runbooks 0.7.9__py3-none-any.whl → 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. runbooks/__init__.py +1 -1
  2. runbooks/cfat/README.md +12 -1
  3. runbooks/cfat/__init__.py +1 -1
  4. runbooks/cfat/assessment/runner.py +42 -34
  5. runbooks/cfat/models.py +1 -1
  6. runbooks/common/__init__.py +152 -0
  7. runbooks/common/accuracy_validator.py +1039 -0
  8. runbooks/common/context_logger.py +440 -0
  9. runbooks/common/cross_module_integration.py +594 -0
  10. runbooks/common/enhanced_exception_handler.py +1108 -0
  11. runbooks/common/enterprise_audit_integration.py +634 -0
  12. runbooks/common/mcp_integration.py +539 -0
  13. runbooks/common/performance_monitor.py +387 -0
  14. runbooks/common/profile_utils.py +216 -0
  15. runbooks/common/rich_utils.py +171 -0
  16. runbooks/feedback/user_feedback_collector.py +440 -0
  17. runbooks/finops/README.md +339 -451
  18. runbooks/finops/__init__.py +4 -21
  19. runbooks/finops/account_resolver.py +279 -0
  20. runbooks/finops/accuracy_cross_validator.py +638 -0
  21. runbooks/finops/aws_client.py +721 -36
  22. runbooks/finops/budget_integration.py +313 -0
  23. runbooks/finops/cli.py +59 -5
  24. runbooks/finops/cost_processor.py +211 -37
  25. runbooks/finops/dashboard_router.py +900 -0
  26. runbooks/finops/dashboard_runner.py +990 -232
  27. runbooks/finops/embedded_mcp_validator.py +288 -0
  28. runbooks/finops/enhanced_dashboard_runner.py +8 -7
  29. runbooks/finops/enhanced_progress.py +327 -0
  30. runbooks/finops/enhanced_trend_visualization.py +423 -0
  31. runbooks/finops/finops_dashboard.py +29 -1880
  32. runbooks/finops/helpers.py +509 -196
  33. runbooks/finops/iam_guidance.py +400 -0
  34. runbooks/finops/markdown_exporter.py +466 -0
  35. runbooks/finops/multi_dashboard.py +1502 -0
  36. runbooks/finops/optimizer.py +15 -15
  37. runbooks/finops/profile_processor.py +2 -2
  38. runbooks/finops/runbooks.inventory.organizations_discovery.log +0 -0
  39. runbooks/finops/runbooks.security.report_generator.log +0 -0
  40. runbooks/finops/runbooks.security.run_script.log +0 -0
  41. runbooks/finops/runbooks.security.security_export.log +0 -0
  42. runbooks/finops/service_mapping.py +195 -0
  43. runbooks/finops/single_dashboard.py +710 -0
  44. runbooks/finops/tests/test_reference_images_validation.py +1 -1
  45. runbooks/inventory/README.md +12 -1
  46. runbooks/inventory/core/collector.py +157 -29
  47. runbooks/inventory/list_ec2_instances.py +9 -6
  48. runbooks/inventory/list_ssm_parameters.py +10 -10
  49. runbooks/inventory/organizations_discovery.py +210 -164
  50. runbooks/inventory/rich_inventory_display.py +74 -107
  51. runbooks/inventory/run_on_multi_accounts.py +13 -13
  52. runbooks/main.py +740 -134
  53. runbooks/metrics/dora_metrics_engine.py +711 -17
  54. runbooks/monitoring/performance_monitor.py +433 -0
  55. runbooks/operate/README.md +394 -0
  56. runbooks/operate/base.py +215 -47
  57. runbooks/operate/ec2_operations.py +7 -5
  58. runbooks/operate/privatelink_operations.py +1 -1
  59. runbooks/operate/vpc_endpoints.py +1 -1
  60. runbooks/remediation/README.md +489 -13
  61. runbooks/remediation/commons.py +8 -4
  62. runbooks/security/ENTERPRISE_SECURITY_FRAMEWORK.md +506 -0
  63. runbooks/security/README.md +12 -1
  64. runbooks/security/__init__.py +164 -33
  65. runbooks/security/compliance_automation.py +12 -10
  66. runbooks/security/compliance_automation_engine.py +1021 -0
  67. runbooks/security/enterprise_security_framework.py +931 -0
  68. runbooks/security/enterprise_security_policies.json +293 -0
  69. runbooks/security/integration_test_enterprise_security.py +879 -0
  70. runbooks/security/module_security_integrator.py +641 -0
  71. runbooks/security/report_generator.py +1 -1
  72. runbooks/security/run_script.py +4 -8
  73. runbooks/security/security_baseline_tester.py +36 -49
  74. runbooks/security/security_export.py +99 -120
  75. runbooks/sre/README.md +472 -0
  76. runbooks/sre/__init__.py +33 -0
  77. runbooks/sre/mcp_reliability_engine.py +1049 -0
  78. runbooks/sre/performance_optimization_engine.py +1032 -0
  79. runbooks/sre/reliability_monitoring_framework.py +1011 -0
  80. runbooks/validation/__init__.py +2 -2
  81. runbooks/validation/benchmark.py +154 -149
  82. runbooks/validation/cli.py +159 -147
  83. runbooks/validation/mcp_validator.py +265 -236
  84. runbooks/vpc/README.md +478 -0
  85. runbooks/vpc/__init__.py +2 -2
  86. runbooks/vpc/manager_interface.py +366 -351
  87. runbooks/vpc/networking_wrapper.py +62 -33
  88. runbooks/vpc/rich_formatters.py +22 -8
  89. {runbooks-0.7.9.dist-info → runbooks-0.9.0.dist-info}/METADATA +136 -54
  90. {runbooks-0.7.9.dist-info → runbooks-0.9.0.dist-info}/RECORD +94 -55
  91. {runbooks-0.7.9.dist-info → runbooks-0.9.0.dist-info}/entry_points.txt +1 -1
  92. runbooks/finops/cross_validation.py +0 -375
  93. {runbooks-0.7.9.dist-info → runbooks-0.9.0.dist-info}/WHEEL +0 -0
  94. {runbooks-0.7.9.dist-info → runbooks-0.9.0.dist-info}/licenses/LICENSE +0 -0
  95. {runbooks-0.7.9.dist-info → runbooks-0.9.0.dist-info}/top_level.txt +0 -0
@@ -21,29 +21,32 @@ Usage:
21
21
 
22
22
  import asyncio
23
23
  import json
24
+ import logging
24
25
  import time
26
+ from dataclasses import asdict, dataclass
25
27
  from datetime import datetime, timedelta
26
- from pathlib import Path
27
- from typing import Dict, List, Optional, Any, Tuple, Union
28
- from dataclasses import dataclass, asdict
29
28
  from enum import Enum
30
- import logging
29
+ from pathlib import Path
30
+ from typing import Any, Dict, List, Optional, Tuple, Union
31
+
32
+ from rich import box
31
33
 
32
34
  # Rich console for enterprise output
33
35
  from rich.console import Console
34
- from rich.table import Table
35
36
  from rich.panel import Panel
36
- from rich.progress import track, Progress, TaskID
37
+ from rich.progress import Progress, TaskID, track
37
38
  from rich.status import Status
38
- from rich import box
39
+ from rich.table import Table
39
40
 
40
41
  # Import existing modules
41
42
  try:
43
+ # Import functions dynamically to avoid circular imports
42
44
  from runbooks.inventory.core.collector import InventoryCollector
43
- from runbooks.finops.main import FinOpsRunner
44
- from runbooks.security.run_script import SecurityBaselineRunner
45
45
  from runbooks.operate.base import BaseOperation
46
+ from runbooks.security.run_script import SecurityBaselineRunner
46
47
  from runbooks.vpc.networking_wrapper import NetworkingWrapper
48
+ # FinOps runner will be imported dynamically when needed
49
+ run_dashboard = None
47
50
  except ImportError as e:
48
51
  logging.warning(f"Optional module import failed: {e}")
49
52
 
@@ -56,17 +59,21 @@ except ImportError:
56
59
 
57
60
  console = Console()
58
61
 
62
+
59
63
  class ValidationStatus(Enum):
60
64
  """Validation status enumeration."""
65
+
61
66
  PASSED = "PASSED"
62
67
  FAILED = "FAILED"
63
68
  WARNING = "WARNING"
64
69
  ERROR = "ERROR"
65
70
  TIMEOUT = "TIMEOUT"
66
71
 
72
+
67
73
  @dataclass
68
74
  class ValidationResult:
69
75
  """Individual validation result."""
76
+
70
77
  operation_name: str
71
78
  status: ValidationStatus
72
79
  runbooks_result: Any
@@ -77,9 +84,11 @@ class ValidationResult:
77
84
  timestamp: datetime
78
85
  error_message: Optional[str] = None
79
86
 
87
+
80
88
  @dataclass
81
89
  class ValidationReport:
82
90
  """Comprehensive validation report."""
91
+
83
92
  overall_accuracy: float
84
93
  total_validations: int
85
94
  passed_validations: int
@@ -91,94 +100,107 @@ class ValidationReport:
91
100
  validation_results: List[ValidationResult]
92
101
  recommendations: List[str]
93
102
 
103
+
94
104
  class MCPValidator:
95
105
  """
96
106
  Enterprise MCP Validation Framework with 99.5% accuracy target.
97
-
107
+
98
108
  Validates critical operations across:
99
109
  - Cost Explorer data
100
110
  - Organizations API
101
111
  - EC2 inventory
102
- - Security baselines
112
+ - Security baselines
103
113
  - VPC analysis
104
114
  """
105
-
106
- def __init__(self,
107
- profiles: Dict[str, str] = None,
108
- tolerance_percentage: float = 5.0,
109
- performance_target_seconds: float = 30.0):
115
+
116
+ def __init__(
117
+ self,
118
+ profiles: Dict[str, str] = None,
119
+ tolerance_percentage: float = 5.0,
120
+ performance_target_seconds: float = 30.0,
121
+ ):
110
122
  """Initialize MCP validator."""
111
-
123
+
112
124
  # Default AWS profiles
113
125
  self.profiles = profiles or {
114
- 'billing': 'ams-admin-Billing-ReadOnlyAccess-909135376185',
115
- 'management': 'ams-admin-ReadOnlyAccess-909135376185',
116
- 'centralised_ops': 'ams-centralised-ops-ReadOnlyAccess-335083429030',
117
- 'single_aws': 'ams-shared-services-non-prod-ReadOnlyAccess-499201730520'
126
+ "billing": "ams-admin-Billing-ReadOnlyAccess-909135376185",
127
+ "management": "ams-admin-ReadOnlyAccess-909135376185",
128
+ "centralised_ops": "ams-centralised-ops-ReadOnlyAccess-335083429030",
129
+ "single_aws": "ams-shared-services-non-prod-ReadOnlyAccess-499201730520",
118
130
  }
119
-
131
+
120
132
  self.tolerance_percentage = tolerance_percentage
121
133
  self.performance_target = performance_target_seconds
122
134
  self.validation_results: List[ValidationResult] = []
123
-
135
+
124
136
  # Initialize MCP integration if available
125
137
  self.mcp_enabled = MCPIntegrationManager is not None
126
138
  if self.mcp_enabled:
127
139
  self.mcp_manager = create_mcp_manager_for_multi_account()
128
140
  else:
129
141
  console.print("[yellow]Warning: MCP integration not available[/yellow]")
130
-
142
+
131
143
  # Configure logging
132
144
  logging.basicConfig(
133
145
  level=logging.INFO,
134
- format='%(asctime)s - %(levelname)s - %(message)s',
135
- handlers=[
136
- logging.FileHandler('./artifacts/mcp_validation.log'),
137
- logging.StreamHandler()
138
- ]
146
+ format="%(asctime)s - %(levelname)s - %(message)s",
147
+ handlers=[logging.FileHandler("./artifacts/mcp_validation.log"), logging.StreamHandler()],
139
148
  )
140
149
  self.logger = logging.getLogger(__name__)
141
-
142
- console.print(Panel(
143
- f"[green]MCP Validator Initialized[/green]\n"
144
- f"Target Accuracy: 99.5%\n"
145
- f"Tolerance: ±{tolerance_percentage}%\n"
146
- f"Performance Target: <{performance_target_seconds}s\n"
147
- f"MCP Integration: {'✅ Enabled' if self.mcp_enabled else '❌ Disabled'}",
148
- title="Enterprise Validation Framework"
149
- ))
150
-
150
+
151
+ console.print(
152
+ Panel(
153
+ f"[green]MCP Validator Initialized[/green]\n"
154
+ f"Target Accuracy: 99.5%\n"
155
+ f"Tolerance: ±{tolerance_percentage}%\n"
156
+ f"Performance Target: <{performance_target_seconds}s\n"
157
+ f"MCP Integration: {'✅ Enabled' if self.mcp_enabled else '❌ Disabled'}",
158
+ title="Enterprise Validation Framework",
159
+ )
160
+ )
161
+
151
162
  async def validate_cost_explorer(self) -> ValidationResult:
152
163
  """Validate Cost Explorer data accuracy."""
153
164
  start_time = time.time()
154
165
  operation_name = "cost_explorer_validation"
155
-
166
+
156
167
  try:
157
168
  with Status("[bold green]Validating Cost Explorer data...") as status:
158
- # Get runbooks FinOps result
159
- finops_runner = FinOpsRunner(profile=self.profiles['billing'])
160
- runbooks_result = finops_runner.get_cost_summary()
161
-
169
+ # Get runbooks FinOps result using dynamic import
170
+ import argparse
171
+ from runbooks.finops.dashboard_runner import run_dashboard
172
+ temp_args = argparse.Namespace(
173
+ profile=self.profiles["billing"],
174
+ profiles=None,
175
+ all=False,
176
+ combine=False,
177
+ regions=None,
178
+ time_range=None,
179
+ tag=None,
180
+ export_type=None,
181
+ report_name=None,
182
+ dir=None
183
+ )
184
+ runbooks_result = run_dashboard(temp_args)
185
+
162
186
  # Get MCP validation if available
163
187
  if self.mcp_enabled:
164
- end_date = datetime.now().strftime('%Y-%m-%d')
165
- start_date = (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')
166
- mcp_result = self.mcp_manager.billing_client.get_cost_data_raw(
167
- start_date, end_date
168
- )
188
+ end_date = datetime.now().strftime("%Y-%m-%d")
189
+ start_date = (datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d")
190
+ mcp_result = self.mcp_manager.billing_client.get_cost_data_raw(start_date, end_date)
169
191
  else:
170
192
  mcp_result = {"status": "disabled", "message": "MCP not available"}
171
-
193
+
172
194
  # Calculate accuracy
173
195
  accuracy = self._calculate_cost_accuracy(runbooks_result, mcp_result)
174
-
196
+
175
197
  execution_time = time.time() - start_time
176
-
198
+
177
199
  # Determine status
178
200
  status_val = ValidationStatus.PASSED if accuracy >= 99.5 else ValidationStatus.WARNING
179
201
  if accuracy < 95.0:
180
202
  status_val = ValidationStatus.FAILED
181
-
203
+
182
204
  result = ValidationResult(
183
205
  operation_name=operation_name,
184
206
  status=status_val,
@@ -187,11 +209,11 @@ class MCPValidator:
187
209
  accuracy_percentage=accuracy,
188
210
  variance_details=self._analyze_cost_variance(runbooks_result, mcp_result),
189
211
  execution_time=execution_time,
190
- timestamp=datetime.now()
212
+ timestamp=datetime.now(),
191
213
  )
192
-
214
+
193
215
  return result
194
-
216
+
195
217
  except Exception as e:
196
218
  execution_time = time.time() - start_time
197
219
  return ValidationResult(
@@ -203,34 +225,34 @@ class MCPValidator:
203
225
  variance_details={},
204
226
  execution_time=execution_time,
205
227
  timestamp=datetime.now(),
206
- error_message=str(e)
228
+ error_message=str(e),
207
229
  )
208
-
230
+
209
231
  async def validate_organizations_data(self) -> ValidationResult:
210
232
  """Validate Organizations API data accuracy."""
211
233
  start_time = time.time()
212
234
  operation_name = "organizations_validation"
213
-
235
+
214
236
  try:
215
237
  with Status("[bold green]Validating Organizations data...") as status:
216
238
  # Get runbooks inventory result
217
- inventory = InventoryCollector(profile=self.profiles['management'])
239
+ inventory = InventoryCollector(profile=self.profiles["management"])
218
240
  runbooks_result = inventory.collect_organizations_data()
219
-
241
+
220
242
  # Get MCP validation if available
221
243
  if self.mcp_enabled:
222
244
  mcp_result = self.mcp_manager.management_client.get_organizations_data()
223
245
  else:
224
246
  mcp_result = {"status": "disabled", "total_accounts": 0}
225
-
247
+
226
248
  # Calculate accuracy (exact match required for account counts)
227
249
  accuracy = self._calculate_organizations_accuracy(runbooks_result, mcp_result)
228
-
250
+
229
251
  execution_time = time.time() - start_time
230
-
252
+
231
253
  # Organizations data must be exact match
232
254
  status_val = ValidationStatus.PASSED if accuracy == 100.0 else ValidationStatus.FAILED
233
-
255
+
234
256
  result = ValidationResult(
235
257
  operation_name=operation_name,
236
258
  status=status_val,
@@ -239,11 +261,11 @@ class MCPValidator:
239
261
  accuracy_percentage=accuracy,
240
262
  variance_details=self._analyze_organizations_variance(runbooks_result, mcp_result),
241
263
  execution_time=execution_time,
242
- timestamp=datetime.now()
264
+ timestamp=datetime.now(),
243
265
  )
244
-
266
+
245
267
  return result
246
-
268
+
247
269
  except Exception as e:
248
270
  execution_time = time.time() - start_time
249
271
  return ValidationResult(
@@ -255,32 +277,32 @@ class MCPValidator:
255
277
  variance_details={},
256
278
  execution_time=execution_time,
257
279
  timestamp=datetime.now(),
258
- error_message=str(e)
280
+ error_message=str(e),
259
281
  )
260
-
282
+
261
283
  async def validate_ec2_inventory(self) -> ValidationResult:
262
284
  """Validate EC2 inventory accuracy."""
263
285
  start_time = time.time()
264
286
  operation_name = "ec2_inventory_validation"
265
-
287
+
266
288
  try:
267
289
  with Status("[bold green]Validating EC2 inventory...") as status:
268
290
  # Get runbooks EC2 inventory
269
- inventory = InventoryCollector(profile=self.profiles['centralised_ops'])
291
+ inventory = InventoryCollector(profile=self.profiles["centralised_ops"])
270
292
  runbooks_result = inventory.collect_ec2_instances()
271
-
293
+
272
294
  # For MCP validation, we would collect via direct boto3 calls
273
295
  # This simulates the MCP server providing independent data
274
296
  mcp_result = self._get_mcp_ec2_data() if self.mcp_enabled else {"instances": []}
275
-
297
+
276
298
  # Calculate accuracy (exact match for instance counts)
277
299
  accuracy = self._calculate_ec2_accuracy(runbooks_result, mcp_result)
278
-
300
+
279
301
  execution_time = time.time() - start_time
280
-
302
+
281
303
  # EC2 inventory should be exact match
282
304
  status_val = ValidationStatus.PASSED if accuracy >= 99.0 else ValidationStatus.FAILED
283
-
305
+
284
306
  result = ValidationResult(
285
307
  operation_name=operation_name,
286
308
  status=status_val,
@@ -289,11 +311,11 @@ class MCPValidator:
289
311
  accuracy_percentage=accuracy,
290
312
  variance_details=self._analyze_ec2_variance(runbooks_result, mcp_result),
291
313
  execution_time=execution_time,
292
- timestamp=datetime.now()
314
+ timestamp=datetime.now(),
293
315
  )
294
-
316
+
295
317
  return result
296
-
318
+
297
319
  except Exception as e:
298
320
  execution_time = time.time() - start_time
299
321
  return ValidationResult(
@@ -305,33 +327,33 @@ class MCPValidator:
305
327
  variance_details={},
306
328
  execution_time=execution_time,
307
329
  timestamp=datetime.now(),
308
- error_message=str(e)
330
+ error_message=str(e),
309
331
  )
310
-
332
+
311
333
  async def validate_security_baseline(self) -> ValidationResult:
312
334
  """Validate security baseline checks accuracy."""
313
335
  start_time = time.time()
314
336
  operation_name = "security_baseline_validation"
315
-
337
+
316
338
  try:
317
339
  with Status("[bold green]Validating security baseline...") as status:
318
340
  # Get runbooks security assessment
319
341
  security_runner = SecurityBaselineRunner()
320
- runbooks_result = security_runner.run_assessment(profile=self.profiles['single_aws'])
321
-
342
+ runbooks_result = security_runner.run_assessment(profile=self.profiles["single_aws"])
343
+
322
344
  # MCP validation would run independent security checks
323
345
  mcp_result = self._get_mcp_security_data() if self.mcp_enabled else {"checks": []}
324
-
346
+
325
347
  # Calculate accuracy (95%+ agreement required)
326
348
  accuracy = self._calculate_security_accuracy(runbooks_result, mcp_result)
327
-
349
+
328
350
  execution_time = time.time() - start_time
329
-
351
+
330
352
  # Security checks require high agreement
331
353
  status_val = ValidationStatus.PASSED if accuracy >= 95.0 else ValidationStatus.WARNING
332
354
  if accuracy < 90.0:
333
355
  status_val = ValidationStatus.FAILED
334
-
356
+
335
357
  result = ValidationResult(
336
358
  operation_name=operation_name,
337
359
  status=status_val,
@@ -340,11 +362,11 @@ class MCPValidator:
340
362
  accuracy_percentage=accuracy,
341
363
  variance_details=self._analyze_security_variance(runbooks_result, mcp_result),
342
364
  execution_time=execution_time,
343
- timestamp=datetime.now()
365
+ timestamp=datetime.now(),
344
366
  )
345
-
367
+
346
368
  return result
347
-
369
+
348
370
  except Exception as e:
349
371
  execution_time = time.time() - start_time
350
372
  return ValidationResult(
@@ -356,31 +378,31 @@ class MCPValidator:
356
378
  variance_details={},
357
379
  execution_time=execution_time,
358
380
  timestamp=datetime.now(),
359
- error_message=str(e)
381
+ error_message=str(e),
360
382
  )
361
-
383
+
362
384
  async def validate_vpc_analysis(self) -> ValidationResult:
363
385
  """Validate VPC analysis accuracy."""
364
386
  start_time = time.time()
365
387
  operation_name = "vpc_analysis_validation"
366
-
388
+
367
389
  try:
368
390
  with Status("[bold green]Validating VPC analysis...") as status:
369
391
  # Get runbooks VPC analysis
370
- vpc_wrapper = NetworkingWrapper(profile=self.profiles['centralised_ops'])
392
+ vpc_wrapper = NetworkingWrapper(profile=self.profiles["centralised_ops"])
371
393
  runbooks_result = vpc_wrapper.analyze_vpc_costs()
372
-
394
+
373
395
  # MCP validation for VPC data
374
396
  mcp_result = self._get_mcp_vpc_data() if self.mcp_enabled else {"vpcs": []}
375
-
397
+
376
398
  # Calculate accuracy (exact match for topology)
377
399
  accuracy = self._calculate_vpc_accuracy(runbooks_result, mcp_result)
378
-
400
+
379
401
  execution_time = time.time() - start_time
380
-
402
+
381
403
  # VPC topology should be exact match
382
404
  status_val = ValidationStatus.PASSED if accuracy >= 99.0 else ValidationStatus.FAILED
383
-
405
+
384
406
  result = ValidationResult(
385
407
  operation_name=operation_name,
386
408
  status=status_val,
@@ -389,11 +411,11 @@ class MCPValidator:
389
411
  accuracy_percentage=accuracy,
390
412
  variance_details=self._analyze_vpc_variance(runbooks_result, mcp_result),
391
413
  execution_time=execution_time,
392
- timestamp=datetime.now()
414
+ timestamp=datetime.now(),
393
415
  )
394
-
416
+
395
417
  return result
396
-
418
+
397
419
  except Exception as e:
398
420
  execution_time = time.time() - start_time
399
421
  return ValidationResult(
@@ -405,53 +427,57 @@ class MCPValidator:
405
427
  variance_details={},
406
428
  execution_time=execution_time,
407
429
  timestamp=datetime.now(),
408
- error_message=str(e)
430
+ error_message=str(e),
409
431
  )
410
-
432
+
411
433
  async def validate_all_operations(self) -> ValidationReport:
412
434
  """
413
435
  Run comprehensive validation across all critical operations.
414
-
436
+
415
437
  Returns:
416
438
  ValidationReport with overall accuracy and detailed results
417
439
  """
418
440
  start_time = time.time()
419
-
420
- console.print(Panel(
421
- "[bold blue]Starting Comprehensive MCP Validation[/bold blue]\n"
422
- "Target: 99.5% accuracy across all operations",
423
- title="Enterprise Validation Suite"
424
- ))
425
-
441
+
442
+ console.print(
443
+ Panel(
444
+ "[bold blue]Starting Comprehensive MCP Validation[/bold blue]\n"
445
+ "Target: 99.5% accuracy across all operations",
446
+ title="Enterprise Validation Suite",
447
+ )
448
+ )
449
+
426
450
  # Define validation operations
427
451
  validation_tasks = [
428
452
  ("Cost Explorer", self.validate_cost_explorer()),
429
453
  ("Organizations", self.validate_organizations_data()),
430
454
  ("EC2 Inventory", self.validate_ec2_inventory()),
431
455
  ("Security Baseline", self.validate_security_baseline()),
432
- ("VPC Analysis", self.validate_vpc_analysis())
456
+ ("VPC Analysis", self.validate_vpc_analysis()),
433
457
  ]
434
-
458
+
435
459
  results = []
436
-
460
+
437
461
  # Run validations with progress tracking
438
462
  with Progress() as progress:
439
463
  task = progress.add_task("[cyan]Validating operations...", total=len(validation_tasks))
440
-
464
+
441
465
  for operation_name, validation_coro in validation_tasks:
442
466
  progress.console.print(f"[bold green]→[/bold green] Validating {operation_name}")
443
-
467
+
444
468
  try:
445
469
  # Run with timeout
446
470
  result = await asyncio.wait_for(validation_coro, timeout=self.performance_target)
447
471
  results.append(result)
448
-
472
+
449
473
  # Log result
450
474
  status_color = "green" if result.status == ValidationStatus.PASSED else "red"
451
- progress.console.print(f" [{status_color}]{result.status.value}[/{status_color}] "
452
- f"{result.accuracy_percentage:.1f}% accuracy "
453
- f"({result.execution_time:.1f}s)")
454
-
475
+ progress.console.print(
476
+ f" [{status_color}]{result.status.value}[/{status_color}] "
477
+ f"{result.accuracy_percentage:.1f}% accuracy "
478
+ f"({result.execution_time:.1f}s)"
479
+ )
480
+
455
481
  except asyncio.TimeoutError:
456
482
  timeout_result = ValidationResult(
457
483
  operation_name=operation_name.lower().replace(" ", "_"),
@@ -462,31 +488,31 @@ class MCPValidator:
462
488
  variance_details={},
463
489
  execution_time=self.performance_target,
464
490
  timestamp=datetime.now(),
465
- error_message="Validation timeout"
491
+ error_message="Validation timeout",
466
492
  )
467
493
  results.append(timeout_result)
468
494
  progress.console.print(f" [red]TIMEOUT[/red] {operation_name} exceeded {self.performance_target}s")
469
-
495
+
470
496
  progress.advance(task)
471
-
497
+
472
498
  # Calculate overall metrics
473
499
  total_validations = len(results)
474
500
  passed_validations = len([r for r in results if r.status == ValidationStatus.PASSED])
475
501
  failed_validations = len([r for r in results if r.status == ValidationStatus.FAILED])
476
502
  warning_validations = len([r for r in results if r.status == ValidationStatus.WARNING])
477
503
  error_validations = len([r for r in results if r.status in [ValidationStatus.ERROR, ValidationStatus.TIMEOUT]])
478
-
504
+
479
505
  # Calculate overall accuracy (weighted average)
480
506
  if results:
481
507
  overall_accuracy = sum(r.accuracy_percentage for r in results) / len(results)
482
508
  else:
483
509
  overall_accuracy = 0.0
484
-
510
+
485
511
  execution_time = time.time() - start_time
486
-
512
+
487
513
  # Generate recommendations
488
514
  recommendations = self._generate_recommendations(results, overall_accuracy)
489
-
515
+
490
516
  report = ValidationReport(
491
517
  overall_accuracy=overall_accuracy,
492
518
  total_validations=total_validations,
@@ -497,78 +523,84 @@ class MCPValidator:
497
523
  execution_time=execution_time,
498
524
  timestamp=datetime.now(),
499
525
  validation_results=results,
500
- recommendations=recommendations
526
+ recommendations=recommendations,
501
527
  )
502
-
528
+
503
529
  # Store results
504
530
  self.validation_results.extend(results)
505
-
531
+
506
532
  return report
507
-
533
+
508
534
  def display_validation_report(self, report: ValidationReport) -> None:
509
535
  """Display comprehensive validation report."""
510
-
536
+
511
537
  # Overall status
512
- status_color = "green" if report.overall_accuracy >= 99.5 else "red" if report.overall_accuracy < 95.0 else "yellow"
513
-
514
- console.print(Panel(
515
- f"[bold {status_color}]Overall Accuracy: {report.overall_accuracy:.2f}%[/bold {status_color}]\n"
516
- f"Target: 99.5% | Execution Time: {report.execution_time:.1f}s\n"
517
- f"Validations: {report.passed_validations}/{report.total_validations} passed",
518
- title="Validation Summary"
519
- ))
520
-
538
+ status_color = (
539
+ "green" if report.overall_accuracy >= 99.5 else "red" if report.overall_accuracy < 95.0 else "yellow"
540
+ )
541
+
542
+ console.print(
543
+ Panel(
544
+ f"[bold {status_color}]Overall Accuracy: {report.overall_accuracy:.2f}%[/bold {status_color}]\n"
545
+ f"Target: 99.5% | Execution Time: {report.execution_time:.1f}s\n"
546
+ f"Validations: {report.passed_validations}/{report.total_validations} passed",
547
+ title="Validation Summary",
548
+ )
549
+ )
550
+
521
551
  # Detailed results table
522
552
  table = Table(title="Detailed Validation Results", box=box.ROUNDED)
523
553
  table.add_column("Operation", style="cyan", no_wrap=True)
524
554
  table.add_column("Status", style="bold")
525
555
  table.add_column("Accuracy", justify="right")
526
- table.add_column("Time (s)", justify="right")
556
+ table.add_column("Time (s)", justify="right")
527
557
  table.add_column("Details")
528
-
558
+
529
559
  for result in report.validation_results:
530
560
  status_style = {
531
561
  ValidationStatus.PASSED: "green",
532
- ValidationStatus.WARNING: "yellow",
562
+ ValidationStatus.WARNING: "yellow",
533
563
  ValidationStatus.FAILED: "red",
534
564
  ValidationStatus.ERROR: "red",
535
- ValidationStatus.TIMEOUT: "red"
565
+ ValidationStatus.TIMEOUT: "red",
536
566
  }[result.status]
537
-
567
+
538
568
  details = result.error_message or f"Variance: {result.variance_details.get('summary', 'N/A')}"
539
-
569
+
540
570
  table.add_row(
541
571
  result.operation_name.replace("_", " ").title(),
542
572
  f"[{status_style}]{result.status.value}[/{status_style}]",
543
573
  f"{result.accuracy_percentage:.1f}%",
544
574
  f"{result.execution_time:.1f}",
545
- details[:50] + "..." if len(details) > 50 else details
575
+ details[:50] + "..." if len(details) > 50 else details,
546
576
  )
547
-
577
+
548
578
  console.print(table)
549
-
579
+
550
580
  # Recommendations
551
581
  if report.recommendations:
552
- console.print(Panel(
553
- "\n".join(f"• {rec}" for rec in report.recommendations),
554
- title="Recommendations",
555
- border_style="blue"
556
- ))
557
-
582
+ console.print(
583
+ Panel(
584
+ "\n".join(f"• {rec}" for rec in report.recommendations),
585
+ title="Recommendations",
586
+ border_style="blue",
587
+ )
588
+ )
589
+
558
590
  # Save report
559
591
  self._save_validation_report(report)
560
-
592
+
561
593
  def _save_validation_report(self, report: ValidationReport) -> None:
562
594
  """Save validation report to artifacts directory."""
563
595
  artifacts_dir = Path("./artifacts/validation")
564
596
  artifacts_dir.mkdir(parents=True, exist_ok=True)
565
-
597
+
566
598
  timestamp = report.timestamp.strftime("%Y%m%d_%H%M%S")
567
599
  report_file = artifacts_dir / f"mcp_validation_{timestamp}.json"
568
-
600
+
569
601
  # Convert to dict for JSON serialization
570
602
  report_dict = asdict(report)
571
-
603
+
572
604
  # Convert datetime and enum objects
573
605
  def serialize_special(obj):
574
606
  if isinstance(obj, datetime):
@@ -576,51 +608,51 @@ class MCPValidator:
576
608
  elif isinstance(obj, ValidationStatus):
577
609
  return obj.value
578
610
  return str(obj)
579
-
580
- with open(report_file, 'w') as f:
611
+
612
+ with open(report_file, "w") as f:
581
613
  json.dump(report_dict, f, indent=2, default=serialize_special)
582
-
614
+
583
615
  console.print(f"[green]Validation report saved:[/green] {report_file}")
584
616
  self.logger.info(f"Validation report saved: {report_file}")
585
-
617
+
586
618
  # Accuracy calculation methods
587
619
  def _calculate_cost_accuracy(self, runbooks_result: Any, mcp_result: Any) -> float:
588
620
  """Calculate Cost Explorer accuracy."""
589
- if not mcp_result or mcp_result.get('status') != 'success':
621
+ if not mcp_result or mcp_result.get("status") != "success":
590
622
  return 50.0 # Partial score when MCP unavailable
591
-
623
+
592
624
  try:
593
- runbooks_total = runbooks_result.get('total_cost', 0)
594
- mcp_total = float(mcp_result.get('data', {}).get('total_amount', 0))
595
-
625
+ runbooks_total = runbooks_result.get("total_cost", 0)
626
+ mcp_total = float(mcp_result.get("data", {}).get("total_amount", 0))
627
+
596
628
  if runbooks_total > 0 and mcp_total > 0:
597
629
  variance = abs(runbooks_total - mcp_total) / runbooks_total * 100
598
630
  accuracy = max(0, 100 - variance)
599
631
  return min(100.0, accuracy)
600
632
  except:
601
633
  pass
602
-
634
+
603
635
  return 0.0
604
-
636
+
605
637
  def _calculate_organizations_accuracy(self, runbooks_result: Any, mcp_result: Any) -> float:
606
638
  """Calculate Organizations data accuracy."""
607
- if not mcp_result or mcp_result.get('status') != 'success':
639
+ if not mcp_result or mcp_result.get("status") != "success":
608
640
  return 50.0
609
-
641
+
610
642
  try:
611
- runbooks_count = runbooks_result.get('total_accounts', 0)
612
- mcp_count = mcp_result.get('total_accounts', 0)
613
-
643
+ runbooks_count = runbooks_result.get("total_accounts", 0)
644
+ mcp_count = mcp_result.get("total_accounts", 0)
645
+
614
646
  return 100.0 if runbooks_count == mcp_count else 0.0
615
647
  except:
616
648
  return 0.0
617
-
649
+
618
650
  def _calculate_ec2_accuracy(self, runbooks_result: Any, mcp_result: Any) -> float:
619
651
  """Calculate EC2 inventory accuracy."""
620
652
  try:
621
- runbooks_count = len(runbooks_result.get('instances', []))
622
- mcp_count = len(mcp_result.get('instances', []))
623
-
653
+ runbooks_count = len(runbooks_result.get("instances", []))
654
+ mcp_count = len(mcp_result.get("instances", []))
655
+
624
656
  if runbooks_count == mcp_count:
625
657
  return 100.0
626
658
  elif runbooks_count > 0:
@@ -628,37 +660,37 @@ class MCPValidator:
628
660
  return max(0, 100 - variance)
629
661
  except:
630
662
  pass
631
-
663
+
632
664
  return 0.0
633
-
665
+
634
666
  def _calculate_security_accuracy(self, runbooks_result: Any, mcp_result: Any) -> float:
635
667
  """Calculate security baseline accuracy."""
636
668
  try:
637
- runbooks_checks = runbooks_result.get('checks_passed', 0)
638
- mcp_checks = mcp_result.get('checks_passed', 0)
639
-
640
- total_checks = max(runbooks_result.get('total_checks', 1), 1)
641
-
669
+ runbooks_checks = runbooks_result.get("checks_passed", 0)
670
+ mcp_checks = mcp_result.get("checks_passed", 0)
671
+
672
+ total_checks = max(runbooks_result.get("total_checks", 1), 1)
673
+
642
674
  # Calculate agreement percentage
643
675
  agreement = 1.0 - abs(runbooks_checks - mcp_checks) / total_checks
644
676
  return agreement * 100
645
677
  except:
646
678
  pass
647
-
679
+
648
680
  return 85.0 # Default reasonable score for security
649
-
681
+
650
682
  def _calculate_vpc_accuracy(self, runbooks_result: Any, mcp_result: Any) -> float:
651
683
  """Calculate VPC analysis accuracy."""
652
684
  try:
653
- runbooks_vpcs = len(runbooks_result.get('vpcs', []))
654
- mcp_vpcs = len(mcp_result.get('vpcs', []))
655
-
685
+ runbooks_vpcs = len(runbooks_result.get("vpcs", []))
686
+ mcp_vpcs = len(mcp_result.get("vpcs", []))
687
+
656
688
  return 100.0 if runbooks_vpcs == mcp_vpcs else 90.0
657
689
  except:
658
690
  pass
659
-
691
+
660
692
  return 90.0
661
-
693
+
662
694
  # Variance analysis methods
663
695
  def _analyze_cost_variance(self, runbooks_result: Any, mcp_result: Any) -> Dict[str, Any]:
664
696
  """Analyze cost data variance."""
@@ -666,82 +698,78 @@ class MCPValidator:
666
698
  "type": "cost_variance",
667
699
  "summary": "Cost data comparison between runbooks and MCP",
668
700
  "details": {
669
- "runbooks_total": runbooks_result.get('total_cost', 0) if runbooks_result else 0,
670
- "mcp_available": mcp_result.get('status') == 'success' if mcp_result else False
671
- }
701
+ "runbooks_total": runbooks_result.get("total_cost", 0) if runbooks_result else 0,
702
+ "mcp_available": mcp_result.get("status") == "success" if mcp_result else False,
703
+ },
672
704
  }
673
-
705
+
674
706
  def _analyze_organizations_variance(self, runbooks_result: Any, mcp_result: Any) -> Dict[str, Any]:
675
707
  """Analyze organizations data variance."""
676
708
  return {
677
- "type": "organizations_variance",
709
+ "type": "organizations_variance",
678
710
  "summary": "Account count comparison",
679
711
  "details": {
680
- "runbooks_accounts": runbooks_result.get('total_accounts', 0) if runbooks_result else 0,
681
- "mcp_accounts": mcp_result.get('total_accounts', 0) if mcp_result else 0
682
- }
712
+ "runbooks_accounts": runbooks_result.get("total_accounts", 0) if runbooks_result else 0,
713
+ "mcp_accounts": mcp_result.get("total_accounts", 0) if mcp_result else 0,
714
+ },
683
715
  }
684
-
716
+
685
717
  def _analyze_ec2_variance(self, runbooks_result: Any, mcp_result: Any) -> Dict[str, Any]:
686
718
  """Analyze EC2 inventory variance."""
687
719
  return {
688
720
  "type": "ec2_variance",
689
- "summary": "Instance count comparison",
721
+ "summary": "Instance count comparison",
690
722
  "details": {
691
- "runbooks_instances": len(runbooks_result.get('instances', [])) if runbooks_result else 0,
692
- "mcp_instances": len(mcp_result.get('instances', [])) if mcp_result else 0
693
- }
723
+ "runbooks_instances": len(runbooks_result.get("instances", [])) if runbooks_result else 0,
724
+ "mcp_instances": len(mcp_result.get("instances", [])) if mcp_result else 0,
725
+ },
694
726
  }
695
-
727
+
696
728
  def _analyze_security_variance(self, runbooks_result: Any, mcp_result: Any) -> Dict[str, Any]:
697
729
  """Analyze security baseline variance."""
698
730
  return {
699
731
  "type": "security_variance",
700
732
  "summary": "Security check agreement",
701
733
  "details": {
702
- "runbooks_checks": runbooks_result.get('checks_passed', 0) if runbooks_result else 0,
703
- "mcp_checks": mcp_result.get('checks_passed', 0) if mcp_result else 0
704
- }
734
+ "runbooks_checks": runbooks_result.get("checks_passed", 0) if runbooks_result else 0,
735
+ "mcp_checks": mcp_result.get("checks_passed", 0) if mcp_result else 0,
736
+ },
705
737
  }
706
-
738
+
707
739
  def _analyze_vpc_variance(self, runbooks_result: Any, mcp_result: Any) -> Dict[str, Any]:
708
740
  """Analyze VPC data variance."""
709
741
  return {
710
742
  "type": "vpc_variance",
711
743
  "summary": "VPC topology comparison",
712
744
  "details": {
713
- "runbooks_vpcs": len(runbooks_result.get('vpcs', [])) if runbooks_result else 0,
714
- "mcp_vpcs": len(mcp_result.get('vpcs', [])) if mcp_result else 0
715
- }
745
+ "runbooks_vpcs": len(runbooks_result.get("vpcs", [])) if runbooks_result else 0,
746
+ "mcp_vpcs": len(mcp_result.get("vpcs", [])) if mcp_result else 0,
747
+ },
716
748
  }
717
-
749
+
718
750
  # MCP data collection methods (simulated)
719
751
  def _get_mcp_ec2_data(self) -> Dict[str, Any]:
720
752
  """Get MCP EC2 data (simulated)."""
721
753
  return {
722
754
  "instances": ["i-123", "i-456", "i-789"], # Simulated
723
- "status": "success"
755
+ "status": "success",
724
756
  }
725
-
757
+
726
758
  def _get_mcp_security_data(self) -> Dict[str, Any]:
727
759
  """Get MCP security data (simulated)."""
728
- return {
729
- "checks_passed": 12,
730
- "total_checks": 15,
731
- "status": "success"
732
- }
733
-
760
+ return {"checks_passed": 12, "total_checks": 15, "status": "success"}
761
+
734
762
  def _get_mcp_vpc_data(self) -> Dict[str, Any]:
735
763
  """Get MCP VPC data (simulated)."""
736
764
  return {
737
765
  "vpcs": ["vpc-123", "vpc-456"], # Simulated
738
- "status": "success"
766
+ "status": "success",
739
767
  }
740
-
768
+
741
769
  def _generate_recommendations(self, results: List[ValidationResult], overall_accuracy: float) -> List[str]:
742
770
  """Generate actionable recommendations."""
743
771
  recommendations = []
744
-
772
+
745
773
  if overall_accuracy >= 99.5:
746
774
  recommendations.append("✅ All validations passed - runbooks data is highly accurate")
747
775
  recommendations.append("🎯 Deploy with confidence - 99.5%+ accuracy achieved")
@@ -751,18 +779,19 @@ class MCPValidator:
751
779
  else:
752
780
  recommendations.append("❌ Accuracy below acceptable threshold - investigate data sources")
753
781
  recommendations.append("🔧 Check AWS API permissions and MCP connectivity")
754
-
782
+
755
783
  # Performance recommendations
756
784
  slow_operations = [r for r in results if r.execution_time > self.performance_target * 0.8]
757
785
  if slow_operations:
758
786
  recommendations.append("⚡ Consider performance optimization for slow operations")
759
-
760
- # Error-specific recommendations
787
+
788
+ # Error-specific recommendations
761
789
  error_operations = [r for r in results if r.status in [ValidationStatus.ERROR, ValidationStatus.TIMEOUT]]
762
790
  if error_operations:
763
791
  recommendations.append("🔧 Address errors in failed operations before production deployment")
764
-
792
+
765
793
  return recommendations
766
794
 
795
+
767
796
  # Export main class
768
- __all__ = ['MCPValidator', 'ValidationResult', 'ValidationReport', 'ValidationStatus']
797
+ __all__ = ["MCPValidator", "ValidationResult", "ValidationReport", "ValidationStatus"]