runbooks 1.1.4__py3-none-any.whl → 1.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +31 -2
- runbooks/__init___optimized.py +18 -4
- runbooks/_platform/__init__.py +1 -5
- runbooks/_platform/core/runbooks_wrapper.py +141 -138
- runbooks/aws2/accuracy_validator.py +812 -0
- runbooks/base.py +7 -0
- runbooks/cfat/assessment/compliance.py +1 -1
- runbooks/cfat/assessment/runner.py +1 -0
- runbooks/cfat/cloud_foundations_assessment.py +227 -239
- runbooks/cli/__init__.py +1 -1
- runbooks/cli/commands/cfat.py +64 -23
- runbooks/cli/commands/finops.py +1005 -54
- runbooks/cli/commands/inventory.py +138 -35
- runbooks/cli/commands/operate.py +9 -36
- runbooks/cli/commands/security.py +42 -18
- runbooks/cli/commands/validation.py +432 -18
- runbooks/cli/commands/vpc.py +81 -17
- runbooks/cli/registry.py +22 -10
- runbooks/cloudops/__init__.py +20 -27
- runbooks/cloudops/base.py +96 -107
- runbooks/cloudops/cost_optimizer.py +544 -542
- runbooks/cloudops/infrastructure_optimizer.py +5 -4
- runbooks/cloudops/interfaces.py +224 -225
- runbooks/cloudops/lifecycle_manager.py +5 -4
- runbooks/cloudops/mcp_cost_validation.py +252 -235
- runbooks/cloudops/models.py +78 -53
- runbooks/cloudops/monitoring_automation.py +5 -4
- runbooks/cloudops/notebook_framework.py +177 -213
- runbooks/cloudops/security_enforcer.py +125 -159
- runbooks/common/accuracy_validator.py +11 -0
- runbooks/common/aws_pricing.py +349 -326
- runbooks/common/aws_pricing_api.py +211 -212
- runbooks/common/aws_profile_manager.py +40 -36
- runbooks/common/aws_utils.py +74 -79
- runbooks/common/business_logic.py +126 -104
- runbooks/common/cli_decorators.py +36 -60
- runbooks/common/comprehensive_cost_explorer_integration.py +455 -463
- runbooks/common/cross_account_manager.py +197 -204
- runbooks/common/date_utils.py +27 -39
- runbooks/common/decorators.py +29 -19
- runbooks/common/dry_run_examples.py +173 -208
- runbooks/common/dry_run_framework.py +157 -155
- runbooks/common/enhanced_exception_handler.py +15 -4
- runbooks/common/enhanced_logging_example.py +50 -64
- runbooks/common/enhanced_logging_integration_example.py +65 -37
- runbooks/common/env_utils.py +16 -16
- runbooks/common/error_handling.py +40 -38
- runbooks/common/lazy_loader.py +41 -23
- runbooks/common/logging_integration_helper.py +79 -86
- runbooks/common/mcp_cost_explorer_integration.py +476 -493
- runbooks/common/mcp_integration.py +63 -74
- runbooks/common/memory_optimization.py +140 -118
- runbooks/common/module_cli_base.py +37 -58
- runbooks/common/organizations_client.py +175 -193
- runbooks/common/patterns.py +23 -25
- runbooks/common/performance_monitoring.py +67 -71
- runbooks/common/performance_optimization_engine.py +283 -274
- runbooks/common/profile_utils.py +111 -37
- runbooks/common/rich_utils.py +201 -141
- runbooks/common/sre_performance_suite.py +177 -186
- runbooks/enterprise/__init__.py +1 -1
- runbooks/enterprise/logging.py +144 -106
- runbooks/enterprise/security.py +187 -204
- runbooks/enterprise/validation.py +43 -56
- runbooks/finops/__init__.py +26 -30
- runbooks/finops/account_resolver.py +1 -1
- runbooks/finops/advanced_optimization_engine.py +980 -0
- runbooks/finops/automation_core.py +268 -231
- runbooks/finops/business_case_config.py +184 -179
- runbooks/finops/cli.py +660 -139
- runbooks/finops/commvault_ec2_analysis.py +157 -164
- runbooks/finops/compute_cost_optimizer.py +336 -320
- runbooks/finops/config.py +20 -20
- runbooks/finops/cost_optimizer.py +484 -618
- runbooks/finops/cost_processor.py +332 -214
- runbooks/finops/dashboard_runner.py +1006 -172
- runbooks/finops/ebs_cost_optimizer.py +991 -657
- runbooks/finops/elastic_ip_optimizer.py +317 -257
- runbooks/finops/enhanced_mcp_integration.py +340 -0
- runbooks/finops/enhanced_progress.py +32 -29
- runbooks/finops/enhanced_trend_visualization.py +3 -2
- runbooks/finops/enterprise_wrappers.py +223 -285
- runbooks/finops/executive_export.py +203 -160
- runbooks/finops/helpers.py +130 -288
- runbooks/finops/iam_guidance.py +1 -1
- runbooks/finops/infrastructure/__init__.py +80 -0
- runbooks/finops/infrastructure/commands.py +506 -0
- runbooks/finops/infrastructure/load_balancer_optimizer.py +866 -0
- runbooks/finops/infrastructure/vpc_endpoint_optimizer.py +832 -0
- runbooks/finops/markdown_exporter.py +337 -174
- runbooks/finops/mcp_validator.py +1952 -0
- runbooks/finops/nat_gateway_optimizer.py +1512 -481
- runbooks/finops/network_cost_optimizer.py +657 -587
- runbooks/finops/notebook_utils.py +226 -188
- runbooks/finops/optimization_engine.py +1136 -0
- runbooks/finops/optimizer.py +19 -23
- runbooks/finops/rds_snapshot_optimizer.py +367 -411
- runbooks/finops/reservation_optimizer.py +427 -363
- runbooks/finops/scenario_cli_integration.py +64 -65
- runbooks/finops/scenarios.py +1277 -438
- runbooks/finops/schemas.py +218 -182
- runbooks/finops/snapshot_manager.py +2289 -0
- runbooks/finops/types.py +3 -3
- runbooks/finops/validation_framework.py +259 -265
- runbooks/finops/vpc_cleanup_exporter.py +189 -144
- runbooks/finops/vpc_cleanup_optimizer.py +591 -573
- runbooks/finops/workspaces_analyzer.py +171 -182
- runbooks/integration/__init__.py +89 -0
- runbooks/integration/mcp_integration.py +1920 -0
- runbooks/inventory/CLAUDE.md +816 -0
- runbooks/inventory/__init__.py +2 -2
- runbooks/inventory/cloud_foundations_integration.py +144 -149
- runbooks/inventory/collectors/aws_comprehensive.py +1 -1
- runbooks/inventory/collectors/aws_networking.py +109 -99
- runbooks/inventory/collectors/base.py +4 -0
- runbooks/inventory/core/collector.py +495 -313
- runbooks/inventory/drift_detection_cli.py +69 -96
- runbooks/inventory/inventory_mcp_cli.py +48 -46
- runbooks/inventory/list_rds_snapshots_aggregator.py +192 -208
- runbooks/inventory/mcp_inventory_validator.py +549 -465
- runbooks/inventory/mcp_vpc_validator.py +359 -442
- runbooks/inventory/organizations_discovery.py +55 -51
- runbooks/inventory/rich_inventory_display.py +33 -32
- runbooks/inventory/unified_validation_engine.py +278 -251
- runbooks/inventory/vpc_analyzer.py +732 -695
- runbooks/inventory/vpc_architecture_validator.py +293 -348
- runbooks/inventory/vpc_dependency_analyzer.py +382 -378
- runbooks/inventory/vpc_flow_analyzer.py +1 -1
- runbooks/main.py +49 -34
- runbooks/main_final.py +91 -60
- runbooks/main_minimal.py +22 -10
- runbooks/main_optimized.py +131 -100
- runbooks/main_ultra_minimal.py +7 -2
- runbooks/mcp/__init__.py +36 -0
- runbooks/mcp/integration.py +679 -0
- runbooks/monitoring/performance_monitor.py +9 -4
- runbooks/operate/dynamodb_operations.py +3 -1
- runbooks/operate/ec2_operations.py +145 -137
- runbooks/operate/iam_operations.py +146 -152
- runbooks/operate/networking_cost_heatmap.py +29 -8
- runbooks/operate/rds_operations.py +223 -254
- runbooks/operate/s3_operations.py +107 -118
- runbooks/operate/vpc_operations.py +646 -616
- runbooks/remediation/base.py +1 -1
- runbooks/remediation/commons.py +10 -7
- runbooks/remediation/commvault_ec2_analysis.py +70 -66
- runbooks/remediation/ec2_unattached_ebs_volumes.py +1 -0
- runbooks/remediation/multi_account.py +24 -21
- runbooks/remediation/rds_snapshot_list.py +86 -60
- runbooks/remediation/remediation_cli.py +92 -146
- runbooks/remediation/universal_account_discovery.py +83 -79
- runbooks/remediation/workspaces_list.py +46 -41
- runbooks/security/__init__.py +19 -0
- runbooks/security/assessment_runner.py +1150 -0
- runbooks/security/baseline_checker.py +812 -0
- runbooks/security/cloudops_automation_security_validator.py +509 -535
- runbooks/security/compliance_automation_engine.py +17 -17
- runbooks/security/config/__init__.py +2 -2
- runbooks/security/config/compliance_config.py +50 -50
- runbooks/security/config_template_generator.py +63 -76
- runbooks/security/enterprise_security_framework.py +1 -1
- runbooks/security/executive_security_dashboard.py +519 -508
- runbooks/security/multi_account_security_controls.py +959 -1210
- runbooks/security/real_time_security_monitor.py +422 -444
- runbooks/security/security_baseline_tester.py +1 -1
- runbooks/security/security_cli.py +143 -112
- runbooks/security/test_2way_validation.py +439 -0
- runbooks/security/two_way_validation_framework.py +852 -0
- runbooks/sre/production_monitoring_framework.py +167 -177
- runbooks/tdd/__init__.py +15 -0
- runbooks/tdd/cli.py +1071 -0
- runbooks/utils/__init__.py +14 -17
- runbooks/utils/logger.py +7 -2
- runbooks/utils/version_validator.py +50 -47
- runbooks/validation/__init__.py +6 -6
- runbooks/validation/cli.py +9 -3
- runbooks/validation/comprehensive_2way_validator.py +745 -704
- runbooks/validation/mcp_validator.py +906 -228
- runbooks/validation/terraform_citations_validator.py +104 -115
- runbooks/validation/terraform_drift_detector.py +447 -451
- runbooks/vpc/README.md +617 -0
- runbooks/vpc/__init__.py +8 -1
- runbooks/vpc/analyzer.py +577 -0
- runbooks/vpc/cleanup_wrapper.py +476 -413
- runbooks/vpc/cli_cloudtrail_commands.py +339 -0
- runbooks/vpc/cli_mcp_validation_commands.py +480 -0
- runbooks/vpc/cloudtrail_audit_integration.py +717 -0
- runbooks/vpc/config.py +92 -97
- runbooks/vpc/cost_engine.py +411 -148
- runbooks/vpc/cost_explorer_integration.py +553 -0
- runbooks/vpc/cross_account_session.py +101 -106
- runbooks/vpc/enhanced_mcp_validation.py +917 -0
- runbooks/vpc/eni_gate_validator.py +961 -0
- runbooks/vpc/heatmap_engine.py +185 -160
- runbooks/vpc/mcp_no_eni_validator.py +680 -639
- runbooks/vpc/nat_gateway_optimizer.py +358 -0
- runbooks/vpc/networking_wrapper.py +15 -8
- runbooks/vpc/pdca_remediation_planner.py +528 -0
- runbooks/vpc/performance_optimized_analyzer.py +219 -231
- runbooks/vpc/runbooks_adapter.py +1167 -241
- runbooks/vpc/tdd_red_phase_stubs.py +601 -0
- runbooks/vpc/test_data_loader.py +358 -0
- runbooks/vpc/tests/conftest.py +314 -4
- runbooks/vpc/tests/test_cleanup_framework.py +1022 -0
- runbooks/vpc/tests/test_cost_engine.py +0 -2
- runbooks/vpc/topology_generator.py +326 -0
- runbooks/vpc/unified_scenarios.py +1297 -1124
- runbooks/vpc/vpc_cleanup_integration.py +1943 -1115
- runbooks-1.1.5.dist-info/METADATA +328 -0
- {runbooks-1.1.4.dist-info → runbooks-1.1.5.dist-info}/RECORD +214 -193
- runbooks/finops/README.md +0 -414
- runbooks/finops/accuracy_cross_validator.py +0 -647
- runbooks/finops/business_cases.py +0 -950
- runbooks/finops/dashboard_router.py +0 -922
- runbooks/finops/ebs_optimizer.py +0 -973
- runbooks/finops/embedded_mcp_validator.py +0 -1629
- runbooks/finops/enhanced_dashboard_runner.py +0 -527
- runbooks/finops/finops_dashboard.py +0 -584
- runbooks/finops/finops_scenarios.py +0 -1218
- runbooks/finops/legacy_migration.py +0 -730
- runbooks/finops/multi_dashboard.py +0 -1519
- runbooks/finops/single_dashboard.py +0 -1113
- runbooks/finops/unlimited_scenarios.py +0 -393
- runbooks-1.1.4.dist-info/METADATA +0 -800
- {runbooks-1.1.4.dist-info → runbooks-1.1.5.dist-info}/WHEEL +0 -0
- {runbooks-1.1.4.dist-info → runbooks-1.1.5.dist-info}/entry_points.txt +0 -0
- {runbooks-1.1.4.dist-info → runbooks-1.1.5.dist-info}/licenses/LICENSE +0 -0
- {runbooks-1.1.4.dist-info → runbooks-1.1.5.dist-info}/top_level.txt +0 -0
@@ -58,10 +58,10 @@ class EnhancedMCPValidator:
|
|
58
58
|
Provides enterprise-grade validation using actual MCP servers from .mcp.json configuration,
|
59
59
|
with 4-way cross-validation: runbooks inventory + direct AWS APIs + MCP servers + terraform state.
|
60
60
|
Ensures ≥99.5% accuracy for enterprise compliance with comprehensive drift detection.
|
61
|
-
|
61
|
+
|
62
62
|
Enhanced Features:
|
63
63
|
- Real MCP server integration from .mcp.json configuration
|
64
|
-
- Enterprise AWS profile override priority system (User > Environment > Default)
|
64
|
+
- Enterprise AWS profile override priority system (User > Environment > Default)
|
65
65
|
- Multi-server validation: aws-api, cost-explorer, iam, cloudwatch, terraform-mcp
|
66
66
|
- 4-way validation: runbooks + direct APIs + MCP servers + terraform drift
|
67
67
|
- Real-time variance detection with configurable tolerance
|
@@ -70,11 +70,16 @@ class EnhancedMCPValidator:
|
|
70
70
|
- Complete audit trails with evidence-based validation
|
71
71
|
"""
|
72
72
|
|
73
|
-
def __init__(
|
74
|
-
|
73
|
+
def __init__(
|
74
|
+
self,
|
75
|
+
user_profile: Optional[str] = None,
|
76
|
+
console: Optional[Console] = None,
|
77
|
+
mcp_config_path: Optional[str] = None,
|
78
|
+
terraform_directory: Optional[str] = None,
|
79
|
+
):
|
75
80
|
"""
|
76
81
|
Initialize enhanced MCP validator with enterprise profile management and MCP server integration.
|
77
|
-
|
82
|
+
|
78
83
|
Args:
|
79
84
|
user_profile: User-specified profile (--profile parameter) - takes priority over environment
|
80
85
|
console: Rich console for output (optional)
|
@@ -87,16 +92,16 @@ class EnhancedMCPValidator:
|
|
87
92
|
self.tolerance_percent = 5.0 # ±5% tolerance for resource count validation
|
88
93
|
self.validation_cache = {} # Cache for performance optimization
|
89
94
|
self.cache_ttl = 300 # 5 minutes cache TTL
|
90
|
-
|
95
|
+
|
91
96
|
# MCP Server Integration
|
92
97
|
self.mcp_config_path = mcp_config_path or "/Volumes/Working/1xOps/CloudOps-Runbooks/.mcp.json"
|
93
98
|
self.mcp_servers = {}
|
94
99
|
self.mcp_processes = {} # Track running MCP server processes
|
95
|
-
|
100
|
+
|
96
101
|
# AWS Profile Management following proven patterns
|
97
102
|
self.enterprise_profiles = self._resolve_enterprise_profiles()
|
98
103
|
self.aws_sessions = {}
|
99
|
-
|
104
|
+
|
100
105
|
# Terraform integration
|
101
106
|
self.terraform_directory = terraform_directory or "/Volumes/Working/1xOps/CloudOps-Runbooks/terraform-aws"
|
102
107
|
self.terraform_cache = {} # Cache terraform state parsing
|
@@ -104,18 +109,18 @@ class EnhancedMCPValidator:
|
|
104
109
|
|
105
110
|
# Supported AWS services for inventory validation
|
106
111
|
self.supported_services = {
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
112
|
+
"ec2": "EC2 Instances",
|
113
|
+
"s3": "S3 Buckets",
|
114
|
+
"rds": "RDS Instances",
|
115
|
+
"lambda": "Lambda Functions",
|
116
|
+
"vpc": "VPCs",
|
117
|
+
"iam": "IAM Roles",
|
118
|
+
"cloudformation": "CloudFormation Stacks",
|
119
|
+
"elbv2": "Load Balancers",
|
120
|
+
"route53": "Route53 Hosted Zones",
|
121
|
+
"sns": "SNS Topics",
|
122
|
+
"eni": "Network Interfaces",
|
123
|
+
"ebs": "EBS Volumes",
|
119
124
|
}
|
120
125
|
|
121
126
|
# Initialize components
|
@@ -126,7 +131,7 @@ class EnhancedMCPValidator:
|
|
126
131
|
def _resolve_enterprise_profiles(self) -> Dict[str, str]:
|
127
132
|
"""
|
128
133
|
Resolve enterprise AWS profiles using proven 3-tier priority system.
|
129
|
-
|
134
|
+
|
130
135
|
Returns:
|
131
136
|
Dict mapping operation types to resolved profile names
|
132
137
|
"""
|
@@ -136,7 +141,7 @@ class EnhancedMCPValidator:
|
|
136
141
|
"operational": resolve_profile_for_operation_silent("operational", self.user_profile),
|
137
142
|
"single_account": resolve_profile_for_operation_silent("single_account", self.user_profile),
|
138
143
|
}
|
139
|
-
|
144
|
+
|
140
145
|
def _load_mcp_configuration(self) -> None:
|
141
146
|
"""Load and parse MCP server configuration from .mcp.json."""
|
142
147
|
try:
|
@@ -144,90 +149,88 @@ class EnhancedMCPValidator:
|
|
144
149
|
print_warning(f"MCP configuration not found: {self.mcp_config_path}")
|
145
150
|
self.mcp_servers = {}
|
146
151
|
return
|
147
|
-
|
148
|
-
with open(self.mcp_config_path,
|
152
|
+
|
153
|
+
with open(self.mcp_config_path, "r") as f:
|
149
154
|
config = json.load(f)
|
150
|
-
|
155
|
+
|
151
156
|
self.mcp_servers = config.get("mcpServers", {})
|
152
|
-
|
157
|
+
|
153
158
|
# Log MCP server availability
|
154
159
|
available_servers = list(self.mcp_servers.keys())
|
155
|
-
relevant_servers = [
|
156
|
-
|
157
|
-
|
160
|
+
relevant_servers = [
|
161
|
+
s for s in available_servers if s in ["aws-api", "cost-explorer", "iam", "cloudwatch", "terraform-mcp"]
|
162
|
+
]
|
163
|
+
|
164
|
+
print_info(
|
165
|
+
f"MCP servers available: {len(available_servers)} total, {len(relevant_servers)} validation-relevant"
|
166
|
+
)
|
158
167
|
if relevant_servers:
|
159
168
|
self.console.log(f"[dim cyan]Validation servers: {', '.join(relevant_servers)}[/]")
|
160
|
-
|
169
|
+
|
161
170
|
except Exception as e:
|
162
171
|
print_warning(f"Failed to load MCP configuration: {str(e)}")
|
163
172
|
self.mcp_servers = {}
|
164
|
-
|
173
|
+
|
165
174
|
def _substitute_environment_variables(self, server_config: Dict[str, Any]) -> Dict[str, Any]:
|
166
175
|
"""
|
167
176
|
Substitute environment variables in MCP server configuration with resolved profiles.
|
168
|
-
|
177
|
+
|
169
178
|
Args:
|
170
179
|
server_config: MCP server configuration dictionary
|
171
|
-
|
180
|
+
|
172
181
|
Returns:
|
173
182
|
Configuration with environment variables resolved
|
174
183
|
"""
|
175
184
|
config = server_config.copy()
|
176
|
-
|
185
|
+
|
177
186
|
if "env" in config:
|
178
187
|
env = config["env"].copy()
|
179
|
-
|
188
|
+
|
180
189
|
# Substitute profile environment variables with resolved enterprise profiles
|
181
190
|
profile_substitutions = {
|
182
191
|
"${AWS_BILLING_PROFILE}": self.enterprise_profiles["billing"],
|
183
192
|
"${AWS_MANAGEMENT_PROFILE}": self.enterprise_profiles["management"],
|
184
193
|
"${AWS_CENTRALISED_OPS_PROFILE}": self.enterprise_profiles["operational"],
|
185
194
|
}
|
186
|
-
|
195
|
+
|
187
196
|
for key, value in env.items():
|
188
197
|
if isinstance(value, str):
|
189
198
|
for placeholder, resolved_profile in profile_substitutions.items():
|
190
199
|
if placeholder in value:
|
191
200
|
env[key] = value.replace(placeholder, resolved_profile)
|
192
201
|
self.console.log(f"[dim]MCP {key}: {placeholder} → {resolved_profile}[/]")
|
193
|
-
|
202
|
+
|
194
203
|
config["env"] = env
|
195
|
-
|
204
|
+
|
196
205
|
return config
|
197
|
-
|
206
|
+
|
198
207
|
async def _start_mcp_server(self, server_name: str, server_config: Dict[str, Any]) -> Optional[subprocess.Popen]:
|
199
208
|
"""
|
200
209
|
Start an MCP server process with resolved environment variables.
|
201
|
-
|
210
|
+
|
202
211
|
Args:
|
203
212
|
server_name: Name of the MCP server
|
204
213
|
server_config: Server configuration dictionary
|
205
|
-
|
214
|
+
|
206
215
|
Returns:
|
207
216
|
Popen process object if successful, None if failed
|
208
217
|
"""
|
209
218
|
try:
|
210
219
|
# Substitute environment variables
|
211
220
|
resolved_config = self._substitute_environment_variables(server_config)
|
212
|
-
|
221
|
+
|
213
222
|
# Build command
|
214
223
|
command = [resolved_config["command"]] + resolved_config.get("args", [])
|
215
224
|
env = os.environ.copy()
|
216
225
|
env.update(resolved_config.get("env", {}))
|
217
|
-
|
226
|
+
|
218
227
|
# Start process
|
219
228
|
self.console.log(f"[dim]Starting MCP server: {server_name}[/]")
|
220
|
-
process = subprocess.Popen(
|
221
|
-
|
222
|
-
stdout=subprocess.PIPE,
|
223
|
-
stderr=subprocess.PIPE,
|
224
|
-
env=env,
|
225
|
-
text=True
|
226
|
-
)
|
227
|
-
|
229
|
+
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, text=True)
|
230
|
+
|
228
231
|
# Give process time to start
|
229
232
|
await asyncio.sleep(2)
|
230
|
-
|
233
|
+
|
231
234
|
# Check if process is still running
|
232
235
|
if process.poll() is None:
|
233
236
|
self.mcp_processes[server_name] = process
|
@@ -237,11 +240,11 @@ class EnhancedMCPValidator:
|
|
237
240
|
stdout, stderr = process.communicate()
|
238
241
|
print_warning(f"MCP server '{server_name}' failed to start: {stderr[:100]}")
|
239
242
|
return None
|
240
|
-
|
243
|
+
|
241
244
|
except Exception as e:
|
242
245
|
print_warning(f"Failed to start MCP server '{server_name}': {str(e)}")
|
243
246
|
return None
|
244
|
-
|
247
|
+
|
245
248
|
def _stop_mcp_servers(self) -> None:
|
246
249
|
"""Stop all running MCP server processes."""
|
247
250
|
for server_name, process in self.mcp_processes.items():
|
@@ -251,13 +254,13 @@ class EnhancedMCPValidator:
|
|
251
254
|
self.console.log(f"[dim]Stopped MCP server: {server_name}[/]")
|
252
255
|
except Exception as e:
|
253
256
|
self.console.log(f"[yellow]Warning: Could not stop MCP server {server_name}: {str(e)}[/]")
|
254
|
-
|
257
|
+
|
255
258
|
self.mcp_processes.clear()
|
256
259
|
|
257
260
|
def _initialize_aws_sessions(self) -> None:
|
258
261
|
"""Initialize AWS sessions for all enterprise profiles with enhanced error handling."""
|
259
262
|
successful_sessions = 0
|
260
|
-
|
263
|
+
|
261
264
|
for operation_type, profile_name in self.enterprise_profiles.items():
|
262
265
|
try:
|
263
266
|
# Validate profile exists in AWS config
|
@@ -265,42 +268,50 @@ class EnhancedMCPValidator:
|
|
265
268
|
if profile_name not in available_profiles:
|
266
269
|
print_warning(f"Profile '{profile_name}' not found in AWS config for {operation_type}")
|
267
270
|
continue
|
268
|
-
|
271
|
+
|
269
272
|
session = boto3.Session(profile_name=profile_name)
|
270
|
-
|
273
|
+
|
271
274
|
# Test session validity with timeout
|
272
275
|
try:
|
273
276
|
sts_client = session.client("sts")
|
274
277
|
identity = sts_client.get_caller_identity()
|
275
|
-
|
278
|
+
|
276
279
|
self.aws_sessions[operation_type] = {
|
277
280
|
"session": session,
|
278
281
|
"profile": profile_name,
|
279
282
|
"account_id": identity.get("Account"),
|
280
283
|
"user_id": identity.get("UserId", "Unknown"),
|
281
|
-
"region": session.region_name or "us-east-1"
|
284
|
+
"region": session.region_name or "us-east-1",
|
282
285
|
}
|
283
|
-
|
286
|
+
|
284
287
|
successful_sessions += 1
|
285
|
-
print_info(
|
286
|
-
|
288
|
+
print_info(
|
289
|
+
f"✅ MCP session for {operation_type}: {profile_name[:30]}... → Account {identity.get('Account', 'Unknown')}"
|
290
|
+
)
|
291
|
+
|
287
292
|
except Exception as sts_error:
|
288
293
|
if "expired" in str(sts_error).lower() or "token" in str(sts_error).lower():
|
289
|
-
print_warning(
|
294
|
+
print_warning(
|
295
|
+
f"AWS SSO token expired for {operation_type}. Run: aws sso login --profile {profile_name}"
|
296
|
+
)
|
290
297
|
else:
|
291
298
|
print_warning(f"STS validation failed for {operation_type}: {str(sts_error)[:40]}")
|
292
|
-
|
299
|
+
|
293
300
|
except Exception as e:
|
294
301
|
print_warning(f"Session creation failed for {operation_type} ({profile_name[:20]}...): {str(e)[:40]}")
|
295
|
-
|
302
|
+
|
296
303
|
# Log overall session status
|
297
304
|
total_profiles = len(self.enterprise_profiles)
|
298
|
-
self.console.log(
|
299
|
-
|
305
|
+
self.console.log(
|
306
|
+
f"[dim]AWS sessions: {successful_sessions}/{total_profiles} profiles initialized successfully[/]"
|
307
|
+
)
|
308
|
+
|
300
309
|
if successful_sessions == 0:
|
301
310
|
print_error("No AWS sessions could be initialized. Check profile configuration and SSO status.")
|
302
311
|
elif successful_sessions < total_profiles:
|
303
|
-
print_warning(
|
312
|
+
print_warning(
|
313
|
+
f"Only {successful_sessions}/{total_profiles} AWS sessions initialized. Some validations may be limited."
|
314
|
+
)
|
304
315
|
|
305
316
|
def _discover_terraform_state_files(self) -> None:
|
306
317
|
"""Discover terraform state files and configurations in the terraform directory."""
|
@@ -309,22 +320,22 @@ class EnhancedMCPValidator:
|
|
309
320
|
if not terraform_path.exists():
|
310
321
|
print_warning(f"Terraform directory not found: {self.terraform_directory}")
|
311
322
|
return
|
312
|
-
|
323
|
+
|
313
324
|
# Look for terraform configuration files and state references
|
314
325
|
config_files = []
|
315
326
|
state_references = []
|
316
|
-
|
327
|
+
|
317
328
|
# Search for terraform files recursively
|
318
329
|
for tf_file in terraform_path.rglob("*.tf"):
|
319
330
|
config_files.append(str(tf_file))
|
320
|
-
|
321
|
-
# Search for state configuration files
|
331
|
+
|
332
|
+
# Search for state configuration files
|
322
333
|
for state_file in terraform_path.rglob("state.tf"):
|
323
334
|
state_references.append(str(state_file))
|
324
|
-
|
335
|
+
|
325
336
|
self.terraform_state_files = state_references
|
326
337
|
print_info(f"Discovered {len(config_files)} terraform files, {len(state_references)} state configurations")
|
327
|
-
|
338
|
+
|
328
339
|
except Exception as e:
|
329
340
|
print_warning(f"Failed to discover terraform files: {str(e)[:50]}")
|
330
341
|
self.terraform_state_files = []
|
@@ -332,17 +343,17 @@ class EnhancedMCPValidator:
|
|
332
343
|
def _parse_terraform_state_config(self, state_file: str) -> Dict[str, Any]:
|
333
344
|
"""
|
334
345
|
Parse terraform state configuration to extract resource declarations.
|
335
|
-
|
346
|
+
|
336
347
|
Args:
|
337
348
|
state_file: Path to terraform state.tf file
|
338
|
-
|
349
|
+
|
339
350
|
Returns:
|
340
351
|
Dictionary containing parsed terraform configuration
|
341
352
|
"""
|
342
353
|
try:
|
343
|
-
with open(state_file,
|
354
|
+
with open(state_file, "r") as f:
|
344
355
|
content = f.read()
|
345
|
-
|
356
|
+
|
346
357
|
# Extract account ID from directory structure
|
347
358
|
account_id = None
|
348
359
|
path_parts = Path(state_file).parts
|
@@ -352,73 +363,73 @@ class EnhancedMCPValidator:
|
|
352
363
|
if potential_account.isdigit() and len(potential_account) == 12:
|
353
364
|
account_id = potential_account
|
354
365
|
break
|
355
|
-
|
366
|
+
|
356
367
|
# Extract backend configuration
|
357
368
|
backend_bucket = None
|
358
369
|
backend_key = None
|
359
370
|
dynamodb_table = None
|
360
|
-
|
371
|
+
|
361
372
|
# Simple parsing for S3 backend configuration
|
362
|
-
lines = content.split(
|
373
|
+
lines = content.split("\n")
|
363
374
|
in_backend = False
|
364
375
|
for line in lines:
|
365
376
|
line = line.strip()
|
366
377
|
if 'backend "s3"' in line:
|
367
378
|
in_backend = True
|
368
379
|
continue
|
369
|
-
if in_backend and line.startswith(
|
370
|
-
backend_bucket = line.split(
|
371
|
-
elif in_backend and line.startswith(
|
372
|
-
backend_key = line.split(
|
373
|
-
elif in_backend and line.startswith(
|
374
|
-
dynamodb_table = line.split(
|
375
|
-
elif in_backend and line ==
|
380
|
+
if in_backend and line.startswith("bucket"):
|
381
|
+
backend_bucket = line.split("=")[1].strip().strip('"')
|
382
|
+
elif in_backend and line.startswith("key"):
|
383
|
+
backend_key = line.split("=")[1].strip().strip('"')
|
384
|
+
elif in_backend and line.startswith("dynamodb_table"):
|
385
|
+
dynamodb_table = line.split("=")[1].strip().strip('"')
|
386
|
+
elif in_backend and line == "}":
|
376
387
|
in_backend = False
|
377
|
-
|
388
|
+
|
378
389
|
return {
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
390
|
+
"file_path": state_file,
|
391
|
+
"account_id": account_id,
|
392
|
+
"backend_bucket": backend_bucket,
|
393
|
+
"backend_key": backend_key,
|
394
|
+
"dynamodb_table": dynamodb_table,
|
395
|
+
"directory": str(Path(state_file).parent),
|
396
|
+
"parsed_timestamp": datetime.now().isoformat(),
|
386
397
|
}
|
387
|
-
|
398
|
+
|
388
399
|
except Exception as e:
|
389
400
|
print_warning(f"Failed to parse terraform state file {state_file}: {str(e)[:50]}")
|
390
401
|
return {
|
391
|
-
|
392
|
-
|
393
|
-
|
402
|
+
"file_path": state_file,
|
403
|
+
"error": str(e),
|
404
|
+
"parsed_timestamp": datetime.now().isoformat(),
|
394
405
|
}
|
395
406
|
|
396
407
|
def _get_terraform_declared_resources(self, account_id: Optional[str] = None) -> Dict[str, Any]:
|
397
408
|
"""
|
398
409
|
Extract resource declarations from terraform configuration files.
|
399
|
-
|
410
|
+
|
400
411
|
Args:
|
401
412
|
account_id: AWS account ID to filter terraform configurations
|
402
|
-
|
413
|
+
|
403
414
|
Returns:
|
404
415
|
Dictionary containing terraform declared resources by type
|
405
416
|
"""
|
406
417
|
try:
|
407
418
|
declared_resources = {
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
419
|
+
"ec2": 0,
|
420
|
+
"s3": 0,
|
421
|
+
"rds": 0,
|
422
|
+
"lambda": 0,
|
423
|
+
"vpc": 0,
|
424
|
+
"iam": 0,
|
425
|
+
"cloudformation": 0,
|
426
|
+
"elbv2": 0,
|
427
|
+
"route53": 0,
|
428
|
+
"sns": 0,
|
418
429
|
}
|
419
|
-
|
430
|
+
|
420
431
|
config_files = []
|
421
|
-
|
432
|
+
|
422
433
|
# If account_id provided, look for account-specific terraform files
|
423
434
|
if account_id:
|
424
435
|
account_path = Path(self.terraform_directory) / "account" / account_id
|
@@ -428,76 +439,76 @@ class EnhancedMCPValidator:
|
|
428
439
|
# Look in all terraform files
|
429
440
|
terraform_path = Path(self.terraform_directory)
|
430
441
|
config_files.extend(terraform_path.rglob("*.tf"))
|
431
|
-
|
442
|
+
|
432
443
|
resource_patterns = {
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
444
|
+
"ec2": ["aws_instance", "aws_launch_template"],
|
445
|
+
"s3": ["aws_s3_bucket"],
|
446
|
+
"rds": ["aws_db_instance", "aws_rds_cluster"],
|
447
|
+
"lambda": ["aws_lambda_function"],
|
448
|
+
"vpc": ["aws_vpc"],
|
449
|
+
"iam": ["aws_iam_role", "aws_iam_user"],
|
450
|
+
"cloudformation": ["aws_cloudformation_stack"],
|
451
|
+
"elbv2": ["aws_lb", "aws_alb"],
|
452
|
+
"route53": ["aws_route53_zone"],
|
453
|
+
"sns": ["aws_sns_topic"],
|
443
454
|
}
|
444
|
-
|
455
|
+
|
445
456
|
# Parse terraform files for resource declarations
|
446
457
|
for config_file in config_files:
|
447
458
|
try:
|
448
|
-
with open(config_file,
|
459
|
+
with open(config_file, "r") as f:
|
449
460
|
content = f.read()
|
450
|
-
|
461
|
+
|
451
462
|
# Count resource declarations
|
452
463
|
for service, patterns in resource_patterns.items():
|
453
464
|
for pattern in patterns:
|
454
465
|
declared_resources[service] += content.count(f'resource "{pattern}"')
|
455
|
-
|
466
|
+
|
456
467
|
except Exception as e:
|
457
468
|
continue # Skip files that can't be read
|
458
|
-
|
469
|
+
|
459
470
|
return {
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
471
|
+
"account_id": account_id,
|
472
|
+
"declared_resources": declared_resources,
|
473
|
+
"files_parsed": len(config_files),
|
474
|
+
"data_source": "terraform_configuration_files",
|
475
|
+
"timestamp": datetime.now().isoformat(),
|
465
476
|
}
|
466
|
-
|
477
|
+
|
467
478
|
except Exception as e:
|
468
479
|
print_warning(f"Failed to extract terraform declared resources: {str(e)[:50]}")
|
469
480
|
return {
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
481
|
+
"account_id": account_id,
|
482
|
+
"declared_resources": {service: 0 for service in self.supported_services.keys()},
|
483
|
+
"error": str(e),
|
484
|
+
"data_source": "terraform_configuration_error",
|
485
|
+
"timestamp": datetime.now().isoformat(),
|
475
486
|
}
|
476
487
|
|
477
488
|
async def validate_with_mcp_servers(self, runbooks_inventory: Dict[str, Any]) -> Dict[str, Any]:
|
478
489
|
"""
|
479
490
|
Enhanced validation using real MCP servers from .mcp.json configuration.
|
480
|
-
|
491
|
+
|
481
492
|
Provides comprehensive 4-way validation:
|
482
493
|
1. Runbooks inventory data
|
483
|
-
2. Direct AWS API calls
|
494
|
+
2. Direct AWS API calls
|
484
495
|
3. Real MCP server responses
|
485
496
|
4. Terraform state drift detection
|
486
|
-
|
497
|
+
|
487
498
|
Args:
|
488
499
|
runbooks_inventory: Inventory data from runbooks collection
|
489
|
-
|
500
|
+
|
490
501
|
Returns:
|
491
502
|
Enhanced validation results with MCP server integration
|
492
503
|
"""
|
493
504
|
validation_results = {
|
494
505
|
"validation_timestamp": datetime.now().isoformat(),
|
495
|
-
"validation_method": "enhanced_mcp_server_integration",
|
506
|
+
"validation_method": "enhanced_mcp_server_integration",
|
496
507
|
"mcp_integration": {
|
497
508
|
"config_loaded": bool(self.mcp_servers),
|
498
509
|
"servers_available": list(self.mcp_servers.keys()),
|
499
510
|
"servers_started": {},
|
500
|
-
"validation_sources": []
|
511
|
+
"validation_sources": [],
|
501
512
|
},
|
502
513
|
"enterprise_profiles": self.enterprise_profiles,
|
503
514
|
"profiles_validated": 0,
|
@@ -508,35 +519,32 @@ class EnhancedMCPValidator:
|
|
508
519
|
"start_time": time.time(),
|
509
520
|
"mcp_server_startup_time": 0,
|
510
521
|
"validation_execution_time": 0,
|
511
|
-
"total_execution_time": 0
|
512
|
-
}
|
522
|
+
"total_execution_time": 0,
|
523
|
+
},
|
513
524
|
}
|
514
525
|
|
515
526
|
self.console.log(f"[blue]⚡ Starting enhanced MCP server validation[/]")
|
516
|
-
|
527
|
+
|
517
528
|
# Start relevant MCP servers
|
518
529
|
await self._start_relevant_mcp_servers(validation_results)
|
519
|
-
|
530
|
+
|
520
531
|
# Execute validation with all available sources
|
521
532
|
with Progress(
|
522
533
|
SpinnerColumn(),
|
523
534
|
TextColumn("[progress.description]{task.description}"),
|
524
535
|
BarColumn(),
|
525
|
-
TaskProgressColumn(),
|
536
|
+
TaskProgressColumn(),
|
526
537
|
TimeElapsedColumn(),
|
527
538
|
console=self.console,
|
528
539
|
) as progress:
|
529
540
|
task = progress.add_task("MCP server validation...", total=len(self.aws_sessions))
|
530
|
-
|
541
|
+
|
531
542
|
# Parallel execution for <20s target
|
532
543
|
with ThreadPoolExecutor(max_workers=min(3, len(self.aws_sessions))) as executor:
|
533
544
|
future_to_operation = {}
|
534
545
|
for operation_type, session_info in self.aws_sessions.items():
|
535
546
|
future = executor.submit(
|
536
|
-
self._validate_operation_with_mcp_servers,
|
537
|
-
operation_type,
|
538
|
-
session_info,
|
539
|
-
runbooks_inventory
|
547
|
+
self._validate_operation_with_mcp_servers, operation_type, session_info, runbooks_inventory
|
540
548
|
)
|
541
549
|
future_to_operation[future] = operation_type
|
542
550
|
|
@@ -555,17 +563,17 @@ class EnhancedMCPValidator:
|
|
555
563
|
# Finalize results and cleanup
|
556
564
|
self._finalize_mcp_validation_results(validation_results)
|
557
565
|
self._stop_mcp_servers()
|
558
|
-
|
566
|
+
|
559
567
|
return validation_results
|
560
|
-
|
568
|
+
|
561
569
|
async def _start_relevant_mcp_servers(self, validation_results: Dict[str, Any]) -> None:
|
562
570
|
"""Start MCP servers relevant to validation operations."""
|
563
571
|
startup_start = time.time()
|
564
|
-
|
572
|
+
|
565
573
|
# Priority servers for validation
|
566
|
-
relevant_servers = [
|
574
|
+
relevant_servers = ["aws-api", "cost-explorer", "iam", "cloudwatch"]
|
567
575
|
started_servers = []
|
568
|
-
|
576
|
+
|
569
577
|
for server_name in relevant_servers:
|
570
578
|
if server_name in self.mcp_servers:
|
571
579
|
server_config = self.mcp_servers[server_name]
|
@@ -575,30 +583,30 @@ class EnhancedMCPValidator:
|
|
575
583
|
validation_results["mcp_integration"]["servers_started"][server_name] = {
|
576
584
|
"status": "started",
|
577
585
|
"pid": process.pid,
|
578
|
-
"profile_used": self._get_server_profile(server_config)
|
586
|
+
"profile_used": self._get_server_profile(server_config),
|
579
587
|
}
|
580
588
|
else:
|
581
589
|
validation_results["mcp_integration"]["servers_started"][server_name] = {
|
582
590
|
"status": "failed",
|
583
|
-
"error": "Failed to start process"
|
591
|
+
"error": "Failed to start process",
|
584
592
|
}
|
585
|
-
|
593
|
+
|
586
594
|
validation_results["mcp_integration"]["validation_sources"] = [
|
587
595
|
"runbooks_inventory",
|
588
|
-
"direct_aws_apis",
|
589
|
-
f"mcp_servers_{len(started_servers)}"
|
596
|
+
"direct_aws_apis",
|
597
|
+
f"mcp_servers_{len(started_servers)}",
|
590
598
|
]
|
591
|
-
|
599
|
+
|
592
600
|
if self.terraform_state_files:
|
593
601
|
validation_results["mcp_integration"]["validation_sources"].append("terraform_state")
|
594
|
-
|
602
|
+
|
595
603
|
validation_results["performance_metrics"]["mcp_server_startup_time"] = time.time() - startup_start
|
596
|
-
|
604
|
+
|
597
605
|
if started_servers:
|
598
606
|
print_success(f"✅ MCP servers started: {', '.join(started_servers)}")
|
599
607
|
else:
|
600
608
|
print_warning("⚠️ No MCP servers started - using direct API validation only")
|
601
|
-
|
609
|
+
|
602
610
|
def _get_server_profile(self, server_config: Dict[str, Any]) -> Optional[str]:
|
603
611
|
"""Extract the profile name used by an MCP server configuration."""
|
604
612
|
env = server_config.get("env", {})
|
@@ -606,29 +614,35 @@ class EnhancedMCPValidator:
|
|
606
614
|
if "PROFILE" in key and isinstance(value, str) and not value.startswith("${"):
|
607
615
|
return value
|
608
616
|
return None
|
609
|
-
|
610
|
-
def _validate_operation_with_mcp_servers(
|
611
|
-
|
617
|
+
|
618
|
+
def _validate_operation_with_mcp_servers(
|
619
|
+
self, operation_type: str, session_info: Dict[str, Any], runbooks_inventory: Dict[str, Any]
|
620
|
+
) -> Optional[Dict[str, Any]]:
|
612
621
|
"""Validate a single operation using all available validation sources."""
|
613
622
|
try:
|
614
623
|
session = session_info["session"]
|
615
624
|
profile_name = session_info["profile"]
|
616
625
|
account_id = session_info["account_id"]
|
617
|
-
|
626
|
+
|
618
627
|
# Get validation data from all sources
|
619
628
|
runbooks_data = self._extract_runbooks_inventory_data(runbooks_inventory, operation_type, account_id)
|
620
629
|
direct_aws_data = asyncio.run(self._get_independent_inventory_data(session, profile_name))
|
621
630
|
mcp_server_data = self._get_mcp_server_data(operation_type, account_id)
|
622
631
|
terraform_data = self._get_terraform_declared_resources(account_id)
|
623
|
-
|
632
|
+
|
624
633
|
# Calculate comprehensive validation accuracy
|
625
634
|
validation_result = self._calculate_comprehensive_accuracy(
|
626
|
-
runbooks_data,
|
627
|
-
|
635
|
+
runbooks_data,
|
636
|
+
direct_aws_data,
|
637
|
+
mcp_server_data,
|
638
|
+
terraform_data,
|
639
|
+
operation_type,
|
640
|
+
profile_name,
|
641
|
+
account_id,
|
628
642
|
)
|
629
|
-
|
643
|
+
|
630
644
|
return validation_result
|
631
|
-
|
645
|
+
|
632
646
|
except Exception as e:
|
633
647
|
return {
|
634
648
|
"operation_type": operation_type,
|
@@ -637,17 +651,17 @@ class EnhancedMCPValidator:
|
|
637
651
|
"overall_accuracy_percent": 0.0,
|
638
652
|
"passed_validation": False,
|
639
653
|
"error": str(e),
|
640
|
-
"validation_status": "ERROR"
|
654
|
+
"validation_status": "ERROR",
|
641
655
|
}
|
642
|
-
|
656
|
+
|
643
657
|
def _get_mcp_server_data(self, operation_type: str, account_id: Optional[str]) -> Dict[str, Any]:
|
644
658
|
"""
|
645
659
|
Get validation data from MCP servers (placeholder for actual MCP client implementation).
|
646
|
-
|
660
|
+
|
647
661
|
Args:
|
648
662
|
operation_type: Type of operation (billing, management, operational)
|
649
663
|
account_id: AWS account ID for context
|
650
|
-
|
664
|
+
|
651
665
|
Returns:
|
652
666
|
MCP server validation data
|
653
667
|
"""
|
@@ -659,28 +673,34 @@ class EnhancedMCPValidator:
|
|
659
673
|
"account_id": account_id,
|
660
674
|
"resource_counts": {},
|
661
675
|
"servers_queried": [],
|
662
|
-
"validation_timestamp": datetime.now().isoformat()
|
676
|
+
"validation_timestamp": datetime.now().isoformat(),
|
663
677
|
}
|
664
|
-
|
678
|
+
|
665
679
|
# Check which servers are running and could provide data
|
666
680
|
for server_name, process in self.mcp_processes.items():
|
667
681
|
if process and process.poll() is None: # Server is running
|
668
682
|
mcp_data["servers_queried"].append(server_name)
|
669
|
-
|
683
|
+
|
670
684
|
# For demonstration, populate with placeholder data structure
|
671
685
|
# Real implementation would use MCP client to query running servers
|
672
686
|
for service in self.supported_services.keys():
|
673
687
|
mcp_data["resource_counts"][service] = 0 # Placeholder
|
674
|
-
|
688
|
+
|
675
689
|
return mcp_data
|
676
690
|
|
677
|
-
def _calculate_comprehensive_accuracy(
|
678
|
-
|
679
|
-
|
680
|
-
|
691
|
+
def _calculate_comprehensive_accuracy(
|
692
|
+
self,
|
693
|
+
runbooks_data: Dict,
|
694
|
+
direct_aws_data: Dict,
|
695
|
+
mcp_server_data: Dict,
|
696
|
+
terraform_data: Dict,
|
697
|
+
operation_type: str,
|
698
|
+
profile_name: str,
|
699
|
+
account_id: Optional[str],
|
700
|
+
) -> Dict[str, Any]:
|
681
701
|
"""
|
682
702
|
Calculate comprehensive accuracy across all validation sources.
|
683
|
-
|
703
|
+
|
684
704
|
Args:
|
685
705
|
runbooks_data: Data from runbooks inventory
|
686
706
|
direct_aws_data: Data from direct AWS API calls
|
@@ -689,7 +709,7 @@ class EnhancedMCPValidator:
|
|
689
709
|
operation_type: Operation type being validated
|
690
710
|
profile_name: AWS profile name
|
691
711
|
account_id: AWS account ID
|
692
|
-
|
712
|
+
|
693
713
|
Returns:
|
694
714
|
Comprehensive validation result
|
695
715
|
"""
|
@@ -702,7 +722,7 @@ class EnhancedMCPValidator:
|
|
702
722
|
resource_validations = {}
|
703
723
|
total_variance = 0.0
|
704
724
|
valid_comparisons = 0
|
705
|
-
|
725
|
+
|
706
726
|
# Comprehensive validation for each resource type
|
707
727
|
for resource_type in self.supported_services.keys():
|
708
728
|
runbooks_count = runbooks_counts.get(resource_type, 0)
|
@@ -713,7 +733,7 @@ class EnhancedMCPValidator:
|
|
713
733
|
# Calculate variance across all sources
|
714
734
|
all_counts = [runbooks_count, direct_aws_count, mcp_server_count, terraform_count]
|
715
735
|
active_counts = [c for c in all_counts if c > 0]
|
716
|
-
|
736
|
+
|
717
737
|
if not active_counts:
|
718
738
|
# All sources report zero - perfect alignment
|
719
739
|
accuracy_percent = 100.0
|
@@ -742,7 +762,7 @@ class EnhancedMCPValidator:
|
|
742
762
|
"variance_percent": variance,
|
743
763
|
"validation_status": validation_status,
|
744
764
|
"passed_validation": accuracy_percent >= self.validation_threshold,
|
745
|
-
"sources_with_data": len(active_counts)
|
765
|
+
"sources_with_data": len(active_counts),
|
746
766
|
}
|
747
767
|
|
748
768
|
# Include in total variance calculation
|
@@ -767,9 +787,9 @@ class EnhancedMCPValidator:
|
|
767
787
|
"runbooks_inventory": bool(runbooks_counts),
|
768
788
|
"direct_aws_apis": bool(direct_aws_counts),
|
769
789
|
"mcp_servers": len(mcp_server_data.get("servers_queried", [])),
|
770
|
-
"terraform_state": bool(terraform_counts)
|
790
|
+
"terraform_state": bool(terraform_counts),
|
771
791
|
},
|
772
|
-
"accuracy_category": self._categorize_inventory_accuracy(overall_accuracy)
|
792
|
+
"accuracy_category": self._categorize_inventory_accuracy(overall_accuracy),
|
773
793
|
}
|
774
794
|
|
775
795
|
except Exception as e:
|
@@ -780,19 +800,19 @@ class EnhancedMCPValidator:
|
|
780
800
|
"overall_accuracy_percent": 0.0,
|
781
801
|
"passed_validation": False,
|
782
802
|
"error": str(e),
|
783
|
-
"validation_status": "ERROR"
|
803
|
+
"validation_status": "ERROR",
|
784
804
|
}
|
785
|
-
|
805
|
+
|
786
806
|
def _finalize_mcp_validation_results(self, validation_results: Dict[str, Any]) -> None:
|
787
807
|
"""Finalize MCP validation results with comprehensive metrics."""
|
788
808
|
profile_results = validation_results["profile_results"]
|
789
|
-
|
809
|
+
|
790
810
|
# Calculate performance metrics
|
791
811
|
start_time = validation_results["performance_metrics"]["start_time"]
|
792
812
|
validation_results["performance_metrics"]["total_execution_time"] = time.time() - start_time
|
793
813
|
validation_results["performance_metrics"]["validation_execution_time"] = (
|
794
|
-
validation_results["performance_metrics"]["total_execution_time"]
|
795
|
-
validation_results["performance_metrics"]["mcp_server_startup_time"]
|
814
|
+
validation_results["performance_metrics"]["total_execution_time"]
|
815
|
+
- validation_results["performance_metrics"]["mcp_server_startup_time"]
|
796
816
|
)
|
797
817
|
|
798
818
|
if not profile_results:
|
@@ -811,7 +831,7 @@ class EnhancedMCPValidator:
|
|
811
831
|
|
812
832
|
# Display enhanced results
|
813
833
|
self._display_mcp_validation_results(validation_results)
|
814
|
-
|
834
|
+
|
815
835
|
def _display_mcp_validation_results(self, results: Dict[str, Any]) -> None:
|
816
836
|
"""Display enhanced MCP validation results with server integration details."""
|
817
837
|
overall_accuracy = results.get("total_accuracy", 0)
|
@@ -820,36 +840,38 @@ class EnhancedMCPValidator:
|
|
820
840
|
performance_metrics = results.get("performance_metrics", {})
|
821
841
|
|
822
842
|
self.console.print(f"\n[bright_cyan]🔍 Enhanced MCP Server Validation Results[/]")
|
823
|
-
|
843
|
+
|
824
844
|
# Display MCP integration summary
|
825
845
|
servers_started = mcp_integration.get("servers_started", {})
|
826
846
|
if servers_started:
|
827
847
|
successful_servers = [name for name, info in servers_started.items() if info.get("status") == "started"]
|
828
848
|
failed_servers = [name for name, info in servers_started.items() if info.get("status") == "failed"]
|
829
|
-
|
849
|
+
|
830
850
|
if successful_servers:
|
831
851
|
self.console.print(f"[dim green]✅ MCP Servers: {', '.join(successful_servers)}[/]")
|
832
852
|
if failed_servers:
|
833
853
|
self.console.print(f"[dim red]❌ Failed Servers: {', '.join(failed_servers)}[/]")
|
834
|
-
|
854
|
+
|
835
855
|
# Display validation sources
|
836
856
|
validation_sources = mcp_integration.get("validation_sources", [])
|
837
857
|
self.console.print(f"[dim cyan]🔗 Validation Sources: {', '.join(validation_sources)}[/]")
|
838
|
-
|
858
|
+
|
839
859
|
# Display performance metrics
|
840
860
|
total_time = performance_metrics.get("total_execution_time", 0)
|
841
861
|
startup_time = performance_metrics.get("mcp_server_startup_time", 0)
|
842
862
|
validation_time = performance_metrics.get("validation_execution_time", 0)
|
843
|
-
|
844
|
-
self.console.print(
|
845
|
-
|
863
|
+
|
864
|
+
self.console.print(
|
865
|
+
f"[dim]⚡ Performance: {total_time:.1f}s total ({startup_time:.1f}s startup, {validation_time:.1f}s validation)[/]"
|
866
|
+
)
|
867
|
+
|
846
868
|
# Display per-operation results
|
847
869
|
for result in results.get("profile_results", []):
|
848
870
|
operation_type = result.get("operation_type", "Unknown")
|
849
871
|
accuracy = result.get("overall_accuracy_percent", 0)
|
850
872
|
status = result.get("validation_status", "UNKNOWN")
|
851
873
|
account_id = result.get("account_id", "Unknown")
|
852
|
-
|
874
|
+
|
853
875
|
# Determine display formatting
|
854
876
|
if status == "PASSED" and accuracy >= 99.5:
|
855
877
|
icon = "✅"
|
@@ -863,27 +885,32 @@ class EnhancedMCPValidator:
|
|
863
885
|
else:
|
864
886
|
icon = "❌"
|
865
887
|
color = "red"
|
866
|
-
|
867
|
-
self.console.print(
|
868
|
-
|
888
|
+
|
889
|
+
self.console.print(
|
890
|
+
f"[dim] {operation_type:12s} ({account_id}): {icon} [{color}]{accuracy:.1f}% accuracy[/]"
|
891
|
+
)
|
892
|
+
|
869
893
|
# Show resource-level details for significant variances
|
870
894
|
resource_validations = result.get("resource_validations", {})
|
871
895
|
for resource_type, resource_data in resource_validations.items():
|
872
896
|
if resource_data.get("variance_percent", 0) > 10: # Show resources with >10% variance
|
873
897
|
variance = resource_data["variance_percent"]
|
874
898
|
sources_count = resource_data["sources_with_data"]
|
875
|
-
self.console.print(
|
899
|
+
self.console.print(
|
900
|
+
f"[dim] {self.supported_services.get(resource_type, resource_type):15s}: ⚠️ {variance:.1f}% variance ({sources_count} sources)[/]"
|
901
|
+
)
|
876
902
|
|
877
903
|
# Overall validation summary
|
878
904
|
if passed:
|
879
905
|
print_success(f"✅ Enhanced MCP Validation PASSED: {overall_accuracy:.1f}% accuracy achieved")
|
880
906
|
else:
|
881
907
|
print_warning(f"⚠️ Enhanced MCP Validation: {overall_accuracy:.1f}% accuracy (≥99.5% required)")
|
882
|
-
|
908
|
+
|
883
909
|
print_info(f"Enterprise compliance: {results.get('profiles_validated', 0)} operations validated")
|
884
910
|
|
885
|
-
def _extract_runbooks_inventory_data(
|
886
|
-
|
911
|
+
def _extract_runbooks_inventory_data(
|
912
|
+
self, runbooks_inventory: Dict[str, Any], operation_type: str, account_id: Optional[str] = None
|
913
|
+
) -> Dict[str, Any]:
|
887
914
|
"""
|
888
915
|
Extract inventory data from runbooks results for comprehensive validation.
|
889
916
|
Enhanced to work with operation types instead of profile names.
|
@@ -892,7 +919,7 @@ class EnhancedMCPValidator:
|
|
892
919
|
# Handle various runbooks inventory data structures
|
893
920
|
resource_counts = {}
|
894
921
|
regions_discovered = []
|
895
|
-
|
922
|
+
|
896
923
|
# Try operation_type key first
|
897
924
|
if operation_type in runbooks_inventory:
|
898
925
|
operation_data = runbooks_inventory[operation_type]
|
@@ -907,14 +934,16 @@ class EnhancedMCPValidator:
|
|
907
934
|
else:
|
908
935
|
resource_counts = runbooks_inventory.get("resource_counts", {})
|
909
936
|
regions_discovered = runbooks_inventory.get("regions", [])
|
910
|
-
|
937
|
+
|
911
938
|
return {
|
912
939
|
"operation_type": operation_type,
|
913
940
|
"account_id": account_id,
|
914
941
|
"resource_counts": resource_counts,
|
915
942
|
"regions_discovered": regions_discovered,
|
916
943
|
"data_source": "runbooks_inventory_collection",
|
917
|
-
"extraction_method": f"operation_type_{operation_type}"
|
944
|
+
"extraction_method": f"operation_type_{operation_type}"
|
945
|
+
if operation_type in runbooks_inventory
|
946
|
+
else "fallback",
|
918
947
|
}
|
919
948
|
except Exception as e:
|
920
949
|
self.console.log(f"[yellow]Warning: Error extracting runbooks inventory data: {str(e)}[/]")
|
@@ -924,13 +953,13 @@ class EnhancedMCPValidator:
|
|
924
953
|
"resource_counts": {},
|
925
954
|
"regions_discovered": [],
|
926
955
|
"data_source": "runbooks_inventory_collection_error",
|
927
|
-
"error": str(e)
|
956
|
+
"error": str(e),
|
928
957
|
}
|
929
958
|
|
930
959
|
async def validate_inventory_data_async(self, runbooks_inventory: Dict[str, Any]) -> Dict[str, Any]:
|
931
960
|
"""
|
932
961
|
Enhanced 3-way validation: runbooks inventory vs AWS API vs terraform state.
|
933
|
-
|
962
|
+
|
934
963
|
Provides comprehensive drift detection between declared infrastructure
|
935
964
|
and actual deployed resources with enterprise accuracy requirements.
|
936
965
|
|
@@ -951,18 +980,20 @@ class EnhancedMCPValidator:
|
|
951
980
|
"terraform_integration": {
|
952
981
|
"enabled": len(self.terraform_state_files) > 0,
|
953
982
|
"state_files_discovered": len(self.terraform_state_files),
|
954
|
-
"drift_analysis": {}
|
983
|
+
"drift_analysis": {},
|
955
984
|
},
|
956
985
|
}
|
957
986
|
|
958
987
|
# Enhanced parallel processing with terraform integration for <20s performance target
|
959
|
-
self.console.log(
|
960
|
-
|
988
|
+
self.console.log(
|
989
|
+
f"[blue]⚡ Starting enhanced 3-way validation with {min(5, len(self.aws_sessions))} workers[/]"
|
990
|
+
)
|
991
|
+
|
961
992
|
with Progress(
|
962
993
|
SpinnerColumn(),
|
963
994
|
TextColumn("[progress.description]{task.description}"),
|
964
995
|
BarColumn(),
|
965
|
-
TaskProgressColumn(),
|
996
|
+
TaskProgressColumn(),
|
966
997
|
TimeElapsedColumn(),
|
967
998
|
console=self.console,
|
968
999
|
) as progress:
|
@@ -973,7 +1004,9 @@ class EnhancedMCPValidator:
|
|
973
1004
|
# Submit all validation tasks
|
974
1005
|
future_to_profile = {}
|
975
1006
|
for profile, session in self.aws_sessions.items():
|
976
|
-
future = executor.submit(
|
1007
|
+
future = executor.submit(
|
1008
|
+
self._validate_profile_with_drift_detection, profile, session, runbooks_inventory
|
1009
|
+
)
|
977
1010
|
future_to_profile[future] = profile
|
978
1011
|
|
979
1012
|
# Collect results as they complete (maintain progress visibility)
|
@@ -992,7 +1025,9 @@ class EnhancedMCPValidator:
|
|
992
1025
|
self._finalize_enhanced_validation_results(validation_results)
|
993
1026
|
return validation_results
|
994
1027
|
|
995
|
-
def _validate_profile_with_drift_detection(
|
1028
|
+
def _validate_profile_with_drift_detection(
|
1029
|
+
self, profile: str, session: boto3.Session, runbooks_inventory: Dict[str, Any]
|
1030
|
+
) -> Optional[Dict[str, Any]]:
|
996
1031
|
"""Enhanced validation with 3-way drift detection: runbooks vs API vs terraform."""
|
997
1032
|
try:
|
998
1033
|
# Get AWS account ID for terraform state correlation
|
@@ -1014,11 +1049,7 @@ class EnhancedMCPValidator:
|
|
1014
1049
|
|
1015
1050
|
# Calculate 3-way accuracy and drift detection
|
1016
1051
|
drift_result = self._calculate_drift_analysis(
|
1017
|
-
runbooks_inventory_data,
|
1018
|
-
aws_inventory_data,
|
1019
|
-
terraform_data,
|
1020
|
-
profile,
|
1021
|
-
account_id
|
1052
|
+
runbooks_inventory_data, aws_inventory_data, terraform_data, profile, account_id
|
1022
1053
|
)
|
1023
1054
|
return drift_result
|
1024
1055
|
|
@@ -1031,10 +1062,12 @@ class EnhancedMCPValidator:
|
|
1031
1062
|
"error": str(e),
|
1032
1063
|
"validation_status": "ERROR",
|
1033
1064
|
"account_id": None,
|
1034
|
-
"drift_analysis": {}
|
1065
|
+
"drift_analysis": {},
|
1035
1066
|
}
|
1036
1067
|
|
1037
|
-
def _validate_profile_inventory_sync(
|
1068
|
+
def _validate_profile_inventory_sync(
|
1069
|
+
self, profile: str, session: boto3.Session, runbooks_inventory: Dict[str, Any]
|
1070
|
+
) -> Optional[Dict[str, Any]]:
|
1038
1071
|
"""Synchronous wrapper for profile inventory validation (for parallel execution)."""
|
1039
1072
|
try:
|
1040
1073
|
# Get independent resource counts from AWS API
|
@@ -1051,17 +1084,19 @@ class EnhancedMCPValidator:
|
|
1051
1084
|
# Return None for failed validations (handled in calling function)
|
1052
1085
|
return None
|
1053
1086
|
|
1054
|
-
def _calculate_drift_analysis(
|
1087
|
+
def _calculate_drift_analysis(
|
1088
|
+
self, runbooks_data: Dict, aws_data: Dict, terraform_data: Dict, profile: str, account_id: Optional[str]
|
1089
|
+
) -> Dict[str, Any]:
|
1055
1090
|
"""
|
1056
1091
|
Calculate comprehensive drift analysis between runbooks, AWS API, and terraform.
|
1057
|
-
|
1092
|
+
|
1058
1093
|
Args:
|
1059
1094
|
runbooks_data: Inventory data from runbooks
|
1060
1095
|
aws_data: Inventory data from AWS API
|
1061
1096
|
terraform_data: Declared resources from terraform
|
1062
1097
|
profile: Profile name for validation
|
1063
1098
|
account_id: AWS account ID
|
1064
|
-
|
1099
|
+
|
1065
1100
|
Returns:
|
1066
1101
|
Comprehensive drift analysis with accuracy metrics
|
1067
1102
|
"""
|
@@ -1073,7 +1108,7 @@ class EnhancedMCPValidator:
|
|
1073
1108
|
resource_drift_analysis = {}
|
1074
1109
|
total_variance = 0.0
|
1075
1110
|
valid_comparisons = 0
|
1076
|
-
|
1111
|
+
|
1077
1112
|
# Analyze each resource type across all 3 sources
|
1078
1113
|
for resource_type in self.supported_services.keys():
|
1079
1114
|
runbooks_count = runbooks_counts.get(resource_type, 0)
|
@@ -1084,10 +1119,10 @@ class EnhancedMCPValidator:
|
|
1084
1119
|
api_drift = abs(runbooks_count - aws_count) if runbooks_count > 0 or aws_count > 0 else 0
|
1085
1120
|
iac_drift = abs(aws_count - terraform_count) if aws_count > 0 or terraform_count > 0 else 0
|
1086
1121
|
total_drift = abs(runbooks_count - terraform_count) if runbooks_count > 0 or terraform_count > 0 else 0
|
1087
|
-
|
1122
|
+
|
1088
1123
|
# Determine max count for percentage calculations
|
1089
1124
|
max_count = max(runbooks_count, aws_count, terraform_count)
|
1090
|
-
|
1125
|
+
|
1091
1126
|
# Calculate accuracy percentages
|
1092
1127
|
if max_count == 0:
|
1093
1128
|
# All sources report zero - perfect alignment
|
@@ -1112,10 +1147,14 @@ class EnhancedMCPValidator:
|
|
1112
1147
|
recommendations = []
|
1113
1148
|
if iac_drift > 0:
|
1114
1149
|
if aws_count > terraform_count:
|
1115
|
-
recommendations.append(
|
1150
|
+
recommendations.append(
|
1151
|
+
f"Consider updating terraform to declare {aws_count - terraform_count} additional {resource_type} resources"
|
1152
|
+
)
|
1116
1153
|
elif terraform_count > aws_count:
|
1117
|
-
recommendations.append(
|
1118
|
-
|
1154
|
+
recommendations.append(
|
1155
|
+
f"Investigate {terraform_count - aws_count} terraform-declared {resource_type} resources not found in AWS"
|
1156
|
+
)
|
1157
|
+
|
1119
1158
|
if api_drift > 0:
|
1120
1159
|
recommendations.append(f"Review inventory collection accuracy for {resource_type} resources")
|
1121
1160
|
|
@@ -1131,12 +1170,12 @@ class EnhancedMCPValidator:
|
|
1131
1170
|
"overall_accuracy_percent": overall_accuracy,
|
1132
1171
|
"drift_status": drift_status,
|
1133
1172
|
"passed_validation": overall_accuracy >= self.validation_threshold,
|
1134
|
-
"recommendations": recommendations
|
1173
|
+
"recommendations": recommendations,
|
1135
1174
|
}
|
1136
1175
|
|
1137
1176
|
# Include in total variance calculation if any resources exist
|
1138
1177
|
if max_count > 0:
|
1139
|
-
total_variance +=
|
1178
|
+
total_variance += 100.0 - overall_accuracy
|
1140
1179
|
valid_comparisons += 1
|
1141
1180
|
|
1142
1181
|
# Calculate overall metrics
|
@@ -1146,15 +1185,20 @@ class EnhancedMCPValidator:
|
|
1146
1185
|
# Generate account-level recommendations
|
1147
1186
|
account_recommendations = []
|
1148
1187
|
high_drift_resources = [
|
1149
|
-
resource
|
1188
|
+
resource
|
1189
|
+
for resource, data in resource_drift_analysis.items()
|
1150
1190
|
if data["drift_status"] != "NO_DRIFT" and data["total_drift"] > 0
|
1151
1191
|
]
|
1152
|
-
|
1192
|
+
|
1153
1193
|
if high_drift_resources:
|
1154
|
-
account_recommendations.append(
|
1155
|
-
|
1194
|
+
account_recommendations.append(
|
1195
|
+
f"Review terraform configuration for {len(high_drift_resources)} resource types with detected drift"
|
1196
|
+
)
|
1197
|
+
|
1156
1198
|
if terraform_data.get("files_parsed", 0) == 0:
|
1157
|
-
account_recommendations.append(
|
1199
|
+
account_recommendations.append(
|
1200
|
+
"No terraform configuration found for this account - consider implementing Infrastructure as Code"
|
1201
|
+
)
|
1158
1202
|
|
1159
1203
|
return {
|
1160
1204
|
"profile": profile,
|
@@ -1171,9 +1215,12 @@ class EnhancedMCPValidator:
|
|
1171
1215
|
"total_resource_types": len(resource_drift_analysis),
|
1172
1216
|
"drift_detected": len(high_drift_resources),
|
1173
1217
|
"no_drift": len(resource_drift_analysis) - len(high_drift_resources),
|
1174
|
-
"highest_drift_resource": max(
|
1175
|
-
|
1176
|
-
|
1218
|
+
"highest_drift_resource": max(
|
1219
|
+
resource_drift_analysis.keys(), key=lambda x: resource_drift_analysis[x]["total_drift"]
|
1220
|
+
)
|
1221
|
+
if resource_drift_analysis
|
1222
|
+
else None,
|
1223
|
+
},
|
1177
1224
|
}
|
1178
1225
|
|
1179
1226
|
except Exception as e:
|
@@ -1184,7 +1231,7 @@ class EnhancedMCPValidator:
|
|
1184
1231
|
"passed_validation": False,
|
1185
1232
|
"error": str(e),
|
1186
1233
|
"validation_status": "ERROR",
|
1187
|
-
"drift_analysis": {}
|
1234
|
+
"drift_analysis": {},
|
1188
1235
|
}
|
1189
1236
|
|
1190
1237
|
async def _get_independent_inventory_data(self, session: boto3.Session, profile: str) -> Dict[str, Any]:
|
@@ -1204,10 +1251,10 @@ class EnhancedMCPValidator:
|
|
1204
1251
|
if session is None:
|
1205
1252
|
print_warning(f"Session not initialized for {profile}, using default profile")
|
1206
1253
|
session = boto3.Session(profile_name=profile)
|
1207
|
-
|
1254
|
+
|
1208
1255
|
ec2_client = session.client("ec2", region_name="us-east-1")
|
1209
1256
|
regions_response = ec2_client.describe_regions()
|
1210
|
-
regions = [region[
|
1257
|
+
regions = [region["RegionName"] for region in regions_response["Regions"]]
|
1211
1258
|
inventory_data["regions_discovered"] = regions
|
1212
1259
|
except Exception as e:
|
1213
1260
|
print_warning(f"Could not discover regions for {profile}: {str(e)[:50]}")
|
@@ -1216,57 +1263,59 @@ class EnhancedMCPValidator:
|
|
1216
1263
|
|
1217
1264
|
# Validate resource counts for each supported service
|
1218
1265
|
resource_counts = {}
|
1219
|
-
|
1266
|
+
|
1220
1267
|
# EC2 Instances - Enhanced comprehensive discovery
|
1221
1268
|
try:
|
1222
1269
|
total_ec2_instances = 0
|
1223
1270
|
successful_regions = 0
|
1224
1271
|
failed_regions = 0
|
1225
|
-
|
1272
|
+
|
1226
1273
|
# Use all available regions for comprehensive coverage
|
1227
1274
|
for region in regions:
|
1228
1275
|
try:
|
1229
1276
|
ec2_client = session.client("ec2", region_name=region)
|
1230
|
-
|
1277
|
+
|
1231
1278
|
# Get all instances using pagination for large accounts
|
1232
|
-
paginator = ec2_client.get_paginator(
|
1279
|
+
paginator = ec2_client.get_paginator("describe_instances")
|
1233
1280
|
region_instances = 0
|
1234
|
-
|
1281
|
+
|
1235
1282
|
for page in paginator.paginate():
|
1236
|
-
for reservation in page.get(
|
1283
|
+
for reservation in page.get("Reservations", []):
|
1237
1284
|
# Count all instances regardless of state for accurate inventory
|
1238
|
-
instances = reservation.get(
|
1285
|
+
instances = reservation.get("Instances", [])
|
1239
1286
|
region_instances += len(instances)
|
1240
|
-
|
1287
|
+
|
1241
1288
|
total_ec2_instances += region_instances
|
1242
1289
|
successful_regions += 1
|
1243
|
-
|
1290
|
+
|
1244
1291
|
# Log progress for debugging
|
1245
1292
|
if region_instances > 0:
|
1246
1293
|
self.console.log(f"[dim] EC2 {region}: {region_instances} instances[/]")
|
1247
|
-
|
1294
|
+
|
1248
1295
|
except Exception as e:
|
1249
1296
|
failed_regions += 1
|
1250
1297
|
# Log specific errors for troubleshooting
|
1251
1298
|
if "UnauthorizedOperation" not in str(e):
|
1252
1299
|
self.console.log(f"[dim yellow] EC2 {region}: Access denied or unavailable[/]")
|
1253
|
-
|
1254
|
-
resource_counts[
|
1255
|
-
|
1300
|
+
|
1301
|
+
resource_counts["ec2"] = total_ec2_instances
|
1302
|
+
|
1256
1303
|
# Track validation quality metrics
|
1257
|
-
self.console.log(
|
1258
|
-
|
1304
|
+
self.console.log(
|
1305
|
+
f"[dim]EC2 validation: {successful_regions} regions accessible, {failed_regions} failed[/]"
|
1306
|
+
)
|
1307
|
+
|
1259
1308
|
except Exception as e:
|
1260
1309
|
self.console.log(f"[red]EC2 validation failed: {str(e)[:50]}[/]")
|
1261
|
-
resource_counts[
|
1310
|
+
resource_counts["ec2"] = 0
|
1262
1311
|
|
1263
1312
|
# S3 Buckets (global service)
|
1264
1313
|
try:
|
1265
1314
|
s3_client = session.client("s3", region_name="us-east-1")
|
1266
1315
|
buckets_response = s3_client.list_buckets()
|
1267
|
-
resource_counts[
|
1316
|
+
resource_counts["s3"] = len(buckets_response.get("Buckets", []))
|
1268
1317
|
except Exception:
|
1269
|
-
resource_counts[
|
1318
|
+
resource_counts["s3"] = 0
|
1270
1319
|
|
1271
1320
|
# RDS Instances - Enhanced comprehensive discovery
|
1272
1321
|
try:
|
@@ -1274,23 +1323,23 @@ class EnhancedMCPValidator:
|
|
1274
1323
|
for region in regions:
|
1275
1324
|
try:
|
1276
1325
|
rds_client = session.client("rds", region_name=region)
|
1277
|
-
|
1326
|
+
|
1278
1327
|
# Use pagination for large RDS deployments
|
1279
|
-
paginator = rds_client.get_paginator(
|
1328
|
+
paginator = rds_client.get_paginator("describe_db_instances")
|
1280
1329
|
region_instances = 0
|
1281
|
-
|
1330
|
+
|
1282
1331
|
for page in paginator.paginate():
|
1283
|
-
region_instances += len(page.get(
|
1284
|
-
|
1332
|
+
region_instances += len(page.get("DBInstances", []))
|
1333
|
+
|
1285
1334
|
total_rds_instances += region_instances
|
1286
|
-
|
1335
|
+
|
1287
1336
|
if region_instances > 0:
|
1288
1337
|
self.console.log(f"[dim] RDS {region}: {region_instances} instances[/]")
|
1289
1338
|
except Exception:
|
1290
1339
|
continue
|
1291
|
-
resource_counts[
|
1340
|
+
resource_counts["rds"] = total_rds_instances
|
1292
1341
|
except Exception:
|
1293
|
-
resource_counts[
|
1342
|
+
resource_counts["rds"] = 0
|
1294
1343
|
|
1295
1344
|
# Lambda Functions - Enhanced comprehensive discovery
|
1296
1345
|
try:
|
@@ -1298,23 +1347,23 @@ class EnhancedMCPValidator:
|
|
1298
1347
|
for region in regions:
|
1299
1348
|
try:
|
1300
1349
|
lambda_client = session.client("lambda", region_name=region)
|
1301
|
-
|
1350
|
+
|
1302
1351
|
# Use pagination for large Lambda deployments
|
1303
|
-
paginator = lambda_client.get_paginator(
|
1352
|
+
paginator = lambda_client.get_paginator("list_functions")
|
1304
1353
|
region_functions = 0
|
1305
|
-
|
1354
|
+
|
1306
1355
|
for page in paginator.paginate():
|
1307
|
-
region_functions += len(page.get(
|
1308
|
-
|
1356
|
+
region_functions += len(page.get("Functions", []))
|
1357
|
+
|
1309
1358
|
total_lambda_functions += region_functions
|
1310
|
-
|
1359
|
+
|
1311
1360
|
if region_functions > 0:
|
1312
1361
|
self.console.log(f"[dim] Lambda {region}: {region_functions} functions[/]")
|
1313
1362
|
except Exception:
|
1314
1363
|
continue
|
1315
|
-
resource_counts[
|
1364
|
+
resource_counts["lambda"] = total_lambda_functions
|
1316
1365
|
except Exception:
|
1317
|
-
resource_counts[
|
1366
|
+
resource_counts["lambda"] = 0
|
1318
1367
|
|
1319
1368
|
# VPCs - Enhanced comprehensive discovery
|
1320
1369
|
try:
|
@@ -1322,43 +1371,43 @@ class EnhancedMCPValidator:
|
|
1322
1371
|
for region in regions:
|
1323
1372
|
try:
|
1324
1373
|
ec2_client = session.client("ec2", region_name=region)
|
1325
|
-
|
1374
|
+
|
1326
1375
|
# Use pagination for VPC discovery
|
1327
|
-
paginator = ec2_client.get_paginator(
|
1376
|
+
paginator = ec2_client.get_paginator("describe_vpcs")
|
1328
1377
|
region_vpcs = 0
|
1329
|
-
|
1378
|
+
|
1330
1379
|
for page in paginator.paginate():
|
1331
|
-
region_vpcs += len(page.get(
|
1332
|
-
|
1380
|
+
region_vpcs += len(page.get("Vpcs", []))
|
1381
|
+
|
1333
1382
|
total_vpcs += region_vpcs
|
1334
|
-
|
1383
|
+
|
1335
1384
|
if region_vpcs > 0:
|
1336
1385
|
self.console.log(f"[dim] VPC {region}: {region_vpcs} VPCs[/]")
|
1337
1386
|
except Exception:
|
1338
1387
|
continue
|
1339
|
-
resource_counts[
|
1388
|
+
resource_counts["vpc"] = total_vpcs
|
1340
1389
|
except Exception:
|
1341
|
-
resource_counts[
|
1390
|
+
resource_counts["vpc"] = 0
|
1342
1391
|
|
1343
1392
|
# IAM Roles (global service) - Enhanced discovery with pagination
|
1344
1393
|
try:
|
1345
1394
|
iam_client = session.client("iam", region_name="us-east-1")
|
1346
|
-
|
1395
|
+
|
1347
1396
|
# Use pagination for large IAM role deployments
|
1348
|
-
paginator = iam_client.get_paginator(
|
1397
|
+
paginator = iam_client.get_paginator("list_roles")
|
1349
1398
|
total_roles = 0
|
1350
|
-
|
1399
|
+
|
1351
1400
|
for page in paginator.paginate():
|
1352
|
-
total_roles += len(page.get(
|
1353
|
-
|
1354
|
-
resource_counts[
|
1355
|
-
|
1401
|
+
total_roles += len(page.get("Roles", []))
|
1402
|
+
|
1403
|
+
resource_counts["iam"] = total_roles
|
1404
|
+
|
1356
1405
|
if total_roles > 0:
|
1357
1406
|
self.console.log(f"[dim] IAM: {total_roles} roles discovered[/]")
|
1358
|
-
|
1407
|
+
|
1359
1408
|
except Exception as e:
|
1360
1409
|
self.console.log(f"[yellow]IAM roles discovery failed: {str(e)[:40]}[/]")
|
1361
|
-
resource_counts[
|
1410
|
+
resource_counts["iam"] = 0
|
1362
1411
|
|
1363
1412
|
# CloudFormation Stacks - Enhanced comprehensive discovery
|
1364
1413
|
try:
|
@@ -1366,67 +1415,69 @@ class EnhancedMCPValidator:
|
|
1366
1415
|
for region in regions:
|
1367
1416
|
try:
|
1368
1417
|
cf_client = session.client("cloudformation", region_name=region)
|
1369
|
-
|
1418
|
+
|
1370
1419
|
# Use pagination for large CloudFormation deployments
|
1371
|
-
paginator = cf_client.get_paginator(
|
1420
|
+
paginator = cf_client.get_paginator("list_stacks")
|
1372
1421
|
region_stacks = 0
|
1373
|
-
|
1374
|
-
for page in paginator.paginate(
|
1375
|
-
|
1376
|
-
|
1422
|
+
|
1423
|
+
for page in paginator.paginate(
|
1424
|
+
StackStatusFilter=["CREATE_COMPLETE", "UPDATE_COMPLETE", "ROLLBACK_COMPLETE"]
|
1425
|
+
):
|
1426
|
+
region_stacks += len(page.get("StackSummaries", []))
|
1427
|
+
|
1377
1428
|
total_stacks += region_stacks
|
1378
|
-
|
1429
|
+
|
1379
1430
|
if region_stacks > 0:
|
1380
1431
|
self.console.log(f"[dim] CloudFormation {region}: {region_stacks} stacks[/]")
|
1381
1432
|
except Exception:
|
1382
1433
|
continue
|
1383
|
-
resource_counts[
|
1434
|
+
resource_counts["cloudformation"] = total_stacks
|
1384
1435
|
except Exception:
|
1385
|
-
resource_counts[
|
1436
|
+
resource_counts["cloudformation"] = 0
|
1386
1437
|
|
1387
|
-
# Load Balancers (ELBv2) - Enhanced comprehensive discovery
|
1438
|
+
# Load Balancers (ELBv2) - Enhanced comprehensive discovery
|
1388
1439
|
try:
|
1389
1440
|
total_load_balancers = 0
|
1390
1441
|
for region in regions:
|
1391
1442
|
try:
|
1392
1443
|
elbv2_client = session.client("elbv2", region_name=region)
|
1393
|
-
|
1444
|
+
|
1394
1445
|
# Use pagination for large load balancer deployments
|
1395
|
-
paginator = elbv2_client.get_paginator(
|
1446
|
+
paginator = elbv2_client.get_paginator("describe_load_balancers")
|
1396
1447
|
region_lbs = 0
|
1397
|
-
|
1448
|
+
|
1398
1449
|
for page in paginator.paginate():
|
1399
|
-
region_lbs += len(page.get(
|
1400
|
-
|
1450
|
+
region_lbs += len(page.get("LoadBalancers", []))
|
1451
|
+
|
1401
1452
|
total_load_balancers += region_lbs
|
1402
|
-
|
1453
|
+
|
1403
1454
|
if region_lbs > 0:
|
1404
1455
|
self.console.log(f"[dim] ELBv2 {region}: {region_lbs} load balancers[/]")
|
1405
1456
|
except Exception:
|
1406
1457
|
continue
|
1407
|
-
resource_counts[
|
1458
|
+
resource_counts["elbv2"] = total_load_balancers
|
1408
1459
|
except Exception:
|
1409
|
-
resource_counts[
|
1460
|
+
resource_counts["elbv2"] = 0
|
1410
1461
|
|
1411
1462
|
# Route53 Hosted Zones (global service) - Enhanced discovery
|
1412
1463
|
try:
|
1413
1464
|
route53_client = session.client("route53", region_name="us-east-1")
|
1414
|
-
|
1465
|
+
|
1415
1466
|
# Use pagination for large Route53 deployments
|
1416
|
-
paginator = route53_client.get_paginator(
|
1467
|
+
paginator = route53_client.get_paginator("list_hosted_zones")
|
1417
1468
|
total_hosted_zones = 0
|
1418
|
-
|
1469
|
+
|
1419
1470
|
for page in paginator.paginate():
|
1420
|
-
total_hosted_zones += len(page.get(
|
1421
|
-
|
1422
|
-
resource_counts[
|
1423
|
-
|
1471
|
+
total_hosted_zones += len(page.get("HostedZones", []))
|
1472
|
+
|
1473
|
+
resource_counts["route53"] = total_hosted_zones
|
1474
|
+
|
1424
1475
|
if total_hosted_zones > 0:
|
1425
1476
|
self.console.log(f"[dim] Route53: {total_hosted_zones} hosted zones[/]")
|
1426
|
-
|
1477
|
+
|
1427
1478
|
except Exception as e:
|
1428
1479
|
self.console.log(f"[yellow]Route53 discovery failed: {str(e)[:40]}[/]")
|
1429
|
-
resource_counts[
|
1480
|
+
resource_counts["route53"] = 0
|
1430
1481
|
|
1431
1482
|
# SNS Topics - Enhanced comprehensive discovery
|
1432
1483
|
try:
|
@@ -1434,23 +1485,23 @@ class EnhancedMCPValidator:
|
|
1434
1485
|
for region in regions:
|
1435
1486
|
try:
|
1436
1487
|
sns_client = session.client("sns", region_name=region)
|
1437
|
-
|
1488
|
+
|
1438
1489
|
# Use pagination for large SNS deployments
|
1439
|
-
paginator = sns_client.get_paginator(
|
1490
|
+
paginator = sns_client.get_paginator("list_topics")
|
1440
1491
|
region_topics = 0
|
1441
|
-
|
1492
|
+
|
1442
1493
|
for page in paginator.paginate():
|
1443
|
-
region_topics += len(page.get(
|
1444
|
-
|
1494
|
+
region_topics += len(page.get("Topics", []))
|
1495
|
+
|
1445
1496
|
total_topics += region_topics
|
1446
|
-
|
1497
|
+
|
1447
1498
|
if region_topics > 0:
|
1448
1499
|
self.console.log(f"[dim] SNS {region}: {region_topics} topics[/]")
|
1449
1500
|
except Exception:
|
1450
1501
|
continue
|
1451
|
-
resource_counts[
|
1502
|
+
resource_counts["sns"] = total_topics
|
1452
1503
|
except Exception:
|
1453
|
-
resource_counts[
|
1504
|
+
resource_counts["sns"] = 0
|
1454
1505
|
|
1455
1506
|
# Network Interfaces (ENI) - Enhanced comprehensive discovery
|
1456
1507
|
try:
|
@@ -1458,23 +1509,23 @@ class EnhancedMCPValidator:
|
|
1458
1509
|
for region in regions:
|
1459
1510
|
try:
|
1460
1511
|
ec2_client = session.client("ec2", region_name=region)
|
1461
|
-
|
1512
|
+
|
1462
1513
|
# Use pagination for large ENI deployments
|
1463
|
-
paginator = ec2_client.get_paginator(
|
1514
|
+
paginator = ec2_client.get_paginator("describe_network_interfaces")
|
1464
1515
|
region_enis = 0
|
1465
|
-
|
1516
|
+
|
1466
1517
|
for page in paginator.paginate():
|
1467
|
-
region_enis += len(page.get(
|
1468
|
-
|
1518
|
+
region_enis += len(page.get("NetworkInterfaces", []))
|
1519
|
+
|
1469
1520
|
total_enis += region_enis
|
1470
|
-
|
1521
|
+
|
1471
1522
|
if region_enis > 0:
|
1472
1523
|
self.console.log(f"[dim] ENI {region}: {region_enis} network interfaces[/]")
|
1473
1524
|
except Exception:
|
1474
1525
|
continue
|
1475
|
-
resource_counts[
|
1526
|
+
resource_counts["eni"] = total_enis
|
1476
1527
|
except Exception:
|
1477
|
-
resource_counts[
|
1528
|
+
resource_counts["eni"] = 0
|
1478
1529
|
|
1479
1530
|
# EBS Volumes - Enhanced comprehensive discovery
|
1480
1531
|
try:
|
@@ -1482,23 +1533,23 @@ class EnhancedMCPValidator:
|
|
1482
1533
|
for region in regions:
|
1483
1534
|
try:
|
1484
1535
|
ec2_client = session.client("ec2", region_name=region)
|
1485
|
-
|
1536
|
+
|
1486
1537
|
# Use pagination for large EBS deployments
|
1487
|
-
paginator = ec2_client.get_paginator(
|
1538
|
+
paginator = ec2_client.get_paginator("describe_volumes")
|
1488
1539
|
region_volumes = 0
|
1489
|
-
|
1540
|
+
|
1490
1541
|
for page in paginator.paginate():
|
1491
|
-
region_volumes += len(page.get(
|
1492
|
-
|
1542
|
+
region_volumes += len(page.get("Volumes", []))
|
1543
|
+
|
1493
1544
|
total_volumes += region_volumes
|
1494
|
-
|
1545
|
+
|
1495
1546
|
if region_volumes > 0:
|
1496
1547
|
self.console.log(f"[dim] EBS {region}: {region_volumes} volumes[/]")
|
1497
1548
|
except Exception:
|
1498
1549
|
continue
|
1499
|
-
resource_counts[
|
1550
|
+
resource_counts["ebs"] = total_volumes
|
1500
1551
|
except Exception:
|
1501
|
-
resource_counts[
|
1552
|
+
resource_counts["ebs"] = 0
|
1502
1553
|
|
1503
1554
|
inventory_data["resource_counts"] = resource_counts
|
1504
1555
|
|
@@ -1516,11 +1567,11 @@ class EnhancedMCPValidator:
|
|
1516
1567
|
def _extract_runbooks_inventory_data(self, runbooks_inventory: Dict[str, Any], profile: str) -> Dict[str, Any]:
|
1517
1568
|
"""
|
1518
1569
|
Extract inventory data from runbooks results for comparison.
|
1519
|
-
|
1570
|
+
|
1520
1571
|
Args:
|
1521
1572
|
runbooks_inventory: Inventory results from runbooks collection
|
1522
1573
|
profile: Profile name for data extraction
|
1523
|
-
|
1574
|
+
|
1524
1575
|
Returns:
|
1525
1576
|
Extracted inventory data in standardized format
|
1526
1577
|
"""
|
@@ -1534,13 +1585,13 @@ class EnhancedMCPValidator:
|
|
1534
1585
|
# Fallback: Look for direct resource keys (legacy format)
|
1535
1586
|
resource_counts = runbooks_inventory.get("resource_counts", {})
|
1536
1587
|
regions_discovered = runbooks_inventory.get("regions", [])
|
1537
|
-
|
1588
|
+
|
1538
1589
|
return {
|
1539
1590
|
"profile": profile,
|
1540
1591
|
"resource_counts": resource_counts,
|
1541
1592
|
"regions_discovered": regions_discovered,
|
1542
1593
|
"data_source": "runbooks_inventory_collection",
|
1543
|
-
"extraction_method": "profile_nested" if profile in runbooks_inventory else "direct_keys"
|
1594
|
+
"extraction_method": "profile_nested" if profile in runbooks_inventory else "direct_keys",
|
1544
1595
|
}
|
1545
1596
|
except Exception as e:
|
1546
1597
|
self.console.log(f"[yellow]Warning: Error extracting runbooks inventory data for {profile}: {str(e)}[/]")
|
@@ -1549,18 +1600,18 @@ class EnhancedMCPValidator:
|
|
1549
1600
|
"resource_counts": {},
|
1550
1601
|
"regions_discovered": [],
|
1551
1602
|
"data_source": "runbooks_inventory_collection_error",
|
1552
|
-
"error": str(e)
|
1603
|
+
"error": str(e),
|
1553
1604
|
}
|
1554
1605
|
|
1555
1606
|
def _calculate_inventory_accuracy(self, runbooks_data: Dict, aws_data: Dict, profile: str) -> Dict[str, Any]:
|
1556
1607
|
"""
|
1557
1608
|
Calculate accuracy between runbooks and AWS API inventory data.
|
1558
|
-
|
1609
|
+
|
1559
1610
|
Args:
|
1560
1611
|
runbooks_data: Inventory data from runbooks
|
1561
1612
|
aws_data: Inventory data from AWS API
|
1562
1613
|
profile: Profile name for validation
|
1563
|
-
|
1614
|
+
|
1564
1615
|
Returns:
|
1565
1616
|
Accuracy metrics with resource-level breakdown
|
1566
1617
|
"""
|
@@ -1583,11 +1634,15 @@ class EnhancedMCPValidator:
|
|
1583
1634
|
elif runbooks_count == 0 and aws_count > 0:
|
1584
1635
|
# Runbooks missing resources - accuracy issue
|
1585
1636
|
accuracy_percent = 0.0
|
1586
|
-
self.console.log(
|
1637
|
+
self.console.log(
|
1638
|
+
f"[red]⚠️ Profile {profile} {resource_type}: Runbooks shows 0 but MCP shows {aws_count}[/]"
|
1639
|
+
)
|
1587
1640
|
elif aws_count == 0 and runbooks_count > 0:
|
1588
|
-
# MCP missing data - moderate accuracy issue
|
1641
|
+
# MCP missing data - moderate accuracy issue
|
1589
1642
|
accuracy_percent = 50.0 # Give partial credit as MCP may have different access
|
1590
|
-
self.console.log(
|
1643
|
+
self.console.log(
|
1644
|
+
f"[yellow]⚠️ Profile {profile} {resource_type}: MCP shows 0 but Runbooks shows {runbooks_count}[/]"
|
1645
|
+
)
|
1591
1646
|
else:
|
1592
1647
|
# Both have values - calculate variance-based accuracy
|
1593
1648
|
max_count = max(runbooks_count, aws_count)
|
@@ -1600,7 +1655,7 @@ class EnhancedMCPValidator:
|
|
1600
1655
|
"accuracy_percent": accuracy_percent,
|
1601
1656
|
"variance_count": abs(runbooks_count - aws_count),
|
1602
1657
|
"variance_percent": abs(runbooks_count - aws_count) / max(max(runbooks_count, aws_count), 1) * 100,
|
1603
|
-
"passed_validation": accuracy_percent >= self.validation_threshold
|
1658
|
+
"passed_validation": accuracy_percent >= self.validation_threshold,
|
1604
1659
|
}
|
1605
1660
|
|
1606
1661
|
if runbooks_count > 0 or aws_count > 0: # Only count non-zero comparisons
|
@@ -1666,14 +1721,14 @@ class EnhancedMCPValidator:
|
|
1666
1721
|
"total_accounts": len(valid_results),
|
1667
1722
|
"accounts_with_drift": 0,
|
1668
1723
|
"resource_types_with_drift": set(),
|
1669
|
-
"terraform_coverage": 0
|
1724
|
+
"terraform_coverage": 0,
|
1670
1725
|
}
|
1671
|
-
|
1726
|
+
|
1672
1727
|
for result in valid_results:
|
1673
1728
|
# Check if account has terraform coverage
|
1674
1729
|
if result.get("terraform_files_parsed", 0) > 0:
|
1675
1730
|
drift_summary["terraform_coverage"] += 1
|
1676
|
-
|
1731
|
+
|
1677
1732
|
# Collect drift analysis
|
1678
1733
|
has_drift = False
|
1679
1734
|
resource_drift = result.get("resource_drift_analysis", {})
|
@@ -1681,7 +1736,7 @@ class EnhancedMCPValidator:
|
|
1681
1736
|
if drift_data.get("drift_status", "NO_DRIFT") != "NO_DRIFT":
|
1682
1737
|
has_drift = True
|
1683
1738
|
drift_summary["resource_types_with_drift"].add(resource_type)
|
1684
|
-
|
1739
|
+
|
1685
1740
|
# Aggregate resource summary
|
1686
1741
|
if resource_type not in resource_summary:
|
1687
1742
|
resource_summary[resource_type] = {
|
@@ -1689,17 +1744,19 @@ class EnhancedMCPValidator:
|
|
1689
1744
|
"total_aws": 0,
|
1690
1745
|
"total_terraform": 0,
|
1691
1746
|
"accuracy_scores": [],
|
1692
|
-
"drift_incidents": 0
|
1747
|
+
"drift_incidents": 0,
|
1693
1748
|
}
|
1694
|
-
|
1749
|
+
|
1695
1750
|
resource_summary[resource_type]["total_runbooks"] += drift_data.get("runbooks_count", 0)
|
1696
1751
|
resource_summary[resource_type]["total_aws"] += drift_data.get("aws_api_count", 0)
|
1697
1752
|
resource_summary[resource_type]["total_terraform"] += drift_data.get("terraform_count", 0)
|
1698
|
-
resource_summary[resource_type]["accuracy_scores"].append(
|
1699
|
-
|
1753
|
+
resource_summary[resource_type]["accuracy_scores"].append(
|
1754
|
+
drift_data.get("overall_accuracy_percent", 0)
|
1755
|
+
)
|
1756
|
+
|
1700
1757
|
if drift_data.get("drift_status", "NO_DRIFT") != "NO_DRIFT":
|
1701
1758
|
resource_summary[resource_type]["drift_incidents"] += 1
|
1702
|
-
|
1759
|
+
|
1703
1760
|
if has_drift:
|
1704
1761
|
drift_summary["accounts_with_drift"] += 1
|
1705
1762
|
|
@@ -1714,10 +1771,16 @@ class EnhancedMCPValidator:
|
|
1714
1771
|
validation_results["terraform_integration"]["drift_analysis"] = {
|
1715
1772
|
"total_accounts": drift_summary["total_accounts"],
|
1716
1773
|
"accounts_with_drift": drift_summary["accounts_with_drift"],
|
1717
|
-
"drift_percentage": (drift_summary["accounts_with_drift"] / drift_summary["total_accounts"] * 100)
|
1774
|
+
"drift_percentage": (drift_summary["accounts_with_drift"] / drift_summary["total_accounts"] * 100)
|
1775
|
+
if drift_summary["total_accounts"] > 0
|
1776
|
+
else 0,
|
1718
1777
|
"resource_types_with_drift": len(drift_summary["resource_types_with_drift"]),
|
1719
1778
|
"terraform_coverage_accounts": drift_summary["terraform_coverage"],
|
1720
|
-
"terraform_coverage_percentage": (
|
1779
|
+
"terraform_coverage_percentage": (
|
1780
|
+
drift_summary["terraform_coverage"] / drift_summary["total_accounts"] * 100
|
1781
|
+
)
|
1782
|
+
if drift_summary["total_accounts"] > 0
|
1783
|
+
else 0,
|
1721
1784
|
}
|
1722
1785
|
|
1723
1786
|
# Display enhanced results with drift analysis
|
@@ -1745,11 +1808,7 @@ class EnhancedMCPValidator:
|
|
1745
1808
|
for result in valid_results:
|
1746
1809
|
for resource_type, resource_data in result.get("resource_accuracies", {}).items():
|
1747
1810
|
if resource_type not in resource_summary:
|
1748
|
-
resource_summary[resource_type] = {
|
1749
|
-
"total_runbooks": 0,
|
1750
|
-
"total_aws": 0,
|
1751
|
-
"accuracy_scores": []
|
1752
|
-
}
|
1811
|
+
resource_summary[resource_type] = {"total_runbooks": 0, "total_aws": 0, "accuracy_scores": []}
|
1753
1812
|
resource_summary[resource_type]["total_runbooks"] += resource_data["runbooks_count"]
|
1754
1813
|
resource_summary[resource_type]["total_aws"] += resource_data["aws_api_count"]
|
1755
1814
|
resource_summary[resource_type]["accuracy_scores"].append(resource_data["accuracy_percent"])
|
@@ -1795,7 +1854,9 @@ class EnhancedMCPValidator:
|
|
1795
1854
|
color = "red"
|
1796
1855
|
|
1797
1856
|
# Profile summary
|
1798
|
-
self.console.print(
|
1857
|
+
self.console.print(
|
1858
|
+
f"[dim] {profile[:30]}: {icon} [{color}]{accuracy:.1f}% accuracy[/] [dim]({category})[/][/dim]"
|
1859
|
+
)
|
1799
1860
|
|
1800
1861
|
# Resource-level breakdown
|
1801
1862
|
resource_accuracies = profile_result.get("resource_accuracies", {})
|
@@ -1824,7 +1885,7 @@ class EnhancedMCPValidator:
|
|
1824
1885
|
avg_accuracy = summary.get("average_accuracy", 0)
|
1825
1886
|
total_runbooks = summary.get("total_runbooks", 0)
|
1826
1887
|
total_aws = summary.get("total_aws", 0)
|
1827
|
-
|
1888
|
+
|
1828
1889
|
summary_icon = "✅" if avg_accuracy >= 99.5 else "⚠️" if avg_accuracy >= 90.0 else "❌"
|
1829
1890
|
self.console.print(
|
1830
1891
|
f"[dim] {self.supported_services.get(resource_type, resource_type):20s}: {summary_icon} "
|
@@ -1838,21 +1899,23 @@ class EnhancedMCPValidator:
|
|
1838
1899
|
terraform_integration = results.get("terraform_integration", {})
|
1839
1900
|
|
1840
1901
|
self.console.print(f"\n[bright_cyan]🔍 Enhanced Inventory Validation with Drift Detection[/]")
|
1841
|
-
|
1902
|
+
|
1842
1903
|
# Display terraform integration status
|
1843
1904
|
if terraform_integration.get("enabled", False):
|
1844
1905
|
tf_files = terraform_integration.get("state_files_discovered", 0)
|
1845
1906
|
drift_analysis = terraform_integration.get("drift_analysis", {})
|
1846
|
-
|
1907
|
+
|
1847
1908
|
self.console.print(f"[dim]🏗️ Terraform Integration: {tf_files} state files discovered[/]")
|
1848
|
-
|
1909
|
+
|
1849
1910
|
if drift_analysis:
|
1850
1911
|
total_accounts = drift_analysis.get("total_accounts", 0)
|
1851
1912
|
accounts_with_drift = drift_analysis.get("accounts_with_drift", 0)
|
1852
1913
|
drift_percentage = drift_analysis.get("drift_percentage", 0)
|
1853
1914
|
tf_coverage = drift_analysis.get("terraform_coverage_percentage", 0)
|
1854
|
-
|
1855
|
-
self.console.print(
|
1915
|
+
|
1916
|
+
self.console.print(
|
1917
|
+
f"[dim]📊 Drift Analysis: {accounts_with_drift}/{total_accounts} accounts ({drift_percentage:.1f}%) with detected drift[/]"
|
1918
|
+
)
|
1856
1919
|
self.console.print(f"[dim]🎯 IaC Coverage: {tf_coverage:.1f}% accounts have terraform configuration[/]")
|
1857
1920
|
|
1858
1921
|
# Display per-profile results with enhanced drift breakdown
|
@@ -1877,7 +1940,7 @@ class EnhancedMCPValidator:
|
|
1877
1940
|
# Profile summary with drift information
|
1878
1941
|
drift_count = drift_summary.get("drift_detected", 0)
|
1879
1942
|
total_resources = drift_summary.get("total_resource_types", 0)
|
1880
|
-
|
1943
|
+
|
1881
1944
|
self.console.print(f"[dim] {profile[:30]} ({account_id}): {icon} [{color}]{accuracy:.1f}% accuracy[/]")
|
1882
1945
|
if drift_count > 0:
|
1883
1946
|
self.console.print(f"[dim] 🔄 Drift detected in {drift_count}/{total_resources} resource types[/]")
|
@@ -1885,21 +1948,25 @@ class EnhancedMCPValidator:
|
|
1885
1948
|
# Enhanced resource-level breakdown with 3-way comparison
|
1886
1949
|
drift_analysis = profile_result.get("resource_drift_analysis", {})
|
1887
1950
|
for resource_type, drift_data in drift_analysis.items():
|
1888
|
-
if
|
1951
|
+
if (
|
1952
|
+
drift_data.get("runbooks_count", 0) > 0
|
1953
|
+
or drift_data.get("aws_api_count", 0) > 0
|
1954
|
+
or drift_data.get("terraform_count", 0) > 0
|
1955
|
+
):
|
1889
1956
|
drift_status = drift_data.get("drift_status", "NO_DRIFT")
|
1890
1957
|
resource_icon = "✅" if drift_status == "NO_DRIFT" else "🔄" if "DRIFT" in drift_status else "⚠️"
|
1891
|
-
|
1958
|
+
|
1892
1959
|
runbooks_count = drift_data.get("runbooks_count", 0)
|
1893
1960
|
aws_count = drift_data.get("aws_api_count", 0)
|
1894
1961
|
terraform_count = drift_data.get("terraform_count", 0)
|
1895
1962
|
overall_acc = drift_data.get("overall_accuracy_percent", 0)
|
1896
|
-
|
1963
|
+
|
1897
1964
|
self.console.print(
|
1898
1965
|
f"[dim] {self.supported_services.get(resource_type, resource_type):20s}: {resource_icon} "
|
1899
1966
|
f"Runbooks: {runbooks_count:3d} | AWS: {aws_count:3d} | Terraform: {terraform_count:3d} | "
|
1900
1967
|
f"Accuracy: {overall_acc:5.1f}%[/dim]"
|
1901
1968
|
)
|
1902
|
-
|
1969
|
+
|
1903
1970
|
# Show recommendations for drift
|
1904
1971
|
recommendations = drift_data.get("recommendations", [])
|
1905
1972
|
for rec in recommendations[:1]: # Show first recommendation only
|
@@ -1915,34 +1982,33 @@ class EnhancedMCPValidator:
|
|
1915
1982
|
print_success(f"✅ Enhanced Validation PASSED: {overall_accuracy:.1f}% accuracy achieved")
|
1916
1983
|
else:
|
1917
1984
|
print_warning(f"🔄 Enhanced Validation: {overall_accuracy:.1f}% accuracy with drift detected")
|
1918
|
-
|
1985
|
+
|
1919
1986
|
print_info(f"Enterprise compliance: {results.get('profiles_validated', 0)} profiles validated")
|
1920
1987
|
|
1921
1988
|
# Enhanced resource validation summary with terraform comparison
|
1922
1989
|
resource_summary = results.get("resource_validation_summary", {})
|
1923
1990
|
if resource_summary:
|
1924
1991
|
self.console.print(f"\n[bright_cyan]📊 Enhanced Resource Validation Summary[/]")
|
1925
|
-
|
1992
|
+
|
1926
1993
|
# Create drift analysis table
|
1927
1994
|
drift_table = create_table(
|
1928
|
-
title="Infrastructure Drift Analysis",
|
1929
|
-
caption="3-way comparison: Runbooks | AWS API | Terraform IaC"
|
1995
|
+
title="Infrastructure Drift Analysis", caption="3-way comparison: Runbooks | AWS API | Terraform IaC"
|
1930
1996
|
)
|
1931
|
-
|
1997
|
+
|
1932
1998
|
drift_table.add_column("Resource Type", style="cyan", no_wrap=True)
|
1933
1999
|
drift_table.add_column("Runbooks", style="green", justify="right")
|
1934
2000
|
drift_table.add_column("AWS API", style="blue", justify="right")
|
1935
2001
|
drift_table.add_column("Terraform", style="magenta", justify="right")
|
1936
2002
|
drift_table.add_column("Accuracy", justify="right")
|
1937
2003
|
drift_table.add_column("Drift Status", style="yellow")
|
1938
|
-
|
2004
|
+
|
1939
2005
|
for resource_type, summary in resource_summary.items():
|
1940
2006
|
avg_accuracy = summary.get("average_accuracy", 0)
|
1941
2007
|
total_runbooks = summary.get("total_runbooks", 0)
|
1942
2008
|
total_aws = summary.get("total_aws", 0)
|
1943
2009
|
total_terraform = summary.get("total_terraform", 0)
|
1944
2010
|
drift_incidents = summary.get("drift_incidents", 0)
|
1945
|
-
|
2011
|
+
|
1946
2012
|
# Determine status
|
1947
2013
|
if drift_incidents > 0:
|
1948
2014
|
status = f"🔄 {drift_incidents} drift(s)"
|
@@ -1950,18 +2016,18 @@ class EnhancedMCPValidator:
|
|
1950
2016
|
else:
|
1951
2017
|
status = "✅ Aligned"
|
1952
2018
|
status_style = "green"
|
1953
|
-
|
2019
|
+
|
1954
2020
|
accuracy_icon = "✅" if avg_accuracy >= 99.5 else "⚠️" if avg_accuracy >= 90.0 else "❌"
|
1955
|
-
|
2021
|
+
|
1956
2022
|
drift_table.add_row(
|
1957
2023
|
self.supported_services.get(resource_type, resource_type),
|
1958
2024
|
str(total_runbooks),
|
1959
2025
|
str(total_aws),
|
1960
2026
|
str(total_terraform),
|
1961
2027
|
f"{accuracy_icon} {avg_accuracy:5.1f}%",
|
1962
|
-
status
|
2028
|
+
status,
|
1963
2029
|
)
|
1964
|
-
|
2030
|
+
|
1965
2031
|
self.console.print(drift_table)
|
1966
2032
|
|
1967
2033
|
def validate_inventory_data(self, runbooks_inventory: Dict[str, Any]) -> Dict[str, Any]:
|
@@ -1974,74 +2040,76 @@ class EnhancedMCPValidator:
|
|
1974
2040
|
|
1975
2041
|
return loop.run_until_complete(self.validate_inventory_data_async(runbooks_inventory))
|
1976
2042
|
|
1977
|
-
def validate_resource_counts(
|
2043
|
+
def validate_resource_counts(
|
2044
|
+
self, resource_counts: Dict[str, int], profile: Optional[str] = None
|
2045
|
+
) -> Dict[str, Any]:
|
1978
2046
|
"""
|
1979
2047
|
Cross-validate individual resource counts with AWS API.
|
1980
|
-
|
2048
|
+
|
1981
2049
|
Args:
|
1982
2050
|
resource_counts: Dictionary of resource types to counts (e.g., {'ec2': 45, 's3': 12})
|
1983
2051
|
profile: Profile to use for validation (uses first available if None)
|
1984
|
-
|
2052
|
+
|
1985
2053
|
Returns:
|
1986
2054
|
Resource-level validation results
|
1987
2055
|
"""
|
1988
2056
|
profile = profile or (self.profiles[0] if self.profiles else None)
|
1989
2057
|
if not profile or profile not in self.aws_sessions:
|
1990
|
-
return {
|
1991
|
-
|
2058
|
+
return {"error": "No valid profile for resource count validation"}
|
2059
|
+
|
1992
2060
|
session_info = self.aws_sessions[profile]
|
1993
2061
|
session = session_info["session"] # Extract actual boto3.Session object
|
1994
2062
|
validations = {}
|
1995
|
-
|
2063
|
+
|
1996
2064
|
# Get MCP resource counts
|
1997
2065
|
try:
|
1998
2066
|
mcp_data = asyncio.run(self._get_independent_inventory_data(session, profile))
|
1999
|
-
mcp_counts = mcp_data.get(
|
2000
|
-
|
2067
|
+
mcp_counts = mcp_data.get("resource_counts", {})
|
2068
|
+
|
2001
2069
|
# Validate each resource type
|
2002
2070
|
for resource_type, runbooks_count in resource_counts.items():
|
2003
2071
|
if resource_type in self.supported_services:
|
2004
2072
|
mcp_count = mcp_counts.get(resource_type, 0)
|
2005
|
-
|
2073
|
+
|
2006
2074
|
variance = 0.0
|
2007
2075
|
if runbooks_count > 0:
|
2008
2076
|
variance = abs(runbooks_count - mcp_count) / runbooks_count * 100
|
2009
|
-
|
2077
|
+
|
2010
2078
|
validations[resource_type] = {
|
2011
|
-
|
2012
|
-
|
2013
|
-
|
2014
|
-
|
2015
|
-
|
2079
|
+
"runbooks_count": runbooks_count,
|
2080
|
+
"mcp_count": mcp_count,
|
2081
|
+
"variance_percent": variance,
|
2082
|
+
"passed": variance <= self.tolerance_percent,
|
2083
|
+
"status": "PASSED" if variance <= self.tolerance_percent else "VARIANCE",
|
2016
2084
|
}
|
2017
|
-
|
2085
|
+
|
2018
2086
|
# Display resource validation results
|
2019
2087
|
self._display_resource_count_validation(validations)
|
2020
|
-
|
2088
|
+
|
2021
2089
|
except Exception as e:
|
2022
2090
|
print_error(f"Resource count validation failed: {str(e)[:50]}")
|
2023
|
-
return {
|
2024
|
-
|
2091
|
+
return {"error": str(e)}
|
2092
|
+
|
2025
2093
|
return {
|
2026
|
-
|
2027
|
-
|
2028
|
-
|
2029
|
-
|
2094
|
+
"resources": validations,
|
2095
|
+
"validated_count": len(validations),
|
2096
|
+
"passed_count": sum(1 for v in validations.values() if v["passed"]),
|
2097
|
+
"timestamp": datetime.now().isoformat(),
|
2030
2098
|
}
|
2031
|
-
|
2099
|
+
|
2032
2100
|
def _display_resource_count_validation(self, validations: Dict[str, Dict]) -> None:
|
2033
2101
|
"""Display resource count validation results."""
|
2034
2102
|
if validations:
|
2035
2103
|
self.console.print("\n[bright_cyan]Resource Count MCP Validation:[/bright_cyan]")
|
2036
|
-
|
2104
|
+
|
2037
2105
|
for resource_type, validation in validations.items():
|
2038
|
-
if validation[
|
2106
|
+
if validation["passed"]:
|
2039
2107
|
icon = "✅"
|
2040
2108
|
color = "green"
|
2041
2109
|
else:
|
2042
2110
|
icon = "⚠️"
|
2043
2111
|
color = "yellow"
|
2044
|
-
|
2112
|
+
|
2045
2113
|
resource_name = self.supported_services.get(resource_type, resource_type)
|
2046
2114
|
self.console.print(
|
2047
2115
|
f"[dim] {resource_name:20s}: {icon} [{color}]"
|
@@ -2050,30 +2118,38 @@ class EnhancedMCPValidator:
|
|
2050
2118
|
)
|
2051
2119
|
|
2052
2120
|
|
2053
|
-
def create_enhanced_mcp_validator(
|
2054
|
-
|
2121
|
+
def create_enhanced_mcp_validator(
|
2122
|
+
user_profile: Optional[str] = None,
|
2123
|
+
console: Optional[Console] = None,
|
2124
|
+
mcp_config_path: Optional[str] = None,
|
2125
|
+
terraform_directory: Optional[str] = None,
|
2126
|
+
) -> EnhancedMCPValidator:
|
2055
2127
|
"""
|
2056
2128
|
Factory function to create enhanced MCP validator with real server integration.
|
2057
|
-
|
2129
|
+
|
2058
2130
|
Args:
|
2059
2131
|
user_profile: User-specified profile (--profile parameter) - takes priority
|
2060
2132
|
console: Rich console for output
|
2061
2133
|
mcp_config_path: Path to .mcp.json configuration file
|
2062
2134
|
terraform_directory: Path to terraform configurations
|
2063
|
-
|
2135
|
+
|
2064
2136
|
Returns:
|
2065
2137
|
Enhanced MCP validator instance
|
2066
2138
|
"""
|
2067
2139
|
return EnhancedMCPValidator(
|
2068
|
-
user_profile=user_profile,
|
2069
|
-
console=console,
|
2140
|
+
user_profile=user_profile,
|
2141
|
+
console=console,
|
2070
2142
|
mcp_config_path=mcp_config_path,
|
2071
|
-
terraform_directory=terraform_directory
|
2143
|
+
terraform_directory=terraform_directory,
|
2072
2144
|
)
|
2073
2145
|
|
2074
2146
|
|
2075
|
-
def validate_inventory_with_mcp_servers(
|
2076
|
-
|
2147
|
+
def validate_inventory_with_mcp_servers(
|
2148
|
+
runbooks_inventory: Dict[str, Any],
|
2149
|
+
user_profile: Optional[str] = None,
|
2150
|
+
mcp_config_path: Optional[str] = None,
|
2151
|
+
terraform_directory: Optional[str] = None,
|
2152
|
+
) -> Dict[str, Any]:
|
2077
2153
|
"""
|
2078
2154
|
Enhanced convenience function to validate inventory results using real MCP servers.
|
2079
2155
|
|
@@ -2087,42 +2163,50 @@ def validate_inventory_with_mcp_servers(runbooks_inventory: Dict[str, Any], user
|
|
2087
2163
|
Enhanced validation results with MCP server integration and drift detection
|
2088
2164
|
"""
|
2089
2165
|
validator = create_enhanced_mcp_validator(
|
2090
|
-
user_profile=user_profile,
|
2091
|
-
mcp_config_path=mcp_config_path,
|
2092
|
-
terraform_directory=terraform_directory
|
2166
|
+
user_profile=user_profile, mcp_config_path=mcp_config_path, terraform_directory=terraform_directory
|
2093
2167
|
)
|
2094
2168
|
return asyncio.run(validator.validate_with_mcp_servers(runbooks_inventory))
|
2095
2169
|
|
2096
2170
|
|
2097
2171
|
# Legacy compatibility - maintain backward compatibility with existing code
|
2098
|
-
def create_inventory_mcp_validator(
|
2172
|
+
def create_inventory_mcp_validator(
|
2173
|
+
profiles: List[str], console: Optional[Console] = None, terraform_directory: Optional[str] = None
|
2174
|
+
) -> EnhancedMCPValidator:
|
2099
2175
|
"""Legacy compatibility function for existing code."""
|
2100
2176
|
# Convert profile list to single user profile (use first profile)
|
2101
2177
|
user_profile = profiles[0] if profiles else None
|
2102
|
-
return create_enhanced_mcp_validator(
|
2178
|
+
return create_enhanced_mcp_validator(
|
2179
|
+
user_profile=user_profile, console=console, terraform_directory=terraform_directory
|
2180
|
+
)
|
2103
2181
|
|
2104
2182
|
|
2105
|
-
def validate_inventory_results_with_mcp(
|
2183
|
+
def validate_inventory_results_with_mcp(
|
2184
|
+
profiles: List[str], runbooks_inventory: Dict[str, Any], terraform_directory: Optional[str] = None
|
2185
|
+
) -> Dict[str, Any]:
|
2106
2186
|
"""Legacy compatibility function for existing code."""
|
2107
2187
|
user_profile = profiles[0] if profiles else None
|
2108
|
-
return validate_inventory_with_mcp_servers(
|
2188
|
+
return validate_inventory_with_mcp_servers(
|
2189
|
+
runbooks_inventory, user_profile=user_profile, terraform_directory=terraform_directory
|
2190
|
+
)
|
2109
2191
|
|
2110
2192
|
|
2111
|
-
def generate_drift_report(
|
2193
|
+
def generate_drift_report(
|
2194
|
+
profiles: List[str], runbooks_inventory: Dict[str, Any], terraform_directory: Optional[str] = None
|
2195
|
+
) -> Dict[str, Any]:
|
2112
2196
|
"""
|
2113
2197
|
Generate comprehensive infrastructure drift report.
|
2114
|
-
|
2198
|
+
|
2115
2199
|
Args:
|
2116
2200
|
profiles: List of AWS profiles to analyze
|
2117
2201
|
runbooks_inventory: Inventory results from runbooks collection
|
2118
2202
|
terraform_directory: Path to terraform configurations
|
2119
|
-
|
2203
|
+
|
2120
2204
|
Returns:
|
2121
2205
|
Comprehensive drift analysis report with recommendations
|
2122
2206
|
"""
|
2123
2207
|
validator = create_inventory_mcp_validator(profiles, terraform_directory=terraform_directory)
|
2124
2208
|
validation_results = validator.validate_inventory_data(runbooks_inventory)
|
2125
|
-
|
2209
|
+
|
2126
2210
|
# Extract drift-specific information for reporting
|
2127
2211
|
drift_report = {
|
2128
2212
|
"report_type": "infrastructure_drift_analysis",
|
@@ -2131,9 +2215,9 @@ def generate_drift_report(profiles: List[str], runbooks_inventory: Dict[str, Any
|
|
2131
2215
|
"accounts_analyzed": validation_results.get("profiles_validated", 0),
|
2132
2216
|
"overall_accuracy": validation_results.get("total_accuracy", 0),
|
2133
2217
|
"drift_detected": not validation_results.get("passed_validation", False),
|
2134
|
-
"detailed_analysis": []
|
2218
|
+
"detailed_analysis": [],
|
2135
2219
|
}
|
2136
|
-
|
2220
|
+
|
2137
2221
|
# Add detailed per-account drift analysis
|
2138
2222
|
for profile_result in validation_results.get("profile_results", []):
|
2139
2223
|
account_drift = {
|
@@ -2143,8 +2227,8 @@ def generate_drift_report(profiles: List[str], runbooks_inventory: Dict[str, Any
|
|
2143
2227
|
"drift_summary": profile_result.get("drift_summary", {}),
|
2144
2228
|
"terraform_coverage": profile_result.get("terraform_files_parsed", 0) > 0,
|
2145
2229
|
"recommendations": profile_result.get("account_recommendations", []),
|
2146
|
-
"resource_drift_details": profile_result.get("resource_drift_analysis", {})
|
2230
|
+
"resource_drift_details": profile_result.get("resource_drift_analysis", {}),
|
2147
2231
|
}
|
2148
2232
|
drift_report["detailed_analysis"].append(account_drift)
|
2149
|
-
|
2150
|
-
return drift_report
|
2233
|
+
|
2234
|
+
return drift_report
|