runbooks 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +31 -2
- runbooks/__init___optimized.py +18 -4
- runbooks/_platform/__init__.py +1 -5
- runbooks/_platform/core/runbooks_wrapper.py +141 -138
- runbooks/aws2/accuracy_validator.py +812 -0
- runbooks/base.py +7 -0
- runbooks/cfat/WEIGHT_CONFIG_README.md +1 -1
- runbooks/cfat/assessment/compliance.py +8 -8
- runbooks/cfat/assessment/runner.py +1 -0
- runbooks/cfat/cloud_foundations_assessment.py +227 -239
- runbooks/cfat/models.py +6 -2
- runbooks/cfat/tests/__init__.py +6 -1
- runbooks/cli/__init__.py +13 -0
- runbooks/cli/commands/cfat.py +274 -0
- runbooks/cli/commands/finops.py +1164 -0
- runbooks/cli/commands/inventory.py +379 -0
- runbooks/cli/commands/operate.py +239 -0
- runbooks/cli/commands/security.py +248 -0
- runbooks/cli/commands/validation.py +825 -0
- runbooks/cli/commands/vpc.py +310 -0
- runbooks/cli/registry.py +107 -0
- runbooks/cloudops/__init__.py +23 -30
- runbooks/cloudops/base.py +96 -107
- runbooks/cloudops/cost_optimizer.py +549 -547
- runbooks/cloudops/infrastructure_optimizer.py +5 -4
- runbooks/cloudops/interfaces.py +226 -227
- runbooks/cloudops/lifecycle_manager.py +5 -4
- runbooks/cloudops/mcp_cost_validation.py +252 -235
- runbooks/cloudops/models.py +78 -53
- runbooks/cloudops/monitoring_automation.py +5 -4
- runbooks/cloudops/notebook_framework.py +179 -215
- runbooks/cloudops/security_enforcer.py +125 -159
- runbooks/common/accuracy_validator.py +11 -0
- runbooks/common/aws_pricing.py +349 -326
- runbooks/common/aws_pricing_api.py +211 -212
- runbooks/common/aws_profile_manager.py +341 -0
- runbooks/common/aws_utils.py +75 -80
- runbooks/common/business_logic.py +127 -105
- runbooks/common/cli_decorators.py +36 -60
- runbooks/common/comprehensive_cost_explorer_integration.py +456 -464
- runbooks/common/cross_account_manager.py +198 -205
- runbooks/common/date_utils.py +27 -39
- runbooks/common/decorators.py +235 -0
- runbooks/common/dry_run_examples.py +173 -208
- runbooks/common/dry_run_framework.py +157 -155
- runbooks/common/enhanced_exception_handler.py +15 -4
- runbooks/common/enhanced_logging_example.py +50 -64
- runbooks/common/enhanced_logging_integration_example.py +65 -37
- runbooks/common/env_utils.py +16 -16
- runbooks/common/error_handling.py +40 -38
- runbooks/common/lazy_loader.py +41 -23
- runbooks/common/logging_integration_helper.py +79 -86
- runbooks/common/mcp_cost_explorer_integration.py +478 -495
- runbooks/common/mcp_integration.py +63 -74
- runbooks/common/memory_optimization.py +140 -118
- runbooks/common/module_cli_base.py +37 -58
- runbooks/common/organizations_client.py +176 -194
- runbooks/common/patterns.py +204 -0
- runbooks/common/performance_monitoring.py +67 -71
- runbooks/common/performance_optimization_engine.py +283 -274
- runbooks/common/profile_utils.py +248 -39
- runbooks/common/rich_utils.py +643 -92
- runbooks/common/sre_performance_suite.py +177 -186
- runbooks/enterprise/__init__.py +1 -1
- runbooks/enterprise/logging.py +144 -106
- runbooks/enterprise/security.py +187 -204
- runbooks/enterprise/validation.py +43 -56
- runbooks/finops/__init__.py +29 -33
- runbooks/finops/account_resolver.py +1 -1
- runbooks/finops/advanced_optimization_engine.py +980 -0
- runbooks/finops/automation_core.py +268 -231
- runbooks/finops/business_case_config.py +184 -179
- runbooks/finops/cli.py +660 -139
- runbooks/finops/commvault_ec2_analysis.py +157 -164
- runbooks/finops/compute_cost_optimizer.py +336 -320
- runbooks/finops/config.py +20 -20
- runbooks/finops/cost_optimizer.py +488 -622
- runbooks/finops/cost_processor.py +332 -214
- runbooks/finops/dashboard_runner.py +1006 -172
- runbooks/finops/ebs_cost_optimizer.py +991 -657
- runbooks/finops/elastic_ip_optimizer.py +317 -257
- runbooks/finops/enhanced_mcp_integration.py +340 -0
- runbooks/finops/enhanced_progress.py +40 -37
- runbooks/finops/enhanced_trend_visualization.py +3 -2
- runbooks/finops/enterprise_wrappers.py +230 -292
- runbooks/finops/executive_export.py +203 -160
- runbooks/finops/helpers.py +130 -288
- runbooks/finops/iam_guidance.py +1 -1
- runbooks/finops/infrastructure/__init__.py +80 -0
- runbooks/finops/infrastructure/commands.py +506 -0
- runbooks/finops/infrastructure/load_balancer_optimizer.py +866 -0
- runbooks/finops/infrastructure/vpc_endpoint_optimizer.py +832 -0
- runbooks/finops/markdown_exporter.py +338 -175
- runbooks/finops/mcp_validator.py +1952 -0
- runbooks/finops/nat_gateway_optimizer.py +1513 -482
- runbooks/finops/network_cost_optimizer.py +657 -587
- runbooks/finops/notebook_utils.py +226 -188
- runbooks/finops/optimization_engine.py +1136 -0
- runbooks/finops/optimizer.py +25 -29
- runbooks/finops/rds_snapshot_optimizer.py +367 -411
- runbooks/finops/reservation_optimizer.py +427 -363
- runbooks/finops/scenario_cli_integration.py +77 -78
- runbooks/finops/scenarios.py +1278 -439
- runbooks/finops/schemas.py +218 -182
- runbooks/finops/snapshot_manager.py +2289 -0
- runbooks/finops/tests/test_finops_dashboard.py +3 -3
- runbooks/finops/tests/test_reference_images_validation.py +2 -2
- runbooks/finops/tests/test_single_account_features.py +17 -17
- runbooks/finops/tests/validate_test_suite.py +1 -1
- runbooks/finops/types.py +3 -3
- runbooks/finops/validation_framework.py +263 -269
- runbooks/finops/vpc_cleanup_exporter.py +191 -146
- runbooks/finops/vpc_cleanup_optimizer.py +593 -575
- runbooks/finops/workspaces_analyzer.py +171 -182
- runbooks/hitl/enhanced_workflow_engine.py +1 -1
- runbooks/integration/__init__.py +89 -0
- runbooks/integration/mcp_integration.py +1920 -0
- runbooks/inventory/CLAUDE.md +816 -0
- runbooks/inventory/README.md +3 -3
- runbooks/inventory/Tests/common_test_data.py +30 -30
- runbooks/inventory/__init__.py +2 -2
- runbooks/inventory/cloud_foundations_integration.py +144 -149
- runbooks/inventory/collectors/aws_comprehensive.py +28 -11
- runbooks/inventory/collectors/aws_networking.py +111 -101
- runbooks/inventory/collectors/base.py +4 -0
- runbooks/inventory/core/collector.py +495 -313
- runbooks/inventory/discovery.md +2 -2
- runbooks/inventory/drift_detection_cli.py +69 -96
- runbooks/inventory/find_ec2_security_groups.py +1 -1
- runbooks/inventory/inventory_mcp_cli.py +48 -46
- runbooks/inventory/list_rds_snapshots_aggregator.py +192 -208
- runbooks/inventory/mcp_inventory_validator.py +549 -465
- runbooks/inventory/mcp_vpc_validator.py +359 -442
- runbooks/inventory/organizations_discovery.py +56 -52
- runbooks/inventory/rich_inventory_display.py +33 -32
- runbooks/inventory/unified_validation_engine.py +278 -251
- runbooks/inventory/vpc_analyzer.py +733 -696
- runbooks/inventory/vpc_architecture_validator.py +293 -348
- runbooks/inventory/vpc_dependency_analyzer.py +382 -378
- runbooks/inventory/vpc_flow_analyzer.py +3 -3
- runbooks/main.py +152 -9147
- runbooks/main_final.py +91 -60
- runbooks/main_minimal.py +22 -10
- runbooks/main_optimized.py +131 -100
- runbooks/main_ultra_minimal.py +7 -2
- runbooks/mcp/__init__.py +36 -0
- runbooks/mcp/integration.py +679 -0
- runbooks/metrics/dora_metrics_engine.py +2 -2
- runbooks/monitoring/performance_monitor.py +9 -4
- runbooks/operate/dynamodb_operations.py +3 -1
- runbooks/operate/ec2_operations.py +145 -137
- runbooks/operate/iam_operations.py +146 -152
- runbooks/operate/mcp_integration.py +1 -1
- runbooks/operate/networking_cost_heatmap.py +33 -10
- runbooks/operate/privatelink_operations.py +1 -1
- runbooks/operate/rds_operations.py +223 -254
- runbooks/operate/s3_operations.py +107 -118
- runbooks/operate/vpc_endpoints.py +1 -1
- runbooks/operate/vpc_operations.py +648 -618
- runbooks/remediation/base.py +1 -1
- runbooks/remediation/commons.py +10 -7
- runbooks/remediation/commvault_ec2_analysis.py +71 -67
- runbooks/remediation/ec2_unattached_ebs_volumes.py +1 -0
- runbooks/remediation/multi_account.py +24 -21
- runbooks/remediation/rds_snapshot_list.py +91 -65
- runbooks/remediation/remediation_cli.py +92 -146
- runbooks/remediation/universal_account_discovery.py +83 -79
- runbooks/remediation/workspaces_list.py +49 -44
- runbooks/security/__init__.py +19 -0
- runbooks/security/assessment_runner.py +1150 -0
- runbooks/security/baseline_checker.py +812 -0
- runbooks/security/cloudops_automation_security_validator.py +509 -535
- runbooks/security/compliance_automation_engine.py +17 -17
- runbooks/security/config/__init__.py +2 -2
- runbooks/security/config/compliance_config.py +50 -50
- runbooks/security/config_template_generator.py +63 -76
- runbooks/security/enterprise_security_framework.py +1 -1
- runbooks/security/executive_security_dashboard.py +519 -508
- runbooks/security/integration_test_enterprise_security.py +5 -3
- runbooks/security/multi_account_security_controls.py +959 -1210
- runbooks/security/real_time_security_monitor.py +422 -444
- runbooks/security/run_script.py +1 -1
- runbooks/security/security_baseline_tester.py +1 -1
- runbooks/security/security_cli.py +143 -112
- runbooks/security/test_2way_validation.py +439 -0
- runbooks/security/two_way_validation_framework.py +852 -0
- runbooks/sre/mcp_reliability_engine.py +6 -6
- runbooks/sre/production_monitoring_framework.py +167 -177
- runbooks/tdd/__init__.py +15 -0
- runbooks/tdd/cli.py +1071 -0
- runbooks/utils/__init__.py +14 -17
- runbooks/utils/logger.py +7 -2
- runbooks/utils/version_validator.py +51 -48
- runbooks/validation/__init__.py +6 -6
- runbooks/validation/cli.py +9 -3
- runbooks/validation/comprehensive_2way_validator.py +754 -708
- runbooks/validation/mcp_validator.py +906 -228
- runbooks/validation/terraform_citations_validator.py +104 -115
- runbooks/validation/terraform_drift_detector.py +447 -451
- runbooks/vpc/README.md +617 -0
- runbooks/vpc/__init__.py +8 -1
- runbooks/vpc/analyzer.py +577 -0
- runbooks/vpc/cleanup_wrapper.py +476 -413
- runbooks/vpc/cli_cloudtrail_commands.py +339 -0
- runbooks/vpc/cli_mcp_validation_commands.py +480 -0
- runbooks/vpc/cloudtrail_audit_integration.py +717 -0
- runbooks/vpc/config.py +92 -97
- runbooks/vpc/cost_engine.py +411 -148
- runbooks/vpc/cost_explorer_integration.py +553 -0
- runbooks/vpc/cross_account_session.py +101 -106
- runbooks/vpc/enhanced_mcp_validation.py +917 -0
- runbooks/vpc/eni_gate_validator.py +961 -0
- runbooks/vpc/heatmap_engine.py +190 -162
- runbooks/vpc/mcp_no_eni_validator.py +681 -640
- runbooks/vpc/nat_gateway_optimizer.py +358 -0
- runbooks/vpc/networking_wrapper.py +15 -8
- runbooks/vpc/pdca_remediation_planner.py +528 -0
- runbooks/vpc/performance_optimized_analyzer.py +219 -231
- runbooks/vpc/runbooks_adapter.py +1167 -241
- runbooks/vpc/tdd_red_phase_stubs.py +601 -0
- runbooks/vpc/test_data_loader.py +358 -0
- runbooks/vpc/tests/conftest.py +314 -4
- runbooks/vpc/tests/test_cleanup_framework.py +1022 -0
- runbooks/vpc/tests/test_cost_engine.py +0 -2
- runbooks/vpc/topology_generator.py +326 -0
- runbooks/vpc/unified_scenarios.py +1302 -1129
- runbooks/vpc/vpc_cleanup_integration.py +1943 -1115
- runbooks-1.1.5.dist-info/METADATA +328 -0
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/RECORD +233 -200
- runbooks/finops/README.md +0 -414
- runbooks/finops/accuracy_cross_validator.py +0 -647
- runbooks/finops/business_cases.py +0 -950
- runbooks/finops/dashboard_router.py +0 -922
- runbooks/finops/ebs_optimizer.py +0 -956
- runbooks/finops/embedded_mcp_validator.py +0 -1629
- runbooks/finops/enhanced_dashboard_runner.py +0 -527
- runbooks/finops/finops_dashboard.py +0 -584
- runbooks/finops/finops_scenarios.py +0 -1218
- runbooks/finops/legacy_migration.py +0 -730
- runbooks/finops/multi_dashboard.py +0 -1519
- runbooks/finops/single_dashboard.py +0 -1113
- runbooks/finops/unlimited_scenarios.py +0 -393
- runbooks-1.1.3.dist-info/METADATA +0 -799
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/WHEEL +0 -0
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/entry_points.txt +0 -0
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/licenses/LICENSE +0 -0
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/top_level.txt +0 -0
@@ -44,6 +44,7 @@ except ImportError:
|
|
44
44
|
ENHANCED_PROFILES_AVAILABLE = False
|
45
45
|
# Fallback profile definitions with universal environment support
|
46
46
|
import os
|
47
|
+
|
47
48
|
ENTERPRISE_PROFILES = {
|
48
49
|
"BILLING_PROFILE": os.getenv("BILLING_PROFILE", "default-billing-profile"),
|
49
50
|
"MANAGEMENT_PROFILE": os.getenv("MANAGEMENT_PROFILE", "default-management-profile"),
|
@@ -106,11 +107,12 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
106
107
|
self.mcp_integrator = EnterpriseMCPIntegrator(profile)
|
107
108
|
self.cross_module_integrator = EnterpriseCrossModuleIntegrator(profile)
|
108
109
|
self.enable_mcp_validation = True
|
109
|
-
|
110
|
+
|
110
111
|
# Initialize inventory-specific MCP validator
|
111
112
|
self.inventory_mcp_validator = None
|
112
113
|
try:
|
113
114
|
from ..mcp_inventory_validator import create_inventory_mcp_validator
|
115
|
+
|
114
116
|
# Use profiles that would work for inventory operations
|
115
117
|
validator_profiles = [self.active_profile]
|
116
118
|
self.inventory_mcp_validator = create_inventory_mcp_validator(validator_profiles)
|
@@ -139,12 +141,12 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
139
141
|
def _initialize_profile_architecture(self) -> str:
|
140
142
|
"""
|
141
143
|
Initialize profile management following --profile or --all patterns.
|
142
|
-
|
144
|
+
|
143
145
|
Strategic Alignment: "Do one thing and do it well"
|
144
146
|
- Single profile override pattern: --profile takes precedence
|
145
147
|
- Universal AWS environment compatibility: works with ANY profile configuration
|
146
148
|
- Graceful fallback system for discovery across different AWS setups
|
147
|
-
|
149
|
+
|
148
150
|
Returns:
|
149
151
|
str: The active profile to use for all operations
|
150
152
|
"""
|
@@ -153,24 +155,24 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
153
155
|
print_info(f"✅ Universal AWS Compatibility: Using user-specified profile '{self.profile}'")
|
154
156
|
logger.info("Profile override via --profile parameter - universal environment support")
|
155
157
|
return self.profile
|
156
|
-
|
158
|
+
|
157
159
|
# SECONDARY: Environment variable fallback with intelligent prioritization
|
158
160
|
# Priority order: Management > Billing > Operations > Default (Organizations discovery preference)
|
159
161
|
env_profile = (
|
160
|
-
os.getenv("MANAGEMENT_PROFILE")
|
161
|
-
os.getenv("BILLING_PROFILE")
|
162
|
-
os.getenv("CENTRALISED_OPS_PROFILE")
|
163
|
-
os.getenv("SINGLE_AWS_PROFILE")
|
164
|
-
"default"
|
162
|
+
os.getenv("MANAGEMENT_PROFILE")
|
163
|
+
or os.getenv("BILLING_PROFILE")
|
164
|
+
or os.getenv("CENTRALISED_OPS_PROFILE")
|
165
|
+
or os.getenv("SINGLE_AWS_PROFILE")
|
166
|
+
or "default"
|
165
167
|
)
|
166
|
-
|
168
|
+
|
167
169
|
if env_profile != "default":
|
168
170
|
print_info(f"✅ Universal AWS Compatibility: Using environment profile '{env_profile}'")
|
169
171
|
logger.info(f"Environment variable profile selected: {env_profile}")
|
170
172
|
else:
|
171
173
|
print_info("✅ Universal AWS Compatibility: Using 'default' profile - works with any AWS CLI configuration")
|
172
174
|
logger.info("Using default profile - universal compatibility mode")
|
173
|
-
|
175
|
+
|
174
176
|
return env_profile
|
175
177
|
|
176
178
|
def _initialize_collectors(self) -> Dict[str, str]:
|
@@ -190,19 +192,19 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
190
192
|
|
191
193
|
logger.debug(f"Initialized {len(collectors)} resource collectors")
|
192
194
|
return collectors
|
193
|
-
|
195
|
+
|
194
196
|
def _extract_resource_counts(self, resource_data: Dict[str, Any]) -> Dict[str, int]:
|
195
197
|
"""
|
196
198
|
Extract resource counts from collected inventory data for MCP validation.
|
197
|
-
|
199
|
+
|
198
200
|
Args:
|
199
201
|
resource_data: Raw resource data from inventory collection
|
200
|
-
|
202
|
+
|
201
203
|
Returns:
|
202
204
|
Dictionary mapping resource types to counts
|
203
205
|
"""
|
204
206
|
resource_counts = {}
|
205
|
-
|
207
|
+
|
206
208
|
try:
|
207
209
|
# Handle various data structures from inventory collection
|
208
210
|
if isinstance(resource_data, dict):
|
@@ -215,15 +217,15 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
215
217
|
for region_data in resources.values():
|
216
218
|
if isinstance(region_data, list):
|
217
219
|
total_count += len(region_data)
|
218
|
-
elif isinstance(region_data, dict) and
|
219
|
-
total_count += len(region_data[
|
220
|
+
elif isinstance(region_data, dict) and "resources" in region_data:
|
221
|
+
total_count += len(region_data["resources"])
|
220
222
|
resource_counts[resource_type] = total_count
|
221
223
|
elif isinstance(resources, int):
|
222
224
|
resource_counts[resource_type] = resources
|
223
|
-
|
225
|
+
|
224
226
|
logger.debug(f"Extracted resource counts for validation: {resource_counts}")
|
225
227
|
return resource_counts
|
226
|
-
|
228
|
+
|
227
229
|
except Exception as e:
|
228
230
|
logger.warning(f"Failed to extract resource counts for MCP validation: {e}")
|
229
231
|
return {}
|
@@ -235,17 +237,17 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
235
237
|
def get_organization_accounts(self) -> List[str]:
|
236
238
|
"""
|
237
239
|
Get list of accounts in AWS Organization with universal compatibility.
|
238
|
-
|
239
|
-
Strategic Alignment: "Do one thing and do it well"
|
240
|
+
|
241
|
+
Strategic Alignment: "Do one thing and do it well"
|
240
242
|
- Universal AWS environment compatibility: works with ANY Organizations setup
|
241
243
|
- Intelligent fallback system: Organizations → standalone account detection
|
242
244
|
- Graceful handling of different permission scenarios
|
243
245
|
"""
|
244
246
|
try:
|
245
247
|
# Use active profile for Organizations operations (Universal Compatibility)
|
246
|
-
management_session = create_management_session(
|
248
|
+
management_session = create_management_session(profile_name=self.active_profile)
|
247
249
|
organizations_client = management_session.client("organizations")
|
248
|
-
|
250
|
+
|
249
251
|
print_info(f"🔍 Universal Discovery: Attempting Organizations API with profile '{self.active_profile}'...")
|
250
252
|
response = self._make_aws_call(organizations_client.list_accounts)
|
251
253
|
|
@@ -256,7 +258,9 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
256
258
|
|
257
259
|
if accounts:
|
258
260
|
print_success(f"✅ Organizations Discovery: Found {len(accounts)} active accounts in organization")
|
259
|
-
logger.info(
|
261
|
+
logger.info(
|
262
|
+
f"Organizations discovery successful: {len(accounts)} accounts with profile {self.active_profile}"
|
263
|
+
)
|
260
264
|
return accounts
|
261
265
|
else:
|
262
266
|
print_warning("⚠️ Organizations Discovery: No active accounts found in organization")
|
@@ -265,9 +269,11 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
265
269
|
except Exception as e:
|
266
270
|
# Enhanced error messages for different AWS environment scenarios
|
267
271
|
error_message = str(e).lower()
|
268
|
-
|
272
|
+
|
269
273
|
if "accessdenied" in error_message or "unauthorized" in error_message:
|
270
|
-
print_warning(
|
274
|
+
print_warning(
|
275
|
+
f"⚠️ Universal Compatibility: Profile '{self.active_profile}' lacks Organizations permissions"
|
276
|
+
)
|
271
277
|
print_info("💡 Single Account Mode: Continuing with current account (universal compatibility)")
|
272
278
|
elif "organizationsnotinuse" in error_message:
|
273
279
|
print_info(f"ℹ️ Standalone Account: Profile '{self.active_profile}' not in an AWS Organization")
|
@@ -275,9 +281,9 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
275
281
|
else:
|
276
282
|
print_warning(f"⚠️ Organizations Discovery Failed: {e}")
|
277
283
|
print_info("💡 Fallback Mode: Continuing with current account for universal compatibility")
|
278
|
-
|
284
|
+
|
279
285
|
logger.warning(f"Organization discovery failed, graceful fallback: {e}")
|
280
|
-
|
286
|
+
|
281
287
|
# Universal fallback: always return current account for single-account operations
|
282
288
|
return [self.get_account_id()]
|
283
289
|
|
@@ -345,37 +351,37 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
345
351
|
if self.enable_mcp_validation and self.inventory_mcp_validator:
|
346
352
|
try:
|
347
353
|
print_info("Validating inventory results with specialized inventory MCP validator")
|
348
|
-
|
354
|
+
|
349
355
|
# Extract resource counts for validation
|
350
356
|
# Build validation data structure that matches what the validator expects
|
351
357
|
resource_counts = self._extract_resource_counts(resource_data)
|
352
|
-
|
358
|
+
|
353
359
|
# Add resource counts to results for the validator to find
|
354
360
|
results["resource_counts"] = resource_counts
|
355
|
-
|
361
|
+
|
356
362
|
validation_data = {
|
357
363
|
"resource_counts": resource_counts,
|
358
364
|
"regions": results["metadata"].get("regions_scanned", []),
|
359
365
|
self.active_profile: {
|
360
366
|
"resource_counts": resource_counts,
|
361
|
-
"regions": results["metadata"].get("regions_scanned", [])
|
362
|
-
}
|
367
|
+
"regions": results["metadata"].get("regions_scanned", []),
|
368
|
+
},
|
363
369
|
}
|
364
|
-
|
370
|
+
|
365
371
|
# Run inventory-specific MCP validation
|
366
372
|
inventory_validation = self.inventory_mcp_validator.validate_inventory_data(validation_data)
|
367
|
-
|
373
|
+
|
368
374
|
results["inventory_mcp_validation"] = inventory_validation
|
369
|
-
|
375
|
+
|
370
376
|
overall_accuracy = inventory_validation.get("total_accuracy", 0)
|
371
377
|
if inventory_validation.get("passed_validation", False):
|
372
378
|
print_success(f"✅ Inventory MCP validation PASSED: {overall_accuracy:.1f}% accuracy achieved")
|
373
379
|
else:
|
374
380
|
print_warning(f"⚠️ Inventory MCP validation: {overall_accuracy:.1f}% accuracy (≥99.5% required)")
|
375
|
-
|
376
|
-
# Also try the generic MCP integrator as backup
|
381
|
+
|
382
|
+
# Also try the generic MCP integrator as backup - using proper async handling
|
377
383
|
try:
|
378
|
-
validation_result =
|
384
|
+
validation_result = self._run_async_validation_safely(results)
|
379
385
|
results["mcp_validation"] = validation_result.to_dict()
|
380
386
|
except Exception:
|
381
387
|
pass # Skip generic validation if it fails
|
@@ -383,10 +389,10 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
383
389
|
except Exception as e:
|
384
390
|
print_warning(f"Inventory MCP validation failed: {str(e)[:50]}... - continuing without validation")
|
385
391
|
results["inventory_mcp_validation"] = {"error": str(e), "validation_skipped": True}
|
386
|
-
|
387
|
-
# Fallback to generic MCP integration
|
392
|
+
|
393
|
+
# Fallback to generic MCP integration with proper async handling
|
388
394
|
try:
|
389
|
-
validation_result =
|
395
|
+
validation_result = self._run_async_validation_safely(results)
|
390
396
|
results["mcp_validation"] = validation_result.to_dict()
|
391
397
|
except Exception as fallback_e:
|
392
398
|
results["mcp_validation"] = {"error": str(fallback_e), "validation_skipped": True}
|
@@ -438,12 +444,64 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
438
444
|
results["errors"].append(error_msg)
|
439
445
|
return results
|
440
446
|
|
447
|
+
def _run_async_validation_safely(self, results: Dict[str, Any]):
|
448
|
+
"""
|
449
|
+
Safely run async MCP validation handling event loop conflicts.
|
450
|
+
|
451
|
+
This method properly handles the case where an event loop is already running
|
452
|
+
by using proper async execution patterns instead of skipping validation.
|
453
|
+
|
454
|
+
Args:
|
455
|
+
results: Inventory results to validate
|
456
|
+
|
457
|
+
Returns:
|
458
|
+
Validation result from MCP integrator
|
459
|
+
"""
|
460
|
+
try:
|
461
|
+
# Check if event loop is already running
|
462
|
+
try:
|
463
|
+
loop = asyncio.get_running_loop()
|
464
|
+
# Event loop is running, we need to use a different approach
|
465
|
+
# Create a task that can be run in the current loop
|
466
|
+
import concurrent.futures
|
467
|
+
import threading
|
468
|
+
|
469
|
+
# Use ThreadPoolExecutor to run async code in a separate thread
|
470
|
+
def run_validation():
|
471
|
+
new_loop = asyncio.new_event_loop()
|
472
|
+
asyncio.set_event_loop(new_loop)
|
473
|
+
try:
|
474
|
+
return new_loop.run_until_complete(self.mcp_integrator.validate_inventory_operations(results))
|
475
|
+
finally:
|
476
|
+
new_loop.close()
|
477
|
+
|
478
|
+
with concurrent.futures.ThreadPoolExecutor() as executor:
|
479
|
+
future = executor.submit(run_validation)
|
480
|
+
return future.result(timeout=30) # 30 second timeout
|
481
|
+
|
482
|
+
except RuntimeError:
|
483
|
+
# No event loop running, safe to use asyncio.run()
|
484
|
+
return asyncio.run(self.mcp_integrator.validate_inventory_operations(results))
|
485
|
+
|
486
|
+
except Exception as e:
|
487
|
+
# Create a fallback result with error information
|
488
|
+
class ValidationResult:
|
489
|
+
def to_dict(self):
|
490
|
+
return {
|
491
|
+
"error": f"Async validation failed: {str(e)[:100]}",
|
492
|
+
"validation_skipped": True,
|
493
|
+
"total_accuracy": 0.0,
|
494
|
+
"passed_validation": False,
|
495
|
+
}
|
496
|
+
|
497
|
+
return ValidationResult()
|
498
|
+
|
441
499
|
def _collect_parallel(
|
442
500
|
self, resource_types: List[str], account_ids: List[str], include_costs: bool
|
443
501
|
) -> Dict[str, Any]:
|
444
502
|
"""
|
445
503
|
Collect inventory in parallel with enhanced performance monitoring.
|
446
|
-
|
504
|
+
|
447
505
|
Follows the same pattern as legacy implementation but with enterprise
|
448
506
|
performance monitoring and error handling.
|
449
507
|
"""
|
@@ -486,7 +544,7 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
486
544
|
) -> Dict[str, Any]:
|
487
545
|
"""
|
488
546
|
Collect inventory sequentially with enhanced error handling.
|
489
|
-
|
547
|
+
|
490
548
|
Follows the same pattern as legacy implementation but with enhanced
|
491
549
|
error handling and progress tracking.
|
492
550
|
"""
|
@@ -514,16 +572,18 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
514
572
|
def _collect_resource_for_account(self, resource_type: str, account_id: str, include_costs: bool) -> Dict[str, Any]:
|
515
573
|
"""
|
516
574
|
Collect specific resource type for an account using REAL AWS API calls.
|
517
|
-
|
575
|
+
|
518
576
|
This method makes actual AWS API calls to discover resources, following
|
519
577
|
the proven patterns from the existing inventory modules.
|
520
578
|
"""
|
521
579
|
try:
|
522
580
|
# Use active profile for AWS API calls
|
523
581
|
session = boto3.Session(profile_name=self.active_profile)
|
524
|
-
|
525
|
-
print_info(
|
526
|
-
|
582
|
+
|
583
|
+
print_info(
|
584
|
+
f"Collecting {resource_type} resources from account {account_id} using profile {self.active_profile}"
|
585
|
+
)
|
586
|
+
|
527
587
|
if resource_type == "ec2":
|
528
588
|
return self._collect_ec2_instances(session, account_id)
|
529
589
|
elif resource_type == "rds":
|
@@ -546,13 +606,13 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
546
606
|
print_warning(f"Resource type '{resource_type}' not supported yet")
|
547
607
|
return {
|
548
608
|
"resources": [],
|
549
|
-
"count": 0,
|
609
|
+
"count": 0,
|
550
610
|
"resource_type": resource_type,
|
551
611
|
"account_id": account_id,
|
552
612
|
"collection_timestamp": datetime.now().isoformat(),
|
553
|
-
"warning": f"Resource type {resource_type} not implemented yet"
|
613
|
+
"warning": f"Resource type {resource_type} not implemented yet",
|
554
614
|
}
|
555
|
-
|
615
|
+
|
556
616
|
except Exception as e:
|
557
617
|
error_msg = f"Failed to collect {resource_type} for account {account_id}: {e}"
|
558
618
|
logger.error(error_msg)
|
@@ -569,13 +629,13 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
569
629
|
try:
|
570
630
|
region = self.region or session.region_name or "us-east-1"
|
571
631
|
ec2_client = session.client("ec2", region_name=region)
|
572
|
-
|
632
|
+
|
573
633
|
print_info(f"Calling EC2 describe_instances API for account {account_id} in region {region}")
|
574
|
-
|
634
|
+
|
575
635
|
# Make real AWS API call with pagination support
|
576
636
|
instances = []
|
577
|
-
paginator = ec2_client.get_paginator(
|
578
|
-
|
637
|
+
paginator = ec2_client.get_paginator("describe_instances")
|
638
|
+
|
579
639
|
for page in paginator.paginate():
|
580
640
|
for reservation in page.get("Reservations", []):
|
581
641
|
for instance in reservation.get("Instances", []):
|
@@ -586,7 +646,9 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
586
646
|
"state": instance["State"]["Name"],
|
587
647
|
"region": region,
|
588
648
|
"account_id": account_id,
|
589
|
-
"launch_time": instance.get("LaunchTime", "").isoformat()
|
649
|
+
"launch_time": instance.get("LaunchTime", "").isoformat()
|
650
|
+
if instance.get("LaunchTime")
|
651
|
+
else "",
|
590
652
|
"availability_zone": instance.get("Placement", {}).get("AvailabilityZone", ""),
|
591
653
|
"vpc_id": instance.get("VpcId", ""),
|
592
654
|
"subnet_id": instance.get("SubnetId", ""),
|
@@ -594,7 +656,7 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
594
656
|
"public_ip_address": instance.get("PublicIpAddress", ""),
|
595
657
|
"public_dns_name": instance.get("PublicDnsName", ""),
|
596
658
|
}
|
597
|
-
|
659
|
+
|
598
660
|
# Extract tags
|
599
661
|
tags = {}
|
600
662
|
name = "No Name Tag"
|
@@ -602,20 +664,20 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
602
664
|
tags[tag["Key"]] = tag["Value"]
|
603
665
|
if tag["Key"] == "Name":
|
604
666
|
name = tag["Value"]
|
605
|
-
|
667
|
+
|
606
668
|
instance_data["tags"] = tags
|
607
669
|
instance_data["name"] = name
|
608
|
-
|
670
|
+
|
609
671
|
# Extract security groups
|
610
672
|
instance_data["security_groups"] = [
|
611
|
-
{"group_id": sg["GroupId"], "group_name": sg["GroupName"]}
|
673
|
+
{"group_id": sg["GroupId"], "group_name": sg["GroupName"]}
|
612
674
|
for sg in instance.get("SecurityGroups", [])
|
613
675
|
]
|
614
|
-
|
676
|
+
|
615
677
|
instances.append(instance_data)
|
616
|
-
|
678
|
+
|
617
679
|
print_success(f"Found {len(instances)} EC2 instances in account {account_id}")
|
618
|
-
|
680
|
+
|
619
681
|
return {
|
620
682
|
"instances": instances,
|
621
683
|
"count": len(instances),
|
@@ -623,7 +685,7 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
623
685
|
"region": region,
|
624
686
|
"account_id": account_id,
|
625
687
|
}
|
626
|
-
|
688
|
+
|
627
689
|
except Exception as e:
|
628
690
|
print_error(f"Failed to collect EC2 instances: {e}")
|
629
691
|
raise
|
@@ -633,13 +695,13 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
633
695
|
try:
|
634
696
|
region = self.region or session.region_name or "us-east-1"
|
635
697
|
rds_client = session.client("rds", region_name=region)
|
636
|
-
|
698
|
+
|
637
699
|
print_info(f"Calling RDS describe_db_instances API for account {account_id} in region {region}")
|
638
|
-
|
700
|
+
|
639
701
|
# Make real AWS API call with pagination support
|
640
702
|
instances = []
|
641
|
-
paginator = rds_client.get_paginator(
|
642
|
-
|
703
|
+
paginator = rds_client.get_paginator("describe_db_instances")
|
704
|
+
|
643
705
|
for page in paginator.paginate():
|
644
706
|
for db_instance in page.get("DBInstances", []):
|
645
707
|
instance_data = {
|
@@ -653,15 +715,19 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
653
715
|
"multi_az": db_instance.get("MultiAZ", False),
|
654
716
|
"storage_type": db_instance.get("StorageType", ""),
|
655
717
|
"allocated_storage": db_instance.get("AllocatedStorage", 0),
|
656
|
-
"endpoint": db_instance.get("Endpoint", {}).get("Address", "")
|
718
|
+
"endpoint": db_instance.get("Endpoint", {}).get("Address", "")
|
719
|
+
if db_instance.get("Endpoint")
|
720
|
+
else "",
|
657
721
|
"port": db_instance.get("Endpoint", {}).get("Port", 0) if db_instance.get("Endpoint") else 0,
|
658
|
-
"vpc_id": db_instance.get("DBSubnetGroup", {}).get("VpcId", "")
|
722
|
+
"vpc_id": db_instance.get("DBSubnetGroup", {}).get("VpcId", "")
|
723
|
+
if db_instance.get("DBSubnetGroup")
|
724
|
+
else "",
|
659
725
|
}
|
660
|
-
|
726
|
+
|
661
727
|
instances.append(instance_data)
|
662
|
-
|
728
|
+
|
663
729
|
print_success(f"Found {len(instances)} RDS instances in account {account_id}")
|
664
|
-
|
730
|
+
|
665
731
|
return {
|
666
732
|
"instances": instances,
|
667
733
|
"count": len(instances),
|
@@ -669,7 +735,7 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
669
735
|
"region": region,
|
670
736
|
"account_id": account_id,
|
671
737
|
}
|
672
|
-
|
738
|
+
|
673
739
|
except Exception as e:
|
674
740
|
print_error(f"Failed to collect RDS instances: {e}")
|
675
741
|
raise
|
@@ -678,20 +744,20 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
678
744
|
"""Collect S3 buckets using real AWS API calls."""
|
679
745
|
try:
|
680
746
|
s3_client = session.client("s3")
|
681
|
-
|
747
|
+
|
682
748
|
print_info(f"Calling S3 list_buckets API for account {account_id}")
|
683
|
-
|
749
|
+
|
684
750
|
# Make real AWS API call - S3 buckets are global
|
685
751
|
response = s3_client.list_buckets()
|
686
752
|
buckets = []
|
687
|
-
|
753
|
+
|
688
754
|
for bucket in response.get("Buckets", []):
|
689
755
|
bucket_data = {
|
690
756
|
"name": bucket["Name"],
|
691
757
|
"creation_date": bucket["CreationDate"].isoformat(),
|
692
758
|
"account_id": account_id,
|
693
759
|
}
|
694
|
-
|
760
|
+
|
695
761
|
# Try to get bucket location (region)
|
696
762
|
try:
|
697
763
|
location_response = s3_client.get_bucket_location(Bucket=bucket["Name"])
|
@@ -702,7 +768,7 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
702
768
|
except Exception as e:
|
703
769
|
logger.warning(f"Could not get location for bucket {bucket['Name']}: {e}")
|
704
770
|
bucket_data["region"] = "unknown"
|
705
|
-
|
771
|
+
|
706
772
|
# Try to get bucket versioning
|
707
773
|
try:
|
708
774
|
versioning_response = s3_client.get_bucket_versioning(Bucket=bucket["Name"])
|
@@ -710,18 +776,18 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
710
776
|
except Exception as e:
|
711
777
|
logger.warning(f"Could not get versioning for bucket {bucket['Name']}: {e}")
|
712
778
|
bucket_data["versioning"] = "unknown"
|
713
|
-
|
779
|
+
|
714
780
|
buckets.append(bucket_data)
|
715
|
-
|
781
|
+
|
716
782
|
print_success(f"Found {len(buckets)} S3 buckets in account {account_id}")
|
717
|
-
|
783
|
+
|
718
784
|
return {
|
719
785
|
"buckets": buckets,
|
720
786
|
"count": len(buckets),
|
721
787
|
"collection_timestamp": datetime.now().isoformat(),
|
722
788
|
"account_id": account_id,
|
723
789
|
}
|
724
|
-
|
790
|
+
|
725
791
|
except Exception as e:
|
726
792
|
print_error(f"Failed to collect S3 buckets: {e}")
|
727
793
|
raise
|
@@ -731,13 +797,13 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
731
797
|
try:
|
732
798
|
region = self.region or session.region_name or "us-east-1"
|
733
799
|
lambda_client = session.client("lambda", region_name=region)
|
734
|
-
|
800
|
+
|
735
801
|
print_info(f"Calling Lambda list_functions API for account {account_id} in region {region}")
|
736
|
-
|
802
|
+
|
737
803
|
# Make real AWS API call with pagination support
|
738
804
|
functions = []
|
739
|
-
paginator = lambda_client.get_paginator(
|
740
|
-
|
805
|
+
paginator = lambda_client.get_paginator("list_functions")
|
806
|
+
|
741
807
|
for page in paginator.paginate():
|
742
808
|
for function in page.get("Functions", []):
|
743
809
|
function_data = {
|
@@ -753,11 +819,11 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
753
819
|
"account_id": account_id,
|
754
820
|
"region": region,
|
755
821
|
}
|
756
|
-
|
822
|
+
|
757
823
|
functions.append(function_data)
|
758
|
-
|
824
|
+
|
759
825
|
print_success(f"Found {len(functions)} Lambda functions in account {account_id}")
|
760
|
-
|
826
|
+
|
761
827
|
return {
|
762
828
|
"functions": functions,
|
763
829
|
"count": len(functions),
|
@@ -765,7 +831,7 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
765
831
|
"region": region,
|
766
832
|
"account_id": account_id,
|
767
833
|
}
|
768
|
-
|
834
|
+
|
769
835
|
except Exception as e:
|
770
836
|
print_error(f"Failed to collect Lambda functions: {e}")
|
771
837
|
raise
|
@@ -774,13 +840,13 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
774
840
|
"""Collect IAM resources using real AWS API calls."""
|
775
841
|
try:
|
776
842
|
iam_client = session.client("iam")
|
777
|
-
|
843
|
+
|
778
844
|
print_info(f"Calling IAM APIs for account {account_id}")
|
779
|
-
|
845
|
+
|
780
846
|
resources = {"users": [], "roles": [], "policies": [], "groups": []}
|
781
|
-
|
847
|
+
|
782
848
|
# Collect users
|
783
|
-
paginator = iam_client.get_paginator(
|
849
|
+
paginator = iam_client.get_paginator("list_users")
|
784
850
|
for page in paginator.paginate():
|
785
851
|
for user in page.get("Users", []):
|
786
852
|
user_data = {
|
@@ -792,9 +858,9 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
792
858
|
"account_id": account_id,
|
793
859
|
}
|
794
860
|
resources["users"].append(user_data)
|
795
|
-
|
861
|
+
|
796
862
|
# Collect roles
|
797
|
-
paginator = iam_client.get_paginator(
|
863
|
+
paginator = iam_client.get_paginator("list_roles")
|
798
864
|
for page in paginator.paginate():
|
799
865
|
for role in page.get("Roles", []):
|
800
866
|
role_data = {
|
@@ -806,17 +872,17 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
806
872
|
"account_id": account_id,
|
807
873
|
}
|
808
874
|
resources["roles"].append(role_data)
|
809
|
-
|
875
|
+
|
810
876
|
total_count = len(resources["users"]) + len(resources["roles"])
|
811
877
|
print_success(f"Found {total_count} IAM resources in account {account_id}")
|
812
|
-
|
878
|
+
|
813
879
|
return {
|
814
880
|
"resources": resources,
|
815
881
|
"count": total_count,
|
816
882
|
"collection_timestamp": datetime.now().isoformat(),
|
817
883
|
"account_id": account_id,
|
818
884
|
}
|
819
|
-
|
885
|
+
|
820
886
|
except Exception as e:
|
821
887
|
print_error(f"Failed to collect IAM resources: {e}")
|
822
888
|
raise
|
@@ -826,12 +892,12 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
826
892
|
try:
|
827
893
|
region = self.region or session.region_name or "us-east-1"
|
828
894
|
ec2_client = session.client("ec2", region_name=region)
|
829
|
-
|
895
|
+
|
830
896
|
print_info(f"Calling EC2 VPC APIs for account {account_id} in region {region}")
|
831
|
-
|
897
|
+
|
832
898
|
vpcs = []
|
833
|
-
paginator = ec2_client.get_paginator(
|
834
|
-
|
899
|
+
paginator = ec2_client.get_paginator("describe_vpcs")
|
900
|
+
|
835
901
|
for page in paginator.paginate():
|
836
902
|
for vpc in page.get("Vpcs", []):
|
837
903
|
vpc_data = {
|
@@ -843,7 +909,7 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
843
909
|
"account_id": account_id,
|
844
910
|
"region": region,
|
845
911
|
}
|
846
|
-
|
912
|
+
|
847
913
|
# Extract tags
|
848
914
|
tags = {}
|
849
915
|
name = "No Name Tag"
|
@@ -851,14 +917,14 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
851
917
|
tags[tag["Key"]] = tag["Value"]
|
852
918
|
if tag["Key"] == "Name":
|
853
919
|
name = tag["Value"]
|
854
|
-
|
920
|
+
|
855
921
|
vpc_data["tags"] = tags
|
856
922
|
vpc_data["name"] = name
|
857
|
-
|
923
|
+
|
858
924
|
vpcs.append(vpc_data)
|
859
|
-
|
925
|
+
|
860
926
|
print_success(f"Found {len(vpcs)} VPCs in account {account_id}")
|
861
|
-
|
927
|
+
|
862
928
|
return {
|
863
929
|
"vpcs": vpcs,
|
864
930
|
"count": len(vpcs),
|
@@ -866,7 +932,7 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
866
932
|
"region": region,
|
867
933
|
"account_id": account_id,
|
868
934
|
}
|
869
|
-
|
935
|
+
|
870
936
|
except Exception as e:
|
871
937
|
print_error(f"Failed to collect VPC resources: {e}")
|
872
938
|
raise
|
@@ -876,12 +942,12 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
876
942
|
try:
|
877
943
|
region = self.region or session.region_name or "us-east-1"
|
878
944
|
cf_client = session.client("cloudformation", region_name=region)
|
879
|
-
|
945
|
+
|
880
946
|
print_info(f"Calling CloudFormation describe_stacks API for account {account_id} in region {region}")
|
881
|
-
|
947
|
+
|
882
948
|
stacks = []
|
883
|
-
paginator = cf_client.get_paginator(
|
884
|
-
|
949
|
+
paginator = cf_client.get_paginator("describe_stacks")
|
950
|
+
|
885
951
|
for page in paginator.paginate():
|
886
952
|
for stack in page.get("Stacks", []):
|
887
953
|
stack_data = {
|
@@ -893,14 +959,14 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
893
959
|
"account_id": account_id,
|
894
960
|
"region": region,
|
895
961
|
}
|
896
|
-
|
962
|
+
|
897
963
|
if "LastUpdatedTime" in stack:
|
898
964
|
stack_data["last_updated_time"] = stack["LastUpdatedTime"].isoformat()
|
899
|
-
|
965
|
+
|
900
966
|
stacks.append(stack_data)
|
901
|
-
|
967
|
+
|
902
968
|
print_success(f"Found {len(stacks)} CloudFormation stacks in account {account_id}")
|
903
|
-
|
969
|
+
|
904
970
|
return {
|
905
971
|
"stacks": stacks,
|
906
972
|
"count": len(stacks),
|
@@ -908,7 +974,7 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
908
974
|
"region": region,
|
909
975
|
"account_id": account_id,
|
910
976
|
}
|
911
|
-
|
977
|
+
|
912
978
|
except Exception as e:
|
913
979
|
print_error(f"Failed to collect CloudFormation stacks: {e}")
|
914
980
|
raise
|
@@ -919,17 +985,17 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
919
985
|
# Note: Cost Explorer requires specific billing permissions
|
920
986
|
print_warning("Cost data collection requires AWS Cost Explorer permissions")
|
921
987
|
print_info(f"Attempting to collect cost data for account {account_id}")
|
922
|
-
|
988
|
+
|
923
989
|
# For now, return placeholder - would need billing profile for actual cost data
|
924
990
|
return {
|
925
991
|
"monthly_costs": {
|
926
992
|
"note": "Cost data collection requires proper billing permissions and profile",
|
927
|
-
"suggestion": "Use BILLING_PROFILE environment variable or --profile with billing access"
|
993
|
+
"suggestion": "Use BILLING_PROFILE environment variable or --profile with billing access",
|
928
994
|
},
|
929
995
|
"account_id": account_id,
|
930
996
|
"collection_timestamp": datetime.now().isoformat(),
|
931
997
|
}
|
932
|
-
|
998
|
+
|
933
999
|
except Exception as e:
|
934
1000
|
print_error(f"Failed to collect cost data: {e}")
|
935
1001
|
raise
|
@@ -938,12 +1004,12 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
938
1004
|
"""Collect AWS Organizations data using existing organizations discovery module."""
|
939
1005
|
try:
|
940
1006
|
print_info(f"Collecting Organizations data for account {account_id}")
|
941
|
-
|
1007
|
+
|
942
1008
|
# Use the session's profile name for organizations discovery
|
943
1009
|
profile_name = session.profile_name or self.active_profile
|
944
|
-
|
945
|
-
org_client = session.client(
|
946
|
-
|
1010
|
+
|
1011
|
+
org_client = session.client("organizations", region_name="us-east-1") # Organizations is always us-east-1
|
1012
|
+
|
947
1013
|
# Collect organization structure and accounts
|
948
1014
|
organizations_data = {
|
949
1015
|
"organization_info": {},
|
@@ -951,56 +1017,58 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
951
1017
|
"organizational_units": [],
|
952
1018
|
"resource_type": "organizations",
|
953
1019
|
"account_id": account_id,
|
954
|
-
"collection_timestamp": datetime.now().isoformat()
|
1020
|
+
"collection_timestamp": datetime.now().isoformat(),
|
955
1021
|
}
|
956
|
-
|
1022
|
+
|
957
1023
|
try:
|
958
1024
|
# Get organization details
|
959
1025
|
org_response = org_client.describe_organization()
|
960
1026
|
organizations_data["organization_info"] = org_response.get("Organization", {})
|
961
|
-
|
1027
|
+
|
962
1028
|
# Get all accounts in the organization
|
963
|
-
paginator = org_client.get_paginator(
|
1029
|
+
paginator = org_client.get_paginator("list_accounts")
|
964
1030
|
accounts = []
|
965
1031
|
for page in paginator.paginate():
|
966
|
-
accounts.extend(page.get(
|
967
|
-
|
1032
|
+
accounts.extend(page.get("Accounts", []))
|
1033
|
+
|
968
1034
|
organizations_data["accounts"] = accounts
|
969
1035
|
organizations_data["count"] = len(accounts)
|
970
|
-
|
1036
|
+
|
971
1037
|
# Get organizational units
|
972
1038
|
try:
|
973
1039
|
roots_response = org_client.list_roots()
|
974
|
-
for root in roots_response.get(
|
975
|
-
ou_paginator = org_client.get_paginator(
|
976
|
-
for ou_page in ou_paginator.paginate(ParentId=root[
|
977
|
-
organizations_data["organizational_units"].extend(ou_page.get(
|
1040
|
+
for root in roots_response.get("Roots", []):
|
1041
|
+
ou_paginator = org_client.get_paginator("list_organizational_units_for_parent")
|
1042
|
+
for ou_page in ou_paginator.paginate(ParentId=root["Id"]):
|
1043
|
+
organizations_data["organizational_units"].extend(ou_page.get("OrganizationalUnits", []))
|
978
1044
|
except Exception as ou_e:
|
979
1045
|
print_warning(f"Could not collect organizational units: {ou_e}")
|
980
1046
|
organizations_data["organizational_units"] = []
|
981
|
-
|
1047
|
+
|
982
1048
|
print_success(f"Successfully collected {len(accounts)} accounts from organization")
|
983
|
-
|
1049
|
+
|
984
1050
|
except Exception as org_e:
|
985
1051
|
print_warning(f"Organization data collection limited: {org_e}")
|
986
1052
|
# Try to collect at least basic account info if not in an organization
|
987
1053
|
try:
|
988
|
-
sts_client = session.client(
|
1054
|
+
sts_client = session.client("sts")
|
989
1055
|
caller_identity = sts_client.get_caller_identity()
|
990
|
-
organizations_data["accounts"] = [
|
991
|
-
|
992
|
-
|
993
|
-
|
994
|
-
|
995
|
-
|
1056
|
+
organizations_data["accounts"] = [
|
1057
|
+
{
|
1058
|
+
"Id": caller_identity.get("Account"),
|
1059
|
+
"Name": f"Account-{caller_identity.get('Account')}",
|
1060
|
+
"Status": "ACTIVE",
|
1061
|
+
"JoinedMethod": "STANDALONE",
|
1062
|
+
}
|
1063
|
+
]
|
996
1064
|
organizations_data["count"] = 1
|
997
1065
|
print_info("Collected standalone account information")
|
998
1066
|
except Exception as sts_e:
|
999
1067
|
print_error(f"Could not collect account information: {sts_e}")
|
1000
1068
|
organizations_data["count"] = 0
|
1001
|
-
|
1069
|
+
|
1002
1070
|
return organizations_data
|
1003
|
-
|
1071
|
+
|
1004
1072
|
except Exception as e:
|
1005
1073
|
print_error(f"Failed to collect organizations data: {e}")
|
1006
1074
|
raise
|
@@ -1008,7 +1076,7 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
1008
1076
|
def _generate_summary(self, resource_data: Dict[str, Any]) -> Dict[str, Any]:
|
1009
1077
|
"""
|
1010
1078
|
Generate comprehensive summary statistics from collected data.
|
1011
|
-
|
1079
|
+
|
1012
1080
|
Enhanced implementation with better error handling and metrics.
|
1013
1081
|
"""
|
1014
1082
|
summary = {
|
@@ -1022,7 +1090,7 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
1022
1090
|
"failed_collections": 0,
|
1023
1091
|
"accounts_processed": set(),
|
1024
1092
|
"resource_types_processed": set(),
|
1025
|
-
}
|
1093
|
+
},
|
1026
1094
|
}
|
1027
1095
|
|
1028
1096
|
for resource_type, accounts_data in resource_data.items():
|
@@ -1031,7 +1099,7 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
1031
1099
|
|
1032
1100
|
for account_id, account_data in accounts_data.items():
|
1033
1101
|
summary["collection_summary"]["accounts_processed"].add(account_id)
|
1034
|
-
|
1102
|
+
|
1035
1103
|
if "error" in account_data:
|
1036
1104
|
summary["errors"].append(f"{resource_type}/{account_id}: {account_data['error']}")
|
1037
1105
|
summary["collection_summary"]["failed_collections"] += 1
|
@@ -1065,8 +1133,10 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
1065
1133
|
|
1066
1134
|
# Convert sets to lists for JSON serialization
|
1067
1135
|
summary["collection_summary"]["accounts_processed"] = list(summary["collection_summary"]["accounts_processed"])
|
1068
|
-
summary["collection_summary"]["resource_types_processed"] = list(
|
1069
|
-
|
1136
|
+
summary["collection_summary"]["resource_types_processed"] = list(
|
1137
|
+
summary["collection_summary"]["resource_types_processed"]
|
1138
|
+
)
|
1139
|
+
|
1070
1140
|
# Update collection status based on errors
|
1071
1141
|
if summary["errors"]:
|
1072
1142
|
if summary["collection_summary"]["successful_collections"] == 0:
|
@@ -1075,21 +1145,18 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
1075
1145
|
summary["collection_status"] = "completed_with_errors"
|
1076
1146
|
|
1077
1147
|
return summary
|
1078
|
-
|
1148
|
+
|
1079
1149
|
def export_inventory_results(
|
1080
|
-
self,
|
1081
|
-
results: Dict[str, Any],
|
1082
|
-
export_format: str = "json",
|
1083
|
-
output_file: Optional[str] = None
|
1150
|
+
self, results: Dict[str, Any], export_format: str = "json", output_file: Optional[str] = None
|
1084
1151
|
) -> str:
|
1085
1152
|
"""
|
1086
1153
|
Export inventory results to multiple formats following proven finops patterns.
|
1087
|
-
|
1154
|
+
|
1088
1155
|
Args:
|
1089
1156
|
results: Inventory results dictionary
|
1090
1157
|
export_format: Export format (json, csv, markdown, pdf, yaml)
|
1091
1158
|
output_file: Optional output file path
|
1092
|
-
|
1159
|
+
|
1093
1160
|
Returns:
|
1094
1161
|
Export file path or formatted string content
|
1095
1162
|
"""
|
@@ -1097,15 +1164,15 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
1097
1164
|
import csv
|
1098
1165
|
from datetime import datetime
|
1099
1166
|
from pathlib import Path
|
1100
|
-
|
1167
|
+
|
1101
1168
|
# Determine output file path
|
1102
1169
|
if not output_file:
|
1103
1170
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
1104
1171
|
output_file = f"/Volumes/Working/1xOps/CloudOps-Runbooks/tmp/inventory_export_{timestamp}.{export_format}"
|
1105
|
-
|
1172
|
+
|
1106
1173
|
# Ensure tmp directory exists
|
1107
1174
|
Path(output_file).parent.mkdir(parents=True, exist_ok=True)
|
1108
|
-
|
1175
|
+
|
1109
1176
|
try:
|
1110
1177
|
if export_format.lower() == "json":
|
1111
1178
|
return self._export_json(results, output_file)
|
@@ -1119,169 +1186,189 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
1119
1186
|
return self._export_pdf(results, output_file)
|
1120
1187
|
else:
|
1121
1188
|
raise ValueError(f"Unsupported export format: {export_format}")
|
1122
|
-
|
1189
|
+
|
1123
1190
|
except Exception as e:
|
1124
1191
|
error_msg = f"Export failed for format {export_format}: {e}"
|
1125
1192
|
print_error(error_msg)
|
1126
1193
|
logger.error(error_msg)
|
1127
1194
|
raise
|
1128
|
-
|
1195
|
+
|
1129
1196
|
def _export_json(self, results: Dict[str, Any], output_file: str) -> str:
|
1130
1197
|
"""Export results to JSON format."""
|
1131
|
-
with open(output_file,
|
1198
|
+
with open(output_file, "w") as f:
|
1132
1199
|
json.dump(results, f, indent=2, default=str)
|
1133
|
-
|
1200
|
+
|
1134
1201
|
print_success(f"Inventory exported to JSON: {output_file}")
|
1135
1202
|
return output_file
|
1136
|
-
|
1203
|
+
|
1137
1204
|
def _export_csv(self, results: Dict[str, Any], output_file: str) -> str:
|
1138
1205
|
"""Export results to CSV format with real AWS data structure."""
|
1139
1206
|
import csv
|
1140
|
-
|
1141
|
-
with open(output_file,
|
1207
|
+
|
1208
|
+
with open(output_file, "w", newline="") as f:
|
1142
1209
|
writer = csv.writer(f)
|
1143
|
-
|
1210
|
+
|
1144
1211
|
# Write header
|
1145
1212
|
writer.writerow(["Account", "Region", "Resource Type", "Resource ID", "Name", "Status", "Additional Info"])
|
1146
|
-
|
1213
|
+
|
1147
1214
|
# Write data rows from real AWS resource structure
|
1148
1215
|
resource_data = results.get("resources", {})
|
1149
|
-
|
1216
|
+
|
1150
1217
|
for resource_type, accounts_data in resource_data.items():
|
1151
1218
|
for account_id, account_data in accounts_data.items():
|
1152
1219
|
if "error" in account_data:
|
1153
1220
|
# Handle error cases
|
1154
|
-
writer.writerow(
|
1155
|
-
|
1156
|
-
|
1157
|
-
|
1158
|
-
|
1159
|
-
|
1160
|
-
|
1161
|
-
|
1162
|
-
|
1221
|
+
writer.writerow(
|
1222
|
+
[
|
1223
|
+
account_id,
|
1224
|
+
account_data.get("region", "unknown"),
|
1225
|
+
resource_type,
|
1226
|
+
"",
|
1227
|
+
"",
|
1228
|
+
"ERROR",
|
1229
|
+
account_data.get("error", ""),
|
1230
|
+
]
|
1231
|
+
)
|
1163
1232
|
continue
|
1164
|
-
|
1233
|
+
|
1165
1234
|
account_region = account_data.get("region", "unknown")
|
1166
|
-
|
1235
|
+
|
1167
1236
|
# Handle different resource types with their specific data structures
|
1168
1237
|
if resource_type == "ec2" and "instances" in account_data:
|
1169
1238
|
for instance in account_data["instances"]:
|
1170
|
-
writer.writerow(
|
1171
|
-
|
1172
|
-
|
1173
|
-
|
1174
|
-
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1239
|
+
writer.writerow(
|
1240
|
+
[
|
1241
|
+
account_id,
|
1242
|
+
instance.get("region", account_region),
|
1243
|
+
"ec2-instance",
|
1244
|
+
instance.get("instance_id", ""),
|
1245
|
+
instance.get("name", "No Name Tag"),
|
1246
|
+
instance.get("state", ""),
|
1247
|
+
f"Type: {instance.get('instance_type', '')}, AZ: {instance.get('availability_zone', '')}",
|
1248
|
+
]
|
1249
|
+
)
|
1250
|
+
|
1180
1251
|
elif resource_type == "rds" and "instances" in account_data:
|
1181
1252
|
for instance in account_data["instances"]:
|
1182
|
-
writer.writerow(
|
1183
|
-
|
1184
|
-
|
1185
|
-
|
1186
|
-
|
1187
|
-
|
1188
|
-
|
1189
|
-
|
1190
|
-
|
1191
|
-
|
1253
|
+
writer.writerow(
|
1254
|
+
[
|
1255
|
+
account_id,
|
1256
|
+
instance.get("region", account_region),
|
1257
|
+
"rds-instance",
|
1258
|
+
instance.get("db_instance_identifier", ""),
|
1259
|
+
instance.get("db_instance_identifier", ""),
|
1260
|
+
instance.get("status", ""),
|
1261
|
+
f"Engine: {instance.get('engine', '')}, Class: {instance.get('instance_class', '')}",
|
1262
|
+
]
|
1263
|
+
)
|
1264
|
+
|
1192
1265
|
elif resource_type == "s3" and "buckets" in account_data:
|
1193
1266
|
for bucket in account_data["buckets"]:
|
1194
|
-
writer.writerow(
|
1195
|
-
|
1196
|
-
|
1197
|
-
|
1198
|
-
|
1199
|
-
|
1200
|
-
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1267
|
+
writer.writerow(
|
1268
|
+
[
|
1269
|
+
account_id,
|
1270
|
+
bucket.get("region", account_region),
|
1271
|
+
"s3-bucket",
|
1272
|
+
bucket.get("name", ""),
|
1273
|
+
bucket.get("name", ""),
|
1274
|
+
"",
|
1275
|
+
f"Created: {bucket.get('creation_date', '')}",
|
1276
|
+
]
|
1277
|
+
)
|
1278
|
+
|
1204
1279
|
elif resource_type == "lambda" and "functions" in account_data:
|
1205
1280
|
for function in account_data["functions"]:
|
1206
|
-
writer.writerow(
|
1207
|
-
|
1208
|
-
|
1209
|
-
|
1210
|
-
|
1211
|
-
|
1212
|
-
|
1213
|
-
|
1214
|
-
|
1215
|
-
|
1281
|
+
writer.writerow(
|
1282
|
+
[
|
1283
|
+
account_id,
|
1284
|
+
function.get("region", account_region),
|
1285
|
+
"lambda-function",
|
1286
|
+
function.get("function_name", ""),
|
1287
|
+
function.get("function_name", ""),
|
1288
|
+
"",
|
1289
|
+
f"Runtime: {function.get('runtime', '')}, Memory: {function.get('memory_size', '')}MB",
|
1290
|
+
]
|
1291
|
+
)
|
1292
|
+
|
1216
1293
|
elif resource_type == "iam" and "resources" in account_data:
|
1217
1294
|
iam_resources = account_data["resources"]
|
1218
1295
|
for user in iam_resources.get("users", []):
|
1219
|
-
writer.writerow(
|
1220
|
-
|
1221
|
-
|
1222
|
-
|
1223
|
-
|
1224
|
-
|
1225
|
-
|
1226
|
-
|
1227
|
-
|
1296
|
+
writer.writerow(
|
1297
|
+
[
|
1298
|
+
account_id,
|
1299
|
+
"global",
|
1300
|
+
"iam-user",
|
1301
|
+
user.get("user_name", ""),
|
1302
|
+
user.get("user_name", ""),
|
1303
|
+
"",
|
1304
|
+
f"ARN: {user.get('arn', '')}",
|
1305
|
+
]
|
1306
|
+
)
|
1228
1307
|
for role in iam_resources.get("roles", []):
|
1229
|
-
writer.writerow(
|
1230
|
-
|
1231
|
-
|
1232
|
-
|
1233
|
-
|
1234
|
-
|
1235
|
-
|
1236
|
-
|
1237
|
-
|
1238
|
-
|
1308
|
+
writer.writerow(
|
1309
|
+
[
|
1310
|
+
account_id,
|
1311
|
+
"global",
|
1312
|
+
"iam-role",
|
1313
|
+
role.get("role_name", ""),
|
1314
|
+
role.get("role_name", ""),
|
1315
|
+
"",
|
1316
|
+
f"ARN: {role.get('arn', '')}",
|
1317
|
+
]
|
1318
|
+
)
|
1319
|
+
|
1239
1320
|
elif resource_type == "vpc" and "vpcs" in account_data:
|
1240
1321
|
for vpc in account_data["vpcs"]:
|
1241
|
-
writer.writerow(
|
1242
|
-
|
1243
|
-
|
1244
|
-
|
1245
|
-
|
1246
|
-
|
1247
|
-
|
1248
|
-
|
1249
|
-
|
1250
|
-
|
1322
|
+
writer.writerow(
|
1323
|
+
[
|
1324
|
+
account_id,
|
1325
|
+
vpc.get("region", account_region),
|
1326
|
+
"vpc",
|
1327
|
+
vpc.get("vpc_id", ""),
|
1328
|
+
vpc.get("name", "No Name Tag"),
|
1329
|
+
vpc.get("state", ""),
|
1330
|
+
f"CIDR: {vpc.get('cidr_block', '')}, Default: {vpc.get('is_default', False)}",
|
1331
|
+
]
|
1332
|
+
)
|
1333
|
+
|
1251
1334
|
elif resource_type == "cloudformation" and "stacks" in account_data:
|
1252
1335
|
for stack in account_data["stacks"]:
|
1253
|
-
writer.writerow(
|
1254
|
-
|
1255
|
-
|
1256
|
-
|
1257
|
-
|
1258
|
-
|
1259
|
-
|
1260
|
-
|
1261
|
-
|
1262
|
-
|
1336
|
+
writer.writerow(
|
1337
|
+
[
|
1338
|
+
account_id,
|
1339
|
+
stack.get("region", account_region),
|
1340
|
+
"cloudformation-stack",
|
1341
|
+
stack.get("stack_name", ""),
|
1342
|
+
stack.get("stack_name", ""),
|
1343
|
+
stack.get("stack_status", ""),
|
1344
|
+
f"Created: {stack.get('creation_time', '')}",
|
1345
|
+
]
|
1346
|
+
)
|
1347
|
+
|
1263
1348
|
# Handle cases where no specific resources were found but collection was successful
|
1264
1349
|
elif account_data.get("count", 0) == 0:
|
1265
|
-
writer.writerow(
|
1266
|
-
|
1267
|
-
|
1268
|
-
|
1269
|
-
|
1270
|
-
|
1271
|
-
|
1272
|
-
|
1273
|
-
|
1274
|
-
|
1350
|
+
writer.writerow(
|
1351
|
+
[
|
1352
|
+
account_id,
|
1353
|
+
account_region,
|
1354
|
+
resource_type,
|
1355
|
+
"",
|
1356
|
+
"",
|
1357
|
+
"NO_RESOURCES",
|
1358
|
+
f"No {resource_type} resources found",
|
1359
|
+
]
|
1360
|
+
)
|
1361
|
+
|
1275
1362
|
print_success(f"Inventory exported to CSV: {output_file}")
|
1276
1363
|
return output_file
|
1277
|
-
|
1364
|
+
|
1278
1365
|
def _export_markdown(self, results: Dict[str, Any], output_file: str) -> str:
|
1279
1366
|
"""Export results to Markdown format with tables."""
|
1280
1367
|
content = []
|
1281
1368
|
content.append("# AWS Inventory Report")
|
1282
1369
|
content.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
1283
1370
|
content.append("")
|
1284
|
-
|
1371
|
+
|
1285
1372
|
# Summary section
|
1286
1373
|
total_resources = sum(
|
1287
1374
|
len(resources)
|
@@ -1289,30 +1376,32 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
1289
1376
|
for region_data in account_data.get("regions", {}).values()
|
1290
1377
|
for resources in region_data.get("resources", {}).values()
|
1291
1378
|
)
|
1292
|
-
|
1379
|
+
|
1293
1380
|
content.append("## Summary")
|
1294
1381
|
content.append(f"- Total Accounts: {len(results.get('accounts', {}))}")
|
1295
1382
|
content.append(f"- Total Resources: {total_resources}")
|
1296
1383
|
content.append("")
|
1297
|
-
|
1384
|
+
|
1298
1385
|
# Detailed inventory
|
1299
1386
|
content.append("## Detailed Inventory")
|
1300
1387
|
content.append("")
|
1301
1388
|
content.append("| Account | Region | Resource Type | Resource ID | Name | Status |")
|
1302
1389
|
content.append("|---------|--------|---------------|-------------|------|--------|")
|
1303
|
-
|
1390
|
+
|
1304
1391
|
for account_id, account_data in results.get("accounts", {}).items():
|
1305
1392
|
for region, region_data in account_data.get("regions", {}).items():
|
1306
1393
|
for resource_type, resources in region_data.get("resources", {}).items():
|
1307
1394
|
for resource in resources:
|
1308
|
-
content.append(
|
1309
|
-
|
1310
|
-
|
1311
|
-
|
1312
|
-
|
1395
|
+
content.append(
|
1396
|
+
f"| {account_id} | {region} | {resource_type} | {resource.get('id', '')} | {resource.get('name', '')} | {resource.get('state', '')} |"
|
1397
|
+
)
|
1398
|
+
|
1399
|
+
with open(output_file, "w") as f:
|
1400
|
+
f.write("\n".join(content))
|
1401
|
+
|
1313
1402
|
print_success(f"Inventory exported to Markdown: {output_file}")
|
1314
1403
|
return output_file
|
1315
|
-
|
1404
|
+
|
1316
1405
|
def _export_yaml(self, results: Dict[str, Any], output_file: str) -> str:
|
1317
1406
|
"""Export results to YAML format."""
|
1318
1407
|
try:
|
@@ -1320,13 +1409,13 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
1320
1409
|
except ImportError:
|
1321
1410
|
print_error("PyYAML not available. Install with: pip install pyyaml")
|
1322
1411
|
raise
|
1323
|
-
|
1324
|
-
with open(output_file,
|
1412
|
+
|
1413
|
+
with open(output_file, "w") as f:
|
1325
1414
|
yaml.dump(results, f, default_flow_style=False, sort_keys=False)
|
1326
|
-
|
1415
|
+
|
1327
1416
|
print_success(f"Inventory exported to YAML: {output_file}")
|
1328
1417
|
return output_file
|
1329
|
-
|
1418
|
+
|
1330
1419
|
def _export_pdf(self, results: Dict[str, Any], output_file: str) -> str:
|
1331
1420
|
"""Export results to executive PDF report."""
|
1332
1421
|
try:
|
@@ -1338,43 +1427,39 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
1338
1427
|
except ImportError:
|
1339
1428
|
# Graceful fallback to markdown if reportlab not available
|
1340
1429
|
print_warning("ReportLab not available, exporting to markdown instead")
|
1341
|
-
return self._export_markdown(results, output_file.replace(
|
1342
|
-
|
1430
|
+
return self._export_markdown(results, output_file.replace(".pdf", ".md"))
|
1431
|
+
|
1343
1432
|
doc = SimpleDocTemplate(output_file, pagesize=A4)
|
1344
1433
|
styles = getSampleStyleSheet()
|
1345
1434
|
story = []
|
1346
|
-
|
1435
|
+
|
1347
1436
|
# Title
|
1348
1437
|
title_style = ParagraphStyle(
|
1349
|
-
|
1350
|
-
parent=styles['Heading1'],
|
1351
|
-
fontSize=24,
|
1352
|
-
spaceAfter=30,
|
1353
|
-
textColor=colors.darkblue
|
1438
|
+
"CustomTitle", parent=styles["Heading1"], fontSize=24, spaceAfter=30, textColor=colors.darkblue
|
1354
1439
|
)
|
1355
1440
|
story.append(Paragraph("AWS Inventory Report", title_style))
|
1356
1441
|
story.append(Spacer(1, 20))
|
1357
|
-
|
1442
|
+
|
1358
1443
|
# Executive Summary
|
1359
|
-
story.append(Paragraph("Executive Summary", styles[
|
1360
|
-
|
1444
|
+
story.append(Paragraph("Executive Summary", styles["Heading2"]))
|
1445
|
+
|
1361
1446
|
total_resources = sum(
|
1362
1447
|
len(resources)
|
1363
1448
|
for account_data in results.get("accounts", {}).values()
|
1364
1449
|
for region_data in account_data.get("regions", {}).values()
|
1365
1450
|
for resources in region_data.get("resources", {}).values()
|
1366
1451
|
)
|
1367
|
-
|
1452
|
+
|
1368
1453
|
summary_text = f"""
|
1369
|
-
This report provides a comprehensive inventory of AWS resources across {len(results.get(
|
1454
|
+
This report provides a comprehensive inventory of AWS resources across {len(results.get("accounts", {}))} accounts.
|
1370
1455
|
A total of {total_resources} resources were discovered and catalogued.
|
1371
1456
|
"""
|
1372
|
-
story.append(Paragraph(summary_text, styles[
|
1457
|
+
story.append(Paragraph(summary_text, styles["Normal"]))
|
1373
1458
|
story.append(Spacer(1, 20))
|
1374
|
-
|
1459
|
+
|
1375
1460
|
# Build the PDF
|
1376
1461
|
doc.build(story)
|
1377
|
-
|
1462
|
+
|
1378
1463
|
print_success(f"Inventory exported to PDF: {output_file}")
|
1379
1464
|
return output_file
|
1380
1465
|
|
@@ -1492,7 +1577,7 @@ class InventoryCollector(EnhancedInventoryCollector):
|
|
1492
1577
|
"instances": [], # Replace with real EC2 API response processing
|
1493
1578
|
"count": 0,
|
1494
1579
|
"account_id": account_id,
|
1495
|
-
"region": self.region or "us-east-1"
|
1580
|
+
"region": self.region or "us-east-1",
|
1496
1581
|
}
|
1497
1582
|
elif resource_type == "rds":
|
1498
1583
|
# TODO: Implement real RDS API call
|
@@ -1502,7 +1587,7 @@ class InventoryCollector(EnhancedInventoryCollector):
|
|
1502
1587
|
"instances": [], # Replace with real RDS API response processing
|
1503
1588
|
"count": 0,
|
1504
1589
|
"account_id": account_id,
|
1505
|
-
"region": self.region or "us-east-1"
|
1590
|
+
"region": self.region or "us-east-1",
|
1506
1591
|
}
|
1507
1592
|
elif resource_type == "s3":
|
1508
1593
|
# TODO: Implement real S3 API call
|
@@ -1512,16 +1597,11 @@ class InventoryCollector(EnhancedInventoryCollector):
|
|
1512
1597
|
"buckets": [], # Replace with real S3 API response processing
|
1513
1598
|
"count": 0,
|
1514
1599
|
"account_id": account_id,
|
1515
|
-
"region": self.region or "us-east-1"
|
1600
|
+
"region": self.region or "us-east-1",
|
1516
1601
|
}
|
1517
1602
|
except Exception as e:
|
1518
1603
|
# Proper error handling for AWS API failures
|
1519
|
-
return {
|
1520
|
-
"error": str(e),
|
1521
|
-
"resource_type": resource_type,
|
1522
|
-
"account_id": account_id,
|
1523
|
-
"count": 0
|
1524
|
-
}
|
1604
|
+
return {"error": str(e), "resource_type": resource_type, "account_id": account_id, "count": 0}
|
1525
1605
|
else:
|
1526
1606
|
return {"resources": [], "count": 0, "resource_type": resource_type, "account_id": account_id}
|
1527
1607
|
|
@@ -1671,3 +1751,105 @@ class InventoryCollector(EnhancedInventoryCollector):
|
|
1671
1751
|
status = "enabled" if enable else "disabled"
|
1672
1752
|
print_info(f"Cross-module integration {status}")
|
1673
1753
|
logger.info(f"Cross-module integration {status} for inventory collector")
|
1754
|
+
|
1755
|
+
|
1756
|
+
# Aliases for backward compatibility
|
1757
|
+
ResourceCollector = InventoryCollector
|
1758
|
+
CollectionResult = dict # Simple dict for now
|
1759
|
+
CollectionError = Exception # Simple exception for now
|
1760
|
+
|
1761
|
+
|
1762
|
+
def run_inventory_collection(**kwargs) -> Dict[str, Any]:
|
1763
|
+
"""
|
1764
|
+
CLI wrapper function for inventory collection.
|
1765
|
+
|
1766
|
+
Provides a simple function interface to the InventoryCollector class
|
1767
|
+
for CLI command integration.
|
1768
|
+
|
1769
|
+
Args:
|
1770
|
+
**kwargs: All arguments passed to InventoryCollector and collect_inventory
|
1771
|
+
|
1772
|
+
Returns:
|
1773
|
+
Dict containing inventory results
|
1774
|
+
"""
|
1775
|
+
# Extract initialization parameters
|
1776
|
+
profile = kwargs.pop("profile", None)
|
1777
|
+
region = kwargs.pop("region", "us-east-1")
|
1778
|
+
dry_run = kwargs.pop("dry_run", False)
|
1779
|
+
all_regions = kwargs.pop("all_regions", False)
|
1780
|
+
|
1781
|
+
# Extract collection parameters
|
1782
|
+
resources = kwargs.pop("resources", ())
|
1783
|
+
all_resources = kwargs.pop("all_resources", False)
|
1784
|
+
all_profiles = kwargs.pop("all_profiles", False)
|
1785
|
+
include_costs = kwargs.pop("include_costs", False)
|
1786
|
+
include_security = kwargs.pop("include_security", False)
|
1787
|
+
include_cost_recommendations = kwargs.pop("include_cost_recommendations", False)
|
1788
|
+
parallel = kwargs.pop("parallel", True)
|
1789
|
+
validate = kwargs.pop("validate", False)
|
1790
|
+
validate_all = kwargs.pop("validate_all", False)
|
1791
|
+
|
1792
|
+
# Extract export parameters
|
1793
|
+
export_formats = kwargs.pop("export_formats", [])
|
1794
|
+
output_dir = kwargs.pop("output_dir", "./awso_evidence")
|
1795
|
+
report_name = kwargs.pop("report_name", None)
|
1796
|
+
|
1797
|
+
# Remaining kwargs (all, combine, etc.)
|
1798
|
+
use_all_profiles = kwargs.pop("all", False) or all_profiles
|
1799
|
+
combine_results = kwargs.pop("combine", False)
|
1800
|
+
|
1801
|
+
# Initialize collector
|
1802
|
+
collector = InventoryCollector(profile=profile, region=region, parallel=parallel)
|
1803
|
+
|
1804
|
+
# Enable MCP validation if requested
|
1805
|
+
if validate or validate_all:
|
1806
|
+
collector.enable_mcp_validation = True
|
1807
|
+
|
1808
|
+
# Determine resource types
|
1809
|
+
resource_types = list(resources) if resources else None
|
1810
|
+
if all_resources:
|
1811
|
+
resource_types = None # None means all resources
|
1812
|
+
|
1813
|
+
# Determine regions
|
1814
|
+
regions_to_scan = [region]
|
1815
|
+
if all_regions:
|
1816
|
+
# Get all enabled regions from AWS
|
1817
|
+
try:
|
1818
|
+
import boto3
|
1819
|
+
|
1820
|
+
session = boto3.Session(profile_name=profile) if profile else boto3.Session()
|
1821
|
+
ec2 = session.client("ec2", region_name=region)
|
1822
|
+
all_regions_response = ec2.describe_regions(AllRegions=False)
|
1823
|
+
regions_to_scan = [r["RegionName"] for r in all_regions_response["Regions"]]
|
1824
|
+
except Exception as e:
|
1825
|
+
logger.warning(f"Failed to get all regions, using {region}: {e}")
|
1826
|
+
regions_to_scan = [region]
|
1827
|
+
|
1828
|
+
# Determine account IDs
|
1829
|
+
account_ids = [collector.get_current_account_id()]
|
1830
|
+
if use_all_profiles:
|
1831
|
+
try:
|
1832
|
+
account_ids = collector.get_organization_accounts()
|
1833
|
+
except Exception as e:
|
1834
|
+
logger.warning(f"Failed to get organization accounts: {e}")
|
1835
|
+
|
1836
|
+
# Collect inventory
|
1837
|
+
try:
|
1838
|
+
results = collector.collect_inventory(
|
1839
|
+
resource_types=resource_types or collector.get_all_resource_types(),
|
1840
|
+
account_ids=account_ids,
|
1841
|
+
include_costs=include_costs,
|
1842
|
+
)
|
1843
|
+
|
1844
|
+
# Export if requested
|
1845
|
+
if export_formats and export_formats != ["table"]:
|
1846
|
+
export_results = collector.export_inventory_results(
|
1847
|
+
results=results, formats=export_formats, output_dir=output_dir, report_name=report_name
|
1848
|
+
)
|
1849
|
+
results["exports"] = export_results
|
1850
|
+
|
1851
|
+
return results
|
1852
|
+
|
1853
|
+
except Exception as e:
|
1854
|
+
logger.error(f"Inventory collection failed: {e}")
|
1855
|
+
raise
|