runbooks 0.9.9__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/cfat/cloud_foundations_assessment.py +626 -0
- runbooks/cloudops/cost_optimizer.py +95 -33
- runbooks/common/aws_pricing.py +388 -0
- runbooks/common/aws_pricing_api.py +205 -0
- runbooks/common/aws_utils.py +2 -2
- runbooks/common/comprehensive_cost_explorer_integration.py +979 -0
- runbooks/common/cross_account_manager.py +606 -0
- runbooks/common/enhanced_exception_handler.py +4 -0
- runbooks/common/env_utils.py +96 -0
- runbooks/common/mcp_integration.py +49 -2
- runbooks/common/organizations_client.py +579 -0
- runbooks/common/profile_utils.py +96 -2
- runbooks/finops/cost_optimizer.py +2 -1
- runbooks/finops/elastic_ip_optimizer.py +13 -9
- runbooks/finops/embedded_mcp_validator.py +31 -0
- runbooks/finops/enhanced_trend_visualization.py +3 -2
- runbooks/finops/markdown_exporter.py +217 -2
- runbooks/finops/nat_gateway_optimizer.py +57 -20
- runbooks/finops/vpc_cleanup_exporter.py +28 -26
- runbooks/finops/vpc_cleanup_optimizer.py +370 -16
- runbooks/inventory/__init__.py +10 -1
- runbooks/inventory/cloud_foundations_integration.py +409 -0
- runbooks/inventory/core/collector.py +1148 -88
- runbooks/inventory/discovery.md +389 -0
- runbooks/inventory/drift_detection_cli.py +327 -0
- runbooks/inventory/inventory_mcp_cli.py +171 -0
- runbooks/inventory/inventory_modules.py +4 -7
- runbooks/inventory/mcp_inventory_validator.py +2149 -0
- runbooks/inventory/mcp_vpc_validator.py +23 -6
- runbooks/inventory/organizations_discovery.py +91 -1
- runbooks/inventory/rich_inventory_display.py +129 -1
- runbooks/inventory/unified_validation_engine.py +1292 -0
- runbooks/inventory/verify_ec2_security_groups.py +3 -1
- runbooks/inventory/vpc_analyzer.py +825 -7
- runbooks/inventory/vpc_flow_analyzer.py +36 -42
- runbooks/main.py +654 -35
- runbooks/monitoring/performance_monitor.py +11 -7
- runbooks/operate/dynamodb_operations.py +6 -5
- runbooks/operate/ec2_operations.py +3 -2
- runbooks/operate/networking_cost_heatmap.py +4 -3
- runbooks/operate/s3_operations.py +13 -12
- runbooks/operate/vpc_operations.py +49 -1
- runbooks/remediation/base.py +1 -1
- runbooks/remediation/commvault_ec2_analysis.py +6 -1
- runbooks/remediation/ec2_unattached_ebs_volumes.py +6 -3
- runbooks/remediation/rds_snapshot_list.py +5 -3
- runbooks/validation/__init__.py +21 -1
- runbooks/validation/comprehensive_2way_validator.py +1996 -0
- runbooks/validation/mcp_validator.py +904 -94
- runbooks/validation/terraform_citations_validator.py +363 -0
- runbooks/validation/terraform_drift_detector.py +1098 -0
- runbooks/vpc/cleanup_wrapper.py +231 -10
- runbooks/vpc/config.py +310 -62
- runbooks/vpc/cross_account_session.py +308 -0
- runbooks/vpc/heatmap_engine.py +96 -29
- runbooks/vpc/manager_interface.py +9 -9
- runbooks/vpc/mcp_no_eni_validator.py +1551 -0
- runbooks/vpc/networking_wrapper.py +14 -8
- runbooks/vpc/runbooks.inventory.organizations_discovery.log +0 -0
- runbooks/vpc/runbooks.security.report_generator.log +0 -0
- runbooks/vpc/runbooks.security.run_script.log +0 -0
- runbooks/vpc/runbooks.security.security_export.log +0 -0
- runbooks/vpc/tests/test_cost_engine.py +1 -1
- runbooks/vpc/unified_scenarios.py +73 -3
- runbooks/vpc/vpc_cleanup_integration.py +512 -78
- {runbooks-0.9.9.dist-info → runbooks-1.0.0.dist-info}/METADATA +94 -52
- {runbooks-0.9.9.dist-info → runbooks-1.0.0.dist-info}/RECORD +71 -49
- {runbooks-0.9.9.dist-info → runbooks-1.0.0.dist-info}/WHEEL +0 -0
- {runbooks-0.9.9.dist-info → runbooks-1.0.0.dist-info}/entry_points.txt +0 -0
- {runbooks-0.9.9.dist-info → runbooks-1.0.0.dist-info}/licenses/LICENSE +0 -0
- {runbooks-0.9.9.dist-info → runbooks-1.0.0.dist-info}/top_level.txt +0 -0
@@ -1,23 +1,31 @@
|
|
1
1
|
"""
|
2
|
-
Enhanced Inventory
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
-
|
10
|
-
-
|
11
|
-
-
|
12
|
-
-
|
13
|
-
-
|
2
|
+
Enhanced Inventory Collector - AWS Resource Discovery with Enterprise Profile Management.
|
3
|
+
|
4
|
+
Strategic Alignment:
|
5
|
+
- "Do one thing and do it well" - Focused inventory collection with proven patterns
|
6
|
+
- "Move Fast, But Not So Fast We Crash" - Performance with enterprise reliability
|
7
|
+
|
8
|
+
Core Capabilities:
|
9
|
+
- Single profile architecture: --profile override pattern for all operations
|
10
|
+
- Multi-account discovery leveraging existing enterprise infrastructure
|
11
|
+
- Performance targets: <45s inventory operations across 60+ accounts
|
12
|
+
- MCP integration for real-time AWS API validation and accuracy
|
13
|
+
- Rich CLI output following enterprise UX standards
|
14
|
+
|
15
|
+
Business Value:
|
16
|
+
- Enables systematic AWS resource governance across enterprise landing zones
|
17
|
+
- Provides foundation for cost optimization and security compliance initiatives
|
18
|
+
- Supports terraform IaC validation and configuration drift detection
|
14
19
|
"""
|
15
20
|
|
16
21
|
import asyncio
|
22
|
+
import json
|
23
|
+
import os
|
17
24
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
18
25
|
from datetime import datetime, timezone
|
19
26
|
from typing import Any, Dict, List, Optional, Set
|
20
27
|
|
28
|
+
import boto3
|
21
29
|
from loguru import logger
|
22
30
|
|
23
31
|
from runbooks.base import CloudFoundationsBase, ProgressTracker
|
@@ -87,8 +95,8 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
87
95
|
self.benchmarks = []
|
88
96
|
self.current_benchmark = None
|
89
97
|
|
90
|
-
#
|
91
|
-
self.
|
98
|
+
# Simplified profile management: single profile for all operations
|
99
|
+
self.active_profile = self._initialize_profile_architecture()
|
92
100
|
|
93
101
|
# Resource collectors
|
94
102
|
self._resource_collectors = self._initialize_collectors()
|
@@ -97,9 +105,20 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
97
105
|
self.mcp_integrator = EnterpriseMCPIntegrator(profile)
|
98
106
|
self.cross_module_integrator = EnterpriseCrossModuleIntegrator(profile)
|
99
107
|
self.enable_mcp_validation = True
|
108
|
+
|
109
|
+
# Initialize inventory-specific MCP validator
|
110
|
+
self.inventory_mcp_validator = None
|
111
|
+
try:
|
112
|
+
from ..mcp_inventory_validator import create_inventory_mcp_validator
|
113
|
+
# Use profiles that would work for inventory operations
|
114
|
+
validator_profiles = [self.active_profile]
|
115
|
+
self.inventory_mcp_validator = create_inventory_mcp_validator(validator_profiles)
|
116
|
+
print_info("Inventory MCP validator initialized for real-time validation")
|
117
|
+
except Exception as e:
|
118
|
+
print_warning(f"Inventory MCP validator initialization failed: {str(e)[:50]}...")
|
100
119
|
|
101
120
|
print_info("Enhanced inventory collector with MCP integration initialized")
|
102
|
-
logger.info(f"Enhanced inventory collector initialized with {
|
121
|
+
logger.info(f"Enhanced inventory collector initialized with active profile: {self.active_profile}")
|
103
122
|
|
104
123
|
def run(self, **kwargs) -> Dict[str, Any]:
|
105
124
|
"""
|
@@ -116,17 +135,35 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
116
135
|
resource_types=resource_types, account_ids=account_ids, include_costs=include_costs
|
117
136
|
)
|
118
137
|
|
119
|
-
def _initialize_profile_architecture(self) ->
|
120
|
-
"""
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
138
|
+
def _initialize_profile_architecture(self) -> str:
|
139
|
+
"""
|
140
|
+
Initialize profile management following --profile or --all patterns.
|
141
|
+
|
142
|
+
Strategic Alignment: "Do one thing and do it well"
|
143
|
+
- Single profile override pattern: --profile takes precedence
|
144
|
+
- Simple fallback to environment variables when no --profile specified
|
145
|
+
- No hardcoded profile mappings - dynamic based on user input
|
146
|
+
|
147
|
+
Returns:
|
148
|
+
str: The active profile to use for all operations
|
149
|
+
"""
|
150
|
+
# Primary profile determination: user --profile parameter takes absolute precedence
|
151
|
+
if self.profile:
|
152
|
+
print_info(f"Using user-specified profile: {self.profile}")
|
153
|
+
logger.info("Profile override via --profile parameter - enterprise priority system")
|
154
|
+
return self.profile
|
155
|
+
|
156
|
+
# Fallback to environment variables when no --profile specified
|
157
|
+
env_profile = (
|
158
|
+
os.getenv("MANAGEMENT_PROFILE") or
|
159
|
+
os.getenv("BILLING_PROFILE") or
|
160
|
+
os.getenv("CENTRALISED_OPS_PROFILE") or
|
161
|
+
"default"
|
162
|
+
)
|
163
|
+
|
164
|
+
print_info(f"No --profile specified, using environment fallback: {env_profile}")
|
165
|
+
logger.info("Using environment variable fallback profile")
|
166
|
+
return env_profile
|
130
167
|
|
131
168
|
def _initialize_collectors(self) -> Dict[str, str]:
|
132
169
|
"""Initialize available resource collectors."""
|
@@ -140,19 +177,70 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
140
177
|
"vpc": "VPCCollector",
|
141
178
|
"cloudformation": "CloudFormationCollector",
|
142
179
|
"costs": "CostCollector",
|
180
|
+
"organizations": "ManagementResourceCollector",
|
143
181
|
}
|
144
182
|
|
145
183
|
logger.debug(f"Initialized {len(collectors)} resource collectors")
|
146
184
|
return collectors
|
185
|
+
|
186
|
+
def _extract_resource_counts(self, resource_data: Dict[str, Any]) -> Dict[str, int]:
|
187
|
+
"""
|
188
|
+
Extract resource counts from collected inventory data for MCP validation.
|
189
|
+
|
190
|
+
Args:
|
191
|
+
resource_data: Raw resource data from inventory collection
|
192
|
+
|
193
|
+
Returns:
|
194
|
+
Dictionary mapping resource types to counts
|
195
|
+
"""
|
196
|
+
resource_counts = {}
|
197
|
+
|
198
|
+
try:
|
199
|
+
# Handle various data structures from inventory collection
|
200
|
+
if isinstance(resource_data, dict):
|
201
|
+
for resource_type, resources in resource_data.items():
|
202
|
+
if isinstance(resources, list):
|
203
|
+
resource_counts[resource_type] = len(resources)
|
204
|
+
elif isinstance(resources, dict):
|
205
|
+
# Handle nested structures (e.g., by region)
|
206
|
+
total_count = 0
|
207
|
+
for region_data in resources.values():
|
208
|
+
if isinstance(region_data, list):
|
209
|
+
total_count += len(region_data)
|
210
|
+
elif isinstance(region_data, dict) and 'resources' in region_data:
|
211
|
+
total_count += len(region_data['resources'])
|
212
|
+
resource_counts[resource_type] = total_count
|
213
|
+
elif isinstance(resources, int):
|
214
|
+
resource_counts[resource_type] = resources
|
215
|
+
|
216
|
+
logger.debug(f"Extracted resource counts for validation: {resource_counts}")
|
217
|
+
return resource_counts
|
218
|
+
|
219
|
+
except Exception as e:
|
220
|
+
logger.warning(f"Failed to extract resource counts for MCP validation: {e}")
|
221
|
+
return {}
|
147
222
|
|
148
223
|
def get_all_resource_types(self) -> List[str]:
|
149
224
|
"""Get list of all available resource types."""
|
150
225
|
return list(self._resource_collectors.keys())
|
151
226
|
|
152
227
|
def get_organization_accounts(self) -> List[str]:
|
153
|
-
"""
|
228
|
+
"""
|
229
|
+
Get list of accounts in AWS Organization.
|
230
|
+
|
231
|
+
Strategic Alignment: "Do one thing and do it well"
|
232
|
+
- Single responsibility: discover organization structure
|
233
|
+
- Uses active profile for Organizations API access
|
234
|
+
- Enables multi-account inventory operations
|
235
|
+
"""
|
154
236
|
try:
|
155
|
-
|
237
|
+
# Use management profile for Organizations operations
|
238
|
+
# Use single active profile for all operations following --profile pattern
|
239
|
+
management_profile = self.active_profile
|
240
|
+
management_session = create_management_session(profile=management_profile)
|
241
|
+
organizations_client = management_session.client("organizations")
|
242
|
+
|
243
|
+
print_info("Discovering organization accounts with management profile...")
|
156
244
|
response = self._make_aws_call(organizations_client.list_accounts)
|
157
245
|
|
158
246
|
accounts = []
|
@@ -160,11 +248,13 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
160
248
|
if account["Status"] == "ACTIVE":
|
161
249
|
accounts.append(account["Id"])
|
162
250
|
|
163
|
-
|
251
|
+
print_success(f"Found {len(accounts)} active accounts in organization")
|
252
|
+
logger.info(f"Found {len(accounts)} active accounts using management profile: {management_profile}")
|
164
253
|
return accounts
|
165
254
|
|
166
255
|
except Exception as e:
|
167
|
-
|
256
|
+
print_warning(f"Could not list organization accounts: {e}")
|
257
|
+
logger.warning(f"Organization discovery failed, falling back to current account: {e}")
|
168
258
|
# Fallback to current account
|
169
259
|
return [self.get_account_id()]
|
170
260
|
|
@@ -210,13 +300,13 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
210
300
|
"collector_profile": self.profile,
|
211
301
|
"collector_region": self.region,
|
212
302
|
"enterprise_profiles_used": self.use_enterprise_profiles,
|
213
|
-
"
|
303
|
+
"active_profile": self.active_profile,
|
214
304
|
"performance_target": self.performance_target_seconds,
|
215
305
|
},
|
216
306
|
"resources": {},
|
217
307
|
"summary": {},
|
218
308
|
"errors": [],
|
219
|
-
"profile_info": self.
|
309
|
+
"profile_info": {"active_profile": self.active_profile},
|
220
310
|
}
|
221
311
|
|
222
312
|
try:
|
@@ -228,22 +318,55 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
228
318
|
results["resources"] = resource_data
|
229
319
|
results["summary"] = self._generate_summary(resource_data)
|
230
320
|
|
231
|
-
# Phase 4: MCP Validation Integration
|
232
|
-
if self.enable_mcp_validation:
|
321
|
+
# Phase 4: Enhanced Inventory MCP Validation Integration
|
322
|
+
if self.enable_mcp_validation and self.inventory_mcp_validator:
|
233
323
|
try:
|
234
|
-
print_info("Validating inventory results with MCP
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
324
|
+
print_info("Validating inventory results with specialized inventory MCP validator")
|
325
|
+
|
326
|
+
# Extract resource counts for validation
|
327
|
+
# Build validation data structure that matches what the validator expects
|
328
|
+
resource_counts = self._extract_resource_counts(resource_data)
|
329
|
+
|
330
|
+
# Add resource counts to results for the validator to find
|
331
|
+
results["resource_counts"] = resource_counts
|
332
|
+
|
333
|
+
validation_data = {
|
334
|
+
"resource_counts": resource_counts,
|
335
|
+
"regions": results["metadata"].get("regions_scanned", []),
|
336
|
+
self.active_profile: {
|
337
|
+
"resource_counts": resource_counts,
|
338
|
+
"regions": results["metadata"].get("regions_scanned", [])
|
339
|
+
}
|
340
|
+
}
|
341
|
+
|
342
|
+
# Run inventory-specific MCP validation
|
343
|
+
inventory_validation = self.inventory_mcp_validator.validate_inventory_data(validation_data)
|
344
|
+
|
345
|
+
results["inventory_mcp_validation"] = inventory_validation
|
346
|
+
|
347
|
+
overall_accuracy = inventory_validation.get("total_accuracy", 0)
|
348
|
+
if inventory_validation.get("passed_validation", False):
|
349
|
+
print_success(f"✅ Inventory MCP validation PASSED: {overall_accuracy:.1f}% accuracy achieved")
|
241
350
|
else:
|
242
|
-
print_warning("MCP validation
|
351
|
+
print_warning(f"⚠️ Inventory MCP validation: {overall_accuracy:.1f}% accuracy (≥99.5% required)")
|
352
|
+
|
353
|
+
# Also try the generic MCP integrator as backup
|
354
|
+
try:
|
355
|
+
validation_result = asyncio.run(self.mcp_integrator.validate_inventory_operations(results))
|
356
|
+
results["mcp_validation"] = validation_result.to_dict()
|
357
|
+
except Exception:
|
358
|
+
pass # Skip generic validation if it fails
|
243
359
|
|
244
360
|
except Exception as e:
|
245
|
-
print_warning(f"MCP validation failed: {str(e)[:50]}... - continuing without validation")
|
246
|
-
results["
|
361
|
+
print_warning(f"Inventory MCP validation failed: {str(e)[:50]}... - continuing without validation")
|
362
|
+
results["inventory_mcp_validation"] = {"error": str(e), "validation_skipped": True}
|
363
|
+
|
364
|
+
# Fallback to generic MCP integration
|
365
|
+
try:
|
366
|
+
validation_result = asyncio.run(self.mcp_integrator.validate_inventory_operations(results))
|
367
|
+
results["mcp_validation"] = validation_result.to_dict()
|
368
|
+
except Exception as fallback_e:
|
369
|
+
results["mcp_validation"] = {"error": str(fallback_e), "validation_skipped": True}
|
247
370
|
|
248
371
|
# Complete performance benchmark
|
249
372
|
end_time = datetime.now()
|
@@ -292,6 +415,946 @@ class EnhancedInventoryCollector(CloudFoundationsBase):
|
|
292
415
|
results["errors"].append(error_msg)
|
293
416
|
return results
|
294
417
|
|
418
|
+
def _collect_parallel(
|
419
|
+
self, resource_types: List[str], account_ids: List[str], include_costs: bool
|
420
|
+
) -> Dict[str, Any]:
|
421
|
+
"""
|
422
|
+
Collect inventory in parallel with enhanced performance monitoring.
|
423
|
+
|
424
|
+
Follows the same pattern as legacy implementation but with enterprise
|
425
|
+
performance monitoring and error handling.
|
426
|
+
"""
|
427
|
+
results = {}
|
428
|
+
total_tasks = len(resource_types) * len(account_ids)
|
429
|
+
progress = ProgressTracker(total_tasks, "Collecting inventory")
|
430
|
+
|
431
|
+
with ThreadPoolExecutor(max_workers=10) as executor:
|
432
|
+
# Submit collection tasks
|
433
|
+
future_to_params = {}
|
434
|
+
|
435
|
+
for resource_type in resource_types:
|
436
|
+
for account_id in account_ids:
|
437
|
+
future = executor.submit(
|
438
|
+
self._collect_resource_for_account, resource_type, account_id, include_costs
|
439
|
+
)
|
440
|
+
future_to_params[future] = (resource_type, account_id)
|
441
|
+
|
442
|
+
# Collect results
|
443
|
+
for future in as_completed(future_to_params):
|
444
|
+
resource_type, account_id = future_to_params[future]
|
445
|
+
try:
|
446
|
+
resource_data = future.result()
|
447
|
+
|
448
|
+
if resource_type not in results:
|
449
|
+
results[resource_type] = {}
|
450
|
+
|
451
|
+
results[resource_type][account_id] = resource_data
|
452
|
+
progress.update(status=f"Completed {resource_type} for {account_id}")
|
453
|
+
|
454
|
+
except Exception as e:
|
455
|
+
logger.error(f"Failed to collect {resource_type} for account {account_id}: {e}")
|
456
|
+
progress.update(status=f"Failed {resource_type} for {account_id}")
|
457
|
+
|
458
|
+
progress.complete()
|
459
|
+
return results
|
460
|
+
|
461
|
+
def _collect_sequential(
|
462
|
+
self, resource_types: List[str], account_ids: List[str], include_costs: bool
|
463
|
+
) -> Dict[str, Any]:
|
464
|
+
"""
|
465
|
+
Collect inventory sequentially with enhanced error handling.
|
466
|
+
|
467
|
+
Follows the same pattern as legacy implementation but with enhanced
|
468
|
+
error handling and progress tracking.
|
469
|
+
"""
|
470
|
+
results = {}
|
471
|
+
total_tasks = len(resource_types) * len(account_ids)
|
472
|
+
progress = ProgressTracker(total_tasks, "Collecting inventory")
|
473
|
+
|
474
|
+
for resource_type in resource_types:
|
475
|
+
results[resource_type] = {}
|
476
|
+
|
477
|
+
for account_id in account_ids:
|
478
|
+
try:
|
479
|
+
resource_data = self._collect_resource_for_account(resource_type, account_id, include_costs)
|
480
|
+
results[resource_type][account_id] = resource_data
|
481
|
+
progress.update(status=f"Completed {resource_type} for {account_id}")
|
482
|
+
|
483
|
+
except Exception as e:
|
484
|
+
logger.error(f"Failed to collect {resource_type} for account {account_id}: {e}")
|
485
|
+
results[resource_type][account_id] = {"error": str(e)}
|
486
|
+
progress.update(status=f"Failed {resource_type} for {account_id}")
|
487
|
+
|
488
|
+
progress.complete()
|
489
|
+
return results
|
490
|
+
|
491
|
+
def _collect_resource_for_account(self, resource_type: str, account_id: str, include_costs: bool) -> Dict[str, Any]:
|
492
|
+
"""
|
493
|
+
Collect specific resource type for an account using REAL AWS API calls.
|
494
|
+
|
495
|
+
This method makes actual AWS API calls to discover resources, following
|
496
|
+
the proven patterns from the existing inventory modules.
|
497
|
+
"""
|
498
|
+
try:
|
499
|
+
# Use active profile for AWS API calls
|
500
|
+
session = boto3.Session(profile_name=self.active_profile)
|
501
|
+
|
502
|
+
print_info(f"Collecting {resource_type} resources from account {account_id} using profile {self.active_profile}")
|
503
|
+
|
504
|
+
if resource_type == "ec2":
|
505
|
+
return self._collect_ec2_instances(session, account_id)
|
506
|
+
elif resource_type == "rds":
|
507
|
+
return self._collect_rds_instances(session, account_id)
|
508
|
+
elif resource_type == "s3":
|
509
|
+
return self._collect_s3_buckets(session, account_id)
|
510
|
+
elif resource_type == "lambda":
|
511
|
+
return self._collect_lambda_functions(session, account_id)
|
512
|
+
elif resource_type == "iam":
|
513
|
+
return self._collect_iam_resources(session, account_id)
|
514
|
+
elif resource_type == "vpc":
|
515
|
+
return self._collect_vpc_resources(session, account_id)
|
516
|
+
elif resource_type == "cloudformation":
|
517
|
+
return self._collect_cloudformation_stacks(session, account_id)
|
518
|
+
elif resource_type == "organizations":
|
519
|
+
return self._collect_organizations_data(session, account_id)
|
520
|
+
elif resource_type == "costs" and include_costs:
|
521
|
+
return self._collect_cost_data(session, account_id)
|
522
|
+
else:
|
523
|
+
print_warning(f"Resource type '{resource_type}' not supported yet")
|
524
|
+
return {
|
525
|
+
"resources": [],
|
526
|
+
"count": 0,
|
527
|
+
"resource_type": resource_type,
|
528
|
+
"account_id": account_id,
|
529
|
+
"collection_timestamp": datetime.now().isoformat(),
|
530
|
+
"warning": f"Resource type {resource_type} not implemented yet"
|
531
|
+
}
|
532
|
+
|
533
|
+
except Exception as e:
|
534
|
+
error_msg = f"Failed to collect {resource_type} for account {account_id}: {e}"
|
535
|
+
logger.error(error_msg)
|
536
|
+
print_error(error_msg)
|
537
|
+
return {
|
538
|
+
"error": str(e),
|
539
|
+
"resource_type": resource_type,
|
540
|
+
"account_id": account_id,
|
541
|
+
"collection_timestamp": datetime.now().isoformat(),
|
542
|
+
}
|
543
|
+
|
544
|
+
def _collect_ec2_instances(self, session: boto3.Session, account_id: str) -> Dict[str, Any]:
|
545
|
+
"""Collect EC2 instances using real AWS API calls."""
|
546
|
+
try:
|
547
|
+
region = self.region or session.region_name or "us-east-1"
|
548
|
+
ec2_client = session.client("ec2", region_name=region)
|
549
|
+
|
550
|
+
print_info(f"Calling EC2 describe_instances API for account {account_id} in region {region}")
|
551
|
+
|
552
|
+
# Make real AWS API call with pagination support
|
553
|
+
instances = []
|
554
|
+
paginator = ec2_client.get_paginator('describe_instances')
|
555
|
+
|
556
|
+
for page in paginator.paginate():
|
557
|
+
for reservation in page.get("Reservations", []):
|
558
|
+
for instance in reservation.get("Instances", []):
|
559
|
+
# Extract instance data
|
560
|
+
instance_data = {
|
561
|
+
"instance_id": instance["InstanceId"],
|
562
|
+
"instance_type": instance["InstanceType"],
|
563
|
+
"state": instance["State"]["Name"],
|
564
|
+
"region": region,
|
565
|
+
"account_id": account_id,
|
566
|
+
"launch_time": instance.get("LaunchTime", "").isoformat() if instance.get("LaunchTime") else "",
|
567
|
+
"availability_zone": instance.get("Placement", {}).get("AvailabilityZone", ""),
|
568
|
+
"vpc_id": instance.get("VpcId", ""),
|
569
|
+
"subnet_id": instance.get("SubnetId", ""),
|
570
|
+
"private_ip_address": instance.get("PrivateIpAddress", ""),
|
571
|
+
"public_ip_address": instance.get("PublicIpAddress", ""),
|
572
|
+
"public_dns_name": instance.get("PublicDnsName", ""),
|
573
|
+
}
|
574
|
+
|
575
|
+
# Extract tags
|
576
|
+
tags = {}
|
577
|
+
name = "No Name Tag"
|
578
|
+
for tag in instance.get("Tags", []):
|
579
|
+
tags[tag["Key"]] = tag["Value"]
|
580
|
+
if tag["Key"] == "Name":
|
581
|
+
name = tag["Value"]
|
582
|
+
|
583
|
+
instance_data["tags"] = tags
|
584
|
+
instance_data["name"] = name
|
585
|
+
|
586
|
+
# Extract security groups
|
587
|
+
instance_data["security_groups"] = [
|
588
|
+
{"group_id": sg["GroupId"], "group_name": sg["GroupName"]}
|
589
|
+
for sg in instance.get("SecurityGroups", [])
|
590
|
+
]
|
591
|
+
|
592
|
+
instances.append(instance_data)
|
593
|
+
|
594
|
+
print_success(f"Found {len(instances)} EC2 instances in account {account_id}")
|
595
|
+
|
596
|
+
return {
|
597
|
+
"instances": instances,
|
598
|
+
"count": len(instances),
|
599
|
+
"collection_timestamp": datetime.now().isoformat(),
|
600
|
+
"region": region,
|
601
|
+
"account_id": account_id,
|
602
|
+
}
|
603
|
+
|
604
|
+
except Exception as e:
|
605
|
+
print_error(f"Failed to collect EC2 instances: {e}")
|
606
|
+
raise
|
607
|
+
|
608
|
+
def _collect_rds_instances(self, session: boto3.Session, account_id: str) -> Dict[str, Any]:
|
609
|
+
"""Collect RDS instances using real AWS API calls."""
|
610
|
+
try:
|
611
|
+
region = self.region or session.region_name or "us-east-1"
|
612
|
+
rds_client = session.client("rds", region_name=region)
|
613
|
+
|
614
|
+
print_info(f"Calling RDS describe_db_instances API for account {account_id} in region {region}")
|
615
|
+
|
616
|
+
# Make real AWS API call with pagination support
|
617
|
+
instances = []
|
618
|
+
paginator = rds_client.get_paginator('describe_db_instances')
|
619
|
+
|
620
|
+
for page in paginator.paginate():
|
621
|
+
for db_instance in page.get("DBInstances", []):
|
622
|
+
instance_data = {
|
623
|
+
"db_instance_identifier": db_instance["DBInstanceIdentifier"],
|
624
|
+
"engine": db_instance["Engine"],
|
625
|
+
"engine_version": db_instance["EngineVersion"],
|
626
|
+
"instance_class": db_instance["DBInstanceClass"],
|
627
|
+
"status": db_instance["DBInstanceStatus"],
|
628
|
+
"account_id": account_id,
|
629
|
+
"region": region,
|
630
|
+
"multi_az": db_instance.get("MultiAZ", False),
|
631
|
+
"storage_type": db_instance.get("StorageType", ""),
|
632
|
+
"allocated_storage": db_instance.get("AllocatedStorage", 0),
|
633
|
+
"endpoint": db_instance.get("Endpoint", {}).get("Address", "") if db_instance.get("Endpoint") else "",
|
634
|
+
"port": db_instance.get("Endpoint", {}).get("Port", 0) if db_instance.get("Endpoint") else 0,
|
635
|
+
"vpc_id": db_instance.get("DBSubnetGroup", {}).get("VpcId", "") if db_instance.get("DBSubnetGroup") else "",
|
636
|
+
}
|
637
|
+
|
638
|
+
instances.append(instance_data)
|
639
|
+
|
640
|
+
print_success(f"Found {len(instances)} RDS instances in account {account_id}")
|
641
|
+
|
642
|
+
return {
|
643
|
+
"instances": instances,
|
644
|
+
"count": len(instances),
|
645
|
+
"collection_timestamp": datetime.now().isoformat(),
|
646
|
+
"region": region,
|
647
|
+
"account_id": account_id,
|
648
|
+
}
|
649
|
+
|
650
|
+
except Exception as e:
|
651
|
+
print_error(f"Failed to collect RDS instances: {e}")
|
652
|
+
raise
|
653
|
+
|
654
|
+
def _collect_s3_buckets(self, session: boto3.Session, account_id: str) -> Dict[str, Any]:
|
655
|
+
"""Collect S3 buckets using real AWS API calls."""
|
656
|
+
try:
|
657
|
+
s3_client = session.client("s3")
|
658
|
+
|
659
|
+
print_info(f"Calling S3 list_buckets API for account {account_id}")
|
660
|
+
|
661
|
+
# Make real AWS API call - S3 buckets are global
|
662
|
+
response = s3_client.list_buckets()
|
663
|
+
buckets = []
|
664
|
+
|
665
|
+
for bucket in response.get("Buckets", []):
|
666
|
+
bucket_data = {
|
667
|
+
"name": bucket["Name"],
|
668
|
+
"creation_date": bucket["CreationDate"].isoformat(),
|
669
|
+
"account_id": account_id,
|
670
|
+
}
|
671
|
+
|
672
|
+
# Try to get bucket location (region)
|
673
|
+
try:
|
674
|
+
location_response = s3_client.get_bucket_location(Bucket=bucket["Name"])
|
675
|
+
bucket_region = location_response.get("LocationConstraint")
|
676
|
+
if bucket_region is None:
|
677
|
+
bucket_region = "us-east-1" # Default for US Standard
|
678
|
+
bucket_data["region"] = bucket_region
|
679
|
+
except Exception as e:
|
680
|
+
logger.warning(f"Could not get location for bucket {bucket['Name']}: {e}")
|
681
|
+
bucket_data["region"] = "unknown"
|
682
|
+
|
683
|
+
# Try to get bucket versioning
|
684
|
+
try:
|
685
|
+
versioning_response = s3_client.get_bucket_versioning(Bucket=bucket["Name"])
|
686
|
+
bucket_data["versioning"] = versioning_response.get("Status", "Suspended")
|
687
|
+
except Exception as e:
|
688
|
+
logger.warning(f"Could not get versioning for bucket {bucket['Name']}: {e}")
|
689
|
+
bucket_data["versioning"] = "unknown"
|
690
|
+
|
691
|
+
buckets.append(bucket_data)
|
692
|
+
|
693
|
+
print_success(f"Found {len(buckets)} S3 buckets in account {account_id}")
|
694
|
+
|
695
|
+
return {
|
696
|
+
"buckets": buckets,
|
697
|
+
"count": len(buckets),
|
698
|
+
"collection_timestamp": datetime.now().isoformat(),
|
699
|
+
"account_id": account_id,
|
700
|
+
}
|
701
|
+
|
702
|
+
except Exception as e:
|
703
|
+
print_error(f"Failed to collect S3 buckets: {e}")
|
704
|
+
raise
|
705
|
+
|
706
|
+
def _collect_lambda_functions(self, session: boto3.Session, account_id: str) -> Dict[str, Any]:
|
707
|
+
"""Collect Lambda functions using real AWS API calls."""
|
708
|
+
try:
|
709
|
+
region = self.region or session.region_name or "us-east-1"
|
710
|
+
lambda_client = session.client("lambda", region_name=region)
|
711
|
+
|
712
|
+
print_info(f"Calling Lambda list_functions API for account {account_id} in region {region}")
|
713
|
+
|
714
|
+
# Make real AWS API call with pagination support
|
715
|
+
functions = []
|
716
|
+
paginator = lambda_client.get_paginator('list_functions')
|
717
|
+
|
718
|
+
for page in paginator.paginate():
|
719
|
+
for function in page.get("Functions", []):
|
720
|
+
function_data = {
|
721
|
+
"function_name": function["FunctionName"],
|
722
|
+
"runtime": function.get("Runtime", ""),
|
723
|
+
"handler": function.get("Handler", ""),
|
724
|
+
"code_size": function.get("CodeSize", 0),
|
725
|
+
"description": function.get("Description", ""),
|
726
|
+
"timeout": function.get("Timeout", 0),
|
727
|
+
"memory_size": function.get("MemorySize", 0),
|
728
|
+
"last_modified": function.get("LastModified", ""),
|
729
|
+
"role": function.get("Role", ""),
|
730
|
+
"account_id": account_id,
|
731
|
+
"region": region,
|
732
|
+
}
|
733
|
+
|
734
|
+
functions.append(function_data)
|
735
|
+
|
736
|
+
print_success(f"Found {len(functions)} Lambda functions in account {account_id}")
|
737
|
+
|
738
|
+
return {
|
739
|
+
"functions": functions,
|
740
|
+
"count": len(functions),
|
741
|
+
"collection_timestamp": datetime.now().isoformat(),
|
742
|
+
"region": region,
|
743
|
+
"account_id": account_id,
|
744
|
+
}
|
745
|
+
|
746
|
+
except Exception as e:
|
747
|
+
print_error(f"Failed to collect Lambda functions: {e}")
|
748
|
+
raise
|
749
|
+
|
750
|
+
def _collect_iam_resources(self, session: boto3.Session, account_id: str) -> Dict[str, Any]:
|
751
|
+
"""Collect IAM resources using real AWS API calls."""
|
752
|
+
try:
|
753
|
+
iam_client = session.client("iam")
|
754
|
+
|
755
|
+
print_info(f"Calling IAM APIs for account {account_id}")
|
756
|
+
|
757
|
+
resources = {"users": [], "roles": [], "policies": [], "groups": []}
|
758
|
+
|
759
|
+
# Collect users
|
760
|
+
paginator = iam_client.get_paginator('list_users')
|
761
|
+
for page in paginator.paginate():
|
762
|
+
for user in page.get("Users", []):
|
763
|
+
user_data = {
|
764
|
+
"user_name": user["UserName"],
|
765
|
+
"user_id": user["UserId"],
|
766
|
+
"arn": user["Arn"],
|
767
|
+
"create_date": user["CreateDate"].isoformat(),
|
768
|
+
"path": user["Path"],
|
769
|
+
"account_id": account_id,
|
770
|
+
}
|
771
|
+
resources["users"].append(user_data)
|
772
|
+
|
773
|
+
# Collect roles
|
774
|
+
paginator = iam_client.get_paginator('list_roles')
|
775
|
+
for page in paginator.paginate():
|
776
|
+
for role in page.get("Roles", []):
|
777
|
+
role_data = {
|
778
|
+
"role_name": role["RoleName"],
|
779
|
+
"role_id": role["RoleId"],
|
780
|
+
"arn": role["Arn"],
|
781
|
+
"create_date": role["CreateDate"].isoformat(),
|
782
|
+
"path": role["Path"],
|
783
|
+
"account_id": account_id,
|
784
|
+
}
|
785
|
+
resources["roles"].append(role_data)
|
786
|
+
|
787
|
+
total_count = len(resources["users"]) + len(resources["roles"])
|
788
|
+
print_success(f"Found {total_count} IAM resources in account {account_id}")
|
789
|
+
|
790
|
+
return {
|
791
|
+
"resources": resources,
|
792
|
+
"count": total_count,
|
793
|
+
"collection_timestamp": datetime.now().isoformat(),
|
794
|
+
"account_id": account_id,
|
795
|
+
}
|
796
|
+
|
797
|
+
except Exception as e:
|
798
|
+
print_error(f"Failed to collect IAM resources: {e}")
|
799
|
+
raise
|
800
|
+
|
801
|
+
def _collect_vpc_resources(self, session: boto3.Session, account_id: str) -> Dict[str, Any]:
|
802
|
+
"""Collect VPC resources using real AWS API calls."""
|
803
|
+
try:
|
804
|
+
region = self.region or session.region_name or "us-east-1"
|
805
|
+
ec2_client = session.client("ec2", region_name=region)
|
806
|
+
|
807
|
+
print_info(f"Calling EC2 VPC APIs for account {account_id} in region {region}")
|
808
|
+
|
809
|
+
vpcs = []
|
810
|
+
paginator = ec2_client.get_paginator('describe_vpcs')
|
811
|
+
|
812
|
+
for page in paginator.paginate():
|
813
|
+
for vpc in page.get("Vpcs", []):
|
814
|
+
vpc_data = {
|
815
|
+
"vpc_id": vpc["VpcId"],
|
816
|
+
"cidr_block": vpc["CidrBlock"],
|
817
|
+
"state": vpc["State"],
|
818
|
+
"is_default": vpc.get("IsDefault", False),
|
819
|
+
"instance_tenancy": vpc.get("InstanceTenancy", ""),
|
820
|
+
"account_id": account_id,
|
821
|
+
"region": region,
|
822
|
+
}
|
823
|
+
|
824
|
+
# Extract tags
|
825
|
+
tags = {}
|
826
|
+
name = "No Name Tag"
|
827
|
+
for tag in vpc.get("Tags", []):
|
828
|
+
tags[tag["Key"]] = tag["Value"]
|
829
|
+
if tag["Key"] == "Name":
|
830
|
+
name = tag["Value"]
|
831
|
+
|
832
|
+
vpc_data["tags"] = tags
|
833
|
+
vpc_data["name"] = name
|
834
|
+
|
835
|
+
vpcs.append(vpc_data)
|
836
|
+
|
837
|
+
print_success(f"Found {len(vpcs)} VPCs in account {account_id}")
|
838
|
+
|
839
|
+
return {
|
840
|
+
"vpcs": vpcs,
|
841
|
+
"count": len(vpcs),
|
842
|
+
"collection_timestamp": datetime.now().isoformat(),
|
843
|
+
"region": region,
|
844
|
+
"account_id": account_id,
|
845
|
+
}
|
846
|
+
|
847
|
+
except Exception as e:
|
848
|
+
print_error(f"Failed to collect VPC resources: {e}")
|
849
|
+
raise
|
850
|
+
|
851
|
+
def _collect_cloudformation_stacks(self, session: boto3.Session, account_id: str) -> Dict[str, Any]:
|
852
|
+
"""Collect CloudFormation stacks using real AWS API calls."""
|
853
|
+
try:
|
854
|
+
region = self.region or session.region_name or "us-east-1"
|
855
|
+
cf_client = session.client("cloudformation", region_name=region)
|
856
|
+
|
857
|
+
print_info(f"Calling CloudFormation describe_stacks API for account {account_id} in region {region}")
|
858
|
+
|
859
|
+
stacks = []
|
860
|
+
paginator = cf_client.get_paginator('describe_stacks')
|
861
|
+
|
862
|
+
for page in paginator.paginate():
|
863
|
+
for stack in page.get("Stacks", []):
|
864
|
+
stack_data = {
|
865
|
+
"stack_name": stack["StackName"],
|
866
|
+
"stack_id": stack["StackId"],
|
867
|
+
"stack_status": stack["StackStatus"],
|
868
|
+
"creation_time": stack["CreationTime"].isoformat(),
|
869
|
+
"description": stack.get("Description", ""),
|
870
|
+
"account_id": account_id,
|
871
|
+
"region": region,
|
872
|
+
}
|
873
|
+
|
874
|
+
if "LastUpdatedTime" in stack:
|
875
|
+
stack_data["last_updated_time"] = stack["LastUpdatedTime"].isoformat()
|
876
|
+
|
877
|
+
stacks.append(stack_data)
|
878
|
+
|
879
|
+
print_success(f"Found {len(stacks)} CloudFormation stacks in account {account_id}")
|
880
|
+
|
881
|
+
return {
|
882
|
+
"stacks": stacks,
|
883
|
+
"count": len(stacks),
|
884
|
+
"collection_timestamp": datetime.now().isoformat(),
|
885
|
+
"region": region,
|
886
|
+
"account_id": account_id,
|
887
|
+
}
|
888
|
+
|
889
|
+
except Exception as e:
|
890
|
+
print_error(f"Failed to collect CloudFormation stacks: {e}")
|
891
|
+
raise
|
892
|
+
|
893
|
+
def _collect_cost_data(self, session: boto3.Session, account_id: str) -> Dict[str, Any]:
|
894
|
+
"""Collect cost data using real AWS Cost Explorer API calls."""
|
895
|
+
try:
|
896
|
+
# Note: Cost Explorer requires specific billing permissions
|
897
|
+
print_warning("Cost data collection requires AWS Cost Explorer permissions")
|
898
|
+
print_info(f"Attempting to collect cost data for account {account_id}")
|
899
|
+
|
900
|
+
# For now, return placeholder - would need billing profile for actual cost data
|
901
|
+
return {
|
902
|
+
"monthly_costs": {
|
903
|
+
"note": "Cost data collection requires proper billing permissions and profile",
|
904
|
+
"suggestion": "Use BILLING_PROFILE environment variable or --profile with billing access"
|
905
|
+
},
|
906
|
+
"account_id": account_id,
|
907
|
+
"collection_timestamp": datetime.now().isoformat(),
|
908
|
+
}
|
909
|
+
|
910
|
+
except Exception as e:
|
911
|
+
print_error(f"Failed to collect cost data: {e}")
|
912
|
+
raise
|
913
|
+
|
914
|
+
def _collect_organizations_data(self, session: boto3.Session, account_id: str) -> Dict[str, Any]:
|
915
|
+
"""Collect AWS Organizations data using existing organizations discovery module."""
|
916
|
+
try:
|
917
|
+
print_info(f"Collecting Organizations data for account {account_id}")
|
918
|
+
|
919
|
+
# Use the session's profile name for organizations discovery
|
920
|
+
profile_name = session.profile_name or self.active_profile
|
921
|
+
|
922
|
+
org_client = session.client('organizations', region_name='us-east-1') # Organizations is always us-east-1
|
923
|
+
|
924
|
+
# Collect organization structure and accounts
|
925
|
+
organizations_data = {
|
926
|
+
"organization_info": {},
|
927
|
+
"accounts": [],
|
928
|
+
"organizational_units": [],
|
929
|
+
"resource_type": "organizations",
|
930
|
+
"account_id": account_id,
|
931
|
+
"collection_timestamp": datetime.now().isoformat()
|
932
|
+
}
|
933
|
+
|
934
|
+
try:
|
935
|
+
# Get organization details
|
936
|
+
org_response = org_client.describe_organization()
|
937
|
+
organizations_data["organization_info"] = org_response.get("Organization", {})
|
938
|
+
|
939
|
+
# Get all accounts in the organization
|
940
|
+
paginator = org_client.get_paginator('list_accounts')
|
941
|
+
accounts = []
|
942
|
+
for page in paginator.paginate():
|
943
|
+
accounts.extend(page.get('Accounts', []))
|
944
|
+
|
945
|
+
organizations_data["accounts"] = accounts
|
946
|
+
organizations_data["count"] = len(accounts)
|
947
|
+
|
948
|
+
# Get organizational units
|
949
|
+
try:
|
950
|
+
roots_response = org_client.list_roots()
|
951
|
+
for root in roots_response.get('Roots', []):
|
952
|
+
ou_paginator = org_client.get_paginator('list_organizational_units_for_parent')
|
953
|
+
for ou_page in ou_paginator.paginate(ParentId=root['Id']):
|
954
|
+
organizations_data["organizational_units"].extend(ou_page.get('OrganizationalUnits', []))
|
955
|
+
except Exception as ou_e:
|
956
|
+
print_warning(f"Could not collect organizational units: {ou_e}")
|
957
|
+
organizations_data["organizational_units"] = []
|
958
|
+
|
959
|
+
print_success(f"Successfully collected {len(accounts)} accounts from organization")
|
960
|
+
|
961
|
+
except Exception as org_e:
|
962
|
+
print_warning(f"Organization data collection limited: {org_e}")
|
963
|
+
# Try to collect at least basic account info if not in an organization
|
964
|
+
try:
|
965
|
+
sts_client = session.client('sts')
|
966
|
+
caller_identity = sts_client.get_caller_identity()
|
967
|
+
organizations_data["accounts"] = [{
|
968
|
+
"Id": caller_identity.get("Account"),
|
969
|
+
"Name": f"Account-{caller_identity.get('Account')}",
|
970
|
+
"Status": "ACTIVE",
|
971
|
+
"JoinedMethod": "STANDALONE"
|
972
|
+
}]
|
973
|
+
organizations_data["count"] = 1
|
974
|
+
print_info("Collected standalone account information")
|
975
|
+
except Exception as sts_e:
|
976
|
+
print_error(f"Could not collect account information: {sts_e}")
|
977
|
+
organizations_data["count"] = 0
|
978
|
+
|
979
|
+
return organizations_data
|
980
|
+
|
981
|
+
except Exception as e:
|
982
|
+
print_error(f"Failed to collect organizations data: {e}")
|
983
|
+
raise
|
984
|
+
|
985
|
+
def _generate_summary(self, resource_data: Dict[str, Any]) -> Dict[str, Any]:
|
986
|
+
"""
|
987
|
+
Generate comprehensive summary statistics from collected data.
|
988
|
+
|
989
|
+
Enhanced implementation with better error handling and metrics.
|
990
|
+
"""
|
991
|
+
summary = {
|
992
|
+
"total_resources": 0,
|
993
|
+
"resources_by_type": {},
|
994
|
+
"resources_by_account": {},
|
995
|
+
"collection_status": "completed",
|
996
|
+
"errors": [],
|
997
|
+
"collection_summary": {
|
998
|
+
"successful_collections": 0,
|
999
|
+
"failed_collections": 0,
|
1000
|
+
"accounts_processed": set(),
|
1001
|
+
"resource_types_processed": set(),
|
1002
|
+
}
|
1003
|
+
}
|
1004
|
+
|
1005
|
+
for resource_type, accounts_data in resource_data.items():
|
1006
|
+
type_count = 0
|
1007
|
+
summary["collection_summary"]["resource_types_processed"].add(resource_type)
|
1008
|
+
|
1009
|
+
for account_id, account_data in accounts_data.items():
|
1010
|
+
summary["collection_summary"]["accounts_processed"].add(account_id)
|
1011
|
+
|
1012
|
+
if "error" in account_data:
|
1013
|
+
summary["errors"].append(f"{resource_type}/{account_id}: {account_data['error']}")
|
1014
|
+
summary["collection_summary"]["failed_collections"] += 1
|
1015
|
+
continue
|
1016
|
+
|
1017
|
+
summary["collection_summary"]["successful_collections"] += 1
|
1018
|
+
|
1019
|
+
# Count resources based on type
|
1020
|
+
account_count = account_data.get("count", 0)
|
1021
|
+
if account_count == 0:
|
1022
|
+
# Try to calculate from actual resource lists
|
1023
|
+
if resource_type == "ec2":
|
1024
|
+
account_count = len(account_data.get("instances", []))
|
1025
|
+
elif resource_type == "rds":
|
1026
|
+
account_count = len(account_data.get("instances", []))
|
1027
|
+
elif resource_type == "s3":
|
1028
|
+
account_count = len(account_data.get("buckets", []))
|
1029
|
+
elif resource_type == "lambda":
|
1030
|
+
account_count = len(account_data.get("functions", []))
|
1031
|
+
else:
|
1032
|
+
account_count = len(account_data.get("resources", []))
|
1033
|
+
|
1034
|
+
type_count += account_count
|
1035
|
+
|
1036
|
+
if account_id not in summary["resources_by_account"]:
|
1037
|
+
summary["resources_by_account"][account_id] = 0
|
1038
|
+
summary["resources_by_account"][account_id] += account_count
|
1039
|
+
|
1040
|
+
summary["resources_by_type"][resource_type] = type_count
|
1041
|
+
summary["total_resources"] += type_count
|
1042
|
+
|
1043
|
+
# Convert sets to lists for JSON serialization
|
1044
|
+
summary["collection_summary"]["accounts_processed"] = list(summary["collection_summary"]["accounts_processed"])
|
1045
|
+
summary["collection_summary"]["resource_types_processed"] = list(summary["collection_summary"]["resource_types_processed"])
|
1046
|
+
|
1047
|
+
# Update collection status based on errors
|
1048
|
+
if summary["errors"]:
|
1049
|
+
if summary["collection_summary"]["successful_collections"] == 0:
|
1050
|
+
summary["collection_status"] = "failed"
|
1051
|
+
else:
|
1052
|
+
summary["collection_status"] = "completed_with_errors"
|
1053
|
+
|
1054
|
+
return summary
|
1055
|
+
|
1056
|
+
def export_inventory_results(
|
1057
|
+
self,
|
1058
|
+
results: Dict[str, Any],
|
1059
|
+
export_format: str = "json",
|
1060
|
+
output_file: Optional[str] = None
|
1061
|
+
) -> str:
|
1062
|
+
"""
|
1063
|
+
Export inventory results to multiple formats following proven finops patterns.
|
1064
|
+
|
1065
|
+
Args:
|
1066
|
+
results: Inventory results dictionary
|
1067
|
+
export_format: Export format (json, csv, markdown, pdf, yaml)
|
1068
|
+
output_file: Optional output file path
|
1069
|
+
|
1070
|
+
Returns:
|
1071
|
+
Export file path or formatted string content
|
1072
|
+
"""
|
1073
|
+
import json
|
1074
|
+
import csv
|
1075
|
+
from datetime import datetime
|
1076
|
+
from pathlib import Path
|
1077
|
+
|
1078
|
+
# Determine output file path
|
1079
|
+
if not output_file:
|
1080
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
1081
|
+
output_file = f"/Volumes/Working/1xOps/CloudOps-Runbooks/tmp/inventory_export_{timestamp}.{export_format}"
|
1082
|
+
|
1083
|
+
# Ensure tmp directory exists
|
1084
|
+
Path(output_file).parent.mkdir(parents=True, exist_ok=True)
|
1085
|
+
|
1086
|
+
try:
|
1087
|
+
if export_format.lower() == "json":
|
1088
|
+
return self._export_json(results, output_file)
|
1089
|
+
elif export_format.lower() == "csv":
|
1090
|
+
return self._export_csv(results, output_file)
|
1091
|
+
elif export_format.lower() == "markdown":
|
1092
|
+
return self._export_markdown(results, output_file)
|
1093
|
+
elif export_format.lower() == "yaml":
|
1094
|
+
return self._export_yaml(results, output_file)
|
1095
|
+
elif export_format.lower() == "pdf":
|
1096
|
+
return self._export_pdf(results, output_file)
|
1097
|
+
else:
|
1098
|
+
raise ValueError(f"Unsupported export format: {export_format}")
|
1099
|
+
|
1100
|
+
except Exception as e:
|
1101
|
+
error_msg = f"Export failed for format {export_format}: {e}"
|
1102
|
+
print_error(error_msg)
|
1103
|
+
logger.error(error_msg)
|
1104
|
+
raise
|
1105
|
+
|
1106
|
+
def _export_json(self, results: Dict[str, Any], output_file: str) -> str:
|
1107
|
+
"""Export results to JSON format."""
|
1108
|
+
with open(output_file, 'w') as f:
|
1109
|
+
json.dump(results, f, indent=2, default=str)
|
1110
|
+
|
1111
|
+
print_success(f"Inventory exported to JSON: {output_file}")
|
1112
|
+
return output_file
|
1113
|
+
|
1114
|
+
def _export_csv(self, results: Dict[str, Any], output_file: str) -> str:
|
1115
|
+
"""Export results to CSV format with real AWS data structure."""
|
1116
|
+
import csv
|
1117
|
+
|
1118
|
+
with open(output_file, 'w', newline='') as f:
|
1119
|
+
writer = csv.writer(f)
|
1120
|
+
|
1121
|
+
# Write header
|
1122
|
+
writer.writerow(["Account", "Region", "Resource Type", "Resource ID", "Name", "Status", "Additional Info"])
|
1123
|
+
|
1124
|
+
# Write data rows from real AWS resource structure
|
1125
|
+
resource_data = results.get("resources", {})
|
1126
|
+
|
1127
|
+
for resource_type, accounts_data in resource_data.items():
|
1128
|
+
for account_id, account_data in accounts_data.items():
|
1129
|
+
if "error" in account_data:
|
1130
|
+
# Handle error cases
|
1131
|
+
writer.writerow([
|
1132
|
+
account_id,
|
1133
|
+
account_data.get("region", "unknown"),
|
1134
|
+
resource_type,
|
1135
|
+
"",
|
1136
|
+
"",
|
1137
|
+
"ERROR",
|
1138
|
+
account_data.get("error", "")
|
1139
|
+
])
|
1140
|
+
continue
|
1141
|
+
|
1142
|
+
account_region = account_data.get("region", "unknown")
|
1143
|
+
|
1144
|
+
# Handle different resource types with their specific data structures
|
1145
|
+
if resource_type == "ec2" and "instances" in account_data:
|
1146
|
+
for instance in account_data["instances"]:
|
1147
|
+
writer.writerow([
|
1148
|
+
account_id,
|
1149
|
+
instance.get("region", account_region),
|
1150
|
+
"ec2-instance",
|
1151
|
+
instance.get("instance_id", ""),
|
1152
|
+
instance.get("name", "No Name Tag"),
|
1153
|
+
instance.get("state", ""),
|
1154
|
+
f"Type: {instance.get('instance_type', '')}, AZ: {instance.get('availability_zone', '')}"
|
1155
|
+
])
|
1156
|
+
|
1157
|
+
elif resource_type == "rds" and "instances" in account_data:
|
1158
|
+
for instance in account_data["instances"]:
|
1159
|
+
writer.writerow([
|
1160
|
+
account_id,
|
1161
|
+
instance.get("region", account_region),
|
1162
|
+
"rds-instance",
|
1163
|
+
instance.get("db_instance_identifier", ""),
|
1164
|
+
instance.get("db_instance_identifier", ""),
|
1165
|
+
instance.get("status", ""),
|
1166
|
+
f"Engine: {instance.get('engine', '')}, Class: {instance.get('instance_class', '')}"
|
1167
|
+
])
|
1168
|
+
|
1169
|
+
elif resource_type == "s3" and "buckets" in account_data:
|
1170
|
+
for bucket in account_data["buckets"]:
|
1171
|
+
writer.writerow([
|
1172
|
+
account_id,
|
1173
|
+
bucket.get("region", account_region),
|
1174
|
+
"s3-bucket",
|
1175
|
+
bucket.get("name", ""),
|
1176
|
+
bucket.get("name", ""),
|
1177
|
+
"",
|
1178
|
+
f"Created: {bucket.get('creation_date', '')}"
|
1179
|
+
])
|
1180
|
+
|
1181
|
+
elif resource_type == "lambda" and "functions" in account_data:
|
1182
|
+
for function in account_data["functions"]:
|
1183
|
+
writer.writerow([
|
1184
|
+
account_id,
|
1185
|
+
function.get("region", account_region),
|
1186
|
+
"lambda-function",
|
1187
|
+
function.get("function_name", ""),
|
1188
|
+
function.get("function_name", ""),
|
1189
|
+
"",
|
1190
|
+
f"Runtime: {function.get('runtime', '')}, Memory: {function.get('memory_size', '')}MB"
|
1191
|
+
])
|
1192
|
+
|
1193
|
+
elif resource_type == "iam" and "resources" in account_data:
|
1194
|
+
iam_resources = account_data["resources"]
|
1195
|
+
for user in iam_resources.get("users", []):
|
1196
|
+
writer.writerow([
|
1197
|
+
account_id,
|
1198
|
+
"global",
|
1199
|
+
"iam-user",
|
1200
|
+
user.get("user_name", ""),
|
1201
|
+
user.get("user_name", ""),
|
1202
|
+
"",
|
1203
|
+
f"ARN: {user.get('arn', '')}"
|
1204
|
+
])
|
1205
|
+
for role in iam_resources.get("roles", []):
|
1206
|
+
writer.writerow([
|
1207
|
+
account_id,
|
1208
|
+
"global",
|
1209
|
+
"iam-role",
|
1210
|
+
role.get("role_name", ""),
|
1211
|
+
role.get("role_name", ""),
|
1212
|
+
"",
|
1213
|
+
f"ARN: {role.get('arn', '')}"
|
1214
|
+
])
|
1215
|
+
|
1216
|
+
elif resource_type == "vpc" and "vpcs" in account_data:
|
1217
|
+
for vpc in account_data["vpcs"]:
|
1218
|
+
writer.writerow([
|
1219
|
+
account_id,
|
1220
|
+
vpc.get("region", account_region),
|
1221
|
+
"vpc",
|
1222
|
+
vpc.get("vpc_id", ""),
|
1223
|
+
vpc.get("name", "No Name Tag"),
|
1224
|
+
vpc.get("state", ""),
|
1225
|
+
f"CIDR: {vpc.get('cidr_block', '')}, Default: {vpc.get('is_default', False)}"
|
1226
|
+
])
|
1227
|
+
|
1228
|
+
elif resource_type == "cloudformation" and "stacks" in account_data:
|
1229
|
+
for stack in account_data["stacks"]:
|
1230
|
+
writer.writerow([
|
1231
|
+
account_id,
|
1232
|
+
stack.get("region", account_region),
|
1233
|
+
"cloudformation-stack",
|
1234
|
+
stack.get("stack_name", ""),
|
1235
|
+
stack.get("stack_name", ""),
|
1236
|
+
stack.get("stack_status", ""),
|
1237
|
+
f"Created: {stack.get('creation_time', '')}"
|
1238
|
+
])
|
1239
|
+
|
1240
|
+
# Handle cases where no specific resources were found but collection was successful
|
1241
|
+
elif account_data.get("count", 0) == 0:
|
1242
|
+
writer.writerow([
|
1243
|
+
account_id,
|
1244
|
+
account_region,
|
1245
|
+
resource_type,
|
1246
|
+
"",
|
1247
|
+
"",
|
1248
|
+
"NO_RESOURCES",
|
1249
|
+
f"No {resource_type} resources found"
|
1250
|
+
])
|
1251
|
+
|
1252
|
+
print_success(f"Inventory exported to CSV: {output_file}")
|
1253
|
+
return output_file
|
1254
|
+
|
1255
|
+
def _export_markdown(self, results: Dict[str, Any], output_file: str) -> str:
|
1256
|
+
"""Export results to Markdown format with tables."""
|
1257
|
+
content = []
|
1258
|
+
content.append("# AWS Inventory Report")
|
1259
|
+
content.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
1260
|
+
content.append("")
|
1261
|
+
|
1262
|
+
# Summary section
|
1263
|
+
total_resources = sum(
|
1264
|
+
len(resources)
|
1265
|
+
for account_data in results.get("accounts", {}).values()
|
1266
|
+
for region_data in account_data.get("regions", {}).values()
|
1267
|
+
for resources in region_data.get("resources", {}).values()
|
1268
|
+
)
|
1269
|
+
|
1270
|
+
content.append("## Summary")
|
1271
|
+
content.append(f"- Total Accounts: {len(results.get('accounts', {}))}")
|
1272
|
+
content.append(f"- Total Resources: {total_resources}")
|
1273
|
+
content.append("")
|
1274
|
+
|
1275
|
+
# Detailed inventory
|
1276
|
+
content.append("## Detailed Inventory")
|
1277
|
+
content.append("")
|
1278
|
+
content.append("| Account | Region | Resource Type | Resource ID | Name | Status |")
|
1279
|
+
content.append("|---------|--------|---------------|-------------|------|--------|")
|
1280
|
+
|
1281
|
+
for account_id, account_data in results.get("accounts", {}).items():
|
1282
|
+
for region, region_data in account_data.get("regions", {}).items():
|
1283
|
+
for resource_type, resources in region_data.get("resources", {}).items():
|
1284
|
+
for resource in resources:
|
1285
|
+
content.append(f"| {account_id} | {region} | {resource_type} | {resource.get('id', '')} | {resource.get('name', '')} | {resource.get('state', '')} |")
|
1286
|
+
|
1287
|
+
with open(output_file, 'w') as f:
|
1288
|
+
f.write('\n'.join(content))
|
1289
|
+
|
1290
|
+
print_success(f"Inventory exported to Markdown: {output_file}")
|
1291
|
+
return output_file
|
1292
|
+
|
1293
|
+
def _export_yaml(self, results: Dict[str, Any], output_file: str) -> str:
|
1294
|
+
"""Export results to YAML format."""
|
1295
|
+
try:
|
1296
|
+
import yaml
|
1297
|
+
except ImportError:
|
1298
|
+
print_error("PyYAML not available. Install with: pip install pyyaml")
|
1299
|
+
raise
|
1300
|
+
|
1301
|
+
with open(output_file, 'w') as f:
|
1302
|
+
yaml.dump(results, f, default_flow_style=False, sort_keys=False)
|
1303
|
+
|
1304
|
+
print_success(f"Inventory exported to YAML: {output_file}")
|
1305
|
+
return output_file
|
1306
|
+
|
1307
|
+
def _export_pdf(self, results: Dict[str, Any], output_file: str) -> str:
|
1308
|
+
"""Export results to executive PDF report."""
|
1309
|
+
try:
|
1310
|
+
from reportlab.lib.pagesizes import letter, A4
|
1311
|
+
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle
|
1312
|
+
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
|
1313
|
+
from reportlab.lib.units import inch
|
1314
|
+
from reportlab.lib import colors
|
1315
|
+
except ImportError:
|
1316
|
+
# Graceful fallback to markdown if reportlab not available
|
1317
|
+
print_warning("ReportLab not available, exporting to markdown instead")
|
1318
|
+
return self._export_markdown(results, output_file.replace('.pdf', '.md'))
|
1319
|
+
|
1320
|
+
doc = SimpleDocTemplate(output_file, pagesize=A4)
|
1321
|
+
styles = getSampleStyleSheet()
|
1322
|
+
story = []
|
1323
|
+
|
1324
|
+
# Title
|
1325
|
+
title_style = ParagraphStyle(
|
1326
|
+
'CustomTitle',
|
1327
|
+
parent=styles['Heading1'],
|
1328
|
+
fontSize=24,
|
1329
|
+
spaceAfter=30,
|
1330
|
+
textColor=colors.darkblue
|
1331
|
+
)
|
1332
|
+
story.append(Paragraph("AWS Inventory Report", title_style))
|
1333
|
+
story.append(Spacer(1, 20))
|
1334
|
+
|
1335
|
+
# Executive Summary
|
1336
|
+
story.append(Paragraph("Executive Summary", styles['Heading2']))
|
1337
|
+
|
1338
|
+
total_resources = sum(
|
1339
|
+
len(resources)
|
1340
|
+
for account_data in results.get("accounts", {}).values()
|
1341
|
+
for region_data in account_data.get("regions", {}).values()
|
1342
|
+
for resources in region_data.get("resources", {}).values()
|
1343
|
+
)
|
1344
|
+
|
1345
|
+
summary_text = f"""
|
1346
|
+
This report provides a comprehensive inventory of AWS resources across {len(results.get('accounts', {}))} accounts.
|
1347
|
+
A total of {total_resources} resources were discovered and catalogued.
|
1348
|
+
"""
|
1349
|
+
story.append(Paragraph(summary_text, styles['Normal']))
|
1350
|
+
story.append(Spacer(1, 20))
|
1351
|
+
|
1352
|
+
# Build the PDF
|
1353
|
+
doc.build(story)
|
1354
|
+
|
1355
|
+
print_success(f"Inventory exported to PDF: {output_file}")
|
1356
|
+
return output_file
|
1357
|
+
|
295
1358
|
|
296
1359
|
# Legacy compatibility class - maintain backward compatibility
|
297
1360
|
class InventoryCollector(EnhancedInventoryCollector):
|
@@ -390,54 +1453,51 @@ class InventoryCollector(EnhancedInventoryCollector):
|
|
390
1453
|
this would delegate to specific resource collectors.
|
391
1454
|
"""
|
392
1455
|
# Mock implementation - replace with actual collectors
|
393
|
-
import random
|
394
1456
|
import time
|
395
1457
|
|
396
|
-
#
|
397
|
-
time.sleep(
|
1458
|
+
# Deterministic collection timing
|
1459
|
+
time.sleep(0.2) # Fixed 200ms delay for testing
|
398
1460
|
|
399
|
-
#
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
1461
|
+
# REMOVED: Mock data generation violates enterprise standards
|
1462
|
+
# Use real AWS API calls with proper authentication and error handling
|
1463
|
+
try:
|
1464
|
+
if resource_type == "ec2":
|
1465
|
+
# TODO: Implement real EC2 API call
|
1466
|
+
# ec2_client = self.session.client('ec2', region_name=self.region)
|
1467
|
+
# response = ec2_client.describe_instances()
|
1468
|
+
return {
|
1469
|
+
"instances": [], # Replace with real EC2 API response processing
|
1470
|
+
"count": 0,
|
1471
|
+
"account_id": account_id,
|
1472
|
+
"region": self.region or "us-east-1"
|
1473
|
+
}
|
1474
|
+
elif resource_type == "rds":
|
1475
|
+
# TODO: Implement real RDS API call
|
1476
|
+
# rds_client = self.session.client('rds', region_name=self.region)
|
1477
|
+
# response = rds_client.describe_db_instances()
|
1478
|
+
return {
|
1479
|
+
"instances": [], # Replace with real RDS API response processing
|
1480
|
+
"count": 0,
|
1481
|
+
"account_id": account_id,
|
1482
|
+
"region": self.region or "us-east-1"
|
1483
|
+
}
|
1484
|
+
elif resource_type == "s3":
|
1485
|
+
# TODO: Implement real S3 API call
|
1486
|
+
# s3_client = self.session.client('s3')
|
1487
|
+
# response = s3_client.list_buckets()
|
1488
|
+
return {
|
1489
|
+
"buckets": [], # Replace with real S3 API response processing
|
1490
|
+
"count": 0,
|
1491
|
+
"account_id": account_id,
|
1492
|
+
"region": self.region or "us-east-1"
|
1493
|
+
}
|
1494
|
+
except Exception as e:
|
1495
|
+
# Proper error handling for AWS API failures
|
430
1496
|
return {
|
431
|
-
"
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
"region": self.region or "us-east-1",
|
436
|
-
"account_id": account_id,
|
437
|
-
}
|
438
|
-
for _ in range(random.randint(1, 10))
|
439
|
-
],
|
440
|
-
"count": random.randint(1, 10),
|
1497
|
+
"error": str(e),
|
1498
|
+
"resource_type": resource_type,
|
1499
|
+
"account_id": account_id,
|
1500
|
+
"count": 0
|
441
1501
|
}
|
442
1502
|
else:
|
443
1503
|
return {"resources": [], "count": 0, "resource_type": resource_type, "account_id": account_id}
|