runbooks 0.9.8__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +1 -1
- runbooks/cfat/cloud_foundations_assessment.py +626 -0
- runbooks/cloudops/cost_optimizer.py +95 -33
- runbooks/common/aws_pricing.py +388 -0
- runbooks/common/aws_pricing_api.py +205 -0
- runbooks/common/aws_utils.py +2 -2
- runbooks/common/comprehensive_cost_explorer_integration.py +979 -0
- runbooks/common/cross_account_manager.py +606 -0
- runbooks/common/enhanced_exception_handler.py +4 -0
- runbooks/common/env_utils.py +96 -0
- runbooks/common/mcp_integration.py +49 -2
- runbooks/common/organizations_client.py +579 -0
- runbooks/common/profile_utils.py +96 -2
- runbooks/common/rich_utils.py +3 -0
- runbooks/finops/cost_optimizer.py +2 -1
- runbooks/finops/elastic_ip_optimizer.py +13 -9
- runbooks/finops/embedded_mcp_validator.py +31 -0
- runbooks/finops/enhanced_trend_visualization.py +3 -2
- runbooks/finops/markdown_exporter.py +441 -0
- runbooks/finops/nat_gateway_optimizer.py +57 -20
- runbooks/finops/optimizer.py +2 -0
- runbooks/finops/single_dashboard.py +2 -2
- runbooks/finops/vpc_cleanup_exporter.py +330 -0
- runbooks/finops/vpc_cleanup_optimizer.py +895 -40
- runbooks/inventory/__init__.py +10 -1
- runbooks/inventory/cloud_foundations_integration.py +409 -0
- runbooks/inventory/core/collector.py +1148 -88
- runbooks/inventory/discovery.md +389 -0
- runbooks/inventory/drift_detection_cli.py +327 -0
- runbooks/inventory/inventory_mcp_cli.py +171 -0
- runbooks/inventory/inventory_modules.py +4 -7
- runbooks/inventory/mcp_inventory_validator.py +2149 -0
- runbooks/inventory/mcp_vpc_validator.py +23 -6
- runbooks/inventory/organizations_discovery.py +91 -1
- runbooks/inventory/rich_inventory_display.py +129 -1
- runbooks/inventory/unified_validation_engine.py +1292 -0
- runbooks/inventory/verify_ec2_security_groups.py +3 -1
- runbooks/inventory/vpc_analyzer.py +825 -7
- runbooks/inventory/vpc_flow_analyzer.py +36 -42
- runbooks/main.py +969 -42
- runbooks/monitoring/performance_monitor.py +11 -7
- runbooks/operate/dynamodb_operations.py +6 -5
- runbooks/operate/ec2_operations.py +3 -2
- runbooks/operate/networking_cost_heatmap.py +4 -3
- runbooks/operate/s3_operations.py +13 -12
- runbooks/operate/vpc_operations.py +50 -2
- runbooks/remediation/base.py +1 -1
- runbooks/remediation/commvault_ec2_analysis.py +6 -1
- runbooks/remediation/ec2_unattached_ebs_volumes.py +6 -3
- runbooks/remediation/rds_snapshot_list.py +5 -3
- runbooks/validation/__init__.py +21 -1
- runbooks/validation/comprehensive_2way_validator.py +1996 -0
- runbooks/validation/mcp_validator.py +904 -94
- runbooks/validation/terraform_citations_validator.py +363 -0
- runbooks/validation/terraform_drift_detector.py +1098 -0
- runbooks/vpc/cleanup_wrapper.py +231 -10
- runbooks/vpc/config.py +310 -62
- runbooks/vpc/cross_account_session.py +308 -0
- runbooks/vpc/heatmap_engine.py +96 -29
- runbooks/vpc/manager_interface.py +9 -9
- runbooks/vpc/mcp_no_eni_validator.py +1551 -0
- runbooks/vpc/networking_wrapper.py +14 -8
- runbooks/vpc/runbooks.inventory.organizations_discovery.log +0 -0
- runbooks/vpc/runbooks.security.report_generator.log +0 -0
- runbooks/vpc/runbooks.security.run_script.log +0 -0
- runbooks/vpc/runbooks.security.security_export.log +0 -0
- runbooks/vpc/tests/test_cost_engine.py +1 -1
- runbooks/vpc/unified_scenarios.py +3269 -0
- runbooks/vpc/vpc_cleanup_integration.py +516 -82
- {runbooks-0.9.8.dist-info → runbooks-1.0.0.dist-info}/METADATA +94 -52
- {runbooks-0.9.8.dist-info → runbooks-1.0.0.dist-info}/RECORD +75 -51
- {runbooks-0.9.8.dist-info → runbooks-1.0.0.dist-info}/WHEEL +0 -0
- {runbooks-0.9.8.dist-info → runbooks-1.0.0.dist-info}/entry_points.txt +0 -0
- {runbooks-0.9.8.dist-info → runbooks-1.0.0.dist-info}/licenses/LICENSE +0 -0
- {runbooks-0.9.8.dist-info → runbooks-1.0.0.dist-info}/top_level.txt +0 -0
@@ -47,9 +47,11 @@ from ..common.rich_utils import (
|
|
47
47
|
console, print_header, print_success, print_error, print_warning, print_info,
|
48
48
|
create_table, create_progress_bar, format_cost, create_panel, STATUS_INDICATORS
|
49
49
|
)
|
50
|
+
from ..common.aws_pricing import DynamicAWSPricing
|
50
51
|
from .embedded_mcp_validator import EmbeddedMCPValidator
|
51
52
|
from ..common.profile_utils import get_profile_for_operation
|
52
|
-
from ..
|
53
|
+
from ..security.enterprise_security_framework import EnterpriseSecurityFramework
|
54
|
+
from ..vpc.mcp_no_eni_validator import NOENIVPCMCPValidator, DynamicDiscoveryResults
|
53
55
|
|
54
56
|
logger = logging.getLogger(__name__)
|
55
57
|
|
@@ -87,6 +89,28 @@ class VPCCleanupCandidate(BaseModel):
|
|
87
89
|
risk_assessment: str = "medium" # low, medium, high
|
88
90
|
business_impact: str = "minimal" # minimal, moderate, significant
|
89
91
|
tags: Dict[str, str] = Field(default_factory=dict)
|
92
|
+
|
93
|
+
# Enhanced fields for advanced filtering
|
94
|
+
account_id: Optional[str] = None
|
95
|
+
flow_logs_enabled: bool = False
|
96
|
+
load_balancers: List[str] = Field(default_factory=list)
|
97
|
+
iac_detected: bool = False
|
98
|
+
owners_approvals: List[str] = Field(default_factory=list)
|
99
|
+
|
100
|
+
@property
|
101
|
+
def is_no_eni_vpc(self) -> bool:
|
102
|
+
"""Check if VPC has zero ENI attachments (safe for cleanup)."""
|
103
|
+
return self.dependency_analysis.eni_count == 0
|
104
|
+
|
105
|
+
@property
|
106
|
+
def is_nil_vpc(self) -> bool:
|
107
|
+
"""Check if VPC has no resources (empty VPC)."""
|
108
|
+
return (
|
109
|
+
self.dependency_analysis.eni_count == 0 and
|
110
|
+
len(self.dependency_analysis.route_tables) <= 1 and # Only default route table
|
111
|
+
len(self.dependency_analysis.nat_gateways) == 0 and
|
112
|
+
len(self.dependency_analysis.vpc_endpoints) == 0
|
113
|
+
)
|
90
114
|
|
91
115
|
|
92
116
|
class VPCCleanupResults(BaseModel):
|
@@ -102,6 +126,7 @@ class VPCCleanupResults(BaseModel):
|
|
102
126
|
evidence_hash: Optional[str] = None
|
103
127
|
safety_assessment: str = "graduated_risk_approach"
|
104
128
|
security_assessment: Optional[Dict[str, Any]] = None
|
129
|
+
multi_account_context: Optional[Dict[str, Any]] = Field(default_factory=dict)
|
105
130
|
|
106
131
|
|
107
132
|
class VPCCleanupOptimizer:
|
@@ -129,7 +154,55 @@ class VPCCleanupOptimizer:
|
|
129
154
|
|
130
155
|
print_info(f"VPC Cleanup Optimizer initialized with profile: {self.profile}")
|
131
156
|
|
132
|
-
def
|
157
|
+
def filter_vpcs_by_criteria(self, candidates: List[VPCCleanupCandidate],
|
158
|
+
no_eni_only: bool = False,
|
159
|
+
filter_type: str = "all") -> List[VPCCleanupCandidate]:
|
160
|
+
"""
|
161
|
+
Filter VPC candidates based on specified criteria.
|
162
|
+
|
163
|
+
Args:
|
164
|
+
candidates: List of VPC cleanup candidates
|
165
|
+
no_eni_only: If True, show only VPCs with zero ENI attachments
|
166
|
+
filter_type: Filter type - 'none', 'default', or 'all'
|
167
|
+
|
168
|
+
Returns:
|
169
|
+
Filtered list of VPC candidates
|
170
|
+
"""
|
171
|
+
filtered_candidates = candidates.copy()
|
172
|
+
|
173
|
+
# Apply no-ENI-only filter
|
174
|
+
if no_eni_only:
|
175
|
+
filtered_candidates = [
|
176
|
+
candidate for candidate in filtered_candidates
|
177
|
+
if candidate.is_no_eni_vpc
|
178
|
+
]
|
179
|
+
print_info(f"🔍 No-ENI filter applied - {len(filtered_candidates)} VPCs with zero ENI attachments")
|
180
|
+
|
181
|
+
# Apply type-based filters
|
182
|
+
if filter_type == "none":
|
183
|
+
# Show only VPCs with no resources (nil VPCs)
|
184
|
+
filtered_candidates = [
|
185
|
+
candidate for candidate in filtered_candidates
|
186
|
+
if candidate.is_nil_vpc
|
187
|
+
]
|
188
|
+
print_info(f"📋 'None' filter applied - {len(filtered_candidates)} VPCs with no resources")
|
189
|
+
|
190
|
+
elif filter_type == "default":
|
191
|
+
# Show only default VPCs
|
192
|
+
filtered_candidates = [
|
193
|
+
candidate for candidate in filtered_candidates
|
194
|
+
if candidate.is_default
|
195
|
+
]
|
196
|
+
print_info(f"🏠 'Default' filter applied - {len(filtered_candidates)} default VPCs")
|
197
|
+
|
198
|
+
elif filter_type == "all":
|
199
|
+
# Show all VPCs (no additional filtering)
|
200
|
+
print_info(f"📊 'All' filter applied - {len(filtered_candidates)} total VPCs")
|
201
|
+
|
202
|
+
return filtered_candidates
|
203
|
+
|
204
|
+
def analyze_vpc_cleanup_opportunities(self, no_eni_only: bool = False,
|
205
|
+
filter_type: str = "all") -> VPCCleanupResults:
|
133
206
|
"""
|
134
207
|
Comprehensive VPC cleanup analysis with three-bucket strategy.
|
135
208
|
|
@@ -143,7 +216,7 @@ class VPCCleanupOptimizer:
|
|
143
216
|
|
144
217
|
Returns: Comprehensive cleanup analysis with validated savings
|
145
218
|
"""
|
146
|
-
print_header("VPC Cleanup Cost Optimization Engine", "v0.9.
|
219
|
+
print_header("VPC Cleanup Cost Optimization Engine", "v0.9.9")
|
147
220
|
print_info("AWSO-05 Implementation - Three-Bucket Strategy")
|
148
221
|
|
149
222
|
# Initialize MCP validator for accuracy validation
|
@@ -155,12 +228,22 @@ class VPCCleanupOptimizer:
|
|
155
228
|
# Step 2: Dependency analysis for each VPC
|
156
229
|
analyzed_candidates = self._analyze_vpc_dependencies(vpc_candidates)
|
157
230
|
|
231
|
+
# Step 2.5: Apply filtering based on criteria
|
232
|
+
filtered_candidates = self.filter_vpcs_by_criteria(
|
233
|
+
analyzed_candidates,
|
234
|
+
no_eni_only=no_eni_only,
|
235
|
+
filter_type=filter_type
|
236
|
+
)
|
237
|
+
|
158
238
|
# Step 3: Three-bucket classification
|
159
|
-
bucket_classification = self._classify_three_bucket_strategy(
|
239
|
+
bucket_classification = self._classify_three_bucket_strategy(filtered_candidates)
|
160
240
|
|
161
241
|
# Step 4: Enhanced VPC security assessment integration
|
162
242
|
security_assessment = self._perform_vpc_security_assessment(analyzed_candidates)
|
163
243
|
|
244
|
+
# Step 4.5: Re-classify buckets after security assessment (ensure NO-ENI VPCs stay in Bucket 1)
|
245
|
+
bucket_classification = self._ensure_no_eni_bucket_1_classification(bucket_classification)
|
246
|
+
|
164
247
|
# Step 5: Cost calculation and savings estimation
|
165
248
|
cost_analysis = self._calculate_vpc_cleanup_costs(bucket_classification)
|
166
249
|
|
@@ -175,6 +258,295 @@ class VPCCleanupOptimizer:
|
|
175
258
|
|
176
259
|
return results
|
177
260
|
|
261
|
+
def analyze_vpc_cleanup_opportunities_multi_account(self,
|
262
|
+
account_ids: List[str],
|
263
|
+
accounts_info: Dict[str, Any],
|
264
|
+
no_eni_only: bool = False,
|
265
|
+
filter_type: str = "all",
|
266
|
+
progress_callback=None) -> 'VPCCleanupResults':
|
267
|
+
"""
|
268
|
+
ENHANCED: Multi-account VPC cleanup analysis with Organizations integration.
|
269
|
+
|
270
|
+
Critical Fix: Instead of assuming cross-account access, this method aggregates
|
271
|
+
VPC discovery from the current profile's accessible scope, which may span
|
272
|
+
multiple accounts if the profile has appropriate permissions.
|
273
|
+
|
274
|
+
Args:
|
275
|
+
account_ids: List of account IDs from Organizations discovery
|
276
|
+
accounts_info: Account metadata from Organizations API
|
277
|
+
no_eni_only: Filter to only VPCs with zero ENI attachments
|
278
|
+
filter_type: Filter criteria ("all", "no-eni", "default", "tagged")
|
279
|
+
progress_callback: Optional callback for progress updates
|
280
|
+
|
281
|
+
Returns: Aggregated VPC cleanup analysis across accessible accounts
|
282
|
+
"""
|
283
|
+
print_header("Multi-Account VPC Cleanup Analysis", "v0.9.9")
|
284
|
+
print_info(f"🏢 Analyzing VPCs across {len(account_ids)} organization accounts")
|
285
|
+
print_info(f"🔐 Using profile: {self.profile} (scope: accessible accounts only)")
|
286
|
+
|
287
|
+
# Initialize analysis timing
|
288
|
+
self.analysis_start_time = time.time()
|
289
|
+
|
290
|
+
# Initialize MCP validator for accuracy validation
|
291
|
+
self.mcp_validator = EmbeddedMCPValidator([self.profile])
|
292
|
+
|
293
|
+
if progress_callback:
|
294
|
+
progress_callback("Initializing multi-account discovery...")
|
295
|
+
|
296
|
+
# Enhanced VPC discovery with organization context
|
297
|
+
vpc_candidates = self._discover_vpc_candidates_multi_account(account_ids, accounts_info, progress_callback)
|
298
|
+
|
299
|
+
if progress_callback:
|
300
|
+
progress_callback("Analyzing VPC dependencies...")
|
301
|
+
|
302
|
+
# Follow standard analysis pipeline
|
303
|
+
analyzed_candidates = self._analyze_vpc_dependencies(vpc_candidates)
|
304
|
+
filtered_candidates = self.filter_vpcs_by_criteria(
|
305
|
+
analyzed_candidates,
|
306
|
+
no_eni_only=no_eni_only,
|
307
|
+
filter_type=filter_type
|
308
|
+
)
|
309
|
+
|
310
|
+
if progress_callback:
|
311
|
+
progress_callback("Performing three-bucket classification...")
|
312
|
+
|
313
|
+
bucket_classification = self._classify_three_bucket_strategy(filtered_candidates)
|
314
|
+
security_assessment = self._perform_vpc_security_assessment(analyzed_candidates)
|
315
|
+
bucket_classification = self._ensure_no_eni_bucket_1_classification(bucket_classification)
|
316
|
+
|
317
|
+
if progress_callback:
|
318
|
+
progress_callback("Calculating cost analysis...")
|
319
|
+
|
320
|
+
cost_analysis = self._calculate_vpc_cleanup_costs(bucket_classification)
|
321
|
+
validation_results = self._validate_analysis_with_mcp(cost_analysis)
|
322
|
+
|
323
|
+
if progress_callback:
|
324
|
+
progress_callback("Generating comprehensive results...")
|
325
|
+
|
326
|
+
# Generate results with multi-account context
|
327
|
+
results = self._generate_comprehensive_results(cost_analysis, validation_results, security_assessment)
|
328
|
+
|
329
|
+
# Add multi-account metadata
|
330
|
+
results.multi_account_context = {
|
331
|
+
'total_accounts_analyzed': len(account_ids),
|
332
|
+
'accounts_with_vpcs': len(set(candidate.account_id for candidate in vpc_candidates if candidate.account_id)),
|
333
|
+
'organization_id': accounts_info.get('organization_id', 'unknown'),
|
334
|
+
'accounts': [
|
335
|
+
{
|
336
|
+
'account_id': account_id,
|
337
|
+
'account_name': accounts_info.get('accounts', {}).get(account_id, {}).get('Name', 'unknown'),
|
338
|
+
'status': accounts_info.get('accounts', {}).get(account_id, {}).get('Status', 'unknown')
|
339
|
+
}
|
340
|
+
for account_id in account_ids
|
341
|
+
],
|
342
|
+
'vpc_count_by_account': self._calculate_vpc_count_by_account(vpc_candidates),
|
343
|
+
'analysis_scope': 'organization',
|
344
|
+
'profile_access_scope': self.profile
|
345
|
+
}
|
346
|
+
|
347
|
+
print_success(f"✅ Multi-account analysis complete: {results.total_vpcs_analyzed} VPCs across organization")
|
348
|
+
return results
|
349
|
+
|
350
|
+
def _calculate_vpc_count_by_account(self, vpc_candidates: List[VPCCleanupCandidate]) -> Dict[str, int]:
|
351
|
+
"""Calculate VPC count by account from the candidate list."""
|
352
|
+
vpc_count_by_account = {}
|
353
|
+
|
354
|
+
for candidate in vpc_candidates:
|
355
|
+
account_id = candidate.account_id or 'unknown'
|
356
|
+
if account_id in vpc_count_by_account:
|
357
|
+
vpc_count_by_account[account_id] += 1
|
358
|
+
else:
|
359
|
+
vpc_count_by_account[account_id] = 1
|
360
|
+
|
361
|
+
return vpc_count_by_account
|
362
|
+
|
363
|
+
def _discover_vpc_candidates_multi_account(self, account_ids: List[str],
|
364
|
+
accounts_info: Dict[str, Any],
|
365
|
+
progress_callback=None) -> List[VPCCleanupCandidate]:
|
366
|
+
"""
|
367
|
+
Enhanced VPC discovery with organization account context.
|
368
|
+
|
369
|
+
CRITICAL FIX: This method now attempts to discover VPCs across multiple accounts
|
370
|
+
by trying different access patterns:
|
371
|
+
1. Direct access with current profile
|
372
|
+
2. Cross-account role assumption (if available)
|
373
|
+
3. Aggregation from multiple AWS SSO profiles
|
374
|
+
"""
|
375
|
+
vpc_candidates = []
|
376
|
+
total_accounts_checked = 0
|
377
|
+
accounts_with_vpcs = set()
|
378
|
+
|
379
|
+
if progress_callback:
|
380
|
+
progress_callback(f"Discovering VPCs across {len(account_ids)} organization accounts...")
|
381
|
+
|
382
|
+
# Get list of all regions
|
383
|
+
ec2_client = self.session.client('ec2', region_name='us-east-1')
|
384
|
+
regions = [region['RegionName'] for region in ec2_client.describe_regions()['Regions']]
|
385
|
+
|
386
|
+
print_info(f"🌍 Scanning {len(regions)} AWS regions across {len(account_ids)} accounts...")
|
387
|
+
|
388
|
+
# First, discover VPCs in current profile's accessible accounts
|
389
|
+
current_account_vpcs = self._discover_vpcs_current_profile(regions, progress_callback)
|
390
|
+
vpc_candidates.extend(current_account_vpcs)
|
391
|
+
|
392
|
+
# Extract unique account IDs from discovered VPCs
|
393
|
+
for vpc in current_account_vpcs:
|
394
|
+
if vpc.account_id:
|
395
|
+
accounts_with_vpcs.add(vpc.account_id)
|
396
|
+
|
397
|
+
# Attempt cross-account discovery for remaining accounts
|
398
|
+
remaining_accounts = [acc for acc in account_ids if acc not in accounts_with_vpcs]
|
399
|
+
|
400
|
+
if remaining_accounts:
|
401
|
+
print_info(f"🔄 Attempting cross-account discovery for {len(remaining_accounts)} additional accounts...")
|
402
|
+
|
403
|
+
# Try different access patterns for remaining accounts
|
404
|
+
for account_id in remaining_accounts[:10]: # Limit to first 10 for performance
|
405
|
+
total_accounts_checked += 1
|
406
|
+
account_name = accounts_info.get(account_id, {}).get('name', 'Unknown')
|
407
|
+
|
408
|
+
if progress_callback:
|
409
|
+
progress_callback(f"Checking account {account_name} ({account_id[:12]}...)")
|
410
|
+
|
411
|
+
# Attempt cross-account access
|
412
|
+
cross_account_vpcs = self._attempt_cross_account_discovery(
|
413
|
+
account_id, account_name, regions
|
414
|
+
)
|
415
|
+
|
416
|
+
if cross_account_vpcs:
|
417
|
+
vpc_candidates.extend(cross_account_vpcs)
|
418
|
+
accounts_with_vpcs.add(account_id)
|
419
|
+
print_success(f" ✅ Found {len(cross_account_vpcs)} VPCs in {account_name}")
|
420
|
+
|
421
|
+
# Summary
|
422
|
+
print_success(f"✅ Discovered {len(vpc_candidates)} total VPCs across {len(accounts_with_vpcs)} accounts")
|
423
|
+
print_info(f"📊 Organization scope: {len(account_ids)} accounts, {total_accounts_checked} checked, {len(accounts_with_vpcs)} with VPCs")
|
424
|
+
|
425
|
+
# If we still have < 13 VPCs, provide guidance
|
426
|
+
if len(vpc_candidates) < 13:
|
427
|
+
print_warning(f"⚠️ Only {len(vpc_candidates)} VPCs found (target: ≥13). Consider:")
|
428
|
+
print_info(" 1. Using MANAGEMENT_PROFILE with broader cross-account access")
|
429
|
+
print_info(" 2. Configuring cross-account roles for VPC discovery")
|
430
|
+
print_info(" 3. Running discovery from each account individually")
|
431
|
+
|
432
|
+
return vpc_candidates
|
433
|
+
|
434
|
+
def _discover_vpcs_current_profile(self, regions: List[str], progress_callback=None) -> List[VPCCleanupCandidate]:
|
435
|
+
"""Discover VPCs accessible with current profile."""
|
436
|
+
vpc_candidates = []
|
437
|
+
regions_with_vpcs = 0
|
438
|
+
|
439
|
+
for region in regions:
|
440
|
+
try:
|
441
|
+
regional_ec2 = self.session.client('ec2', region_name=region)
|
442
|
+
response = regional_ec2.describe_vpcs()
|
443
|
+
|
444
|
+
region_vpc_count = 0
|
445
|
+
for vpc in response['Vpcs']:
|
446
|
+
# Enhanced VPC candidate with account detection
|
447
|
+
candidate = VPCCleanupCandidate(
|
448
|
+
vpc_id=vpc['VpcId'],
|
449
|
+
region=region,
|
450
|
+
state=vpc['State'],
|
451
|
+
cidr_block=vpc['CidrBlock'],
|
452
|
+
is_default=vpc.get('IsDefault', False),
|
453
|
+
account_id=self._detect_vpc_account_id(vpc), # Enhanced account detection
|
454
|
+
dependency_analysis=VPCDependencyAnalysis(
|
455
|
+
vpc_id=vpc['VpcId'],
|
456
|
+
region=region,
|
457
|
+
is_default_vpc=vpc.get('IsDefault', False)
|
458
|
+
),
|
459
|
+
tags={tag['Key']: tag['Value'] for tag in vpc.get('Tags', [])}
|
460
|
+
)
|
461
|
+
vpc_candidates.append(candidate)
|
462
|
+
region_vpc_count += 1
|
463
|
+
|
464
|
+
if region_vpc_count > 0:
|
465
|
+
regions_with_vpcs += 1
|
466
|
+
|
467
|
+
except ClientError as e:
|
468
|
+
# Silently skip regions with no access
|
469
|
+
pass
|
470
|
+
|
471
|
+
return vpc_candidates
|
472
|
+
|
473
|
+
def _attempt_cross_account_discovery(self, account_id: str, account_name: str,
|
474
|
+
regions: List[str]) -> List[VPCCleanupCandidate]:
|
475
|
+
"""Attempt to discover VPCs in a specific account using cross-account access."""
|
476
|
+
vpc_candidates = []
|
477
|
+
|
478
|
+
# Try to assume a cross-account role (if configured)
|
479
|
+
role_name = "OrganizationAccountAccessRole" # Standard AWS Organizations role
|
480
|
+
|
481
|
+
try:
|
482
|
+
# Attempt to assume role in target account
|
483
|
+
sts_client = self.session.client('sts')
|
484
|
+
assumed_role = sts_client.assume_role(
|
485
|
+
RoleArn=f"arn:aws:iam::{account_id}:role/{role_name}",
|
486
|
+
RoleSessionName=f"VPCDiscovery-{account_id[:12]}"
|
487
|
+
)
|
488
|
+
|
489
|
+
# Create session with assumed role credentials
|
490
|
+
assumed_session = boto3.Session(
|
491
|
+
aws_access_key_id=assumed_role['Credentials']['AccessKeyId'],
|
492
|
+
aws_secret_access_key=assumed_role['Credentials']['SecretAccessKey'],
|
493
|
+
aws_session_token=assumed_role['Credentials']['SessionToken']
|
494
|
+
)
|
495
|
+
|
496
|
+
# Discover VPCs in target account
|
497
|
+
for region in regions[:3]: # Check first 3 regions for performance
|
498
|
+
try:
|
499
|
+
ec2_client = assumed_session.client('ec2', region_name=region)
|
500
|
+
response = ec2_client.describe_vpcs()
|
501
|
+
|
502
|
+
for vpc in response['Vpcs']:
|
503
|
+
candidate = VPCCleanupCandidate(
|
504
|
+
vpc_id=vpc['VpcId'],
|
505
|
+
region=region,
|
506
|
+
state=vpc['State'],
|
507
|
+
cidr_block=vpc['CidrBlock'],
|
508
|
+
is_default=vpc.get('IsDefault', False),
|
509
|
+
account_id=account_id, # Set explicit account ID
|
510
|
+
dependency_analysis=VPCDependencyAnalysis(
|
511
|
+
vpc_id=vpc['VpcId'],
|
512
|
+
region=region,
|
513
|
+
is_default_vpc=vpc.get('IsDefault', False)
|
514
|
+
),
|
515
|
+
tags={tag['Key']: tag['Value'] for tag in vpc.get('Tags', [])}
|
516
|
+
)
|
517
|
+
vpc_candidates.append(candidate)
|
518
|
+
|
519
|
+
except Exception:
|
520
|
+
# Skip regions with access issues
|
521
|
+
pass
|
522
|
+
|
523
|
+
except Exception as e:
|
524
|
+
# Cross-account access not available - this is expected for most profiles
|
525
|
+
pass
|
526
|
+
|
527
|
+
return vpc_candidates
|
528
|
+
|
529
|
+
def _detect_vpc_account_id(self, vpc_data: Dict[str, Any]) -> Optional[str]:
|
530
|
+
"""
|
531
|
+
Detect account ID for VPC (enhanced for multi-account context).
|
532
|
+
|
533
|
+
In multi-account scenarios, VPC ARN or tags may contain account information.
|
534
|
+
"""
|
535
|
+
# Try to extract account ID from VPC ARN if available
|
536
|
+
if 'VpcArn' in vpc_data:
|
537
|
+
# VPC ARN format: arn:aws:ec2:region:account-id:vpc/vpc-id
|
538
|
+
arn_parts = vpc_data['VpcArn'].split(':')
|
539
|
+
if len(arn_parts) >= 5:
|
540
|
+
return arn_parts[4]
|
541
|
+
|
542
|
+
# Fallback: Try to get from current session context
|
543
|
+
try:
|
544
|
+
sts_client = self.session.client('sts')
|
545
|
+
response = sts_client.get_caller_identity()
|
546
|
+
return response.get('Account')
|
547
|
+
except Exception:
|
548
|
+
return None
|
549
|
+
|
178
550
|
def _discover_vpc_candidates(self) -> List[VPCCleanupCandidate]:
|
179
551
|
"""Discover VPC candidates across all AWS regions."""
|
180
552
|
vpc_candidates = []
|
@@ -217,6 +589,244 @@ class VPCCleanupOptimizer:
|
|
217
589
|
print_success(f"✅ Discovered {len(vpc_candidates)} VPC candidates across {len(regions)} regions")
|
218
590
|
return vpc_candidates
|
219
591
|
|
592
|
+
async def discover_no_eni_vpcs_with_mcp_validation(self,
|
593
|
+
target_regions: List[str] = None,
|
594
|
+
max_concurrent_accounts: int = 10) -> Tuple[List[VPCCleanupCandidate], DynamicDiscoveryResults]:
|
595
|
+
"""
|
596
|
+
Discover NO-ENI VPCs across all AWS accounts using real-time MCP validation.
|
597
|
+
|
598
|
+
This method integrates with the dynamic MCP validator to discover the actual
|
599
|
+
count of NO-ENI VPCs across all accessible accounts, not hardcoded numbers.
|
600
|
+
|
601
|
+
Args:
|
602
|
+
target_regions: List of regions to scan (default: ['ap-southeast-2'])
|
603
|
+
max_concurrent_accounts: Maximum concurrent account scans
|
604
|
+
|
605
|
+
Returns:
|
606
|
+
Tuple of (VPC cleanup candidates, Dynamic discovery results)
|
607
|
+
"""
|
608
|
+
if target_regions is None:
|
609
|
+
target_regions = ['ap-southeast-2']
|
610
|
+
|
611
|
+
print_header("🌐 Real-Time NO-ENI VPC Discovery", "MCP-Validated VPC Cleanup Analysis")
|
612
|
+
|
613
|
+
# Configure enterprise profiles for MCP validation - Universal compatibility
|
614
|
+
from runbooks.common.profile_utils import get_enterprise_profile_mapping
|
615
|
+
enterprise_profiles = get_enterprise_profile_mapping()
|
616
|
+
|
617
|
+
# Override with current profile if available
|
618
|
+
current_profile_type = self._determine_profile_type(self.profile)
|
619
|
+
if current_profile_type:
|
620
|
+
enterprise_profiles[current_profile_type] = self.profile
|
621
|
+
|
622
|
+
# Initialize MCP validator for dynamic discovery
|
623
|
+
print_info("🔧 Initializing dynamic MCP validator...")
|
624
|
+
mcp_validator = NOENIVPCMCPValidator(enterprise_profiles)
|
625
|
+
|
626
|
+
# Perform dynamic discovery across all accounts
|
627
|
+
print_info("🚀 Starting real-time discovery across all AWS accounts...")
|
628
|
+
discovery_results = await mcp_validator.discover_all_no_eni_vpcs_dynamically(
|
629
|
+
target_regions=target_regions,
|
630
|
+
max_concurrent_accounts=max_concurrent_accounts
|
631
|
+
)
|
632
|
+
|
633
|
+
# Convert MCP discovery results to VPC cleanup candidates
|
634
|
+
print_info("🔄 Converting MCP results to VPC cleanup candidates...")
|
635
|
+
cleanup_candidates = []
|
636
|
+
|
637
|
+
for target in discovery_results.account_region_results:
|
638
|
+
if not target.has_access or not target.no_eni_vpcs:
|
639
|
+
continue
|
640
|
+
|
641
|
+
# Get detailed VPC information for each NO-ENI VPC
|
642
|
+
try:
|
643
|
+
# Use appropriate session for this account
|
644
|
+
session = self._get_session_for_account(target.account_id, enterprise_profiles)
|
645
|
+
ec2_client = session.client('ec2', region_name=target.region)
|
646
|
+
|
647
|
+
# Get VPC details
|
648
|
+
vpc_response = ec2_client.describe_vpcs(VpcIds=target.no_eni_vpcs)
|
649
|
+
|
650
|
+
for vpc in vpc_response.get('Vpcs', []):
|
651
|
+
# Create comprehensive dependency analysis
|
652
|
+
dependency_analysis = await self._create_dependency_analysis_for_vpc(
|
653
|
+
vpc['VpcId'], target.region, ec2_client
|
654
|
+
)
|
655
|
+
|
656
|
+
# Create VPC cleanup candidate
|
657
|
+
candidate = VPCCleanupCandidate(
|
658
|
+
vpc_id=vpc['VpcId'],
|
659
|
+
region=target.region,
|
660
|
+
state=vpc.get('State', 'unknown'),
|
661
|
+
cidr_block=vpc.get('CidrBlock', ''),
|
662
|
+
is_default=vpc.get('IsDefault', False),
|
663
|
+
dependency_analysis=dependency_analysis,
|
664
|
+
cleanup_bucket='internal', # NO-ENI VPCs go to internal bucket
|
665
|
+
monthly_cost=self._estimate_vpc_monthly_cost(vpc),
|
666
|
+
annual_savings=self._estimate_vpc_monthly_cost(vpc) * 12,
|
667
|
+
cleanup_recommendation='ready' if dependency_analysis.eni_count == 0 else 'investigate',
|
668
|
+
risk_assessment='low' if dependency_analysis.eni_count == 0 else 'medium',
|
669
|
+
business_impact='minimal',
|
670
|
+
tags=self._extract_vpc_tags(vpc),
|
671
|
+
account_id=target.account_id,
|
672
|
+
flow_logs_enabled=await self._check_flow_logs_enabled(vpc['VpcId'], ec2_client)
|
673
|
+
)
|
674
|
+
|
675
|
+
cleanup_candidates.append(candidate)
|
676
|
+
|
677
|
+
except Exception as e:
|
678
|
+
print_warning(f"Failed to analyze VPC details for account {target.account_id}: {e}")
|
679
|
+
continue
|
680
|
+
|
681
|
+
# Display integration results
|
682
|
+
print_header("🎯 MCP-Validated VPC Cleanup Summary", "Real-Time Integration Results")
|
683
|
+
console.print(f"[bold green]✅ NO-ENI VPCs discovered: {len(cleanup_candidates)}[/bold green]")
|
684
|
+
console.print(f"[bold blue]📊 Accounts scanned: {discovery_results.total_accounts_scanned}[/bold blue]")
|
685
|
+
console.print(f"[bold yellow]🌍 Regions scanned: {discovery_results.total_regions_scanned}[/bold yellow]")
|
686
|
+
console.print(f"[bold magenta]🧪 MCP validation accuracy: {discovery_results.mcp_validation_accuracy:.2f}%[/bold magenta]")
|
687
|
+
|
688
|
+
# Calculate potential savings
|
689
|
+
total_annual_savings = sum(candidate.annual_savings for candidate in cleanup_candidates)
|
690
|
+
console.print(f"[bold cyan]💰 Potential annual savings: {format_cost(total_annual_savings)}[/bold cyan]")
|
691
|
+
|
692
|
+
# Validation status
|
693
|
+
if discovery_results.mcp_validation_accuracy >= 99.5:
|
694
|
+
print_success(f"✅ ENTERPRISE VALIDATION PASSED: {discovery_results.mcp_validation_accuracy:.2f}% accuracy")
|
695
|
+
else:
|
696
|
+
print_warning(f"⚠️ VALIDATION REVIEW REQUIRED: {discovery_results.mcp_validation_accuracy:.2f}% accuracy")
|
697
|
+
|
698
|
+
return cleanup_candidates, discovery_results
|
699
|
+
|
700
|
+
def _determine_profile_type(self, profile_name: str) -> Optional[str]:
|
701
|
+
"""Determine profile type from profile name for universal compatibility."""
|
702
|
+
# Universal pattern matching for any AWS profile naming convention
|
703
|
+
if 'billing' in profile_name.lower() or 'Billing' in profile_name:
|
704
|
+
return 'BILLING'
|
705
|
+
elif 'management' in profile_name.lower() or 'admin' in profile_name.lower():
|
706
|
+
return 'MANAGEMENT'
|
707
|
+
elif 'ops' in profile_name.lower() or 'operational' in profile_name.lower():
|
708
|
+
return 'CENTRALISED_OPS'
|
709
|
+
return None
|
710
|
+
|
711
|
+
def _get_session_for_account(self, account_id: str, enterprise_profiles: Dict[str, str]) -> boto3.Session:
|
712
|
+
"""Get appropriate session for accessing a specific account."""
|
713
|
+
# In enterprise setup, would assume role here
|
714
|
+
# For now, return session with best available profile
|
715
|
+
|
716
|
+
# Priority order for account access
|
717
|
+
profile_priority = ['MANAGEMENT', 'CENTRALISED_OPS', 'BILLING']
|
718
|
+
|
719
|
+
for profile_type in profile_priority:
|
720
|
+
if profile_type in enterprise_profiles:
|
721
|
+
try:
|
722
|
+
session = boto3.Session(profile_name=enterprise_profiles[profile_type])
|
723
|
+
# Verify access
|
724
|
+
sts_client = session.client('sts')
|
725
|
+
identity = sts_client.get_caller_identity()
|
726
|
+
|
727
|
+
if identity['Account'] == account_id:
|
728
|
+
return session
|
729
|
+
except Exception:
|
730
|
+
continue
|
731
|
+
|
732
|
+
# Fallback to current session
|
733
|
+
return self.session
|
734
|
+
|
735
|
+
async def _create_dependency_analysis_for_vpc(self,
|
736
|
+
vpc_id: str,
|
737
|
+
region: str,
|
738
|
+
ec2_client) -> VPCDependencyAnalysis:
|
739
|
+
"""Create comprehensive dependency analysis for a VPC."""
|
740
|
+
try:
|
741
|
+
# Get ENI count
|
742
|
+
eni_response = ec2_client.describe_network_interfaces(
|
743
|
+
Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]
|
744
|
+
)
|
745
|
+
eni_count = len(eni_response.get('NetworkInterfaces', []))
|
746
|
+
|
747
|
+
# Get route tables
|
748
|
+
rt_response = ec2_client.describe_route_tables(
|
749
|
+
Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]
|
750
|
+
)
|
751
|
+
route_tables = [rt['RouteTableId'] for rt in rt_response.get('RouteTables', [])]
|
752
|
+
|
753
|
+
# Get security groups
|
754
|
+
sg_response = ec2_client.describe_security_groups(
|
755
|
+
Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]
|
756
|
+
)
|
757
|
+
security_groups = [sg['GroupId'] for sg in sg_response.get('SecurityGroups', [])]
|
758
|
+
|
759
|
+
# Get internet gateways
|
760
|
+
igw_response = ec2_client.describe_internet_gateways(
|
761
|
+
Filters=[{'Name': 'attachment.vpc-id', 'Values': [vpc_id]}]
|
762
|
+
)
|
763
|
+
internet_gateways = [igw['InternetGatewayId'] for igw in igw_response.get('InternetGateways', [])]
|
764
|
+
|
765
|
+
# Get NAT gateways
|
766
|
+
nat_response = ec2_client.describe_nat_gateways(
|
767
|
+
Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]
|
768
|
+
)
|
769
|
+
nat_gateways = [nat['NatGatewayId'] for nat in nat_response.get('NatGateways', [])]
|
770
|
+
|
771
|
+
# Determine risk level
|
772
|
+
if eni_count == 0 and len(nat_gateways) == 0 and len(internet_gateways) <= 1:
|
773
|
+
risk_level = 'low'
|
774
|
+
elif eni_count == 0:
|
775
|
+
risk_level = 'medium'
|
776
|
+
else:
|
777
|
+
risk_level = 'high'
|
778
|
+
|
779
|
+
return VPCDependencyAnalysis(
|
780
|
+
vpc_id=vpc_id,
|
781
|
+
region=region,
|
782
|
+
eni_count=eni_count,
|
783
|
+
route_tables=route_tables,
|
784
|
+
security_groups=security_groups,
|
785
|
+
internet_gateways=internet_gateways,
|
786
|
+
nat_gateways=nat_gateways,
|
787
|
+
dependency_risk_level=risk_level
|
788
|
+
)
|
789
|
+
|
790
|
+
except Exception as e:
|
791
|
+
print_warning(f"Failed to analyze dependencies for {vpc_id}: {e}")
|
792
|
+
return VPCDependencyAnalysis(
|
793
|
+
vpc_id=vpc_id,
|
794
|
+
region=region,
|
795
|
+
dependency_risk_level='unknown'
|
796
|
+
)
|
797
|
+
|
798
|
+
def _estimate_vpc_monthly_cost(self, vpc: Dict[str, Any]) -> float:
|
799
|
+
"""Estimate monthly cost for VPC resources."""
|
800
|
+
# Base VPC cost estimation (simplified)
|
801
|
+
# In enterprise setup, would integrate with Cost Explorer
|
802
|
+
base_cost = 0.0
|
803
|
+
|
804
|
+
# Default VPCs might have default resources
|
805
|
+
if vpc.get('IsDefault', False):
|
806
|
+
base_cost += 5.0 # Estimated monthly cost for default VPC resources
|
807
|
+
|
808
|
+
return base_cost
|
809
|
+
|
810
|
+
def _extract_vpc_tags(self, vpc: Dict[str, Any]) -> Dict[str, str]:
|
811
|
+
"""Extract tags from VPC data."""
|
812
|
+
tags = {}
|
813
|
+
for tag in vpc.get('Tags', []):
|
814
|
+
tags[tag['Key']] = tag['Value']
|
815
|
+
return tags
|
816
|
+
|
817
|
+
async def _check_flow_logs_enabled(self, vpc_id: str, ec2_client) -> bool:
|
818
|
+
"""Check if VPC Flow Logs are enabled."""
|
819
|
+
try:
|
820
|
+
response = ec2_client.describe_flow_logs(
|
821
|
+
Filters=[
|
822
|
+
{'Name': 'resource-id', 'Values': [vpc_id]},
|
823
|
+
{'Name': 'resource-type', 'Values': ['VPC']}
|
824
|
+
]
|
825
|
+
)
|
826
|
+
return len(response.get('FlowLogs', [])) > 0
|
827
|
+
except Exception:
|
828
|
+
return False
|
829
|
+
|
220
830
|
def _analyze_vpc_dependencies(self, candidates: List[VPCCleanupCandidate]) -> List[VPCCleanupCandidate]:
|
221
831
|
"""Analyze VPC dependencies for cleanup safety assessment."""
|
222
832
|
print_info("🔍 Analyzing VPC dependencies for safety assessment...")
|
@@ -295,6 +905,9 @@ class VPCCleanupOptimizer:
|
|
295
905
|
candidate.dependency_analysis
|
296
906
|
)
|
297
907
|
|
908
|
+
# Enhanced data collection for new fields
|
909
|
+
self._collect_enhanced_vpc_data(candidate, ec2_client)
|
910
|
+
|
298
911
|
analyzed_candidates.append(candidate)
|
299
912
|
|
300
913
|
except ClientError as e:
|
@@ -307,20 +920,140 @@ class VPCCleanupOptimizer:
|
|
307
920
|
return analyzed_candidates
|
308
921
|
|
309
922
|
def _calculate_dependency_risk(self, dependency_analysis: VPCDependencyAnalysis) -> str:
|
310
|
-
"""
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
923
|
+
"""
|
924
|
+
Calculate dependency risk level based on VPC resource analysis.
|
925
|
+
|
926
|
+
CRITICAL FIX: Prioritize ENI count = 0 for safe Bucket 1 classification.
|
927
|
+
NO-ENI VPCs are inherently safe regardless of other infrastructure present.
|
928
|
+
"""
|
929
|
+
# PRIORITY 1: NO-ENI VPCs are inherently low risk (safe for immediate deletion)
|
930
|
+
# This overrides all other factors - if no ENI attachments, no active workloads depend on it
|
931
|
+
if dependency_analysis.eni_count == 0:
|
932
|
+
return "low" # Safe for Bucket 1 - Ready for deletion
|
933
|
+
|
934
|
+
# PRIORITY 2: Default VPCs always require careful handling
|
319
935
|
if dependency_analysis.is_default_vpc:
|
320
|
-
return "high"
|
936
|
+
return "high" # Bucket 3 - Manual review required
|
321
937
|
|
322
|
-
#
|
323
|
-
|
938
|
+
# PRIORITY 3: VPCs with ENI attachments require dependency analysis
|
939
|
+
# These have active workloads and need investigation
|
940
|
+
return "medium" # Bucket 2 - Requires analysis
|
941
|
+
|
942
|
+
def _collect_enhanced_vpc_data(self, candidate: VPCCleanupCandidate, ec2_client) -> None:
|
943
|
+
"""Collect enhanced VPC data for new fields."""
|
944
|
+
try:
|
945
|
+
# Get account ID from session
|
946
|
+
sts_client = self.session.client('sts', region_name=candidate.region)
|
947
|
+
account_info = sts_client.get_caller_identity()
|
948
|
+
candidate.account_id = account_info.get('Account')
|
949
|
+
|
950
|
+
# Detect flow logs
|
951
|
+
candidate.flow_logs_enabled = self._detect_flow_logs(candidate.vpc_id, ec2_client)
|
952
|
+
|
953
|
+
# Detect load balancers
|
954
|
+
candidate.load_balancers = self._detect_load_balancers(candidate.vpc_id, candidate.region)
|
955
|
+
|
956
|
+
# Analyze IaC indicators in tags
|
957
|
+
candidate.iac_detected = self._analyze_iac_tags(candidate.tags)
|
958
|
+
|
959
|
+
# Extract owner information from tags
|
960
|
+
candidate.owners_approvals = self._extract_owners_from_tags(candidate.tags)
|
961
|
+
|
962
|
+
except Exception as e:
|
963
|
+
print_warning(f"Enhanced data collection failed for VPC {candidate.vpc_id}: {e}")
|
964
|
+
|
965
|
+
def _detect_flow_logs(self, vpc_id: str, ec2_client) -> bool:
|
966
|
+
"""Detect if VPC Flow Logs are enabled."""
|
967
|
+
try:
|
968
|
+
response = ec2_client.describe_flow_logs(
|
969
|
+
Filters=[
|
970
|
+
{'Name': 'resource-id', 'Values': [vpc_id]},
|
971
|
+
{'Name': 'resource-type', 'Values': ['VPC']}
|
972
|
+
]
|
973
|
+
)
|
974
|
+
return len(response['FlowLogs']) > 0
|
975
|
+
except Exception as e:
|
976
|
+
print_warning(f"Flow logs detection failed for VPC {vpc_id}: {e}")
|
977
|
+
return False
|
978
|
+
|
979
|
+
def _detect_load_balancers(self, vpc_id: str, region: str) -> List[str]:
|
980
|
+
"""Detect load balancers associated with the VPC."""
|
981
|
+
load_balancers = []
|
982
|
+
|
983
|
+
try:
|
984
|
+
# Check Application/Network Load Balancers (ELBv2)
|
985
|
+
elbv2_client = self.session.client('elbv2', region_name=region)
|
986
|
+
response = elbv2_client.describe_load_balancers()
|
987
|
+
|
988
|
+
for lb in response['LoadBalancers']:
|
989
|
+
if lb.get('VpcId') == vpc_id:
|
990
|
+
load_balancers.append(lb['LoadBalancerArn'])
|
991
|
+
|
992
|
+
except Exception as e:
|
993
|
+
print_warning(f"ELBv2 load balancer detection failed for VPC {vpc_id}: {e}")
|
994
|
+
|
995
|
+
try:
|
996
|
+
# Check Classic Load Balancers (ELB)
|
997
|
+
elb_client = self.session.client('elb', region_name=region)
|
998
|
+
response = elb_client.describe_load_balancers()
|
999
|
+
|
1000
|
+
for lb in response['LoadBalancerDescriptions']:
|
1001
|
+
if lb.get('VPCId') == vpc_id:
|
1002
|
+
load_balancers.append(lb['LoadBalancerName'])
|
1003
|
+
|
1004
|
+
except Exception as e:
|
1005
|
+
print_warning(f"Classic load balancer detection failed for VPC {vpc_id}: {e}")
|
1006
|
+
|
1007
|
+
return load_balancers
|
1008
|
+
|
1009
|
+
def _analyze_iac_tags(self, tags: Dict[str, str]) -> bool:
|
1010
|
+
"""Analyze tags for Infrastructure as Code indicators."""
|
1011
|
+
iac_indicators = [
|
1012
|
+
'terraform', 'cloudformation', 'cdk', 'pulumi', 'ansible',
|
1013
|
+
'created-by', 'managed-by', 'provisioned-by', 'stack-name',
|
1014
|
+
'aws:cloudformation:', 'terraform:', 'cdk:'
|
1015
|
+
]
|
1016
|
+
|
1017
|
+
# Check tag keys and values for IaC indicators
|
1018
|
+
all_tag_text = ' '.join([
|
1019
|
+
f"{key} {value}" for key, value in tags.items()
|
1020
|
+
]).lower()
|
1021
|
+
|
1022
|
+
return any(indicator in all_tag_text for indicator in iac_indicators)
|
1023
|
+
|
1024
|
+
def _extract_owners_from_tags(self, tags: Dict[str, str]) -> List[str]:
|
1025
|
+
"""Extract owner/approval information from tags with enhanced patterns."""
|
1026
|
+
owners = []
|
1027
|
+
|
1028
|
+
# Enhanced owner key patterns - Common AWS tagging patterns
|
1029
|
+
owner_keys = [
|
1030
|
+
'Owner', 'owner', 'OWNER', # Direct owner tags
|
1031
|
+
'CreatedBy', 'createdby', 'Created-By', 'created-by', # Creator tags
|
1032
|
+
'ManagedBy', 'managedby', 'Managed-By', 'managed-by', # Management tags
|
1033
|
+
'Team', 'team', 'TEAM', # Team tags
|
1034
|
+
'Contact', 'contact', 'CONTACT', # Contact tags
|
1035
|
+
'BusinessOwner', 'business-owner', 'business_owner', # Business owner
|
1036
|
+
'TechnicalOwner', 'technical-owner', 'technical_owner', # Technical owner
|
1037
|
+
'Approver', 'approver', 'APPROVER' # Approval tags
|
1038
|
+
]
|
1039
|
+
|
1040
|
+
for key in owner_keys:
|
1041
|
+
if key in tags and tags[key]:
|
1042
|
+
# Split multiple owners if comma-separated
|
1043
|
+
owner_values = [owner.strip() for owner in tags[key].split(',')]
|
1044
|
+
owners.extend(owner_values)
|
1045
|
+
|
1046
|
+
# Format owners with role context if identifiable
|
1047
|
+
formatted_owners = []
|
1048
|
+
for owner in owners:
|
1049
|
+
if any(business_key in owner.lower() for business_key in ['business', 'manager', 'finance']):
|
1050
|
+
formatted_owners.append(f"{owner} (Business)")
|
1051
|
+
elif any(tech_key in owner.lower() for tech_key in ['ops', 'devops', 'engineering', 'tech']):
|
1052
|
+
formatted_owners.append(f"{owner} (Technical)")
|
1053
|
+
else:
|
1054
|
+
formatted_owners.append(owner)
|
1055
|
+
|
1056
|
+
return list(set(formatted_owners)) # Remove duplicates
|
324
1057
|
|
325
1058
|
def _classify_three_bucket_strategy(self, candidates: List[VPCCleanupCandidate]) -> Dict[str, List[VPCCleanupCandidate]]:
|
326
1059
|
"""Classify VPCs using AWSO-05 three-bucket strategy."""
|
@@ -333,25 +1066,24 @@ class VPCCleanupOptimizer:
|
|
333
1066
|
for candidate in candidates:
|
334
1067
|
dependency = candidate.dependency_analysis
|
335
1068
|
|
336
|
-
#
|
337
|
-
|
338
|
-
|
339
|
-
not dependency.is_default_vpc):
|
1069
|
+
# PRIORITY 1: ENI count = 0 takes precedence (safety-first approach)
|
1070
|
+
# NO-ENI VPCs are inherently safe regardless of default status
|
1071
|
+
if dependency.eni_count == 0 and dependency.dependency_risk_level == "low":
|
340
1072
|
candidate.cleanup_bucket = "internal"
|
341
1073
|
candidate.cleanup_recommendation = "ready"
|
342
1074
|
candidate.risk_assessment = "low"
|
343
1075
|
candidate.business_impact = "minimal"
|
344
1076
|
bucket_1_internal.append(candidate)
|
345
1077
|
|
346
|
-
#
|
347
|
-
elif dependency.is_default_vpc:
|
1078
|
+
# PRIORITY 2: Default VPCs with ENI attachments need careful handling
|
1079
|
+
elif dependency.is_default_vpc and dependency.eni_count > 0:
|
348
1080
|
candidate.cleanup_bucket = "control"
|
349
1081
|
candidate.cleanup_recommendation = "manual_review"
|
350
1082
|
candidate.risk_assessment = "high"
|
351
1083
|
candidate.business_impact = "significant"
|
352
1084
|
bucket_3_control.append(candidate)
|
353
1085
|
|
354
|
-
#
|
1086
|
+
# PRIORITY 3: Non-default VPCs with ENI attachments require analysis
|
355
1087
|
else:
|
356
1088
|
candidate.cleanup_bucket = "external"
|
357
1089
|
candidate.cleanup_recommendation = "investigate"
|
@@ -372,13 +1104,82 @@ class VPCCleanupOptimizer:
|
|
372
1104
|
|
373
1105
|
return classification_results
|
374
1106
|
|
1107
|
+
def _ensure_no_eni_bucket_1_classification(self, bucket_classification: Dict[str, List[VPCCleanupCandidate]]) -> Dict[str, List[VPCCleanupCandidate]]:
|
1108
|
+
"""
|
1109
|
+
Ensure NO-ENI VPCs remain in Bucket 1 after security assessment.
|
1110
|
+
|
1111
|
+
CRITICAL FIX: Security assessment may have modified VPC properties, but
|
1112
|
+
NO-ENI VPCs (ENI count = 0) should ALWAYS remain in Bucket 1 regardless
|
1113
|
+
of default status or security findings. They are inherently safe.
|
1114
|
+
"""
|
1115
|
+
print_info("🔧 Ensuring NO-ENI VPCs remain in Bucket 1 (safety-first approach)...")
|
1116
|
+
|
1117
|
+
# Create new bucket structure
|
1118
|
+
new_bucket_1 = []
|
1119
|
+
new_bucket_2 = []
|
1120
|
+
new_bucket_3 = []
|
1121
|
+
|
1122
|
+
# Collect all VPCs from all buckets
|
1123
|
+
all_vpcs = []
|
1124
|
+
for bucket_vpcs in bucket_classification.values():
|
1125
|
+
all_vpcs.extend(bucket_vpcs)
|
1126
|
+
|
1127
|
+
# Re-classify with NO-ENI priority
|
1128
|
+
for candidate in all_vpcs:
|
1129
|
+
# PRIORITY 1: NO-ENI VPCs ALWAYS go to Bucket 1 (overrides all other factors)
|
1130
|
+
if candidate.dependency_analysis.eni_count == 0:
|
1131
|
+
# Ensure NO-ENI VPCs maintain Bucket 1 properties
|
1132
|
+
candidate.cleanup_bucket = "internal"
|
1133
|
+
candidate.cleanup_recommendation = "ready"
|
1134
|
+
candidate.risk_assessment = "low"
|
1135
|
+
candidate.business_impact = "minimal"
|
1136
|
+
new_bucket_1.append(candidate)
|
1137
|
+
|
1138
|
+
# PRIORITY 2: Default VPCs with ENI attachments go to Bucket 3
|
1139
|
+
elif candidate.is_default and candidate.dependency_analysis.eni_count > 0:
|
1140
|
+
candidate.cleanup_bucket = "control"
|
1141
|
+
candidate.cleanup_recommendation = "manual_review"
|
1142
|
+
candidate.risk_assessment = "high"
|
1143
|
+
candidate.business_impact = "significant"
|
1144
|
+
new_bucket_3.append(candidate)
|
1145
|
+
|
1146
|
+
# PRIORITY 3: All other VPCs go to Bucket 2
|
1147
|
+
else:
|
1148
|
+
candidate.cleanup_bucket = "external"
|
1149
|
+
candidate.cleanup_recommendation = "investigate"
|
1150
|
+
candidate.risk_assessment = "medium"
|
1151
|
+
candidate.business_impact = "moderate"
|
1152
|
+
new_bucket_2.append(candidate)
|
1153
|
+
|
1154
|
+
corrected_classification = {
|
1155
|
+
"bucket_1_internal": new_bucket_1,
|
1156
|
+
"bucket_2_external": new_bucket_2,
|
1157
|
+
"bucket_3_control": new_bucket_3
|
1158
|
+
}
|
1159
|
+
|
1160
|
+
# Log corrections if any VPCs were moved
|
1161
|
+
original_b1_count = len(bucket_classification["bucket_1_internal"])
|
1162
|
+
original_b3_count = len(bucket_classification["bucket_3_control"])
|
1163
|
+
new_b1_count = len(new_bucket_1)
|
1164
|
+
new_b3_count = len(new_bucket_3)
|
1165
|
+
|
1166
|
+
if original_b1_count != new_b1_count or original_b3_count != new_b3_count:
|
1167
|
+
print_warning(f"🔧 Bucket re-classification applied:")
|
1168
|
+
print_info(f" • Bucket 1: {original_b1_count} → {new_b1_count} VPCs")
|
1169
|
+
print_info(f" • Bucket 3: {original_b3_count} → {new_b3_count} VPCs")
|
1170
|
+
print_success("✅ NO-ENI VPCs prioritized for Bucket 1 (safety-first)")
|
1171
|
+
else:
|
1172
|
+
print_success("✅ NO-ENI VPC classification already correct")
|
1173
|
+
|
1174
|
+
return corrected_classification
|
1175
|
+
|
375
1176
|
def _perform_vpc_security_assessment(self, candidates: List[VPCCleanupCandidate]) -> Dict[str, Any]:
|
376
1177
|
"""Perform comprehensive VPC security assessment using enterprise security module."""
|
377
1178
|
print_info("🔒 Performing comprehensive VPC security assessment...")
|
378
1179
|
|
379
1180
|
try:
|
380
|
-
# Initialize enterprise security
|
381
|
-
|
1181
|
+
# Initialize enterprise security framework
|
1182
|
+
security_framework = EnterpriseSecurityFramework(profile=self.profile)
|
382
1183
|
|
383
1184
|
security_results = {
|
384
1185
|
"assessed_vpcs": 0,
|
@@ -401,19 +1202,30 @@ class VPCCleanupOptimizer:
|
|
401
1202
|
for candidate in candidates:
|
402
1203
|
try:
|
403
1204
|
# Enhanced security assessment for each VPC
|
404
|
-
vpc_security = self._assess_individual_vpc_security(candidate,
|
1205
|
+
vpc_security = self._assess_individual_vpc_security(candidate, security_framework)
|
405
1206
|
|
406
1207
|
# Classify security risk level
|
1208
|
+
# CRITICAL FIX: Don't override NO-ENI VPC classifications (they're inherently safe)
|
1209
|
+
# NO-ENI VPCs should remain in Bucket 1 regardless of default status
|
407
1210
|
if candidate.is_default or vpc_security["high_risk_findings"] > 2:
|
408
1211
|
security_results["security_risks"]["high_risk"].append(candidate.vpc_id)
|
409
|
-
|
410
|
-
|
1212
|
+
|
1213
|
+
# Only override classification if VPC has ENI attachments
|
1214
|
+
# NO-ENI VPCs (ENI count = 0) remain safe for Bucket 1 regardless of default status
|
1215
|
+
if candidate.dependency_analysis.eni_count > 0:
|
1216
|
+
candidate.risk_assessment = "high"
|
1217
|
+
candidate.cleanup_recommendation = "manual_review"
|
1218
|
+
# NO-ENI VPCs keep their original Bucket 1 classification
|
411
1219
|
elif vpc_security["medium_risk_findings"] > 1:
|
412
1220
|
security_results["security_risks"]["medium_risk"].append(candidate.vpc_id)
|
413
|
-
|
1221
|
+
# Only override classification if VPC has ENI attachments
|
1222
|
+
if candidate.dependency_analysis.eni_count > 0:
|
1223
|
+
candidate.risk_assessment = "medium"
|
414
1224
|
else:
|
415
1225
|
security_results["security_risks"]["low_risk"].append(candidate.vpc_id)
|
416
|
-
|
1226
|
+
# Only override classification if VPC has ENI attachments
|
1227
|
+
if candidate.dependency_analysis.eni_count > 0:
|
1228
|
+
candidate.risk_assessment = "low"
|
417
1229
|
|
418
1230
|
# Track compliance issues
|
419
1231
|
if candidate.is_default:
|
@@ -447,7 +1259,7 @@ class VPCCleanupOptimizer:
|
|
447
1259
|
print_error(f"Security assessment failed: {e}")
|
448
1260
|
return {"error": str(e), "assessed_vpcs": 0}
|
449
1261
|
|
450
|
-
def _assess_individual_vpc_security(self, candidate: VPCCleanupCandidate,
|
1262
|
+
def _assess_individual_vpc_security(self, candidate: VPCCleanupCandidate, security_framework) -> Dict[str, Any]:
|
451
1263
|
"""Assess individual VPC security posture."""
|
452
1264
|
security_findings = {
|
453
1265
|
"high_risk_findings": 0,
|
@@ -505,12 +1317,21 @@ class VPCCleanupOptimizer:
|
|
505
1317
|
"""Calculate VPC cleanup costs and savings estimation."""
|
506
1318
|
print_info("💰 Calculating VPC cleanup costs and savings...")
|
507
1319
|
|
508
|
-
#
|
509
|
-
#
|
1320
|
+
# Dynamic VPC cost calculation (enterprise compliance)
|
1321
|
+
# Using dynamic pricing engine for accurate regional costs
|
1322
|
+
pricing_engine = DynamicAWSPricing()
|
1323
|
+
default_region = 'us-east-1' # Default region for cost estimation
|
1324
|
+
|
510
1325
|
monthly_vpc_base_cost = 0.0 # VPCs themselves are free
|
511
|
-
|
512
|
-
|
513
|
-
|
1326
|
+
nat_result = pricing_engine.get_service_pricing('nat_gateway', default_region)
|
1327
|
+
monthly_nat_gateway_cost = nat_result.monthly_cost
|
1328
|
+
|
1329
|
+
endpoint_result = pricing_engine.get_service_pricing('vpc_endpoint', default_region)
|
1330
|
+
monthly_vpc_endpoint_cost = endpoint_result.monthly_cost
|
1331
|
+
|
1332
|
+
# Data processing costs vary by usage - using conservative estimate per region
|
1333
|
+
regional_multiplier = pricing_engine.regional_multipliers.get(default_region, 1.0)
|
1334
|
+
monthly_data_processing_cost = 50.0 * regional_multiplier # Base estimate adjusted for region
|
514
1335
|
|
515
1336
|
total_annual_savings = 0.0
|
516
1337
|
cost_details = {}
|
@@ -613,7 +1434,16 @@ class VPCCleanupOptimizer:
|
|
613
1434
|
total_annual_savings=cost_analysis["total_annual_savings"],
|
614
1435
|
mcp_validation_accuracy=validation_results["overall_accuracy"],
|
615
1436
|
analysis_timestamp=datetime.now(),
|
616
|
-
security_assessment=security_assessment
|
1437
|
+
security_assessment=security_assessment,
|
1438
|
+
multi_account_context={
|
1439
|
+
'total_accounts_analyzed': 1,
|
1440
|
+
'accounts_with_vpcs': 1 if all_candidates else 0,
|
1441
|
+
'organization_id': 'single_account_analysis',
|
1442
|
+
'accounts': [{'account_id': 'current', 'account_name': 'current', 'status': 'active'}],
|
1443
|
+
'vpc_count_by_account': {'current': len(all_candidates)},
|
1444
|
+
'analysis_scope': 'single_account',
|
1445
|
+
'profile_access_scope': self.profile
|
1446
|
+
}
|
617
1447
|
)
|
618
1448
|
|
619
1449
|
# Generate SHA256 evidence hash for audit compliance
|
@@ -760,6 +1590,9 @@ class VPCCleanupOptimizer:
|
|
760
1590
|
ready_table.add_column("Region", style="blue", width=12)
|
761
1591
|
ready_table.add_column("CIDR Block", style="yellow", width=18)
|
762
1592
|
ready_table.add_column("ENI Count", justify="right", style="green")
|
1593
|
+
ready_table.add_column("Flow Logs", justify="center", style="magenta")
|
1594
|
+
ready_table.add_column("Load Balancers", justify="right", style="red")
|
1595
|
+
ready_table.add_column("IaC", justify="center", style="cyan")
|
763
1596
|
ready_table.add_column("Annual Savings", justify="right", style="green")
|
764
1597
|
|
765
1598
|
for candidate in results.bucket_1_internal[:10]: # Show first 10
|
@@ -768,6 +1601,9 @@ class VPCCleanupOptimizer:
|
|
768
1601
|
candidate.region,
|
769
1602
|
candidate.cidr_block,
|
770
1603
|
str(candidate.dependency_analysis.eni_count),
|
1604
|
+
"✅" if candidate.flow_logs_enabled else "❌",
|
1605
|
+
str(len(candidate.load_balancers)),
|
1606
|
+
"✅" if candidate.iac_detected else "❌",
|
771
1607
|
format_cost(candidate.annual_savings)
|
772
1608
|
)
|
773
1609
|
|
@@ -784,7 +1620,11 @@ class VPCCleanupOptimizer:
|
|
784
1620
|
@click.option('--export', default='json', help='Export format: json, csv, pdf')
|
785
1621
|
@click.option('--evidence-bundle', is_flag=True, help='Generate SHA256 evidence bundle')
|
786
1622
|
@click.option('--dry-run', is_flag=True, default=True, help='Perform analysis only (default: true)')
|
787
|
-
|
1623
|
+
@click.option('--no-eni-only', is_flag=True, help='Show only VPCs with zero ENI attachments')
|
1624
|
+
@click.option('--filter', type=click.Choice(['none', 'default', 'all']), default='all',
|
1625
|
+
help='Filter VPCs: none=no resources, default=default VPCs only, all=show all')
|
1626
|
+
def vpc_cleanup_command(profile: str, export: str, evidence_bundle: bool, dry_run: bool,
|
1627
|
+
no_eni_only: bool, filter: str):
|
788
1628
|
"""
|
789
1629
|
AWSO-05 VPC Cleanup Cost Optimization Engine
|
790
1630
|
|
@@ -798,13 +1638,28 @@ def vpc_cleanup_command(profile: str, export: str, evidence_bundle: bool, dry_ru
|
|
798
1638
|
|
799
1639
|
try:
|
800
1640
|
optimizer = VPCCleanupOptimizer(profile=profile)
|
801
|
-
results = optimizer.analyze_vpc_cleanup_opportunities(
|
1641
|
+
results = optimizer.analyze_vpc_cleanup_opportunities(
|
1642
|
+
no_eni_only=no_eni_only,
|
1643
|
+
filter_type=filter
|
1644
|
+
)
|
802
1645
|
|
803
1646
|
if evidence_bundle:
|
804
1647
|
print_info(f"📁 Evidence bundle generated: SHA256 {results.evidence_hash}")
|
805
1648
|
|
806
1649
|
if export:
|
807
|
-
|
1650
|
+
from .vpc_cleanup_exporter import export_vpc_cleanup_results
|
1651
|
+
export_formats = [format.strip() for format in export.split(',')]
|
1652
|
+
export_results = export_vpc_cleanup_results(
|
1653
|
+
results,
|
1654
|
+
export_formats=export_formats,
|
1655
|
+
output_dir="./tmp"
|
1656
|
+
)
|
1657
|
+
|
1658
|
+
for format_type, filename in export_results.items():
|
1659
|
+
if filename:
|
1660
|
+
print_success(f"📄 {format_type.upper()} export: {filename}")
|
1661
|
+
else:
|
1662
|
+
print_warning(f"⚠️ {format_type.upper()} export failed")
|
808
1663
|
|
809
1664
|
print_success("🎯 AWSO-05 VPC cleanup analysis completed successfully")
|
810
1665
|
|