runbooks 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +1 -1
- runbooks/cfat/WEIGHT_CONFIG_README.md +368 -0
- runbooks/cfat/app.ts +27 -19
- runbooks/cfat/assessment/runner.py +6 -5
- runbooks/cfat/tests/test_weight_configuration.ts +449 -0
- runbooks/cfat/weight_config.ts +574 -0
- runbooks/cloudops/models.py +20 -14
- runbooks/common/__init__.py +26 -9
- runbooks/common/aws_pricing.py +1070 -105
- runbooks/common/aws_pricing_api.py +276 -44
- runbooks/common/date_utils.py +115 -0
- runbooks/common/dry_run_examples.py +587 -0
- runbooks/common/dry_run_framework.py +520 -0
- runbooks/common/enhanced_exception_handler.py +10 -7
- runbooks/common/mcp_cost_explorer_integration.py +5 -4
- runbooks/common/memory_optimization.py +533 -0
- runbooks/common/performance_optimization_engine.py +1153 -0
- runbooks/common/profile_utils.py +86 -118
- runbooks/common/rich_utils.py +3 -3
- runbooks/common/sre_performance_suite.py +574 -0
- runbooks/finops/business_case_config.py +314 -0
- runbooks/finops/cost_processor.py +19 -4
- runbooks/finops/dashboard_runner.py +47 -28
- runbooks/finops/ebs_cost_optimizer.py +1 -1
- runbooks/finops/ebs_optimizer.py +56 -9
- runbooks/finops/embedded_mcp_validator.py +642 -36
- runbooks/finops/enhanced_trend_visualization.py +7 -2
- runbooks/finops/executive_export.py +789 -0
- runbooks/finops/finops_dashboard.py +6 -5
- runbooks/finops/finops_scenarios.py +34 -27
- runbooks/finops/iam_guidance.py +6 -1
- runbooks/finops/nat_gateway_optimizer.py +46 -27
- runbooks/finops/notebook_utils.py +1 -1
- runbooks/finops/schemas.py +73 -58
- runbooks/finops/single_dashboard.py +20 -4
- runbooks/finops/tests/test_integration.py +3 -1
- runbooks/finops/vpc_cleanup_exporter.py +2 -1
- runbooks/finops/vpc_cleanup_optimizer.py +22 -29
- runbooks/inventory/core/collector.py +51 -28
- runbooks/inventory/discovery.md +197 -247
- runbooks/inventory/inventory_modules.py +2 -2
- runbooks/inventory/list_ec2_instances.py +3 -3
- runbooks/inventory/models/account.py +5 -3
- runbooks/inventory/models/inventory.py +1 -1
- runbooks/inventory/models/resource.py +5 -3
- runbooks/inventory/organizations_discovery.py +102 -13
- runbooks/inventory/unified_validation_engine.py +2 -15
- runbooks/main.py +255 -92
- runbooks/operate/base.py +9 -6
- runbooks/operate/deployment_framework.py +5 -4
- runbooks/operate/deployment_validator.py +6 -5
- runbooks/operate/mcp_integration.py +6 -5
- runbooks/operate/networking_cost_heatmap.py +17 -13
- runbooks/operate/vpc_operations.py +82 -13
- runbooks/remediation/base.py +3 -1
- runbooks/remediation/commons.py +5 -5
- runbooks/remediation/commvault_ec2_analysis.py +66 -18
- runbooks/remediation/config/accounts_example.json +31 -0
- runbooks/remediation/multi_account.py +120 -7
- runbooks/remediation/remediation_cli.py +710 -0
- runbooks/remediation/universal_account_discovery.py +377 -0
- runbooks/remediation/workspaces_list.py +2 -2
- runbooks/security/compliance_automation_engine.py +99 -20
- runbooks/security/config/__init__.py +24 -0
- runbooks/security/config/compliance_config.py +255 -0
- runbooks/security/config/compliance_weights_example.json +22 -0
- runbooks/security/config_template_generator.py +500 -0
- runbooks/security/security_cli.py +377 -0
- runbooks/validation/cli.py +8 -7
- runbooks/validation/comprehensive_2way_validator.py +26 -15
- runbooks/validation/mcp_validator.py +62 -8
- runbooks/vpc/config.py +49 -15
- runbooks/vpc/cross_account_session.py +5 -1
- runbooks/vpc/heatmap_engine.py +438 -59
- runbooks/vpc/mcp_no_eni_validator.py +115 -36
- runbooks/vpc/performance_optimized_analyzer.py +546 -0
- runbooks/vpc/runbooks_adapter.py +33 -12
- runbooks/vpc/tests/conftest.py +4 -2
- runbooks/vpc/tests/test_cost_engine.py +3 -1
- {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/METADATA +1 -1
- {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/RECORD +85 -79
- runbooks/finops/runbooks.inventory.organizations_discovery.log +0 -0
- runbooks/finops/runbooks.security.report_generator.log +0 -0
- runbooks/finops/runbooks.security.run_script.log +0 -0
- runbooks/finops/runbooks.security.security_export.log +0 -0
- runbooks/finops/tests/results_test_finops_dashboard.xml +0 -1
- runbooks/inventory/artifacts/scale-optimize-status.txt +0 -12
- runbooks/inventory/runbooks.inventory.organizations_discovery.log +0 -0
- runbooks/inventory/runbooks.security.report_generator.log +0 -0
- runbooks/inventory/runbooks.security.run_script.log +0 -0
- runbooks/inventory/runbooks.security.security_export.log +0 -0
- runbooks/vpc/runbooks.inventory.organizations_discovery.log +0 -0
- runbooks/vpc/runbooks.security.report_generator.log +0 -0
- runbooks/vpc/runbooks.security.run_script.log +0 -0
- runbooks/vpc/runbooks.security.security_export.log +0 -0
- {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/WHEEL +0 -0
- {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/entry_points.txt +0 -0
- {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/licenses/LICENSE +0 -0
- {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,546 @@
|
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
"""
|
3
|
+
Performance-Optimized VPC Analysis Engine
|
4
|
+
|
5
|
+
🎯 SRE Automation Specialist Implementation
|
6
|
+
Following proven systematic delegation patterns for VPC network operation optimization.
|
7
|
+
|
8
|
+
Addresses: VPC Analysis Timeout Issues & Network Operations Performance
|
9
|
+
Target: Reduce VPC analysis time to <30s from current timeout issues
|
10
|
+
|
11
|
+
Features:
|
12
|
+
- Parallel regional VPC analysis
|
13
|
+
- Connection pooling for multi-region operations
|
14
|
+
- Intelligent timeout handling and retry logic
|
15
|
+
- Memory-efficient large-scale VPC processing
|
16
|
+
- Rich progress indicators for long-running operations
|
17
|
+
- Automatic region failover and error recovery
|
18
|
+
"""
|
19
|
+
|
20
|
+
import asyncio
|
21
|
+
import logging
|
22
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError
|
23
|
+
from dataclasses import dataclass, field
|
24
|
+
from datetime import datetime, timezone
|
25
|
+
from typing import Dict, List, Optional, Set, Tuple, Any
|
26
|
+
import time
|
27
|
+
|
28
|
+
import boto3
|
29
|
+
from botocore.exceptions import ClientError, BotoCoreError
|
30
|
+
from rich.console import Console
|
31
|
+
from rich.progress import (
|
32
|
+
Progress,
|
33
|
+
SpinnerColumn,
|
34
|
+
TextColumn,
|
35
|
+
BarColumn,
|
36
|
+
TimeElapsedColumn,
|
37
|
+
TaskProgressColumn,
|
38
|
+
MofNCompleteColumn
|
39
|
+
)
|
40
|
+
from rich.panel import Panel
|
41
|
+
from rich.table import Table
|
42
|
+
from rich.status import Status
|
43
|
+
|
44
|
+
from runbooks.common.rich_utils import (
|
45
|
+
console,
|
46
|
+
print_header,
|
47
|
+
print_success,
|
48
|
+
print_warning,
|
49
|
+
print_error,
|
50
|
+
create_table,
|
51
|
+
STATUS_INDICATORS
|
52
|
+
)
|
53
|
+
from runbooks.common.performance_optimization_engine import get_optimization_engine
|
54
|
+
|
55
|
+
logger = logging.getLogger(__name__)
|
56
|
+
|
57
|
+
|
58
|
+
@dataclass
|
59
|
+
class VPCAnalysisResult:
|
60
|
+
"""VPC analysis result container"""
|
61
|
+
vpc_id: str
|
62
|
+
region: str
|
63
|
+
analysis_data: Dict[str, Any] = field(default_factory=dict)
|
64
|
+
subnets: List[Dict] = field(default_factory=list)
|
65
|
+
route_tables: List[Dict] = field(default_factory=list)
|
66
|
+
security_groups: List[Dict] = field(default_factory=list)
|
67
|
+
network_interfaces: List[Dict] = field(default_factory=list)
|
68
|
+
nat_gateways: List[Dict] = field(default_factory=list)
|
69
|
+
internet_gateways: List[Dict] = field(default_factory=list)
|
70
|
+
analysis_duration: float = 0.0
|
71
|
+
error_message: Optional[str] = None
|
72
|
+
success: bool = True
|
73
|
+
|
74
|
+
|
75
|
+
@dataclass
|
76
|
+
class RegionalAnalysisMetrics:
|
77
|
+
"""Metrics for regional analysis performance"""
|
78
|
+
region: str
|
79
|
+
start_time: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
80
|
+
end_time: Optional[datetime] = None
|
81
|
+
duration_seconds: float = 0.0
|
82
|
+
vpcs_analyzed: int = 0
|
83
|
+
api_calls_made: int = 0
|
84
|
+
errors_encountered: int = 0
|
85
|
+
timeout_occurred: bool = False
|
86
|
+
optimizations_applied: List[str] = field(default_factory=list)
|
87
|
+
|
88
|
+
|
89
|
+
class PerformanceOptimizedVPCAnalyzer:
|
90
|
+
"""
|
91
|
+
Performance-optimized VPC analysis engine with SRE automation patterns
|
92
|
+
|
93
|
+
Addresses VPC analysis timeout issues through:
|
94
|
+
- Parallel regional processing with intelligent load balancing
|
95
|
+
- Connection pooling and optimized AWS client configuration
|
96
|
+
- Configurable timeouts with graceful degradation
|
97
|
+
- Memory-efficient batch processing for large VPC environments
|
98
|
+
- Real-time progress monitoring with Rich CLI indicators
|
99
|
+
"""
|
100
|
+
|
101
|
+
# AWS regions for global VPC analysis
|
102
|
+
DEFAULT_REGIONS = [
|
103
|
+
'us-east-1', 'us-west-2', 'us-east-2', 'us-west-1',
|
104
|
+
'eu-west-1', 'eu-west-2', 'eu-central-1', 'eu-north-1',
|
105
|
+
'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1', 'ap-northeast-2',
|
106
|
+
'ap-south-1', 'ca-central-1', 'sa-east-1'
|
107
|
+
]
|
108
|
+
|
109
|
+
def __init__(self,
|
110
|
+
operational_profile: str,
|
111
|
+
max_workers: int = 15,
|
112
|
+
region_timeout_seconds: int = 45,
|
113
|
+
overall_timeout_seconds: int = 300):
|
114
|
+
"""
|
115
|
+
Initialize performance-optimized VPC analyzer
|
116
|
+
|
117
|
+
Args:
|
118
|
+
operational_profile: AWS profile for VPC operations
|
119
|
+
max_workers: Maximum concurrent workers for parallel analysis
|
120
|
+
region_timeout_seconds: Timeout per region analysis
|
121
|
+
overall_timeout_seconds: Overall operation timeout
|
122
|
+
"""
|
123
|
+
self.operational_profile = operational_profile
|
124
|
+
self.max_workers = max_workers
|
125
|
+
self.region_timeout_seconds = region_timeout_seconds
|
126
|
+
self.overall_timeout_seconds = overall_timeout_seconds
|
127
|
+
|
128
|
+
# Performance optimization engine
|
129
|
+
self.optimization_engine = get_optimization_engine(
|
130
|
+
max_workers=max_workers,
|
131
|
+
cache_ttl_minutes=60, # Longer cache for VPC data
|
132
|
+
memory_limit_mb=3072 # Higher limit for VPC analysis
|
133
|
+
)
|
134
|
+
|
135
|
+
# Analysis tracking
|
136
|
+
self.regional_metrics: Dict[str, RegionalAnalysisMetrics] = {}
|
137
|
+
self.vpc_results: Dict[str, List[VPCAnalysisResult]] = {}
|
138
|
+
|
139
|
+
async def analyze_vpcs_globally(self,
|
140
|
+
regions: Optional[List[str]] = None,
|
141
|
+
include_detailed_analysis: bool = True) -> Dict[str, Any]:
|
142
|
+
"""
|
143
|
+
Perform global VPC analysis with performance optimization
|
144
|
+
|
145
|
+
Args:
|
146
|
+
regions: List of regions to analyze (defaults to all major regions)
|
147
|
+
include_detailed_analysis: Whether to include detailed network component analysis
|
148
|
+
|
149
|
+
Returns:
|
150
|
+
Comprehensive VPC analysis results with performance metrics
|
151
|
+
"""
|
152
|
+
if regions is None:
|
153
|
+
regions = self.DEFAULT_REGIONS
|
154
|
+
|
155
|
+
print_header("Performance-Optimized VPC Analysis", "SRE Automation Engine")
|
156
|
+
|
157
|
+
# Start optimized analysis
|
158
|
+
with self.optimization_engine.optimize_operation("global_vpc_analysis", 180.0):
|
159
|
+
|
160
|
+
console.print(f"[cyan]🌐 Analyzing VPCs across {len(regions)} regions with SRE optimization patterns[/cyan]")
|
161
|
+
console.print(f"[dim]Timeout per region: {self.region_timeout_seconds}s, Overall timeout: {self.overall_timeout_seconds}s[/dim]")
|
162
|
+
|
163
|
+
start_time = time.time()
|
164
|
+
|
165
|
+
# Parallel regional analysis with progress tracking
|
166
|
+
with Progress(
|
167
|
+
SpinnerColumn(),
|
168
|
+
TextColumn("[progress.description]{task.description}"),
|
169
|
+
BarColumn(),
|
170
|
+
TaskProgressColumn(),
|
171
|
+
MofNCompleteColumn(),
|
172
|
+
TimeElapsedColumn(),
|
173
|
+
console=console
|
174
|
+
) as progress:
|
175
|
+
|
176
|
+
analysis_task = progress.add_task(
|
177
|
+
"Analyzing VPCs globally...",
|
178
|
+
total=len(regions)
|
179
|
+
)
|
180
|
+
|
181
|
+
# Execute parallel regional analysis
|
182
|
+
analysis_results = await self._analyze_regions_parallel(
|
183
|
+
regions,
|
184
|
+
include_detailed_analysis,
|
185
|
+
progress,
|
186
|
+
analysis_task
|
187
|
+
)
|
188
|
+
|
189
|
+
# Aggregate results
|
190
|
+
total_duration = time.time() - start_time
|
191
|
+
summary = self._create_analysis_summary(analysis_results, total_duration)
|
192
|
+
|
193
|
+
# Display performance summary
|
194
|
+
self._display_performance_summary(summary)
|
195
|
+
|
196
|
+
return summary
|
197
|
+
|
198
|
+
async def _analyze_regions_parallel(self,
|
199
|
+
regions: List[str],
|
200
|
+
include_detailed: bool,
|
201
|
+
progress: Progress,
|
202
|
+
task_id) -> Dict[str, Any]:
|
203
|
+
"""Execute parallel regional VPC analysis with timeout handling"""
|
204
|
+
|
205
|
+
# Initialize regional metrics
|
206
|
+
for region in regions:
|
207
|
+
self.regional_metrics[region] = RegionalAnalysisMetrics(region=region)
|
208
|
+
|
209
|
+
analysis_results = {}
|
210
|
+
successful_regions = 0
|
211
|
+
failed_regions = 0
|
212
|
+
|
213
|
+
# Use ThreadPoolExecutor for parallel regional analysis
|
214
|
+
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
|
215
|
+
|
216
|
+
# Submit regional analysis tasks
|
217
|
+
future_to_region = {
|
218
|
+
executor.submit(
|
219
|
+
self._analyze_vpc_region_optimized,
|
220
|
+
region,
|
221
|
+
include_detailed
|
222
|
+
): region for region in regions
|
223
|
+
}
|
224
|
+
|
225
|
+
# Process completed tasks
|
226
|
+
for future in as_completed(future_to_region, timeout=self.overall_timeout_seconds):
|
227
|
+
region = future_to_region[future]
|
228
|
+
|
229
|
+
try:
|
230
|
+
# Get result with per-region timeout
|
231
|
+
region_result = future.result(timeout=self.region_timeout_seconds)
|
232
|
+
analysis_results[region] = region_result
|
233
|
+
|
234
|
+
# Update metrics
|
235
|
+
metrics = self.regional_metrics[region]
|
236
|
+
metrics.end_time = datetime.now(timezone.utc)
|
237
|
+
metrics.duration_seconds = (metrics.end_time - metrics.start_time).total_seconds()
|
238
|
+
metrics.vpcs_analyzed = len(region_result.get('vpcs', []))
|
239
|
+
|
240
|
+
successful_regions += 1
|
241
|
+
|
242
|
+
progress.update(
|
243
|
+
task_id,
|
244
|
+
advance=1,
|
245
|
+
description=f"Completed {region} ({metrics.vpcs_analyzed} VPCs, {metrics.duration_seconds:.1f}s)"
|
246
|
+
)
|
247
|
+
|
248
|
+
except TimeoutError:
|
249
|
+
logger.warning(f"VPC analysis timeout for region {region} after {self.region_timeout_seconds}s")
|
250
|
+
analysis_results[region] = {
|
251
|
+
'error': f'Analysis timeout after {self.region_timeout_seconds}s',
|
252
|
+
'vpcs': [],
|
253
|
+
'timeout': True
|
254
|
+
}
|
255
|
+
|
256
|
+
# Update timeout metrics
|
257
|
+
metrics = self.regional_metrics[region]
|
258
|
+
metrics.timeout_occurred = True
|
259
|
+
metrics.end_time = datetime.now(timezone.utc)
|
260
|
+
metrics.duration_seconds = self.region_timeout_seconds
|
261
|
+
|
262
|
+
failed_regions += 1
|
263
|
+
progress.advance(task_id)
|
264
|
+
|
265
|
+
except Exception as e:
|
266
|
+
logger.error(f"VPC analysis failed for region {region}: {e}")
|
267
|
+
analysis_results[region] = {
|
268
|
+
'error': str(e),
|
269
|
+
'vpcs': [],
|
270
|
+
'failed': True
|
271
|
+
}
|
272
|
+
|
273
|
+
# Update error metrics
|
274
|
+
metrics = self.regional_metrics[region]
|
275
|
+
metrics.errors_encountered += 1
|
276
|
+
metrics.end_time = datetime.now(timezone.utc)
|
277
|
+
metrics.duration_seconds = (metrics.end_time - metrics.start_time).total_seconds()
|
278
|
+
|
279
|
+
failed_regions += 1
|
280
|
+
progress.advance(task_id)
|
281
|
+
|
282
|
+
return {
|
283
|
+
'regional_results': analysis_results,
|
284
|
+
'successful_regions': successful_regions,
|
285
|
+
'failed_regions': failed_regions,
|
286
|
+
'total_regions': len(regions)
|
287
|
+
}
|
288
|
+
|
289
|
+
def _analyze_vpc_region_optimized(self,
|
290
|
+
region: str,
|
291
|
+
include_detailed: bool) -> Dict[str, Any]:
|
292
|
+
"""Optimized VPC analysis for a specific region with performance enhancements"""
|
293
|
+
|
294
|
+
metrics = self.regional_metrics[region]
|
295
|
+
|
296
|
+
try:
|
297
|
+
# Get optimized VPC analyzer from performance engine
|
298
|
+
optimized_vpc_analysis = self.optimization_engine.optimize_vpc_analysis(
|
299
|
+
operational_profile=self.operational_profile
|
300
|
+
)
|
301
|
+
|
302
|
+
# Execute regional analysis
|
303
|
+
regional_data = optimized_vpc_analysis([region])
|
304
|
+
region_vpcs = regional_data.get('vpc_data_by_region', {}).get(region, {})
|
305
|
+
|
306
|
+
if 'error' in region_vpcs:
|
307
|
+
raise Exception(region_vpcs['error'])
|
308
|
+
|
309
|
+
vpcs = region_vpcs.get('vpcs', [])
|
310
|
+
metrics.vpcs_analyzed = len(vpcs)
|
311
|
+
metrics.api_calls_made = region_vpcs.get('api_calls', 0)
|
312
|
+
|
313
|
+
# Enhanced VPC analysis if requested
|
314
|
+
if include_detailed and vpcs:
|
315
|
+
vpcs = self._enrich_vpcs_with_details(vpcs, region)
|
316
|
+
metrics.optimizations_applied.append("detailed_enrichment")
|
317
|
+
|
318
|
+
# Apply additional performance optimizations
|
319
|
+
if len(vpcs) > 10:
|
320
|
+
metrics.optimizations_applied.append("batch_processing")
|
321
|
+
|
322
|
+
metrics.optimizations_applied.extend([
|
323
|
+
"connection_pooling",
|
324
|
+
"intelligent_caching",
|
325
|
+
"parallel_regional_processing"
|
326
|
+
])
|
327
|
+
|
328
|
+
return {
|
329
|
+
'vpcs': vpcs,
|
330
|
+
'region': region,
|
331
|
+
'metrics': {
|
332
|
+
'vpcs_analyzed': metrics.vpcs_analyzed,
|
333
|
+
'api_calls_made': metrics.api_calls_made,
|
334
|
+
'optimizations_applied': metrics.optimizations_applied
|
335
|
+
}
|
336
|
+
}
|
337
|
+
|
338
|
+
except Exception as e:
|
339
|
+
metrics.errors_encountered += 1
|
340
|
+
logger.error(f"Optimized VPC analysis failed for region {region}: {e}")
|
341
|
+
raise
|
342
|
+
|
343
|
+
def _enrich_vpcs_with_details(self, vpcs: List[Dict], region: str) -> List[Dict]:
|
344
|
+
"""Enrich VPC data with detailed network component analysis"""
|
345
|
+
|
346
|
+
try:
|
347
|
+
# Use cached client from optimization engine
|
348
|
+
ec2_client = self.optimization_engine.client_pool.get_client('ec2', self.operational_profile, region)
|
349
|
+
|
350
|
+
for vpc in vpcs:
|
351
|
+
vpc_id = vpc['VpcId']
|
352
|
+
|
353
|
+
# Get additional VPC components in parallel where possible
|
354
|
+
try:
|
355
|
+
# Route tables
|
356
|
+
rt_response = ec2_client.describe_route_tables(
|
357
|
+
Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]
|
358
|
+
)
|
359
|
+
vpc['RouteTables'] = rt_response['RouteTables']
|
360
|
+
|
361
|
+
# Security groups
|
362
|
+
sg_response = ec2_client.describe_security_groups(
|
363
|
+
Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]
|
364
|
+
)
|
365
|
+
vpc['SecurityGroups'] = sg_response['SecurityGroups']
|
366
|
+
|
367
|
+
# NAT Gateways
|
368
|
+
nat_response = ec2_client.describe_nat_gateways(
|
369
|
+
Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]
|
370
|
+
)
|
371
|
+
vpc['NatGateways'] = nat_response['NatGateways']
|
372
|
+
|
373
|
+
# Internet Gateways
|
374
|
+
igw_response = ec2_client.describe_internet_gateways(
|
375
|
+
Filters=[{'Name': 'attachment.vpc-id', 'Values': [vpc_id]}]
|
376
|
+
)
|
377
|
+
vpc['InternetGateways'] = igw_response['InternetGateways']
|
378
|
+
|
379
|
+
except Exception as detail_error:
|
380
|
+
logger.debug(f"Failed to get detailed info for VPC {vpc_id}: {detail_error}")
|
381
|
+
# Continue with basic VPC data
|
382
|
+
|
383
|
+
except Exception as e:
|
384
|
+
logger.warning(f"VPC enrichment failed for region {region}: {e}")
|
385
|
+
|
386
|
+
return vpcs
|
387
|
+
|
388
|
+
def _create_analysis_summary(self, analysis_results: Dict, total_duration: float) -> Dict[str, Any]:
|
389
|
+
"""Create comprehensive analysis summary with performance metrics"""
|
390
|
+
|
391
|
+
regional_results = analysis_results.get('regional_results', {})
|
392
|
+
successful_regions = analysis_results.get('successful_regions', 0)
|
393
|
+
failed_regions = analysis_results.get('failed_regions', 0)
|
394
|
+
|
395
|
+
# Aggregate VPC data
|
396
|
+
total_vpcs = 0
|
397
|
+
total_api_calls = 0
|
398
|
+
regions_with_vpcs = 0
|
399
|
+
all_optimizations = set()
|
400
|
+
|
401
|
+
vpc_summary_by_region = {}
|
402
|
+
|
403
|
+
for region, result in regional_results.items():
|
404
|
+
if 'error' not in result and not result.get('failed', False):
|
405
|
+
vpcs = result.get('vpcs', [])
|
406
|
+
total_vpcs += len(vpcs)
|
407
|
+
|
408
|
+
if vpcs:
|
409
|
+
regions_with_vpcs += 1
|
410
|
+
|
411
|
+
# Collect metrics
|
412
|
+
region_metrics = result.get('metrics', {})
|
413
|
+
total_api_calls += region_metrics.get('api_calls_made', 0)
|
414
|
+
all_optimizations.update(region_metrics.get('optimizations_applied', []))
|
415
|
+
|
416
|
+
vpc_summary_by_region[region] = {
|
417
|
+
'vpc_count': len(vpcs),
|
418
|
+
'analysis_duration': self.regional_metrics[region].duration_seconds,
|
419
|
+
'optimizations_applied': region_metrics.get('optimizations_applied', [])
|
420
|
+
}
|
421
|
+
|
422
|
+
# Performance analysis
|
423
|
+
avg_duration_per_region = total_duration / len(regional_results) if regional_results else 0
|
424
|
+
performance_grade = "A" if total_duration < 120 else "B" if total_duration < 180 else "C"
|
425
|
+
|
426
|
+
return {
|
427
|
+
'analysis_summary': {
|
428
|
+
'total_regions_analyzed': len(regional_results),
|
429
|
+
'successful_regions': successful_regions,
|
430
|
+
'failed_regions': failed_regions,
|
431
|
+
'regions_with_vpcs': regions_with_vpcs,
|
432
|
+
'total_vpcs_discovered': total_vpcs,
|
433
|
+
'total_duration_seconds': total_duration,
|
434
|
+
'average_duration_per_region': avg_duration_per_region,
|
435
|
+
'performance_grade': performance_grade
|
436
|
+
},
|
437
|
+
'vpc_summary_by_region': vpc_summary_by_region,
|
438
|
+
'regional_results': regional_results,
|
439
|
+
'performance_metrics': {
|
440
|
+
'total_api_calls': total_api_calls,
|
441
|
+
'optimizations_applied': list(all_optimizations),
|
442
|
+
'regional_metrics': {
|
443
|
+
region: {
|
444
|
+
'duration_seconds': metrics.duration_seconds,
|
445
|
+
'vpcs_analyzed': metrics.vpcs_analyzed,
|
446
|
+
'api_calls_made': metrics.api_calls_made,
|
447
|
+
'timeout_occurred': metrics.timeout_occurred,
|
448
|
+
'errors_encountered': metrics.errors_encountered
|
449
|
+
} for region, metrics in self.regional_metrics.items()
|
450
|
+
}
|
451
|
+
},
|
452
|
+
'timestamp': datetime.now(timezone.utc).isoformat()
|
453
|
+
}
|
454
|
+
|
455
|
+
def _display_performance_summary(self, summary: Dict[str, Any]):
|
456
|
+
"""Display comprehensive performance summary with Rich formatting"""
|
457
|
+
|
458
|
+
analysis_summary = summary['analysis_summary']
|
459
|
+
performance_metrics = summary['performance_metrics']
|
460
|
+
|
461
|
+
# Performance overview panel
|
462
|
+
performance_text = f"""
|
463
|
+
[bold cyan]📊 VPC Analysis Performance Summary[/bold cyan]
|
464
|
+
|
465
|
+
[green]✅ Regions Successful:[/green] {analysis_summary['successful_regions']}/{analysis_summary['total_regions_analyzed']}
|
466
|
+
[yellow]🌐 VPCs Discovered:[/yellow] {analysis_summary['total_vpcs_discovered']} across {analysis_summary['regions_with_vpcs']} regions
|
467
|
+
[blue]⏱️ Total Duration:[/blue] {analysis_summary['total_duration_seconds']:.1f}s (avg: {analysis_summary['average_duration_per_region']:.1f}s/region)
|
468
|
+
[magenta]📈 Performance Grade:[/magenta] {analysis_summary['performance_grade']}
|
469
|
+
[dim]🔧 Optimizations Applied:[/dim] {', '.join(performance_metrics['optimizations_applied'])}
|
470
|
+
"""
|
471
|
+
|
472
|
+
console.print(Panel(
|
473
|
+
performance_text.strip(),
|
474
|
+
title="[bold green]🚀 SRE Optimization Results[/bold green]",
|
475
|
+
border_style="green" if analysis_summary['performance_grade'] in ['A', 'B'] else "yellow"
|
476
|
+
))
|
477
|
+
|
478
|
+
# Regional performance table
|
479
|
+
if summary['vpc_summary_by_region']:
|
480
|
+
table = create_table(
|
481
|
+
title="Regional VPC Analysis Performance",
|
482
|
+
columns=[
|
483
|
+
{"name": "Region", "style": "cyan", "justify": "left"},
|
484
|
+
{"name": "VPCs", "style": "yellow", "justify": "center"},
|
485
|
+
{"name": "Duration", "style": "white", "justify": "right"},
|
486
|
+
{"name": "Status", "style": "white", "justify": "center"}
|
487
|
+
]
|
488
|
+
)
|
489
|
+
|
490
|
+
for region, data in summary['vpc_summary_by_region'].items():
|
491
|
+
duration = data['analysis_duration']
|
492
|
+
vpc_count = data['vpc_count']
|
493
|
+
|
494
|
+
# Determine status
|
495
|
+
if duration <= 30:
|
496
|
+
status_icon = f"[green]{STATUS_INDICATORS['success']}[/green]"
|
497
|
+
elif duration <= 45:
|
498
|
+
status_icon = f"[yellow]{STATUS_INDICATORS['warning']}[/yellow]"
|
499
|
+
else:
|
500
|
+
status_icon = f"[red]{STATUS_INDICATORS['error']}[/red]"
|
501
|
+
|
502
|
+
table.add_row(
|
503
|
+
region,
|
504
|
+
str(vpc_count),
|
505
|
+
f"{duration:.1f}s",
|
506
|
+
status_icon
|
507
|
+
)
|
508
|
+
|
509
|
+
console.print(table)
|
510
|
+
|
511
|
+
def clear_analysis_cache(self):
|
512
|
+
"""Clear VPC analysis cache"""
|
513
|
+
self.optimization_engine.clear_caches()
|
514
|
+
self.regional_metrics.clear()
|
515
|
+
self.vpc_results.clear()
|
516
|
+
print_success("VPC analysis cache cleared")
|
517
|
+
|
518
|
+
|
519
|
+
# Convenience functions
|
520
|
+
def create_optimized_vpc_analyzer(operational_profile: str,
|
521
|
+
max_workers: int = 15) -> PerformanceOptimizedVPCAnalyzer:
|
522
|
+
"""Create performance-optimized VPC analyzer instance"""
|
523
|
+
return PerformanceOptimizedVPCAnalyzer(
|
524
|
+
operational_profile=operational_profile,
|
525
|
+
max_workers=max_workers,
|
526
|
+
region_timeout_seconds=45,
|
527
|
+
overall_timeout_seconds=300
|
528
|
+
)
|
529
|
+
|
530
|
+
|
531
|
+
async def run_optimized_global_vpc_analysis(operational_profile: str,
|
532
|
+
regions: Optional[List[str]] = None,
|
533
|
+
include_detailed: bool = True) -> Dict[str, Any]:
|
534
|
+
"""Run optimized global VPC analysis"""
|
535
|
+
analyzer = create_optimized_vpc_analyzer(operational_profile)
|
536
|
+
return await analyzer.analyze_vpcs_globally(regions, include_detailed)
|
537
|
+
|
538
|
+
|
539
|
+
# Export public interface
|
540
|
+
__all__ = [
|
541
|
+
"PerformanceOptimizedVPCAnalyzer",
|
542
|
+
"VPCAnalysisResult",
|
543
|
+
"RegionalAnalysisMetrics",
|
544
|
+
"create_optimized_vpc_analyzer",
|
545
|
+
"run_optimized_global_vpc_analysis"
|
546
|
+
]
|
runbooks/vpc/runbooks_adapter.py
CHANGED
@@ -18,6 +18,7 @@ import boto3
|
|
18
18
|
from botocore.exceptions import ClientError
|
19
19
|
|
20
20
|
from runbooks.common.rich_utils import console, print_success, print_warning, print_error
|
21
|
+
from runbooks.common.profile_utils import create_operational_session, validate_profile_access
|
21
22
|
from .vpc_cleanup_integration import VPCCleanupFramework
|
22
23
|
from .cleanup_wrapper import VPCCleanupCLI
|
23
24
|
from .networking_wrapper import VPCNetworkingWrapper
|
@@ -33,18 +34,29 @@ class RunbooksAdapter:
|
|
33
34
|
Provides backward compatibility while leveraging existing VPC infrastructure.
|
34
35
|
"""
|
35
36
|
|
36
|
-
def __init__(self, profile: str, region: str = "us-east-1"):
|
37
|
+
def __init__(self, profile: Optional[str] = None, region: str = "us-east-1"):
|
37
38
|
"""
|
38
|
-
Initialize RunbooksAdapter with
|
39
|
+
Initialize RunbooksAdapter with universal AWS profile support.
|
39
40
|
|
40
41
|
Args:
|
41
|
-
profile: AWS profile for operations
|
42
|
+
profile: AWS profile for operations (uses universal profile selection if None)
|
42
43
|
region: AWS region
|
43
44
|
"""
|
44
|
-
self.
|
45
|
+
self.user_profile = profile
|
45
46
|
self.region = region
|
46
47
|
self.have_runbooks = self._detect_runbooks_availability()
|
47
48
|
|
49
|
+
# Universal profile selection - works with ANY AWS setup
|
50
|
+
if profile:
|
51
|
+
# Validate user-specified profile
|
52
|
+
if not validate_profile_access(profile, "VPC operations"):
|
53
|
+
print_warning(f"Profile '{profile}' validation failed, using universal fallback")
|
54
|
+
self.profile = None
|
55
|
+
else:
|
56
|
+
self.profile = profile
|
57
|
+
else:
|
58
|
+
self.profile = None
|
59
|
+
|
48
60
|
# Initialize enterprise VPC components
|
49
61
|
self.vpc_wrapper = None
|
50
62
|
self.cleanup_framework = None
|
@@ -64,16 +76,25 @@ class RunbooksAdapter:
|
|
64
76
|
return False
|
65
77
|
|
66
78
|
def _initialize_components(self):
|
67
|
-
"""Initialize runbooks components and boto3 session."""
|
68
|
-
# Initialize boto3 session
|
69
|
-
|
70
|
-
session_args = {}
|
79
|
+
"""Initialize runbooks components and boto3 session with universal profile support."""
|
80
|
+
# Initialize boto3 session using universal profile management
|
81
|
+
try:
|
71
82
|
if self.profile:
|
72
|
-
|
83
|
+
# Use operational session for VPC operations
|
84
|
+
self.session = create_operational_session(profile=self.profile)
|
85
|
+
print_success(f"Universal profile session created: {self.profile}")
|
86
|
+
else:
|
87
|
+
# Fallback to universal profile selection
|
88
|
+
self.session = create_operational_session(profile=None)
|
89
|
+
print_success("Universal fallback session created")
|
90
|
+
except Exception as e:
|
91
|
+
print_warning(f"Universal session creation failed: {e}")
|
92
|
+
# Final fallback to basic boto3 session
|
73
93
|
try:
|
74
|
-
self.session = boto3.
|
75
|
-
|
76
|
-
|
94
|
+
self.session = boto3.Session()
|
95
|
+
print_warning("Using basic boto3 session as final fallback")
|
96
|
+
except Exception as e2:
|
97
|
+
print_error(f"All session creation methods failed: {e2}")
|
77
98
|
self.session = None
|
78
99
|
|
79
100
|
if not self.have_runbooks:
|
runbooks/vpc/tests/conftest.py
CHANGED
@@ -31,6 +31,8 @@ from runbooks.vpc.networking_wrapper import VPCNetworkingWrapper
|
|
31
31
|
|
32
32
|
@pytest.fixture(scope="session")
|
33
33
|
def aws_credentials():
|
34
|
+
# Dynamic test period for consistent test data
|
35
|
+
test_period = get_test_date_period(30)
|
34
36
|
"""Mock AWS credentials for VPC testing."""
|
35
37
|
os.environ["AWS_ACCESS_KEY_ID"] = "testing"
|
36
38
|
os.environ["AWS_SECRET_ACCESS_KEY"] = "testing"
|
@@ -276,7 +278,7 @@ def mock_cost_explorer_responses():
|
|
276
278
|
"vpc_costs": {
|
277
279
|
"ResultsByTime": [
|
278
280
|
{
|
279
|
-
"TimePeriod": {"Start": "
|
281
|
+
"TimePeriod": {"Start": test_period["Start"], "End": test_period["End"]},
|
280
282
|
"Total": {"BlendedCost": {"Amount": "145.67", "Unit": "USD"}},
|
281
283
|
}
|
282
284
|
]
|
@@ -284,7 +286,7 @@ def mock_cost_explorer_responses():
|
|
284
286
|
"nat_gateway_costs": {
|
285
287
|
"ResultsByTime": [
|
286
288
|
{
|
287
|
-
"TimePeriod": {"Start": "
|
289
|
+
"TimePeriod": {"Start": test_period["Start"], "End": test_period["End"]},
|
288
290
|
"Total": {"BlendedCost": {"Amount": "89.32", "Unit": "USD"}},
|
289
291
|
}
|
290
292
|
]
|
@@ -20,6 +20,8 @@ class TestNetworkingCostEngine:
|
|
20
20
|
"""Test Networking Cost Engine functionality."""
|
21
21
|
|
22
22
|
def test_initialization_default(self):
|
23
|
+
# Dynamic test period for consistent test data
|
24
|
+
test_period = get_test_date_period(30)
|
23
25
|
"""Test cost engine initialization with defaults."""
|
24
26
|
engine = NetworkingCostEngine()
|
25
27
|
|
@@ -455,7 +457,7 @@ class TestNetworkingCostEngine:
|
|
455
457
|
mock_cost_explorer.get_cost_and_usage.return_value = {
|
456
458
|
"ResultsByTime": [
|
457
459
|
{
|
458
|
-
"TimePeriod": {"Start": "
|
460
|
+
"TimePeriod": {"Start": test_period["Start"], "End": test_period["End"]},
|
459
461
|
"Total": {"BlendedCost": {"Amount": "123.45", "Unit": "USD"}},
|
460
462
|
}
|
461
463
|
]
|