runbooks 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +31 -2
- runbooks/__init___optimized.py +18 -4
- runbooks/_platform/__init__.py +1 -5
- runbooks/_platform/core/runbooks_wrapper.py +141 -138
- runbooks/aws2/accuracy_validator.py +812 -0
- runbooks/base.py +7 -0
- runbooks/cfat/WEIGHT_CONFIG_README.md +1 -1
- runbooks/cfat/assessment/compliance.py +8 -8
- runbooks/cfat/assessment/runner.py +1 -0
- runbooks/cfat/cloud_foundations_assessment.py +227 -239
- runbooks/cfat/models.py +6 -2
- runbooks/cfat/tests/__init__.py +6 -1
- runbooks/cli/__init__.py +13 -0
- runbooks/cli/commands/cfat.py +274 -0
- runbooks/cli/commands/finops.py +1164 -0
- runbooks/cli/commands/inventory.py +379 -0
- runbooks/cli/commands/operate.py +239 -0
- runbooks/cli/commands/security.py +248 -0
- runbooks/cli/commands/validation.py +825 -0
- runbooks/cli/commands/vpc.py +310 -0
- runbooks/cli/registry.py +107 -0
- runbooks/cloudops/__init__.py +23 -30
- runbooks/cloudops/base.py +96 -107
- runbooks/cloudops/cost_optimizer.py +549 -547
- runbooks/cloudops/infrastructure_optimizer.py +5 -4
- runbooks/cloudops/interfaces.py +226 -227
- runbooks/cloudops/lifecycle_manager.py +5 -4
- runbooks/cloudops/mcp_cost_validation.py +252 -235
- runbooks/cloudops/models.py +78 -53
- runbooks/cloudops/monitoring_automation.py +5 -4
- runbooks/cloudops/notebook_framework.py +179 -215
- runbooks/cloudops/security_enforcer.py +125 -159
- runbooks/common/accuracy_validator.py +11 -0
- runbooks/common/aws_pricing.py +349 -326
- runbooks/common/aws_pricing_api.py +211 -212
- runbooks/common/aws_profile_manager.py +341 -0
- runbooks/common/aws_utils.py +75 -80
- runbooks/common/business_logic.py +127 -105
- runbooks/common/cli_decorators.py +36 -60
- runbooks/common/comprehensive_cost_explorer_integration.py +456 -464
- runbooks/common/cross_account_manager.py +198 -205
- runbooks/common/date_utils.py +27 -39
- runbooks/common/decorators.py +235 -0
- runbooks/common/dry_run_examples.py +173 -208
- runbooks/common/dry_run_framework.py +157 -155
- runbooks/common/enhanced_exception_handler.py +15 -4
- runbooks/common/enhanced_logging_example.py +50 -64
- runbooks/common/enhanced_logging_integration_example.py +65 -37
- runbooks/common/env_utils.py +16 -16
- runbooks/common/error_handling.py +40 -38
- runbooks/common/lazy_loader.py +41 -23
- runbooks/common/logging_integration_helper.py +79 -86
- runbooks/common/mcp_cost_explorer_integration.py +478 -495
- runbooks/common/mcp_integration.py +63 -74
- runbooks/common/memory_optimization.py +140 -118
- runbooks/common/module_cli_base.py +37 -58
- runbooks/common/organizations_client.py +176 -194
- runbooks/common/patterns.py +204 -0
- runbooks/common/performance_monitoring.py +67 -71
- runbooks/common/performance_optimization_engine.py +283 -274
- runbooks/common/profile_utils.py +248 -39
- runbooks/common/rich_utils.py +643 -92
- runbooks/common/sre_performance_suite.py +177 -186
- runbooks/enterprise/__init__.py +1 -1
- runbooks/enterprise/logging.py +144 -106
- runbooks/enterprise/security.py +187 -204
- runbooks/enterprise/validation.py +43 -56
- runbooks/finops/__init__.py +29 -33
- runbooks/finops/account_resolver.py +1 -1
- runbooks/finops/advanced_optimization_engine.py +980 -0
- runbooks/finops/automation_core.py +268 -231
- runbooks/finops/business_case_config.py +184 -179
- runbooks/finops/cli.py +660 -139
- runbooks/finops/commvault_ec2_analysis.py +157 -164
- runbooks/finops/compute_cost_optimizer.py +336 -320
- runbooks/finops/config.py +20 -20
- runbooks/finops/cost_optimizer.py +488 -622
- runbooks/finops/cost_processor.py +332 -214
- runbooks/finops/dashboard_runner.py +1006 -172
- runbooks/finops/ebs_cost_optimizer.py +991 -657
- runbooks/finops/elastic_ip_optimizer.py +317 -257
- runbooks/finops/enhanced_mcp_integration.py +340 -0
- runbooks/finops/enhanced_progress.py +40 -37
- runbooks/finops/enhanced_trend_visualization.py +3 -2
- runbooks/finops/enterprise_wrappers.py +230 -292
- runbooks/finops/executive_export.py +203 -160
- runbooks/finops/helpers.py +130 -288
- runbooks/finops/iam_guidance.py +1 -1
- runbooks/finops/infrastructure/__init__.py +80 -0
- runbooks/finops/infrastructure/commands.py +506 -0
- runbooks/finops/infrastructure/load_balancer_optimizer.py +866 -0
- runbooks/finops/infrastructure/vpc_endpoint_optimizer.py +832 -0
- runbooks/finops/markdown_exporter.py +338 -175
- runbooks/finops/mcp_validator.py +1952 -0
- runbooks/finops/nat_gateway_optimizer.py +1513 -482
- runbooks/finops/network_cost_optimizer.py +657 -587
- runbooks/finops/notebook_utils.py +226 -188
- runbooks/finops/optimization_engine.py +1136 -0
- runbooks/finops/optimizer.py +25 -29
- runbooks/finops/rds_snapshot_optimizer.py +367 -411
- runbooks/finops/reservation_optimizer.py +427 -363
- runbooks/finops/scenario_cli_integration.py +77 -78
- runbooks/finops/scenarios.py +1278 -439
- runbooks/finops/schemas.py +218 -182
- runbooks/finops/snapshot_manager.py +2289 -0
- runbooks/finops/tests/test_finops_dashboard.py +3 -3
- runbooks/finops/tests/test_reference_images_validation.py +2 -2
- runbooks/finops/tests/test_single_account_features.py +17 -17
- runbooks/finops/tests/validate_test_suite.py +1 -1
- runbooks/finops/types.py +3 -3
- runbooks/finops/validation_framework.py +263 -269
- runbooks/finops/vpc_cleanup_exporter.py +191 -146
- runbooks/finops/vpc_cleanup_optimizer.py +593 -575
- runbooks/finops/workspaces_analyzer.py +171 -182
- runbooks/hitl/enhanced_workflow_engine.py +1 -1
- runbooks/integration/__init__.py +89 -0
- runbooks/integration/mcp_integration.py +1920 -0
- runbooks/inventory/CLAUDE.md +816 -0
- runbooks/inventory/README.md +3 -3
- runbooks/inventory/Tests/common_test_data.py +30 -30
- runbooks/inventory/__init__.py +2 -2
- runbooks/inventory/cloud_foundations_integration.py +144 -149
- runbooks/inventory/collectors/aws_comprehensive.py +28 -11
- runbooks/inventory/collectors/aws_networking.py +111 -101
- runbooks/inventory/collectors/base.py +4 -0
- runbooks/inventory/core/collector.py +495 -313
- runbooks/inventory/discovery.md +2 -2
- runbooks/inventory/drift_detection_cli.py +69 -96
- runbooks/inventory/find_ec2_security_groups.py +1 -1
- runbooks/inventory/inventory_mcp_cli.py +48 -46
- runbooks/inventory/list_rds_snapshots_aggregator.py +192 -208
- runbooks/inventory/mcp_inventory_validator.py +549 -465
- runbooks/inventory/mcp_vpc_validator.py +359 -442
- runbooks/inventory/organizations_discovery.py +56 -52
- runbooks/inventory/rich_inventory_display.py +33 -32
- runbooks/inventory/unified_validation_engine.py +278 -251
- runbooks/inventory/vpc_analyzer.py +733 -696
- runbooks/inventory/vpc_architecture_validator.py +293 -348
- runbooks/inventory/vpc_dependency_analyzer.py +382 -378
- runbooks/inventory/vpc_flow_analyzer.py +3 -3
- runbooks/main.py +152 -9147
- runbooks/main_final.py +91 -60
- runbooks/main_minimal.py +22 -10
- runbooks/main_optimized.py +131 -100
- runbooks/main_ultra_minimal.py +7 -2
- runbooks/mcp/__init__.py +36 -0
- runbooks/mcp/integration.py +679 -0
- runbooks/metrics/dora_metrics_engine.py +2 -2
- runbooks/monitoring/performance_monitor.py +9 -4
- runbooks/operate/dynamodb_operations.py +3 -1
- runbooks/operate/ec2_operations.py +145 -137
- runbooks/operate/iam_operations.py +146 -152
- runbooks/operate/mcp_integration.py +1 -1
- runbooks/operate/networking_cost_heatmap.py +33 -10
- runbooks/operate/privatelink_operations.py +1 -1
- runbooks/operate/rds_operations.py +223 -254
- runbooks/operate/s3_operations.py +107 -118
- runbooks/operate/vpc_endpoints.py +1 -1
- runbooks/operate/vpc_operations.py +648 -618
- runbooks/remediation/base.py +1 -1
- runbooks/remediation/commons.py +10 -7
- runbooks/remediation/commvault_ec2_analysis.py +71 -67
- runbooks/remediation/ec2_unattached_ebs_volumes.py +1 -0
- runbooks/remediation/multi_account.py +24 -21
- runbooks/remediation/rds_snapshot_list.py +91 -65
- runbooks/remediation/remediation_cli.py +92 -146
- runbooks/remediation/universal_account_discovery.py +83 -79
- runbooks/remediation/workspaces_list.py +49 -44
- runbooks/security/__init__.py +19 -0
- runbooks/security/assessment_runner.py +1150 -0
- runbooks/security/baseline_checker.py +812 -0
- runbooks/security/cloudops_automation_security_validator.py +509 -535
- runbooks/security/compliance_automation_engine.py +17 -17
- runbooks/security/config/__init__.py +2 -2
- runbooks/security/config/compliance_config.py +50 -50
- runbooks/security/config_template_generator.py +63 -76
- runbooks/security/enterprise_security_framework.py +1 -1
- runbooks/security/executive_security_dashboard.py +519 -508
- runbooks/security/integration_test_enterprise_security.py +5 -3
- runbooks/security/multi_account_security_controls.py +959 -1210
- runbooks/security/real_time_security_monitor.py +422 -444
- runbooks/security/run_script.py +1 -1
- runbooks/security/security_baseline_tester.py +1 -1
- runbooks/security/security_cli.py +143 -112
- runbooks/security/test_2way_validation.py +439 -0
- runbooks/security/two_way_validation_framework.py +852 -0
- runbooks/sre/mcp_reliability_engine.py +6 -6
- runbooks/sre/production_monitoring_framework.py +167 -177
- runbooks/tdd/__init__.py +15 -0
- runbooks/tdd/cli.py +1071 -0
- runbooks/utils/__init__.py +14 -17
- runbooks/utils/logger.py +7 -2
- runbooks/utils/version_validator.py +51 -48
- runbooks/validation/__init__.py +6 -6
- runbooks/validation/cli.py +9 -3
- runbooks/validation/comprehensive_2way_validator.py +754 -708
- runbooks/validation/mcp_validator.py +906 -228
- runbooks/validation/terraform_citations_validator.py +104 -115
- runbooks/validation/terraform_drift_detector.py +447 -451
- runbooks/vpc/README.md +617 -0
- runbooks/vpc/__init__.py +8 -1
- runbooks/vpc/analyzer.py +577 -0
- runbooks/vpc/cleanup_wrapper.py +476 -413
- runbooks/vpc/cli_cloudtrail_commands.py +339 -0
- runbooks/vpc/cli_mcp_validation_commands.py +480 -0
- runbooks/vpc/cloudtrail_audit_integration.py +717 -0
- runbooks/vpc/config.py +92 -97
- runbooks/vpc/cost_engine.py +411 -148
- runbooks/vpc/cost_explorer_integration.py +553 -0
- runbooks/vpc/cross_account_session.py +101 -106
- runbooks/vpc/enhanced_mcp_validation.py +917 -0
- runbooks/vpc/eni_gate_validator.py +961 -0
- runbooks/vpc/heatmap_engine.py +190 -162
- runbooks/vpc/mcp_no_eni_validator.py +681 -640
- runbooks/vpc/nat_gateway_optimizer.py +358 -0
- runbooks/vpc/networking_wrapper.py +15 -8
- runbooks/vpc/pdca_remediation_planner.py +528 -0
- runbooks/vpc/performance_optimized_analyzer.py +219 -231
- runbooks/vpc/runbooks_adapter.py +1167 -241
- runbooks/vpc/tdd_red_phase_stubs.py +601 -0
- runbooks/vpc/test_data_loader.py +358 -0
- runbooks/vpc/tests/conftest.py +314 -4
- runbooks/vpc/tests/test_cleanup_framework.py +1022 -0
- runbooks/vpc/tests/test_cost_engine.py +0 -2
- runbooks/vpc/topology_generator.py +326 -0
- runbooks/vpc/unified_scenarios.py +1302 -1129
- runbooks/vpc/vpc_cleanup_integration.py +1943 -1115
- runbooks-1.1.5.dist-info/METADATA +328 -0
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/RECORD +233 -200
- runbooks/finops/README.md +0 -414
- runbooks/finops/accuracy_cross_validator.py +0 -647
- runbooks/finops/business_cases.py +0 -950
- runbooks/finops/dashboard_router.py +0 -922
- runbooks/finops/ebs_optimizer.py +0 -956
- runbooks/finops/embedded_mcp_validator.py +0 -1629
- runbooks/finops/enhanced_dashboard_runner.py +0 -527
- runbooks/finops/finops_dashboard.py +0 -584
- runbooks/finops/finops_scenarios.py +0 -1218
- runbooks/finops/legacy_migration.py +0 -730
- runbooks/finops/multi_dashboard.py +0 -1519
- runbooks/finops/single_dashboard.py +0 -1113
- runbooks/finops/unlimited_scenarios.py +0 -393
- runbooks-1.1.3.dist-info/METADATA +0 -799
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/WHEEL +0 -0
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/entry_points.txt +0 -0
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/licenses/LICENSE +0 -0
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env python3
|
2
2
|
"""
|
3
3
|
Performance-Optimized VPC Analysis Engine
|
4
|
-
|
4
|
+
|
5
5
|
🎯 SRE Automation Specialist Implementation
|
6
6
|
Following proven systematic delegation patterns for VPC network operation optimization.
|
7
7
|
|
@@ -35,7 +35,7 @@ from rich.progress import (
|
|
35
35
|
BarColumn,
|
36
36
|
TimeElapsedColumn,
|
37
37
|
TaskProgressColumn,
|
38
|
-
MofNCompleteColumn
|
38
|
+
MofNCompleteColumn,
|
39
39
|
)
|
40
40
|
from rich.panel import Panel
|
41
41
|
from rich.table import Table
|
@@ -48,7 +48,7 @@ from runbooks.common.rich_utils import (
|
|
48
48
|
print_warning,
|
49
49
|
print_error,
|
50
50
|
create_table,
|
51
|
-
STATUS_INDICATORS
|
51
|
+
STATUS_INDICATORS,
|
52
52
|
)
|
53
53
|
from runbooks.common.performance_optimization_engine import get_optimization_engine
|
54
54
|
|
@@ -58,6 +58,7 @@ logger = logging.getLogger(__name__)
|
|
58
58
|
@dataclass
|
59
59
|
class VPCAnalysisResult:
|
60
60
|
"""VPC analysis result container"""
|
61
|
+
|
61
62
|
vpc_id: str
|
62
63
|
region: str
|
63
64
|
analysis_data: Dict[str, Any] = field(default_factory=dict)
|
@@ -72,9 +73,10 @@ class VPCAnalysisResult:
|
|
72
73
|
success: bool = True
|
73
74
|
|
74
75
|
|
75
|
-
@dataclass
|
76
|
+
@dataclass
|
76
77
|
class RegionalAnalysisMetrics:
|
77
78
|
"""Metrics for regional analysis performance"""
|
79
|
+
|
78
80
|
region: str
|
79
81
|
start_time: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
80
82
|
end_time: Optional[datetime] = None
|
@@ -89,31 +91,44 @@ class RegionalAnalysisMetrics:
|
|
89
91
|
class PerformanceOptimizedVPCAnalyzer:
|
90
92
|
"""
|
91
93
|
Performance-optimized VPC analysis engine with SRE automation patterns
|
92
|
-
|
94
|
+
|
93
95
|
Addresses VPC analysis timeout issues through:
|
94
96
|
- Parallel regional processing with intelligent load balancing
|
95
|
-
- Connection pooling and optimized AWS client configuration
|
97
|
+
- Connection pooling and optimized AWS client configuration
|
96
98
|
- Configurable timeouts with graceful degradation
|
97
99
|
- Memory-efficient batch processing for large VPC environments
|
98
100
|
- Real-time progress monitoring with Rich CLI indicators
|
99
101
|
"""
|
100
|
-
|
102
|
+
|
101
103
|
# AWS regions for global VPC analysis
|
102
104
|
DEFAULT_REGIONS = [
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
105
|
+
"us-east-1",
|
106
|
+
"us-west-2",
|
107
|
+
"us-east-2",
|
108
|
+
"us-west-1",
|
109
|
+
"eu-west-1",
|
110
|
+
"eu-west-2",
|
111
|
+
"eu-central-1",
|
112
|
+
"eu-north-1",
|
113
|
+
"ap-southeast-1",
|
114
|
+
"ap-southeast-2",
|
115
|
+
"ap-northeast-1",
|
116
|
+
"ap-northeast-2",
|
117
|
+
"ap-south-1",
|
118
|
+
"ca-central-1",
|
119
|
+
"sa-east-1",
|
107
120
|
]
|
108
|
-
|
109
|
-
def __init__(
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
121
|
+
|
122
|
+
def __init__(
|
123
|
+
self,
|
124
|
+
operational_profile: str,
|
125
|
+
max_workers: int = 15,
|
126
|
+
region_timeout_seconds: int = 45,
|
127
|
+
overall_timeout_seconds: int = 300,
|
128
|
+
):
|
114
129
|
"""
|
115
130
|
Initialize performance-optimized VPC analyzer
|
116
|
-
|
131
|
+
|
117
132
|
Args:
|
118
133
|
operational_profile: AWS profile for VPC operations
|
119
134
|
max_workers: Maximum concurrent workers for parallel analysis
|
@@ -124,44 +139,47 @@ class PerformanceOptimizedVPCAnalyzer:
|
|
124
139
|
self.max_workers = max_workers
|
125
140
|
self.region_timeout_seconds = region_timeout_seconds
|
126
141
|
self.overall_timeout_seconds = overall_timeout_seconds
|
127
|
-
|
142
|
+
|
128
143
|
# Performance optimization engine
|
129
144
|
self.optimization_engine = get_optimization_engine(
|
130
145
|
max_workers=max_workers,
|
131
146
|
cache_ttl_minutes=60, # Longer cache for VPC data
|
132
|
-
memory_limit_mb=3072
|
147
|
+
memory_limit_mb=3072, # Higher limit for VPC analysis
|
133
148
|
)
|
134
|
-
|
149
|
+
|
135
150
|
# Analysis tracking
|
136
151
|
self.regional_metrics: Dict[str, RegionalAnalysisMetrics] = {}
|
137
152
|
self.vpc_results: Dict[str, List[VPCAnalysisResult]] = {}
|
138
153
|
|
139
|
-
async def analyze_vpcs_globally(
|
140
|
-
|
141
|
-
|
154
|
+
async def analyze_vpcs_globally(
|
155
|
+
self, regions: Optional[List[str]] = None, include_detailed_analysis: bool = True
|
156
|
+
) -> Dict[str, Any]:
|
142
157
|
"""
|
143
158
|
Perform global VPC analysis with performance optimization
|
144
|
-
|
159
|
+
|
145
160
|
Args:
|
146
161
|
regions: List of regions to analyze (defaults to all major regions)
|
147
162
|
include_detailed_analysis: Whether to include detailed network component analysis
|
148
|
-
|
163
|
+
|
149
164
|
Returns:
|
150
165
|
Comprehensive VPC analysis results with performance metrics
|
151
166
|
"""
|
152
167
|
if regions is None:
|
153
168
|
regions = self.DEFAULT_REGIONS
|
154
|
-
|
169
|
+
|
155
170
|
print_header("Performance-Optimized VPC Analysis", "SRE Automation Engine")
|
156
|
-
|
171
|
+
|
157
172
|
# Start optimized analysis
|
158
173
|
with self.optimization_engine.optimize_operation("global_vpc_analysis", 180.0):
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
174
|
+
console.print(
|
175
|
+
f"[cyan]🌐 Analyzing VPCs across {len(regions)} regions with SRE optimization patterns[/cyan]"
|
176
|
+
)
|
177
|
+
console.print(
|
178
|
+
f"[dim]Timeout per region: {self.region_timeout_seconds}s, Overall timeout: {self.overall_timeout_seconds}s[/dim]"
|
179
|
+
)
|
180
|
+
|
163
181
|
start_time = time.time()
|
164
|
-
|
182
|
+
|
165
183
|
# Parallel regional analysis with progress tracking
|
166
184
|
with Progress(
|
167
185
|
SpinnerColumn(),
|
@@ -170,171 +188,150 @@ class PerformanceOptimizedVPCAnalyzer:
|
|
170
188
|
TaskProgressColumn(),
|
171
189
|
MofNCompleteColumn(),
|
172
190
|
TimeElapsedColumn(),
|
173
|
-
console=console
|
191
|
+
console=console,
|
174
192
|
) as progress:
|
175
|
-
|
176
|
-
|
177
|
-
"Analyzing VPCs globally...",
|
178
|
-
total=len(regions)
|
179
|
-
)
|
180
|
-
|
193
|
+
analysis_task = progress.add_task("Analyzing VPCs globally...", total=len(regions))
|
194
|
+
|
181
195
|
# Execute parallel regional analysis
|
182
196
|
analysis_results = await self._analyze_regions_parallel(
|
183
|
-
regions,
|
184
|
-
include_detailed_analysis,
|
185
|
-
progress,
|
186
|
-
analysis_task
|
197
|
+
regions, include_detailed_analysis, progress, analysis_task
|
187
198
|
)
|
188
|
-
|
199
|
+
|
189
200
|
# Aggregate results
|
190
201
|
total_duration = time.time() - start_time
|
191
202
|
summary = self._create_analysis_summary(analysis_results, total_duration)
|
192
|
-
|
203
|
+
|
193
204
|
# Display performance summary
|
194
205
|
self._display_performance_summary(summary)
|
195
|
-
|
206
|
+
|
196
207
|
return summary
|
197
208
|
|
198
|
-
async def _analyze_regions_parallel(
|
199
|
-
|
200
|
-
|
201
|
-
progress: Progress,
|
202
|
-
task_id) -> Dict[str, Any]:
|
209
|
+
async def _analyze_regions_parallel(
|
210
|
+
self, regions: List[str], include_detailed: bool, progress: Progress, task_id
|
211
|
+
) -> Dict[str, Any]:
|
203
212
|
"""Execute parallel regional VPC analysis with timeout handling"""
|
204
|
-
|
213
|
+
|
205
214
|
# Initialize regional metrics
|
206
215
|
for region in regions:
|
207
216
|
self.regional_metrics[region] = RegionalAnalysisMetrics(region=region)
|
208
|
-
|
217
|
+
|
209
218
|
analysis_results = {}
|
210
219
|
successful_regions = 0
|
211
220
|
failed_regions = 0
|
212
|
-
|
221
|
+
|
213
222
|
# Use ThreadPoolExecutor for parallel regional analysis
|
214
223
|
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
|
215
|
-
|
216
224
|
# Submit regional analysis tasks
|
217
225
|
future_to_region = {
|
218
|
-
executor.submit(
|
219
|
-
|
220
|
-
region,
|
221
|
-
include_detailed
|
222
|
-
): region for region in regions
|
226
|
+
executor.submit(self._analyze_vpc_region_optimized, region, include_detailed): region
|
227
|
+
for region in regions
|
223
228
|
}
|
224
|
-
|
229
|
+
|
225
230
|
# Process completed tasks
|
226
231
|
for future in as_completed(future_to_region, timeout=self.overall_timeout_seconds):
|
227
232
|
region = future_to_region[future]
|
228
|
-
|
233
|
+
|
229
234
|
try:
|
230
235
|
# Get result with per-region timeout
|
231
236
|
region_result = future.result(timeout=self.region_timeout_seconds)
|
232
237
|
analysis_results[region] = region_result
|
233
|
-
|
238
|
+
|
234
239
|
# Update metrics
|
235
240
|
metrics = self.regional_metrics[region]
|
236
241
|
metrics.end_time = datetime.now(timezone.utc)
|
237
242
|
metrics.duration_seconds = (metrics.end_time - metrics.start_time).total_seconds()
|
238
|
-
metrics.vpcs_analyzed = len(region_result.get(
|
239
|
-
|
243
|
+
metrics.vpcs_analyzed = len(region_result.get("vpcs", []))
|
244
|
+
|
240
245
|
successful_regions += 1
|
241
|
-
|
246
|
+
|
242
247
|
progress.update(
|
243
|
-
task_id,
|
248
|
+
task_id,
|
244
249
|
advance=1,
|
245
|
-
description=f"Completed {region} ({metrics.vpcs_analyzed} VPCs, {metrics.duration_seconds:.1f}s)"
|
250
|
+
description=f"Completed {region} ({metrics.vpcs_analyzed} VPCs, {metrics.duration_seconds:.1f}s)",
|
246
251
|
)
|
247
|
-
|
252
|
+
|
248
253
|
except TimeoutError:
|
249
254
|
logger.warning(f"VPC analysis timeout for region {region} after {self.region_timeout_seconds}s")
|
250
255
|
analysis_results[region] = {
|
251
|
-
|
252
|
-
|
253
|
-
|
256
|
+
"error": f"Analysis timeout after {self.region_timeout_seconds}s",
|
257
|
+
"vpcs": [],
|
258
|
+
"timeout": True,
|
254
259
|
}
|
255
|
-
|
260
|
+
|
256
261
|
# Update timeout metrics
|
257
|
-
metrics = self.regional_metrics[region]
|
262
|
+
metrics = self.regional_metrics[region]
|
258
263
|
metrics.timeout_occurred = True
|
259
264
|
metrics.end_time = datetime.now(timezone.utc)
|
260
265
|
metrics.duration_seconds = self.region_timeout_seconds
|
261
|
-
|
266
|
+
|
262
267
|
failed_regions += 1
|
263
268
|
progress.advance(task_id)
|
264
|
-
|
269
|
+
|
265
270
|
except Exception as e:
|
266
271
|
logger.error(f"VPC analysis failed for region {region}: {e}")
|
267
|
-
analysis_results[region] = {
|
268
|
-
|
269
|
-
'vpcs': [],
|
270
|
-
'failed': True
|
271
|
-
}
|
272
|
-
|
272
|
+
analysis_results[region] = {"error": str(e), "vpcs": [], "failed": True}
|
273
|
+
|
273
274
|
# Update error metrics
|
274
275
|
metrics = self.regional_metrics[region]
|
275
276
|
metrics.errors_encountered += 1
|
276
|
-
metrics.end_time = datetime.now(timezone.utc)
|
277
|
+
metrics.end_time = datetime.now(timezone.utc)
|
277
278
|
metrics.duration_seconds = (metrics.end_time - metrics.start_time).total_seconds()
|
278
|
-
|
279
|
+
|
279
280
|
failed_regions += 1
|
280
281
|
progress.advance(task_id)
|
281
|
-
|
282
|
+
|
282
283
|
return {
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
284
|
+
"regional_results": analysis_results,
|
285
|
+
"successful_regions": successful_regions,
|
286
|
+
"failed_regions": failed_regions,
|
287
|
+
"total_regions": len(regions),
|
287
288
|
}
|
288
289
|
|
289
|
-
def _analyze_vpc_region_optimized(self,
|
290
|
-
region: str,
|
291
|
-
include_detailed: bool) -> Dict[str, Any]:
|
290
|
+
def _analyze_vpc_region_optimized(self, region: str, include_detailed: bool) -> Dict[str, Any]:
|
292
291
|
"""Optimized VPC analysis for a specific region with performance enhancements"""
|
293
|
-
|
292
|
+
|
294
293
|
metrics = self.regional_metrics[region]
|
295
|
-
|
294
|
+
|
296
295
|
try:
|
297
296
|
# Get optimized VPC analyzer from performance engine
|
298
297
|
optimized_vpc_analysis = self.optimization_engine.optimize_vpc_analysis(
|
299
298
|
operational_profile=self.operational_profile
|
300
299
|
)
|
301
|
-
|
300
|
+
|
302
301
|
# Execute regional analysis
|
303
302
|
regional_data = optimized_vpc_analysis([region])
|
304
|
-
region_vpcs = regional_data.get(
|
305
|
-
|
306
|
-
if
|
307
|
-
raise Exception(region_vpcs[
|
308
|
-
|
309
|
-
vpcs = region_vpcs.get(
|
303
|
+
region_vpcs = regional_data.get("vpc_data_by_region", {}).get(region, {})
|
304
|
+
|
305
|
+
if "error" in region_vpcs:
|
306
|
+
raise Exception(region_vpcs["error"])
|
307
|
+
|
308
|
+
vpcs = region_vpcs.get("vpcs", [])
|
310
309
|
metrics.vpcs_analyzed = len(vpcs)
|
311
|
-
metrics.api_calls_made = region_vpcs.get(
|
312
|
-
|
310
|
+
metrics.api_calls_made = region_vpcs.get("api_calls", 0)
|
311
|
+
|
313
312
|
# Enhanced VPC analysis if requested
|
314
313
|
if include_detailed and vpcs:
|
315
314
|
vpcs = self._enrich_vpcs_with_details(vpcs, region)
|
316
315
|
metrics.optimizations_applied.append("detailed_enrichment")
|
317
|
-
|
316
|
+
|
318
317
|
# Apply additional performance optimizations
|
319
318
|
if len(vpcs) > 10:
|
320
319
|
metrics.optimizations_applied.append("batch_processing")
|
321
|
-
|
322
|
-
metrics.optimizations_applied.extend(
|
323
|
-
"connection_pooling",
|
324
|
-
|
325
|
-
|
326
|
-
])
|
327
|
-
|
320
|
+
|
321
|
+
metrics.optimizations_applied.extend(
|
322
|
+
["connection_pooling", "intelligent_caching", "parallel_regional_processing"]
|
323
|
+
)
|
324
|
+
|
328
325
|
return {
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
}
|
326
|
+
"vpcs": vpcs,
|
327
|
+
"region": region,
|
328
|
+
"metrics": {
|
329
|
+
"vpcs_analyzed": metrics.vpcs_analyzed,
|
330
|
+
"api_calls_made": metrics.api_calls_made,
|
331
|
+
"optimizations_applied": metrics.optimizations_applied,
|
332
|
+
},
|
336
333
|
}
|
337
|
-
|
334
|
+
|
338
335
|
except Exception as e:
|
339
336
|
metrics.errors_encountered += 1
|
340
337
|
logger.error(f"Optimized VPC analysis failed for region {region}: {e}")
|
@@ -342,155 +339,152 @@ class PerformanceOptimizedVPCAnalyzer:
|
|
342
339
|
|
343
340
|
def _enrich_vpcs_with_details(self, vpcs: List[Dict], region: str) -> List[Dict]:
|
344
341
|
"""Enrich VPC data with detailed network component analysis"""
|
345
|
-
|
342
|
+
|
346
343
|
try:
|
347
344
|
# Use cached client from optimization engine
|
348
|
-
ec2_client = self.optimization_engine.client_pool.get_client(
|
349
|
-
|
345
|
+
ec2_client = self.optimization_engine.client_pool.get_client("ec2", self.operational_profile, region)
|
346
|
+
|
350
347
|
for vpc in vpcs:
|
351
|
-
vpc_id = vpc[
|
352
|
-
|
348
|
+
vpc_id = vpc["VpcId"]
|
349
|
+
|
353
350
|
# Get additional VPC components in parallel where possible
|
354
351
|
try:
|
355
352
|
# Route tables
|
356
|
-
rt_response = ec2_client.describe_route_tables(
|
357
|
-
|
358
|
-
|
359
|
-
vpc['RouteTables'] = rt_response['RouteTables']
|
360
|
-
|
353
|
+
rt_response = ec2_client.describe_route_tables(Filters=[{"Name": "vpc-id", "Values": [vpc_id]}])
|
354
|
+
vpc["RouteTables"] = rt_response["RouteTables"]
|
355
|
+
|
361
356
|
# Security groups
|
362
|
-
sg_response = ec2_client.describe_security_groups(
|
363
|
-
|
364
|
-
|
365
|
-
vpc['SecurityGroups'] = sg_response['SecurityGroups']
|
366
|
-
|
357
|
+
sg_response = ec2_client.describe_security_groups(Filters=[{"Name": "vpc-id", "Values": [vpc_id]}])
|
358
|
+
vpc["SecurityGroups"] = sg_response["SecurityGroups"]
|
359
|
+
|
367
360
|
# NAT Gateways
|
368
|
-
nat_response = ec2_client.describe_nat_gateways(
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
# Internet Gateways
|
361
|
+
nat_response = ec2_client.describe_nat_gateways(Filters=[{"Name": "vpc-id", "Values": [vpc_id]}])
|
362
|
+
vpc["NatGateways"] = nat_response["NatGateways"]
|
363
|
+
|
364
|
+
# Internet Gateways
|
374
365
|
igw_response = ec2_client.describe_internet_gateways(
|
375
|
-
Filters=[{
|
366
|
+
Filters=[{"Name": "attachment.vpc-id", "Values": [vpc_id]}]
|
376
367
|
)
|
377
|
-
vpc[
|
378
|
-
|
368
|
+
vpc["InternetGateways"] = igw_response["InternetGateways"]
|
369
|
+
|
379
370
|
except Exception as detail_error:
|
380
371
|
logger.debug(f"Failed to get detailed info for VPC {vpc_id}: {detail_error}")
|
381
372
|
# Continue with basic VPC data
|
382
|
-
|
373
|
+
|
383
374
|
except Exception as e:
|
384
375
|
logger.warning(f"VPC enrichment failed for region {region}: {e}")
|
385
|
-
|
376
|
+
|
386
377
|
return vpcs
|
387
378
|
|
388
379
|
def _create_analysis_summary(self, analysis_results: Dict, total_duration: float) -> Dict[str, Any]:
|
389
380
|
"""Create comprehensive analysis summary with performance metrics"""
|
390
|
-
|
391
|
-
regional_results = analysis_results.get(
|
392
|
-
successful_regions = analysis_results.get(
|
393
|
-
failed_regions = analysis_results.get(
|
394
|
-
|
381
|
+
|
382
|
+
regional_results = analysis_results.get("regional_results", {})
|
383
|
+
successful_regions = analysis_results.get("successful_regions", 0)
|
384
|
+
failed_regions = analysis_results.get("failed_regions", 0)
|
385
|
+
|
395
386
|
# Aggregate VPC data
|
396
387
|
total_vpcs = 0
|
397
388
|
total_api_calls = 0
|
398
389
|
regions_with_vpcs = 0
|
399
390
|
all_optimizations = set()
|
400
|
-
|
391
|
+
|
401
392
|
vpc_summary_by_region = {}
|
402
|
-
|
393
|
+
|
403
394
|
for region, result in regional_results.items():
|
404
|
-
if
|
405
|
-
vpcs = result.get(
|
395
|
+
if "error" not in result and not result.get("failed", False):
|
396
|
+
vpcs = result.get("vpcs", [])
|
406
397
|
total_vpcs += len(vpcs)
|
407
|
-
|
398
|
+
|
408
399
|
if vpcs:
|
409
400
|
regions_with_vpcs += 1
|
410
|
-
|
401
|
+
|
411
402
|
# Collect metrics
|
412
|
-
region_metrics = result.get(
|
413
|
-
total_api_calls += region_metrics.get(
|
414
|
-
all_optimizations.update(region_metrics.get(
|
415
|
-
|
403
|
+
region_metrics = result.get("metrics", {})
|
404
|
+
total_api_calls += region_metrics.get("api_calls_made", 0)
|
405
|
+
all_optimizations.update(region_metrics.get("optimizations_applied", []))
|
406
|
+
|
416
407
|
vpc_summary_by_region[region] = {
|
417
|
-
|
418
|
-
|
419
|
-
|
408
|
+
"vpc_count": len(vpcs),
|
409
|
+
"analysis_duration": self.regional_metrics[region].duration_seconds,
|
410
|
+
"optimizations_applied": region_metrics.get("optimizations_applied", []),
|
420
411
|
}
|
421
|
-
|
412
|
+
|
422
413
|
# Performance analysis
|
423
414
|
avg_duration_per_region = total_duration / len(regional_results) if regional_results else 0
|
424
415
|
performance_grade = "A" if total_duration < 120 else "B" if total_duration < 180 else "C"
|
425
|
-
|
416
|
+
|
426
417
|
return {
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
418
|
+
"analysis_summary": {
|
419
|
+
"total_regions_analyzed": len(regional_results),
|
420
|
+
"successful_regions": successful_regions,
|
421
|
+
"failed_regions": failed_regions,
|
422
|
+
"regions_with_vpcs": regions_with_vpcs,
|
423
|
+
"total_vpcs_discovered": total_vpcs,
|
424
|
+
"total_duration_seconds": total_duration,
|
425
|
+
"average_duration_per_region": avg_duration_per_region,
|
426
|
+
"performance_grade": performance_grade,
|
436
427
|
},
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
428
|
+
"vpc_summary_by_region": vpc_summary_by_region,
|
429
|
+
"regional_results": regional_results,
|
430
|
+
"performance_metrics": {
|
431
|
+
"total_api_calls": total_api_calls,
|
432
|
+
"optimizations_applied": list(all_optimizations),
|
433
|
+
"regional_metrics": {
|
443
434
|
region: {
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
}
|
450
|
-
|
435
|
+
"duration_seconds": metrics.duration_seconds,
|
436
|
+
"vpcs_analyzed": metrics.vpcs_analyzed,
|
437
|
+
"api_calls_made": metrics.api_calls_made,
|
438
|
+
"timeout_occurred": metrics.timeout_occurred,
|
439
|
+
"errors_encountered": metrics.errors_encountered,
|
440
|
+
}
|
441
|
+
for region, metrics in self.regional_metrics.items()
|
442
|
+
},
|
451
443
|
},
|
452
|
-
|
444
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
453
445
|
}
|
454
446
|
|
455
447
|
def _display_performance_summary(self, summary: Dict[str, Any]):
|
456
448
|
"""Display comprehensive performance summary with Rich formatting"""
|
457
|
-
|
458
|
-
analysis_summary = summary[
|
459
|
-
performance_metrics = summary[
|
460
|
-
|
449
|
+
|
450
|
+
analysis_summary = summary["analysis_summary"]
|
451
|
+
performance_metrics = summary["performance_metrics"]
|
452
|
+
|
461
453
|
# Performance overview panel
|
462
454
|
performance_text = f"""
|
463
455
|
[bold cyan]📊 VPC Analysis Performance Summary[/bold cyan]
|
464
456
|
|
465
|
-
[green]✅ Regions Successful:[/green] {analysis_summary[
|
466
|
-
[yellow]🌐 VPCs Discovered:[/yellow] {analysis_summary[
|
467
|
-
[blue]⏱️ Total Duration:[/blue] {analysis_summary[
|
468
|
-
[magenta]📈 Performance Grade:[/magenta] {analysis_summary[
|
469
|
-
[dim]🔧 Optimizations Applied:[/dim] {
|
457
|
+
[green]✅ Regions Successful:[/green] {analysis_summary["successful_regions"]}/{analysis_summary["total_regions_analyzed"]}
|
458
|
+
[yellow]🌐 VPCs Discovered:[/yellow] {analysis_summary["total_vpcs_discovered"]} across {analysis_summary["regions_with_vpcs"]} regions
|
459
|
+
[blue]⏱️ Total Duration:[/blue] {analysis_summary["total_duration_seconds"]:.1f}s (avg: {analysis_summary["average_duration_per_region"]:.1f}s/region)
|
460
|
+
[magenta]📈 Performance Grade:[/magenta] {analysis_summary["performance_grade"]}
|
461
|
+
[dim]🔧 Optimizations Applied:[/dim] {", ".join(performance_metrics["optimizations_applied"])}
|
470
462
|
"""
|
471
|
-
|
472
|
-
console.print(
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
463
|
+
|
464
|
+
console.print(
|
465
|
+
Panel(
|
466
|
+
performance_text.strip(),
|
467
|
+
title="[bold green]🚀 SRE Optimization Results[/bold green]",
|
468
|
+
border_style="green" if analysis_summary["performance_grade"] in ["A", "B"] else "yellow",
|
469
|
+
)
|
470
|
+
)
|
471
|
+
|
478
472
|
# Regional performance table
|
479
|
-
if summary[
|
473
|
+
if summary["vpc_summary_by_region"]:
|
480
474
|
table = create_table(
|
481
475
|
title="Regional VPC Analysis Performance",
|
482
476
|
columns=[
|
483
477
|
{"name": "Region", "style": "cyan", "justify": "left"},
|
484
478
|
{"name": "VPCs", "style": "yellow", "justify": "center"},
|
485
|
-
{"name": "Duration", "style": "white", "justify": "right"},
|
486
|
-
{"name": "Status", "style": "white", "justify": "center"}
|
487
|
-
]
|
479
|
+
{"name": "Duration", "style": "white", "justify": "right"},
|
480
|
+
{"name": "Status", "style": "white", "justify": "center"},
|
481
|
+
],
|
488
482
|
)
|
489
|
-
|
490
|
-
for region, data in summary[
|
491
|
-
duration = data[
|
492
|
-
vpc_count = data[
|
493
|
-
|
483
|
+
|
484
|
+
for region, data in summary["vpc_summary_by_region"].items():
|
485
|
+
duration = data["analysis_duration"]
|
486
|
+
vpc_count = data["vpc_count"]
|
487
|
+
|
494
488
|
# Determine status
|
495
489
|
if duration <= 30:
|
496
490
|
status_icon = f"[green]{STATUS_INDICATORS['success']}[/green]"
|
@@ -498,14 +492,9 @@ class PerformanceOptimizedVPCAnalyzer:
|
|
498
492
|
status_icon = f"[yellow]{STATUS_INDICATORS['warning']}[/yellow]"
|
499
493
|
else:
|
500
494
|
status_icon = f"[red]{STATUS_INDICATORS['error']}[/red]"
|
501
|
-
|
502
|
-
table.add_row(
|
503
|
-
|
504
|
-
str(vpc_count),
|
505
|
-
f"{duration:.1f}s",
|
506
|
-
status_icon
|
507
|
-
)
|
508
|
-
|
495
|
+
|
496
|
+
table.add_row(region, str(vpc_count), f"{duration:.1f}s", status_icon)
|
497
|
+
|
509
498
|
console.print(table)
|
510
499
|
|
511
500
|
def clear_analysis_cache(self):
|
@@ -517,20 +506,19 @@ class PerformanceOptimizedVPCAnalyzer:
|
|
517
506
|
|
518
507
|
|
519
508
|
# Convenience functions
|
520
|
-
def create_optimized_vpc_analyzer(operational_profile: str,
|
521
|
-
max_workers: int = 15) -> PerformanceOptimizedVPCAnalyzer:
|
509
|
+
def create_optimized_vpc_analyzer(operational_profile: str, max_workers: int = 15) -> PerformanceOptimizedVPCAnalyzer:
|
522
510
|
"""Create performance-optimized VPC analyzer instance"""
|
523
511
|
return PerformanceOptimizedVPCAnalyzer(
|
524
512
|
operational_profile=operational_profile,
|
525
513
|
max_workers=max_workers,
|
526
514
|
region_timeout_seconds=45,
|
527
|
-
overall_timeout_seconds=300
|
515
|
+
overall_timeout_seconds=300,
|
528
516
|
)
|
529
517
|
|
530
518
|
|
531
|
-
async def run_optimized_global_vpc_analysis(
|
532
|
-
|
533
|
-
|
519
|
+
async def run_optimized_global_vpc_analysis(
|
520
|
+
operational_profile: str, regions: Optional[List[str]] = None, include_detailed: bool = True
|
521
|
+
) -> Dict[str, Any]:
|
534
522
|
"""Run optimized global VPC analysis"""
|
535
523
|
analyzer = create_optimized_vpc_analyzer(operational_profile)
|
536
524
|
return await analyzer.analyze_vpcs_globally(regions, include_detailed)
|
@@ -540,7 +528,7 @@ async def run_optimized_global_vpc_analysis(operational_profile: str,
|
|
540
528
|
__all__ = [
|
541
529
|
"PerformanceOptimizedVPCAnalyzer",
|
542
530
|
"VPCAnalysisResult",
|
543
|
-
"RegionalAnalysisMetrics",
|
531
|
+
"RegionalAnalysisMetrics",
|
544
532
|
"create_optimized_vpc_analyzer",
|
545
|
-
"run_optimized_global_vpc_analysis"
|
546
|
-
]
|
533
|
+
"run_optimized_global_vpc_analysis",
|
534
|
+
]
|