runbooks 1.1.4__py3-none-any.whl → 1.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +31 -2
- runbooks/__init___optimized.py +18 -4
- runbooks/_platform/__init__.py +1 -5
- runbooks/_platform/core/runbooks_wrapper.py +141 -138
- runbooks/aws2/accuracy_validator.py +812 -0
- runbooks/base.py +7 -0
- runbooks/cfat/assessment/compliance.py +1 -1
- runbooks/cfat/assessment/runner.py +1 -0
- runbooks/cfat/cloud_foundations_assessment.py +227 -239
- runbooks/cli/__init__.py +1 -1
- runbooks/cli/commands/cfat.py +64 -23
- runbooks/cli/commands/finops.py +1005 -54
- runbooks/cli/commands/inventory.py +138 -35
- runbooks/cli/commands/operate.py +9 -36
- runbooks/cli/commands/security.py +42 -18
- runbooks/cli/commands/validation.py +432 -18
- runbooks/cli/commands/vpc.py +81 -17
- runbooks/cli/registry.py +22 -10
- runbooks/cloudops/__init__.py +20 -27
- runbooks/cloudops/base.py +96 -107
- runbooks/cloudops/cost_optimizer.py +544 -542
- runbooks/cloudops/infrastructure_optimizer.py +5 -4
- runbooks/cloudops/interfaces.py +224 -225
- runbooks/cloudops/lifecycle_manager.py +5 -4
- runbooks/cloudops/mcp_cost_validation.py +252 -235
- runbooks/cloudops/models.py +78 -53
- runbooks/cloudops/monitoring_automation.py +5 -4
- runbooks/cloudops/notebook_framework.py +177 -213
- runbooks/cloudops/security_enforcer.py +125 -159
- runbooks/common/accuracy_validator.py +11 -0
- runbooks/common/aws_pricing.py +349 -326
- runbooks/common/aws_pricing_api.py +211 -212
- runbooks/common/aws_profile_manager.py +40 -36
- runbooks/common/aws_utils.py +74 -79
- runbooks/common/business_logic.py +126 -104
- runbooks/common/cli_decorators.py +36 -60
- runbooks/common/comprehensive_cost_explorer_integration.py +455 -463
- runbooks/common/cross_account_manager.py +197 -204
- runbooks/common/date_utils.py +27 -39
- runbooks/common/decorators.py +29 -19
- runbooks/common/dry_run_examples.py +173 -208
- runbooks/common/dry_run_framework.py +157 -155
- runbooks/common/enhanced_exception_handler.py +15 -4
- runbooks/common/enhanced_logging_example.py +50 -64
- runbooks/common/enhanced_logging_integration_example.py +65 -37
- runbooks/common/env_utils.py +16 -16
- runbooks/common/error_handling.py +40 -38
- runbooks/common/lazy_loader.py +41 -23
- runbooks/common/logging_integration_helper.py +79 -86
- runbooks/common/mcp_cost_explorer_integration.py +476 -493
- runbooks/common/mcp_integration.py +63 -74
- runbooks/common/memory_optimization.py +140 -118
- runbooks/common/module_cli_base.py +37 -58
- runbooks/common/organizations_client.py +175 -193
- runbooks/common/patterns.py +23 -25
- runbooks/common/performance_monitoring.py +67 -71
- runbooks/common/performance_optimization_engine.py +283 -274
- runbooks/common/profile_utils.py +111 -37
- runbooks/common/rich_utils.py +201 -141
- runbooks/common/sre_performance_suite.py +177 -186
- runbooks/enterprise/__init__.py +1 -1
- runbooks/enterprise/logging.py +144 -106
- runbooks/enterprise/security.py +187 -204
- runbooks/enterprise/validation.py +43 -56
- runbooks/finops/__init__.py +26 -30
- runbooks/finops/account_resolver.py +1 -1
- runbooks/finops/advanced_optimization_engine.py +980 -0
- runbooks/finops/automation_core.py +268 -231
- runbooks/finops/business_case_config.py +184 -179
- runbooks/finops/cli.py +660 -139
- runbooks/finops/commvault_ec2_analysis.py +157 -164
- runbooks/finops/compute_cost_optimizer.py +336 -320
- runbooks/finops/config.py +20 -20
- runbooks/finops/cost_optimizer.py +484 -618
- runbooks/finops/cost_processor.py +332 -214
- runbooks/finops/dashboard_runner.py +1006 -172
- runbooks/finops/ebs_cost_optimizer.py +991 -657
- runbooks/finops/elastic_ip_optimizer.py +317 -257
- runbooks/finops/enhanced_mcp_integration.py +340 -0
- runbooks/finops/enhanced_progress.py +32 -29
- runbooks/finops/enhanced_trend_visualization.py +3 -2
- runbooks/finops/enterprise_wrappers.py +223 -285
- runbooks/finops/executive_export.py +203 -160
- runbooks/finops/helpers.py +130 -288
- runbooks/finops/iam_guidance.py +1 -1
- runbooks/finops/infrastructure/__init__.py +80 -0
- runbooks/finops/infrastructure/commands.py +506 -0
- runbooks/finops/infrastructure/load_balancer_optimizer.py +866 -0
- runbooks/finops/infrastructure/vpc_endpoint_optimizer.py +832 -0
- runbooks/finops/markdown_exporter.py +337 -174
- runbooks/finops/mcp_validator.py +1952 -0
- runbooks/finops/nat_gateway_optimizer.py +1512 -481
- runbooks/finops/network_cost_optimizer.py +657 -587
- runbooks/finops/notebook_utils.py +226 -188
- runbooks/finops/optimization_engine.py +1136 -0
- runbooks/finops/optimizer.py +19 -23
- runbooks/finops/rds_snapshot_optimizer.py +367 -411
- runbooks/finops/reservation_optimizer.py +427 -363
- runbooks/finops/scenario_cli_integration.py +64 -65
- runbooks/finops/scenarios.py +1277 -438
- runbooks/finops/schemas.py +218 -182
- runbooks/finops/snapshot_manager.py +2289 -0
- runbooks/finops/types.py +3 -3
- runbooks/finops/validation_framework.py +259 -265
- runbooks/finops/vpc_cleanup_exporter.py +189 -144
- runbooks/finops/vpc_cleanup_optimizer.py +591 -573
- runbooks/finops/workspaces_analyzer.py +171 -182
- runbooks/integration/__init__.py +89 -0
- runbooks/integration/mcp_integration.py +1920 -0
- runbooks/inventory/CLAUDE.md +816 -0
- runbooks/inventory/__init__.py +2 -2
- runbooks/inventory/cloud_foundations_integration.py +144 -149
- runbooks/inventory/collectors/aws_comprehensive.py +1 -1
- runbooks/inventory/collectors/aws_networking.py +109 -99
- runbooks/inventory/collectors/base.py +4 -0
- runbooks/inventory/core/collector.py +495 -313
- runbooks/inventory/drift_detection_cli.py +69 -96
- runbooks/inventory/inventory_mcp_cli.py +48 -46
- runbooks/inventory/list_rds_snapshots_aggregator.py +192 -208
- runbooks/inventory/mcp_inventory_validator.py +549 -465
- runbooks/inventory/mcp_vpc_validator.py +359 -442
- runbooks/inventory/organizations_discovery.py +55 -51
- runbooks/inventory/rich_inventory_display.py +33 -32
- runbooks/inventory/unified_validation_engine.py +278 -251
- runbooks/inventory/vpc_analyzer.py +732 -695
- runbooks/inventory/vpc_architecture_validator.py +293 -348
- runbooks/inventory/vpc_dependency_analyzer.py +382 -378
- runbooks/inventory/vpc_flow_analyzer.py +1 -1
- runbooks/main.py +49 -34
- runbooks/main_final.py +91 -60
- runbooks/main_minimal.py +22 -10
- runbooks/main_optimized.py +131 -100
- runbooks/main_ultra_minimal.py +7 -2
- runbooks/mcp/__init__.py +36 -0
- runbooks/mcp/integration.py +679 -0
- runbooks/monitoring/performance_monitor.py +9 -4
- runbooks/operate/dynamodb_operations.py +3 -1
- runbooks/operate/ec2_operations.py +145 -137
- runbooks/operate/iam_operations.py +146 -152
- runbooks/operate/networking_cost_heatmap.py +29 -8
- runbooks/operate/rds_operations.py +223 -254
- runbooks/operate/s3_operations.py +107 -118
- runbooks/operate/vpc_operations.py +646 -616
- runbooks/remediation/base.py +1 -1
- runbooks/remediation/commons.py +10 -7
- runbooks/remediation/commvault_ec2_analysis.py +70 -66
- runbooks/remediation/ec2_unattached_ebs_volumes.py +1 -0
- runbooks/remediation/multi_account.py +24 -21
- runbooks/remediation/rds_snapshot_list.py +86 -60
- runbooks/remediation/remediation_cli.py +92 -146
- runbooks/remediation/universal_account_discovery.py +83 -79
- runbooks/remediation/workspaces_list.py +46 -41
- runbooks/security/__init__.py +19 -0
- runbooks/security/assessment_runner.py +1150 -0
- runbooks/security/baseline_checker.py +812 -0
- runbooks/security/cloudops_automation_security_validator.py +509 -535
- runbooks/security/compliance_automation_engine.py +17 -17
- runbooks/security/config/__init__.py +2 -2
- runbooks/security/config/compliance_config.py +50 -50
- runbooks/security/config_template_generator.py +63 -76
- runbooks/security/enterprise_security_framework.py +1 -1
- runbooks/security/executive_security_dashboard.py +519 -508
- runbooks/security/multi_account_security_controls.py +959 -1210
- runbooks/security/real_time_security_monitor.py +422 -444
- runbooks/security/security_baseline_tester.py +1 -1
- runbooks/security/security_cli.py +143 -112
- runbooks/security/test_2way_validation.py +439 -0
- runbooks/security/two_way_validation_framework.py +852 -0
- runbooks/sre/production_monitoring_framework.py +167 -177
- runbooks/tdd/__init__.py +15 -0
- runbooks/tdd/cli.py +1071 -0
- runbooks/utils/__init__.py +14 -17
- runbooks/utils/logger.py +7 -2
- runbooks/utils/version_validator.py +50 -47
- runbooks/validation/__init__.py +6 -6
- runbooks/validation/cli.py +9 -3
- runbooks/validation/comprehensive_2way_validator.py +745 -704
- runbooks/validation/mcp_validator.py +906 -228
- runbooks/validation/terraform_citations_validator.py +104 -115
- runbooks/validation/terraform_drift_detector.py +447 -451
- runbooks/vpc/README.md +617 -0
- runbooks/vpc/__init__.py +8 -1
- runbooks/vpc/analyzer.py +577 -0
- runbooks/vpc/cleanup_wrapper.py +476 -413
- runbooks/vpc/cli_cloudtrail_commands.py +339 -0
- runbooks/vpc/cli_mcp_validation_commands.py +480 -0
- runbooks/vpc/cloudtrail_audit_integration.py +717 -0
- runbooks/vpc/config.py +92 -97
- runbooks/vpc/cost_engine.py +411 -148
- runbooks/vpc/cost_explorer_integration.py +553 -0
- runbooks/vpc/cross_account_session.py +101 -106
- runbooks/vpc/enhanced_mcp_validation.py +917 -0
- runbooks/vpc/eni_gate_validator.py +961 -0
- runbooks/vpc/heatmap_engine.py +185 -160
- runbooks/vpc/mcp_no_eni_validator.py +680 -639
- runbooks/vpc/nat_gateway_optimizer.py +358 -0
- runbooks/vpc/networking_wrapper.py +15 -8
- runbooks/vpc/pdca_remediation_planner.py +528 -0
- runbooks/vpc/performance_optimized_analyzer.py +219 -231
- runbooks/vpc/runbooks_adapter.py +1167 -241
- runbooks/vpc/tdd_red_phase_stubs.py +601 -0
- runbooks/vpc/test_data_loader.py +358 -0
- runbooks/vpc/tests/conftest.py +314 -4
- runbooks/vpc/tests/test_cleanup_framework.py +1022 -0
- runbooks/vpc/tests/test_cost_engine.py +0 -2
- runbooks/vpc/topology_generator.py +326 -0
- runbooks/vpc/unified_scenarios.py +1297 -1124
- runbooks/vpc/vpc_cleanup_integration.py +1943 -1115
- runbooks-1.1.5.dist-info/METADATA +328 -0
- {runbooks-1.1.4.dist-info → runbooks-1.1.5.dist-info}/RECORD +214 -193
- runbooks/finops/README.md +0 -414
- runbooks/finops/accuracy_cross_validator.py +0 -647
- runbooks/finops/business_cases.py +0 -950
- runbooks/finops/dashboard_router.py +0 -922
- runbooks/finops/ebs_optimizer.py +0 -973
- runbooks/finops/embedded_mcp_validator.py +0 -1629
- runbooks/finops/enhanced_dashboard_runner.py +0 -527
- runbooks/finops/finops_dashboard.py +0 -584
- runbooks/finops/finops_scenarios.py +0 -1218
- runbooks/finops/legacy_migration.py +0 -730
- runbooks/finops/multi_dashboard.py +0 -1519
- runbooks/finops/single_dashboard.py +0 -1113
- runbooks/finops/unlimited_scenarios.py +0 -393
- runbooks-1.1.4.dist-info/METADATA +0 -800
- {runbooks-1.1.4.dist-info → runbooks-1.1.5.dist-info}/WHEEL +0 -0
- {runbooks-1.1.4.dist-info → runbooks-1.1.5.dist-info}/entry_points.txt +0 -0
- {runbooks-1.1.4.dist-info → runbooks-1.1.5.dist-info}/licenses/LICENSE +0 -0
- {runbooks-1.1.4.dist-info → runbooks-1.1.5.dist-info}/top_level.txt +0 -0
@@ -1,20 +1,20 @@
|
|
1
1
|
#!/usr/bin/env python3
|
2
2
|
"""
|
3
3
|
SRE Performance Optimization Suite - Comprehensive Performance Enhancement
|
4
|
-
|
4
|
+
|
5
5
|
🎯 Enterprise SRE Automation Specialist Implementation
|
6
6
|
Following proven systematic delegation patterns for production reliability optimization.
|
7
7
|
|
8
8
|
This suite integrates all performance optimizations identified from PDCA analysis:
|
9
9
|
|
10
10
|
CRITICAL PERFORMANCE BOTTLENECKS ADDRESSED:
|
11
|
-
1. Organization Discovery Performance: 52.3s -> <30s target
|
11
|
+
1. Organization Discovery Performance: 52.3s -> <30s target
|
12
12
|
2. VPC Analysis Timeout Issues: Network operations optimization
|
13
13
|
3. Memory Usage Optimization: Large-scale operation memory management
|
14
14
|
4. Concurrent Processing: Multi-account parallel processing with rate limiting
|
15
15
|
|
16
16
|
ENTERPRISE FEATURES:
|
17
|
-
- Unified performance monitoring dashboard
|
17
|
+
- Unified performance monitoring dashboard
|
18
18
|
- Intelligent caching with TTL management
|
19
19
|
- Connection pooling for AWS API calls
|
20
20
|
- Memory-efficient batch processing
|
@@ -46,24 +46,18 @@ from runbooks.common.rich_utils import (
|
|
46
46
|
print_warning,
|
47
47
|
print_error,
|
48
48
|
create_table,
|
49
|
-
STATUS_INDICATORS
|
49
|
+
STATUS_INDICATORS,
|
50
50
|
)
|
51
51
|
|
52
52
|
from runbooks.common.performance_optimization_engine import (
|
53
53
|
PerformanceOptimizationEngine,
|
54
54
|
OptimizationMetrics,
|
55
|
-
get_optimization_engine
|
55
|
+
get_optimization_engine,
|
56
56
|
)
|
57
57
|
|
58
|
-
from runbooks.common.memory_optimization import
|
59
|
-
MemoryOptimizer,
|
60
|
-
get_memory_optimizer
|
61
|
-
)
|
58
|
+
from runbooks.common.memory_optimization import MemoryOptimizer, get_memory_optimizer
|
62
59
|
|
63
|
-
from runbooks.common.performance_monitor import
|
64
|
-
PerformanceBenchmark,
|
65
|
-
get_performance_benchmark
|
66
|
-
)
|
60
|
+
from runbooks.common.performance_monitor import PerformanceBenchmark, get_performance_benchmark
|
67
61
|
|
68
62
|
logger = logging.getLogger(__name__)
|
69
63
|
|
@@ -71,23 +65,24 @@ logger = logging.getLogger(__name__)
|
|
71
65
|
@dataclass
|
72
66
|
class SREPerformanceMetrics:
|
73
67
|
"""Comprehensive SRE performance metrics"""
|
68
|
+
|
74
69
|
operation_name: str
|
75
70
|
start_time: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
76
71
|
end_time: Optional[datetime] = None
|
77
72
|
total_duration_seconds: float = 0.0
|
78
73
|
target_duration_seconds: float = 30.0
|
79
|
-
|
74
|
+
|
80
75
|
# Performance optimization metrics
|
81
76
|
optimization_metrics: Optional[OptimizationMetrics] = None
|
82
77
|
memory_peak_mb: float = 0.0
|
83
78
|
memory_saved_mb: float = 0.0
|
84
|
-
|
85
|
-
# Infrastructure metrics
|
79
|
+
|
80
|
+
# Infrastructure metrics
|
86
81
|
aws_api_calls: int = 0
|
87
82
|
cache_hits: int = 0
|
88
83
|
cache_misses: int = 0
|
89
84
|
parallel_workers_used: int = 0
|
90
|
-
|
85
|
+
|
91
86
|
# Success metrics
|
92
87
|
success: bool = False
|
93
88
|
error_message: Optional[str] = None
|
@@ -109,23 +104,25 @@ class SREPerformanceMetrics:
|
|
109
104
|
class SREPerformanceSuite:
|
110
105
|
"""
|
111
106
|
Comprehensive SRE performance optimization suite
|
112
|
-
|
107
|
+
|
113
108
|
Integrates all performance optimization components:
|
114
|
-
- Performance optimization engine for AWS API optimization
|
109
|
+
- Performance optimization engine for AWS API optimization
|
115
110
|
- Memory optimization for large-scale operations
|
116
111
|
- Performance monitoring and benchmarking
|
117
112
|
- Real-time performance dashboard
|
118
113
|
- Automated performance reporting
|
119
114
|
"""
|
120
|
-
|
121
|
-
def __init__(
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
115
|
+
|
116
|
+
def __init__(
|
117
|
+
self,
|
118
|
+
max_workers: int = 20,
|
119
|
+
memory_limit_mb: int = 2048,
|
120
|
+
cache_ttl_minutes: int = 30,
|
121
|
+
performance_target_seconds: float = 30.0,
|
122
|
+
):
|
126
123
|
"""
|
127
124
|
Initialize comprehensive SRE performance suite
|
128
|
-
|
125
|
+
|
129
126
|
Args:
|
130
127
|
max_workers: Maximum concurrent workers for parallel operations
|
131
128
|
memory_limit_mb: Memory usage limit in MB
|
@@ -136,32 +133,31 @@ class SREPerformanceSuite:
|
|
136
133
|
self.memory_limit_mb = memory_limit_mb
|
137
134
|
self.cache_ttl_minutes = cache_ttl_minutes
|
138
135
|
self.performance_target_seconds = performance_target_seconds
|
139
|
-
|
136
|
+
|
140
137
|
# Initialize optimization components
|
141
138
|
self.optimization_engine = get_optimization_engine(
|
142
|
-
max_workers=max_workers,
|
143
|
-
cache_ttl_minutes=cache_ttl_minutes,
|
144
|
-
memory_limit_mb=memory_limit_mb
|
139
|
+
max_workers=max_workers, cache_ttl_minutes=cache_ttl_minutes, memory_limit_mb=memory_limit_mb
|
145
140
|
)
|
146
|
-
|
141
|
+
|
147
142
|
self.memory_optimizer = get_memory_optimizer(
|
148
|
-
warning_threshold_mb=memory_limit_mb * 0.7,
|
149
|
-
critical_threshold_mb=memory_limit_mb * 0.9
|
143
|
+
warning_threshold_mb=memory_limit_mb * 0.7, critical_threshold_mb=memory_limit_mb * 0.9
|
150
144
|
)
|
151
|
-
|
145
|
+
|
152
146
|
# Performance tracking
|
153
147
|
self.performance_metrics: List[SREPerformanceMetrics] = []
|
154
148
|
self.current_operation: Optional[SREPerformanceMetrics] = None
|
155
149
|
|
156
150
|
@contextmanager
|
157
|
-
def optimized_operation(
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
151
|
+
def optimized_operation(
|
152
|
+
self,
|
153
|
+
operation_name: str,
|
154
|
+
target_seconds: Optional[float] = None,
|
155
|
+
enable_memory_monitoring: bool = True,
|
156
|
+
enable_caching: bool = True,
|
157
|
+
):
|
162
158
|
"""
|
163
159
|
Context manager for comprehensive SRE-optimized operations
|
164
|
-
|
160
|
+
|
165
161
|
Integrates:
|
166
162
|
- Performance optimization engine
|
167
163
|
- Memory optimization and monitoring
|
@@ -169,53 +165,49 @@ class SREPerformanceSuite:
|
|
169
165
|
- Real-time metrics collection
|
170
166
|
"""
|
171
167
|
target = target_seconds or self.performance_target_seconds
|
172
|
-
|
168
|
+
|
173
169
|
# Initialize comprehensive metrics
|
174
|
-
metrics = SREPerformanceMetrics(
|
175
|
-
operation_name=operation_name,
|
176
|
-
target_duration_seconds=target
|
177
|
-
)
|
170
|
+
metrics = SREPerformanceMetrics(operation_name=operation_name, target_duration_seconds=target)
|
178
171
|
self.current_operation = metrics
|
179
|
-
|
172
|
+
|
180
173
|
# Start all optimization components
|
181
174
|
with self.optimization_engine.optimize_operation(operation_name, target) as opt_metrics:
|
182
175
|
with self.memory_optimizer.optimize_memory_usage(operation_name, enable_memory_monitoring) as mem_metrics:
|
183
|
-
|
184
176
|
# Start performance benchmark
|
185
177
|
benchmark = get_performance_benchmark("sre_suite")
|
186
|
-
|
178
|
+
|
187
179
|
with benchmark.measure_operation(operation_name, show_progress=True) as perf_metrics:
|
188
180
|
try:
|
189
181
|
console.log(f"[cyan]🚀 SRE-optimized operation: {operation_name} (target: {target}s)[/cyan]")
|
190
|
-
|
182
|
+
|
191
183
|
yield metrics
|
192
|
-
|
184
|
+
|
193
185
|
# Operation succeeded - collect metrics
|
194
186
|
metrics.end_time = datetime.now(timezone.utc)
|
195
187
|
metrics.total_duration_seconds = (metrics.end_time - metrics.start_time).total_seconds()
|
196
188
|
metrics.success = True
|
197
|
-
|
189
|
+
|
198
190
|
# Collect optimization metrics
|
199
191
|
metrics.optimization_metrics = opt_metrics
|
200
192
|
metrics.memory_peak_mb = mem_metrics.memory_peak_mb
|
201
193
|
metrics.memory_saved_mb = mem_metrics.memory_saved_mb
|
202
|
-
metrics.optimizations_applied = list(
|
203
|
-
opt_metrics.optimization_applied + mem_metrics.optimization_techniques_applied
|
204
|
-
)
|
205
|
-
|
194
|
+
metrics.optimizations_applied = list(
|
195
|
+
set(opt_metrics.optimization_applied + mem_metrics.optimization_techniques_applied)
|
196
|
+
)
|
197
|
+
|
206
198
|
# Calculate performance grade
|
207
199
|
improvement = metrics.calculate_performance_improvement()
|
208
200
|
if improvement >= 20:
|
209
201
|
metrics.performance_grade = "A"
|
210
202
|
elif improvement >= 10:
|
211
|
-
metrics.performance_grade = "B"
|
203
|
+
metrics.performance_grade = "B"
|
212
204
|
elif metrics.total_duration_seconds <= target:
|
213
205
|
metrics.performance_grade = "C"
|
214
206
|
else:
|
215
207
|
metrics.performance_grade = "D"
|
216
|
-
|
208
|
+
|
217
209
|
self._log_comprehensive_results(metrics)
|
218
|
-
|
210
|
+
|
219
211
|
except Exception as e:
|
220
212
|
# Handle operation failure
|
221
213
|
metrics.end_time = datetime.now(timezone.utc)
|
@@ -223,112 +215,104 @@ class SREPerformanceSuite:
|
|
223
215
|
metrics.success = False
|
224
216
|
metrics.error_message = str(e)
|
225
217
|
metrics.performance_grade = "F"
|
226
|
-
|
218
|
+
|
227
219
|
print_error(f"SRE-optimized operation failed: {operation_name}", e)
|
228
220
|
raise
|
229
|
-
|
221
|
+
|
230
222
|
finally:
|
231
223
|
# Store metrics and cleanup
|
232
224
|
self.performance_metrics.append(metrics)
|
233
225
|
self.current_operation = None
|
234
226
|
|
235
|
-
async def optimize_organization_discovery(
|
236
|
-
|
237
|
-
|
227
|
+
async def optimize_organization_discovery(
|
228
|
+
self, management_profile: str, target_seconds: float = 30.0
|
229
|
+
) -> Dict[str, Any]:
|
238
230
|
"""
|
239
231
|
Optimize organization discovery with comprehensive SRE patterns
|
240
|
-
|
232
|
+
|
241
233
|
Addresses: Organization Discovery Performance (52.3s -> <30s target)
|
242
234
|
"""
|
243
235
|
with self.optimized_operation("organization_discovery_optimization", target_seconds):
|
244
|
-
|
245
236
|
# Use optimized discovery function from performance engine
|
246
237
|
optimized_discovery = self.optimization_engine.optimize_organization_discovery(
|
247
|
-
management_profile=management_profile,
|
248
|
-
use_parallel_processing=True,
|
249
|
-
batch_size=20
|
238
|
+
management_profile=management_profile, use_parallel_processing=True, batch_size=20
|
250
239
|
)
|
251
|
-
|
240
|
+
|
252
241
|
# Execute optimized discovery
|
253
242
|
result = optimized_discovery()
|
254
|
-
|
243
|
+
|
255
244
|
# Update metrics
|
256
245
|
if self.current_operation:
|
257
|
-
self.current_operation.aws_api_calls = result.get(
|
246
|
+
self.current_operation.aws_api_calls = result.get("api_calls", 0)
|
258
247
|
self.current_operation.cache_hits = self.optimization_engine.cache.hits
|
259
248
|
self.current_operation.cache_misses = self.optimization_engine.cache.misses
|
260
|
-
|
249
|
+
|
261
250
|
return {
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
}
|
251
|
+
"discovery_result": result,
|
252
|
+
"performance_metrics": self.current_operation,
|
253
|
+
"optimization_summary": {
|
254
|
+
"target_achieved": self.current_operation.total_duration_seconds <= target_seconds,
|
255
|
+
"performance_improvement": self.current_operation.calculate_performance_improvement(),
|
256
|
+
"optimizations_applied": self.current_operation.optimizations_applied,
|
257
|
+
},
|
269
258
|
}
|
270
259
|
|
271
|
-
async def optimize_vpc_analysis(
|
272
|
-
|
273
|
-
|
274
|
-
target_seconds: float = 180.0) -> Dict[str, Any]:
|
260
|
+
async def optimize_vpc_analysis(
|
261
|
+
self, operational_profile: str, regions: Optional[List[str]] = None, target_seconds: float = 180.0
|
262
|
+
) -> Dict[str, Any]:
|
275
263
|
"""
|
276
264
|
Optimize VPC analysis with comprehensive SRE patterns
|
277
|
-
|
265
|
+
|
278
266
|
Addresses: VPC Analysis Timeout Issues
|
279
267
|
"""
|
280
268
|
with self.optimized_operation("vpc_analysis_optimization", target_seconds):
|
281
|
-
|
282
269
|
# Import and use the optimized VPC analyzer
|
283
270
|
from runbooks.vpc.performance_optimized_analyzer import create_optimized_vpc_analyzer
|
284
|
-
|
271
|
+
|
285
272
|
analyzer = create_optimized_vpc_analyzer(
|
286
273
|
operational_profile=operational_profile,
|
287
|
-
max_workers=min(self.max_workers, 15) # Limit workers for VPC analysis
|
274
|
+
max_workers=min(self.max_workers, 15), # Limit workers for VPC analysis
|
288
275
|
)
|
289
|
-
|
276
|
+
|
290
277
|
# Execute optimized global VPC analysis
|
291
|
-
result = await analyzer.analyze_vpcs_globally(
|
292
|
-
|
293
|
-
include_detailed_analysis=True
|
294
|
-
)
|
295
|
-
|
278
|
+
result = await analyzer.analyze_vpcs_globally(regions=regions, include_detailed_analysis=True)
|
279
|
+
|
296
280
|
# Update metrics
|
297
281
|
if self.current_operation:
|
298
|
-
analysis_summary = result.get(
|
299
|
-
perf_metrics = result.get(
|
300
|
-
|
301
|
-
self.current_operation.aws_api_calls = perf_metrics.get(
|
282
|
+
analysis_summary = result.get("analysis_summary", {})
|
283
|
+
perf_metrics = result.get("performance_metrics", {})
|
284
|
+
|
285
|
+
self.current_operation.aws_api_calls = perf_metrics.get("total_api_calls", 0)
|
302
286
|
self.current_operation.parallel_workers_used = self.max_workers
|
303
|
-
|
287
|
+
|
304
288
|
return {
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
}
|
289
|
+
"vpc_analysis_result": result,
|
290
|
+
"performance_metrics": self.current_operation,
|
291
|
+
"optimization_summary": {
|
292
|
+
"target_achieved": self.current_operation.total_duration_seconds <= target_seconds,
|
293
|
+
"regions_analyzed": result.get("analysis_summary", {}).get("total_regions_analyzed", 0),
|
294
|
+
"vpcs_discovered": result.get("analysis_summary", {}).get("total_vpcs_discovered", 0),
|
295
|
+
"performance_grade": result.get("analysis_summary", {}).get("performance_grade", "N/A"),
|
296
|
+
},
|
313
297
|
}
|
314
298
|
|
315
299
|
def create_performance_dashboard(self) -> None:
|
316
300
|
"""Create comprehensive SRE performance dashboard"""
|
317
301
|
print_header("SRE Performance Optimization Dashboard", "Enterprise Performance Suite")
|
318
|
-
|
302
|
+
|
319
303
|
if not self.performance_metrics:
|
320
304
|
console.print("[yellow]No performance metrics available yet[/yellow]")
|
321
305
|
return
|
322
|
-
|
306
|
+
|
323
307
|
# Performance summary table
|
324
308
|
self._create_performance_summary_table()
|
325
|
-
|
309
|
+
|
326
310
|
# Optimization details table
|
327
311
|
self._create_optimization_details_table()
|
328
|
-
|
312
|
+
|
329
313
|
# System resource status
|
330
314
|
self._create_resource_status_panel()
|
331
|
-
|
315
|
+
|
332
316
|
# Performance recommendations
|
333
317
|
self._create_performance_recommendations()
|
334
318
|
|
@@ -344,19 +328,19 @@ class SREPerformanceSuite:
|
|
344
328
|
{"name": "Improvement", "style": "green", "justify": "right"},
|
345
329
|
{"name": "Memory (MB)", "style": "blue", "justify": "right"},
|
346
330
|
{"name": "API Calls", "style": "yellow", "justify": "right"},
|
347
|
-
{"name": "Status", "style": "white", "justify": "center"}
|
348
|
-
]
|
331
|
+
{"name": "Status", "style": "white", "justify": "center"},
|
332
|
+
],
|
349
333
|
)
|
350
|
-
|
334
|
+
|
351
335
|
for metrics in self.performance_metrics:
|
352
336
|
improvement = metrics.calculate_performance_improvement()
|
353
|
-
status_icon = STATUS_INDICATORS[
|
354
|
-
status_color =
|
355
|
-
|
337
|
+
status_icon = STATUS_INDICATORS["success"] if metrics.success else STATUS_INDICATORS["error"]
|
338
|
+
status_color = "green" if metrics.success else "red"
|
339
|
+
|
356
340
|
# Grade color coding
|
357
|
-
grade_colors = {
|
358
|
-
grade_color = grade_colors.get(metrics.performance_grade,
|
359
|
-
|
341
|
+
grade_colors = {"A": "green", "B": "green", "C": "yellow", "D": "red", "F": "red"}
|
342
|
+
grade_color = grade_colors.get(metrics.performance_grade, "white")
|
343
|
+
|
360
344
|
table.add_row(
|
361
345
|
metrics.operation_name,
|
362
346
|
f"{metrics.total_duration_seconds:.1f}s",
|
@@ -365,9 +349,9 @@ class SREPerformanceSuite:
|
|
365
349
|
f"+{improvement:.1f}%" if improvement > 0 else f"{improvement:.1f}%",
|
366
350
|
f"{metrics.memory_peak_mb:.1f}",
|
367
351
|
str(metrics.aws_api_calls),
|
368
|
-
f"[{status_color}]{status_icon}[/{status_color}]"
|
352
|
+
f"[{status_color}]{status_icon}[/{status_color}]",
|
369
353
|
)
|
370
|
-
|
354
|
+
|
371
355
|
console.print(table)
|
372
356
|
|
373
357
|
def _create_optimization_details_table(self):
|
@@ -379,22 +363,23 @@ class SREPerformanceSuite:
|
|
379
363
|
{"name": "Cache Efficiency", "style": "blue", "justify": "right"},
|
380
364
|
{"name": "Memory Saved", "style": "green", "justify": "right"},
|
381
365
|
{"name": "Workers Used", "style": "yellow", "justify": "right"},
|
382
|
-
{"name": "Optimizations", "style": "dim", "justify": "left", "max_width": 40}
|
383
|
-
]
|
366
|
+
{"name": "Optimizations", "style": "dim", "justify": "left", "max_width": 40},
|
367
|
+
],
|
384
368
|
)
|
385
|
-
|
369
|
+
|
386
370
|
for metrics in self.performance_metrics:
|
387
371
|
cache_efficiency = metrics.get_cache_efficiency()
|
388
372
|
memory_saved = f"+{metrics.memory_saved_mb:.1f}MB" if metrics.memory_saved_mb > 0 else "0MB"
|
389
|
-
|
373
|
+
|
390
374
|
table.add_row(
|
391
375
|
metrics.operation_name,
|
392
376
|
f"{cache_efficiency:.1f}%",
|
393
377
|
memory_saved,
|
394
378
|
str(metrics.parallel_workers_used),
|
395
|
-
", ".join(metrics.optimizations_applied[:3])
|
379
|
+
", ".join(metrics.optimizations_applied[:3])
|
380
|
+
+ ("..." if len(metrics.optimizations_applied) > 3 else ""),
|
396
381
|
)
|
397
|
-
|
382
|
+
|
398
383
|
console.print(table)
|
399
384
|
|
400
385
|
def _create_resource_status_panel(self):
|
@@ -402,75 +387,74 @@ class SREPerformanceSuite:
|
|
402
387
|
# Get current resource status
|
403
388
|
memory_report = self.memory_optimizer.get_memory_usage_report()
|
404
389
|
cache_stats = self.optimization_engine.cache.get_stats()
|
405
|
-
|
390
|
+
|
406
391
|
# Status colors
|
407
|
-
memory_color = {
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
'critical': 'red'
|
412
|
-
}.get(memory_report.get('memory_status', 'good'), 'white')
|
413
|
-
|
392
|
+
memory_color = {"good": "green", "moderate": "yellow", "warning": "yellow", "critical": "red"}.get(
|
393
|
+
memory_report.get("memory_status", "good"), "white"
|
394
|
+
)
|
395
|
+
|
414
396
|
status_text = f"""
|
415
|
-
[bold cyan]💾 Memory Status:[/bold cyan] [{memory_color}]{memory_report[
|
397
|
+
[bold cyan]💾 Memory Status:[/bold cyan] [{memory_color}]{memory_report["memory_status"].upper()}[/{memory_color}] ({memory_report["current_memory_mb"]:.1f}MB / {memory_report["critical_threshold_mb"]:.0f}MB)
|
416
398
|
|
417
|
-
[bold blue]🗄️ Cache Performance:[/bold blue] {cache_stats[
|
399
|
+
[bold blue]🗄️ Cache Performance:[/bold blue] {cache_stats["hit_rate"]:.1f}% hit rate ({cache_stats["hits"]} hits, {cache_stats["misses"]} misses)
|
418
400
|
|
419
|
-
[bold yellow]🔧 System Resources:[/bold yellow] {cache_stats[
|
401
|
+
[bold yellow]🔧 System Resources:[/bold yellow] {cache_stats["size"]}/{cache_stats["max_size"]} cache entries, {memory_report["active_objects"]:,} active objects
|
420
402
|
"""
|
421
|
-
|
422
|
-
console.print(
|
423
|
-
status_text.strip(),
|
424
|
-
|
425
|
-
border_style=memory_color
|
426
|
-
))
|
403
|
+
|
404
|
+
console.print(
|
405
|
+
Panel(status_text.strip(), title="[bold]System Resource Status[/bold]", border_style=memory_color)
|
406
|
+
)
|
427
407
|
|
428
408
|
def _create_performance_recommendations(self):
|
429
409
|
"""Create performance optimization recommendations"""
|
430
410
|
recommendations = []
|
431
|
-
|
411
|
+
|
432
412
|
# Analyze recent performance metrics
|
433
413
|
if self.performance_metrics:
|
434
414
|
recent_metrics = self.performance_metrics[-5:] # Last 5 operations
|
435
|
-
|
415
|
+
|
436
416
|
# Check for performance issues
|
437
417
|
slow_operations = [m for m in recent_metrics if m.total_duration_seconds > m.target_duration_seconds]
|
438
418
|
if slow_operations:
|
439
419
|
recommendations.append(f"🐌 {len(slow_operations)} operations exceeded target duration")
|
440
|
-
|
420
|
+
|
441
421
|
# Check memory usage
|
442
422
|
high_memory_ops = [m for m in recent_metrics if m.memory_peak_mb > self.memory_limit_mb * 0.8]
|
443
423
|
if high_memory_ops:
|
444
424
|
recommendations.append(f"🧠 {len(high_memory_ops)} operations had high memory usage")
|
445
|
-
|
425
|
+
|
446
426
|
# Check cache efficiency
|
447
427
|
low_cache_ops = [m for m in recent_metrics if m.get_cache_efficiency() < 50]
|
448
428
|
if low_cache_ops:
|
449
429
|
recommendations.append(f"📦 {len(low_cache_ops)} operations had low cache efficiency")
|
450
|
-
|
430
|
+
|
451
431
|
# System-level recommendations
|
452
432
|
memory_report = self.memory_optimizer.get_memory_usage_report()
|
453
|
-
recommendations.extend(memory_report.get(
|
454
|
-
|
433
|
+
recommendations.extend(memory_report.get("optimization_recommendations", []))
|
434
|
+
|
455
435
|
if recommendations:
|
456
436
|
recommendations_text = "\n".join(f"• {rec}" for rec in recommendations[:8])
|
457
|
-
console.print(
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
437
|
+
console.print(
|
438
|
+
Panel(
|
439
|
+
recommendations_text,
|
440
|
+
title="[bold yellow]Performance Recommendations[/bold yellow]",
|
441
|
+
border_style="yellow",
|
442
|
+
)
|
443
|
+
)
|
462
444
|
else:
|
463
|
-
console.print(
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
445
|
+
console.print(
|
446
|
+
Panel(
|
447
|
+
"[green]✅ System performance is optimal - no recommendations at this time[/green]",
|
448
|
+
title="[bold green]Performance Status[/bold green]",
|
449
|
+
border_style="green",
|
450
|
+
)
|
451
|
+
)
|
468
452
|
|
469
453
|
def _log_comprehensive_results(self, metrics: SREPerformanceMetrics):
|
470
454
|
"""Log comprehensive SRE optimization results"""
|
471
455
|
improvement = metrics.calculate_performance_improvement()
|
472
456
|
cache_efficiency = metrics.get_cache_efficiency()
|
473
|
-
|
457
|
+
|
474
458
|
if metrics.success:
|
475
459
|
if improvement > 0:
|
476
460
|
print_success(
|
@@ -482,35 +466,43 @@ class SREPerformanceSuite:
|
|
482
466
|
f"[green]SRE operation completed: {metrics.operation_name} "
|
483
467
|
f"({metrics.total_duration_seconds:.1f}s, Grade: {metrics.performance_grade})[/green]"
|
484
468
|
)
|
485
|
-
|
469
|
+
|
486
470
|
# Log optimization details
|
487
471
|
if metrics.optimizations_applied:
|
488
472
|
console.log(f"[dim]🔧 Optimizations: {', '.join(metrics.optimizations_applied)}[/dim]")
|
489
|
-
|
473
|
+
|
490
474
|
if cache_efficiency > 0:
|
491
475
|
console.log(f"[dim]📦 Cache: {cache_efficiency:.1f}% efficiency ({metrics.cache_hits} hits)[/dim]")
|
492
|
-
|
476
|
+
|
493
477
|
if metrics.memory_saved_mb > 0:
|
494
|
-
console.log(
|
478
|
+
console.log(
|
479
|
+
f"[dim]🧠 Memory: {metrics.memory_saved_mb:.1f}MB saved (peak: {metrics.memory_peak_mb:.1f}MB)[/dim]"
|
480
|
+
)
|
495
481
|
|
496
482
|
def get_performance_summary(self) -> Dict[str, Any]:
|
497
483
|
"""Get comprehensive performance summary"""
|
498
484
|
if not self.performance_metrics:
|
499
485
|
return {"status": "no_data", "message": "No performance metrics available"}
|
500
|
-
|
486
|
+
|
501
487
|
# Calculate aggregate statistics
|
502
488
|
successful_ops = [m for m in self.performance_metrics if m.success]
|
503
489
|
total_ops = len(self.performance_metrics)
|
504
|
-
|
505
|
-
avg_duration =
|
506
|
-
|
490
|
+
|
491
|
+
avg_duration = (
|
492
|
+
sum(m.total_duration_seconds for m in successful_ops) / len(successful_ops) if successful_ops else 0
|
493
|
+
)
|
494
|
+
avg_improvement = (
|
495
|
+
sum(m.calculate_performance_improvement() for m in successful_ops) / len(successful_ops)
|
496
|
+
if successful_ops
|
497
|
+
else 0
|
498
|
+
)
|
507
499
|
success_rate = len(successful_ops) / total_ops if total_ops > 0 else 0
|
508
|
-
|
500
|
+
|
509
501
|
# Performance grade distribution
|
510
502
|
grade_counts = {}
|
511
503
|
for metrics in self.performance_metrics:
|
512
504
|
grade_counts[metrics.performance_grade] = grade_counts.get(metrics.performance_grade, 0) + 1
|
513
|
-
|
505
|
+
|
514
506
|
return {
|
515
507
|
"status": "active",
|
516
508
|
"total_operations": total_ops,
|
@@ -521,16 +513,17 @@ class SREPerformanceSuite:
|
|
521
513
|
"performance_grade_distribution": grade_counts,
|
522
514
|
"system_resources": {
|
523
515
|
"memory_status": self.memory_optimizer.get_memory_usage_report(),
|
524
|
-
"cache_status": self.optimization_engine.cache.get_stats()
|
516
|
+
"cache_status": self.optimization_engine.cache.get_stats(),
|
525
517
|
},
|
526
518
|
"recent_operations": [
|
527
519
|
{
|
528
520
|
"name": m.operation_name,
|
529
521
|
"duration": m.total_duration_seconds,
|
530
522
|
"grade": m.performance_grade,
|
531
|
-
"success": m.success
|
532
|
-
}
|
533
|
-
|
523
|
+
"success": m.success,
|
524
|
+
}
|
525
|
+
for m in self.performance_metrics[-5:]
|
526
|
+
],
|
534
527
|
}
|
535
528
|
|
536
529
|
def clear_performance_data(self):
|
@@ -545,16 +538,14 @@ class SREPerformanceSuite:
|
|
545
538
|
_sre_performance_suite: Optional[SREPerformanceSuite] = None
|
546
539
|
|
547
540
|
|
548
|
-
def get_sre_performance_suite(
|
549
|
-
|
550
|
-
|
541
|
+
def get_sre_performance_suite(
|
542
|
+
max_workers: int = 20, memory_limit_mb: int = 2048, cache_ttl_minutes: int = 30
|
543
|
+
) -> SREPerformanceSuite:
|
551
544
|
"""Get or create global SRE performance suite instance"""
|
552
545
|
global _sre_performance_suite
|
553
546
|
if _sre_performance_suite is None:
|
554
547
|
_sre_performance_suite = SREPerformanceSuite(
|
555
|
-
max_workers=max_workers,
|
556
|
-
memory_limit_mb=memory_limit_mb,
|
557
|
-
cache_ttl_minutes=cache_ttl_minutes
|
548
|
+
max_workers=max_workers, memory_limit_mb=memory_limit_mb, cache_ttl_minutes=cache_ttl_minutes
|
558
549
|
)
|
559
550
|
return _sre_performance_suite
|
560
551
|
|
@@ -569,6 +560,6 @@ def create_sre_performance_dashboard():
|
|
569
560
|
__all__ = [
|
570
561
|
"SREPerformanceSuite",
|
571
562
|
"SREPerformanceMetrics",
|
572
|
-
"get_sre_performance_suite",
|
573
|
-
"create_sre_performance_dashboard"
|
574
|
-
]
|
563
|
+
"get_sre_performance_suite",
|
564
|
+
"create_sre_performance_dashboard",
|
565
|
+
]
|