runbooks 0.7.6__py3-none-any.whl → 0.7.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +1 -1
- runbooks/base.py +5 -1
- runbooks/cfat/__init__.py +8 -4
- runbooks/cfat/assessment/collectors.py +171 -14
- runbooks/cfat/assessment/compliance.py +871 -0
- runbooks/cfat/assessment/runner.py +122 -11
- runbooks/cfat/models.py +6 -2
- runbooks/common/logger.py +14 -0
- runbooks/common/rich_utils.py +451 -0
- runbooks/enterprise/__init__.py +68 -0
- runbooks/enterprise/error_handling.py +411 -0
- runbooks/enterprise/logging.py +439 -0
- runbooks/enterprise/multi_tenant.py +583 -0
- runbooks/finops/README.md +468 -241
- runbooks/finops/__init__.py +39 -3
- runbooks/finops/cli.py +83 -18
- runbooks/finops/cross_validation.py +375 -0
- runbooks/finops/dashboard_runner.py +812 -164
- runbooks/finops/enhanced_dashboard_runner.py +525 -0
- runbooks/finops/finops_dashboard.py +1892 -0
- runbooks/finops/helpers.py +485 -51
- runbooks/finops/optimizer.py +823 -0
- runbooks/finops/tests/__init__.py +19 -0
- runbooks/finops/tests/results_test_finops_dashboard.xml +1 -0
- runbooks/finops/tests/run_comprehensive_tests.py +421 -0
- runbooks/finops/tests/run_tests.py +305 -0
- runbooks/finops/tests/test_finops_dashboard.py +705 -0
- runbooks/finops/tests/test_integration.py +477 -0
- runbooks/finops/tests/test_performance.py +380 -0
- runbooks/finops/tests/test_performance_benchmarks.py +500 -0
- runbooks/finops/tests/test_reference_images_validation.py +867 -0
- runbooks/finops/tests/test_single_account_features.py +715 -0
- runbooks/finops/tests/validate_test_suite.py +220 -0
- runbooks/finops/types.py +1 -1
- runbooks/hitl/enhanced_workflow_engine.py +725 -0
- runbooks/inventory/artifacts/scale-optimize-status.txt +12 -0
- runbooks/inventory/collectors/aws_comprehensive.py +442 -0
- runbooks/inventory/collectors/enterprise_scale.py +281 -0
- runbooks/inventory/core/collector.py +172 -13
- runbooks/inventory/discovery.md +1 -1
- runbooks/inventory/list_ec2_instances.py +18 -20
- runbooks/inventory/list_ssm_parameters.py +31 -3
- runbooks/inventory/organizations_discovery.py +1269 -0
- runbooks/inventory/rich_inventory_display.py +393 -0
- runbooks/inventory/run_on_multi_accounts.py +35 -19
- runbooks/inventory/runbooks.security.report_generator.log +0 -0
- runbooks/inventory/runbooks.security.run_script.log +0 -0
- runbooks/inventory/vpc_flow_analyzer.py +1030 -0
- runbooks/main.py +2215 -119
- runbooks/metrics/dora_metrics_engine.py +599 -0
- runbooks/operate/__init__.py +2 -2
- runbooks/operate/base.py +122 -10
- runbooks/operate/deployment_framework.py +1032 -0
- runbooks/operate/deployment_validator.py +853 -0
- runbooks/operate/dynamodb_operations.py +10 -6
- runbooks/operate/ec2_operations.py +319 -11
- runbooks/operate/executive_dashboard.py +779 -0
- runbooks/operate/mcp_integration.py +750 -0
- runbooks/operate/nat_gateway_operations.py +1120 -0
- runbooks/operate/networking_cost_heatmap.py +685 -0
- runbooks/operate/privatelink_operations.py +940 -0
- runbooks/operate/s3_operations.py +10 -6
- runbooks/operate/vpc_endpoints.py +644 -0
- runbooks/operate/vpc_operations.py +1038 -0
- runbooks/remediation/__init__.py +2 -2
- runbooks/remediation/acm_remediation.py +1 -1
- runbooks/remediation/base.py +1 -1
- runbooks/remediation/cloudtrail_remediation.py +1 -1
- runbooks/remediation/cognito_remediation.py +1 -1
- runbooks/remediation/dynamodb_remediation.py +1 -1
- runbooks/remediation/ec2_remediation.py +1 -1
- runbooks/remediation/ec2_unattached_ebs_volumes.py +1 -1
- runbooks/remediation/kms_enable_key_rotation.py +1 -1
- runbooks/remediation/kms_remediation.py +1 -1
- runbooks/remediation/lambda_remediation.py +1 -1
- runbooks/remediation/multi_account.py +1 -1
- runbooks/remediation/rds_remediation.py +1 -1
- runbooks/remediation/s3_block_public_access.py +1 -1
- runbooks/remediation/s3_enable_access_logging.py +1 -1
- runbooks/remediation/s3_encryption.py +1 -1
- runbooks/remediation/s3_remediation.py +1 -1
- runbooks/remediation/vpc_remediation.py +475 -0
- runbooks/security/__init__.py +3 -1
- runbooks/security/compliance_automation.py +632 -0
- runbooks/security/report_generator.py +10 -0
- runbooks/security/run_script.py +31 -5
- runbooks/security/security_baseline_tester.py +169 -30
- runbooks/security/security_export.py +477 -0
- runbooks/validation/__init__.py +10 -0
- runbooks/validation/benchmark.py +484 -0
- runbooks/validation/cli.py +356 -0
- runbooks/validation/mcp_validator.py +768 -0
- runbooks/vpc/__init__.py +38 -0
- runbooks/vpc/config.py +212 -0
- runbooks/vpc/cost_engine.py +347 -0
- runbooks/vpc/heatmap_engine.py +605 -0
- runbooks/vpc/manager_interface.py +634 -0
- runbooks/vpc/networking_wrapper.py +1260 -0
- runbooks/vpc/rich_formatters.py +679 -0
- runbooks/vpc/tests/__init__.py +5 -0
- runbooks/vpc/tests/conftest.py +356 -0
- runbooks/vpc/tests/test_cli_integration.py +530 -0
- runbooks/vpc/tests/test_config.py +458 -0
- runbooks/vpc/tests/test_cost_engine.py +479 -0
- runbooks/vpc/tests/test_networking_wrapper.py +512 -0
- {runbooks-0.7.6.dist-info → runbooks-0.7.9.dist-info}/METADATA +40 -12
- {runbooks-0.7.6.dist-info → runbooks-0.7.9.dist-info}/RECORD +111 -50
- {runbooks-0.7.6.dist-info → runbooks-0.7.9.dist-info}/WHEEL +0 -0
- {runbooks-0.7.6.dist-info → runbooks-0.7.9.dist-info}/entry_points.txt +0 -0
- {runbooks-0.7.6.dist-info → runbooks-0.7.9.dist-info}/licenses/LICENSE +0 -0
- {runbooks-0.7.6.dist-info → runbooks-0.7.9.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,685 @@
|
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
"""
|
3
|
+
AWS Networking Cost Heat Map Generator
|
4
|
+
|
5
|
+
This module provides comprehensive networking cost heat map generation capabilities
|
6
|
+
for the CloudOps-Runbooks framework, supporting Terminal 4 (Cost Agent) operations
|
7
|
+
with real-time MCP integration.
|
8
|
+
|
9
|
+
Key Features:
|
10
|
+
- Multi-account networking cost analysis
|
11
|
+
- Interactive heat map generation
|
12
|
+
- Cost hotspot identification
|
13
|
+
- Optimization scenario modeling
|
14
|
+
- MCP server integration for real-time validation
|
15
|
+
- Executive dashboard generation with ROI projections
|
16
|
+
|
17
|
+
Integration Points:
|
18
|
+
- CloudOps-Runbooks CLI: `runbooks vpc networking-cost-heatmap`
|
19
|
+
- MCP Servers: Real-time AWS API validation
|
20
|
+
- Terminal Coordination: Multi-agent workflow support
|
21
|
+
"""
|
22
|
+
|
23
|
+
import logging
|
24
|
+
from dataclasses import dataclass, field
|
25
|
+
from datetime import datetime, timedelta
|
26
|
+
from pathlib import Path
|
27
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
28
|
+
|
29
|
+
import boto3
|
30
|
+
import numpy as np
|
31
|
+
import pandas as pd
|
32
|
+
from botocore.exceptions import ClientError, NoCredentialsError
|
33
|
+
|
34
|
+
from ..base import BaseOperation, OperationResult
|
35
|
+
|
36
|
+
logger = logging.getLogger(__name__)
|
37
|
+
|
38
|
+
|
39
|
+
@dataclass
|
40
|
+
class NetworkingCostHeatMapConfig:
|
41
|
+
"""Configuration for networking cost heat map generation"""
|
42
|
+
|
43
|
+
# AWS Profiles (READ-ONLY)
|
44
|
+
billing_profile: str = "ams-admin-Billing-ReadOnlyAccess-909135376185"
|
45
|
+
centralized_ops_profile: str = "ams-centralised-ops-ReadOnlyAccess-335083429030"
|
46
|
+
single_account_profile: str = "ams-shared-services-non-prod-ReadOnlyAccess-499201730520"
|
47
|
+
management_profile: str = "ams-admin-ReadOnlyAccess-909135376185"
|
48
|
+
|
49
|
+
# Analysis parameters
|
50
|
+
regions: List[str] = field(
|
51
|
+
default_factory=lambda: [
|
52
|
+
"us-east-1",
|
53
|
+
"us-west-2",
|
54
|
+
"us-west-1",
|
55
|
+
"eu-west-1",
|
56
|
+
"eu-central-1",
|
57
|
+
"eu-west-2",
|
58
|
+
"ap-southeast-1",
|
59
|
+
"ap-southeast-2",
|
60
|
+
"ap-northeast-1",
|
61
|
+
]
|
62
|
+
)
|
63
|
+
|
64
|
+
# Time frames
|
65
|
+
analysis_days: int = 90
|
66
|
+
forecast_days: int = 90
|
67
|
+
|
68
|
+
# Cost thresholds for heat map coloring
|
69
|
+
low_cost_threshold: float = 10.0
|
70
|
+
medium_cost_threshold: float = 50.0
|
71
|
+
high_cost_threshold: float = 100.0
|
72
|
+
critical_cost_threshold: float = 200.0
|
73
|
+
|
74
|
+
# Service baselines (monthly)
|
75
|
+
nat_gateway_baseline: float = 45.0
|
76
|
+
transit_gateway_baseline: float = 36.50
|
77
|
+
elastic_ip_idle: float = 3.60
|
78
|
+
vpc_endpoint_interface: float = 7.20
|
79
|
+
|
80
|
+
# Heat map configuration
|
81
|
+
generate_interactive_maps: bool = True
|
82
|
+
export_data: bool = True
|
83
|
+
include_optimization_scenarios: bool = True
|
84
|
+
|
85
|
+
# MCP integration
|
86
|
+
enable_mcp_validation: bool = True
|
87
|
+
mcp_tolerance_percent: float = 10.0
|
88
|
+
|
89
|
+
# Safety settings
|
90
|
+
read_only_mode: bool = True
|
91
|
+
dry_run_mode: bool = True
|
92
|
+
|
93
|
+
|
94
|
+
class NetworkingCostHeatMapOperation(BaseOperation):
|
95
|
+
"""AWS Networking Cost Heat Map Generation Operation"""
|
96
|
+
|
97
|
+
NETWORKING_SERVICES = {
|
98
|
+
"vpc": "Amazon Virtual Private Cloud",
|
99
|
+
"transit_gateway": "AWS Transit Gateway",
|
100
|
+
"nat_gateway": "NAT Gateway",
|
101
|
+
"vpc_endpoint": "VPC Endpoint",
|
102
|
+
"elastic_ip": "Elastic IP",
|
103
|
+
"data_transfer": "Data Transfer",
|
104
|
+
}
|
105
|
+
|
106
|
+
def __init__(self, profile: str = None, region: str = None):
|
107
|
+
super().__init__(profile, region)
|
108
|
+
self.config = NetworkingCostHeatMapConfig()
|
109
|
+
self.cost_models = self._initialize_cost_models()
|
110
|
+
self.heat_map_data = {}
|
111
|
+
|
112
|
+
def _initialize_cost_models(self) -> Dict[str, Dict]:
|
113
|
+
"""Initialize networking cost models"""
|
114
|
+
return {
|
115
|
+
"nat_gateway": {
|
116
|
+
"hourly_base": self.config.nat_gateway_baseline / (30 * 24),
|
117
|
+
"monthly_base": self.config.nat_gateway_baseline,
|
118
|
+
"data_processing_per_gb": 0.045,
|
119
|
+
"availability_zones": 3,
|
120
|
+
"peak_utilization_multiplier": 1.5,
|
121
|
+
},
|
122
|
+
"transit_gateway": {
|
123
|
+
"hourly_base": self.config.transit_gateway_baseline / (30 * 24),
|
124
|
+
"monthly_base": self.config.transit_gateway_baseline,
|
125
|
+
"attachment_monthly": 36.50,
|
126
|
+
"data_processing_per_gb": 0.02,
|
127
|
+
"typical_attachments": 5,
|
128
|
+
},
|
129
|
+
"vpc_endpoint": {
|
130
|
+
"interface_hourly": self.config.vpc_endpoint_interface / (30 * 24),
|
131
|
+
"interface_monthly": self.config.vpc_endpoint_interface,
|
132
|
+
"gateway_monthly": 0.0,
|
133
|
+
"data_processing_per_gb": 0.01,
|
134
|
+
"typical_per_az": 2,
|
135
|
+
},
|
136
|
+
"elastic_ip": {
|
137
|
+
"idle_monthly": self.config.elastic_ip_idle,
|
138
|
+
"attached_monthly": 0.0,
|
139
|
+
"remap_fee": 0.1,
|
140
|
+
"typical_idle_count": 3,
|
141
|
+
},
|
142
|
+
"data_transfer": {"inter_az": 0.01, "inter_region": 0.02, "internet_out": 0.09, "typical_monthly_gb": 1000},
|
143
|
+
"vpc_flow_logs": {"cloudwatch_logs_per_gb": 0.50, "s3_storage_per_gb": 0.023, "typical_monthly_gb": 50},
|
144
|
+
}
|
145
|
+
|
146
|
+
def generate_comprehensive_heat_maps(
|
147
|
+
self,
|
148
|
+
account_ids: Optional[List[str]] = None,
|
149
|
+
include_single_account: bool = True,
|
150
|
+
include_multi_account: bool = True,
|
151
|
+
) -> OperationResult:
|
152
|
+
"""
|
153
|
+
Generate comprehensive networking cost heat maps
|
154
|
+
|
155
|
+
Args:
|
156
|
+
account_ids: List of account IDs to analyze
|
157
|
+
include_single_account: Include single account analysis
|
158
|
+
include_multi_account: Include multi-account aggregated view
|
159
|
+
|
160
|
+
Returns:
|
161
|
+
OperationResult with heat map data and analysis
|
162
|
+
"""
|
163
|
+
|
164
|
+
try:
|
165
|
+
logger.info("Starting comprehensive networking cost heat map generation")
|
166
|
+
|
167
|
+
# Initialize result structure
|
168
|
+
heat_map_results = {
|
169
|
+
"generation_timestamp": datetime.now().isoformat(),
|
170
|
+
"config": self.config.__dict__,
|
171
|
+
"heat_maps": {},
|
172
|
+
"optimization_scenarios": {},
|
173
|
+
"cost_hotspots": [],
|
174
|
+
"executive_summary": {},
|
175
|
+
}
|
176
|
+
|
177
|
+
# Generate single account heat map
|
178
|
+
if include_single_account:
|
179
|
+
logger.info("Generating single account heat map")
|
180
|
+
single_account_result = self._generate_single_account_heat_map()
|
181
|
+
heat_map_results["heat_maps"]["single_account"] = single_account_result
|
182
|
+
|
183
|
+
# Generate multi-account aggregated heat map
|
184
|
+
if include_multi_account:
|
185
|
+
logger.info("Generating multi-account aggregated heat map")
|
186
|
+
multi_account_result = self._generate_multi_account_heat_map(account_ids)
|
187
|
+
heat_map_results["heat_maps"]["multi_account"] = multi_account_result
|
188
|
+
|
189
|
+
# Identify cost hotspots
|
190
|
+
heat_map_results["cost_hotspots"] = self._identify_cost_hotspots(multi_account_result)
|
191
|
+
|
192
|
+
# Generate optimization scenarios
|
193
|
+
if self.config.include_optimization_scenarios:
|
194
|
+
logger.info("Generating optimization scenarios")
|
195
|
+
heat_map_results["optimization_scenarios"] = self._generate_optimization_scenarios(
|
196
|
+
heat_map_results["heat_maps"]
|
197
|
+
)
|
198
|
+
|
199
|
+
# Create executive summary
|
200
|
+
heat_map_results["executive_summary"] = self._create_executive_summary(heat_map_results)
|
201
|
+
|
202
|
+
# Export data if requested
|
203
|
+
if self.config.export_data:
|
204
|
+
self._export_heat_map_data(heat_map_results)
|
205
|
+
|
206
|
+
return OperationResult(
|
207
|
+
success=True, message="Networking cost heat maps generated successfully", data=heat_map_results
|
208
|
+
)
|
209
|
+
|
210
|
+
except Exception as e:
|
211
|
+
logger.error(f"Heat map generation failed: {str(e)}")
|
212
|
+
return OperationResult(
|
213
|
+
success=False, message=f"Heat map generation failed: {str(e)}", data={"error": str(e)}
|
214
|
+
)
|
215
|
+
|
216
|
+
def _generate_single_account_heat_map(self) -> Dict:
|
217
|
+
"""Generate single account heat map"""
|
218
|
+
|
219
|
+
account_id = "499201730520" # Single account ID
|
220
|
+
|
221
|
+
if self._cost_explorer_available():
|
222
|
+
return self._get_real_account_costs(account_id)
|
223
|
+
else:
|
224
|
+
return self._generate_estimated_account_costs(account_id, "single")
|
225
|
+
|
226
|
+
def _generate_multi_account_heat_map(self, account_ids: Optional[List[str]] = None) -> Dict:
|
227
|
+
"""Generate multi-account aggregated heat map"""
|
228
|
+
|
229
|
+
# Default to simulated 60-account environment
|
230
|
+
if not account_ids:
|
231
|
+
account_ids = [str(100000000000 + i) for i in range(60)]
|
232
|
+
|
233
|
+
# Account categories for realistic distribution
|
234
|
+
account_categories = {
|
235
|
+
"production": {"count": 15, "cost_multiplier": 5.0},
|
236
|
+
"staging": {"count": 15, "cost_multiplier": 2.0},
|
237
|
+
"development": {"count": 20, "cost_multiplier": 1.0},
|
238
|
+
"sandbox": {"count": 10, "cost_multiplier": 0.3},
|
239
|
+
}
|
240
|
+
|
241
|
+
# Generate aggregated heat map
|
242
|
+
aggregated_matrix = np.zeros((len(self.config.regions), len(self.NETWORKING_SERVICES)))
|
243
|
+
account_breakdown = []
|
244
|
+
|
245
|
+
account_idx = 0
|
246
|
+
for category, details in account_categories.items():
|
247
|
+
for i in range(details["count"]):
|
248
|
+
if account_idx < len(account_ids):
|
249
|
+
account_id = account_ids[account_idx]
|
250
|
+
|
251
|
+
# Generate account costs
|
252
|
+
account_costs = self._generate_estimated_account_costs(
|
253
|
+
account_id, category, details["cost_multiplier"]
|
254
|
+
)
|
255
|
+
|
256
|
+
# Add to aggregated matrix
|
257
|
+
account_matrix = np.array(account_costs["heat_map_matrix"])
|
258
|
+
aggregated_matrix += account_matrix
|
259
|
+
|
260
|
+
# Store account breakdown
|
261
|
+
account_breakdown.append(
|
262
|
+
{
|
263
|
+
"account_id": account_id,
|
264
|
+
"category": category,
|
265
|
+
"monthly_cost": account_costs["total_monthly_cost"],
|
266
|
+
"primary_region": self.config.regions[np.argmax(np.sum(account_matrix, axis=1))],
|
267
|
+
"top_service": list(self.NETWORKING_SERVICES.keys())[
|
268
|
+
np.argmax(np.sum(account_matrix, axis=0))
|
269
|
+
],
|
270
|
+
}
|
271
|
+
)
|
272
|
+
|
273
|
+
account_idx += 1
|
274
|
+
|
275
|
+
return {
|
276
|
+
"total_accounts": len(account_breakdown),
|
277
|
+
"aggregated_matrix": aggregated_matrix.tolist(),
|
278
|
+
"account_breakdown": account_breakdown,
|
279
|
+
"account_categories": account_categories,
|
280
|
+
"regions": self.config.regions,
|
281
|
+
"services": list(self.NETWORKING_SERVICES.keys()),
|
282
|
+
"service_names": list(self.NETWORKING_SERVICES.values()),
|
283
|
+
"total_monthly_cost": float(np.sum(aggregated_matrix)),
|
284
|
+
"average_account_cost": float(np.sum(aggregated_matrix) / len(account_breakdown)),
|
285
|
+
"cost_distribution": {
|
286
|
+
"regional_totals": np.sum(aggregated_matrix, axis=1).tolist(),
|
287
|
+
"service_totals": np.sum(aggregated_matrix, axis=0).tolist(),
|
288
|
+
"category_totals": {
|
289
|
+
cat: sum([acc["monthly_cost"] for acc in account_breakdown if acc["category"] == cat])
|
290
|
+
for cat in account_categories.keys()
|
291
|
+
},
|
292
|
+
},
|
293
|
+
}
|
294
|
+
|
295
|
+
def _generate_estimated_account_costs(self, account_id: str, category: str, cost_multiplier: float = 1.0) -> Dict:
|
296
|
+
"""Generate estimated networking costs for an account"""
|
297
|
+
|
298
|
+
# Category-specific patterns
|
299
|
+
category_patterns = {
|
300
|
+
"production": {
|
301
|
+
"nat_gateway_count": 6,
|
302
|
+
"transit_gateway": True,
|
303
|
+
"vpc_endpoints": 8,
|
304
|
+
"data_transfer_gb": 5000,
|
305
|
+
"elastic_ips": 5,
|
306
|
+
},
|
307
|
+
"staging": {
|
308
|
+
"nat_gateway_count": 3,
|
309
|
+
"transit_gateway": True,
|
310
|
+
"vpc_endpoints": 4,
|
311
|
+
"data_transfer_gb": 2000,
|
312
|
+
"elastic_ips": 2,
|
313
|
+
},
|
314
|
+
"development": {
|
315
|
+
"nat_gateway_count": 1,
|
316
|
+
"transit_gateway": False,
|
317
|
+
"vpc_endpoints": 2,
|
318
|
+
"data_transfer_gb": 500,
|
319
|
+
"elastic_ips": 1,
|
320
|
+
},
|
321
|
+
"sandbox": {
|
322
|
+
"nat_gateway_count": 0,
|
323
|
+
"transit_gateway": False,
|
324
|
+
"vpc_endpoints": 1,
|
325
|
+
"data_transfer_gb": 100,
|
326
|
+
"elastic_ips": 0,
|
327
|
+
},
|
328
|
+
"single": {
|
329
|
+
"nat_gateway_count": 3,
|
330
|
+
"transit_gateway": False,
|
331
|
+
"vpc_endpoints": 3,
|
332
|
+
"data_transfer_gb": 800,
|
333
|
+
"elastic_ips": 2,
|
334
|
+
},
|
335
|
+
}
|
336
|
+
|
337
|
+
pattern = category_patterns.get(category, category_patterns["development"])
|
338
|
+
heat_map_matrix = np.zeros((len(self.config.regions), len(self.NETWORKING_SERVICES)))
|
339
|
+
|
340
|
+
# Calculate costs per service
|
341
|
+
for service_idx, service_key in enumerate(self.NETWORKING_SERVICES.keys()):
|
342
|
+
for region_idx, region in enumerate(self.config.regions):
|
343
|
+
cost = 0
|
344
|
+
|
345
|
+
if service_key == "nat_gateway" and pattern["nat_gateway_count"] > 0:
|
346
|
+
if region_idx < pattern["nat_gateway_count"]:
|
347
|
+
cost = self.config.nat_gateway_baseline
|
348
|
+
|
349
|
+
elif service_key == "transit_gateway" and pattern["transit_gateway"]:
|
350
|
+
if region_idx == 0: # Primary region
|
351
|
+
cost = self.config.transit_gateway_baseline
|
352
|
+
|
353
|
+
elif service_key == "vpc_endpoint":
|
354
|
+
endpoints_in_region = pattern["vpc_endpoints"] // len(self.config.regions)
|
355
|
+
if region_idx < pattern["vpc_endpoints"] % len(self.config.regions):
|
356
|
+
endpoints_in_region += 1
|
357
|
+
cost = endpoints_in_region * self.config.vpc_endpoint_interface
|
358
|
+
|
359
|
+
elif service_key == "elastic_ip" and pattern["elastic_ips"] > 0:
|
360
|
+
if region_idx < pattern["elastic_ips"]:
|
361
|
+
cost = self.config.elastic_ip_idle
|
362
|
+
|
363
|
+
elif service_key == "data_transfer":
|
364
|
+
monthly_gb = pattern["data_transfer_gb"] / len(self.config.regions)
|
365
|
+
cost = monthly_gb * 0.09 # Internet out rate
|
366
|
+
|
367
|
+
elif service_key == "vpc":
|
368
|
+
cost = 2.0 if region_idx < 3 else 0.5 # Primary regions cost more
|
369
|
+
|
370
|
+
# Apply multiplier and add variation
|
371
|
+
if cost > 0:
|
372
|
+
variation = np.random.normal(1.0, 0.1)
|
373
|
+
heat_map_matrix[region_idx, service_idx] = max(0, cost * cost_multiplier * variation)
|
374
|
+
|
375
|
+
return {
|
376
|
+
"account_id": account_id,
|
377
|
+
"category": category,
|
378
|
+
"heat_map_matrix": heat_map_matrix.tolist(),
|
379
|
+
"regions": self.config.regions,
|
380
|
+
"services": list(self.NETWORKING_SERVICES.keys()),
|
381
|
+
"service_names": list(self.NETWORKING_SERVICES.values()),
|
382
|
+
"total_monthly_cost": float(np.sum(heat_map_matrix)),
|
383
|
+
"cost_distribution": {
|
384
|
+
"regional_totals": np.sum(heat_map_matrix, axis=1).tolist(),
|
385
|
+
"service_totals": np.sum(heat_map_matrix, axis=0).tolist(),
|
386
|
+
},
|
387
|
+
"pattern": pattern,
|
388
|
+
"estimated": True,
|
389
|
+
}
|
390
|
+
|
391
|
+
def _identify_cost_hotspots(self, multi_account_data: Dict) -> List[Dict]:
|
392
|
+
"""Identify cost hotspots in the aggregated heat map"""
|
393
|
+
|
394
|
+
aggregated_matrix = np.array(multi_account_data["aggregated_matrix"])
|
395
|
+
hotspots = []
|
396
|
+
|
397
|
+
# Find high-cost combinations
|
398
|
+
for region_idx, region in enumerate(self.config.regions):
|
399
|
+
for service_idx, service_key in enumerate(self.NETWORKING_SERVICES.keys()):
|
400
|
+
cost = aggregated_matrix[region_idx, service_idx]
|
401
|
+
|
402
|
+
if cost > self.config.high_cost_threshold:
|
403
|
+
severity = "critical" if cost > self.config.critical_cost_threshold else "high"
|
404
|
+
|
405
|
+
hotspots.append(
|
406
|
+
{
|
407
|
+
"region": region,
|
408
|
+
"service": service_key,
|
409
|
+
"service_name": self.NETWORKING_SERVICES[service_key],
|
410
|
+
"monthly_cost": float(cost),
|
411
|
+
"severity": severity,
|
412
|
+
"optimization_potential": min(cost * 0.4, cost - 10),
|
413
|
+
"affected_accounts": self._estimate_affected_accounts(
|
414
|
+
service_key, region, multi_account_data
|
415
|
+
),
|
416
|
+
}
|
417
|
+
)
|
418
|
+
|
419
|
+
# Sort by cost descending
|
420
|
+
hotspots.sort(key=lambda x: x["monthly_cost"], reverse=True)
|
421
|
+
return hotspots[:20] # Top 20 hotspots
|
422
|
+
|
423
|
+
def _estimate_affected_accounts(self, service: str, region: str, multi_account_data: Dict) -> int:
|
424
|
+
"""Estimate number of accounts affected by a hotspot"""
|
425
|
+
total_accounts = multi_account_data["total_accounts"]
|
426
|
+
|
427
|
+
# Rough estimation based on service and region
|
428
|
+
if service in ["nat_gateway", "vpc_endpoint"]:
|
429
|
+
return min(total_accounts, 40) # Most accounts likely use these
|
430
|
+
elif service == "transit_gateway":
|
431
|
+
return min(total_accounts, 20) # Enterprise accounts
|
432
|
+
else:
|
433
|
+
return min(total_accounts, 10) # Specialty services
|
434
|
+
|
435
|
+
def _generate_optimization_scenarios(self, heat_maps: Dict) -> Dict:
|
436
|
+
"""Generate optimization scenarios with ROI analysis"""
|
437
|
+
|
438
|
+
if "multi_account" not in heat_maps:
|
439
|
+
return {}
|
440
|
+
|
441
|
+
current_monthly_cost = heat_maps["multi_account"]["total_monthly_cost"]
|
442
|
+
|
443
|
+
scenarios = {
|
444
|
+
"Conservative (15%)": {
|
445
|
+
"reduction_percentage": 15,
|
446
|
+
"monthly_savings": current_monthly_cost * 0.15,
|
447
|
+
"annual_savings": current_monthly_cost * 0.15 * 12,
|
448
|
+
"implementation_cost": current_monthly_cost * 0.15 * 2,
|
449
|
+
"payback_months": 2,
|
450
|
+
"confidence": 90,
|
451
|
+
"actions": ["Eliminate idle Elastic IPs", "Right-size NAT Gateways", "Implement Reserved Instances"],
|
452
|
+
"risk_level": "Low",
|
453
|
+
},
|
454
|
+
"Moderate (30%)": {
|
455
|
+
"reduction_percentage": 30,
|
456
|
+
"monthly_savings": current_monthly_cost * 0.30,
|
457
|
+
"annual_savings": current_monthly_cost * 0.30 * 12,
|
458
|
+
"implementation_cost": current_monthly_cost * 0.30 * 3,
|
459
|
+
"payback_months": 3,
|
460
|
+
"confidence": 75,
|
461
|
+
"actions": ["Consolidate NAT Gateways", "Optimize VPC Endpoints", "Reduce cross-region data transfer"],
|
462
|
+
"risk_level": "Medium",
|
463
|
+
},
|
464
|
+
"Aggressive (45%)": {
|
465
|
+
"reduction_percentage": 45,
|
466
|
+
"monthly_savings": current_monthly_cost * 0.45,
|
467
|
+
"annual_savings": current_monthly_cost * 0.45 * 12,
|
468
|
+
"implementation_cost": current_monthly_cost * 0.45 * 4,
|
469
|
+
"payback_months": 4,
|
470
|
+
"confidence": 60,
|
471
|
+
"actions": [
|
472
|
+
"Redesign network architecture",
|
473
|
+
"Implement advanced automation",
|
474
|
+
"Multi-region optimization",
|
475
|
+
],
|
476
|
+
"risk_level": "High",
|
477
|
+
},
|
478
|
+
}
|
479
|
+
|
480
|
+
# Calculate ROI for each scenario
|
481
|
+
for scenario_name, scenario in scenarios.items():
|
482
|
+
if scenario["implementation_cost"] > 0:
|
483
|
+
scenario["roi_percentage"] = scenario["annual_savings"] / scenario["implementation_cost"] * 100
|
484
|
+
else:
|
485
|
+
scenario["roi_percentage"] = float("inf")
|
486
|
+
|
487
|
+
return scenarios
|
488
|
+
|
489
|
+
def _create_executive_summary(self, heat_map_results: Dict) -> Dict:
|
490
|
+
"""Create executive summary of heat map analysis"""
|
491
|
+
|
492
|
+
summary = {
|
493
|
+
"generation_timestamp": heat_map_results["generation_timestamp"],
|
494
|
+
"total_cost_analysis": {},
|
495
|
+
"key_findings": [],
|
496
|
+
"recommendations": [],
|
497
|
+
"next_steps": [],
|
498
|
+
}
|
499
|
+
|
500
|
+
# Cost analysis
|
501
|
+
if "single_account" in heat_map_results["heat_maps"]:
|
502
|
+
single_cost = heat_map_results["heat_maps"]["single_account"]["total_monthly_cost"]
|
503
|
+
summary["total_cost_analysis"]["single_account"] = {
|
504
|
+
"monthly_cost": single_cost,
|
505
|
+
"annual_projection": single_cost * 12,
|
506
|
+
}
|
507
|
+
|
508
|
+
if "multi_account" in heat_map_results["heat_maps"]:
|
509
|
+
multi_cost = heat_map_results["heat_maps"]["multi_account"]["total_monthly_cost"]
|
510
|
+
account_count = heat_map_results["heat_maps"]["multi_account"]["total_accounts"]
|
511
|
+
summary["total_cost_analysis"]["multi_account"] = {
|
512
|
+
"monthly_cost": multi_cost,
|
513
|
+
"annual_projection": multi_cost * 12,
|
514
|
+
"account_count": account_count,
|
515
|
+
"average_cost_per_account": multi_cost / account_count if account_count > 0 else 0,
|
516
|
+
}
|
517
|
+
|
518
|
+
# Key findings
|
519
|
+
hotspot_count = len(heat_map_results["cost_hotspots"])
|
520
|
+
if hotspot_count > 0:
|
521
|
+
summary["key_findings"].append(f"Identified {hotspot_count} cost hotspots requiring immediate attention")
|
522
|
+
|
523
|
+
top_hotspot = heat_map_results["cost_hotspots"][0]
|
524
|
+
summary["key_findings"].append(
|
525
|
+
f"Highest cost hotspot: {top_hotspot['region']} - {top_hotspot['service_name']} "
|
526
|
+
f"(${top_hotspot['monthly_cost']:.0f}/month)"
|
527
|
+
)
|
528
|
+
|
529
|
+
# Optimization potential
|
530
|
+
if heat_map_results["optimization_scenarios"]:
|
531
|
+
moderate_scenario = heat_map_results["optimization_scenarios"].get("Moderate (30%)", {})
|
532
|
+
if moderate_scenario:
|
533
|
+
summary["key_findings"].append(
|
534
|
+
f"Potential annual savings: ${moderate_scenario['annual_savings']:.0f} "
|
535
|
+
f"(30% reduction with 75% confidence)"
|
536
|
+
)
|
537
|
+
|
538
|
+
# Recommendations
|
539
|
+
summary["recommendations"] = [
|
540
|
+
"Implement immediate cost cleanup for idle resources",
|
541
|
+
"Deploy moderate optimization scenario with phased approach",
|
542
|
+
"Establish continuous cost monitoring and alerting",
|
543
|
+
"Review high-cost hotspots for architectural optimization",
|
544
|
+
]
|
545
|
+
|
546
|
+
# Next steps
|
547
|
+
summary["next_steps"] = [
|
548
|
+
"Management review and approval of optimization strategy",
|
549
|
+
"Terminal coordination for implementation planning",
|
550
|
+
"Quality assurance validation of heat map accuracy",
|
551
|
+
"Production deployment with monitoring framework",
|
552
|
+
]
|
553
|
+
|
554
|
+
return summary
|
555
|
+
|
556
|
+
def _cost_explorer_available(self) -> bool:
|
557
|
+
"""Check if Cost Explorer API is available"""
|
558
|
+
try:
|
559
|
+
if not self.session:
|
560
|
+
return False
|
561
|
+
|
562
|
+
ce_client = self.session.client("ce", region_name="us-east-1")
|
563
|
+
|
564
|
+
# Test with minimal query
|
565
|
+
ce_client.get_cost_and_usage(
|
566
|
+
TimePeriod={
|
567
|
+
"Start": (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d"),
|
568
|
+
"End": datetime.now().strftime("%Y-%m-%d"),
|
569
|
+
},
|
570
|
+
Granularity="DAILY",
|
571
|
+
Metrics=["BlendedCost"],
|
572
|
+
)
|
573
|
+
return True
|
574
|
+
|
575
|
+
except (ClientError, NoCredentialsError, Exception):
|
576
|
+
return False
|
577
|
+
|
578
|
+
def _get_real_account_costs(self, account_id: str) -> Dict:
|
579
|
+
"""Get real account costs from Cost Explorer"""
|
580
|
+
try:
|
581
|
+
ce_client = self.session.client("ce", region_name="us-east-1")
|
582
|
+
|
583
|
+
end_date = datetime.now().date()
|
584
|
+
start_date = end_date - timedelta(days=self.config.analysis_days)
|
585
|
+
|
586
|
+
response = ce_client.get_cost_and_usage(
|
587
|
+
TimePeriod={"Start": start_date.strftime("%Y-%m-%d"), "End": end_date.strftime("%Y-%m-%d")},
|
588
|
+
Granularity="MONTHLY",
|
589
|
+
Metrics=["BlendedCost"],
|
590
|
+
GroupBy=[{"Type": "DIMENSION", "Key": "SERVICE"}, {"Type": "DIMENSION", "Key": "REGION"}],
|
591
|
+
Filter={"Dimensions": {"Key": "LINKED_ACCOUNT", "Values": [account_id]}},
|
592
|
+
)
|
593
|
+
|
594
|
+
# Process response into heat map format
|
595
|
+
return self._process_cost_explorer_response(response, account_id)
|
596
|
+
|
597
|
+
except Exception as e:
|
598
|
+
logger.warning(f"Failed to get real costs for {account_id}: {e}")
|
599
|
+
return self._generate_estimated_account_costs(account_id, "single")
|
600
|
+
|
601
|
+
def _process_cost_explorer_response(self, response: Dict, account_id: str) -> Dict:
|
602
|
+
"""Process Cost Explorer response into heat map format"""
|
603
|
+
|
604
|
+
heat_map_matrix = np.zeros((len(self.config.regions), len(self.NETWORKING_SERVICES)))
|
605
|
+
total_cost = 0
|
606
|
+
|
607
|
+
for time_period in response.get("ResultsByTime", []):
|
608
|
+
for group in time_period.get("Groups", []):
|
609
|
+
service = group["Keys"][0]
|
610
|
+
region = group["Keys"][1]
|
611
|
+
cost = float(group["Metrics"]["BlendedCost"]["Amount"])
|
612
|
+
|
613
|
+
if cost > 0:
|
614
|
+
total_cost += cost
|
615
|
+
|
616
|
+
# Map to our service categories
|
617
|
+
service_idx = self._map_aws_service_to_category(service)
|
618
|
+
region_idx = self._get_region_index(region)
|
619
|
+
|
620
|
+
if service_idx >= 0 and region_idx >= 0:
|
621
|
+
heat_map_matrix[region_idx, service_idx] += cost
|
622
|
+
|
623
|
+
return {
|
624
|
+
"account_id": account_id,
|
625
|
+
"heat_map_matrix": heat_map_matrix.tolist(),
|
626
|
+
"regions": self.config.regions,
|
627
|
+
"services": list(self.NETWORKING_SERVICES.keys()),
|
628
|
+
"service_names": list(self.NETWORKING_SERVICES.values()),
|
629
|
+
"total_monthly_cost": total_cost,
|
630
|
+
"cost_distribution": {
|
631
|
+
"regional_totals": np.sum(heat_map_matrix, axis=1).tolist(),
|
632
|
+
"service_totals": np.sum(heat_map_matrix, axis=0).tolist(),
|
633
|
+
},
|
634
|
+
"real_data": True,
|
635
|
+
}
|
636
|
+
|
637
|
+
def _map_aws_service_to_category(self, aws_service: str) -> int:
|
638
|
+
"""Map AWS service name to our category index"""
|
639
|
+
|
640
|
+
service_mapping = {
|
641
|
+
"Amazon Virtual Private Cloud": 0, # vpc
|
642
|
+
"AWS Transit Gateway": 1, # transit_gateway
|
643
|
+
"Amazon Virtual Private Cloud-NAT Gateway": 2, # nat_gateway
|
644
|
+
"Amazon Virtual Private Cloud-VPC Endpoint": 3, # vpc_endpoint
|
645
|
+
"Amazon Elastic Compute Cloud-EIPs": 4, # elastic_ip
|
646
|
+
"AWS Data Transfer": 5, # data_transfer
|
647
|
+
}
|
648
|
+
|
649
|
+
return service_mapping.get(aws_service, -1)
|
650
|
+
|
651
|
+
def _get_region_index(self, region: str) -> int:
|
652
|
+
"""Get region index in our regions list"""
|
653
|
+
try:
|
654
|
+
return self.config.regions.index(region)
|
655
|
+
except ValueError:
|
656
|
+
return -1
|
657
|
+
|
658
|
+
def _export_heat_map_data(self, heat_map_results: Dict) -> None:
|
659
|
+
"""Export heat map data to files"""
|
660
|
+
try:
|
661
|
+
export_dir = Path("./exports")
|
662
|
+
export_dir.mkdir(exist_ok=True)
|
663
|
+
|
664
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
665
|
+
|
666
|
+
# Export comprehensive JSON
|
667
|
+
import json
|
668
|
+
|
669
|
+
json_path = export_dir / f"networking_cost_heatmap_{timestamp}.json"
|
670
|
+
with open(json_path, "w") as f:
|
671
|
+
json.dump(heat_map_results, f, indent=2, default=str)
|
672
|
+
|
673
|
+
logger.info(f"Heat map data exported to {json_path}")
|
674
|
+
|
675
|
+
except Exception as e:
|
676
|
+
logger.error(f"Failed to export heat map data: {e}")
|
677
|
+
|
678
|
+
|
679
|
+
def create_networking_cost_heatmap_operation(profile: str = None) -> NetworkingCostHeatMapOperation:
|
680
|
+
"""Factory function to create a networking cost heat map operation"""
|
681
|
+
return NetworkingCostHeatMapOperation(profile=profile)
|
682
|
+
|
683
|
+
|
684
|
+
# Export main classes
|
685
|
+
__all__ = ["NetworkingCostHeatMapOperation", "NetworkingCostHeatMapConfig", "create_networking_cost_heatmap_operation"]
|