runbooks 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +31 -2
- runbooks/__init___optimized.py +18 -4
- runbooks/_platform/__init__.py +1 -5
- runbooks/_platform/core/runbooks_wrapper.py +141 -138
- runbooks/aws2/accuracy_validator.py +812 -0
- runbooks/base.py +7 -0
- runbooks/cfat/WEIGHT_CONFIG_README.md +1 -1
- runbooks/cfat/assessment/compliance.py +8 -8
- runbooks/cfat/assessment/runner.py +1 -0
- runbooks/cfat/cloud_foundations_assessment.py +227 -239
- runbooks/cfat/models.py +6 -2
- runbooks/cfat/tests/__init__.py +6 -1
- runbooks/cli/__init__.py +13 -0
- runbooks/cli/commands/cfat.py +274 -0
- runbooks/cli/commands/finops.py +1164 -0
- runbooks/cli/commands/inventory.py +379 -0
- runbooks/cli/commands/operate.py +239 -0
- runbooks/cli/commands/security.py +248 -0
- runbooks/cli/commands/validation.py +825 -0
- runbooks/cli/commands/vpc.py +310 -0
- runbooks/cli/registry.py +107 -0
- runbooks/cloudops/__init__.py +23 -30
- runbooks/cloudops/base.py +96 -107
- runbooks/cloudops/cost_optimizer.py +549 -547
- runbooks/cloudops/infrastructure_optimizer.py +5 -4
- runbooks/cloudops/interfaces.py +226 -227
- runbooks/cloudops/lifecycle_manager.py +5 -4
- runbooks/cloudops/mcp_cost_validation.py +252 -235
- runbooks/cloudops/models.py +78 -53
- runbooks/cloudops/monitoring_automation.py +5 -4
- runbooks/cloudops/notebook_framework.py +179 -215
- runbooks/cloudops/security_enforcer.py +125 -159
- runbooks/common/accuracy_validator.py +11 -0
- runbooks/common/aws_pricing.py +349 -326
- runbooks/common/aws_pricing_api.py +211 -212
- runbooks/common/aws_profile_manager.py +341 -0
- runbooks/common/aws_utils.py +75 -80
- runbooks/common/business_logic.py +127 -105
- runbooks/common/cli_decorators.py +36 -60
- runbooks/common/comprehensive_cost_explorer_integration.py +456 -464
- runbooks/common/cross_account_manager.py +198 -205
- runbooks/common/date_utils.py +27 -39
- runbooks/common/decorators.py +235 -0
- runbooks/common/dry_run_examples.py +173 -208
- runbooks/common/dry_run_framework.py +157 -155
- runbooks/common/enhanced_exception_handler.py +15 -4
- runbooks/common/enhanced_logging_example.py +50 -64
- runbooks/common/enhanced_logging_integration_example.py +65 -37
- runbooks/common/env_utils.py +16 -16
- runbooks/common/error_handling.py +40 -38
- runbooks/common/lazy_loader.py +41 -23
- runbooks/common/logging_integration_helper.py +79 -86
- runbooks/common/mcp_cost_explorer_integration.py +478 -495
- runbooks/common/mcp_integration.py +63 -74
- runbooks/common/memory_optimization.py +140 -118
- runbooks/common/module_cli_base.py +37 -58
- runbooks/common/organizations_client.py +176 -194
- runbooks/common/patterns.py +204 -0
- runbooks/common/performance_monitoring.py +67 -71
- runbooks/common/performance_optimization_engine.py +283 -274
- runbooks/common/profile_utils.py +248 -39
- runbooks/common/rich_utils.py +643 -92
- runbooks/common/sre_performance_suite.py +177 -186
- runbooks/enterprise/__init__.py +1 -1
- runbooks/enterprise/logging.py +144 -106
- runbooks/enterprise/security.py +187 -204
- runbooks/enterprise/validation.py +43 -56
- runbooks/finops/__init__.py +29 -33
- runbooks/finops/account_resolver.py +1 -1
- runbooks/finops/advanced_optimization_engine.py +980 -0
- runbooks/finops/automation_core.py +268 -231
- runbooks/finops/business_case_config.py +184 -179
- runbooks/finops/cli.py +660 -139
- runbooks/finops/commvault_ec2_analysis.py +157 -164
- runbooks/finops/compute_cost_optimizer.py +336 -320
- runbooks/finops/config.py +20 -20
- runbooks/finops/cost_optimizer.py +488 -622
- runbooks/finops/cost_processor.py +332 -214
- runbooks/finops/dashboard_runner.py +1006 -172
- runbooks/finops/ebs_cost_optimizer.py +991 -657
- runbooks/finops/elastic_ip_optimizer.py +317 -257
- runbooks/finops/enhanced_mcp_integration.py +340 -0
- runbooks/finops/enhanced_progress.py +40 -37
- runbooks/finops/enhanced_trend_visualization.py +3 -2
- runbooks/finops/enterprise_wrappers.py +230 -292
- runbooks/finops/executive_export.py +203 -160
- runbooks/finops/helpers.py +130 -288
- runbooks/finops/iam_guidance.py +1 -1
- runbooks/finops/infrastructure/__init__.py +80 -0
- runbooks/finops/infrastructure/commands.py +506 -0
- runbooks/finops/infrastructure/load_balancer_optimizer.py +866 -0
- runbooks/finops/infrastructure/vpc_endpoint_optimizer.py +832 -0
- runbooks/finops/markdown_exporter.py +338 -175
- runbooks/finops/mcp_validator.py +1952 -0
- runbooks/finops/nat_gateway_optimizer.py +1513 -482
- runbooks/finops/network_cost_optimizer.py +657 -587
- runbooks/finops/notebook_utils.py +226 -188
- runbooks/finops/optimization_engine.py +1136 -0
- runbooks/finops/optimizer.py +25 -29
- runbooks/finops/rds_snapshot_optimizer.py +367 -411
- runbooks/finops/reservation_optimizer.py +427 -363
- runbooks/finops/scenario_cli_integration.py +77 -78
- runbooks/finops/scenarios.py +1278 -439
- runbooks/finops/schemas.py +218 -182
- runbooks/finops/snapshot_manager.py +2289 -0
- runbooks/finops/tests/test_finops_dashboard.py +3 -3
- runbooks/finops/tests/test_reference_images_validation.py +2 -2
- runbooks/finops/tests/test_single_account_features.py +17 -17
- runbooks/finops/tests/validate_test_suite.py +1 -1
- runbooks/finops/types.py +3 -3
- runbooks/finops/validation_framework.py +263 -269
- runbooks/finops/vpc_cleanup_exporter.py +191 -146
- runbooks/finops/vpc_cleanup_optimizer.py +593 -575
- runbooks/finops/workspaces_analyzer.py +171 -182
- runbooks/hitl/enhanced_workflow_engine.py +1 -1
- runbooks/integration/__init__.py +89 -0
- runbooks/integration/mcp_integration.py +1920 -0
- runbooks/inventory/CLAUDE.md +816 -0
- runbooks/inventory/README.md +3 -3
- runbooks/inventory/Tests/common_test_data.py +30 -30
- runbooks/inventory/__init__.py +2 -2
- runbooks/inventory/cloud_foundations_integration.py +144 -149
- runbooks/inventory/collectors/aws_comprehensive.py +28 -11
- runbooks/inventory/collectors/aws_networking.py +111 -101
- runbooks/inventory/collectors/base.py +4 -0
- runbooks/inventory/core/collector.py +495 -313
- runbooks/inventory/discovery.md +2 -2
- runbooks/inventory/drift_detection_cli.py +69 -96
- runbooks/inventory/find_ec2_security_groups.py +1 -1
- runbooks/inventory/inventory_mcp_cli.py +48 -46
- runbooks/inventory/list_rds_snapshots_aggregator.py +192 -208
- runbooks/inventory/mcp_inventory_validator.py +549 -465
- runbooks/inventory/mcp_vpc_validator.py +359 -442
- runbooks/inventory/organizations_discovery.py +56 -52
- runbooks/inventory/rich_inventory_display.py +33 -32
- runbooks/inventory/unified_validation_engine.py +278 -251
- runbooks/inventory/vpc_analyzer.py +733 -696
- runbooks/inventory/vpc_architecture_validator.py +293 -348
- runbooks/inventory/vpc_dependency_analyzer.py +382 -378
- runbooks/inventory/vpc_flow_analyzer.py +3 -3
- runbooks/main.py +152 -9147
- runbooks/main_final.py +91 -60
- runbooks/main_minimal.py +22 -10
- runbooks/main_optimized.py +131 -100
- runbooks/main_ultra_minimal.py +7 -2
- runbooks/mcp/__init__.py +36 -0
- runbooks/mcp/integration.py +679 -0
- runbooks/metrics/dora_metrics_engine.py +2 -2
- runbooks/monitoring/performance_monitor.py +9 -4
- runbooks/operate/dynamodb_operations.py +3 -1
- runbooks/operate/ec2_operations.py +145 -137
- runbooks/operate/iam_operations.py +146 -152
- runbooks/operate/mcp_integration.py +1 -1
- runbooks/operate/networking_cost_heatmap.py +33 -10
- runbooks/operate/privatelink_operations.py +1 -1
- runbooks/operate/rds_operations.py +223 -254
- runbooks/operate/s3_operations.py +107 -118
- runbooks/operate/vpc_endpoints.py +1 -1
- runbooks/operate/vpc_operations.py +648 -618
- runbooks/remediation/base.py +1 -1
- runbooks/remediation/commons.py +10 -7
- runbooks/remediation/commvault_ec2_analysis.py +71 -67
- runbooks/remediation/ec2_unattached_ebs_volumes.py +1 -0
- runbooks/remediation/multi_account.py +24 -21
- runbooks/remediation/rds_snapshot_list.py +91 -65
- runbooks/remediation/remediation_cli.py +92 -146
- runbooks/remediation/universal_account_discovery.py +83 -79
- runbooks/remediation/workspaces_list.py +49 -44
- runbooks/security/__init__.py +19 -0
- runbooks/security/assessment_runner.py +1150 -0
- runbooks/security/baseline_checker.py +812 -0
- runbooks/security/cloudops_automation_security_validator.py +509 -535
- runbooks/security/compliance_automation_engine.py +17 -17
- runbooks/security/config/__init__.py +2 -2
- runbooks/security/config/compliance_config.py +50 -50
- runbooks/security/config_template_generator.py +63 -76
- runbooks/security/enterprise_security_framework.py +1 -1
- runbooks/security/executive_security_dashboard.py +519 -508
- runbooks/security/integration_test_enterprise_security.py +5 -3
- runbooks/security/multi_account_security_controls.py +959 -1210
- runbooks/security/real_time_security_monitor.py +422 -444
- runbooks/security/run_script.py +1 -1
- runbooks/security/security_baseline_tester.py +1 -1
- runbooks/security/security_cli.py +143 -112
- runbooks/security/test_2way_validation.py +439 -0
- runbooks/security/two_way_validation_framework.py +852 -0
- runbooks/sre/mcp_reliability_engine.py +6 -6
- runbooks/sre/production_monitoring_framework.py +167 -177
- runbooks/tdd/__init__.py +15 -0
- runbooks/tdd/cli.py +1071 -0
- runbooks/utils/__init__.py +14 -17
- runbooks/utils/logger.py +7 -2
- runbooks/utils/version_validator.py +51 -48
- runbooks/validation/__init__.py +6 -6
- runbooks/validation/cli.py +9 -3
- runbooks/validation/comprehensive_2way_validator.py +754 -708
- runbooks/validation/mcp_validator.py +906 -228
- runbooks/validation/terraform_citations_validator.py +104 -115
- runbooks/validation/terraform_drift_detector.py +447 -451
- runbooks/vpc/README.md +617 -0
- runbooks/vpc/__init__.py +8 -1
- runbooks/vpc/analyzer.py +577 -0
- runbooks/vpc/cleanup_wrapper.py +476 -413
- runbooks/vpc/cli_cloudtrail_commands.py +339 -0
- runbooks/vpc/cli_mcp_validation_commands.py +480 -0
- runbooks/vpc/cloudtrail_audit_integration.py +717 -0
- runbooks/vpc/config.py +92 -97
- runbooks/vpc/cost_engine.py +411 -148
- runbooks/vpc/cost_explorer_integration.py +553 -0
- runbooks/vpc/cross_account_session.py +101 -106
- runbooks/vpc/enhanced_mcp_validation.py +917 -0
- runbooks/vpc/eni_gate_validator.py +961 -0
- runbooks/vpc/heatmap_engine.py +190 -162
- runbooks/vpc/mcp_no_eni_validator.py +681 -640
- runbooks/vpc/nat_gateway_optimizer.py +358 -0
- runbooks/vpc/networking_wrapper.py +15 -8
- runbooks/vpc/pdca_remediation_planner.py +528 -0
- runbooks/vpc/performance_optimized_analyzer.py +219 -231
- runbooks/vpc/runbooks_adapter.py +1167 -241
- runbooks/vpc/tdd_red_phase_stubs.py +601 -0
- runbooks/vpc/test_data_loader.py +358 -0
- runbooks/vpc/tests/conftest.py +314 -4
- runbooks/vpc/tests/test_cleanup_framework.py +1022 -0
- runbooks/vpc/tests/test_cost_engine.py +0 -2
- runbooks/vpc/topology_generator.py +326 -0
- runbooks/vpc/unified_scenarios.py +1302 -1129
- runbooks/vpc/vpc_cleanup_integration.py +1943 -1115
- runbooks-1.1.5.dist-info/METADATA +328 -0
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/RECORD +233 -200
- runbooks/finops/README.md +0 -414
- runbooks/finops/accuracy_cross_validator.py +0 -647
- runbooks/finops/business_cases.py +0 -950
- runbooks/finops/dashboard_router.py +0 -922
- runbooks/finops/ebs_optimizer.py +0 -956
- runbooks/finops/embedded_mcp_validator.py +0 -1629
- runbooks/finops/enhanced_dashboard_runner.py +0 -527
- runbooks/finops/finops_dashboard.py +0 -584
- runbooks/finops/finops_scenarios.py +0 -1218
- runbooks/finops/legacy_migration.py +0 -730
- runbooks/finops/multi_dashboard.py +0 -1519
- runbooks/finops/single_dashboard.py +0 -1113
- runbooks/finops/unlimited_scenarios.py +0 -393
- runbooks-1.1.3.dist-info/METADATA +0 -799
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/WHEEL +0 -0
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/entry_points.txt +0 -0
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/licenses/LICENSE +0 -0
- {runbooks-1.1.3.dist-info → runbooks-1.1.5.dist-info}/top_level.txt +0 -0
runbooks/vpc/heatmap_engine.py
CHANGED
@@ -17,6 +17,7 @@ from .cost_engine import NetworkingCostEngine
|
|
17
17
|
from ..common.env_utils import get_required_env_float
|
18
18
|
from ..common.aws_pricing_api import AWSPricingAPI
|
19
19
|
from ..common.rich_utils import console
|
20
|
+
from ..common.aws_profile_manager import AWSProfileManager, get_current_account_id
|
20
21
|
|
21
22
|
logger = logging.getLogger(__name__)
|
22
23
|
|
@@ -107,7 +108,6 @@ class NetworkingCostHeatMapEngine:
|
|
107
108
|
# Heat map data storage
|
108
109
|
self.heat_map_data = {}
|
109
110
|
|
110
|
-
|
111
111
|
def _initialize_aws_sessions(self):
|
112
112
|
"""Initialize AWS sessions for all profiles"""
|
113
113
|
profiles = {
|
@@ -184,8 +184,9 @@ class NetworkingCostHeatMapEngine:
|
|
184
184
|
"""Generate detailed single account heat map"""
|
185
185
|
logger.info("Generating single account heat map")
|
186
186
|
|
187
|
-
# Use
|
188
|
-
|
187
|
+
# Use dynamic account ID resolution for universal compatibility
|
188
|
+
profile_manager = AWSProfileManager(self.config.profile if hasattr(self.config, "profile") else None)
|
189
|
+
account_id = profile_manager.get_account_id()
|
189
190
|
|
190
191
|
# Create cost distribution matrix
|
191
192
|
heat_map_matrix = np.zeros((len(self.config.regions), len(NETWORKING_SERVICES)))
|
@@ -240,18 +241,31 @@ class NetworkingCostHeatMapEngine:
|
|
240
241
|
|
241
242
|
# Account categories with dynamic environment configuration
|
242
243
|
account_categories = {
|
243
|
-
"production": {
|
244
|
-
|
245
|
-
|
246
|
-
|
244
|
+
"production": {
|
245
|
+
"count": int(os.getenv("AWS_PROD_ACCOUNTS", "15")),
|
246
|
+
"cost_multiplier": float(os.getenv("PROD_COST_MULTIPLIER", "5.0")),
|
247
|
+
},
|
248
|
+
"staging": {
|
249
|
+
"count": int(os.getenv("AWS_STAGING_ACCOUNTS", "15")),
|
250
|
+
"cost_multiplier": float(os.getenv("STAGING_COST_MULTIPLIER", "2.0")),
|
251
|
+
},
|
252
|
+
"development": {
|
253
|
+
"count": int(os.getenv("AWS_DEV_ACCOUNTS", "20")),
|
254
|
+
"cost_multiplier": float(os.getenv("DEV_COST_MULTIPLIER", "1.0")),
|
255
|
+
},
|
256
|
+
"sandbox": {
|
257
|
+
"count": int(os.getenv("AWS_SANDBOX_ACCOUNTS", "10")),
|
258
|
+
"cost_multiplier": float(os.getenv("SANDBOX_COST_MULTIPLIER", "0.3")),
|
259
|
+
},
|
247
260
|
}
|
248
261
|
|
249
262
|
# Generate aggregated matrix
|
250
263
|
aggregated_matrix = np.zeros((len(self.config.regions), len(NETWORKING_SERVICES)))
|
251
264
|
account_breakdown = []
|
252
265
|
|
253
|
-
# Dynamic base account ID from
|
254
|
-
|
266
|
+
# Dynamic base account ID from current AWS credentials
|
267
|
+
profile_manager = AWSProfileManager(self.config.profile if hasattr(self.config, "profile") else None)
|
268
|
+
account_id = int(profile_manager.get_account_id())
|
255
269
|
|
256
270
|
for category, details in account_categories.items():
|
257
271
|
for i in range(details["count"]):
|
@@ -269,7 +283,9 @@ class NetworkingCostHeatMapEngine:
|
|
269
283
|
"category": category,
|
270
284
|
"monthly_cost": float(np.sum(account_matrix)),
|
271
285
|
"primary_region": self.config.regions[int(np.argmax(np.sum(account_matrix, axis=1)))],
|
272
|
-
"top_service": list(NETWORKING_SERVICES.keys())[
|
286
|
+
"top_service": list(NETWORKING_SERVICES.keys())[
|
287
|
+
int(np.argmax(np.sum(account_matrix, axis=0)))
|
288
|
+
],
|
273
289
|
}
|
274
290
|
)
|
275
291
|
|
@@ -425,7 +441,7 @@ class NetworkingCostHeatMapEngine:
|
|
425
441
|
|
426
442
|
for region in self.config.regions:
|
427
443
|
# Dynamic cost calculation with real AWS Cost Explorer integration
|
428
|
-
if hasattr(self,
|
444
|
+
if hasattr(self, "cost_engine") and self.cost_engine and self.cost_explorer_available:
|
429
445
|
# Real AWS Cost Explorer data
|
430
446
|
base_cost = self.cost_engine.get_service_cost(service_key, region)
|
431
447
|
else:
|
@@ -537,14 +553,14 @@ class NetworkingCostHeatMapEngine:
|
|
537
553
|
|
538
554
|
# Apply costs based on pattern using dynamic pricing (NO hardcoded fallbacks)
|
539
555
|
region = self.config.regions[0] if self.config.regions else "us-east-1" # Use first configured region
|
540
|
-
|
556
|
+
|
541
557
|
# Get dynamic service pricing
|
542
558
|
service_pricing = self._get_dynamic_service_pricing(region)
|
543
|
-
|
559
|
+
|
544
560
|
for service_idx, service_key in enumerate(NETWORKING_SERVICES.keys()):
|
545
561
|
for region_idx in range(len(self.config.regions)):
|
546
562
|
cost = 0.0 # Default to free
|
547
|
-
|
563
|
+
|
548
564
|
# Only apply costs if we have valid pricing data
|
549
565
|
if service_key in service_pricing and service_pricing[service_key] > 0:
|
550
566
|
if service_key == "nat_gateway" and region_idx < pattern["nat_gateways"]:
|
@@ -555,7 +571,7 @@ class NetworkingCostHeatMapEngine:
|
|
555
571
|
cost = service_pricing[service_key] * multiplier
|
556
572
|
elif service_key == "elastic_ip" and region_idx < pattern.get("elastic_ips", 0):
|
557
573
|
cost = service_pricing[service_key] * multiplier
|
558
|
-
|
574
|
+
|
559
575
|
matrix[region_idx, service_idx] = cost
|
560
576
|
|
561
577
|
return matrix
|
@@ -607,7 +623,7 @@ class NetworkingCostHeatMapEngine:
|
|
607
623
|
def _calculate_dynamic_baseline_cost(self, service_key: str, region: str) -> float:
|
608
624
|
"""
|
609
625
|
Calculate dynamic baseline costs using AWS pricing patterns and region multipliers.
|
610
|
-
|
626
|
+
|
611
627
|
This replaces hardcoded values with calculation based on:
|
612
628
|
- AWS pricing calculator patterns
|
613
629
|
- Regional pricing differences
|
@@ -615,66 +631,68 @@ class NetworkingCostHeatMapEngine:
|
|
615
631
|
"""
|
616
632
|
# Regional cost multipliers based on AWS pricing
|
617
633
|
regional_multipliers = {
|
618
|
-
"us-east-1": 1.0,
|
619
|
-
"us-west-2": 1.05,
|
620
|
-
"us-west-1": 1.15,
|
621
|
-
"eu-west-1": 1.10,
|
634
|
+
"us-east-1": 1.0, # Base region (N. Virginia)
|
635
|
+
"us-west-2": 1.05, # Oregon - slight premium
|
636
|
+
"us-west-1": 1.15, # N. California - higher cost
|
637
|
+
"eu-west-1": 1.10, # Ireland - EU pricing
|
622
638
|
"eu-central-1": 1.12, # Frankfurt - slightly higher
|
623
|
-
"eu-west-2": 1.08,
|
624
|
-
"ap-southeast-1": 1.18,
|
625
|
-
"ap-southeast-2": 1.16,
|
626
|
-
"ap-northeast-1": 1.20,
|
639
|
+
"eu-west-2": 1.08, # London - competitive EU pricing
|
640
|
+
"ap-southeast-1": 1.18, # Singapore - APAC premium
|
641
|
+
"ap-southeast-2": 1.16, # Sydney - competitive APAC
|
642
|
+
"ap-northeast-1": 1.20, # Tokyo - highest APAC
|
627
643
|
}
|
628
|
-
|
644
|
+
|
629
645
|
# AWS service pricing patterns (monthly USD) - DYNAMIC PRICING REQUIRED
|
630
646
|
# ENTERPRISE COMPLIANCE: All pricing must be fetched from AWS Pricing API
|
631
647
|
service_base_costs = self._get_dynamic_service_pricing(region)
|
632
|
-
|
648
|
+
|
633
649
|
base_cost = service_base_costs.get(service_key, 0.0)
|
634
650
|
region_multiplier = regional_multipliers.get(region, 1.0)
|
635
|
-
|
651
|
+
|
636
652
|
return base_cost * region_multiplier
|
637
653
|
|
638
654
|
def _get_dynamic_service_pricing(self, region: str) -> Dict[str, float]:
|
639
655
|
"""
|
640
656
|
Get dynamic AWS service pricing following enterprise cascade:
|
641
|
-
|
642
|
-
a. ✅ Try Runbooks API with boto3 (dynamic)
|
657
|
+
|
658
|
+
a. ✅ Try Runbooks API with boto3 (dynamic)
|
643
659
|
b. ✅ Try MCP-Servers (dynamic) & gaps analysis with real AWS data
|
644
660
|
→ If failed, identify WHY option 'a' didn't work, then UPGRADE option 'a'
|
645
661
|
c. ✅ Fail gracefully with user guidance (NO hardcoded fallback)
|
646
|
-
|
662
|
+
|
647
663
|
ENTERPRISE COMPLIANCE: Zero tolerance for hardcoded pricing fallbacks.
|
648
|
-
|
664
|
+
|
649
665
|
Args:
|
650
666
|
region: AWS region for pricing lookup
|
651
|
-
|
667
|
+
|
652
668
|
Returns:
|
653
669
|
Dictionary of service pricing (monthly USD)
|
654
670
|
"""
|
655
671
|
service_costs = {}
|
656
672
|
pricing_errors = []
|
657
|
-
|
673
|
+
|
658
674
|
# VPC itself is always free
|
659
675
|
service_costs["vpc"] = 0.0
|
660
|
-
|
676
|
+
|
661
677
|
# Step A: Try Runbooks Pricing API (Enhanced)
|
662
678
|
console.print(f"[blue]🔄 Step A: Attempting Runbooks Pricing API for {region}[/blue]")
|
663
679
|
try:
|
664
680
|
from ..common.aws_pricing_api import AWSPricingAPI
|
665
|
-
|
681
|
+
|
666
682
|
# Initialize with proper session management
|
667
|
-
profile = getattr(self,
|
683
|
+
profile = getattr(self, "profile", None)
|
668
684
|
pricing_api = AWSPricingAPI(profile=profile)
|
669
|
-
|
685
|
+
|
670
686
|
# NAT Gateway pricing (primary VPC cost component)
|
671
687
|
try:
|
672
688
|
service_costs["nat_gateway"] = pricing_api.get_nat_gateway_monthly_cost(region)
|
673
|
-
console.print(
|
689
|
+
console.print(
|
690
|
+
f"[green]✅ NAT Gateway pricing: ${service_costs['nat_gateway']:.2f}/month from Runbooks API[/green]"
|
691
|
+
)
|
674
692
|
except Exception as e:
|
675
693
|
pricing_errors.append(f"NAT Gateway: {str(e)}")
|
676
694
|
logger.warning(f"Runbooks API NAT Gateway pricing failed: {e}")
|
677
|
-
|
695
|
+
|
678
696
|
# Try other services with existing API methods
|
679
697
|
for service_key in ["vpc_endpoint", "transit_gateway", "elastic_ip"]:
|
680
698
|
try:
|
@@ -682,20 +700,22 @@ class NetworkingCostHeatMapEngine:
|
|
682
700
|
if hasattr(pricing_api, f"get_{service_key}_monthly_cost"):
|
683
701
|
method = getattr(pricing_api, f"get_{service_key}_monthly_cost")
|
684
702
|
service_costs[service_key] = method(region)
|
685
|
-
console.print(
|
703
|
+
console.print(
|
704
|
+
f"[green]✅ {service_key} pricing: ${service_costs[service_key]:.2f}/month from Runbooks API[/green]"
|
705
|
+
)
|
686
706
|
else:
|
687
707
|
pricing_errors.append(f"{service_key}: API method not implemented")
|
688
708
|
except Exception as e:
|
689
709
|
pricing_errors.append(f"{service_key}: {str(e)}")
|
690
|
-
|
710
|
+
|
691
711
|
# Data transfer pricing (if API available)
|
692
712
|
if "data_transfer" not in service_costs:
|
693
713
|
pricing_errors.append("data_transfer: API method not implemented")
|
694
|
-
|
714
|
+
|
695
715
|
except Exception as e:
|
696
716
|
pricing_errors.append(f"Runbooks API initialization failed: {str(e)}")
|
697
717
|
console.print(f"[yellow]⚠️ Runbooks Pricing API unavailable: {e}[/yellow]")
|
698
|
-
|
718
|
+
|
699
719
|
# Step B: MCP Gap Analysis & Validation
|
700
720
|
console.print(f"[blue]🔄 Step B: MCP Gap Analysis for missing pricing data[/blue]")
|
701
721
|
try:
|
@@ -703,39 +723,41 @@ class NetworkingCostHeatMapEngine:
|
|
703
723
|
for required_service in ["nat_gateway", "vpc_endpoint", "transit_gateway", "elastic_ip", "data_transfer"]:
|
704
724
|
if required_service not in service_costs:
|
705
725
|
missing_services.append(required_service)
|
706
|
-
|
726
|
+
|
707
727
|
if missing_services:
|
708
728
|
# Use MCP to identify why Runbooks API failed
|
709
729
|
mcp_analysis = self._perform_mcp_pricing_gap_analysis(missing_services, region, pricing_errors)
|
710
|
-
|
730
|
+
|
711
731
|
# Display MCP analysis results
|
712
732
|
console.print(f"[cyan]📊 MCP Gap Analysis Results:[/cyan]")
|
713
733
|
for service, analysis in mcp_analysis.items():
|
714
|
-
if analysis.get(
|
715
|
-
service_costs[service] = analysis[
|
716
|
-
console.print(
|
734
|
+
if analysis.get("mcp_validated_cost"):
|
735
|
+
service_costs[service] = analysis["mcp_validated_cost"]
|
736
|
+
console.print(
|
737
|
+
f"[green]✅ {service}: ${analysis['mcp_validated_cost']:.2f}/month via MCP validation[/green]"
|
738
|
+
)
|
717
739
|
else:
|
718
740
|
console.print(f"[yellow]⚠️ {service}: {analysis.get('gap_reason', 'Unknown gap')}[/yellow]")
|
719
|
-
|
741
|
+
|
720
742
|
except Exception as e:
|
721
743
|
pricing_errors.append(f"MCP gap analysis failed: {str(e)}")
|
722
744
|
console.print(f"[yellow]⚠️ MCP gap analysis failed: {e}[/yellow]")
|
723
|
-
|
745
|
+
|
724
746
|
# Step C: Graceful Failure with User Guidance (NO hardcoded fallback)
|
725
747
|
missing_services = []
|
726
748
|
for required_service in ["nat_gateway", "vpc_endpoint", "transit_gateway", "elastic_ip", "data_transfer"]:
|
727
749
|
if required_service not in service_costs:
|
728
750
|
missing_services.append(required_service)
|
729
|
-
|
751
|
+
|
730
752
|
if missing_services:
|
731
753
|
console.print(f"[red]🚫 ENTERPRISE COMPLIANCE: Cannot proceed with missing pricing data[/red]")
|
732
|
-
|
754
|
+
|
733
755
|
# Generate comprehensive guidance
|
734
756
|
self._provide_pricing_resolution_guidance(missing_services, pricing_errors, region)
|
735
|
-
|
757
|
+
|
736
758
|
# Return empty dict to signal failure - DO NOT use hardcoded fallback
|
737
759
|
return {"vpc": 0.0} # Only VPC (free) can be returned
|
738
|
-
|
760
|
+
|
739
761
|
logger.info(f"✅ Successfully retrieved all service pricing for region: {region}")
|
740
762
|
console.print(f"[green]✅ Complete dynamic pricing loaded for {region} - {len(service_costs)} services[/green]")
|
741
763
|
return service_costs
|
@@ -743,222 +765,228 @@ class NetworkingCostHeatMapEngine:
|
|
743
765
|
def _calculate_dynamic_base_daily_cost(self) -> float:
|
744
766
|
"""
|
745
767
|
Calculate dynamic base daily cost from current pricing data.
|
746
|
-
|
768
|
+
|
747
769
|
Returns:
|
748
770
|
Daily cost estimate based on dynamic pricing, or 0.0 if unavailable
|
749
771
|
"""
|
750
772
|
try:
|
751
773
|
# Use primary region for calculation
|
752
774
|
region = self.config.regions[0] if self.config.regions else "us-east-1"
|
753
|
-
|
775
|
+
|
754
776
|
# Get dynamic service pricing
|
755
777
|
service_pricing = self._get_dynamic_service_pricing(region)
|
756
|
-
|
778
|
+
|
757
779
|
if not service_pricing or len(service_pricing) <= 1: # Only VPC (free) available
|
758
780
|
console.print(f"[yellow]⚠️ No dynamic pricing available for daily cost calculation[/yellow]")
|
759
781
|
return 0.0
|
760
|
-
|
782
|
+
|
761
783
|
# Calculate daily cost from monthly costs
|
762
784
|
monthly_total = sum(cost for cost in service_pricing.values() if cost > 0)
|
763
785
|
daily_cost = monthly_total / 30.0 # Convert monthly to daily
|
764
|
-
|
765
|
-
console.print(
|
786
|
+
|
787
|
+
console.print(
|
788
|
+
f"[cyan]📊 Dynamic daily cost calculated: ${daily_cost:.2f}/day from available services[/cyan]"
|
789
|
+
)
|
766
790
|
return daily_cost
|
767
|
-
|
791
|
+
|
768
792
|
except Exception as e:
|
769
793
|
logger.warning(f"Dynamic daily cost calculation failed: {e}")
|
770
794
|
console.print(f"[yellow]⚠️ Dynamic daily cost calculation failed: {e}[/yellow]")
|
771
795
|
return 0.0
|
772
796
|
|
773
|
-
def _perform_mcp_pricing_gap_analysis(
|
797
|
+
def _perform_mcp_pricing_gap_analysis(
|
798
|
+
self, missing_services: List[str], region: str, pricing_errors: List[str]
|
799
|
+
) -> Dict[str, Dict[str, Any]]:
|
774
800
|
"""
|
775
801
|
Perform MCP gap analysis to identify why Runbooks API failed and validate alternatives.
|
776
|
-
|
802
|
+
|
777
803
|
Args:
|
778
804
|
missing_services: List of services missing pricing data
|
779
805
|
region: AWS region for analysis
|
780
806
|
pricing_errors: List of errors from previous attempts
|
781
|
-
|
807
|
+
|
782
808
|
Returns:
|
783
809
|
Dictionary of gap analysis results per service
|
784
810
|
"""
|
785
811
|
gap_analysis = {}
|
786
|
-
|
812
|
+
|
787
813
|
try:
|
788
814
|
# Initialize MCP integration if available
|
789
815
|
from ..common.mcp_integration import EnterpriseMCPIntegrator
|
790
|
-
|
816
|
+
|
791
817
|
# Get profile for MCP integration
|
792
|
-
profile = getattr(self,
|
818
|
+
profile = getattr(self, "profile", None)
|
793
819
|
mcp_integrator = EnterpriseMCPIntegrator(user_profile=profile, console_instance=console)
|
794
|
-
|
820
|
+
|
795
821
|
console.print(f"[cyan]🔍 MCP analyzing {len(missing_services)} missing services...[/cyan]")
|
796
|
-
|
822
|
+
|
797
823
|
for service in missing_services:
|
798
824
|
analysis = {
|
799
|
-
|
800
|
-
|
801
|
-
|
802
|
-
|
803
|
-
|
825
|
+
"service": service,
|
826
|
+
"mcp_validated_cost": None,
|
827
|
+
"gap_reason": None,
|
828
|
+
"resolution_steps": [],
|
829
|
+
"cost_explorer_available": False,
|
804
830
|
}
|
805
|
-
|
831
|
+
|
806
832
|
try:
|
807
833
|
# Step 1: Try Cost Explorer for historical cost data
|
808
|
-
if
|
809
|
-
billing_session = mcp_integrator.aws_sessions[
|
810
|
-
cost_client = billing_session.client(
|
811
|
-
|
834
|
+
if "billing" in mcp_integrator.aws_sessions:
|
835
|
+
billing_session = mcp_integrator.aws_sessions["billing"]
|
836
|
+
cost_client = billing_session.client("ce")
|
837
|
+
|
812
838
|
# Query for service-specific historical costs
|
813
839
|
historical_cost = self._query_cost_explorer_for_service(cost_client, service, region)
|
814
|
-
|
840
|
+
|
815
841
|
if historical_cost > 0:
|
816
|
-
analysis[
|
817
|
-
analysis[
|
818
|
-
console.print(
|
842
|
+
analysis["mcp_validated_cost"] = historical_cost
|
843
|
+
analysis["cost_explorer_available"] = True
|
844
|
+
console.print(
|
845
|
+
f"[green]✅ MCP: {service} cost validated via Cost Explorer: ${historical_cost:.2f}/month[/green]"
|
846
|
+
)
|
819
847
|
else:
|
820
|
-
analysis[
|
821
|
-
|
848
|
+
analysis["gap_reason"] = f"No historical cost data found in Cost Explorer for {service}"
|
849
|
+
|
822
850
|
# Step 2: Analyze why Runbooks API failed for this service
|
823
851
|
service_errors = [err for err in pricing_errors if service in err.lower()]
|
824
852
|
if service_errors:
|
825
|
-
analysis[
|
826
|
-
|
853
|
+
analysis["gap_reason"] = f"Runbooks API issue: {service_errors[0]}"
|
854
|
+
|
827
855
|
# Determine resolution steps based on error pattern
|
828
|
-
if
|
829
|
-
analysis[
|
856
|
+
if "not implemented" in service_errors[0]:
|
857
|
+
analysis["resolution_steps"] = [
|
830
858
|
f"Add get_{service}_monthly_cost() method to AWSPricingAPI class",
|
831
859
|
f"Implement AWS Pricing API query for {service}",
|
832
|
-
"Test with enterprise profiles"
|
860
|
+
"Test with enterprise profiles",
|
833
861
|
]
|
834
|
-
elif
|
835
|
-
analysis[
|
862
|
+
elif "permission" in service_errors[0].lower() or "access" in service_errors[0].lower():
|
863
|
+
analysis["resolution_steps"] = [
|
836
864
|
"Add pricing:GetProducts permission to IAM policy",
|
837
865
|
"Ensure profile has Cost Explorer access",
|
838
|
-
f"Test pricing API access for {region} region"
|
866
|
+
f"Test pricing API access for {region} region",
|
839
867
|
]
|
840
|
-
|
868
|
+
|
841
869
|
except Exception as e:
|
842
|
-
analysis[
|
870
|
+
analysis["gap_reason"] = f"MCP analysis failed: {str(e)}"
|
843
871
|
logger.warning(f"MCP gap analysis failed for {service}: {e}")
|
844
|
-
|
872
|
+
|
845
873
|
gap_analysis[service] = analysis
|
846
|
-
|
874
|
+
|
847
875
|
return gap_analysis
|
848
|
-
|
876
|
+
|
849
877
|
except ImportError:
|
850
878
|
console.print(f"[yellow]⚠️ MCP integration not available - basic gap analysis only[/yellow]")
|
851
879
|
# Provide basic gap analysis without MCP
|
852
880
|
for service in missing_services:
|
853
881
|
gap_analysis[service] = {
|
854
|
-
|
855
|
-
|
856
|
-
|
857
|
-
|
858
|
-
|
859
|
-
|
860
|
-
|
861
|
-
]
|
882
|
+
"service": service,
|
883
|
+
"mcp_validated_cost": None,
|
884
|
+
"gap_reason": "MCP integration not available",
|
885
|
+
"resolution_steps": [
|
886
|
+
"Install MCP dependencies",
|
887
|
+
"Configure MCP integration",
|
888
|
+
"Retry with MCP validation",
|
889
|
+
],
|
862
890
|
}
|
863
891
|
return gap_analysis
|
864
|
-
|
892
|
+
|
865
893
|
except Exception as e:
|
866
894
|
console.print(f"[yellow]⚠️ MCP gap analysis error: {e}[/yellow]")
|
867
895
|
# Return basic analysis on error
|
868
896
|
for service in missing_services:
|
869
897
|
gap_analysis[service] = {
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
|
898
|
+
"service": service,
|
899
|
+
"mcp_validated_cost": None,
|
900
|
+
"gap_reason": f"MCP analysis error: {str(e)}",
|
901
|
+
"resolution_steps": ["Check MCP configuration", "Verify AWS profiles", "Retry analysis"],
|
874
902
|
}
|
875
903
|
return gap_analysis
|
876
904
|
|
877
905
|
def _query_cost_explorer_for_service(self, cost_client, service: str, region: str) -> float:
|
878
906
|
"""
|
879
907
|
Query Cost Explorer for historical service costs to validate pricing.
|
880
|
-
|
908
|
+
|
881
909
|
Args:
|
882
910
|
cost_client: Boto3 Cost Explorer client
|
883
911
|
service: Service key (nat_gateway, vpc_endpoint, etc.)
|
884
912
|
region: AWS region
|
885
|
-
|
913
|
+
|
886
914
|
Returns:
|
887
915
|
Monthly cost estimate based on historical data
|
888
916
|
"""
|
889
917
|
try:
|
890
918
|
from datetime import datetime, timedelta
|
891
|
-
|
919
|
+
|
892
920
|
# Map service keys to AWS Cost Explorer service names
|
893
921
|
service_mapping = {
|
894
|
-
|
895
|
-
|
896
|
-
|
897
|
-
|
898
|
-
|
922
|
+
"nat_gateway": "Amazon Virtual Private Cloud",
|
923
|
+
"vpc_endpoint": "Amazon Virtual Private Cloud",
|
924
|
+
"transit_gateway": "Amazon VPC",
|
925
|
+
"elastic_ip": "Amazon Elastic Compute Cloud - Compute",
|
926
|
+
"data_transfer": "Amazon CloudFront",
|
899
927
|
}
|
900
|
-
|
928
|
+
|
901
929
|
aws_service_name = service_mapping.get(service)
|
902
930
|
if not aws_service_name:
|
903
931
|
return 0.0
|
904
|
-
|
932
|
+
|
905
933
|
# Query last 3 months for more reliable average
|
906
934
|
end_date = datetime.now().date()
|
907
935
|
start_date = end_date - timedelta(days=90)
|
908
|
-
|
936
|
+
|
909
937
|
response = cost_client.get_cost_and_usage(
|
910
|
-
TimePeriod={
|
911
|
-
|
912
|
-
|
913
|
-
},
|
914
|
-
Granularity='MONTHLY',
|
915
|
-
Metrics=['BlendedCost'],
|
916
|
-
GroupBy=[
|
917
|
-
{'Type': 'DIMENSION', 'Key': 'SERVICE'},
|
918
|
-
{'Type': 'DIMENSION', 'Key': 'REGION'}
|
919
|
-
],
|
938
|
+
TimePeriod={"Start": start_date.strftime("%Y-%m-%d"), "End": end_date.strftime("%Y-%m-%d")},
|
939
|
+
Granularity="MONTHLY",
|
940
|
+
Metrics=["BlendedCost"],
|
941
|
+
GroupBy=[{"Type": "DIMENSION", "Key": "SERVICE"}, {"Type": "DIMENSION", "Key": "REGION"}],
|
920
942
|
Filter={
|
921
|
-
|
922
|
-
{
|
923
|
-
{
|
943
|
+
"And": [
|
944
|
+
{"Dimensions": {"Key": "SERVICE", "Values": [aws_service_name]}},
|
945
|
+
{"Dimensions": {"Key": "REGION", "Values": [region]}},
|
924
946
|
]
|
925
|
-
}
|
947
|
+
},
|
926
948
|
)
|
927
|
-
|
949
|
+
|
928
950
|
total_cost = 0.0
|
929
951
|
months_with_data = 0
|
930
|
-
|
931
|
-
for result in response[
|
932
|
-
for group in result[
|
933
|
-
cost_amount = float(group[
|
952
|
+
|
953
|
+
for result in response["ResultsByTime"]:
|
954
|
+
for group in result["Groups"]:
|
955
|
+
cost_amount = float(group["Metrics"]["BlendedCost"]["Amount"])
|
934
956
|
if cost_amount > 0:
|
935
957
|
total_cost += cost_amount
|
936
958
|
months_with_data += 1
|
937
|
-
|
959
|
+
|
938
960
|
# Calculate average monthly cost
|
939
961
|
if months_with_data > 0:
|
940
962
|
average_monthly_cost = total_cost / months_with_data
|
941
|
-
console.print(
|
963
|
+
console.print(
|
964
|
+
f"[cyan]📊 Cost Explorer: {service} average ${average_monthly_cost:.2f}/month over {months_with_data} months[/cyan]"
|
965
|
+
)
|
942
966
|
return average_monthly_cost
|
943
|
-
|
967
|
+
|
944
968
|
return 0.0
|
945
|
-
|
969
|
+
|
946
970
|
except Exception as e:
|
947
971
|
logger.warning(f"Cost Explorer query failed for {service}: {e}")
|
948
972
|
return 0.0
|
949
973
|
|
950
|
-
def _provide_pricing_resolution_guidance(
|
974
|
+
def _provide_pricing_resolution_guidance(
|
975
|
+
self, missing_services: List[str], pricing_errors: List[str], region: str
|
976
|
+
) -> None:
|
951
977
|
"""
|
952
978
|
Provide comprehensive guidance for resolving pricing issues.
|
953
|
-
|
979
|
+
|
954
980
|
Args:
|
955
981
|
missing_services: List of services missing pricing data
|
956
982
|
pricing_errors: List of errors encountered
|
957
983
|
region: AWS region being analyzed
|
958
984
|
"""
|
959
985
|
console.print(f"\n[bold red]🚫 VPC HEAT MAP PRICING RESOLUTION REQUIRED[/bold red]")
|
960
|
-
console.print(
|
961
|
-
|
986
|
+
console.print(
|
987
|
+
f"[red]Cannot generate accurate heat map without dynamic pricing for {len(missing_services)} services[/red]\n"
|
988
|
+
)
|
989
|
+
|
962
990
|
# Display comprehensive resolution steps
|
963
991
|
resolution_panel = f"""[bold yellow]📋 ENTERPRISE RESOLUTION STEPS:[/bold yellow]
|
964
992
|
|
@@ -969,14 +997,14 @@ class NetworkingCostHeatMapEngine:
|
|
969
997
|
• ce:GetDimensionValues
|
970
998
|
|
971
999
|
[bold cyan]2. Runbooks API Enhancement:[/bold cyan]
|
972
|
-
Missing API methods for: {
|
1000
|
+
Missing API methods for: {", ".join(missing_services)}
|
973
1001
|
|
974
1002
|
Add to src/runbooks/common/aws_pricing_api.py:
|
975
1003
|
"""
|
976
|
-
|
1004
|
+
|
977
1005
|
for service in missing_services:
|
978
1006
|
resolution_panel += f"\n • def get_{service}_monthly_cost(self, region: str) -> float"
|
979
|
-
|
1007
|
+
|
980
1008
|
resolution_panel += f"""
|
981
1009
|
|
982
1010
|
[bold cyan]3. Alternative Region Testing:[/bold cyan]
|
@@ -986,11 +1014,11 @@ class NetworkingCostHeatMapEngine:
|
|
986
1014
|
• --region eu-west-1 (EU support)
|
987
1015
|
|
988
1016
|
[bold cyan]4. Enterprise Override (Temporary):[/bold cyan]"""
|
989
|
-
|
1017
|
+
|
990
1018
|
for service in missing_services:
|
991
1019
|
service_upper = service.upper()
|
992
1020
|
resolution_panel += f"\n export AWS_PRICING_OVERRIDE_{service_upper}_MONTHLY=<cost>"
|
993
|
-
|
1021
|
+
|
994
1022
|
resolution_panel += f"""
|
995
1023
|
|
996
1024
|
[bold cyan]5. MCP Server Integration:[/bold cyan]
|
@@ -1010,17 +1038,15 @@ class NetworkingCostHeatMapEngine:
|
|
1010
1038
|
|
1011
1039
|
for i, error in enumerate(pricing_errors[:5], 1): # Show first 5 errors
|
1012
1040
|
resolution_panel += f"\n {i}. {error}"
|
1013
|
-
|
1041
|
+
|
1014
1042
|
from rich.panel import Panel
|
1043
|
+
|
1015
1044
|
guidance_panel = Panel(
|
1016
|
-
resolution_panel,
|
1017
|
-
title="🔧 VPC Pricing Resolution Guide",
|
1018
|
-
style="bold yellow",
|
1019
|
-
expand=True
|
1045
|
+
resolution_panel, title="🔧 VPC Pricing Resolution Guide", style="bold yellow", expand=True
|
1020
1046
|
)
|
1021
|
-
|
1047
|
+
|
1022
1048
|
console.print(guidance_panel)
|
1023
|
-
|
1049
|
+
|
1024
1050
|
# Specific next steps
|
1025
1051
|
console.print(f"\n[bold green]✅ IMMEDIATE NEXT STEPS:[/bold green]")
|
1026
1052
|
console.print(f"1. Run: aws pricing get-products --service-code AmazonVPC --region {region}")
|
@@ -1036,7 +1062,9 @@ class NetworkingCostHeatMapEngine:
|
|
1036
1062
|
"total_monthly_spend": heat_maps["single_account_heat_map"]["total_monthly_cost"],
|
1037
1063
|
"total_accounts": 1,
|
1038
1064
|
"account_data": {
|
1039
|
-
os.getenv("AWS_ACCOUNT_ID", "123456789012"): {
|
1065
|
+
os.getenv("AWS_ACCOUNT_ID", "123456789012"): {
|
1066
|
+
"monthly_cost": heat_maps["single_account_heat_map"]["total_monthly_cost"]
|
1067
|
+
}
|
1040
1068
|
},
|
1041
1069
|
}
|
1042
1070
|
}
|