runbooks 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. runbooks/__init__.py +1 -1
  2. runbooks/cfat/WEIGHT_CONFIG_README.md +368 -0
  3. runbooks/cfat/app.ts +27 -19
  4. runbooks/cfat/assessment/runner.py +6 -5
  5. runbooks/cfat/tests/test_weight_configuration.ts +449 -0
  6. runbooks/cfat/weight_config.ts +574 -0
  7. runbooks/cloudops/models.py +20 -14
  8. runbooks/common/__init__.py +26 -9
  9. runbooks/common/aws_pricing.py +1070 -105
  10. runbooks/common/aws_pricing_api.py +276 -44
  11. runbooks/common/date_utils.py +115 -0
  12. runbooks/common/dry_run_examples.py +587 -0
  13. runbooks/common/dry_run_framework.py +520 -0
  14. runbooks/common/enhanced_exception_handler.py +10 -7
  15. runbooks/common/mcp_cost_explorer_integration.py +5 -4
  16. runbooks/common/memory_optimization.py +533 -0
  17. runbooks/common/performance_optimization_engine.py +1153 -0
  18. runbooks/common/profile_utils.py +86 -118
  19. runbooks/common/rich_utils.py +3 -3
  20. runbooks/common/sre_performance_suite.py +574 -0
  21. runbooks/finops/business_case_config.py +314 -0
  22. runbooks/finops/cost_processor.py +19 -4
  23. runbooks/finops/dashboard_runner.py +47 -28
  24. runbooks/finops/ebs_cost_optimizer.py +1 -1
  25. runbooks/finops/ebs_optimizer.py +56 -9
  26. runbooks/finops/embedded_mcp_validator.py +642 -36
  27. runbooks/finops/enhanced_trend_visualization.py +7 -2
  28. runbooks/finops/executive_export.py +789 -0
  29. runbooks/finops/finops_dashboard.py +6 -5
  30. runbooks/finops/finops_scenarios.py +34 -27
  31. runbooks/finops/iam_guidance.py +6 -1
  32. runbooks/finops/nat_gateway_optimizer.py +46 -27
  33. runbooks/finops/notebook_utils.py +1 -1
  34. runbooks/finops/schemas.py +73 -58
  35. runbooks/finops/single_dashboard.py +20 -4
  36. runbooks/finops/tests/test_integration.py +3 -1
  37. runbooks/finops/vpc_cleanup_exporter.py +2 -1
  38. runbooks/finops/vpc_cleanup_optimizer.py +22 -29
  39. runbooks/inventory/core/collector.py +51 -28
  40. runbooks/inventory/discovery.md +197 -247
  41. runbooks/inventory/inventory_modules.py +2 -2
  42. runbooks/inventory/list_ec2_instances.py +3 -3
  43. runbooks/inventory/models/account.py +5 -3
  44. runbooks/inventory/models/inventory.py +1 -1
  45. runbooks/inventory/models/resource.py +5 -3
  46. runbooks/inventory/organizations_discovery.py +102 -13
  47. runbooks/inventory/unified_validation_engine.py +2 -15
  48. runbooks/main.py +255 -92
  49. runbooks/operate/base.py +9 -6
  50. runbooks/operate/deployment_framework.py +5 -4
  51. runbooks/operate/deployment_validator.py +6 -5
  52. runbooks/operate/mcp_integration.py +6 -5
  53. runbooks/operate/networking_cost_heatmap.py +17 -13
  54. runbooks/operate/vpc_operations.py +82 -13
  55. runbooks/remediation/base.py +3 -1
  56. runbooks/remediation/commons.py +5 -5
  57. runbooks/remediation/commvault_ec2_analysis.py +66 -18
  58. runbooks/remediation/config/accounts_example.json +31 -0
  59. runbooks/remediation/multi_account.py +120 -7
  60. runbooks/remediation/remediation_cli.py +710 -0
  61. runbooks/remediation/universal_account_discovery.py +377 -0
  62. runbooks/remediation/workspaces_list.py +2 -2
  63. runbooks/security/compliance_automation_engine.py +99 -20
  64. runbooks/security/config/__init__.py +24 -0
  65. runbooks/security/config/compliance_config.py +255 -0
  66. runbooks/security/config/compliance_weights_example.json +22 -0
  67. runbooks/security/config_template_generator.py +500 -0
  68. runbooks/security/security_cli.py +377 -0
  69. runbooks/validation/cli.py +8 -7
  70. runbooks/validation/comprehensive_2way_validator.py +26 -15
  71. runbooks/validation/mcp_validator.py +62 -8
  72. runbooks/vpc/config.py +49 -15
  73. runbooks/vpc/cross_account_session.py +5 -1
  74. runbooks/vpc/heatmap_engine.py +438 -59
  75. runbooks/vpc/mcp_no_eni_validator.py +115 -36
  76. runbooks/vpc/performance_optimized_analyzer.py +546 -0
  77. runbooks/vpc/runbooks_adapter.py +33 -12
  78. runbooks/vpc/tests/conftest.py +4 -2
  79. runbooks/vpc/tests/test_cost_engine.py +3 -1
  80. {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/METADATA +1 -1
  81. {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/RECORD +85 -79
  82. runbooks/finops/runbooks.inventory.organizations_discovery.log +0 -0
  83. runbooks/finops/runbooks.security.report_generator.log +0 -0
  84. runbooks/finops/runbooks.security.run_script.log +0 -0
  85. runbooks/finops/runbooks.security.security_export.log +0 -0
  86. runbooks/finops/tests/results_test_finops_dashboard.xml +0 -1
  87. runbooks/inventory/artifacts/scale-optimize-status.txt +0 -12
  88. runbooks/inventory/runbooks.inventory.organizations_discovery.log +0 -0
  89. runbooks/inventory/runbooks.security.report_generator.log +0 -0
  90. runbooks/inventory/runbooks.security.run_script.log +0 -0
  91. runbooks/inventory/runbooks.security.security_export.log +0 -0
  92. runbooks/vpc/runbooks.inventory.organizations_discovery.log +0 -0
  93. runbooks/vpc/runbooks.security.report_generator.log +0 -0
  94. runbooks/vpc/runbooks.security.run_script.log +0 -0
  95. runbooks/vpc/runbooks.security.security_export.log +0 -0
  96. {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/WHEEL +0 -0
  97. {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/entry_points.txt +0 -0
  98. {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/licenses/LICENSE +0 -0
  99. {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/top_level.txt +0 -0
@@ -3,6 +3,7 @@ Networking Cost Heat Map Engine - Advanced heat map generation with all required
3
3
  """
4
4
 
5
5
  import logging
6
+ import os
6
7
  from dataclasses import dataclass, field
7
8
  from datetime import datetime, timedelta
8
9
  from typing import Any, Dict, List, Optional, Tuple
@@ -14,6 +15,8 @@ from botocore.exceptions import ClientError
14
15
  from .config import VPCNetworkingConfig
15
16
  from .cost_engine import NetworkingCostEngine
16
17
  from ..common.env_utils import get_required_env_float
18
+ from ..common.aws_pricing_api import AWSPricingAPI
19
+ from ..common.rich_utils import console
17
20
 
18
21
  logger = logging.getLogger(__name__)
19
22
 
@@ -181,7 +184,8 @@ class NetworkingCostHeatMapEngine:
181
184
  """Generate detailed single account heat map"""
182
185
  logger.info("Generating single account heat map")
183
186
 
184
- account_id = "499201730520" # Default single account
187
+ # Use environment-driven account ID for universal compatibility
188
+ account_id = os.getenv("AWS_ACCOUNT_ID", "123456789012")
185
189
 
186
190
  # Create cost distribution matrix
187
191
  heat_map_matrix = np.zeros((len(self.config.regions), len(NETWORKING_SERVICES)))
@@ -231,40 +235,43 @@ class NetworkingCostHeatMapEngine:
231
235
  """Generate multi-account aggregated heat map"""
232
236
  logger.info("Generating multi-account heat map (60 accounts)")
233
237
 
234
- num_accounts = 60
238
+ # Environment-driven account configuration for universal compatibility
239
+ num_accounts = int(os.getenv("AWS_TOTAL_ACCOUNTS", "60"))
235
240
 
236
- # Account categories
241
+ # Account categories with dynamic environment configuration
237
242
  account_categories = {
238
- "production": {"count": 15, "cost_multiplier": 5.0},
239
- "staging": {"count": 15, "cost_multiplier": 2.0},
240
- "development": {"count": 20, "cost_multiplier": 1.0},
241
- "sandbox": {"count": 10, "cost_multiplier": 0.3},
243
+ "production": {"count": int(os.getenv("AWS_PROD_ACCOUNTS", "15")), "cost_multiplier": float(os.getenv("PROD_COST_MULTIPLIER", "5.0"))},
244
+ "staging": {"count": int(os.getenv("AWS_STAGING_ACCOUNTS", "15")), "cost_multiplier": float(os.getenv("STAGING_COST_MULTIPLIER", "2.0"))},
245
+ "development": {"count": int(os.getenv("AWS_DEV_ACCOUNTS", "20")), "cost_multiplier": float(os.getenv("DEV_COST_MULTIPLIER", "1.0"))},
246
+ "sandbox": {"count": int(os.getenv("AWS_SANDBOX_ACCOUNTS", "10")), "cost_multiplier": float(os.getenv("SANDBOX_COST_MULTIPLIER", "0.3"))},
242
247
  }
243
248
 
244
249
  # Generate aggregated matrix
245
250
  aggregated_matrix = np.zeros((len(self.config.regions), len(NETWORKING_SERVICES)))
246
251
  account_breakdown = []
247
252
 
248
- account_id = 100000000000
253
+ # Dynamic base account ID from environment for universal compatibility
254
+ account_id = int(os.getenv("AWS_BASE_ACCOUNT_ID", "100000000000"))
249
255
 
250
256
  for category, details in account_categories.items():
251
257
  for i in range(details["count"]):
252
- # Generate account costs
258
+ # Generate account costs with dynamic pricing (no hardcoded fallbacks)
253
259
  account_matrix = self._generate_account_costs(str(account_id), category, details["cost_multiplier"])
254
260
 
255
- # Add to aggregated
256
- aggregated_matrix += account_matrix
257
-
258
- # Store breakdown
259
- account_breakdown.append(
260
- {
261
- "account_id": str(account_id),
262
- "category": category,
263
- "monthly_cost": float(np.sum(account_matrix)),
264
- "primary_region": self.config.regions[int(np.argmax(np.sum(account_matrix, axis=1)))],
265
- "top_service": list(NETWORKING_SERVICES.keys())[int(np.argmax(np.sum(account_matrix, axis=0)))],
266
- }
267
- )
261
+ # Add to aggregated only if pricing data is available
262
+ if np.sum(account_matrix) > 0: # Only include accounts with valid pricing
263
+ aggregated_matrix += account_matrix
264
+
265
+ # Store breakdown
266
+ account_breakdown.append(
267
+ {
268
+ "account_id": str(account_id),
269
+ "category": category,
270
+ "monthly_cost": float(np.sum(account_matrix)),
271
+ "primary_region": self.config.regions[int(np.argmax(np.sum(account_matrix, axis=1)))],
272
+ "top_service": list(NETWORKING_SERVICES.keys())[int(np.argmax(np.sum(account_matrix, axis=0)))],
273
+ }
274
+ )
268
275
 
269
276
  account_id += 1
270
277
 
@@ -300,8 +307,8 @@ class NetworkingCostHeatMapEngine:
300
307
  time_series_data = {}
301
308
 
302
309
  for period_name, days in periods.items():
303
- # Dynamic base daily cost from environment variable - NO hardcoded defaults
304
- base_daily_cost = get_required_env_float('VPC_BASE_DAILY_COST')
310
+ # Dynamic base daily cost - calculate from dynamic pricing (no hardcoded fallback)
311
+ base_daily_cost = self._calculate_dynamic_base_daily_cost()
305
312
 
306
313
  if period_name == "forecast_90_days":
307
314
  # Forecast with growth trend
@@ -378,11 +385,15 @@ class NetworkingCostHeatMapEngine:
378
385
  region_total = 0
379
386
 
380
387
  for service_idx, (service_key, service_name) in enumerate(NETWORKING_SERVICES.items()):
381
- base_cost = base_service_costs.get(service_key, 10.0)
382
- # REMOVED: Random variation violates enterprise standards
383
- final_cost = base_cost * region_multiplier
384
- regional_matrix[region_idx, service_idx] = max(0, final_cost)
385
- region_total += final_cost
388
+ base_cost = base_service_costs.get(service_key, 0.0) # Default to free (no hardcoded fallback)
389
+ # Only calculate final cost if base cost is available
390
+ if base_cost > 0:
391
+ final_cost = base_cost * region_multiplier
392
+ regional_matrix[region_idx, service_idx] = max(0, final_cost)
393
+ region_total += final_cost
394
+ else:
395
+ # No pricing data available - set to zero
396
+ regional_matrix[region_idx, service_idx] = 0.0
386
397
 
387
398
  # Track service breakdown
388
399
  if service_key not in service_regional_breakdown:
@@ -524,15 +535,28 @@ class NetworkingCostHeatMapEngine:
524
535
 
525
536
  pattern = patterns.get(category, patterns["development"])
526
537
 
527
- # Apply costs based on pattern
538
+ # Apply costs based on pattern using dynamic pricing (NO hardcoded fallbacks)
539
+ region = self.config.regions[0] if self.config.regions else "us-east-1" # Use first configured region
540
+
541
+ # Get dynamic service pricing
542
+ service_pricing = self._get_dynamic_service_pricing(region)
543
+
528
544
  for service_idx, service_key in enumerate(NETWORKING_SERVICES.keys()):
529
545
  for region_idx in range(len(self.config.regions)):
530
- if service_key == "nat_gateway" and region_idx < pattern["nat_gateways"]:
531
- matrix[region_idx, service_idx] = 45.0 * multiplier
532
- elif service_key == "transit_gateway" and pattern["transit_gateway"] and region_idx == 0:
533
- matrix[region_idx, service_idx] = 36.5 * multiplier
534
- elif service_key == "vpc_endpoint" and region_idx < pattern["vpc_endpoints"]:
535
- matrix[region_idx, service_idx] = 10.0 * multiplier
546
+ cost = 0.0 # Default to free
547
+
548
+ # Only apply costs if we have valid pricing data
549
+ if service_key in service_pricing and service_pricing[service_key] > 0:
550
+ if service_key == "nat_gateway" and region_idx < pattern["nat_gateways"]:
551
+ cost = service_pricing[service_key] * multiplier
552
+ elif service_key == "transit_gateway" and pattern["transit_gateway"] and region_idx == 0:
553
+ cost = service_pricing[service_key] * multiplier
554
+ elif service_key == "vpc_endpoint" and region_idx < pattern["vpc_endpoints"]:
555
+ cost = service_pricing[service_key] * multiplier
556
+ elif service_key == "elastic_ip" and region_idx < pattern.get("elastic_ips", 0):
557
+ cost = service_pricing[service_key] * multiplier
558
+
559
+ matrix[region_idx, service_idx] = cost
536
560
 
537
561
  return matrix
538
562
 
@@ -613,10 +637,14 @@ class NetworkingCostHeatMapEngine:
613
637
 
614
638
  def _get_dynamic_service_pricing(self, region: str) -> Dict[str, float]:
615
639
  """
616
- Get dynamic AWS service pricing from AWS Pricing API or Cost Explorer.
640
+ Get dynamic AWS service pricing following enterprise cascade:
641
+
642
+ a. ✅ Try Runbooks API with boto3 (dynamic)
643
+ b. ✅ Try MCP-Servers (dynamic) & gaps analysis with real AWS data
644
+ → If failed, identify WHY option 'a' didn't work, then UPGRADE option 'a'
645
+ c. ✅ Fail gracefully with user guidance (NO hardcoded fallback)
617
646
 
618
- ENTERPRISE COMPLIANCE: Zero tolerance for hardcoded values.
619
- All pricing must be fetched from AWS APIs.
647
+ ENTERPRISE COMPLIANCE: Zero tolerance for hardcoded pricing fallbacks.
620
648
 
621
649
  Args:
622
650
  region: AWS region for pricing lookup
@@ -624,30 +652,381 @@ class NetworkingCostHeatMapEngine:
624
652
  Returns:
625
653
  Dictionary of service pricing (monthly USD)
626
654
  """
655
+ service_costs = {}
656
+ pricing_errors = []
657
+
658
+ # VPC itself is always free
659
+ service_costs["vpc"] = 0.0
660
+
661
+ # Step A: Try Runbooks Pricing API (Enhanced)
662
+ console.print(f"[blue]🔄 Step A: Attempting Runbooks Pricing API for {region}[/blue]")
627
663
  try:
628
- # Try to get pricing from AWS Pricing API
629
- pricing_client = boto3.client('pricing', region_name='us-east-1') # Pricing API only in us-east-1
664
+ from ..common.aws_pricing_api import AWSPricingAPI
630
665
 
631
- # For now, return error to force proper implementation
632
- logging.error("ENTERPRISE VIOLATION: Dynamic pricing not yet implemented")
633
- raise NotImplementedError(
634
- "CRITICAL: Dynamic pricing integration required. "
635
- "Hardcoded values violate enterprise zero-tolerance policy. "
636
- "Must integrate AWS Pricing API or Cost Explorer."
637
- )
666
+ # Initialize with proper session management
667
+ profile = getattr(self, 'profile', None)
668
+ pricing_api = AWSPricingAPI(profile=profile)
669
+
670
+ # NAT Gateway pricing (primary VPC cost component)
671
+ try:
672
+ service_costs["nat_gateway"] = pricing_api.get_nat_gateway_monthly_cost(region)
673
+ console.print(f"[green]✅ NAT Gateway pricing: ${service_costs['nat_gateway']:.2f}/month from Runbooks API[/green]")
674
+ except Exception as e:
675
+ pricing_errors.append(f"NAT Gateway: {str(e)}")
676
+ logger.warning(f"Runbooks API NAT Gateway pricing failed: {e}")
677
+
678
+ # Try other services with existing API methods
679
+ for service_key in ["vpc_endpoint", "transit_gateway", "elastic_ip"]:
680
+ try:
681
+ # Check if API method exists for this service
682
+ if hasattr(pricing_api, f"get_{service_key}_monthly_cost"):
683
+ method = getattr(pricing_api, f"get_{service_key}_monthly_cost")
684
+ service_costs[service_key] = method(region)
685
+ console.print(f"[green]✅ {service_key} pricing: ${service_costs[service_key]:.2f}/month from Runbooks API[/green]")
686
+ else:
687
+ pricing_errors.append(f"{service_key}: API method not implemented")
688
+ except Exception as e:
689
+ pricing_errors.append(f"{service_key}: {str(e)}")
690
+
691
+ # Data transfer pricing (if API available)
692
+ if "data_transfer" not in service_costs:
693
+ pricing_errors.append("data_transfer: API method not implemented")
638
694
 
639
695
  except Exception as e:
640
- logging.error(f"Failed to get dynamic pricing: {e}")
641
- # TEMPORARY: Return minimal structure to prevent crashes
642
- # THIS MUST BE REPLACED WITH REAL AWS PRICING API INTEGRATION
643
- return {
644
- "vpc": 0.0, # VPC itself is free
645
- "nat_gateway": 0.0, # MUST be calculated from AWS Pricing API
646
- "vpc_endpoint": 0.0, # MUST be calculated from AWS Pricing API
647
- "transit_gateway": 0.0, # MUST be calculated from AWS Pricing API
648
- "elastic_ip": 0.0, # MUST be calculated from AWS Pricing API
649
- "data_transfer": 0.0, # MUST be calculated from AWS Pricing API
696
+ pricing_errors.append(f"Runbooks API initialization failed: {str(e)}")
697
+ console.print(f"[yellow]⚠️ Runbooks Pricing API unavailable: {e}[/yellow]")
698
+
699
+ # Step B: MCP Gap Analysis & Validation
700
+ console.print(f"[blue]🔄 Step B: MCP Gap Analysis for missing pricing data[/blue]")
701
+ try:
702
+ missing_services = []
703
+ for required_service in ["nat_gateway", "vpc_endpoint", "transit_gateway", "elastic_ip", "data_transfer"]:
704
+ if required_service not in service_costs:
705
+ missing_services.append(required_service)
706
+
707
+ if missing_services:
708
+ # Use MCP to identify why Runbooks API failed
709
+ mcp_analysis = self._perform_mcp_pricing_gap_analysis(missing_services, region, pricing_errors)
710
+
711
+ # Display MCP analysis results
712
+ console.print(f"[cyan]📊 MCP Gap Analysis Results:[/cyan]")
713
+ for service, analysis in mcp_analysis.items():
714
+ if analysis.get('mcp_validated_cost'):
715
+ service_costs[service] = analysis['mcp_validated_cost']
716
+ console.print(f"[green]✅ {service}: ${analysis['mcp_validated_cost']:.2f}/month via MCP validation[/green]")
717
+ else:
718
+ console.print(f"[yellow]⚠️ {service}: {analysis.get('gap_reason', 'Unknown gap')}[/yellow]")
719
+
720
+ except Exception as e:
721
+ pricing_errors.append(f"MCP gap analysis failed: {str(e)}")
722
+ console.print(f"[yellow]⚠️ MCP gap analysis failed: {e}[/yellow]")
723
+
724
+ # Step C: Graceful Failure with User Guidance (NO hardcoded fallback)
725
+ missing_services = []
726
+ for required_service in ["nat_gateway", "vpc_endpoint", "transit_gateway", "elastic_ip", "data_transfer"]:
727
+ if required_service not in service_costs:
728
+ missing_services.append(required_service)
729
+
730
+ if missing_services:
731
+ console.print(f"[red]🚫 ENTERPRISE COMPLIANCE: Cannot proceed with missing pricing data[/red]")
732
+
733
+ # Generate comprehensive guidance
734
+ self._provide_pricing_resolution_guidance(missing_services, pricing_errors, region)
735
+
736
+ # Return empty dict to signal failure - DO NOT use hardcoded fallback
737
+ return {"vpc": 0.0} # Only VPC (free) can be returned
738
+
739
+ logger.info(f"✅ Successfully retrieved all service pricing for region: {region}")
740
+ console.print(f"[green]✅ Complete dynamic pricing loaded for {region} - {len(service_costs)} services[/green]")
741
+ return service_costs
742
+
743
+ def _calculate_dynamic_base_daily_cost(self) -> float:
744
+ """
745
+ Calculate dynamic base daily cost from current pricing data.
746
+
747
+ Returns:
748
+ Daily cost estimate based on dynamic pricing, or 0.0 if unavailable
749
+ """
750
+ try:
751
+ # Use primary region for calculation
752
+ region = self.config.regions[0] if self.config.regions else "us-east-1"
753
+
754
+ # Get dynamic service pricing
755
+ service_pricing = self._get_dynamic_service_pricing(region)
756
+
757
+ if not service_pricing or len(service_pricing) <= 1: # Only VPC (free) available
758
+ console.print(f"[yellow]⚠️ No dynamic pricing available for daily cost calculation[/yellow]")
759
+ return 0.0
760
+
761
+ # Calculate daily cost from monthly costs
762
+ monthly_total = sum(cost for cost in service_pricing.values() if cost > 0)
763
+ daily_cost = monthly_total / 30.0 # Convert monthly to daily
764
+
765
+ console.print(f"[cyan]📊 Dynamic daily cost calculated: ${daily_cost:.2f}/day from available services[/cyan]")
766
+ return daily_cost
767
+
768
+ except Exception as e:
769
+ logger.warning(f"Dynamic daily cost calculation failed: {e}")
770
+ console.print(f"[yellow]⚠️ Dynamic daily cost calculation failed: {e}[/yellow]")
771
+ return 0.0
772
+
773
+ def _perform_mcp_pricing_gap_analysis(self, missing_services: List[str], region: str, pricing_errors: List[str]) -> Dict[str, Dict[str, Any]]:
774
+ """
775
+ Perform MCP gap analysis to identify why Runbooks API failed and validate alternatives.
776
+
777
+ Args:
778
+ missing_services: List of services missing pricing data
779
+ region: AWS region for analysis
780
+ pricing_errors: List of errors from previous attempts
781
+
782
+ Returns:
783
+ Dictionary of gap analysis results per service
784
+ """
785
+ gap_analysis = {}
786
+
787
+ try:
788
+ # Initialize MCP integration if available
789
+ from ..common.mcp_integration import EnterpriseMCPIntegrator
790
+
791
+ # Get profile for MCP integration
792
+ profile = getattr(self, 'profile', None)
793
+ mcp_integrator = EnterpriseMCPIntegrator(user_profile=profile, console_instance=console)
794
+
795
+ console.print(f"[cyan]🔍 MCP analyzing {len(missing_services)} missing services...[/cyan]")
796
+
797
+ for service in missing_services:
798
+ analysis = {
799
+ 'service': service,
800
+ 'mcp_validated_cost': None,
801
+ 'gap_reason': None,
802
+ 'resolution_steps': [],
803
+ 'cost_explorer_available': False
804
+ }
805
+
806
+ try:
807
+ # Step 1: Try Cost Explorer for historical cost data
808
+ if 'billing' in mcp_integrator.aws_sessions:
809
+ billing_session = mcp_integrator.aws_sessions['billing']
810
+ cost_client = billing_session.client('ce')
811
+
812
+ # Query for service-specific historical costs
813
+ historical_cost = self._query_cost_explorer_for_service(cost_client, service, region)
814
+
815
+ if historical_cost > 0:
816
+ analysis['mcp_validated_cost'] = historical_cost
817
+ analysis['cost_explorer_available'] = True
818
+ console.print(f"[green]✅ MCP: {service} cost validated via Cost Explorer: ${historical_cost:.2f}/month[/green]")
819
+ else:
820
+ analysis['gap_reason'] = f"No historical cost data found in Cost Explorer for {service}"
821
+
822
+ # Step 2: Analyze why Runbooks API failed for this service
823
+ service_errors = [err for err in pricing_errors if service in err.lower()]
824
+ if service_errors:
825
+ analysis['gap_reason'] = f"Runbooks API issue: {service_errors[0]}"
826
+
827
+ # Determine resolution steps based on error pattern
828
+ if 'not implemented' in service_errors[0]:
829
+ analysis['resolution_steps'] = [
830
+ f"Add get_{service}_monthly_cost() method to AWSPricingAPI class",
831
+ f"Implement AWS Pricing API query for {service}",
832
+ "Test with enterprise profiles"
833
+ ]
834
+ elif 'permission' in service_errors[0].lower() or 'access' in service_errors[0].lower():
835
+ analysis['resolution_steps'] = [
836
+ "Add pricing:GetProducts permission to IAM policy",
837
+ "Ensure profile has Cost Explorer access",
838
+ f"Test pricing API access for {region} region"
839
+ ]
840
+
841
+ except Exception as e:
842
+ analysis['gap_reason'] = f"MCP analysis failed: {str(e)}"
843
+ logger.warning(f"MCP gap analysis failed for {service}: {e}")
844
+
845
+ gap_analysis[service] = analysis
846
+
847
+ return gap_analysis
848
+
849
+ except ImportError:
850
+ console.print(f"[yellow]⚠️ MCP integration not available - basic gap analysis only[/yellow]")
851
+ # Provide basic gap analysis without MCP
852
+ for service in missing_services:
853
+ gap_analysis[service] = {
854
+ 'service': service,
855
+ 'mcp_validated_cost': None,
856
+ 'gap_reason': 'MCP integration not available',
857
+ 'resolution_steps': [
858
+ 'Install MCP dependencies',
859
+ 'Configure MCP integration',
860
+ 'Retry with MCP validation'
861
+ ]
862
+ }
863
+ return gap_analysis
864
+
865
+ except Exception as e:
866
+ console.print(f"[yellow]⚠️ MCP gap analysis error: {e}[/yellow]")
867
+ # Return basic analysis on error
868
+ for service in missing_services:
869
+ gap_analysis[service] = {
870
+ 'service': service,
871
+ 'mcp_validated_cost': None,
872
+ 'gap_reason': f'MCP analysis error: {str(e)}',
873
+ 'resolution_steps': ['Check MCP configuration', 'Verify AWS profiles', 'Retry analysis']
874
+ }
875
+ return gap_analysis
876
+
877
+ def _query_cost_explorer_for_service(self, cost_client, service: str, region: str) -> float:
878
+ """
879
+ Query Cost Explorer for historical service costs to validate pricing.
880
+
881
+ Args:
882
+ cost_client: Boto3 Cost Explorer client
883
+ service: Service key (nat_gateway, vpc_endpoint, etc.)
884
+ region: AWS region
885
+
886
+ Returns:
887
+ Monthly cost estimate based on historical data
888
+ """
889
+ try:
890
+ from datetime import datetime, timedelta
891
+
892
+ # Map service keys to AWS Cost Explorer service names
893
+ service_mapping = {
894
+ 'nat_gateway': 'Amazon Virtual Private Cloud',
895
+ 'vpc_endpoint': 'Amazon Virtual Private Cloud',
896
+ 'transit_gateway': 'Amazon VPC',
897
+ 'elastic_ip': 'Amazon Elastic Compute Cloud - Compute',
898
+ 'data_transfer': 'Amazon CloudFront'
650
899
  }
900
+
901
+ aws_service_name = service_mapping.get(service)
902
+ if not aws_service_name:
903
+ return 0.0
904
+
905
+ # Query last 3 months for more reliable average
906
+ end_date = datetime.now().date()
907
+ start_date = end_date - timedelta(days=90)
908
+
909
+ response = cost_client.get_cost_and_usage(
910
+ TimePeriod={
911
+ 'Start': start_date.strftime('%Y-%m-%d'),
912
+ 'End': end_date.strftime('%Y-%m-%d')
913
+ },
914
+ Granularity='MONTHLY',
915
+ Metrics=['BlendedCost'],
916
+ GroupBy=[
917
+ {'Type': 'DIMENSION', 'Key': 'SERVICE'},
918
+ {'Type': 'DIMENSION', 'Key': 'REGION'}
919
+ ],
920
+ Filter={
921
+ 'And': [
922
+ {'Dimensions': {'Key': 'SERVICE', 'Values': [aws_service_name]}},
923
+ {'Dimensions': {'Key': 'REGION', 'Values': [region]}}
924
+ ]
925
+ }
926
+ )
927
+
928
+ total_cost = 0.0
929
+ months_with_data = 0
930
+
931
+ for result in response['ResultsByTime']:
932
+ for group in result['Groups']:
933
+ cost_amount = float(group['Metrics']['BlendedCost']['Amount'])
934
+ if cost_amount > 0:
935
+ total_cost += cost_amount
936
+ months_with_data += 1
937
+
938
+ # Calculate average monthly cost
939
+ if months_with_data > 0:
940
+ average_monthly_cost = total_cost / months_with_data
941
+ console.print(f"[cyan]📊 Cost Explorer: {service} average ${average_monthly_cost:.2f}/month over {months_with_data} months[/cyan]")
942
+ return average_monthly_cost
943
+
944
+ return 0.0
945
+
946
+ except Exception as e:
947
+ logger.warning(f"Cost Explorer query failed for {service}: {e}")
948
+ return 0.0
949
+
950
+ def _provide_pricing_resolution_guidance(self, missing_services: List[str], pricing_errors: List[str], region: str) -> None:
951
+ """
952
+ Provide comprehensive guidance for resolving pricing issues.
953
+
954
+ Args:
955
+ missing_services: List of services missing pricing data
956
+ pricing_errors: List of errors encountered
957
+ region: AWS region being analyzed
958
+ """
959
+ console.print(f"\n[bold red]🚫 VPC HEAT MAP PRICING RESOLUTION REQUIRED[/bold red]")
960
+ console.print(f"[red]Cannot generate accurate heat map without dynamic pricing for {len(missing_services)} services[/red]\n")
961
+
962
+ # Display comprehensive resolution steps
963
+ resolution_panel = f"""[bold yellow]📋 ENTERPRISE RESOLUTION STEPS:[/bold yellow]
964
+
965
+ [bold cyan]1. IAM Permissions (Most Common Fix):[/bold cyan]
966
+ Add these policies to your AWS profile:
967
+ • pricing:GetProducts
968
+ • ce:GetCostAndUsage
969
+ • ce:GetDimensionValues
970
+
971
+ [bold cyan]2. Runbooks API Enhancement:[/bold cyan]
972
+ Missing API methods for: {', '.join(missing_services)}
973
+
974
+ Add to src/runbooks/common/aws_pricing_api.py:
975
+ """
976
+
977
+ for service in missing_services:
978
+ resolution_panel += f"\n • def get_{service}_monthly_cost(self, region: str) -> float"
979
+
980
+ resolution_panel += f"""
981
+
982
+ [bold cyan]3. Alternative Region Testing:[/bold cyan]
983
+ Try with regions with better Pricing API support:
984
+ • --region us-east-1 (best support)
985
+ • --region us-west-2 (good support)
986
+ • --region eu-west-1 (EU support)
987
+
988
+ [bold cyan]4. Enterprise Override (Temporary):[/bold cyan]"""
989
+
990
+ for service in missing_services:
991
+ service_upper = service.upper()
992
+ resolution_panel += f"\n export AWS_PRICING_OVERRIDE_{service_upper}_MONTHLY=<cost>"
993
+
994
+ resolution_panel += f"""
995
+
996
+ [bold cyan]5. MCP Server Integration:[/bold cyan]
997
+ Ensure MCP servers are accessible and operational
998
+ Check .mcp.json configuration
999
+
1000
+ [bold cyan]6. Profile Validation:[/bold cyan]
1001
+ Current region: {region}
1002
+ Verify profile has access to:
1003
+ • AWS Pricing API (pricing:GetProducts)
1004
+ • Cost Explorer API (ce:GetCostAndUsage)
1005
+
1006
+ 💡 [bold green]QUICK TEST:[/bold green]
1007
+ aws pricing get-products --service-code AmazonVPC --region {region}
1008
+
1009
+ 🔧 [bold green]DEBUG ERRORS:[/bold green]"""
1010
+
1011
+ for i, error in enumerate(pricing_errors[:5], 1): # Show first 5 errors
1012
+ resolution_panel += f"\n {i}. {error}"
1013
+
1014
+ from rich.panel import Panel
1015
+ guidance_panel = Panel(
1016
+ resolution_panel,
1017
+ title="🔧 VPC Pricing Resolution Guide",
1018
+ style="bold yellow",
1019
+ expand=True
1020
+ )
1021
+
1022
+ console.print(guidance_panel)
1023
+
1024
+ # Specific next steps
1025
+ console.print(f"\n[bold green]✅ IMMEDIATE NEXT STEPS:[/bold green]")
1026
+ console.print(f"1. Run: aws pricing get-products --service-code AmazonVPC --region {region}")
1027
+ console.print(f"2. Check IAM permissions for your current profile")
1028
+ console.print(f"3. Try alternative region: runbooks vpc --region us-east-1")
1029
+ console.print(f"4. Contact CloudOps team if pricing API access is restricted\n")
651
1030
 
652
1031
  def _add_mcp_validation(self, heat_maps: Dict) -> Dict:
653
1032
  """Add MCP validation results"""
@@ -657,7 +1036,7 @@ class NetworkingCostHeatMapEngine:
657
1036
  "total_monthly_spend": heat_maps["single_account_heat_map"]["total_monthly_cost"],
658
1037
  "total_accounts": 1,
659
1038
  "account_data": {
660
- "499201730520": {"monthly_cost": heat_maps["single_account_heat_map"]["total_monthly_cost"]}
1039
+ os.getenv("AWS_ACCOUNT_ID", "123456789012"): {"monthly_cost": heat_maps["single_account_heat_map"]["total_monthly_cost"]}
661
1040
  },
662
1041
  }
663
1042
  }