runbooks 0.9.9__py3-none-any.whl → 1.0.1__py3-none-any.whl

runbooks/finops/vpc_cleanup_optimizer.py

@@ -47,9 +47,11 @@ from ..common.rich_utils import (
     console, print_header, print_success, print_error, print_warning, print_info,
     create_table, create_progress_bar, format_cost, create_panel, STATUS_INDICATORS
 )
+from ..common.aws_pricing import DynamicAWSPricing
 from .embedded_mcp_validator import EmbeddedMCPValidator
 from ..common.profile_utils import get_profile_for_operation
 from ..security.enterprise_security_framework import EnterpriseSecurityFramework
+from ..vpc.mcp_no_eni_validator import NOENIVPCMCPValidator, DynamicDiscoveryResults
 
 logger = logging.getLogger(__name__)
 
@@ -364,22 +366,76 @@ class VPCCleanupOptimizer:
         """
         Enhanced VPC discovery with organization account context.
         
-        CRITICAL INSIGHT: This method discovers VPCs that the current profile can access,
-        which may include VPCs from multiple accounts if the profile has cross-account permissions
-        (e.g., via AWS SSO or cross-account roles).
+        CRITICAL FIX: This method now attempts to discover VPCs across multiple accounts
+        by trying different access patterns:
+        1. Direct access with current profile
+        2. Cross-account role assumption (if available)
+        3. Aggregation from multiple AWS SSO profiles
         """
         vpc_candidates = []
+        total_accounts_checked = 0
+        accounts_with_vpcs = set()
         
         if progress_callback:
-            progress_callback("Discovering VPCs across regions...")
+            progress_callback(f"Discovering VPCs across {len(account_ids)} organization accounts...")
         
         # Get list of all regions
         ec2_client = self.session.client('ec2', region_name='us-east-1')
         regions = [region['RegionName'] for region in ec2_client.describe_regions()['Regions']]
         
-        print_info(f"🌍 Scanning {len(regions)} AWS regions for accessible VPCs...")
+        print_info(f"🌍 Scanning {len(regions)} AWS regions across {len(account_ids)} accounts...")
         
+        # First, discover VPCs in current profile's accessible accounts
+        current_account_vpcs = self._discover_vpcs_current_profile(regions, progress_callback)
+        vpc_candidates.extend(current_account_vpcs)
+        
+        # Extract unique account IDs from discovered VPCs
+        for vpc in current_account_vpcs:
+            if vpc.account_id:
+                accounts_with_vpcs.add(vpc.account_id)
+        
+        # Attempt cross-account discovery for remaining accounts
+        remaining_accounts = [acc for acc in account_ids if acc not in accounts_with_vpcs]
+        
+        if remaining_accounts:
+            print_info(f"🔄 Attempting cross-account discovery for {len(remaining_accounts)} additional accounts...")
+            
+            # Try different access patterns for remaining accounts
+            for account_id in remaining_accounts[:10]:  # Limit to first 10 for performance
+                total_accounts_checked += 1
+                account_name = accounts_info.get(account_id, {}).get('name', 'Unknown')
+                
+                if progress_callback:
+                    progress_callback(f"Checking account {account_name} ({account_id[:12]}...)")
+                
+                # Attempt cross-account access
+                cross_account_vpcs = self._attempt_cross_account_discovery(
+                    account_id, account_name, regions
+                )
+                
+                if cross_account_vpcs:
+                    vpc_candidates.extend(cross_account_vpcs)
+                    accounts_with_vpcs.add(account_id)
+                    print_success(f"  ✅ Found {len(cross_account_vpcs)} VPCs in {account_name}")
+        
+        # Summary
+        print_success(f"✅ Discovered {len(vpc_candidates)} total VPCs across {len(accounts_with_vpcs)} accounts")
+        print_info(f"📊 Organization scope: {len(account_ids)} accounts, {total_accounts_checked} checked, {len(accounts_with_vpcs)} with VPCs")
+        
+        # If we still have < 13 VPCs, provide guidance
+        if len(vpc_candidates) < 13:
+            print_warning(f"⚠️ Only {len(vpc_candidates)} VPCs found (target: ≥13). Consider:")
+            print_info("  1. Using MANAGEMENT_PROFILE with broader cross-account access")
+            print_info("  2. Configuring cross-account roles for VPC discovery")
+            print_info("  3. Running discovery from each account individually")
+        
+        return vpc_candidates
+    
+    def _discover_vpcs_current_profile(self, regions: List[str], progress_callback=None) -> List[VPCCleanupCandidate]:
+        """Discover VPCs accessible with current profile."""
+        vpc_candidates = []
         regions_with_vpcs = 0
+        
         for region in regions:
             try:
                 regional_ec2 = self.session.client('ec2', region_name=region)
@@ -409,13 +465,64 @@ class VPCCleanupOptimizer:
                     regions_with_vpcs += 1
                     
             except ClientError as e:
-                if "UnauthorizedOperation" in str(e):
-                    print_warning(f"No access to region {region} (expected for cross-account)")
-                else:
-                    print_warning(f"Could not access region {region}: {e}")
+                # Silently skip regions with no access
+                pass
         
-        print_success(f"✅ Discovered {len(vpc_candidates)} VPCs across {regions_with_vpcs} accessible regions")
-        print_info(f"📊 Organization context: {len(account_ids)} accounts in scope")
+        return vpc_candidates
+    
+    def _attempt_cross_account_discovery(self, account_id: str, account_name: str, 
+                                        regions: List[str]) -> List[VPCCleanupCandidate]:
+        """Attempt to discover VPCs in a specific account using cross-account access."""
+        vpc_candidates = []
+        
+        # Try to assume a cross-account role (if configured)
+        role_name = "OrganizationAccountAccessRole"  # Standard AWS Organizations role
+        
+        try:
+            # Attempt to assume role in target account
+            sts_client = self.session.client('sts')
+            assumed_role = sts_client.assume_role(
+                RoleArn=f"arn:aws:iam::{account_id}:role/{role_name}",
+                RoleSessionName=f"VPCDiscovery-{account_id[:12]}"
+            )
+            
+            # Create session with assumed role credentials
+            assumed_session = boto3.Session(
+                aws_access_key_id=assumed_role['Credentials']['AccessKeyId'],
+                aws_secret_access_key=assumed_role['Credentials']['SecretAccessKey'],
+                aws_session_token=assumed_role['Credentials']['SessionToken']
+            )
+            
+            # Discover VPCs in target account
+            for region in regions[:3]:  # Check first 3 regions for performance
+                try:
+                    ec2_client = assumed_session.client('ec2', region_name=region)
+                    response = ec2_client.describe_vpcs()
+                    
+                    for vpc in response['Vpcs']:
+                        candidate = VPCCleanupCandidate(
+                            vpc_id=vpc['VpcId'],
+                            region=region,
+                            state=vpc['State'],
+                            cidr_block=vpc['CidrBlock'],
+                            is_default=vpc.get('IsDefault', False),
+                            account_id=account_id,  # Set explicit account ID
+                            dependency_analysis=VPCDependencyAnalysis(
+                                vpc_id=vpc['VpcId'],
+                                region=region,
+                                is_default_vpc=vpc.get('IsDefault', False)
+                            ),
+                            tags={tag['Key']: tag['Value'] for tag in vpc.get('Tags', [])}
+                        )
+                        vpc_candidates.append(candidate)
+                        
+                except Exception:
+                    # Skip regions with access issues
+                    pass
+                    
+        except Exception as e:
+            # Cross-account access not available - this is expected for most profiles
+            pass
         
         return vpc_candidates
     
@@ -482,6 +589,237 @@ class VPCCleanupOptimizer:
         print_success(f"✅ Discovered {len(vpc_candidates)} VPC candidates across {len(regions)} regions")
         return vpc_candidates
 
+    async def discover_no_eni_vpcs_with_mcp_validation(self, 
+                                                     target_regions: List[str] = None,
+                                                     max_concurrent_accounts: int = 10) -> Tuple[List[VPCCleanupCandidate], DynamicDiscoveryResults]:
+        """
+        Discover NO-ENI VPCs across all AWS accounts using real-time MCP validation.
+        
+        This method integrates with the dynamic MCP validator to discover the actual
+        count of NO-ENI VPCs across all accessible accounts, not hardcoded numbers.
+        
+        Args:
+            target_regions: List of regions to scan (default: ['ap-southeast-2'])
+            max_concurrent_accounts: Maximum concurrent account scans
+            
+        Returns:
+            Tuple of (VPC cleanup candidates, Dynamic discovery results)
+        """
+        if target_regions is None:
+            target_regions = ['ap-southeast-2']
+        
+        print_header("🌐 Real-Time NO-ENI VPC Discovery", "MCP-Validated VPC Cleanup Analysis")
+        
+        # Initialize MCP validator with universal profile support
+        print_info("🔧 Initializing dynamic MCP validator...")
+        mcp_validator = NOENIVPCMCPValidator(user_profile=self.profile)
+        
+        # Perform dynamic discovery across all accounts
+        print_info("🚀 Starting real-time discovery across all AWS accounts...")
+        discovery_results = await mcp_validator.discover_all_no_eni_vpcs_dynamically(
+            target_regions=target_regions,
+            max_concurrent_accounts=max_concurrent_accounts
+        )
+        
+        # Convert MCP discovery results to VPC cleanup candidates
+        print_info("🔄 Converting MCP results to VPC cleanup candidates...")
+        cleanup_candidates = []
+        
+        for target in discovery_results.account_region_results:
+            if not target.has_access or not target.no_eni_vpcs:
+                continue
+                
+            # Get detailed VPC information for each NO-ENI VPC
+            try:
+                # Use appropriate session for this account
+                session = self._get_session_for_account(target.account_id)
+                ec2_client = session.client('ec2', region_name=target.region)
+                
+                # Get VPC details
+                vpc_response = ec2_client.describe_vpcs(VpcIds=target.no_eni_vpcs)
+                
+                for vpc in vpc_response.get('Vpcs', []):
+                    # Create comprehensive dependency analysis
+                    dependency_analysis = await self._create_dependency_analysis_for_vpc(
+                        vpc['VpcId'], target.region, ec2_client
+                    )
+                    
+                    # Create VPC cleanup candidate
+                    candidate = VPCCleanupCandidate(
+                        vpc_id=vpc['VpcId'],
+                        region=target.region,
+                        state=vpc.get('State', 'unknown'),
+                        cidr_block=vpc.get('CidrBlock', ''),
+                        is_default=vpc.get('IsDefault', False),
+                        dependency_analysis=dependency_analysis,
+                        cleanup_bucket='internal',  # NO-ENI VPCs go to internal bucket
+                        monthly_cost=self._estimate_vpc_monthly_cost(vpc),
+                        annual_savings=self._estimate_vpc_monthly_cost(vpc) * 12,
+                        cleanup_recommendation='ready' if dependency_analysis.eni_count == 0 else 'investigate',
+                        risk_assessment='low' if dependency_analysis.eni_count == 0 else 'medium',
+                        business_impact='minimal',
+                        tags=self._extract_vpc_tags(vpc),
+                        account_id=target.account_id,
+                        flow_logs_enabled=await self._check_flow_logs_enabled(vpc['VpcId'], ec2_client)
+                    )
+                    
+                    cleanup_candidates.append(candidate)
+                    
+            except Exception as e:
+                print_warning(f"Failed to analyze VPC details for account {target.account_id}: {e}")
+                continue
+        
+        # Display integration results
+        print_header("🎯 MCP-Validated VPC Cleanup Summary", "Real-Time Integration Results")
+        console.print(f"[bold green]✅ NO-ENI VPCs discovered: {len(cleanup_candidates)}[/bold green]")
+        console.print(f"[bold blue]📊 Accounts scanned: {discovery_results.total_accounts_scanned}[/bold blue]")
+        console.print(f"[bold yellow]🌍 Regions scanned: {discovery_results.total_regions_scanned}[/bold yellow]")
+        console.print(f"[bold magenta]🧪 MCP validation accuracy: {discovery_results.mcp_validation_accuracy:.2f}%[/bold magenta]")
+        
+        # Calculate potential savings
+        total_annual_savings = sum(candidate.annual_savings for candidate in cleanup_candidates)
+        console.print(f"[bold cyan]💰 Potential annual savings: {format_cost(total_annual_savings)}[/bold cyan]")
+        
+        # Validation status
+        if discovery_results.mcp_validation_accuracy >= 99.5:
+            print_success(f"✅ ENTERPRISE VALIDATION PASSED: {discovery_results.mcp_validation_accuracy:.2f}% accuracy")
+        else:
+            print_warning(f"⚠️ VALIDATION REVIEW REQUIRED: {discovery_results.mcp_validation_accuracy:.2f}% accuracy")
+        
+        return cleanup_candidates, discovery_results
+    
+    def _determine_profile_type(self, profile_name: str) -> Optional[str]:
+        """Determine profile type from profile name for universal compatibility."""
+        # Universal pattern matching for any AWS profile naming convention
+        if 'billing' in profile_name.lower() or 'Billing' in profile_name:
+            return 'BILLING'
+        elif 'management' in profile_name.lower() or 'admin' in profile_name.lower():
+            return 'MANAGEMENT'
+        elif 'ops' in profile_name.lower() or 'operational' in profile_name.lower():
+            return 'CENTRALISED_OPS'
+        return None
+    
+    def _get_session_for_account(self, account_id: str) -> boto3.Session:
+        """Get appropriate session for accessing a specific account using universal profile management."""
+        from runbooks.common.profile_utils import get_profile_for_operation
+        
+        # In enterprise setup, would assume role here
+        # For now, return session with best available profile using three-tier priority system
+        
+        # Try different operation types in priority order
+        profile_types = ['management', 'operational', 'billing']
+        
+        for profile_type in profile_types:
+            try:
+                profile_name = get_profile_for_operation(profile_type, self.profile)
+                session = boto3.Session(profile_name=profile_name)
+                # Verify access
+                sts_client = session.client('sts')
+                identity = sts_client.get_caller_identity()
+                
+                if identity['Account'] == account_id:
+                    return session
+            except Exception:
+                continue
+        
+        # Fallback to current session
+        return self.session
+    
+    async def _create_dependency_analysis_for_vpc(self, 
+                                                vpc_id: str, 
+                                                region: str, 
+                                                ec2_client) -> VPCDependencyAnalysis:
+        """Create comprehensive dependency analysis for a VPC."""
+        try:
+            # Get ENI count
+            eni_response = ec2_client.describe_network_interfaces(
+                Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]
+            )
+            eni_count = len(eni_response.get('NetworkInterfaces', []))
+            
+            # Get route tables
+            rt_response = ec2_client.describe_route_tables(
+                Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]
+            )
+            route_tables = [rt['RouteTableId'] for rt in rt_response.get('RouteTables', [])]
+            
+            # Get security groups
+            sg_response = ec2_client.describe_security_groups(
+                Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]
+            )
+            security_groups = [sg['GroupId'] for sg in sg_response.get('SecurityGroups', [])]
+            
+            # Get internet gateways
+            igw_response = ec2_client.describe_internet_gateways(
+                Filters=[{'Name': 'attachment.vpc-id', 'Values': [vpc_id]}]
+            )
+            internet_gateways = [igw['InternetGatewayId'] for igw in igw_response.get('InternetGateways', [])]
+            
+            # Get NAT gateways
+            nat_response = ec2_client.describe_nat_gateways(
+                Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}]
+            )
+            nat_gateways = [nat['NatGatewayId'] for nat in nat_response.get('NatGateways', [])]
+            
+            # Determine risk level
+            if eni_count == 0 and len(nat_gateways) == 0 and len(internet_gateways) <= 1:
+                risk_level = 'low'
+            elif eni_count == 0:
+                risk_level = 'medium'
+            else:
+                risk_level = 'high'
+            
+            return VPCDependencyAnalysis(
+                vpc_id=vpc_id,
+                region=region,
+                eni_count=eni_count,
+                route_tables=route_tables,
+                security_groups=security_groups,
+                internet_gateways=internet_gateways,
+                nat_gateways=nat_gateways,
+                dependency_risk_level=risk_level
+            )
+            
+        except Exception as e:
+            print_warning(f"Failed to analyze dependencies for {vpc_id}: {e}")
+            return VPCDependencyAnalysis(
+                vpc_id=vpc_id,
+                region=region,
+                dependency_risk_level='unknown'
+            )
+    
+    def _estimate_vpc_monthly_cost(self, vpc: Dict[str, Any]) -> float:
+        """Estimate monthly cost for VPC resources."""
+        # Base VPC cost estimation (simplified)
+        # In enterprise setup, would integrate with Cost Explorer
+        base_cost = 0.0
+        
+        # Default VPCs might have default resources
+        if vpc.get('IsDefault', False):
+            base_cost += 5.0  # Estimated monthly cost for default VPC resources
+        
+        return base_cost
+    
+    def _extract_vpc_tags(self, vpc: Dict[str, Any]) -> Dict[str, str]:
+        """Extract tags from VPC data."""
+        tags = {}
+        for tag in vpc.get('Tags', []):
+            tags[tag['Key']] = tag['Value']
+        return tags
+    
+    async def _check_flow_logs_enabled(self, vpc_id: str, ec2_client) -> bool:
+        """Check if VPC Flow Logs are enabled."""
+        try:
+            response = ec2_client.describe_flow_logs(
+                Filters=[
+                    {'Name': 'resource-id', 'Values': [vpc_id]},
+                    {'Name': 'resource-type', 'Values': ['VPC']}
+                ]
+            )
+            return len(response.get('FlowLogs', [])) > 0
+        except Exception:
+            return False
+
     def _analyze_vpc_dependencies(self, candidates: List[VPCCleanupCandidate]) -> List[VPCCleanupCandidate]:
         """Analyze VPC dependencies for cleanup safety assessment."""
         print_info("🔍 Analyzing VPC dependencies for safety assessment...")
@@ -972,12 +1310,21 @@ class VPCCleanupOptimizer:
         """Calculate VPC cleanup costs and savings estimation."""
         print_info("💰 Calculating VPC cleanup costs and savings...")
         
-        # Standard VPC cost estimation (based on AWSO-05 analysis)
-        # These are conservative estimates for VPC resources
+        # Dynamic VPC cost calculation (enterprise compliance)
+        # Using dynamic pricing engine for accurate regional costs
+        pricing_engine = DynamicAWSPricing()
+        default_region = 'us-east-1'  # Default region for cost estimation
+        
         monthly_vpc_base_cost = 0.0  # VPCs themselves are free
-        monthly_nat_gateway_cost = 45.0  # $45/month per NAT Gateway
-        monthly_vpc_endpoint_cost = 7.2  # $7.20/month per VPC Endpoint
-        monthly_data_processing_cost = 50.0  # Estimated data processing costs
+        nat_result = pricing_engine.get_service_pricing('nat_gateway', default_region)
+        monthly_nat_gateway_cost = nat_result.monthly_cost
+        
+        endpoint_result = pricing_engine.get_service_pricing('vpc_endpoint', default_region)
+        monthly_vpc_endpoint_cost = endpoint_result.monthly_cost
+        
+        # Data processing costs vary by usage - using conservative estimate per region
+        regional_multiplier = pricing_engine.regional_multipliers.get(default_region, 1.0)
+        monthly_data_processing_cost = 50.0 * regional_multiplier  # Base estimate adjusted for region
         
         total_annual_savings = 0.0
         cost_details = {}

runbooks/inventory/__init__.py

@@ -26,6 +26,11 @@ from runbooks.inventory.collectors.base import BaseResourceCollector
 from runbooks.inventory.core.collector import InventoryCollector
 from runbooks.inventory.core.formatter import InventoryFormatter
 
+# Enhanced collector integrated into core collector module
+from runbooks.inventory.core.collector import (
+    EnhancedInventoryCollector,
+)
+
 # Data models
 from runbooks.inventory.models.account import AWSAccount, OrganizationAccount
 from runbooks.inventory.models.inventory import InventoryMetadata, InventoryResult
@@ -38,13 +43,17 @@ from runbooks.inventory.utils.validation import validate_aws_account_id, validat
 # VPC Module Migration Integration
 from runbooks.inventory.vpc_analyzer import VPCAnalyzer, VPCDiscoveryResult, AWSOAnalysis
 
+# Note: EnhancedInventoryCollector now imported above from core.collector
+
 # Import centralized version from main runbooks package
 from runbooks import __version__
 
 __all__ = [
     # Core functionality
     "InventoryCollector",
-    "InventoryFormatter",
+    "InventoryFormatter", 
+    # Enhanced functionality with proven finops patterns
+    "EnhancedInventoryCollector",
     # Base classes for extension
     "BaseResourceCollector",
     # Data models