runbooks 1.0.3__py3-none-any.whl → 1.1.1__py3-none-any.whl

runbooks/cloudops/cost_optimizer.py

@@ -85,8 +85,61 @@ class CostOptimizer(CloudOpsBase):
         
         if dry_run:
             print_warning("🛡️  DRY RUN MODE: No resources will be modified")
-    
-    
+
+        # Performance tracking
+        self.operation_start_time = time.time()
+
+    def _measure_execution_time(self) -> float:
+        """
+        Measure actual execution time from operation start.
+
+        Returns:
+            Execution time in seconds
+        """
+        if hasattr(self, 'operation_start_time'):
+            return time.time() - self.operation_start_time
+        else:
+            # Fallback if start time not tracked
+            import time
+            return time.time() - time.time()  # Returns ~0.0
+
+    def _suggest_smaller_instance_type(self, instance_type: str) -> Optional[str]:
+        """
+        Suggest a smaller instance type for rightsizing.
+
+        Args:
+            instance_type: Current EC2 instance type
+
+        Returns:
+            Suggested smaller instance type or None
+        """
+        # Simple rightsizing mapping - can be enhanced with CloudWatch metrics
+        rightsizing_map = {
+            # T3 family
+            't3.large': 't3.medium',
+            't3.xlarge': 't3.large',
+            't3.2xlarge': 't3.xlarge',
+
+            # M5 family
+            'm5.large': 'm5.medium',
+            'm5.xlarge': 'm5.large',
+            'm5.2xlarge': 'm5.xlarge',
+            'm5.4xlarge': 'm5.2xlarge',
+
+            # C5 family
+            'c5.large': 'c5.medium',
+            'c5.xlarge': 'c5.large',
+            'c5.2xlarge': 'c5.xlarge',
+            'c5.4xlarge': 'c5.2xlarge',
+
+            # R5 family
+            'r5.large': 'r5.medium',
+            'r5.xlarge': 'r5.large',
+            'r5.2xlarge': 'r5.xlarge',
+        }
+
+        return rightsizing_map.get(instance_type)
+
     async def discover_infrastructure(
         self,
         regions: Optional[List[str]] = None,
@@ -383,54 +436,396 @@ class CostOptimizer(CloudOpsBase):
         """Analyze EC2 instances for rightsizing opportunities."""
         print_info("🔍 Analyzing EC2 rightsizing opportunities...")
         
-        # Placeholder implementation - would integrate with CloudWatch metrics
-        return {
-            'instances_analyzed': 45,
-            'oversized_instances': 12,
-            'potential_savings': 2850.00,
-            'resources_analyzed': 45,
-            'resource_impacts': []
-        }
+        # Real AWS integration for rightsizing analysis
+        from runbooks.common.aws_pricing import get_aws_pricing_engine, get_ec2_monthly_cost
+
+        try:
+            pricing_engine = get_aws_pricing_engine(profile=self.profile)
+
+            # Get actual EC2 instances from AWS API
+            ec2_client = self.session.client('ec2')
+            response = ec2_client.describe_instances()
+
+            instances_analyzed = 0
+            oversized_instances = 0
+            potential_monthly_savings = 0.0
+
+            for reservation in response['Reservations']:
+                for instance in reservation['Instances']:
+                    if instance['State']['Name'] in ['running', 'stopped']:
+                        instances_analyzed += 1
+                        instance_type = instance['InstanceType']
+
+                        # Calculate potential savings from rightsizing
+                        current_cost = get_ec2_monthly_cost(instance_type, self.region, self.profile)
+
+                        # Simple rightsizing heuristic - suggest one size smaller if available
+                        smaller_instance = self._suggest_smaller_instance_type(instance_type)
+                        if smaller_instance:
+                            smaller_cost = get_ec2_monthly_cost(smaller_instance, self.region, self.profile)
+                            if smaller_cost < current_cost:
+                                oversized_instances += 1
+                                potential_monthly_savings += (current_cost - smaller_cost)
+
+            return {
+                'instances_analyzed': instances_analyzed,
+                'oversized_instances': oversized_instances,
+                'potential_savings': round(potential_monthly_savings, 2),
+                'resources_analyzed': instances_analyzed,
+                'resource_impacts': []
+            }
+
+        except Exception as e:
+            print_warning(f"Could not get real EC2 data: {e}")
+            # Return minimal fallback
+            return {
+                'instances_analyzed': 0,
+                'oversized_instances': 0,
+                'potential_savings': 0.0,
+                'resources_analyzed': 0,
+                'resource_impacts': []
+            }
     
     async def analyze_ebs_optimization(self) -> Dict[str, Any]:
         """Analyze EBS volumes for optimization opportunities."""
         print_info("🔍 Analyzing EBS optimization opportunities...")
-        
-        return {
-            'volumes_analyzed': 78,
-            'unattached_volumes': 15,
-            'oversized_volumes': 8,
-            'potential_savings': 650.00,
-            'resources_analyzed': 78,
-            'resource_impacts': []
-        }
+
+        # Real AWS integration for EBS analysis
+        from runbooks.common.aws_pricing import get_ebs_gb_monthly_cost
+
+        try:
+            # Get actual EBS volumes from AWS API
+            ec2_client = self.session.client('ec2')
+            response = ec2_client.describe_volumes()
+
+            volumes_analyzed = len(response['Volumes'])
+            unattached_volumes = 0
+            oversized_volumes = 0
+            potential_monthly_savings = 0.0
+
+            for volume in response['Volumes']:
+                # Count unattached volumes
+                if volume['State'] == 'available':
+                    unattached_volumes += 1
+                    volume_size = volume['Size']
+                    volume_type = volume.get('VolumeType', 'gp3')
+                    cost_per_gb = get_ebs_gb_monthly_cost(volume_type, self.region, self.profile)
+                    potential_monthly_savings += (volume_size * cost_per_gb)
+
+                # Identify potentially oversized volumes (basic heuristic)
+                elif volume['State'] == 'in-use' and volume['Size'] > 100:
+                    oversized_volumes += 1
+
+            return {
+                'volumes_analyzed': volumes_analyzed,
+                'unattached_volumes': unattached_volumes,
+                'oversized_volumes': oversized_volumes,
+                'potential_savings': round(potential_monthly_savings, 2),
+                'resources_analyzed': volumes_analyzed,
+                'resource_impacts': []
+            }
+
+        except Exception as e:
+            print_warning(f"Could not get real EBS data: {e}")
+            return {
+                'volumes_analyzed': 0,
+                'unattached_volumes': 0,
+                'oversized_volumes': 0,
+                'potential_savings': 0.0,
+                'resources_analyzed': 0,
+                'resource_impacts': []
+            }
     
     async def analyze_unused_resources(self) -> Dict[str, Any]:
         """Analyze and identify unused AWS resources."""
         print_info("🔍 Analyzing unused resources...")
-        
-        return {
-            'eip_unused': 8,
-            'volumes_unattached': 15,
-            'snapshots_old': 23,
-            'potential_savings': 450.00,
-            'resources_analyzed': 46,
-            'resource_impacts': []
-        }
+
+        # Real AWS integration for unused resources analysis
+        from runbooks.common.aws_pricing import get_eip_monthly_cost, get_ebs_gb_monthly_cost
+
+        try:
+            ec2_client = self.session.client('ec2')
+
+            # Analyze unused Elastic IPs
+            eips_response = ec2_client.describe_addresses()
+            eip_unused = len([eip for eip in eips_response['Addresses'] if 'AssociationId' not in eip])
+
+            # Analyze unattached volumes (already calculated in EBS optimization)
+            volumes_response = ec2_client.describe_volumes()
+            volumes_unattached = len([vol for vol in volumes_response['Volumes'] if vol['State'] == 'available'])
+
+            # Analyze old snapshots (older than 30 days)
+            from datetime import datetime, timedelta
+            cutoff_date = datetime.now() - timedelta(days=30)
+            snapshots_response = ec2_client.describe_snapshots(OwnerIds=['self'])
+            snapshots_old = len([
+                snap for snap in snapshots_response['Snapshots']
+                if datetime.fromisoformat(snap['StartTime'].replace('Z', '+00:00')).replace(tzinfo=None) < cutoff_date
+            ])
+
+            # Calculate potential savings
+            eip_monthly_cost = get_eip_monthly_cost(self.region, self.profile)
+            potential_eip_savings = eip_unused * eip_monthly_cost
+
+            # Estimate EBS snapshot costs (minimal but accumulated)
+            ebs_cost_per_gb = get_ebs_gb_monthly_cost('gp3', self.region, self.profile)
+            estimated_snapshot_savings = snapshots_old * 5.0 * ebs_cost_per_gb  # Assume 5GB average per snapshot
+
+            total_potential_savings = potential_eip_savings + estimated_snapshot_savings
+
+            return {
+                'eip_unused': eip_unused,
+                'volumes_unattached': volumes_unattached,
+                'snapshots_old': snapshots_old,
+                'potential_savings': round(total_potential_savings, 2),
+                'resources_analyzed': eip_unused + volumes_unattached + snapshots_old,
+                'resource_impacts': []
+            }
+
+        except Exception as e:
+            print_warning(f"Could not get real unused resources data: {e}")
+            return {
+                'eip_unused': 0,
+                'volumes_unattached': 0,
+                'snapshots_old': 0,
+                'potential_savings': 0.0,
+                'resources_analyzed': 0,
+                'resource_impacts': []
+            }
     
     async def analyze_s3_optimization(self) -> Dict[str, Any]:
-        """Analyze S3 buckets for storage class optimization."""
+        """Analyze S3 buckets for storage class optimization using real AWS data."""
         print_info("🔍 Analyzing S3 optimization opportunities...")
-        
+
+        buckets_analyzed = 0
+        lifecycle_opportunities = 0
+        storage_class_optimization = 0
+        potential_savings = 0.0
+        resource_impacts = []
+
+        try:
+            s3_client = self.session.client('s3')
+
+            # Get all S3 buckets
+            response = s3_client.list_buckets()
+            all_buckets = response.get('Buckets', [])
+            buckets_analyzed = len(all_buckets)
+
+            print_info(f"Found {buckets_analyzed} S3 buckets for analysis")
+
+            # Analyze each bucket for optimization opportunities
+            with create_progress_bar() as progress:
+                task = progress.add_task("[cyan]Analyzing S3 buckets...", total=len(all_buckets))
+
+                for bucket in all_buckets:
+                    bucket_name = bucket['Name']
+
+                    try:
+                        # Check bucket region to create regional client
+                        bucket_region = await self._get_bucket_region(s3_client, bucket_name)
+                        regional_s3 = self.session.client('s3', region_name=bucket_region)
+
+                        # Analyze lifecycle configuration
+                        lifecycle_needed = await self._analyze_bucket_lifecycle(regional_s3, bucket_name)
+                        if lifecycle_needed:
+                            lifecycle_opportunities += 1
+
+                        # Analyze storage class optimization
+                        storage_optimization = await self._analyze_bucket_storage_classes(regional_s3, bucket_name)
+                        if storage_optimization['has_optimization_opportunity']:
+                            storage_class_optimization += 1
+                            potential_savings += storage_optimization['estimated_monthly_savings']
+
+                            # Create resource impact for this bucket
+                            resource_impacts.append(
+                                ResourceImpact(
+                                    resource_type="s3-bucket",
+                                    resource_id=bucket_name,
+                                    region=bucket_region,
+                                    account_id=self.account_id,
+                                    estimated_monthly_cost=storage_optimization['current_cost'],
+                                    projected_savings=storage_optimization['estimated_monthly_savings'],
+                                    risk_level=RiskLevel.LOW,
+                                    modification_required=True,
+                                    resource_name=f"S3 Bucket {bucket_name}",
+                                    estimated_downtime=0.0
+                                )
+                            )
+
+                        progress.advance(task)
+
+                    except Exception as e:
+                        print_warning(f"Could not analyze bucket {bucket_name}: {str(e)}")
+                        progress.advance(task)
+                        continue
+
+            print_success(f"S3 Analysis Complete:")
+            print_success(f"  • Buckets analyzed: {buckets_analyzed}")
+            print_success(f"  • Lifecycle opportunities: {lifecycle_opportunities}")
+            print_success(f"  • Storage class optimizations: {storage_class_optimization}")
+            print_success(f"  • Potential monthly savings: {format_cost(potential_savings)}")
+
+        except Exception as e:
+            print_error(f"S3 analysis failed: {str(e)}")
+            # Return zero values if analysis fails, but don't use hardcoded success data
+            buckets_analyzed = 0
+            lifecycle_opportunities = 0
+            storage_class_optimization = 0
+            potential_savings = 0.0
+
         return {
-            'buckets_analyzed': 23,
-            'lifecycle_opportunities': 18,
-            'storage_class_optimization': 12,
-            'potential_savings': 1200.00,
-            'resources_analyzed': 23,
-            'resource_impacts': []
+            'buckets_analyzed': buckets_analyzed,
+            'lifecycle_opportunities': lifecycle_opportunities,
+            'storage_class_optimization': storage_class_optimization,
+            'potential_savings': potential_savings,
+            'resources_analyzed': buckets_analyzed,
+            'resource_impacts': resource_impacts
         }
 
+    async def _get_bucket_region(self, s3_client, bucket_name: str) -> str:
+        """Get the region for a specific S3 bucket."""
+        try:
+            response = s3_client.get_bucket_location(Bucket=bucket_name)
+            region = response.get('LocationConstraint')
+
+            # Handle special case for US East 1
+            if region is None:
+                return 'us-east-1'
+
+            return region
+
+        except Exception as e:
+            print_warning(f"Could not determine region for bucket {bucket_name}: {str(e)}")
+            return 'us-east-1'  # Default fallback
+
+    async def _analyze_bucket_lifecycle(self, s3_client, bucket_name: str) -> bool:
+        """
+        Analyze if a bucket would benefit from lifecycle policies.
+
+        Returns True if lifecycle policies would provide cost savings.
+        """
+        try:
+            # Check if lifecycle configuration already exists
+            try:
+                s3_client.get_bucket_lifecycle_configuration(Bucket=bucket_name)
+                # If lifecycle exists, assume it's already optimized
+                return False
+            except ClientError as e:
+                if e.response['Error']['Code'] == 'NoSuchLifecycleConfiguration':
+                    # No lifecycle policy exists - could benefit from one
+                    pass
+                else:
+                    # Other error, skip this bucket
+                    return False
+
+            # Check bucket size and object count to determine if lifecycle is beneficial
+            try:
+                paginator = s3_client.get_paginator('list_objects_v2')
+                page_iterator = paginator.paginate(Bucket=bucket_name, PaginationConfig={'MaxItems': 100})
+
+                object_count = 0
+                total_size = 0
+
+                for page in page_iterator:
+                    if 'Contents' in page:
+                        object_count += len(page['Contents'])
+                        total_size += sum(obj.get('Size', 0) for obj in page['Contents'])
+
+                # Recommend lifecycle if bucket has significant content
+                # and could benefit from automatic transitions
+                if object_count > 50 and total_size > 1024 * 1024 * 100:  # >100MB
+                    return True
+
+            except Exception:
+                # If we can't analyze objects, be conservative
+                pass
+
+            return False
+
+        except Exception:
+            return False
+
+    async def _analyze_bucket_storage_classes(self, s3_client, bucket_name: str) -> Dict[str, Any]:
+        """
+        Analyze bucket storage classes for optimization opportunities.
+
+        Returns analysis results with optimization opportunities and cost estimates.
+        """
+        try:
+            # Get storage class analytics if available
+            paginator = s3_client.get_paginator('list_objects_v2')
+            page_iterator = paginator.paginate(Bucket=bucket_name, PaginationConfig={'MaxItems': 1000})
+
+            storage_analysis = {
+                'standard_objects': 0,
+                'standard_size': 0,
+                'infrequent_access_candidates': 0,
+                'archive_candidates': 0,
+                'current_cost': 0.0,
+                'optimized_cost': 0.0,
+                'has_optimization_opportunity': False,
+                'estimated_monthly_savings': 0.0
+            }
+
+            current_time = datetime.now()
+
+            for page in page_iterator:
+                if 'Contents' not in page:
+                    continue
+
+                for obj in page['Contents']:
+                    size_gb = obj.get('Size', 0) / (1024 * 1024 * 1024)  # Convert to GB
+                    last_modified = obj.get('LastModified', current_time)
+
+                    # Calculate age of object
+                    if hasattr(last_modified, 'replace'):
+                        age_days = (current_time - last_modified.replace(tzinfo=None)).days
+                    else:
+                        age_days = 0
+
+                    storage_class = obj.get('StorageClass', 'STANDARD')
+
+                    # Analyze optimization opportunities
+                    if storage_class == 'STANDARD':
+                        storage_analysis['standard_objects'] += 1
+                        storage_analysis['standard_size'] += size_gb
+
+                        # Current cost (Standard storage ~$0.023/GB/month)
+                        standard_cost = size_gb * 0.023
+                        storage_analysis['current_cost'] += standard_cost
+
+                        # Check if object could be moved to cheaper storage class
+                        if age_days > 30 and size_gb > 0.1:  # Objects older than 30 days and >100MB
+                            storage_analysis['infrequent_access_candidates'] += 1
+                            # IA storage ~$0.0125/GB/month
+                            ia_cost = size_gb * 0.0125
+                            storage_analysis['optimized_cost'] += ia_cost
+                        elif age_days > 90 and size_gb > 0.05:  # Objects older than 90 days
+                            storage_analysis['archive_candidates'] += 1
+                            # Glacier ~$0.004/GB/month
+                            glacier_cost = size_gb * 0.004
+                            storage_analysis['optimized_cost'] += glacier_cost
+                        else:
+                            # No optimization for this object
+                            storage_analysis['optimized_cost'] += standard_cost
+
+            # Calculate potential savings
+            potential_savings = storage_analysis['current_cost'] - storage_analysis['optimized_cost']
+
+            if potential_savings > 1.0:  # Minimum $1/month savings to be worth it
+                storage_analysis['has_optimization_opportunity'] = True
+                storage_analysis['estimated_monthly_savings'] = potential_savings
+
+            return storage_analysis
+
+        except Exception as e:
+            print_warning(f"Could not analyze storage classes for {bucket_name}: {str(e)}")
+            return {
+                'has_optimization_opportunity': False,
+                'estimated_monthly_savings': 0.0,
+                'current_cost': 0.0
+            }
+
     async def optimize_nat_gateways(
         self, 
         regions: Optional[List[str]] = None,
@@ -836,7 +1231,20 @@ class CostOptimizer(CloudOpsBase):
                 execution_timestamp=datetime.now(),
                 execution_mode=self.execution_mode,
                 success=False,
-                error_message="WorkSpaces analyzer module not found"
+                error_message="WorkSpaces analyzer module not found",
+                # Add required fields to prevent Pydantic validation errors
+                execution_time=0.0,
+                resources_analyzed=0,
+                resources_impacted=[],  # Must be a list, not an integer
+                business_metrics={
+                    "total_monthly_savings": 0.0,
+                    "overall_risk_level": "low"
+                },
+                recommendations=[],
+                aws_profile_used=self.profile or "default",
+                current_monthly_spend=0.0,
+                optimized_monthly_spend=0.0,
+                savings_percentage=0.0
             )
         
         with create_progress_bar() as progress:
@@ -891,7 +1299,18 @@ class CostOptimizer(CloudOpsBase):
                     monthly_savings=estimated_savings,
                     risk_level=RiskLevel.LOW
                 )
-            ]
+            ],
+            # Add missing required fields
+            resources_analyzed=len(unused_workspaces),
+            resources_impacted=[],  # Must be a list
+            business_metrics={
+                "total_monthly_savings": estimated_savings,
+                "overall_risk_level": "low"
+            },
+            recommendations=[],
+            aws_profile_used=self.profile or "default",
+            current_monthly_spend=0.0,
+            optimized_monthly_spend=0.0
         )
     
     async def optimize_rds_snapshots(
@@ -984,7 +1403,18 @@ class CostOptimizer(CloudOpsBase):
                     monthly_savings=estimated_monthly_savings,
                     risk_level=RiskLevel.MEDIUM
                 )
-            ]
+            ],
+            # Add missing required fields
+            resources_analyzed=len(all_manual_snapshots),
+            resources_impacted=[],  # Must be a list
+            business_metrics={
+                "total_monthly_savings": estimated_monthly_savings,
+                "overall_risk_level": "medium"
+            },
+            recommendations=[],
+            aws_profile_used=self.profile or "default",
+            current_monthly_spend=0.0,
+            optimized_monthly_spend=0.0
         )
     
     async def investigate_commvault_ec2(
@@ -1039,7 +1469,20 @@ class CostOptimizer(CloudOpsBase):
                     execution_timestamp=datetime.now(),
                     execution_mode=self.execution_mode,
                     success=False,
-                    error_message=f"Cross-account access required for {account_id}"
+                    error_message=f"Cross-account access required for {account_id}",
+                    # Add required fields to prevent Pydantic validation errors
+                    execution_time=0.0,
+                    resources_analyzed=0,
+                    resources_impacted=[],  # Must be a list
+                    business_metrics={
+                        "total_monthly_savings": 0.0,
+                        "overall_risk_level": "high"
+                    },
+                    recommendations=[],
+                    aws_profile_used=self.profile or "default",
+                    current_monthly_spend=0.0,
+                    optimized_monthly_spend=0.0,
+                    savings_percentage=0.0
                 )
             
             # Step 2: Analyze instance utilization patterns
@@ -1105,7 +1548,18 @@ class CostOptimizer(CloudOpsBase):
                     monthly_savings=potential_savings,
                     risk_level=RiskLevel.HIGH  # High risk due to potential backup disruption
                 )
-            ]
+            ],
+            # Add missing required fields
+            resources_analyzed=len(commvault_instances),
+            resources_impacted=[],  # Must be a list
+            business_metrics={
+                "total_monthly_savings": potential_savings,
+                "overall_risk_level": "high"
+            },
+            recommendations=[],
+            aws_profile_used=self.profile or "default",
+            current_monthly_spend=0.0,
+            optimized_monthly_spend=0.0
         )
     
     async def _execute_workspaces_cleanup(self, unused_workspaces: List[dict]) -> None:
@@ -1177,7 +1631,7 @@ class CostOptimizer(CloudOpsBase):
             scenario_name="Emergency Cost Spike Response",
             execution_timestamp=datetime.now(),
             execution_mode=self.execution_mode,
-            execution_time=25.0,  # Target <30 minutes
+            execution_time=self._measure_execution_time(),  # Real measured execution time
             success=True,
             error_message=None,  # Required field for CloudOpsExecutionResult base class
             resources_analyzed=100,  # Estimate for emergency scan