runbooks 0.7.9__py3-none-any.whl → 0.9.1__py3-none-any.whl

runbooks/operate/s3_operations.py

@@ -59,6 +59,10 @@ class S3Operations(BaseOperation):
         "set_public_access_block",
         "get_public_access_block",
         "sync_objects",
+        "find_buckets_without_lifecycle",
+        "add_lifecycle_policy_bulk",
+        "get_bucket_lifecycle",
+        "analyze_lifecycle_compliance",
     }
     requires_confirmation = True
 
@@ -174,6 +178,14 @@ class S3Operations(BaseOperation):
             return self.get_public_access_block(context, kwargs.get("account_id"))
         elif operation_type == "sync_objects":
             return self.sync_objects(context, **kwargs)
+        elif operation_type == "find_buckets_without_lifecycle":
+            return self.find_buckets_without_lifecycle(context, **kwargs)
+        elif operation_type == "add_lifecycle_policy_bulk":
+            return self.add_lifecycle_policy_bulk(context, **kwargs)
+        elif operation_type == "get_bucket_lifecycle":
+            return self.get_bucket_lifecycle(context, **kwargs)
+        elif operation_type == "analyze_lifecycle_compliance":
+            return self.analyze_lifecycle_compliance(context, **kwargs)
         else:
             raise ValueError(f"Unsupported operation: {operation_type}")
 
@@ -723,6 +735,502 @@ class S3Operations(BaseOperation):
 
         return [result]
 
+    def find_buckets_without_lifecycle(
+        self, 
+        context: OperationContext, 
+        region: Optional[str] = None, 
+        bucket_names: Optional[List[str]] = None
+    ) -> List[OperationResult]:
+        """
+        Find S3 buckets without lifecycle policies.
+        
+        Enhanced from unSkript notebook: AWS_Add_Lifecycle_Policy_To_S3_Buckets.ipynb
+        Identifies buckets that do not have any configured lifecycle rules for 
+        managing object lifecycle, valuable for optimizing storage costs.
+
+        Args:
+            context: Operation context
+            region: AWS region to search (if None, searches all regions)
+            bucket_names: Specific bucket names to check (if None, checks all buckets)
+
+        Returns:
+            List of operation results with buckets without lifecycle policies
+        """
+        result = self.create_operation_result(
+            context, "find_buckets_without_lifecycle", "s3:bucket", "lifecycle-audit"
+        )
+
+        try:
+            console.print(f"[bold blue]🔍 Scanning for S3 buckets without lifecycle policies...[/bold blue]")
+            
+            buckets_without_policy = []
+            
+            # Get list of regions to search
+            search_regions = []
+            if region:
+                search_regions = [region]
+            elif bucket_names and region:
+                search_regions = [region]
+            else:
+                # Get all regions if not specified - use boto3 to get all regions
+                try:
+                    ec2_client = self.get_client("ec2", "us-east-1")  # Use us-east-1 to get all regions
+                    regions_response = self.execute_aws_call(ec2_client, "describe_regions")
+                    search_regions = [r["RegionName"] for r in regions_response.get("Regions", [])]
+                except Exception:
+                    # Fallback to common regions
+                    search_regions = ["us-east-1", "us-west-2", "eu-west-1", "ap-southeast-1"]
+
+            # Process each region
+            for reg in search_regions:
+                try:
+                    s3_client = self.get_client("s3", reg)
+                    
+                    # Get buckets to check
+                    if bucket_names:
+                        # Check specific buckets
+                        buckets_to_check = bucket_names
+                    else:
+                        # Get all buckets in region
+                        response = self.execute_aws_call(s3_client, "list_buckets")
+                        buckets_to_check = [bucket["Name"] for bucket in response.get("Buckets", [])]
+
+                    # Check each bucket for lifecycle policies
+                    for bucket_name in buckets_to_check:
+                        try:
+                            # Get bucket location to ensure it's in the current region
+                            try:
+                                bucket_location = self.execute_aws_call(s3_client, "get_bucket_location", Bucket=bucket_name)
+                                bucket_region = bucket_location.get("LocationConstraint")
+                                
+                                # us-east-1 returns None for LocationConstraint
+                                if bucket_region is None:
+                                    bucket_region = "us-east-1"
+                                
+                                # Skip if bucket is not in current region (when checking all regions)
+                                if not bucket_names and bucket_region != reg:
+                                    continue
+                                    
+                            except ClientError as e:
+                                if e.response["Error"]["Code"] in ["NoSuchBucket", "AccessDenied"]:
+                                    continue
+                                raise
+
+                            # Check for lifecycle configuration
+                            try:
+                                lifecycle_response = self.execute_aws_call(
+                                    s3_client, "get_bucket_lifecycle_configuration", Bucket=bucket_name
+                                )
+                                
+                                # If we get here, bucket has lifecycle rules
+                                rules = lifecycle_response.get("Rules", [])
+                                if not rules:
+                                    # Empty rules list means no active lifecycle
+                                    buckets_without_policy.append({
+                                        "bucket_name": bucket_name,
+                                        "region": bucket_region,
+                                        "issue": "Empty lifecycle configuration"
+                                    })
+                                    
+                            except ClientError as e:
+                                if e.response["Error"]["Code"] == "NoSuchLifecycleConfiguration":
+                                    # No lifecycle configuration found
+                                    buckets_without_policy.append({
+                                        "bucket_name": bucket_name,
+                                        "region": bucket_region,
+                                        "issue": "No lifecycle configuration"
+                                    })
+                                else:
+                                    logger.warning(f"Could not check lifecycle for bucket {bucket_name}: {e}")
+
+                        except Exception as bucket_error:
+                            logger.warning(f"Error checking bucket {bucket_name}: {bucket_error}")
+                            continue
+
+                except Exception as region_error:
+                    logger.warning(f"Error processing region {reg}: {region_error}")
+                    continue
+
+            # Format results with Rich console output
+            if buckets_without_policy:
+                console.print(f"[bold yellow]⚠️  Found {len(buckets_without_policy)} bucket(s) without lifecycle policies:[/bold yellow]")
+                
+                from rich.table import Table
+                table = Table(show_header=True, header_style="bold magenta")
+                table.add_column("Bucket Name", style="cyan")
+                table.add_column("Region", style="green")
+                table.add_column("Issue", style="yellow")
+                
+                for bucket_info in buckets_without_policy:
+                    table.add_row(
+                        bucket_info["bucket_name"],
+                        bucket_info["region"], 
+                        bucket_info["issue"]
+                    )
+                
+                console.print(table)
+                
+                result.response_data = {
+                    "buckets_without_lifecycle": buckets_without_policy,
+                    "total_count": len(buckets_without_policy),
+                    "regions_scanned": search_regions
+                }
+                result.mark_completed(OperationStatus.SUCCESS)
+            else:
+                console.print("[bold green]✅ All buckets have lifecycle policies configured![/bold green]")
+                result.response_data = {
+                    "buckets_without_lifecycle": [],
+                    "total_count": 0,
+                    "regions_scanned": search_regions,
+                    "message": "All buckets have lifecycle policies"
+                }
+                result.mark_completed(OperationStatus.SUCCESS)
+
+        except Exception as e:
+            error_msg = f"Failed to scan for buckets without lifecycle policies: {e}"
+            logger.error(error_msg)
+            result.mark_completed(OperationStatus.FAILED, error_msg)
+
+        return [result]
+
+    def add_lifecycle_policy_bulk(
+        self,
+        context: OperationContext,
+        bucket_list: List[Dict[str, str]],
+        expiration_days: int = 30,
+        prefix: str = "",
+        noncurrent_days: int = 30,
+        transition_ia_days: Optional[int] = None,
+        transition_glacier_days: Optional[int] = None,
+    ) -> List[OperationResult]:
+        """
+        Add lifecycle policies to multiple S3 buckets in bulk.
+        
+        Enhanced from unSkript notebook: AWS_Add_Lifecycle_Policy_To_S3_Buckets.ipynb
+        Applies optimized lifecycle configuration for cost management.
+
+        Args:
+            context: Operation context
+            bucket_list: List of dictionaries with 'bucket_name' and 'region' keys
+            expiration_days: Days after which objects expire
+            prefix: Object prefix filter for lifecycle rule
+            noncurrent_days: Days before noncurrent versions are deleted
+            transition_ia_days: Days before transition to IA storage class
+            transition_glacier_days: Days before transition to Glacier
+
+        Returns:
+            List of operation results for each bucket processed
+        """
+        results = []
+        
+        console.print(f"[bold blue]📋 Adding lifecycle policies to {len(bucket_list)} bucket(s)...[/bold blue]")
+        
+        if context.dry_run:
+            console.print("[yellow]🧪 DRY-RUN MODE: No actual changes will be made[/yellow]")
+
+        for i, bucket_info in enumerate(bucket_list, 1):
+            bucket_name = bucket_info.get("bucket_name")
+            bucket_region = bucket_info.get("region")
+            
+            if not bucket_name or not bucket_region:
+                logger.error(f"Invalid bucket info: {bucket_info}")
+                continue
+                
+            console.print(f"[cyan]({i}/{len(bucket_list)}) Processing bucket: {bucket_name}[/cyan]")
+
+            result = self.create_operation_result(
+                context, "add_lifecycle_policy", "s3:bucket", bucket_name
+            )
+
+            try:
+                s3_client = self.get_client("s3", bucket_region)
+
+                # Build lifecycle configuration
+                lifecycle_rules = []
+                
+                # Main lifecycle rule
+                rule = {
+                    "ID": f"lifecycle-rule-{int(datetime.now().timestamp())}",
+                    "Status": "Enabled",
+                    "Filter": {"Prefix": prefix} if prefix else {},
+                    "Expiration": {"Days": expiration_days},
+                }
+
+                # Add noncurrent version expiration
+                if noncurrent_days:
+                    rule["NoncurrentVersionExpiration"] = {"NoncurrentDays": noncurrent_days}
+
+                # Add storage class transitions
+                transitions = []
+                if transition_ia_days and transition_ia_days < expiration_days:
+                    transitions.append({
+                        "Days": transition_ia_days,
+                        "StorageClass": "STANDARD_IA"
+                    })
+                
+                if transition_glacier_days and transition_glacier_days < expiration_days:
+                    transitions.append({
+                        "Days": transition_glacier_days,
+                        "StorageClass": "GLACIER"
+                    })
+                
+                if transitions:
+                    rule["Transitions"] = transitions
+
+                lifecycle_rules.append(rule)
+
+                lifecycle_config = {"Rules": lifecycle_rules}
+
+                if context.dry_run:
+                    console.print(f"[yellow]  [DRY-RUN] Would apply lifecycle policy to {bucket_name}[/yellow]")
+                    result.response_data = {
+                        "bucket_name": bucket_name,
+                        "region": bucket_region,
+                        "lifecycle_config": lifecycle_config,
+                        "dry_run": True
+                    }
+                    result.mark_completed(OperationStatus.DRY_RUN)
+                else:
+                    # Apply lifecycle configuration
+                    response = self.execute_aws_call(
+                        s3_client,
+                        "put_bucket_lifecycle_configuration",
+                        Bucket=bucket_name,
+                        LifecycleConfiguration=lifecycle_config
+                    )
+
+                    console.print(f"[green]  ✅ Successfully applied lifecycle policy to {bucket_name}[/green]")
+                    
+                    result.response_data = {
+                        "bucket_name": bucket_name,
+                        "region": bucket_region,
+                        "lifecycle_config": lifecycle_config,
+                        "aws_response": response
+                    }
+                    result.mark_completed(OperationStatus.SUCCESS)
+
+            except ClientError as e:
+                error_msg = f"Failed to set lifecycle policy on {bucket_name}: {e}"
+                console.print(f"[red]  ❌ {error_msg}[/red]")
+                logger.error(error_msg)
+                result.mark_completed(OperationStatus.FAILED, error_msg)
+            except Exception as e:
+                error_msg = f"Unexpected error for bucket {bucket_name}: {e}"
+                console.print(f"[red]  ❌ {error_msg}[/red]")
+                logger.error(error_msg)
+                result.mark_completed(OperationStatus.FAILED, error_msg)
+
+            results.append(result)
+
+        # Summary report
+        successful = len([r for r in results if r.success])
+        failed = len(results) - successful
+        
+        if context.dry_run:
+            console.print(f"[yellow]🧪 DRY-RUN SUMMARY: Would process {len(results)} bucket(s)[/yellow]")
+        else:
+            console.print(f"[bold blue]📊 BULK OPERATION SUMMARY:[/bold blue]")
+            console.print(f"[green]  ✅ Successful: {successful}[/green]")
+            if failed > 0:
+                console.print(f"[red]  ❌ Failed: {failed}[/red]")
+
+        return results
+
+    def get_bucket_lifecycle(
+        self, context: OperationContext, bucket_name: str
+    ) -> List[OperationResult]:
+        """
+        Get current lifecycle configuration for an S3 bucket.
+
+        Args:
+            context: Operation context
+            bucket_name: Name of bucket to check
+
+        Returns:
+            List of operation results with current lifecycle configuration
+        """
+        s3_client = self.get_client("s3")
+        
+        result = self.create_operation_result(
+            context, "get_bucket_lifecycle", "s3:bucket", bucket_name
+        )
+
+        try:
+            console.print(f"[blue]🔍 Checking lifecycle configuration for bucket: {bucket_name}[/blue]")
+            
+            try:
+                response = self.execute_aws_call(
+                    s3_client, "get_bucket_lifecycle_configuration", Bucket=bucket_name
+                )
+                
+                rules = response.get("Rules", [])
+                console.print(f"[green]✅ Found {len(rules)} lifecycle rule(s) for bucket {bucket_name}[/green]")
+                
+                # Display rules in a formatted table
+                if rules:
+                    from rich.table import Table
+                    table = Table(show_header=True, header_style="bold magenta")
+                    table.add_column("Rule ID", style="cyan")
+                    table.add_column("Status", style="green")
+                    table.add_column("Prefix", style="yellow")
+                    table.add_column("Expiration", style="red")
+                    
+                    for rule in rules:
+                        rule_id = rule.get("ID", "N/A")
+                        status = rule.get("Status", "N/A")
+                        
+                        # Handle different filter formats
+                        filter_info = rule.get("Filter", {})
+                        if isinstance(filter_info, dict):
+                            prefix = filter_info.get("Prefix", "")
+                        else:
+                            prefix = ""
+                            
+                        expiration = rule.get("Expiration", {})
+                        exp_days = expiration.get("Days", "N/A")
+                        
+                        table.add_row(rule_id, status, prefix or "All objects", str(exp_days))
+                    
+                    console.print(table)
+                
+                result.response_data = {
+                    "bucket_name": bucket_name,
+                    "lifecycle_rules": rules,
+                    "rules_count": len(rules)
+                }
+                result.mark_completed(OperationStatus.SUCCESS)
+                
+            except ClientError as e:
+                if e.response["Error"]["Code"] == "NoSuchLifecycleConfiguration":
+                    console.print(f"[yellow]⚠️  No lifecycle configuration found for bucket {bucket_name}[/yellow]")
+                    result.response_data = {
+                        "bucket_name": bucket_name,
+                        "lifecycle_rules": [],
+                        "rules_count": 0,
+                        "message": "No lifecycle configuration"
+                    }
+                    result.mark_completed(OperationStatus.SUCCESS)
+                else:
+                    raise e
+
+        except ClientError as e:
+            error_msg = f"Failed to get lifecycle configuration for {bucket_name}: {e}"
+            logger.error(error_msg)
+            result.mark_completed(OperationStatus.FAILED, error_msg)
+
+        return [result]
+
+    def analyze_lifecycle_compliance(
+        self, context: OperationContext, region: Optional[str] = None
+    ) -> List[OperationResult]:
+        """
+        Analyze lifecycle compliance across S3 buckets and provide cost optimization recommendations.
+        
+        Args:
+            context: Operation context  
+            region: AWS region to analyze (if None, analyzes all regions)
+
+        Returns:
+            List of operation results with compliance analysis and recommendations
+        """
+        result = self.create_operation_result(
+            context, "analyze_lifecycle_compliance", "s3:account", "compliance-analysis"
+        )
+
+        try:
+            console.print("[bold blue]📊 Analyzing S3 lifecycle compliance and cost optimization opportunities...[/bold blue]")
+            
+            # Find buckets without lifecycle policies
+            find_results = self.find_buckets_without_lifecycle(context, region=region)
+            
+            if not find_results or not find_results[0].success:
+                result.mark_completed(OperationStatus.FAILED, "Failed to analyze buckets")
+                return [result]
+                
+            buckets_data = find_results[0].response_data
+            non_compliant_buckets = buckets_data.get("buckets_without_lifecycle", [])
+            total_buckets_scanned = buckets_data.get("total_count", 0)
+            
+            # Calculate compliance metrics
+            s3_client = self.get_client("s3")
+            
+            # Get total bucket count for compliance percentage
+            list_response = self.execute_aws_call(s3_client, "list_buckets")
+            total_buckets = len(list_response.get("Buckets", []))
+            
+            compliance_percentage = ((total_buckets - len(non_compliant_buckets)) / total_buckets * 100) if total_buckets > 0 else 100
+            
+            # Generate recommendations
+            recommendations = []
+            potential_savings = 0
+            
+            if non_compliant_buckets:
+                recommendations.extend([
+                    "🎯 Implement lifecycle policies to automatically transition objects to cheaper storage classes",
+                    "💰 Configure automatic deletion of old object versions to reduce storage costs",
+                    "📈 Set up transitions: Standard → IA (30 days) → Glacier (90 days) → Deep Archive (365 days)",
+                    "🔧 Use prefixes to apply different policies to different object types",
+                    "📊 Monitor lifecycle rule effectiveness with CloudWatch metrics"
+                ])
+                
+                # Estimate potential savings (rough calculation)
+                estimated_savings_per_bucket = 25  # Estimated 25% savings per bucket
+                potential_savings = len(non_compliant_buckets) * estimated_savings_per_bucket
+
+            # Create summary report
+            from rich.panel import Panel
+            from rich.table import Table
+            
+            # Compliance summary table
+            summary_table = Table(show_header=True, header_style="bold magenta")
+            summary_table.add_column("Metric", style="cyan")
+            summary_table.add_column("Value", style="green")
+            
+            summary_table.add_row("Total Buckets", str(total_buckets))
+            summary_table.add_row("Compliant Buckets", str(total_buckets - len(non_compliant_buckets)))
+            summary_table.add_row("Non-Compliant Buckets", str(len(non_compliant_buckets)))
+            summary_table.add_row("Compliance Percentage", f"{compliance_percentage:.1f}%")
+            summary_table.add_row("Potential Cost Savings", f"~{potential_savings}%")
+            
+            console.print(Panel(summary_table, title="S3 Lifecycle Compliance Report", border_style="blue"))
+            
+            # Display recommendations if any
+            if recommendations:
+                console.print("\n[bold yellow]💡 Cost Optimization Recommendations:[/bold yellow]")
+                for i, rec in enumerate(recommendations, 1):
+                    console.print(f"  {i}. {rec}")
+            
+            # Compliance status color coding
+            if compliance_percentage >= 90:
+                compliance_status = "[bold green]EXCELLENT[/bold green]"
+            elif compliance_percentage >= 75:
+                compliance_status = "[bold yellow]GOOD[/bold yellow]"
+            elif compliance_percentage >= 50:
+                compliance_status = "[bold orange]NEEDS IMPROVEMENT[/bold orange]"
+            else:
+                compliance_status = "[bold red]POOR[/bold red]"
+                
+            console.print(f"\n[bold blue]Overall Compliance Status: {compliance_status}[/bold blue]")
+            
+            result.response_data = {
+                "compliance_percentage": compliance_percentage,
+                "total_buckets": total_buckets,
+                "compliant_buckets": total_buckets - len(non_compliant_buckets),
+                "non_compliant_buckets": len(non_compliant_buckets),
+                "non_compliant_details": non_compliant_buckets,
+                "recommendations": recommendations,
+                "potential_savings_percentage": potential_savings,
+                "compliance_status": compliance_status.replace("[bold ", "").replace("[/bold ", "").replace("]", "")
+            }
+            result.mark_completed(OperationStatus.SUCCESS)
+
+        except Exception as e:
+            error_msg = f"Failed to analyze lifecycle compliance: {e}"
+            logger.error(error_msg)
+            result.mark_completed(OperationStatus.FAILED, error_msg)
+
+        return [result]
+
     def delete_bucket_and_objects(self, context: OperationContext, bucket_name: str) -> List[OperationResult]:
         """
         Delete S3 bucket and all its objects/versions (complete cleanup).

runbooks/operate/vpc_endpoints.py

@@ -29,7 +29,6 @@ from typing import Any, Dict, List, Optional, Tuple
 import boto3
 from botocore.exceptions import BotoCoreError, ClientError
 
-from runbooks.operate.base import BaseOperation, OperationResult
 from runbooks.common.rich_utils import (
     console,
     create_panel,
@@ -39,6 +38,7 @@ from runbooks.common.rich_utils import (
     print_status,
     print_success,
 )
+from runbooks.operate.base import BaseOperation, OperationResult
 
 logger = logging.getLogger(__name__)