runbooks 0.7.9__py3-none-any.whl → 0.9.1__py3-none-any.whl

runbooks/finops/embedded_mcp_validator.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+"""
+Embedded MCP Validator - Internal AWS API Validation for Enterprise Accuracy
+
+This module provides self-contained MCP-style validation without external dependencies.
+Direct AWS API integration ensures >=99.5% financial accuracy for enterprise compliance.
+
+User Innovation: "MCP inside runbooks API may be a good feature"
+Implementation: Embedded validation eliminates external MCP server requirements
+"""
+
+import asyncio
+import time
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional, Tuple
+
+import boto3
+from rich.console import Console
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn, TimeElapsedColumn
+
+from ..common.rich_utils import (
+    console as rich_console,
+)
+from ..common.rich_utils import (
+    format_cost,
+    print_error,
+    print_info,
+    print_success,
+    print_warning,
+)
+
+
+class EmbeddedMCPValidator:
+    """
+    Internal MCP-style validator with direct AWS API integration.
+
+    Provides real-time cost validation without external MCP server dependencies.
+    Ensures >=99.5% accuracy for enterprise financial compliance.
+    """
+
+    def __init__(self, profiles: List[str], console: Optional[Console] = None):
+        """Initialize embedded MCP validator with AWS profiles."""
+        self.profiles = profiles
+        self.console = console or rich_console
+        self.aws_sessions = {}
+        self.validation_threshold = 99.5  # Enterprise accuracy requirement
+        self.tolerance_percent = 5.0  # ±5% tolerance for validation
+
+        # Initialize AWS sessions for each profile
+        self._initialize_aws_sessions()
+
+    def _initialize_aws_sessions(self) -> None:
+        """Initialize AWS sessions for all profiles with error handling."""
+        for profile in self.profiles:
+            try:
+                session = boto3.Session(profile_name=profile)
+                # Test session validity
+                session.client("sts").get_caller_identity()
+                self.aws_sessions[profile] = session
+                print_info(f"MCP session initialized for profile: {profile[:30]}...")
+            except Exception as e:
+                print_warning(f"MCP session failed for {profile[:20]}...: {str(e)[:30]}")
+
+    async def validate_cost_data_async(self, runbooks_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Asynchronously validate runbooks cost data against direct AWS API calls.
+
+        Args:
+            runbooks_data: Cost data from runbooks FinOps analysis
+
+        Returns:
+            Validation results with accuracy metrics
+        """
+        validation_results = {
+            "validation_timestamp": datetime.now().isoformat(),
+            "profiles_validated": 0,
+            "total_accuracy": 0.0,
+            "passed_validation": False,
+            "profile_results": [],
+            "validation_method": "embedded_mcp_direct_aws_api",
+        }
+
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TaskProgressColumn(),
+            TimeElapsedColumn(),
+            console=self.console,
+        ) as progress:
+            task = progress.add_task("Validating financial data with embedded MCP...", total=len(self.aws_sessions))
+
+            for profile, session in self.aws_sessions.items():
+                try:
+                    # Get independent cost data from AWS API
+                    aws_cost_data = await self._get_independent_cost_data(session, profile)
+
+                    # Find corresponding runbooks data
+                    runbooks_cost_data = self._extract_runbooks_cost_data(runbooks_data, profile)
+
+                    # Calculate accuracy
+                    accuracy_result = self._calculate_accuracy(runbooks_cost_data, aws_cost_data, profile)
+                    validation_results["profile_results"].append(accuracy_result)
+
+                    progress.advance(task)
+
+                except Exception as e:
+                    print_warning(f"Validation failed for {profile[:20]}...: {str(e)[:40]}")
+                    progress.advance(task)
+
+        # Calculate overall validation metrics
+        self._finalize_validation_results(validation_results)
+        return validation_results
+
+    async def _get_independent_cost_data(self, session: boto3.Session, profile: str) -> Dict[str, Any]:
+        """Get independent cost data directly from AWS Cost Explorer API."""
+        try:
+            ce_client = session.client("ce", region_name="us-east-1")
+
+            # Calculate date range (current month)
+            end_date = datetime.now().date()
+            start_date = end_date.replace(day=1)
+
+            # Get cost and usage data (independent from runbooks)
+            response = ce_client.get_cost_and_usage(
+                TimePeriod={"Start": start_date.isoformat(), "End": end_date.isoformat()},
+                Granularity="MONTHLY",
+                Metrics=["BlendedCost"],
+                GroupBy=[{"Type": "DIMENSION", "Key": "SERVICE"}],
+            )
+
+            # Process AWS response into comparable format
+            total_cost = 0.0
+            services_cost = {}
+
+            if response.get("ResultsByTime"):
+                for result in response["ResultsByTime"]:
+                    for group in result.get("Groups", []):
+                        service = group.get("Keys", ["Unknown"])[0]
+                        cost = float(group.get("Metrics", {}).get("BlendedCost", {}).get("Amount", 0))
+                        services_cost[service] = cost
+                        total_cost += cost
+
+            return {
+                "profile": profile,
+                "total_cost": total_cost,
+                "services": services_cost,
+                "data_source": "direct_aws_cost_explorer",
+                "timestamp": datetime.now().isoformat(),
+            }
+
+        except Exception as e:
+            return {
+                "profile": profile,
+                "error": str(e),
+                "total_cost": 0.0,
+                "services": {},
+                "data_source": "error_fallback",
+            }
+
+    def _extract_runbooks_cost_data(self, runbooks_data: Dict[str, Any], profile: str) -> Dict[str, Any]:
+        """Extract cost data from runbooks results for comparison."""
+        # This method adapts to the actual runbooks data structure
+        # Implementation depends on the runbooks data format
+        return {
+            "profile": profile,
+            "total_cost": runbooks_data.get("total_cost", 0.0),
+            "services": runbooks_data.get("services", {}),
+            "data_source": "runbooks_finops_analysis",
+        }
+
+    def _calculate_accuracy(self, runbooks_data: Dict, aws_data: Dict, profile: str) -> Dict[str, Any]:
+        """Calculate accuracy between runbooks and AWS API data."""
+        try:
+            runbooks_cost = float(runbooks_data.get("total_cost", 0))
+            aws_cost = float(aws_data.get("total_cost", 0))
+
+            if runbooks_cost > 0:
+                accuracy_percent = (1 - abs(runbooks_cost - aws_cost) / runbooks_cost) * 100
+            else:
+                accuracy_percent = 100.0 if aws_cost == 0 else 0.0
+
+            # Determine validation status
+            passed = accuracy_percent >= self.validation_threshold
+
+            return {
+                "profile": profile,
+                "runbooks_cost": runbooks_cost,
+                "aws_api_cost": aws_cost,
+                "accuracy_percent": accuracy_percent,
+                "passed_validation": passed,
+                "tolerance_met": abs(runbooks_cost - aws_cost) / max(runbooks_cost, 1) * 100 <= self.tolerance_percent,
+                "cost_difference": abs(runbooks_cost - aws_cost),
+                "validation_status": "PASSED" if passed else "FAILED",
+            }
+
+        except Exception as e:
+            return {
+                "profile": profile,
+                "accuracy_percent": 0.0,
+                "passed_validation": False,
+                "error": str(e),
+                "validation_status": "ERROR",
+            }
+
+    def _finalize_validation_results(self, validation_results: Dict[str, Any]) -> None:
+        """Calculate overall validation metrics and status."""
+        profile_results = validation_results["profile_results"]
+
+        if not profile_results:
+            validation_results["total_accuracy"] = 0.0
+            validation_results["passed_validation"] = False
+            return
+
+        # Calculate overall accuracy
+        valid_results = [r for r in profile_results if r.get("accuracy_percent", 0) > 0]
+        if valid_results:
+            total_accuracy = sum(r["accuracy_percent"] for r in valid_results) / len(valid_results)
+            validation_results["total_accuracy"] = total_accuracy
+            validation_results["profiles_validated"] = len(valid_results)
+            validation_results["passed_validation"] = total_accuracy >= self.validation_threshold
+
+        # Display results
+        self._display_validation_results(validation_results)
+
+    def _display_validation_results(self, results: Dict[str, Any]) -> None:
+        """Display validation results with Rich CLI formatting."""
+        overall_accuracy = results.get("total_accuracy", 0)
+        passed = results.get("passed_validation", False)
+
+        self.console.print(f"\n[bright_cyan]🔍 Embedded MCP Validation Results[/]")
+
+        # Display per-profile results
+        for profile_result in results.get("profile_results", []):
+            accuracy = profile_result.get("accuracy_percent", 0)
+            status = profile_result.get("validation_status", "UNKNOWN")
+            profile = profile_result.get("profile", "Unknown")
+
+            if status == "PASSED":
+                icon = "✅"
+                color = "green"
+            elif status == "FAILED":
+                icon = "⚠️"
+                color = "yellow"
+            else:
+                icon = "❌"
+                color = "red"
+
+            self.console.print(f"[dim]  {profile[:30]}: {icon} [{color}]{accuracy:.1f}% accuracy[/][/]")
+
+        # Overall summary
+        if passed:
+            print_success(f"✅ MCP Validation PASSED: {overall_accuracy:.1f}% accuracy achieved")
+            print_info(f"Enterprise compliance: {results['profiles_validated']} profiles validated")
+        else:
+            print_warning(f"⚠️ MCP Validation: {overall_accuracy:.1f}% accuracy (≥99.5% required)")
+            print_info("Consider reviewing data sources for accuracy improvements")
+
+    def validate_cost_data(self, runbooks_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Synchronous wrapper for async validation."""
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+        return loop.run_until_complete(self.validate_cost_data_async(runbooks_data))
+
+
+def create_embedded_mcp_validator(profiles: List[str], console: Optional[Console] = None) -> EmbeddedMCPValidator:
+    """Factory function to create embedded MCP validator."""
+    return EmbeddedMCPValidator(profiles=profiles, console=console)
+
+
+# Integration with existing FinOps dashboard
+def validate_finops_results_with_embedded_mcp(profiles: List[str], runbooks_results: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Convenience function to validate FinOps results with embedded MCP.
+
+    Args:
+        profiles: List of AWS profiles to validate
+        runbooks_results: Results from runbooks FinOps analysis
+
+    Returns:
+        Validation results with accuracy metrics
+    """
+    validator = create_embedded_mcp_validator(profiles)
+    return validator.validate_cost_data(runbooks_results)

runbooks/finops/enhanced_dashboard_runner.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 """
 Enhanced FinOps Dashboard Runner
-Migrated and optimized from README/aws-finops-dashboard
 
 This module provides enterprise-grade FinOps dashboard capabilities including:
 - Multi-profile AWS cost analysis with Rich console formatting
@@ -29,16 +28,17 @@ from rich.table import Column, Table
 from rich.tree import Tree
 
 from ..common.rich_utils import get_console
-from .finops_dashboard import FinOpsConfig
+
+# FinOpsConfig dependency removed - using simple dict configuration instead
 
 console = Console()
 
 
 class EnhancedFinOpsDashboard:
-    """Enhanced FinOps Dashboard with production-tested capabilities from aws-finops-dashboard"""
+    """Enhanced FinOps Dashboard with production-tested capabilities from runbooks finops"""
 
-    def __init__(self, config: Optional[FinOpsConfig] = None):
-        self.config = config or FinOpsConfig()
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        self.config = config or {}
         self.console = Console()
         self.rich_console = self.console  # Use the console instance directly
 
@@ -202,8 +202,9 @@ class EnhancedFinOpsDashboard:
             unused_volumes = volumes_response["Volumes"]
             region_data["unused_volumes"] = len(unused_volumes)
 
-            # Calculate EBS volume costs (rough estimate: $0.10/GB/month)
-            volume_savings = sum(vol["Size"] * 0.10 for vol in unused_volumes)
+            # Note: EBS cost calculation requires real AWS Cost Explorer pricing data
+            # Hardcoded pricing estimates removed per compliance requirements
+            volume_savings = 0  # Cannot calculate without real AWS pricing API
             region_data["potential_savings"] += volume_savings
 
             if unused_volumes:

runbooks/finops/enhanced_progress.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env python3
+"""
+Enhanced Progress Bar Implementation - Smooth Progress Tracking
+
+This module provides enhanced progress bar implementations that address the
+0%→100% jump issue by providing meaningful incremental progress updates
+during AWS API operations and data processing.
+
+Features:
+- Smooth incremental progress updates
+- Real-time operation tracking
+- Context-aware progress estimation
+- Rich CLI integration with beautiful progress bars
+- Performance monitoring and timing
+- Operation-specific progress patterns
+
+Author: CloudOps Runbooks Team
+Version: 0.8.0
+"""
+
+import time
+from contextlib import contextmanager
+from typing import Any, Callable, Dict, Iterator, List, Optional, Union
+
+from rich.console import Console
+from rich.progress import (
+    BarColumn,
+    Progress,
+    SpinnerColumn,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+)
+
+from ..common.rich_utils import console as rich_console
+
+
+class EnhancedProgressTracker:
+    """
+    Enhanced progress tracking with smooth incremental updates.
+
+    Provides context-aware progress estimation for AWS operations
+    and prevents jarring 0%→100% jumps by breaking operations into
+    meaningful sub-steps with realistic timing.
+    """
+
+    def __init__(self, console: Optional[Console] = None):
+        self.console = console or rich_console
+        self.operation_timing = {
+            "aws_cost_data": {"steps": 5, "estimated_seconds": 8},
+            "budget_analysis": {"steps": 3, "estimated_seconds": 4},
+            "service_analysis": {"steps": 4, "estimated_seconds": 6},
+            "multi_account_analysis": {"steps": 6, "estimated_seconds": 12},
+            "resource_discovery": {"steps": 8, "estimated_seconds": 15},
+        }
+
+    @contextmanager
+    def create_enhanced_progress(
+        self, operation_type: str = "default", total_items: Optional[int] = None
+    ) -> Iterator["ProgressContext"]:
+        """
+        Create enhanced progress context with smooth incremental updates.
+
+        Args:
+            operation_type: Type of operation for timing estimation
+            total_items: Total number of items to process (for accurate progress)
+
+        Yields:
+            ProgressContext: Enhanced progress context manager
+        """
+        timing_info = self.operation_timing.get(operation_type, {"steps": 5, "estimated_seconds": 8})
+
+        progress = Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(complete_style="bright_green", finished_style="bright_green"),
+            TaskProgressColumn(),
+            TimeElapsedColumn(),
+            TimeRemainingColumn(),
+            console=self.console,
+            transient=False,
+        )
+
+        with progress:
+            context = ProgressContext(progress, timing_info, total_items)
+            yield context
+
+    def create_multi_stage_progress(self, stages: List[Dict[str, Any]]) -> "MultiStageProgress":
+        """
+        Create multi-stage progress for complex operations.
+
+        Args:
+            stages: List of stage definitions with names and estimated durations
+
+        Returns:
+            MultiStageProgress: Multi-stage progress manager
+        """
+        return MultiStageProgress(self.console, stages)
+
+
+class ProgressContext:
+    """
+    Enhanced progress context with smooth incremental updates.
+
+    Provides methods for updating progress with realistic timing
+    and prevents jarring progress jumps.
+    """
+
+    def __init__(self, progress: Progress, timing_info: Dict[str, Any], total_items: Optional[int] = None):
+        self.progress = progress
+        self.timing_info = timing_info
+        self.total_items = total_items or 100
+        self.current_step = 0
+        self.max_steps = timing_info["steps"]
+        self.estimated_seconds = timing_info["estimated_seconds"]
+        self.step_duration = self.estimated_seconds / self.max_steps
+        self.task_id = None
+
+    def start_operation(self, description: str) -> None:
+        """Start the operation with initial progress."""
+        self.task_id = self.progress.add_task(description, total=self.total_items)
+        self.current_step = 0
+
+    def update_step(self, step_name: str, increment: Optional[int] = None) -> None:
+        """
+        Update progress to next step with smooth incremental updates.
+
+        Args:
+            step_name: Name of the current step
+            increment: Optional specific increment amount
+        """
+        if self.task_id is None:
+            return
+
+        self.current_step += 1
+
+        # Calculate target progress based on current step
+        target_progress = (self.current_step / self.max_steps) * self.total_items
+
+        if increment:
+            target_progress = min(self.total_items, increment)
+
+        # Update with smooth incremental steps
+        current_progress = self.progress.tasks[self.task_id].completed
+        steps_needed = max(1, int((target_progress - current_progress) / 5))  # Break into 5 increments
+        increment_size = (target_progress - current_progress) / steps_needed
+
+        for i in range(steps_needed):
+            new_progress = current_progress + (increment_size * (i + 1))
+            self.progress.update(self.task_id, completed=min(self.total_items, new_progress), description=step_name)
+            # Small delay for smooth visual effect
+            time.sleep(0.1)
+
+    def complete_operation(self, final_message: str = "Operation completed") -> None:
+        """Complete the operation with 100% progress."""
+        if self.task_id is not None:
+            self.progress.update(self.task_id, completed=self.total_items, description=final_message)
+
+
+class MultiStageProgress:
+    """
+    Multi-stage progress manager for complex operations.
+
+    Manages multiple progress bars for operations with distinct phases,
+    providing clear visual feedback for each stage of processing.
+    """
+
+    def __init__(self, console: Console, stages: List[Dict[str, Any]]):
+        self.console = console
+        self.stages = stages
+        self.current_stage = 0
+        self.progress = None
+        self.active_tasks = {}
+
+    def __enter__(self) -> "MultiStageProgress":
+        self.progress = Progress(
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(complete_style="bright_green"),
+            TaskProgressColumn(),
+            TimeElapsedColumn(),
+            console=self.console,
+            transient=False,
+        )
+
+        self.progress.__enter__()
+
+        # Initialize all stage tasks
+        for i, stage in enumerate(self.stages):
+            task_id = self.progress.add_task(
+                stage["name"],
+                total=stage.get("total", 100),
+                visible=(i == 0),  # Only show first stage initially
+            )
+            self.active_tasks[i] = task_id
+
+        return self
+
+    def __exit__(self, *args) -> None:
+        if self.progress:
+            self.progress.__exit__(*args)
+
+    def advance_stage(self, stage_index: int, progress_amount: int, description: Optional[str] = None) -> None:
+        """Advance progress for a specific stage."""
+        if stage_index in self.active_tasks and self.progress:
+            task_id = self.active_tasks[stage_index]
+
+            # Make current stage visible if not already
+            if not self.progress.tasks[task_id].visible:
+                self.progress.update(task_id, visible=True)
+
+            # Update progress
+            update_kwargs = {"advance": progress_amount}
+            if description:
+                update_kwargs["description"] = description
+
+            self.progress.update(task_id, **update_kwargs)
+
+    def complete_stage(self, stage_index: int) -> None:
+        """Mark a stage as completed."""
+        if stage_index in self.active_tasks and self.progress:
+            task_id = self.active_tasks[stage_index]
+            stage_total = self.progress.tasks[task_id].total or 100
+            self.progress.update(task_id, completed=stage_total)
+
+    def next_stage(self) -> bool:
+        """Move to the next stage and return True if successful."""
+        if self.current_stage < len(self.stages) - 1:
+            self.current_stage += 1
+
+            # Mark current stage as visible
+            if self.current_stage in self.active_tasks and self.progress:
+                task_id = self.active_tasks[self.current_stage]
+                self.progress.update(task_id, visible=True)
+
+            return True
+        return False
+
+
+# Convenience functions for common progress patterns
+
+
+def track_aws_cost_analysis(items: List[Any], console: Optional[Console] = None) -> Iterator[Any]:
+    """
+    Track AWS cost analysis with enhanced progress.
+
+    Args:
+        items: Items to process
+        console: Optional console instance
+
+    Yields:
+        Items with progress tracking
+    """
+    tracker = EnhancedProgressTracker(console)
+
+    with tracker.create_enhanced_progress("aws_cost_data", len(items)) as progress:
+        progress.start_operation("Analyzing AWS cost data...")
+
+        for i, item in enumerate(items):
+            progress.update_step(f"Processing item {i + 1}/{len(items)}", i + 1)
+            yield item
+
+        progress.complete_operation("Cost analysis completed")
+
+
+def track_multi_account_analysis(accounts: List[str], console: Optional[Console] = None) -> Iterator[str]:
+    """
+    Track multi-account analysis with enhanced progress.
+
+    Args:
+        accounts: Account profiles to process
+        console: Optional console instance
+
+    Yields:
+        Account profiles with progress tracking
+    """
+    tracker = EnhancedProgressTracker(console)
+
+    with tracker.create_enhanced_progress("multi_account_analysis", len(accounts)) as progress:
+        progress.start_operation("Analyzing multiple accounts...")
+
+        for i, account in enumerate(accounts):
+            progress.update_step(f"Analyzing account: {account}", i + 1)
+            yield account
+
+        progress.complete_operation("Multi-account analysis completed")
+
+
+@contextmanager
+def enhanced_finops_progress(
+    operation_name: str, total_steps: int = 100, console: Optional[Console] = None
+) -> Iterator[Callable[[int, str], None]]:
+    """
+    Context manager for enhanced FinOps operations progress.
+
+    Args:
+        operation_name: Name of the operation
+        total_steps: Total number of progress steps
+        console: Optional console instance
+
+    Yields:
+        Progress update function: (step, description) -> None
+    """
+    console = console or rich_console
+
+    progress = Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(complete_style="bright_green", finished_style="bright_green"),
+        TaskProgressColumn(),
+        TimeElapsedColumn(),
+        console=console,
+        transient=False,
+    )
+
+    with progress:
+        task_id = progress.add_task(operation_name, total=total_steps)
+
+        def update_progress(step: int, description: str) -> None:
+            progress.update(task_id, completed=step, description=description)
+
+        yield update_progress
+
+
+def create_progress_tracker(console: Optional[Console] = None) -> EnhancedProgressTracker:
+    """Factory function to create enhanced progress tracker."""
+    return EnhancedProgressTracker(console=console)