runbooks 0.9.2__py3-none-any.whl → 0.9.5__py3-none-any.whl

runbooks/finops/embedded_mcp_validator.py

@@ -11,6 +11,7 @@ Implementation: Embedded validation eliminates external MCP server requirements
 
 import asyncio
 import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timedelta
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -36,6 +37,12 @@ class EmbeddedMCPValidator:
 
     Provides real-time cost validation without external MCP server dependencies.
     Ensures >=99.5% accuracy for enterprise financial compliance.
+    
+    Enhanced Features:
+    - Organization-level total validation
+    - Service-level cost breakdown validation
+    - Real-time variance detection with ±5% tolerance
+    - Visual indicators for validation status
     """
 
     def __init__(self, profiles: List[str], console: Optional[Console] = None):
@@ -45,6 +52,8 @@ class EmbeddedMCPValidator:
         self.aws_sessions = {}
         self.validation_threshold = 99.5  # Enterprise accuracy requirement
         self.tolerance_percent = 5.0  # ±5% tolerance for validation
+        self.validation_cache = {}  # Cache for performance optimization
+        self.cache_ttl = 300  # 5 minutes cache TTL
 
         # Initialize AWS sessions for each profile
         self._initialize_aws_sessions()
@@ -80,52 +89,87 @@ class EmbeddedMCPValidator:
             "validation_method": "embedded_mcp_direct_aws_api",
         }
 
+        # Enhanced parallel processing for <20s performance target
+        self.console.log(f"[blue]⚡ Starting parallel MCP validation with {min(5, len(self.aws_sessions))} workers[/]")
+        
         with Progress(
             SpinnerColumn(),
             TextColumn("[progress.description]{task.description}"),
             BarColumn(),
-            TaskProgressColumn(),
+            TaskProgressColumn(), 
             TimeElapsedColumn(),
             console=self.console,
         ) as progress:
-            task = progress.add_task("Validating financial data with embedded MCP...", total=len(self.aws_sessions))
-
-            for profile, session in self.aws_sessions.items():
-                try:
-                    # Get independent cost data from AWS API
-                    aws_cost_data = await self._get_independent_cost_data(session, profile)
+            task = progress.add_task("Parallel MCP validation (enhanced performance)...", total=len(self.aws_sessions))
+
+            # Parallel execution with ThreadPoolExecutor for <20s target
+            with ThreadPoolExecutor(max_workers=min(5, len(self.aws_sessions))) as executor:
+                # Submit all validation tasks
+                future_to_profile = {}
+                for profile, session in self.aws_sessions.items():
+                    future = executor.submit(self._validate_profile_sync, profile, session, runbooks_data)
+                    future_to_profile[future] = profile
+
+                # Collect results as they complete (maintain progress visibility)
+                for future in as_completed(future_to_profile):
+                    profile = future_to_profile[future]
+                    try:
+                        accuracy_result = future.result()
+                        if accuracy_result:  # Only append successful results
+                            validation_results["profile_results"].append(accuracy_result)
+                        progress.advance(task)
+                    except Exception as e:
+                        print_warning(f"Parallel validation failed for {profile[:20]}...: {str(e)[:40]}")
+                        progress.advance(task)
 
-                    # Find corresponding runbooks data
-                    runbooks_cost_data = self._extract_runbooks_cost_data(runbooks_data, profile)
+        # Calculate overall validation metrics
+        self._finalize_validation_results(validation_results)
+        return validation_results
 
-                    # Calculate accuracy
-                    accuracy_result = self._calculate_accuracy(runbooks_cost_data, aws_cost_data, profile)
-                    validation_results["profile_results"].append(accuracy_result)
+    def _validate_profile_sync(self, profile: str, session: boto3.Session, runbooks_data: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+        """Synchronous wrapper for profile validation (for parallel execution)."""
+        try:
+            # Get independent cost data from AWS API
+            aws_cost_data = asyncio.run(self._get_independent_cost_data(session, profile))
 
-                    progress.advance(task)
+            # Find corresponding runbooks data
+            runbooks_cost_data = self._extract_runbooks_cost_data(runbooks_data, profile)
 
-                except Exception as e:
-                    print_warning(f"Validation failed for {profile[:20]}...: {str(e)[:40]}")
-                    progress.advance(task)
+            # Calculate accuracy
+            accuracy_result = self._calculate_accuracy(runbooks_cost_data, aws_cost_data, profile)
+            return accuracy_result
 
-        # Calculate overall validation metrics
-        self._finalize_validation_results(validation_results)
-        return validation_results
+        except Exception as e:
+            # Return None for failed validations (handled in calling function)
+            return None
 
-    async def _get_independent_cost_data(self, session: boto3.Session, profile: str) -> Dict[str, Any]:
-        """Get independent cost data directly from AWS Cost Explorer API."""
+    async def _get_independent_cost_data(self, session: boto3.Session, profile: str, start_date_override: Optional[str] = None, end_date_override: Optional[str] = None) -> Dict[str, Any]:
+        """Get independent cost data with precise time window alignment to runbooks."""
         try:
             ce_client = session.client("ce", region_name="us-east-1")
 
-            # Calculate date range (current month)
-            end_date = datetime.now().date()
-            start_date = end_date.replace(day=1)
-
-            # Get cost and usage data (independent from runbooks)
+            # CRITICAL FIX: Use exact same time calculation as cost_processor.py
+            if start_date_override and end_date_override:
+                # Use exact time window from calling function (perfect alignment)
+                start_date = start_date_override
+                end_date = end_date_override
+                self.console.log(f"[cyan]🔍 MCP Time Window: {start_date} to {end_date} (aligned with runbooks)[/]")
+            else:
+                # EXACT MATCH: Import and use same logic as cost_processor.py get_cost_data()
+                from datetime import date, timedelta
+                today = date.today()
+                
+                # Use EXACT same logic as cost_processor.py lines 554-567
+                start_date = today.replace(day=1).isoformat()  # First day of current month
+                end_date = (today + timedelta(days=1)).isoformat()  # AWS CE end date is exclusive (today + 1)
+                
+                self.console.log(f"[cyan]📅 MCP Synchronized: {start_date} to {end_date} (matching cost_processor.py)[/]")
+
+            # Get cost and usage data (matching runbooks parameters exactly)
             response = ce_client.get_cost_and_usage(
-                TimePeriod={"Start": start_date.isoformat(), "End": end_date.isoformat()},
+                TimePeriod={"Start": start_date, "End": end_date},
                 Granularity="MONTHLY",
-                Metrics=["BlendedCost"],
+                Metrics=["UnblendedCost"],  # Match CLI using UnblendedCost not BlendedCost
                 GroupBy=[{"Type": "DIMENSION", "Key": "SERVICE"}],
             )
 
@@ -137,7 +181,7 @@ class EmbeddedMCPValidator:
                 for result in response["ResultsByTime"]:
                     for group in result.get("Groups", []):
                         service = group.get("Keys", ["Unknown"])[0]
-                        cost = float(group.get("Metrics", {}).get("BlendedCost", {}).get("Amount", 0))
+                        cost = float(group.get("Metrics", {}).get("UnblendedCost", {}).get("Amount", 0))
                         services_cost[service] = cost
                         total_cost += cost
 
@@ -159,29 +203,75 @@ class EmbeddedMCPValidator:
             }
 
     def _extract_runbooks_cost_data(self, runbooks_data: Dict[str, Any], profile: str) -> Dict[str, Any]:
-        """Extract cost data from runbooks results for comparison."""
-        # This method adapts to the actual runbooks data structure
-        # Implementation depends on the runbooks data format
-        return {
-            "profile": profile,
-            "total_cost": runbooks_data.get("total_cost", 0.0),
-            "services": runbooks_data.get("services", {}),
-            "data_source": "runbooks_finops_analysis",
-        }
+        """
+        Extract cost data from runbooks results for comparison.
+        
+        CRITICAL FIX: Handle the actual data structure from runbooks dashboard.
+        Data format: {profile_name: {total_cost: float, services: dict}}
+        """
+        try:
+            # Handle nested profile structure from single_dashboard.py
+            if profile in runbooks_data:
+                profile_data = runbooks_data[profile]
+                total_cost = profile_data.get("total_cost", 0.0)
+                services = profile_data.get("services", {})
+            else:
+                # Fallback: Look for direct keys (legacy format)
+                total_cost = runbooks_data.get("total_cost", 0.0)
+                services = runbooks_data.get("services", {})
+            
+            # Apply same NON_ANALYTICAL_SERVICES filtering as cost_processor.py
+            from .cost_processor import filter_analytical_services
+            filtered_services = filter_analytical_services(services)
+            
+            return {
+                "profile": profile,
+                "total_cost": float(total_cost),
+                "services": filtered_services,
+                "data_source": "runbooks_finops_analysis",
+                "extraction_method": "profile_nested" if profile in runbooks_data else "direct_keys"
+            }
+        except Exception as e:
+            self.console.log(f"[yellow]Warning: Error extracting runbooks data for {profile}: {str(e)}[/]")
+            return {
+                "profile": profile,
+                "total_cost": 0.0,
+                "services": {},
+                "data_source": "runbooks_finops_analysis_error",
+                "error": str(e)
+            }
 
     def _calculate_accuracy(self, runbooks_data: Dict, aws_data: Dict, profile: str) -> Dict[str, Any]:
-        """Calculate accuracy between runbooks and AWS API data."""
+        """
+        Calculate accuracy between runbooks and AWS API data.
+        
+        CRITICAL FIX: Handle zero values correctly and improve accuracy calculation.
+        """
         try:
             runbooks_cost = float(runbooks_data.get("total_cost", 0))
             aws_cost = float(aws_data.get("total_cost", 0))
 
-            if runbooks_cost > 0:
-                accuracy_percent = (1 - abs(runbooks_cost - aws_cost) / runbooks_cost) * 100
+            # CRITICAL FIX: Improved accuracy calculation for enterprise standards
+            if runbooks_cost == 0 and aws_cost == 0:
+                # Both zero - perfect accuracy
+                accuracy_percent = 100.0
+            elif runbooks_cost == 0 and aws_cost > 0:
+                # Runbooks missing cost data - major accuracy issue
+                accuracy_percent = 0.0
+                self.console.log(f"[red]⚠️  Profile {profile}: Runbooks shows $0.00 but MCP shows ${aws_cost:.2f}[/]")
+            elif aws_cost == 0 and runbooks_cost > 0:
+                # MCP missing data - moderate accuracy issue  
+                accuracy_percent = 50.0  # Give partial credit as MCP may have different data access
+                self.console.log(f"[yellow]⚠️  Profile {profile}: MCP shows $0.00 but Runbooks shows ${runbooks_cost:.2f}[/]")
             else:
-                accuracy_percent = 100.0 if aws_cost == 0 else 0.0
+                # Both have values - calculate variance-based accuracy
+                max_cost = max(runbooks_cost, aws_cost)
+                variance_percent = abs(runbooks_cost - aws_cost) / max_cost * 100
+                accuracy_percent = max(0.0, 100.0 - variance_percent)
 
-            # Determine validation status
+            # Determine validation status with enhanced thresholds
             passed = accuracy_percent >= self.validation_threshold
+            tolerance_met = abs(runbooks_cost - aws_cost) / max(max(runbooks_cost, aws_cost), 0.01) * 100 <= self.tolerance_percent
 
             return {
                 "profile": profile,
@@ -189,9 +279,11 @@ class EmbeddedMCPValidator:
                 "aws_api_cost": aws_cost,
                 "accuracy_percent": accuracy_percent,
                 "passed_validation": passed,
-                "tolerance_met": abs(runbooks_cost - aws_cost) / max(runbooks_cost, 1) * 100 <= self.tolerance_percent,
+                "tolerance_met": tolerance_met,
                 "cost_difference": abs(runbooks_cost - aws_cost),
+                "variance_percent": abs(runbooks_cost - aws_cost) / max(max(runbooks_cost, aws_cost), 0.01) * 100,
                 "validation_status": "PASSED" if passed else "FAILED",
+                "accuracy_category": self._categorize_accuracy(accuracy_percent),
             }
 
         except Exception as e:
@@ -203,6 +295,19 @@ class EmbeddedMCPValidator:
                 "validation_status": "ERROR",
             }
 
+    def _categorize_accuracy(self, accuracy_percent: float) -> str:
+        """Categorize accuracy level for reporting."""
+        if accuracy_percent >= 99.5:
+            return "EXCELLENT"
+        elif accuracy_percent >= 95.0:
+            return "GOOD"
+        elif accuracy_percent >= 90.0:
+            return "ACCEPTABLE"
+        elif accuracy_percent >= 50.0:
+            return "NEEDS_IMPROVEMENT"
+        else:
+            return "CRITICAL_ISSUE"
+
     def _finalize_validation_results(self, validation_results: Dict[str, Any]) -> None:
         """Calculate overall validation metrics and status."""
         profile_results = validation_results["profile_results"]
@@ -230,23 +335,31 @@ class EmbeddedMCPValidator:
 
         self.console.print(f"\n[bright_cyan]🔍 Embedded MCP Validation Results[/]")
 
-        # Display per-profile results
+        # Display per-profile results with enhanced detail
         for profile_result in results.get("profile_results", []):
             accuracy = profile_result.get("accuracy_percent", 0)
             status = profile_result.get("validation_status", "UNKNOWN")
             profile = profile_result.get("profile", "Unknown")
+            runbooks_cost = profile_result.get("runbooks_cost", 0)
+            aws_cost = profile_result.get("aws_api_cost", 0)
+            cost_diff = profile_result.get("cost_difference", 0)
+            category = profile_result.get("accuracy_category", "UNKNOWN")
 
-            if status == "PASSED":
+            if status == "PASSED" and accuracy >= 99.5:
                 icon = "✅"
                 color = "green"
-            elif status == "FAILED":
+            elif status == "PASSED" and accuracy >= 95.0:
+                icon = "✅"
+                color = "bright_green"
+            elif accuracy >= 50.0:
                 icon = "⚠️"
                 color = "yellow"
             else:
                 icon = "❌"
                 color = "red"
 
-            self.console.print(f"[dim]  {profile[:30]}: {icon} [{color}]{accuracy:.1f}% accuracy[/][/]")
+            self.console.print(f"[dim]  {profile[:30]}: {icon} [{color}]{accuracy:.1f}% accuracy[/] "
+                             f"[dim](Runbooks: ${runbooks_cost:.2f}, MCP: ${aws_cost:.2f}, Δ: ${cost_diff:.2f})[/][/dim]")
 
         # Overall summary
         if passed:
@@ -265,6 +378,203 @@ class EmbeddedMCPValidator:
             asyncio.set_event_loop(loop)
 
         return loop.run_until_complete(self.validate_cost_data_async(runbooks_data))
+    
+    def validate_organization_total(self, runbooks_total: float, profiles: Optional[List[str]] = None) -> Dict[str, Any]:
+        """
+        Cross-validate organization total with MCP calculation using parallel processing.
+        
+        Args:
+            runbooks_total: Total cost calculated by runbooks (e.g., $7,254.46)
+            profiles: List of profiles to validate (uses self.profiles if None)
+            
+        Returns:
+            Validation result with variance analysis
+        """
+        profiles = profiles or self.profiles
+        cache_key = f"org_total_{','.join(sorted(profiles))}"
+        
+        # Check cache first
+        if cache_key in self.validation_cache:
+            cached_time, cached_result = self.validation_cache[cache_key]
+            if time.time() - cached_time < self.cache_ttl:
+                self.console.print("[dim]Using cached MCP validation result[/dim]")
+                return cached_result
+        
+        # Calculate MCP total from AWS API using parallel processing
+        mcp_total = 0.0
+        validated_profiles = 0
+        
+        def fetch_profile_cost(profile: str) -> Tuple[str, float, bool]:
+            """Fetch cost for a single profile."""
+            if profile not in self.aws_sessions:
+                return profile, 0.0, False
+            
+            try:
+                session = self.aws_sessions[profile]
+                # Rate limiting for AWS API (max 5 calls per second)
+                time.sleep(0.2)
+                
+                # CRITICAL FIX: Use same time calculation as runbooks for organization totals
+                from datetime import date, timedelta
+                today = date.today()
+                start_date = today.replace(day=1).isoformat()
+                end_date = (today + timedelta(days=1)).isoformat()
+                
+                cost_data = asyncio.run(self._get_independent_cost_data(session, profile, start_date, end_date))
+                return profile, cost_data.get("total_cost", 0), True
+            except Exception as e:
+                print_warning(f"Skipping profile {profile[:20]}... in org validation: {str(e)[:30]}")
+                return profile, 0.0, False
+        
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TaskProgressColumn(),
+            console=self.console,
+            transient=True
+        ) as progress:
+            task = progress.add_task("Validating organization total (parallel)...", total=len(profiles))
+            
+            # Use ThreadPoolExecutor for parallel validation (max 5 workers for AWS API rate limits)
+            max_workers = min(5, len(profiles))
+            
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                # Submit all profile validations
+                future_to_profile = {
+                    executor.submit(fetch_profile_cost, profile): profile 
+                    for profile in profiles
+                }
+                
+                # Process completed validations
+                for future in as_completed(future_to_profile):
+                    profile_name, cost, success = future.result()
+                    if success:
+                        mcp_total += cost
+                        validated_profiles += 1
+                    progress.advance(task)
+        
+        # Calculate variance
+        variance = 0.0
+        if runbooks_total > 0:
+            variance = abs(runbooks_total - mcp_total) / runbooks_total * 100
+        
+        passed = variance <= self.tolerance_percent
+        
+        result = {
+            'runbooks_total': runbooks_total,
+            'mcp_total': mcp_total,
+            'variance_percent': variance,
+            'passed': passed,
+            'tolerance_percent': self.tolerance_percent,
+            'profiles_validated': validated_profiles,
+            'total_profiles': len(profiles),
+            'validation_status': 'PASSED' if passed else 'VARIANCE_DETECTED',
+            'action_required': None if passed else f'Investigate {variance:.2f}% variance',
+            'timestamp': datetime.now().isoformat()
+        }
+        
+        # Cache the result
+        self.validation_cache[cache_key] = (time.time(), result)
+        
+        # Display validation result
+        self._display_organization_validation(result)
+        
+        return result
+    
+    def validate_service_costs(self, service_breakdown: Dict[str, float], profile: Optional[str] = None, start_date: Optional[str] = None, end_date: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Cross-validate individual service costs with time window alignment.
+        
+        Args:
+            service_breakdown: Dictionary of service names to costs (e.g., {'WorkSpaces': 3869.91})
+            profile: Profile to use for validation (uses first available if None)
+            start_date: Start date for validation period (ISO format, matches runbooks)
+            end_date: End date for validation period (ISO format, matches runbooks)
+            
+        Returns:
+            Service-level validation results with time window alignment
+        """
+        profile = profile or (self.profiles[0] if self.profiles else None)
+        if not profile or profile not in self.aws_sessions:
+            return {'error': 'No valid profile for service validation'}
+        
+        session = self.aws_sessions[profile]
+        validations = {}
+        
+        # Get MCP service costs with aligned time window - CRITICAL FIX for synchronization
+        try:
+            # Ensure time window alignment with runbooks dashboard
+            mcp_data = asyncio.run(self._get_independent_cost_data(session, profile, start_date, end_date))
+            mcp_services = mcp_data.get('services', {})
+            
+            # Apply same service filtering as runbooks
+            from .cost_processor import filter_analytical_services
+            mcp_services = filter_analytical_services(mcp_services)
+            
+            # Validate each service
+            for service, runbooks_cost in service_breakdown.items():
+                if runbooks_cost > 100:  # Only validate significant costs
+                    mcp_cost = mcp_services.get(service, 0.0)
+                    
+                    variance = 0.0
+                    if runbooks_cost > 0:
+                        variance = abs(runbooks_cost - mcp_cost) / runbooks_cost * 100
+                    
+                    validations[service] = {
+                        'runbooks_cost': runbooks_cost,
+                        'mcp_cost': mcp_cost,
+                        'variance_percent': variance,
+                        'passed': variance <= self.tolerance_percent,
+                        'status': 'PASSED' if variance <= self.tolerance_percent else 'VARIANCE'
+                    }
+            
+            # Display service validation results
+            self._display_service_validation(validations)
+            
+        except Exception as e:
+            print_error(f"Service validation failed: {str(e)[:50]}")
+            return {'error': str(e)}
+        
+        return {
+            'services': validations,
+            'validated_count': len(validations),
+            'passed_count': sum(1 for v in validations.values() if v['passed']),
+            'timestamp': datetime.now().isoformat()
+        }
+    
+    def _display_organization_validation(self, result: Dict[str, Any]) -> None:
+        """Display organization total validation with visual indicators."""
+        if result['passed']:
+            self.console.print(f"\n[green]✅ Organization Total MCP Validation: PASSED[/green]")
+            self.console.print(f"[dim]   Runbooks: ${result['runbooks_total']:,.2f}[/dim]")
+            self.console.print(f"[dim]   MCP:      ${result['mcp_total']:,.2f}[/dim]")
+            self.console.print(f"[dim]   Variance: {result['variance_percent']:.2f}% (within ±{self.tolerance_percent}%)[/dim]")
+        else:
+            self.console.print(f"\n[yellow]⚠️  Organization Total MCP Variance Detected[/yellow]")
+            self.console.print(f"[yellow]   Runbooks: ${result['runbooks_total']:,.2f}[/yellow]")
+            self.console.print(f"[yellow]   MCP:      ${result['mcp_total']:,.2f}[/yellow]")
+            self.console.print(f"[yellow]   Variance: {result['variance_percent']:.2f}% (exceeds ±{self.tolerance_percent}%)[/yellow]")
+            self.console.print(f"[dim yellow]   Action: {result['action_required']}[/dim yellow]")
+    
+    def _display_service_validation(self, validations: Dict[str, Dict]) -> None:
+        """Display service-level validation results."""
+        if validations:
+            self.console.print("\n[bright_cyan]Service-Level MCP Validation:[/bright_cyan]")
+            
+            for service, validation in validations.items():
+                if validation['passed']:
+                    icon = "✅"
+                    color = "green"
+                else:
+                    icon = "⚠️"
+                    color = "yellow"
+                
+                self.console.print(
+                    f"[dim]  {service:20s}: {icon} [{color}]"
+                    f"${validation['runbooks_cost']:,.2f} vs ${validation['mcp_cost']:,.2f} "
+                    f"({validation['variance_percent']:.1f}% variance)[/][/dim]"
+                )
 
 
 def create_embedded_mcp_validator(profiles: List[str], console: Optional[Console] = None) -> EmbeddedMCPValidator: