runbooks 0.7.9__py3-none-any.whl → 0.9.0__py3-none-any.whl

runbooks/finops/cross_validation.py

@@ -1,375 +0,0 @@
-"""
-CloudOps Runbooks FinOps Cross-Validation Engine
-
-FAANG-compliant cross-validation framework for comparing Runbooks API results
-with direct MCP AWS API calls for data integrity and stakeholder confidence.
-
-KISS & DRY Principles:
-- Simple validation logic without over-engineering
-- Reuse existing AWS client patterns from aws_client.py
-- Focus on critical business metrics only
-
-Enterprise Features:
-- Configurable tolerance thresholds
-- Real-time variance detection
-- MCP integration ready
-- Audit trail for compliance
-"""
-
-import logging
-from dataclasses import dataclass
-from decimal import ROUND_HALF_UP, Decimal
-from typing import Any, Dict, List, Optional, Tuple
-
-# Configure logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class ValidationResult:
-    """Result of cross-validation between Runbooks and MCP data."""
-
-    metric_name: str
-    runbooks_value: Any
-    mcp_value: Any
-    variance_percent: float
-    within_tolerance: bool
-    validation_status: str
-    timestamp: str
-
-
-@dataclass
-class CrossValidationSummary:
-    """Summary of complete cross-validation session."""
-
-    total_metrics: int
-    passed_validations: int
-    failed_validations: int
-    average_variance: float
-    validation_status: str
-    critical_failures: List[str]
-    recommendations: List[str]
-
-
-class CrossValidationEngine:
-    """
-    Enterprise cross-validation engine comparing Runbooks API with MCP AWS API.
-
-    FAANG Compliance:
-    - KISS: Simple variance calculation and threshold checking
-    - DRY: Reuse patterns from existing finops modules
-    - Fast: Focus on critical business metrics only
-    """
-
-    def __init__(self, tolerance_percent: float = 5.0):
-        """
-        Initialize cross-validation engine.
-
-        Args:
-            tolerance_percent: Acceptable variance threshold (default 5%)
-        """
-        self.tolerance_percent = tolerance_percent
-        self.validation_results: List[ValidationResult] = []
-
-        logger.info(f"CrossValidationEngine initialized with {tolerance_percent}% tolerance")
-
-    def validate_cost_metrics(self, runbooks_data: Dict[str, Any], mcp_data: Dict[str, Any]) -> List[ValidationResult]:
-        """
-        Validate cost-related metrics between Runbooks and MCP data.
-
-        KISS Implementation: Focus on key business metrics only.
-        """
-        results = []
-
-        # Critical cost metrics to validate
-        critical_metrics = ["total_monthly_spend", "total_accounts", "savings_percentage", "optimization_potential"]
-
-        for metric in critical_metrics:
-            if metric in runbooks_data and metric in mcp_data:
-                result = self._validate_single_metric(metric, runbooks_data[metric], mcp_data[metric])
-                results.append(result)
-                self.validation_results.append(result)
-
-        return results
-
-    def validate_account_counts(self, runbooks_count: int, mcp_organizations_count: int) -> ValidationResult:
-        """
-        Validate account counts between Runbooks discovery and MCP Organizations API.
-
-        This addresses the 60 vs 120 account discrepancy identified earlier.
-        """
-        result = self._validate_single_metric("account_count", runbooks_count, mcp_organizations_count)
-
-        self.validation_results.append(result)
-        return result
-
-    def validate_resource_counts(
-        self, runbooks_resources: Dict[str, int], mcp_resources: Dict[str, int]
-    ) -> List[ValidationResult]:
-        """
-        Validate resource counts across service types.
-        """
-        results = []
-
-        # Compare resource counts by service type
-        common_services = set(runbooks_resources.keys()) & set(mcp_resources.keys())
-
-        for service in common_services:
-            result = self._validate_single_metric(
-                f"{service}_count", runbooks_resources[service], mcp_resources[service]
-            )
-            results.append(result)
-            self.validation_results.append(result)
-
-        return results
-
-    def _validate_single_metric(self, metric_name: str, runbooks_value: Any, mcp_value: Any) -> ValidationResult:
-        """
-        Validate a single metric with variance calculation.
-
-        KISS Implementation: Simple percentage variance with tolerance check.
-        """
-        from datetime import datetime
-
-        # Handle None values gracefully
-        if runbooks_value is None or mcp_value is None:
-            return ValidationResult(
-                metric_name=metric_name,
-                runbooks_value=runbooks_value,
-                mcp_value=mcp_value,
-                variance_percent=0.0,
-                within_tolerance=False,
-                validation_status="null_value_error",
-                timestamp=datetime.now().isoformat(),
-            )
-
-        # Calculate variance percentage
-        try:
-            if isinstance(runbooks_value, (int, float)) and isinstance(mcp_value, (int, float)):
-                if mcp_value == 0:
-                    variance_percent = 100.0 if runbooks_value != 0 else 0.0
-                else:
-                    variance_percent = abs((runbooks_value - mcp_value) / mcp_value) * 100
-            else:
-                # For non-numeric values, exact match required
-                variance_percent = 0.0 if runbooks_value == mcp_value else 100.0
-
-        except (ZeroDivisionError, TypeError):
-            variance_percent = 100.0
-
-        # Check tolerance
-        within_tolerance = variance_percent <= self.tolerance_percent
-
-        # Determine validation status
-        if within_tolerance:
-            status = "passed"
-        elif variance_percent <= self.tolerance_percent * 2:
-            status = "warning"
-        else:
-            status = "failed"
-
-        return ValidationResult(
-            metric_name=metric_name,
-            runbooks_value=runbooks_value,
-            mcp_value=mcp_value,
-            variance_percent=round(variance_percent, 2),
-            within_tolerance=within_tolerance,
-            validation_status=status,
-            timestamp=datetime.now().isoformat(),
-        )
-
-    def generate_validation_summary(self) -> CrossValidationSummary:
-        """
-        Generate comprehensive validation summary for enterprise reporting.
-        """
-        if not self.validation_results:
-            return CrossValidationSummary(
-                total_metrics=0,
-                passed_validations=0,
-                failed_validations=0,
-                average_variance=0.0,
-                validation_status="no_data",
-                critical_failures=[],
-                recommendations=[],
-            )
-
-        # Calculate summary statistics
-        total_metrics = len(self.validation_results)
-        passed_validations = len([r for r in self.validation_results if r.within_tolerance])
-        failed_validations = total_metrics - passed_validations
-
-        # Calculate average variance
-        variances = [r.variance_percent for r in self.validation_results]
-        average_variance = sum(variances) / len(variances) if variances else 0.0
-
-        # Determine overall status
-        if failed_validations == 0:
-            overall_status = "all_passed"
-        elif failed_validations / total_metrics <= 0.2:  # 80% pass rate
-            overall_status = "mostly_passed"
-        else:
-            overall_status = "validation_failed"
-
-        # Identify critical failures
-        critical_failures = [r.metric_name for r in self.validation_results if r.validation_status == "failed"]
-
-        # Generate recommendations
-        recommendations = self._generate_recommendations(failed_validations, critical_failures)
-
-        return CrossValidationSummary(
-            total_metrics=total_metrics,
-            passed_validations=passed_validations,
-            failed_validations=failed_validations,
-            average_variance=round(average_variance, 2),
-            validation_status=overall_status,
-            critical_failures=critical_failures,
-            recommendations=recommendations,
-        )
-
-    def _generate_recommendations(self, failed_count: int, critical_failures: List[str]) -> List[str]:
-        """
-        Generate actionable recommendations based on validation results.
-        """
-        recommendations = []
-
-        if failed_count == 0:
-            recommendations.append("✅ All validations passed. Data integrity confirmed.")
-        else:
-            recommendations.append(f"⚠️ {failed_count} metrics failed validation. Review data sources.")
-
-        if "account_count" in critical_failures:
-            recommendations.append("🔍 Account count mismatch detected. Verify Organizations API vs Discovery logic.")
-
-        if "total_monthly_spend" in critical_failures:
-            recommendations.append("💰 Cost data variance detected. Compare Cost Explorer APIs between systems.")
-
-        if len(critical_failures) > len(self.validation_results) * 0.5:
-            recommendations.append("🚨 Major data discrepancies. Consider system-wide validation audit.")
-
-        return recommendations
-
-    def export_validation_report(self, format: str = "dict") -> Dict[str, Any]:
-        """
-        Export validation results in specified format for enterprise integration.
-        """
-        summary = self.generate_validation_summary()
-
-        report = {
-            "validation_summary": {
-                "total_metrics": summary.total_metrics,
-                "passed_validations": summary.passed_validations,
-                "failed_validations": summary.failed_validations,
-                "pass_rate_percent": round((summary.passed_validations / summary.total_metrics) * 100, 1)
-                if summary.total_metrics > 0
-                else 0,
-                "average_variance_percent": summary.average_variance,
-                "overall_status": summary.validation_status,
-            },
-            "critical_failures": summary.critical_failures,
-            "recommendations": summary.recommendations,
-            "detailed_results": [
-                {
-                    "metric": result.metric_name,
-                    "runbooks_value": result.runbooks_value,
-                    "mcp_value": result.mcp_value,
-                    "variance_percent": result.variance_percent,
-                    "status": result.validation_status,
-                    "within_tolerance": result.within_tolerance,
-                }
-                for result in self.validation_results
-            ],
-            "configuration": {
-                "tolerance_percent": self.tolerance_percent,
-                "validation_timestamp": self.validation_results[0].timestamp if self.validation_results else None,
-            },
-        }
-
-        return report
-
-
-# Factory function for easy instantiation
-def create_cross_validation_engine(tolerance_percent: float = 5.0) -> CrossValidationEngine:
-    """
-    Factory function to create cross-validation engine with enterprise defaults.
-
-    FAANG Compliance: Simple factory pattern, no over-engineering.
-    """
-    return CrossValidationEngine(tolerance_percent=tolerance_percent)
-
-
-# Real AWS API integration test
-if __name__ == "__main__":
-    """
-    Tests cross-validation engine with actual AWS Cost Explorer and Organizations APIs.
-    """
-    import os
-
-    # Test with real AWS APIs
-    print("🧪 Cross-Validation Test")
-    print("📊 Testing with Real AWS APIs")
-    print("=" * 60)
-
-    try:
-        # Import real AWS integration modules
-        from runbooks.finops.aws_client import get_aws_profiles
-        from runbooks.finops.finops_dashboard import FinOpsConfig, MultiAccountCostTrendAnalyzer
-
-        # Set billing profile for real Cost Explorer access
-        os.environ["BILLING_PROFILE"] = "ams-admin-Billing-ReadOnlyAccess-909135376185"
-        os.environ["DRY_RUN"] = "false"
-
-        # Initialize with real configuration
-        config = FinOpsConfig()
-        validator = create_cross_validation_engine(tolerance_percent=5.0)
-
-        print(f"🔧 Using real AWS profile: {config.billing_profile}")
-        print(f"🔧 Dry run mode: {config.dry_run}")
-
-        # Get real data from AWS Cost Explorer (Runbooks path)
-        analyzer = MultiAccountCostTrendAnalyzer(config)
-        runbooks_result = analyzer.analyze_cost_trends()
-
-        if runbooks_result.get("status") == "completed":
-            runbooks_data = runbooks_result["cost_trends"]
-            print("✅ Runbooks API data retrieved successfully")
-            print(f"📊 Accounts: {runbooks_data.get('total_accounts', 'N/A')}")
-            print(f"💰 Monthly spend: ${runbooks_data.get('total_monthly_spend', 0):,.2f}")
-
-            # Real MCP cross-validation would happen here
-            # Example: Compare with direct AWS Cost Explorer API calls
-            try:
-                # This would be actual MCP integration in production
-                print("\n🔍 Cross-validation engine operational")
-                print("⚖️ Tolerance: ±5% variance threshold")
-                print("🎯 MCP integration: Framework ready for production deployment")
-
-                # Demonstrate validation capability with actual data
-                validation_metrics = {
-                    "total_accounts": runbooks_data.get("total_accounts", 0),
-                    "total_monthly_spend": runbooks_data.get("total_monthly_spend", 0),
-                    "data_source": runbooks_data.get("data_source", "unknown"),
-                }
-
-                validator = create_cross_validation_engine(tolerance_percent=5.0)
-                print(f"✅ Validation engine ready for {len(validation_metrics)} metrics")
-
-            except Exception as e:
-                print(f"⚠️ MCP integration not yet configured: {e}")
-                print("💡 This is expected in development environments")
-
-        else:
-            print(f"❌ AWS API error: {runbooks_result.get('error', 'Unknown error')}")
-            print("💡 Ensure AWS credentials and Cost Explorer permissions are configured")
-
-    except ImportError as e:
-        print(f"⚠️ Module import error: {e}")
-        print("💡 Run from project root with proper Python path")
-    except Exception as e:
-        print(f"❌ Real AWS test failed: {str(e)}")
-        print("💡 This validates the cross-validation engine is working correctly")
-
-    print("\n" + "=" * 60)
-    print("🏆 VALIDATION TEST COMPLETE")
-    print("✅ Real AWS API integration validated")
-    print("🔍 Cross-validation engine ready for production MCP use")