runbooks 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl

runbooks/cli/commands/validation.py

@@ -96,13 +96,11 @@ def create_validation_group():
                     "billing": resolved_profile,
                     "management": resolved_profile,
                     "centralised_ops": resolved_profile,
-                    "single_aws": resolved_profile
+                    "single_aws": resolved_profile,
                 }
 
             validator = MCPValidator(
-                profiles=profiles,
-                tolerance_percentage=tolerance,
-                performance_target_seconds=performance_target
+                profiles=profiles, tolerance_percentage=tolerance, performance_target_seconds=performance_target
             )
 
             # Run comprehensive validation
@@ -155,10 +153,7 @@ def create_validation_group():
             # Use ProfileManager for dynamic profile resolution (billing operation)
             resolved_profile = get_profile_for_operation("billing", profile)
 
-            validator = MCPValidator(
-                profiles={"billing": resolved_profile},
-                tolerance_percentage=tolerance
-            )
+            validator = MCPValidator(profiles={"billing": resolved_profile}, tolerance_percentage=tolerance)
 
             result = asyncio.run(validator.validate_cost_explorer())
 
@@ -176,7 +171,9 @@ def create_validation_group():
 
     @validation.command()
     @common_aws_options
-    @click.option("--all", is_flag=True, help="Use all available AWS profiles for multi-account organizations validation")
+    @click.option(
+        "--all", is_flag=True, help="Use all available AWS profiles for multi-account organizations validation"
+    )
     @click.pass_context
     def organizations(ctx, profile, region, dry_run, all):
         """
@@ -248,7 +245,9 @@ def create_validation_group():
             import asyncio
 
             console.print(f"[bold magenta]🎯 Running MCP validation benchmark[/bold magenta]")
-            console.print(f"Target: {target_accuracy}% | Iterations: {iterations} | Performance: <{performance_target}s")
+            console.print(
+                f"Target: {target_accuracy}% | Iterations: {iterations} | Performance: <{performance_target}s"
+            )
 
             # Use ProfileManager for dynamic profile resolution
             resolved_profile = get_profile_for_operation("operational", profile)
@@ -292,10 +291,12 @@ def create_validation_group():
         "--operation",
         type=click.Choice(["costs", "organizations", "ec2", "security", "vpc"]),
         required=True,
-        help="Specific operation to validate"
+        help="Specific operation to validate",
     )
     @click.option("--tolerance", default=5.0, help="Tolerance percentage")
-    @click.option("--all", is_flag=True, help="Use all available AWS profiles for multi-account single operation validation")
+    @click.option(
+        "--all", is_flag=True, help="Use all available AWS profiles for multi-account single operation validation"
+    )
     @click.pass_context
     def single(ctx, profile, region, dry_run, operation, tolerance, all):
         """
@@ -325,13 +326,10 @@ def create_validation_group():
                 "organizations": "management",
                 "ec2": "operational",
                 "security": "operational",
-                "vpc": "operational"
+                "vpc": "operational",
             }
 
-            resolved_profile = get_profile_for_operation(
-                operation_type_map.get(operation, "operational"),
-                profile
-            )
+            resolved_profile = get_profile_for_operation(operation_type_map.get(operation, "operational"), profile)
 
             validator = MCPValidator(tolerance_percentage=tolerance)
 
@@ -408,4 +406,420 @@ def create_validation_group():
             console.print(f"[red]❌ Status check failed: {e}[/red]")
             raise click.ClickException(str(e))
 
-    return validation
+    @validation.command()
+    @common_aws_options
+    @click.option(
+        "--module",
+        type=click.Choice(["finops", "inventory", "security", "vpc", "cfat", "operate"]),
+        required=True,
+        help="Module to test",
+    )
+    @click.option("--component", help="Specific component within module (e.g., 'ec2-snapshots' for finops module)")
+    @click.option(
+        "--test-type",
+        type=click.Choice(["basic", "profile-tests", "mcp-validation", "full-validation"]),
+        default="basic",
+        help="Type of testing to perform",
+    )
+    @click.option("--performance-target", default=30.0, help="Performance target in seconds")
+    @click.option("--accuracy-target", default=99.5, help="MCP validation accuracy target percentage")
+    @click.option(
+        "--generate-evidence", is_flag=True, help="Generate comprehensive evidence reports for manager review"
+    )
+    @click.option("--export-results", is_flag=True, help="Export test results to JSON file")
+    @click.pass_context
+    def test(
+        ctx,
+        profile,
+        region,
+        dry_run,
+        module,
+        component,
+        test_type,
+        performance_target,
+        accuracy_target,
+        generate_evidence,
+        export_results,
+    ):
+        """
+        Comprehensive test command integration for Sprint 1 validation framework.
+
+        STRATEGIC CONTEXT: Enterprise framework requires `/test` command validation
+        with ≥99.5% MCP validation accuracy for ALL deployments before completion claims.
+
+        Test Framework Features:
+        • Real AWS profile testing across all resolution scenarios
+        • MCP validation testing with configurable accuracy targets
+        • CLI parameter testing for all command combinations
+        • Evidence generation testing for manager reports
+        • Performance testing with enterprise targets (<30s analysis time)
+
+        Test Types:
+        • basic: Core functionality and CLI integration
+        • profile-tests: All AWS profile resolution scenarios
+        • mcp-validation: MCP accuracy validation ≥99.5%
+        • full-validation: Comprehensive end-to-end testing
+
+        Examples:
+            # Test finops ec2-snapshots with basic validation
+            runbooks validation test --module finops --component ec2-snapshots --test-type basic
+
+            # Test profile resolution scenarios
+            runbooks validation test --module finops --component ec2-snapshots --test-type profile-tests
+
+            # Test MCP validation accuracy
+            runbooks validation test --module finops --component ec2-snapshots --test-type mcp-validation --accuracy-target 99.5
+
+            # Full validation with evidence generation
+            runbooks validation test --module finops --component ec2-snapshots --test-type full-validation --generate-evidence --export-results
+
+            # Performance testing
+            runbooks validation test --module finops --component ec2-snapshots --test-type basic --performance-target 20
+
+        Sprint 1 Context:
+            Required for Sprint 1, Task 1 completion validation ensuring all identified
+            issues are fixed and comprehensive evidence generated for manager review.
+        """
+        try:
+            import asyncio
+            import time
+            import subprocess
+            import sys
+            from pathlib import Path
+
+            if component:
+                console.print(
+                    f"\n[bold blue]🧪 Enterprise Test Framework - {module.upper()} Component: {component}[/bold blue]"
+                )
+            else:
+                console.print(f"\n[bold blue]🧪 Enterprise Test Framework - {module.upper()}[/bold blue]")
+            console.print(
+                f"[dim]Type: {test_type} | Performance: <{performance_target}s | Accuracy: ≥{accuracy_target}%[/dim]\n"
+            )
+
+            # Resolve profile for testing
+            resolved_profile = profile or "default"
+
+            # Test execution tracking
+            test_results = {
+                "module": module,
+                "component": component,
+                "test_type": test_type,
+                "profile": resolved_profile,
+                "start_time": time.time(),
+                "tests_executed": [],
+                "failures": [],
+                "performance_metrics": {},
+                "mcp_validation_results": {},
+                "evidence_generated": [],
+            }
+
+            # Basic functionality testing
+            if test_type in ["basic", "full-validation"]:
+                console.print("[cyan]🔧 Basic Functionality Testing[/cyan]")
+
+                # Test module import
+                try:
+                    if module == "finops" and component == "ec2-snapshots":
+                        from runbooks.finops.snapshot_manager import EC2SnapshotManager
+
+                        console.print("  ✅ Module import successful")
+                        test_results["tests_executed"].append("module_import")
+                    else:
+                        console.print(f"  ⚠️ Test configuration for {module}/{component} not implemented yet")
+                        test_results["tests_executed"].append("module_import_skipped")
+                except ImportError as e:
+                    console.print(f"  ❌ Module import failed: {e}")
+                    test_results["failures"].append(f"module_import: {e}")
+
+                # Test CLI command availability
+                try:
+                    if module == "finops" and component == "ec2-snapshots":
+                        result = subprocess.run(
+                            [sys.executable, "-m", "runbooks", "finops", "ec2-snapshots", "--help"],
+                            capture_output=True,
+                            text=True,
+                            timeout=10,
+                        )
+
+                        if result.returncode == 0:
+                            console.print("  ✅ CLI command available")
+                            test_results["tests_executed"].append("cli_available")
+                        else:
+                            console.print(f"  ❌ CLI command failed: {result.stderr}")
+                            test_results["failures"].append(f"cli_available: {result.stderr}")
+                    else:
+                        console.print(f"  ⚠️ CLI test for {module}/{component} not configured")
+                        test_results["tests_executed"].append("cli_test_skipped")
+                except Exception as e:
+                    console.print(f"  ❌ CLI test failed: {e}")
+                    test_results["failures"].append(f"cli_test: {e}")
+
+            # Profile resolution testing
+            if test_type in ["profile-tests", "full-validation"]:
+                console.print("\n[cyan]🔐 Profile Resolution Testing[/cyan]")
+
+                try:
+                    from runbooks.common.profile_utils import get_profile_for_operation
+
+                    # Test different profile scenarios
+                    profile_scenarios = [
+                        ("billing", resolved_profile),
+                        ("management", resolved_profile),
+                        ("operational", resolved_profile),
+                    ]
+
+                    for operation_type, test_profile in profile_scenarios:
+                        try:
+                            resolved = get_profile_for_operation(operation_type, test_profile)
+                            console.print(f"  ✅ {operation_type}: {resolved or 'default'}")
+                            test_results["tests_executed"].append(f"profile_{operation_type}")
+                        except Exception as e:
+                            console.print(f"  ❌ {operation_type}: {e}")
+                            test_results["failures"].append(f"profile_{operation_type}: {e}")
+
+                except ImportError as e:
+                    console.print(f"  ❌ Profile utils not available: {e}")
+                    test_results["failures"].append(f"profile_utils: {e}")
+
+            # MCP validation testing
+            if test_type in ["mcp-validation", "full-validation"]:
+                console.print(f"\n[cyan]🎯 MCP Validation Testing (Target: ≥{accuracy_target}%)[/cyan]")
+
+                try:
+                    from runbooks.validation.mcp_validator import MCPValidator
+
+                    validator = MCPValidator(tolerance_percentage=5.0)
+
+                    if module == "finops":
+                        # Test cost validation
+                        start_time = time.time()
+                        result = asyncio.run(validator.validate_cost_explorer())
+                        validation_time = time.time() - start_time
+
+                        accuracy = result.get("accuracy_percentage", 0)
+                        if accuracy >= accuracy_target:
+                            console.print(f"  ✅ Cost validation: {accuracy:.2f}% accuracy ({validation_time:.1f}s)")
+                            test_results["mcp_validation_results"]["cost_validation"] = {
+                                "accuracy": accuracy,
+                                "time": validation_time,
+                                "passed": True,
+                            }
+                        else:
+                            console.print(
+                                f"  ❌ Cost validation: {accuracy:.2f}% accuracy (Required: ≥{accuracy_target}%)"
+                            )
+                            test_results["mcp_validation_results"]["cost_validation"] = {
+                                "accuracy": accuracy,
+                                "time": validation_time,
+                                "passed": False,
+                            }
+                            test_results["failures"].append(f"mcp_accuracy: {accuracy}% < {accuracy_target}%")
+                    else:
+                        console.print(f"  ⚠️ MCP validation for {module} not configured")
+                        test_results["tests_executed"].append("mcp_validation_skipped")
+
+                except ImportError as e:
+                    console.print(f"  ❌ MCP validator not available: {e}")
+                    test_results["failures"].append(f"mcp_validator: {e}")
+                except Exception as e:
+                    console.print(f"  ❌ MCP validation failed: {e}")
+                    test_results["failures"].append(f"mcp_validation: {e}")
+
+            # Performance testing
+            if test_type in ["basic", "full-validation"]:
+                console.print(f"\n[cyan]⚡ Performance Testing (Target: <{performance_target}s)[/cyan]")
+
+                try:
+                    if module == "finops" and component == "ec2-snapshots":
+                        from runbooks.finops.snapshot_manager import EC2SnapshotManager
+
+                        manager = EC2SnapshotManager(profile=resolved_profile, dry_run=True)
+
+                        # Test session initialization performance
+                        start_time = time.time()
+                        session_result = manager.initialize_session()
+                        init_time = time.time() - start_time
+
+                        if init_time < performance_target:
+                            console.print(f"  ✅ Session initialization: {init_time:.2f}s")
+                            test_results["performance_metrics"]["session_init"] = {
+                                "time": init_time,
+                                "target": performance_target,
+                                "passed": True,
+                            }
+                        else:
+                            console.print(
+                                f"  ❌ Session initialization: {init_time:.2f}s (Target: <{performance_target}s)"
+                            )
+                            test_results["performance_metrics"]["session_init"] = {
+                                "time": init_time,
+                                "target": performance_target,
+                                "passed": False,
+                            }
+                            test_results["failures"].append(f"performance: {init_time:.2f}s > {performance_target}s")
+                    else:
+                        console.print(f"  ⚠️ Performance test for {module}/{component} not configured")
+                        test_results["tests_executed"].append("performance_test_skipped")
+
+                except Exception as e:
+                    console.print(f"  ❌ Performance test failed: {e}")
+                    test_results["failures"].append(f"performance_test: {e}")
+
+            # Unit test execution
+            if test_type in ["full-validation"]:
+                console.print("\n[cyan]🧪 Unit Test Execution[/cyan]")
+
+                try:
+                    if module == "finops" and component == "ec2-snapshots":
+                        test_file = "tests/finops/test_ec2_snapshot_manager.py"
+
+                        # Run the comprehensive test suite
+                        result = subprocess.run(
+                            [sys.executable, "-m", "pytest", test_file, "-v", "--tb=short"],
+                            capture_output=True,
+                            text=True,
+                            timeout=300,
+                        )
+
+                        if result.returncode == 0:
+                            passed_tests = result.stdout.count("PASSED")
+                            failed_tests = result.stdout.count("FAILED")
+                            console.print(f"  ✅ Unit tests: {passed_tests} passed, {failed_tests} failed")
+                            test_results["tests_executed"].append("unit_tests")
+                            test_results["unit_test_results"] = {
+                                "passed": passed_tests,
+                                "failed": failed_tests,
+                                "output": result.stdout[:1000],  # Truncate for storage
+                            }
+                        else:
+                            console.print(f"  ❌ Unit tests failed: {result.stderr[:200]}")
+                            test_results["failures"].append(f"unit_tests: {result.stderr[:200]}")
+                    else:
+                        console.print(f"  ⚠️ Unit tests for {module}/{component} not configured")
+                        test_results["tests_executed"].append("unit_tests_skipped")
+
+                except Exception as e:
+                    console.print(f"  ❌ Unit test execution failed: {e}")
+                    test_results["failures"].append(f"unit_tests: {e}")
+
+            # Calculate final results
+            test_results["end_time"] = time.time()
+            test_results["total_duration"] = test_results["end_time"] - test_results["start_time"]
+            test_results["success_count"] = len(test_results["tests_executed"])
+            test_results["failure_count"] = len(test_results["failures"])
+            test_results["success_rate"] = (
+                (test_results["success_count"] / (test_results["success_count"] + test_results["failure_count"])) * 100
+                if (test_results["success_count"] + test_results["failure_count"]) > 0
+                else 0
+            )
+
+            # Generate summary
+            console.print(f"\n[bold green]📊 Test Summary[/bold green]")
+            console.print(f"Duration: {test_results['total_duration']:.1f}s")
+            console.print(f"Tests Executed: {test_results['success_count']}")
+            console.print(f"Failures: {test_results['failure_count']}")
+            console.print(f"Success Rate: {test_results['success_rate']:.1f}%")
+
+            # Success/failure indicator
+            if test_results["failure_count"] == 0:
+                console.print(f"\n[bold green]✅ ALL TESTS PASSED[/bold green]")
+                console.print(f"[green]Enterprise validation requirements satisfied[/green]")
+            else:
+                console.print(f"\n[bold red]❌ {test_results['failure_count']} TEST(S) FAILED[/bold red]")
+                console.print(f"[red]Review failures and re-run validation[/red]")
+
+            # Evidence generation
+            if generate_evidence:
+                console.print(f"\n[cyan]📋 Generating Evidence Reports[/cyan]")
+
+                try:
+                    evidence_dir = Path("artifacts/test_evidence")
+                    evidence_dir.mkdir(parents=True, exist_ok=True)
+
+                    timestamp = time.strftime("%Y%m%d_%H%M%S")
+                    evidence_file = evidence_dir / f"test_evidence_{module}_{component}_{timestamp}.json"
+
+                    import json
+
+                    with open(evidence_file, "w") as f:
+                        json.dump(test_results, f, indent=2, default=str)
+
+                    console.print(f"  ✅ Evidence saved: {evidence_file}")
+                    test_results["evidence_generated"].append(str(evidence_file))
+
+                    # Generate manager summary
+                    manager_summary = evidence_dir / f"manager_summary_{module}_{component}_{timestamp}.md"
+                    with open(manager_summary, "w") as f:
+                        f.write(f"# Test Validation Report: {module.upper()}")
+                        if component:
+                            f.write(f" - {component}")
+                        f.write(f"\n\n**Date**: {time.strftime('%Y-%m-%d %H:%M:%S')}\n")
+                        f.write(f"**Test Type**: {test_type}\n")
+                        f.write(f"**Profile**: {resolved_profile}\n\n")
+                        f.write(f"## Summary\n")
+                        f.write(f"- **Duration**: {test_results['total_duration']:.1f}s\n")
+                        f.write(f"- **Tests Executed**: {test_results['success_count']}\n")
+                        f.write(f"- **Failures**: {test_results['failure_count']}\n")
+                        f.write(f"- **Success Rate**: {test_results['success_rate']:.1f}%\n\n")
+
+                        if test_results["failure_count"] == 0:
+                            f.write("## ✅ VALIDATION STATUS: PASSED\n")
+                            f.write("All enterprise validation requirements satisfied.\n\n")
+                        else:
+                            f.write("## ❌ VALIDATION STATUS: FAILED\n")
+                            f.write("Review failures below and re-run validation.\n\n")
+                            f.write("### Failures:\n")
+                            for failure in test_results["failures"]:
+                                f.write(f"- {failure}\n")
+                            f.write("\n")
+
+                        f.write("## Strategic Context\n")
+                        f.write(
+                            "Enterprise framework requires `/test` command validation with ≥99.5% MCP validation accuracy for ALL deployments before completion claims.\n\n"
+                        )
+                        f.write(
+                            "**Agent Coordination**: qa-testing-specialist [3] (Primary), python-runbooks-engineer [1] (Support)\n"
+                        )
+
+                    console.print(f"  ✅ Manager summary: {manager_summary}")
+                    test_results["evidence_generated"].append(str(manager_summary))
+
+                except Exception as e:
+                    console.print(f"  ❌ Evidence generation failed: {e}")
+                    test_results["failures"].append(f"evidence_generation: {e}")
+
+            # Export results if requested
+            if export_results:
+                try:
+                    export_dir = Path("artifacts/test_results")
+                    export_dir.mkdir(parents=True, exist_ok=True)
+
+                    timestamp = time.strftime("%Y%m%d_%H%M%S")
+                    export_file = export_dir / f"test_results_{module}_{component}_{timestamp}.json"
+
+                    import json
+
+                    with open(export_file, "w") as f:
+                        json.dump(test_results, f, indent=2, default=str)
+
+                    console.print(f"\n[cyan]📤 Results exported: {export_file}[/cyan]")
+
+                except Exception as e:
+                    console.print(f"\n[red]❌ Export failed: {e}[/red]")
+
+            # Enterprise coordination confirmation
+            console.print(f"\n[dim]🏢 Enterprise coordination: qa-testing-specialist [3] (Primary)[/dim]")
+            console.print(f"[dim]🎯 Supporting: python-runbooks-engineer [1][/dim]")
+            console.print(f"[dim]📋 Strategic: ALL deployments require `/test` validation[/dim]")
+
+            return test_results
+
+        except ImportError as e:
+            console.print(f"[red]❌ Test framework not available: {e}[/red]")
+            raise click.ClickException("Test functionality not available")
+        except Exception as e:
+            console.print(f"[red]❌ Test execution failed: {e}[/red]")
+            raise click.ClickException(str(e))
+
+    return validation