agent-os-kernel 1.1.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

modules/cmvk/examples/example.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""
+Simple example demonstrating the Cross-Model Verification Kernel.
+
+This shows how to use CMVK to solve a simple programming task
+with adversarial verification.
+"""
+import sys
+from pathlib import Path
+
+# Add src to path
+sys.path.insert(0, str(Path(__file__).parent / "src"))
+
+import logging
+
+from src import GeminiVerifier, OpenAIGenerator, VerificationKernel
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+def main():
+    """Run a simple example task."""
+    print("=" * 60)
+    print("Cross-Model Verification Kernel - Example")
+    print("=" * 60)
+    print()
+
+    # Define a task
+    task = """
+Write a Python function called 'fibonacci' that takes an integer n
+and returns the nth Fibonacci number. Handle edge cases for n < 0.
+Include proper error handling and docstrings.
+"""
+
+    print(f"Task: {task.strip()}")
+    print()
+
+    try:
+        # Initialize agents
+        logger.info("Initializing agents...")
+        generator = OpenAIGenerator(model_name="gpt-4o")
+        verifier = GeminiVerifier(model_name="gemini-1.5-pro")
+
+        # Create kernel
+        logger.info("Creating verification kernel...")
+        kernel = VerificationKernel(
+            generator=generator, verifier=verifier, config_path="config/settings.yaml"
+        )
+
+        # Execute verification loop
+        logger.info("Starting verification loop...")
+        print("Executing verification loop...")
+        print()
+
+        result = kernel.execute(task)
+
+        # Display results
+        print("=" * 60)
+        print("RESULTS")
+        print("=" * 60)
+        print(f"Success: {result.is_complete}")
+        print(f"Verification Loops: {result.current_loop}/{result.max_loops}")
+        print(f"Total Verifications: {len(result.verification_history)}")
+        print()
+
+        if result.final_result:
+            print("Final Solution:")
+            print("-" * 60)
+            print(result.final_result)
+            print("-" * 60)
+        else:
+            print("No solution found.")
+
+        # Display verification history
+        if result.verification_history:
+            print()
+            print("Verification History:")
+            for i, verification in enumerate(result.verification_history):
+                print(f"\nLoop {i+1}:")
+                print(f"  Outcome: {verification.outcome.value}")
+                print(f"  Confidence: {verification.confidence:.2f}")
+                if verification.critical_issues:
+                    print(f"  Critical Issues: {len(verification.critical_issues)}")
+                if verification.logic_flaws:
+                    print(f"  Logic Flaws: {len(verification.logic_flaws)}")
+
+        # Display graph stats
+        stats = kernel.get_graph_stats()
+        print()
+        print("Graph Statistics:")
+        print(f"  Total Nodes: {stats['total_nodes']}")
+        print(f"  Verified Nodes: {stats['verified_nodes']}")
+        print(f"  Failed Nodes: {stats['failed_nodes']}")
+
+    except Exception as e:
+        logger.error(f"Error: {e}")
+        print()
+        print("ERROR: Could not complete execution.")
+        print("Make sure you have set your API keys:")
+        print("  export OPENAI_API_KEY='your-key'")
+        print("  export GOOGLE_API_KEY='your-key'")
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

modules/cmvk/examples/model_diversity_comparison.py

@@ -0,0 +1,110 @@
+"""
+Example: Model Diversity Comparison
+
+Demonstrates the difference between single-model and multi-model verification,
+showing the mathematical reduction in blind spot probability.
+"""
+
+from cross_model_verification_kernel.generator import GeneratorConfig
+from cross_model_verification_kernel.kernel import VerificationKernel
+from cross_model_verification_kernel.models import ModelProvider
+from cross_model_verification_kernel.verifier import VerifierConfig
+
+
+def test_single_model_approach() -> None:
+    """Test with same model for both (NOT recommended)"""
+    print("\n" + "=" * 80)
+    print("APPROACH 1: Single Model (Self-Refinement)")
+    print("=" * 80)
+    print("Generator: GPT-4o")
+    print("Verifier: GPT-4o (SAME MODEL)")
+    print("This approach has higher blind spot probability!\n")
+
+    try:
+        generator_config = GeneratorConfig(model=ModelProvider.GPT4O)
+        verifier_config = VerifierConfig(model=ModelProvider.GPT4O)
+
+        # This should raise an error enforcing model diversity!
+        _kernel = VerificationKernel(generator_config, verifier_config)
+        print("WARNING: Should not reach here - model diversity not enforced!")
+    except ValueError as e:
+        print("✓ Model diversity enforcement working:")
+        print(f"  Error: {e}\n")
+
+
+def test_diverse_models() -> None:
+    """Test with different models (recommended)"""
+    print("\n" + "=" * 80)
+    print("APPROACH 2: Model Diversity (Adversarial Architecture)")
+    print("=" * 80)
+    print("Generator: GPT-4o")
+    print("Verifier: Gemini 1.5 Pro (DIFFERENT MODEL)")
+    print("This approach reduces blind spot probability!\n")
+
+    generator_config = GeneratorConfig(model=ModelProvider.GPT4O)
+    verifier_config = VerifierConfig(model=ModelProvider.GEMINI_15_PRO, adversarial_mode=True)
+
+    kernel = VerificationKernel(generator_config, verifier_config)
+
+    task = "Create a function to validate email addresses"
+    result = kernel.verify_task(task_description=task)
+
+    print("Verification completed!")
+    print(f"Status: {'PASSED' if result.success else 'FAILED'}")
+    print(f"\n{result.blind_spot_analysis}")
+
+
+def compare_provider_combinations() -> None:
+    """Compare different provider combinations"""
+    print("\n" + "=" * 80)
+    print("MODEL CORRELATION ANALYSIS")
+    print("=" * 80)
+
+    combinations = [
+        (ModelProvider.GPT4O, ModelProvider.GEMINI_15_PRO, "OpenAI → Google"),
+        (ModelProvider.GPT4O, ModelProvider.CLAUDE_35_SONNET, "OpenAI → Anthropic"),
+        (ModelProvider.GEMINI_15_PRO, ModelProvider.CLAUDE_35_SONNET, "Google → Anthropic"),
+        (ModelProvider.GPT4O, ModelProvider.GPT4_TURBO, "OpenAI → OpenAI (not ideal)"),
+    ]
+
+    task = "Create a secure password hashing function"
+
+    for gen_model, ver_model, desc in combinations:
+        print(f"\n{desc}:")
+        generator_config = GeneratorConfig(model=gen_model)
+        verifier_config = VerifierConfig(model=ver_model)
+
+        kernel = VerificationKernel(generator_config, verifier_config)
+        result = kernel.verify_task(task_description=task)
+
+        analysis = result.blind_spot_analysis
+        print(f"  Correlation: {analysis.correlation_coefficient:.2f}")
+        print(f"  Combined Error Probability: {analysis.combined_error_prob:.4f}")
+        print(f"  Risk Reduction: {analysis.risk_reduction_factor:.2f}x")
+
+
+def main() -> None:
+    """Run model diversity comparison"""
+    print("Cross-Model Verification Kernel - Model Diversity Analysis")
+
+    # Test 1: Show enforcement of model diversity
+    test_single_model_approach()
+
+    # Test 2: Show diverse models working
+    test_diverse_models()
+
+    # Test 3: Compare different combinations
+    compare_provider_combinations()
+
+    print("\n" + "=" * 80)
+    print("KEY TAKEAWAYS")
+    print("=" * 80)
+    print("1. Model diversity is ENFORCED - you cannot use the same model for both")
+    print("2. Different providers (e.g., GPT vs Gemini) have lower correlation")
+    print("3. Lower correlation = better blind spot reduction")
+    print("4. Adversarial verification catches issues that self-refinement misses")
+    print("=" * 80)
+
+
+if __name__ == "__main__":
+    main()

modules/cmvk/examples/real_api_integration.py

@@ -0,0 +1,121 @@
+"""
+Example: Integrating Real API Providers
+
+This example shows how to extend the system with real API providers.
+Currently uses mock implementations - replace with actual API calls.
+"""
+
+import os
+from typing import Any
+
+from cross_model_verification_kernel.models import BaseModelInterface, ModelProvider, ModelResponse
+
+
+class OpenAIInterface(BaseModelInterface):
+    """
+    Example OpenAI interface implementation.
+    Replace with actual OpenAI API calls.
+    """
+
+    def __init__(self, model: ModelProvider, api_key: str):
+        super().__init__(model, api_key)
+        # In real implementation:
+        # import openai
+        # self.client = openai.OpenAI(api_key=api_key)
+
+    def generate(self, prompt: str, **_kwargs: Any) -> ModelResponse:
+        """Generate using OpenAI API"""
+        # In real implementation:
+        # response = self.client.chat.completions.create(
+        #     model=self.model.value,
+        #     messages=[{"role": "user", "content": prompt}],
+        #     **_kwargs
+        # )
+        # content = response.choices[0].message.content
+        _ = prompt  # Mark as used - real implementation would use this
+
+        # For now, return mock response
+        return ModelResponse(
+            content="[Real OpenAI response would go here]",
+            model=self.model.value,
+            provider=self.model,
+            metadata={"api": "openai"},
+        )
+
+    def get_model_info(self) -> dict[str, str]:
+        return {"model": self.model.value, "provider": "openai", "api": "real"}
+
+
+class GeminiInterface(BaseModelInterface):
+    """
+    Example Google Gemini interface implementation.
+    Replace with actual Gemini API calls.
+    """
+
+    def __init__(self, model: ModelProvider, api_key: str):
+        super().__init__(model, api_key)
+        # In real implementation:
+        # import google.generativeai as genai
+        # genai.configure(api_key=api_key)
+        # self.model_instance = genai.GenerativeModel(model.value)
+
+    def generate(self, prompt: str, **_kwargs: Any) -> ModelResponse:
+        """Generate using Gemini API"""
+        # In real implementation:
+        # response = self.model_instance.generate_content(prompt)
+        # content = response.text
+        _ = prompt  # Mark as used - real implementation would use this
+
+        # For now, return mock response
+        return ModelResponse(
+            content="[Real Gemini response would go here]",
+            model=self.model.value,
+            provider=self.model,
+            metadata={"api": "gemini"},
+        )
+
+    def get_model_info(self) -> dict[str, str]:
+        return {"model": self.model.value, "provider": "google", "api": "real"}
+
+
+def main() -> None:
+    """
+    Example of using real API providers (when implemented).
+
+    To use:
+    1. Implement the API calls in OpenAIInterface and GeminiInterface
+    2. Set your API keys as environment variables
+    3. Pass the custom interfaces to Generator/Verifier
+    """
+
+    print("Example: Real API Integration")
+    print("=" * 80)
+    print("\nNOTE: This example shows the structure for real API integration.")
+    print("Replace mock responses with actual API calls to use real models.\n")
+
+    # Get API keys from environment (or your secure key management)
+    openai_key = os.environ.get("OPENAI_API_KEY", "your-key-here")
+    gemini_key = os.environ.get("GEMINI_API_KEY", "your-key-here")
+
+    # Create custom interfaces
+    openai_interface = OpenAIInterface(ModelProvider.GPT4O, openai_key)
+    gemini_interface = GeminiInterface(ModelProvider.GEMINI_15_PRO, gemini_key)
+
+    print("Custom interfaces created:")
+    print(f"  OpenAI: {openai_interface.get_model_info()}")
+    print(f"  Gemini: {gemini_interface.get_model_info()}")
+
+    # Note: To use custom interfaces with the kernel, you would need to:
+    # 1. Create Generator with custom interface: Generator(config, openai_interface)
+    # 2. Create Verifier with custom interface: Verifier(config, gemini_interface)
+    # 3. Create kernel with these custom instances
+
+    print("\nTo enable real API calls:")
+    print("1. Uncomment API code in OpenAIInterface and GeminiInterface")
+    print("2. Install required packages: pip install openai google-generativeai")
+    print("3. Set environment variables: OPENAI_API_KEY, GEMINI_API_KEY")
+    print("4. Run this script")
+
+
+if __name__ == "__main__":
+    main()

modules/cmvk/examples/test_full_pipeline.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+"""
+Full Pipeline Test Script
+
+This script automates Steps 1 and 3 from the project completion checklist:
+1. Run the "Sanity Check" benchmark on sample data
+2. Use the Visualizer to display the debate traces
+
+Usage:
+    python test_full_pipeline.py              # Run with sample dataset
+    python test_full_pipeline.py --full       # Run with full dataset (takes longer)
+    python test_full_pipeline.py --dataset experiments/datasets/humaneval_50.json
+"""
+import argparse
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+
+def print_header(text: str):
+    """Print a formatted header."""
+    print("\n" + "=" * 80)
+    print(text.center(80))
+    print("=" * 80 + "\n")
+
+
+def print_step(step_num: int, description: str):
+    """Print a step header."""
+    print(f"\n{'─' * 80}")
+    print(f"STEP {step_num}: {description}")
+    print("─" * 80 + "\n")
+
+
+def run_command(cmd: list, description: str) -> bool:
+    """
+    Run a command and handle its output.
+
+    Args:
+        cmd: Command to run as a list
+        description: Description of what the command does
+
+    Returns:
+        True if command succeeded, False otherwise
+    """
+    print(f"Running: {' '.join(cmd)}")
+    print(f"Purpose: {description}\n")
+
+    try:
+        result = subprocess.run(cmd, capture_output=False, text=True, check=False)
+
+        if result.returncode == 0:
+            print(f"✅ Success: {description}")
+            return True
+        else:
+            print(f"⚠️  Warning: Command exited with code {result.returncode}")
+            return False
+
+    except Exception as e:
+        print(f"❌ Error running command: {e}")
+        return False
+
+
+def check_prerequisites():
+    """Check if required files and directories exist."""
+    print_step(0, "Checking Prerequisites")
+
+    required_files = [
+        "experiments/blind_spot_benchmark.py",
+        "src/tools/visualizer.py",
+        "requirements.txt",
+    ]
+
+    all_exist = True
+    for file in required_files:
+        filepath = Path(file)
+        if filepath.exists():
+            print(f"✅ Found: {file}")
+        else:
+            print(f"❌ Missing: {file}")
+            all_exist = False
+
+    if not all_exist:
+        print("\n⚠️  Some required files are missing. Please check your installation.")
+        return False
+
+    print("\n✅ All prerequisites satisfied!")
+    return True
+
+
+def run_benchmark(dataset_path: str) -> bool:
+    """
+    Run the blind spot benchmark.
+
+    Args:
+        dataset_path: Path to the dataset file
+
+    Returns:
+        True if benchmark completed successfully
+    """
+    print_step(1, "Running Blind Spot Benchmark (Sanity Check)")
+
+    # Check if dataset exists
+    if not Path(dataset_path).exists():
+        print(f"❌ Dataset not found: {dataset_path}")
+        return False
+
+    print(f"📊 Dataset: {dataset_path}")
+
+    # Run the benchmark
+    cmd = [
+        sys.executable,
+        "experiments/blind_spot_benchmark.py",
+        "--dataset",
+        dataset_path,
+        "--output",
+        "experiments/results",
+    ]
+
+    success = run_command(cmd, "Execute benchmark comparing single-model vs CMVK")
+
+    if success:
+        # Check for results
+        results_dir = Path("experiments/results")
+        if results_dir.exists():
+            json_files = list(results_dir.glob("blind_spot_benchmark_*.json"))
+            txt_files = list(results_dir.glob("blind_spot_summary_*.txt"))
+
+            if json_files:
+                print("\n📁 Results saved to: experiments/results/")
+                print(f"   - {len(json_files)} JSON result file(s)")
+                print(f"   - {len(txt_files)} summary file(s)")
+
+                # Display latest summary
+                if txt_files:
+                    latest_summary = max(txt_files, key=lambda p: p.stat().st_mtime)
+                    print(f"\n📄 Latest Summary ({latest_summary.name}):")
+                    print("-" * 80)
+                    with open(latest_summary) as f:
+                        print(f.read())
+                    print("-" * 80)
+
+    return success
+
+
+def list_traces():
+    """List available trace files."""
+    print_step(2, "Checking for Trace Files")
+
+    traces_dir = Path("logs/traces")
+
+    if not traces_dir.exists():
+        print("⚠️  No traces directory found (logs/traces)")
+        print("Note: Traces are only generated when the kernel runs with enable_trace_logging=True")
+        return []
+
+    trace_files = list(traces_dir.glob("*.json"))
+
+    if not trace_files:
+        print("⚠️  No trace files found in logs/traces/")
+        print("Note: Traces are only generated when the kernel runs with enable_trace_logging=True")
+        return []
+
+    print(f"✅ Found {len(trace_files)} trace file(s):")
+    for i, trace_file in enumerate(sorted(trace_files), 1):
+        mtime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(trace_file.stat().st_mtime))
+        size = trace_file.stat().st_size
+        print(f"   {i}. {trace_file.name}")
+        print(f"      Modified: {mtime} | Size: {size:,} bytes")
+
+    return trace_files
+
+
+def visualize_traces(trace_files: list):
+    """
+    Visualize trace files using the visualizer.
+
+    Args:
+        trace_files: List of trace file paths
+    """
+    print_step(3, "Visualizing Traces (The Debate)")
+
+    if not trace_files:
+        print("⚠️  No traces available to visualize.")
+        print("\nTo enable trace logging, modify the benchmark to initialize the kernel with:")
+        print("    kernel = VerificationKernel(")
+        print("        generator=generator,")
+        print("        verifier=verifier,")
+        print("        enable_trace_logging=True  # Add this parameter")
+        print("    )")
+        return
+
+    # Show the latest trace
+    latest_trace = max(trace_files, key=lambda p: p.stat().st_mtime)
+    print(f"🎭 Replaying latest trace: {latest_trace.name}\n")
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "src.tools.visualizer",
+        "--latest",
+        "--speed",
+        "0.1",  # Fast playback for testing
+    ]
+
+    run_command(cmd, "Replay the adversarial debate")
+
+
+def main():
+    """Main entry point."""
+    parser = argparse.ArgumentParser(
+        description="Run the full CMVK pipeline: benchmark + visualization",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Quick test with sample dataset (5 problems)
+  python test_full_pipeline.py
+
+  # Test with 50-problem dataset (more statistical significance)
+  python test_full_pipeline.py --dataset experiments/datasets/humaneval_50.json
+
+  # Full science run (164 problems, takes 15-20 minutes)
+  python test_full_pipeline.py --full
+
+  # Just list available trace files
+  python test_full_pipeline.py --list-only
+        """,
+    )
+
+    parser.add_argument(
+        "--dataset",
+        default="experiments/datasets/humaneval_sample.json",
+        help="Path to dataset (default: humaneval_sample.json with 5 problems)",
+    )
+
+    parser.add_argument(
+        "--full", action="store_true", help="Run with full HumanEval dataset (164 problems)"
+    )
+
+    parser.add_argument(
+        "--list-only", action="store_true", help="Only list available traces, skip benchmark"
+    )
+
+    parser.add_argument("--skip-viz", action="store_true", help="Skip visualization step")
+
+    args = parser.parse_args()
+
+    # Determine dataset path
+    if args.full:
+        dataset_path = "experiments/datasets/humaneval_full.json"
+    else:
+        dataset_path = args.dataset
+
+    # Print header
+    print_header("🚀 CMVK FULL PIPELINE TEST 🚀")
+    print("This script will:")
+    print("  1. Run the Blind Spot Benchmark")
+    print("  2. Check for execution traces")
+    print("  3. Visualize the adversarial debate (if traces exist)")
+    print()
+
+    # Check prerequisites
+    if not args.list_only:
+        if not check_prerequisites():
+            sys.exit(1)
+
+    # Run benchmark
+    if not args.list_only:
+        benchmark_success = run_benchmark(dataset_path)
+
+        if not benchmark_success:
+            print("\n⚠️  Benchmark completed with warnings. Continuing to trace analysis...")
+
+    # List traces
+    trace_files = list_traces()
+
+    # Visualize traces
+    if not args.skip_viz and trace_files:
+        visualize_traces(trace_files)
+
+    # Final summary
+    print_header("✅ PIPELINE TEST COMPLETE")
+
+    if not args.list_only:
+        print("What was tested:")
+        print("  ✅ Benchmark script execution")
+        print("  ✅ Result file generation")
+        print("  ✅ Trace file detection")
+        if trace_files and not args.skip_viz:
+            print("  ✅ Visualizer playback")
+
+    print("\nNext steps:")
+    print("  1. Set up API keys (OPENAI_API_KEY, GOOGLE_API_KEY) for real runs")
+    print(
+        "  2. Run: python experiments/blind_spot_benchmark.py --dataset experiments/datasets/humaneval_full.json"
+    )
+    print("  3. Update PAPER.md with actual results")
+    print("  4. Take screenshots of visualizer output for README.md")
+    print()
+
+
+if __name__ == "__main__":
+    main()