agent-os-kernel 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

modules/cmvk/src/cross_model_verification_kernel/datasets/humaneval_loader.py

@@ -0,0 +1,271 @@
+"""
+HumanEval Dataset Loader
+
+This module provides utilities for loading and processing the HumanEval dataset,
+the industry standard benchmark for code generation models.
+
+The HumanEval dataset contains 164 hand-written programming problems with function
+signatures, docstrings, and unit tests. It's widely used to evaluate code generation
+capabilities of language models.
+
+Reference: "Evaluating Large Language Models Trained on Code" (Chen et al., 2021)
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+class HumanEvalLoader:
+    """
+    Loader for the HumanEval dataset.
+
+    This class handles loading and formatting HumanEval problems for use
+    with the Cross-Model Verification Kernel.
+    """
+
+    def __init__(self, dataset_path: str | None = None):
+        """
+        Initialize the HumanEval loader.
+
+        Args:
+            dataset_path: Path to the HumanEval JSON file. If None, uses the
+                         sample dataset in experiments/datasets/humaneval_sample.json
+        """
+        if dataset_path is None:
+            # Use default sample dataset
+            # Go up from datasets -> cross_model_verification_kernel -> src -> repo_root
+            base_dir = Path(__file__).parent.parent.parent.parent
+            dataset_path = base_dir / "experiments" / "datasets" / "humaneval_sample.json"
+
+        self.dataset_path = Path(dataset_path)
+        self.problems = []
+        self._load_dataset()
+
+    def _load_dataset(self):
+        """Load the dataset from JSON file."""
+        if not self.dataset_path.exists():
+            raise FileNotFoundError(
+                f"HumanEval dataset not found at {self.dataset_path}. "
+                "Please provide a valid path or ensure the sample dataset exists."
+            )
+
+        with open(self.dataset_path, encoding="utf-8") as f:
+            self.problems = json.load(f)
+
+        print(f"✅ Loaded {len(self.problems)} problems from HumanEval dataset")
+
+    def get_all_problems(self) -> list[dict[str, Any]]:
+        """
+        Get all problems from the dataset.
+
+        Returns:
+            List of problem dictionaries
+        """
+        return self.problems
+
+    def get_problem(self, task_id: str) -> dict[str, Any] | None:
+        """
+        Get a specific problem by task ID.
+
+        Args:
+            task_id: The task ID (e.g., "HumanEval/0")
+
+        Returns:
+            Problem dictionary or None if not found
+        """
+        for problem in self.problems:
+            if problem.get("task_id") == task_id:
+                return problem
+        return None
+
+    def get_problem_by_index(self, index: int) -> dict[str, Any] | None:
+        """
+        Get a problem by its index in the dataset.
+
+        Args:
+            index: Zero-based index
+
+        Returns:
+            Problem dictionary or None if index out of range
+        """
+        if 0 <= index < len(self.problems):
+            return self.problems[index]
+        return None
+
+    def format_for_kernel(self, problem: dict[str, Any]) -> dict[str, Any]:
+        """
+        Format a HumanEval problem for use with the Verification Kernel.
+
+        The kernel expects problems in a specific format with 'id' and 'query' keys.
+
+        Args:
+            problem: Raw HumanEval problem dictionary
+
+        Returns:
+            Formatted problem dictionary with 'id', 'query', and metadata
+        """
+        task_id = problem.get("task_id", "unknown")
+        prompt = problem.get("prompt", "")
+        test = problem.get("test", "")
+        entry_point = problem.get("entry_point", "")
+
+        # Create a detailed query that includes the function signature and docstring
+        query = (
+            f"Complete the following Python function:\n\n"
+            f"{prompt}\n\n"
+            f"Requirements:\n"
+            f"- The function must pass all provided test cases\n"
+            f"- Follow the exact function signature provided\n"
+            f"- Entry point: {entry_point}"
+        )
+
+        return {
+            "id": task_id.replace("/", "_"),  # Make filesystem-safe
+            "query": query,
+            "metadata": {
+                "task_id": task_id,
+                "entry_point": entry_point,
+                "test_code": test,
+                "original_prompt": prompt,
+            },
+        }
+
+    def get_problem_subset(self, start: int = 0, count: int = 10) -> list[dict[str, Any]]:
+        """
+        Get a subset of problems from the dataset.
+
+        This is useful for running experiments on a smaller scale before
+        scaling up to the full dataset.
+
+        Args:
+            start: Starting index (default: 0)
+            count: Number of problems to return (default: 10)
+
+        Returns:
+            List of problem dictionaries
+        """
+        end = min(start + count, len(self.problems))
+        return self.problems[start:end]
+
+    def format_all_for_kernel(
+        self, start: int = 0, count: int | None = None
+    ) -> list[dict[str, Any]]:
+        """
+        Format multiple problems for the kernel.
+
+        Args:
+            start: Starting index (default: 0)
+            count: Number of problems to format (default: all remaining)
+
+        Returns:
+            List of formatted problem dictionaries
+        """
+        if count is None:
+            count = len(self.problems) - start
+
+        subset = self.get_problem_subset(start, count)
+        return [self.format_for_kernel(p) for p in subset]
+
+    def __len__(self) -> int:
+        """Return the number of problems in the dataset."""
+        return len(self.problems)
+
+    def __iter__(self):
+        """Iterate over all problems."""
+        return iter(self.problems)
+
+
+def download_full_humaneval(output_path: str = "experiments/datasets/humaneval_full.jsonl"):
+    """
+    Download the full HumanEval dataset from the official source.
+
+    Note: This requires internet access and the 'requests' library.
+    The full dataset is available at:
+    https://github.com/openai/human-eval
+
+    Args:
+        output_path: Where to save the downloaded dataset
+    """
+    try:
+        import requests
+    except ImportError:
+        print("❌ Error: 'requests' library is required to download the dataset.")
+        print("Install it with: pip install requests")
+        return None
+
+    url = "https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz"
+
+    print(f"Downloading HumanEval dataset from {url}...")
+
+    try:
+        import gzip
+
+        response = requests.get(url)
+        response.raise_for_status()
+
+        # Decompress and save
+        import io
+
+        compressed_file = io.BytesIO(response.content)
+
+        problems = []
+        with gzip.open(compressed_file, "rt", encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    problems.append(json.loads(line))
+
+        # Save as regular JSON for easier handling
+        output_path = Path(output_path)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(output_path.with_suffix(".json"), "w", encoding="utf-8") as f:
+            json.dump(problems, f, indent=2)
+
+        print(f"✅ Downloaded {len(problems)} problems to {output_path.with_suffix('.json')}")
+        return str(output_path.with_suffix(".json"))
+
+    except Exception as e:
+        print(f"❌ Failed to download HumanEval dataset: {e}")
+        print("You can manually download it from: https://github.com/openai/human-eval")
+        return None
+
+
+if __name__ == "__main__":
+    # Example usage
+    print("=" * 80)
+    print("HumanEval Dataset Loader - Demo")
+    print("=" * 80)
+
+    # Load the sample dataset
+    loader = HumanEvalLoader()
+
+    print("\n📊 Dataset Statistics:")
+    print(f"   Total problems: {len(loader)}")
+
+    # Get first problem
+    print("\n📝 Example Problem:")
+    problem = loader.get_problem_by_index(0)
+    if problem:
+        print(f"   Task ID: {problem['task_id']}")
+        print(f"   Entry Point: {problem['entry_point']}")
+        print(f"   Prompt Preview: {problem['prompt'][:100]}...")
+
+    # Format for kernel
+    print("\n🔧 Formatted for Kernel:")
+    formatted = loader.format_for_kernel(problem)
+    print(f"   ID: {formatted['id']}")
+    print(f"   Query (first 150 chars): {formatted['query'][:150]}...")
+
+    # Get a subset
+    print("\n📦 Getting subset (5 problems):")
+    subset = loader.format_all_for_kernel(start=0, count=5)
+    for i, p in enumerate(subset, 1):
+        print(f"   {i}. {p['id']}")
+
+    print("\n" + "=" * 80)
+    print("To download the full HumanEval dataset:")
+    print(
+        "  python -c 'from cross_model_verification_kernel.datasets.humaneval_loader import download_full_humaneval; download_full_humaneval()'"
+    )
+    print("=" * 80)

modules/cmvk/src/cross_model_verification_kernel/generator.py

@@ -0,0 +1,118 @@
+"""
+Generator Module
+
+Implements the Generator component that creates code using one model.
+The Generator is intentionally decoupled from the Verifier to enable adversarial verification.
+"""
+
+from dataclasses import dataclass
+from typing import Any
+
+from .models import BaseModelInterface, MockModelInterface, ModelProvider
+
+
+@dataclass
+class GeneratorConfig:
+    """Configuration for the Generator"""
+
+    model: ModelProvider
+    temperature: float = 0.7
+    max_tokens: int = 2000
+    api_key: str | None = None
+    custom_instructions: str | None = None
+
+
+@dataclass
+class GeneratedCode:
+    """Container for generated code"""
+
+    code: str
+    language: str
+    description: str
+    model_used: str
+    metadata: dict[str, Any] | None = None
+
+
+class Generator:
+    """
+    Generator component that creates code using a specified model.
+
+    This component is intentionally decoupled from verification to enable
+    adversarial testing with different models.
+    """
+
+    def __init__(self, config: GeneratorConfig, model_interface: BaseModelInterface | None = None):
+        """
+        Initialize the Generator
+
+        Args:
+            config: Generator configuration
+            model_interface: Optional custom model interface (uses mock if not provided)
+        """
+        self.config = config
+        self.model_interface = model_interface or MockModelInterface(
+            model=config.model, api_key=config.api_key
+        )
+        self.generation_count = 0
+
+    def generate_code(
+        self, task_description: str, language: str = "python", **kwargs
+    ) -> GeneratedCode:
+        """
+        Generate code based on a task description
+
+        Args:
+            task_description: Description of what the code should do
+            language: Programming language (default: python)
+            **kwargs: Additional generation parameters
+
+        Returns:
+            GeneratedCode object containing the generated code and metadata
+        """
+        self.generation_count += 1
+
+        # Build the prompt for code generation
+        prompt = self._build_generation_prompt(task_description, language)
+
+        # Add custom instructions if provided
+        if self.config.custom_instructions:
+            prompt = f"{self.config.custom_instructions}\n\n{prompt}"
+
+        # Generate using the model
+        response = self.model_interface.generate(
+            prompt, temperature=self.config.temperature, max_tokens=self.config.max_tokens, **kwargs
+        )
+
+        return GeneratedCode(
+            code=response.content,
+            language=language,
+            description=task_description,
+            model_used=response.model,
+            metadata={
+                "generation_count": self.generation_count,
+                "provider": response.provider.value,
+                "response_metadata": response.metadata,
+            },
+        )
+
+    def _build_generation_prompt(self, task_description: str, language: str) -> str:
+        """Build the prompt for code generation"""
+        return f"""Generate {language} code for the following task:
+
+Task: {task_description}
+
+Requirements:
+- Write clean, functional code
+- Include necessary imports
+- Focus on correctness and readability
+
+Generate the code:"""
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get generator statistics"""
+        return {
+            "model": self.config.model.value,
+            "generation_count": self.generation_count,
+            "temperature": self.config.temperature,
+            "max_tokens": self.config.max_tokens,
+        }

modules/cmvk/src/cross_model_verification_kernel/kernel.py

@@ -0,0 +1,292 @@
+"""
+Verification Kernel
+
+The main orchestrator for the adversarial architecture.
+Coordinates Generator and Verifier to provide model-diverse verification.
+"""
+
+from dataclasses import dataclass
+from typing import Any
+
+from .generator import GeneratedCode, Generator, GeneratorConfig
+from .models import ModelProvider
+from .verifier import VerificationReport, Verifier, VerifierConfig
+
+
+@dataclass
+class BlindSpotAnalysis:
+    """Mathematical analysis of blind spot reduction through model diversity"""
+
+    single_model_error_prob: float
+    independent_error_prob: float
+    correlation_coefficient: float
+    combined_error_prob: float
+    risk_reduction_factor: float
+
+    def __str__(self) -> str:
+        return f"""Blind Spot Analysis:
+- Single model error probability: {self.single_model_error_prob:.4f}
+- Independent error probability: {self.independent_error_prob:.4f}
+- Model correlation coefficient: {self.correlation_coefficient:.4f}
+- Combined error probability: {self.combined_error_prob:.4f}
+- Risk reduction factor: {self.risk_reduction_factor:.2f}x"""
+
+
+@dataclass
+class VerificationResult:
+    """Complete result of adversarial verification"""
+
+    generated_code: GeneratedCode
+    verification_report: VerificationReport
+    blind_spot_analysis: BlindSpotAnalysis
+    generator_model: str
+    verifier_model: str
+    success: bool
+
+
+class VerificationKernel:
+    """
+    Main kernel that orchestrates adversarial verification with model diversity.
+
+    This kernel implements the mathematical framework that demonstrates how
+    using different models for generation and verification reduces the
+    probability of shared blind spots.
+    """
+
+    def __init__(self, generator_config: GeneratorConfig, verifier_config: VerifierConfig):
+        """
+        Initialize the Verification Kernel
+
+        Args:
+            generator_config: Configuration for the generator
+            verifier_config: Configuration for the verifier
+
+        Raises:
+            ValueError: If generator and verifier use the same model
+        """
+        # Enforce model diversity
+        if generator_config.model == verifier_config.model:
+            raise ValueError(
+                f"Generator and Verifier must use DIFFERENT models for adversarial verification. "
+                f"Both are configured to use {generator_config.model.value}. "
+                f"This defeats the purpose of model diversity."
+            )
+
+        self.generator = Generator(generator_config)
+        self.verifier = Verifier(verifier_config)
+        self.verification_history = []
+
+    def verify_task(
+        self, task_description: str, language: str = "python", **kwargs
+    ) -> VerificationResult:
+        """
+        Execute the full adversarial verification pipeline
+
+        Args:
+            task_description: Description of the task to generate code for
+            language: Programming language
+            **kwargs: Additional parameters
+
+        Returns:
+            VerificationResult with complete analysis
+        """
+        # Step 1: Generate code with Generator model
+        generated_code = self.generator.generate_code(
+            task_description=task_description, language=language, **kwargs
+        )
+
+        # Step 2: Verify with different Verifier model (adversarial)
+        verification_report = self.verifier.verify_code(
+            code=generated_code.code, description=task_description, language=language, **kwargs
+        )
+
+        # Step 3: Calculate blind spot analysis
+        blind_spot_analysis = self._calculate_blind_spot_reduction(
+            generator_model=self.generator.config.model, verifier_model=self.verifier.config.model
+        )
+
+        # Create result
+        result = VerificationResult(
+            generated_code=generated_code,
+            verification_report=verification_report,
+            blind_spot_analysis=blind_spot_analysis,
+            generator_model=generated_code.model_used,
+            verifier_model=verification_report.model_used,
+            success=verification_report.passed,
+        )
+
+        self.verification_history.append(result)
+        return result
+
+    def _calculate_blind_spot_reduction(
+        self, generator_model: ModelProvider, verifier_model: ModelProvider
+    ) -> BlindSpotAnalysis:
+        """
+        Calculate the mathematical reduction in blind spot probability
+
+        Using probability theory:
+        - P(error) = probability a single model makes an error
+        - P(both_error) = probability both models make the SAME error
+        - If models are independent: P(both_error) = P(error)²
+        - If models are correlated: P(both_error) = P(error)² + ρ*P(error)*(1-P(error))
+
+        Where ρ is the correlation coefficient between models.
+
+        Model diversity reduces ρ, thus reducing P(both_error).
+        """
+        # Estimated error probabilities (can be calibrated from real data)
+        single_model_error_prob = 0.15  # 15% chance of missing a bug
+
+        # Correlation coefficient based on model diversity
+        correlation = self._estimate_model_correlation(generator_model, verifier_model)
+
+        # Independent case (theoretical minimum)
+        independent_error_prob = single_model_error_prob**2
+
+        # Actual combined error probability with correlation
+        combined_error_prob = single_model_error_prob**2 + correlation * single_model_error_prob * (
+            1 - single_model_error_prob
+        )
+
+        # Risk reduction factor
+        risk_reduction = single_model_error_prob / combined_error_prob
+
+        return BlindSpotAnalysis(
+            single_model_error_prob=single_model_error_prob,
+            independent_error_prob=independent_error_prob,
+            correlation_coefficient=correlation,
+            combined_error_prob=combined_error_prob,
+            risk_reduction_factor=risk_reduction,
+        )
+
+    def _estimate_model_correlation(self, model1: ModelProvider, model2: ModelProvider) -> float:
+        """
+        Estimate correlation coefficient between two models
+
+        Models from different providers have lower correlation.
+        Models from the same provider have higher correlation.
+
+        Returns:
+            Correlation coefficient ρ ∈ [0, 1]
+        """
+        # Extract provider families
+        provider1 = self._get_provider_family(model1)
+        provider2 = self._get_provider_family(model2)
+
+        if provider1 != provider2:
+            # Different providers: low correlation (0.1-0.3)
+            return 0.2
+        else:
+            # Same provider: higher correlation (0.4-0.6)
+            return 0.5
+
+    def _get_provider_family(self, model: ModelProvider) -> str:
+        """Get the provider family (GPT, Gemini, Claude)"""
+        if model.value.startswith("gpt"):
+            return "openai"
+        elif model.value.startswith("gemini"):
+            return "google"
+        elif model.value.startswith("claude"):
+            return "anthropic"
+        return "unknown"
+
+    def get_statistics(self) -> dict[str, Any]:
+        """Get comprehensive statistics about the verification kernel"""
+        total_verifications = len(self.verification_history)
+        passed = sum(1 for r in self.verification_history if r.success)
+        failed = total_verifications - passed
+
+        avg_risk_reduction = (
+            sum(r.blind_spot_analysis.risk_reduction_factor for r in self.verification_history)
+            / total_verifications
+            if total_verifications > 0
+            else 0
+        )
+
+        return {
+            "generator": self.generator.get_stats(),
+            "verifier": self.verifier.get_stats(),
+            "total_verifications": total_verifications,
+            "passed": passed,
+            "failed": failed,
+            "success_rate": passed / total_verifications if total_verifications > 0 else 0,
+            "average_risk_reduction_factor": avg_risk_reduction,
+            "model_diversity": {
+                "generator": self.generator.config.model.value,
+                "verifier": self.verifier.config.model.value,
+                "are_different": self.generator.config.model != self.verifier.config.model,
+            },
+        }
+
+    def print_verification_summary(self, result: VerificationResult):
+        """Print a human-readable summary of verification results"""
+        print("=" * 80)
+        print("ADVERSARIAL VERIFICATION REPORT")
+        print("=" * 80)
+        print(f"\nGenerator Model: {result.generator_model}")
+        print(f"Verifier Model: {result.verifier_model}")
+        print(f"\nTask: {result.generated_code.description}")
+        print(f"\nGenerated Code ({result.generated_code.language}):")
+        print("-" * 80)
+        print(result.generated_code.code)
+        print("-" * 80)
+        print(f"\nVerification Status: {'✓ PASSED' if result.success else '✗ FAILED'}")
+        print(f"Summary: {result.verification_report.summary}")
+        print(f"\nIssues Found: {len(result.verification_report.issues)}")
+        for i, issue in enumerate(result.verification_report.issues, 1):
+            print(f"\n  {i}. [{issue.severity.value.upper()}] {issue.category}")
+            print(f"     {issue.description}")
+            if issue.suggestion:
+                print(f"     Suggestion: {issue.suggestion}")
+        print(f"\n{result.blind_spot_analysis}")
+        print("=" * 80)
+
+
+def prosecutor_check(kernel, code_snippet: str) -> bool:
+    """
+    Execute the Prosecutor Workflow: Generate and run hostile tests against code.
+
+    This is a demonstration function showing how to use the Prosecutor Mode
+    to verify code by attempting to break it with adversarial tests.
+
+    Args:
+        kernel: VerificationKernel instance (used for verifier access)
+        code_snippet: The code to test
+
+    Returns:
+        bool: True if code survived the attack, False if it was broken
+
+    Security Note:
+        The code_snippet is executed in a sandbox with timeout and resource limits.
+        While the sandbox provides isolation, this function is intended for trusted
+        code verification workflows, not for arbitrary untrusted code execution.
+    """
+    from .agents.verifier_gemini import GeminiVerifier
+    from .tools.sandbox import Sandbox
+
+    verifier = GeminiVerifier()
+    sandbox = Sandbox()
+
+    print("🕵️ Prosecutor (Gemini) is analyzing code...")
+
+    # 1. Generate the Attack
+    attack_script = verifier.generate_hostile_test(code_snippet)
+    print(f"⚔️ Generated Hostile Test:\n{attack_script}\n")
+
+    # 2. Combine Target + Attack
+    # Note: Both code_snippet and attack_script are executed in the sandbox
+    full_execution_script = f"{code_snippet}\n\n{attack_script}"
+
+    # 3. Run in Sandbox (with timeout and resource limits)
+    print("RUNNING IN SANDBOX...")
+    result = sandbox.execute(full_execution_script)
+
+    # 4. Judgement
+    if result["status"] == "success":
+        # If the script ran without error, the code SURVIVED the attack
+        # (Assuming the attack was meant to assert/crash on bug)
+        print("✅ PASSED: The code survived the hostile test.")
+        return True
+    else:
+        print(f"❌ FAILED: The Prosecutor broke the code.\nError: {result['error']}")
+        return False