ouroboros-ai 0.1.0__py3-none-any.whl

ouroboros/evaluation/mechanical.py

@@ -0,0 +1,351 @@
+"""Stage 1: Mechanical Verification.
+
+Zero-cost verification through automated checks:
+- Lint: Code style and formatting
+- Build: Compilation validation
+- Test: Unit/integration test execution
+- Static: Static analysis (type checking)
+- Coverage: Test coverage threshold (NFR9 >= 0.7)
+
+The MechanicalVerifier is stateless and produces immutable results.
+"""
+
+import asyncio
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from ouroboros.core.errors import ValidationError
+from ouroboros.core.types import Result
+from ouroboros.evaluation.models import CheckResult, CheckType, MechanicalResult
+from ouroboros.events.base import BaseEvent
+from ouroboros.events.evaluation import (
+    create_stage1_completed_event,
+    create_stage1_started_event,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class MechanicalConfig:
+    """Configuration for mechanical verification.
+
+    Attributes:
+        coverage_threshold: Minimum coverage required (default 0.7 per NFR9)
+        lint_command: Command to run linting
+        build_command: Command to run build
+        test_command: Command to run tests
+        static_command: Command to run static analysis
+        timeout_seconds: Timeout for each command
+        working_dir: Working directory for commands
+    """
+
+    coverage_threshold: float = 0.7
+    lint_command: tuple[str, ...] = ("uv", "run", "ruff", "check", ".")
+    build_command: tuple[str, ...] = ("uv", "run", "python", "-m", "py_compile")
+    test_command: tuple[str, ...] = ("uv", "run", "pytest", "--tb=short", "-q")
+    static_command: tuple[str, ...] = ("uv", "run", "mypy", "src/ouroboros", "--ignore-missing-imports")
+    coverage_command: tuple[str, ...] = (
+        "uv", "run", "pytest", "--cov=src/ouroboros", "--cov-report=term-missing", "-q"
+    )
+    timeout_seconds: int = 300
+    working_dir: Path | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class CommandResult:
+    """Result of running a shell command.
+
+    Attributes:
+        return_code: Exit code of the command
+        stdout: Standard output
+        stderr: Standard error
+        timed_out: Whether the command timed out
+    """
+
+    return_code: int
+    stdout: str
+    stderr: str
+    timed_out: bool = False
+
+
+async def run_command(
+    command: tuple[str, ...],
+    timeout: int,
+    working_dir: Path | None = None,
+) -> CommandResult:
+    """Run a shell command asynchronously.
+
+    Args:
+        command: Command and arguments to run
+        timeout: Timeout in seconds
+        working_dir: Working directory
+
+    Returns:
+        CommandResult with output and status
+    """
+    try:
+        process = await asyncio.create_subprocess_exec(
+            *command,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            cwd=working_dir,
+        )
+
+        try:
+            stdout, stderr = await asyncio.wait_for(
+                process.communicate(),
+                timeout=timeout,
+            )
+            return CommandResult(
+                return_code=process.returncode or 0,
+                stdout=stdout.decode("utf-8", errors="replace"),
+                stderr=stderr.decode("utf-8", errors="replace"),
+            )
+        except TimeoutError:
+            process.kill()
+            await process.wait()
+            return CommandResult(
+                return_code=-1,
+                stdout="",
+                stderr="Command timed out",
+                timed_out=True,
+            )
+    except FileNotFoundError as e:
+        return CommandResult(
+            return_code=-1,
+            stdout="",
+            stderr=f"Command not found: {e}",
+        )
+    except OSError as e:
+        return CommandResult(
+            return_code=-1,
+            stdout="",
+            stderr=f"OS error: {e}",
+        )
+
+
+def parse_coverage_from_output(output: str) -> float | None:
+    """Extract coverage percentage from pytest-cov output.
+
+    Args:
+        output: stdout from coverage command
+
+    Returns:
+        Coverage as float (0.0-1.0) or None if not found
+    """
+    # Look for "TOTAL ... XX%" pattern
+    import re
+
+    # Pattern matches lines like "TOTAL   1234   123  90%"
+    pattern = r"TOTAL\s+\d+\s+\d+\s+(\d+)%"
+    match = re.search(pattern, output)
+    if match:
+        return float(match.group(1)) / 100.0
+
+    # Alternative pattern: "Coverage: XX%"
+    alt_pattern = r"Coverage:\s*(\d+(?:\.\d+)?)%"
+    alt_match = re.search(alt_pattern, output)
+    if alt_match:
+        return float(alt_match.group(1)) / 100.0
+
+    return None
+
+
+class MechanicalVerifier:
+    """Stage 1 mechanical verification executor.
+
+    Runs zero-cost automated checks on artifacts.
+    Stateless - all state passed via parameters.
+
+    Example:
+        verifier = MechanicalVerifier(config)
+        result = await verifier.verify(execution_id, checks=[CheckType.LINT, CheckType.TEST])
+    """
+
+    def __init__(self, config: MechanicalConfig | None = None) -> None:
+        """Initialize verifier with configuration.
+
+        Args:
+            config: Verification configuration, uses defaults if None
+        """
+        self.config = config or MechanicalConfig()
+
+    async def verify(
+        self,
+        execution_id: str,
+        checks: list[CheckType] | None = None,
+    ) -> Result[tuple[MechanicalResult, list[BaseEvent]], ValidationError]:
+        """Run mechanical verification checks.
+
+        Args:
+            execution_id: Execution identifier for events
+            checks: List of checks to run, defaults to all
+
+        Returns:
+            Result containing MechanicalResult and events, or error
+        """
+        if checks is None:
+            checks = list(CheckType)
+
+        events: list[BaseEvent] = []
+        check_results: list[CheckResult] = []
+        coverage_score: float | None = None
+
+        # Emit start event
+        events.append(
+            create_stage1_started_event(
+                execution_id=execution_id,
+                checks_to_run=[c.value for c in checks],
+            )
+        )
+
+        # Run each check
+        for check_type in checks:
+            result = await self._run_check(check_type)
+            check_results.append(result)
+
+            # Track coverage if it was a coverage check
+            if check_type == CheckType.COVERAGE and result.passed:
+                coverage_score = result.details.get("coverage_score")
+
+        # Determine overall pass/fail
+        all_passed = all(c.passed for c in check_results)
+
+        # Verify coverage threshold if coverage was checked
+        if coverage_score is not None and coverage_score < self.config.coverage_threshold:
+            # Find and update coverage check to failed
+            updated_results = []
+            for cr in check_results:
+                if cr.check_type == CheckType.COVERAGE:
+                    updated_results.append(
+                        CheckResult(
+                            check_type=CheckType.COVERAGE,
+                            passed=False,
+                            message=f"Coverage {coverage_score:.1%} below threshold {self.config.coverage_threshold:.1%}",
+                            details=cr.details,
+                        )
+                    )
+                else:
+                    updated_results.append(cr)
+            check_results = updated_results
+            all_passed = False
+
+        mechanical_result = MechanicalResult(
+            passed=all_passed,
+            checks=tuple(check_results),
+            coverage_score=coverage_score,
+        )
+
+        # Emit completion event
+        events.append(
+            create_stage1_completed_event(
+                execution_id=execution_id,
+                passed=all_passed,
+                checks=[
+                    {
+                        "check_type": c.check_type.value,
+                        "passed": c.passed,
+                        "message": c.message,
+                    }
+                    for c in check_results
+                ],
+                coverage_score=coverage_score,
+            )
+        )
+
+        return Result.ok((mechanical_result, events))
+
+    async def _run_check(self, check_type: CheckType) -> CheckResult:
+        """Run a single check.
+
+        Args:
+            check_type: Type of check to run
+
+        Returns:
+            CheckResult with pass/fail status
+        """
+        command = self._get_command_for_check(check_type)
+        if command is None:
+            return CheckResult(
+                check_type=check_type,
+                passed=True,
+                message=f"Check {check_type.value} skipped (no command configured)",
+                details={"skipped": True},
+            )
+
+        cmd_result = await run_command(
+            command,
+            timeout=self.config.timeout_seconds,
+            working_dir=self.config.working_dir,
+        )
+
+        if cmd_result.timed_out:
+            return CheckResult(
+                check_type=check_type,
+                passed=False,
+                message=f"Check {check_type.value} timed out after {self.config.timeout_seconds}s",
+                details={"timed_out": True},
+            )
+
+        passed = cmd_result.return_code == 0
+        details: dict[str, Any] = {
+            "return_code": cmd_result.return_code,
+            "stdout_preview": cmd_result.stdout[:500] if cmd_result.stdout else "",
+            "stderr_preview": cmd_result.stderr[:500] if cmd_result.stderr else "",
+        }
+
+        # Extract coverage if this was a coverage check
+        if check_type == CheckType.COVERAGE and passed:
+            coverage = parse_coverage_from_output(cmd_result.stdout)
+            if coverage is not None:
+                details["coverage_score"] = coverage
+
+        message = (
+            f"Check {check_type.value} passed"
+            if passed
+            else f"Check {check_type.value} failed (exit code {cmd_result.return_code})"
+        )
+
+        return CheckResult(
+            check_type=check_type,
+            passed=passed,
+            message=message,
+            details=details,
+        )
+
+    def _get_command_for_check(self, check_type: CheckType) -> tuple[str, ...] | None:
+        """Get the command for a specific check type.
+
+        Args:
+            check_type: Type of check
+
+        Returns:
+            Command tuple or None if not configured
+        """
+        commands = {
+            CheckType.LINT: self.config.lint_command,
+            CheckType.BUILD: self.config.build_command,
+            CheckType.TEST: self.config.test_command,
+            CheckType.STATIC: self.config.static_command,
+            CheckType.COVERAGE: self.config.coverage_command,
+        }
+        return commands.get(check_type)
+
+
+async def run_mechanical_verification(
+    execution_id: str,
+    config: MechanicalConfig | None = None,
+    checks: list[CheckType] | None = None,
+) -> Result[tuple[MechanicalResult, list[BaseEvent]], ValidationError]:
+    """Convenience function for running mechanical verification.
+
+    Args:
+        execution_id: Execution identifier
+        config: Optional configuration
+        checks: Optional list of checks to run
+
+    Returns:
+        Result with MechanicalResult and events
+    """
+    verifier = MechanicalVerifier(config)
+    return await verifier.verify(execution_id, checks)

ouroboros/evaluation/models.py

@@ -0,0 +1,235 @@
+"""Data models for the evaluation pipeline.
+
+This module defines immutable data structures for all three evaluation stages.
+All models use frozen dataclasses with slots for immutability and performance.
+
+Classes:
+    CheckType: Enum of mechanical check types
+    CheckResult: Single mechanical check result
+    MechanicalResult: Aggregated Stage 1 results
+    SemanticResult: Stage 2 LLM evaluation results
+    Vote: Single model vote in consensus
+    ConsensusResult: Aggregated Stage 3 results
+    EvaluationContext: Input context for evaluation
+    EvaluationResult: Complete pipeline output
+"""
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any
+
+from ouroboros.events.base import BaseEvent
+
+
+class CheckType(StrEnum):
+    """Types of mechanical checks in Stage 1.
+
+    Attributes:
+        LINT: Code style and formatting checks
+        BUILD: Compilation and build validation
+        TEST: Unit and integration test execution
+        STATIC: Static analysis (type checking, etc.)
+        COVERAGE: Test coverage threshold verification
+    """
+
+    LINT = "lint"
+    BUILD = "build"
+    TEST = "test"
+    STATIC = "static"
+    COVERAGE = "coverage"
+
+
+@dataclass(frozen=True, slots=True)
+class CheckResult:
+    """Result of a single mechanical check.
+
+    Attributes:
+        check_type: Type of check performed
+        passed: Whether the check passed
+        message: Human-readable result message
+        details: Additional check-specific details
+    """
+
+    check_type: CheckType
+    passed: bool
+    message: str
+    details: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True)
+class MechanicalResult:
+    """Aggregated result of Stage 1 mechanical verification.
+
+    All checks must pass for the overall result to pass.
+    Coverage score is tracked separately for NFR9 compliance.
+
+    Attributes:
+        passed: True if all checks passed
+        checks: Tuple of individual check results
+        coverage_score: Test coverage percentage (0.0-1.0), None if not measured
+    """
+
+    passed: bool
+    checks: tuple[CheckResult, ...]
+    coverage_score: float | None = None
+
+    @property
+    def failed_checks(self) -> tuple[CheckResult, ...]:
+        """Return only the checks that failed."""
+        return tuple(c for c in self.checks if not c.passed)
+
+
+@dataclass(frozen=True, slots=True)
+class SemanticResult:
+    """Result of Stage 2 semantic evaluation.
+
+    Uses LLM to evaluate AC compliance, goal alignment, and drift.
+    Uncertainty score determines if Stage 3 consensus is needed.
+
+    Attributes:
+        score: Overall evaluation score (0.0-1.0)
+        ac_compliance: Whether acceptance criteria are met
+        goal_alignment: Alignment with original goal (0.0-1.0)
+        drift_score: Deviation from seed intent (0.0-1.0, lower is better)
+        uncertainty: Model uncertainty about evaluation (0.0-1.0)
+        reasoning: Explanation of the evaluation
+    """
+
+    score: float
+    ac_compliance: bool
+    goal_alignment: float
+    drift_score: float
+    uncertainty: float
+    reasoning: str
+
+    def __post_init__(self) -> None:
+        """Validate score ranges."""
+        for attr in ("score", "goal_alignment", "drift_score", "uncertainty"):
+            value = getattr(self, attr)
+            if not 0.0 <= value <= 1.0:
+                msg = f"{attr} must be between 0.0 and 1.0, got {value}"
+                raise ValueError(msg)
+
+
+@dataclass(frozen=True, slots=True)
+class Vote:
+    """Single model vote in Stage 3 consensus.
+
+    Attributes:
+        model: Model identifier that cast the vote
+        approved: Whether the model approves the output
+        confidence: Model's confidence in its decision (0.0-1.0)
+        reasoning: Explanation of the vote
+    """
+
+    model: str
+    approved: bool
+    confidence: float
+    reasoning: str
+
+    def __post_init__(self) -> None:
+        """Validate confidence range."""
+        if not 0.0 <= self.confidence <= 1.0:
+            msg = f"confidence must be between 0.0 and 1.0, got {self.confidence}"
+            raise ValueError(msg)
+
+
+@dataclass(frozen=True, slots=True)
+class ConsensusResult:
+    """Aggregated result of Stage 3 multi-model consensus.
+
+    Requires 2/3 majority for approval with minimum 3 models.
+
+    Attributes:
+        approved: True if consensus reached approval
+        votes: Tuple of individual model votes
+        majority_ratio: Ratio of approving votes (0.0-1.0)
+        disagreements: Tuple of reasoning strings from dissenting votes
+    """
+
+    approved: bool
+    votes: tuple[Vote, ...]
+    majority_ratio: float
+    disagreements: tuple[str, ...] = ()
+
+    @property
+    def approving_votes(self) -> int:
+        """Count of votes that approved."""
+        return sum(1 for v in self.votes if v.approved)
+
+    @property
+    def total_votes(self) -> int:
+        """Total number of votes cast."""
+        return len(self.votes)
+
+
+@dataclass(frozen=True, slots=True)
+class EvaluationContext:
+    """Input context for the evaluation pipeline.
+
+    Attributes:
+        execution_id: Unique identifier for the execution
+        seed_id: Identifier of the seed being evaluated against
+        current_ac: The acceptance criterion being evaluated
+        artifact: The output artifact to evaluate
+        artifact_type: Type of artifact (code, document, etc.)
+        goal: Original goal from seed
+        constraints: Constraints from seed
+    """
+
+    execution_id: str
+    seed_id: str
+    current_ac: str
+    artifact: str
+    artifact_type: str = "code"
+    goal: str = ""
+    constraints: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class EvaluationResult:
+    """Complete evaluation pipeline result.
+
+    Contains results from all stages that were executed,
+    final approval status, and generated events for audit trail.
+
+    Attributes:
+        execution_id: Execution identifier for tracing
+        stage1_result: Mechanical verification result (if executed)
+        stage2_result: Semantic evaluation result (if executed)
+        stage3_result: Consensus result (if triggered)
+        final_approved: Overall approval status
+        events: List of events generated during evaluation
+    """
+
+    execution_id: str
+    stage1_result: MechanicalResult | None = None
+    stage2_result: SemanticResult | None = None
+    stage3_result: ConsensusResult | None = None
+    final_approved: bool = False
+    events: list[BaseEvent] = field(default_factory=list)
+
+    @property
+    def highest_stage_completed(self) -> int:
+        """Return the highest stage number that completed."""
+        if self.stage3_result is not None:
+            return 3
+        if self.stage2_result is not None:
+            return 2
+        if self.stage1_result is not None:
+            return 1
+        return 0
+
+    @property
+    def failure_reason(self) -> str | None:
+        """Return the reason for failure, if any."""
+        if self.final_approved:
+            return None
+        if self.stage1_result and not self.stage1_result.passed:
+            failed = self.stage1_result.failed_checks
+            return f"Stage 1 failed: {', '.join(c.check_type for c in failed)}"
+        if self.stage2_result and not self.stage2_result.ac_compliance:
+            return f"Stage 2 failed: AC non-compliance (score={self.stage2_result.score:.2f})"
+        if self.stage3_result and not self.stage3_result.approved:
+            return f"Stage 3 failed: Consensus not reached ({self.stage3_result.majority_ratio:.0%})"
+        return "Unknown failure"