ouroboros-ai 0.1.0__py3-none-any.whl

ouroboros/evaluation/pipeline.py

@@ -0,0 +1,286 @@
+"""Evaluation Pipeline Orchestrator.
+
+Orchestrates the three-stage evaluation pipeline:
+1. Stage 1: Mechanical Verification ($0)
+2. Stage 2: Semantic Evaluation (Standard tier)
+3. Stage 3: Multi-Model Consensus (Frontier tier, if triggered)
+
+The pipeline respects configuration flags and trigger conditions.
+"""
+
+from dataclasses import dataclass
+
+from ouroboros.core.errors import ProviderError, ValidationError
+from ouroboros.core.types import Result
+from ouroboros.evaluation.consensus import ConsensusConfig, ConsensusEvaluator
+from ouroboros.evaluation.mechanical import (
+    MechanicalConfig,
+    MechanicalVerifier,
+)
+from ouroboros.evaluation.models import (
+    CheckType,
+    EvaluationContext,
+    EvaluationResult,
+)
+from ouroboros.evaluation.semantic import SemanticConfig, SemanticEvaluator
+from ouroboros.evaluation.trigger import (
+    ConsensusTrigger,
+    TriggerConfig,
+    TriggerContext,
+)
+from ouroboros.events.base import BaseEvent
+from ouroboros.events.evaluation import create_pipeline_completed_event
+from ouroboros.providers.litellm_adapter import LiteLLMAdapter
+
+
+@dataclass(frozen=True, slots=True)
+class PipelineConfig:
+    """Configuration for the evaluation pipeline.
+
+    Attributes:
+        stage1_enabled: Run mechanical verification
+        stage2_enabled: Run semantic evaluation
+        stage3_enabled: Allow consensus if triggered
+        mechanical: Stage 1 configuration
+        semantic: Stage 2 configuration
+        consensus: Stage 3 configuration
+        trigger: Trigger matrix configuration
+    """
+
+    stage1_enabled: bool = True
+    stage2_enabled: bool = True
+    stage3_enabled: bool = True
+    mechanical: MechanicalConfig | None = None
+    semantic: SemanticConfig | None = None
+    consensus: ConsensusConfig | None = None
+    trigger: TriggerConfig | None = None
+
+
+class EvaluationPipeline:
+    """Orchestrates the three-stage evaluation pipeline.
+
+    Runs stages sequentially, respecting configuration and triggers.
+    Stage 3 is only run if trigger conditions are met.
+
+    Example:
+        pipeline = EvaluationPipeline(llm_adapter, config)
+        result = await pipeline.evaluate(context)
+    """
+
+    def __init__(
+        self,
+        llm_adapter: LiteLLMAdapter,
+        config: PipelineConfig | None = None,
+    ) -> None:
+        """Initialize pipeline.
+
+        Args:
+            llm_adapter: LLM adapter for semantic and consensus
+            config: Pipeline configuration
+        """
+        self._llm = llm_adapter
+        self._config = config or PipelineConfig()
+
+        # Initialize stage evaluators
+        self._mechanical = MechanicalVerifier(self._config.mechanical)
+        self._semantic = SemanticEvaluator(llm_adapter, self._config.semantic)
+        self._consensus = ConsensusEvaluator(llm_adapter, self._config.consensus)
+        self._trigger = ConsensusTrigger(self._config.trigger)
+
+    async def evaluate(
+        self,
+        context: EvaluationContext,
+        trigger_context: TriggerContext | None = None,
+    ) -> Result[EvaluationResult, ProviderError | ValidationError]:
+        """Run the evaluation pipeline.
+
+        Args:
+            context: Evaluation context with artifact
+            trigger_context: Optional pre-populated trigger context
+
+        Returns:
+            Result containing EvaluationResult or error
+        """
+        events: list[BaseEvent] = []
+        stage1_result = None
+        stage2_result = None
+        stage3_result = None
+
+        # Stage 1: Mechanical Verification
+        if self._config.stage1_enabled:
+            result = await self._mechanical.verify(
+                context.execution_id,
+                checks=[CheckType.LINT, CheckType.BUILD, CheckType.TEST, CheckType.STATIC, CheckType.COVERAGE],
+            )
+            if result.is_err:
+                return Result.err(result.error)
+
+            stage1_result, stage1_events = result.value
+            events.extend(stage1_events)
+
+            # If Stage 1 fails, stop here
+            if not stage1_result.passed:
+                return self._build_result(
+                    context.execution_id,
+                    events,
+                    stage1_result=stage1_result,
+                    final_approved=False,
+                )
+
+        # Stage 2: Semantic Evaluation
+        if self._config.stage2_enabled:
+            result = await self._semantic.evaluate(context)
+            if result.is_err:
+                return Result.err(result.error)
+
+            stage2_result, stage2_events = result.value
+            events.extend(stage2_events)
+
+            # Build trigger context if not provided
+            if trigger_context is None:
+                trigger_context = TriggerContext(
+                    execution_id=context.execution_id,
+                    semantic_result=stage2_result,
+                )
+
+            # Check if Stage 2 failed on compliance
+            if not stage2_result.ac_compliance:
+                return self._build_result(
+                    context.execution_id,
+                    events,
+                    stage1_result=stage1_result,
+                    stage2_result=stage2_result,
+                    final_approved=False,
+                )
+
+        # Stage 3: Consensus (if triggered)
+        if self._config.stage3_enabled and trigger_context:
+            trigger_result = self._trigger.evaluate(trigger_context)
+            if trigger_result.is_err:
+                return Result.err(trigger_result.error)
+
+            trigger_decision, trigger_events = trigger_result.value
+            events.extend(trigger_events)
+
+            if trigger_decision.should_trigger:
+                trigger_reason = (
+                    trigger_decision.trigger_type.value
+                    if trigger_decision.trigger_type
+                    else "manual"
+                )
+                result = await self._consensus.evaluate(context, trigger_reason)
+                if result.is_err:
+                    return Result.err(result.error)
+
+                stage3_result, stage3_events = result.value
+                events.extend(stage3_events)
+
+                # Final approval based on consensus
+                return self._build_result(
+                    context.execution_id,
+                    events,
+                    stage1_result=stage1_result,
+                    stage2_result=stage2_result,
+                    stage3_result=stage3_result,
+                    final_approved=stage3_result.approved,
+                )
+
+        # No consensus triggered - approve based on Stage 2
+        final_approved = True
+        if stage2_result:
+            final_approved = stage2_result.ac_compliance and stage2_result.score >= 0.8
+
+        return self._build_result(
+            context.execution_id,
+            events,
+            stage1_result=stage1_result,
+            stage2_result=stage2_result,
+            final_approved=final_approved,
+        )
+
+    def _build_result(
+        self,
+        execution_id: str,
+        events: list[BaseEvent],
+        stage1_result=None,
+        stage2_result=None,
+        stage3_result=None,
+        final_approved: bool = False,
+    ) -> Result[EvaluationResult, ValidationError]:
+        """Build the final evaluation result.
+
+        Args:
+            execution_id: Execution identifier
+            events: Collected events
+            stage1_result: Stage 1 result if completed
+            stage2_result: Stage 2 result if completed
+            stage3_result: Stage 3 result if triggered
+            final_approved: Overall approval status
+
+        Returns:
+            Result containing EvaluationResult
+        """
+        # Calculate highest stage before creating immutable result
+        highest_stage = 0
+        if stage1_result is not None:
+            highest_stage = 1
+        if stage2_result is not None:
+            highest_stage = 2
+        if stage3_result is not None:
+            highest_stage = 3
+
+        # Calculate failure reason before creating immutable result
+        failure_reason: str | None = None
+        if not final_approved:
+            if stage1_result and not stage1_result.passed:
+                failed = stage1_result.failed_checks
+                failure_reason = f"Stage 1 failed: {', '.join(c.check_type for c in failed)}"
+            elif stage2_result and not stage2_result.ac_compliance:
+                failure_reason = f"Stage 2 failed: AC non-compliance (score={stage2_result.score:.2f})"
+            elif stage3_result and not stage3_result.approved:
+                failure_reason = f"Stage 3 failed: Consensus not reached ({stage3_result.majority_ratio:.0%})"
+            else:
+                failure_reason = "Unknown failure"
+
+        # Create completion event
+        completion_event = create_pipeline_completed_event(
+            execution_id=execution_id,
+            final_approved=final_approved,
+            highest_stage=highest_stage,
+            failure_reason=failure_reason,
+        )
+
+        # Build complete event list before creating frozen result
+        all_events = [*events, completion_event]
+
+        result = EvaluationResult(
+            execution_id=execution_id,
+            stage1_result=stage1_result,
+            stage2_result=stage2_result,
+            stage3_result=stage3_result,
+            final_approved=final_approved,
+            events=all_events,
+        )
+
+        return Result.ok(result)
+
+
+async def run_evaluation_pipeline(
+    context: EvaluationContext,
+    llm_adapter: LiteLLMAdapter,
+    config: PipelineConfig | None = None,
+    trigger_context: TriggerContext | None = None,
+) -> Result[EvaluationResult, ProviderError | ValidationError]:
+    """Convenience function for running the evaluation pipeline.
+
+    Args:
+        context: Evaluation context
+        llm_adapter: LLM adapter
+        config: Optional configuration
+        trigger_context: Optional trigger context
+
+    Returns:
+        Result with EvaluationResult
+    """
+    pipeline = EvaluationPipeline(llm_adapter, config)
+    return await pipeline.evaluate(context, trigger_context)

ouroboros/evaluation/semantic.py

@@ -0,0 +1,302 @@
+"""Stage 2: Semantic Evaluation.
+
+LLM-based semantic evaluation using Standard tier:
+- AC Compliance: Whether acceptance criteria are met
+- Goal Alignment: Alignment with original goal
+- Drift Measurement: Deviation from seed intent
+
+The SemanticEvaluator uses the LiteLLM adapter for LLM calls.
+"""
+
+from dataclasses import dataclass
+import json
+
+from ouroboros.core.errors import ProviderError, ValidationError
+from ouroboros.core.types import Result
+from ouroboros.evaluation.models import EvaluationContext, SemanticResult
+from ouroboros.events.base import BaseEvent
+from ouroboros.events.evaluation import (
+    create_stage2_completed_event,
+    create_stage2_started_event,
+)
+from ouroboros.providers.base import CompletionConfig, Message, MessageRole
+from ouroboros.providers.litellm_adapter import LiteLLMAdapter
+
+# Default model for semantic evaluation (Standard tier)
+# Can be overridden via SemanticConfig.model
+DEFAULT_SEMANTIC_MODEL = "openrouter/google/gemini-2.0-flash-001"
+
+
+@dataclass(frozen=True, slots=True)
+class SemanticConfig:
+    """Configuration for semantic evaluation.
+
+    Attributes:
+        model: LLM model to use for evaluation
+        temperature: Sampling temperature (lower for consistency)
+        max_tokens: Maximum tokens for response
+        satisfaction_threshold: Minimum score to pass (default 0.8)
+    """
+
+    model: str = DEFAULT_SEMANTIC_MODEL
+    temperature: float = 0.2
+    max_tokens: int = 2048
+    satisfaction_threshold: float = 0.8
+
+
+EVALUATION_SYSTEM_PROMPT = """You are a rigorous software evaluation assistant. Your task is to evaluate code artifacts against acceptance criteria, goal alignment, and semantic drift.
+
+You must respond ONLY with a valid JSON object in the following exact format:
+{
+    "score": <float between 0.0 and 1.0>,
+    "ac_compliance": <boolean>,
+    "goal_alignment": <float between 0.0 and 1.0>,
+    "drift_score": <float between 0.0 and 1.0>,
+    "uncertainty": <float between 0.0 and 1.0>,
+    "reasoning": "<string explaining your evaluation>"
+}
+
+Evaluation criteria:
+- score: Overall quality score (0.0 = completely fails, 1.0 = perfect)
+- ac_compliance: true if the artifact meets the acceptance criterion
+- goal_alignment: How well the artifact aligns with the original goal
+- drift_score: How much the implementation drifts from intent (0.0 = no drift, 1.0 = complete drift)
+- uncertainty: Your confidence level in this evaluation (0.0 = certain, 1.0 = very uncertain)
+- reasoning: Brief explanation of your evaluation
+
+Be strict but fair. A passing artifact should have:
+- ac_compliance = true
+- score >= 0.8
+- goal_alignment >= 0.7
+- drift_score <= 0.3
+- uncertainty <= 0.3"""
+
+
+def build_evaluation_prompt(context: EvaluationContext) -> str:
+    """Build the user prompt for evaluation.
+
+    Args:
+        context: Evaluation context with artifact and criteria
+
+    Returns:
+        Formatted prompt string
+    """
+    constraints_text = "\n".join(f"- {c}" for c in context.constraints) if context.constraints else "None specified"
+
+    return f"""Evaluate the following artifact:
+
+## Acceptance Criterion
+{context.current_ac}
+
+## Original Goal
+{context.goal if context.goal else "Not specified"}
+
+## Constraints
+{constraints_text}
+
+## Artifact Type
+{context.artifact_type}
+
+## Artifact Content
+```
+{context.artifact}
+```
+
+Provide your evaluation as a JSON object."""
+
+
+def extract_json_payload(text: str) -> str | None:
+    """Extract JSON object from text using index-based approach.
+
+    More reliable than regex for handling nested braces in code snippets.
+
+    Args:
+        text: Raw text potentially containing JSON
+
+    Returns:
+        Extracted JSON string or None if not found
+    """
+    start = text.find("{")
+    end = text.rfind("}")
+    if start != -1 and end != -1 and end > start:
+        return text[start : end + 1]
+    return None
+
+
+def parse_semantic_response(response_text: str) -> Result[SemanticResult, ValidationError]:
+    """Parse LLM response into SemanticResult.
+
+    Args:
+        response_text: Raw LLM response text
+
+    Returns:
+        Result containing SemanticResult or ValidationError
+    """
+    # Extract JSON using index-based approach (handles nested braces)
+    json_str = extract_json_payload(response_text)
+
+    if not json_str:
+        return Result.err(
+            ValidationError(
+                "Could not find JSON in response",
+                field="response",
+                value=response_text[:100],
+            )
+        )
+
+    try:
+        data = json.loads(json_str)
+    except json.JSONDecodeError as e:
+        return Result.err(
+            ValidationError(
+                f"Invalid JSON in response: {e}",
+                field="response",
+                value=json_str[:100],
+            )
+        )
+
+    # Validate required fields
+    required_fields = ["score", "ac_compliance", "goal_alignment", "drift_score", "uncertainty", "reasoning"]
+    missing = [f for f in required_fields if f not in data]
+    if missing:
+        return Result.err(
+            ValidationError(
+                f"Missing required fields: {missing}",
+                field="response",
+                details={"missing_fields": missing},
+            )
+        )
+
+    # Validate and clamp numeric ranges
+    try:
+        score = max(0.0, min(1.0, float(data["score"])))
+        goal_alignment = max(0.0, min(1.0, float(data["goal_alignment"])))
+        drift_score = max(0.0, min(1.0, float(data["drift_score"])))
+        uncertainty = max(0.0, min(1.0, float(data["uncertainty"])))
+
+        return Result.ok(
+            SemanticResult(
+                score=score,
+                ac_compliance=bool(data["ac_compliance"]),
+                goal_alignment=goal_alignment,
+                drift_score=drift_score,
+                uncertainty=uncertainty,
+                reasoning=str(data["reasoning"]),
+            )
+        )
+    except (TypeError, ValueError) as e:
+        return Result.err(
+            ValidationError(
+                f"Invalid field types: {e}",
+                field="response",
+                details={"error": str(e)},
+            )
+        )
+
+
+class SemanticEvaluator:
+    """Stage 2 semantic evaluation using LLM.
+
+    Evaluates artifacts for AC compliance, goal alignment, and drift.
+    Uses Standard tier LLM for balanced cost/quality.
+
+    Example:
+        evaluator = SemanticEvaluator(llm_adapter)
+        result = await evaluator.evaluate(context, execution_id)
+    """
+
+    def __init__(
+        self,
+        llm_adapter: LiteLLMAdapter,
+        config: SemanticConfig | None = None,
+    ) -> None:
+        """Initialize evaluator.
+
+        Args:
+            llm_adapter: LLM adapter for completions
+            config: Evaluation configuration
+        """
+        self._llm = llm_adapter
+        self._config = config or SemanticConfig()
+
+    async def evaluate(
+        self,
+        context: EvaluationContext,
+    ) -> Result[tuple[SemanticResult, list[BaseEvent]], ProviderError | ValidationError]:
+        """Evaluate an artifact semantically.
+
+        Args:
+            context: Evaluation context
+
+        Returns:
+            Result containing SemanticResult and events, or error
+        """
+        events: list[BaseEvent] = []
+
+        # Emit start event
+        events.append(
+            create_stage2_started_event(
+                execution_id=context.execution_id,
+                model=self._config.model,
+                current_ac=context.current_ac,
+            )
+        )
+
+        # Build messages
+        messages = [
+            Message(role=MessageRole.SYSTEM, content=EVALUATION_SYSTEM_PROMPT),
+            Message(role=MessageRole.USER, content=build_evaluation_prompt(context)),
+        ]
+
+        # Call LLM
+        completion_config = CompletionConfig(
+            model=self._config.model,
+            temperature=self._config.temperature,
+            max_tokens=self._config.max_tokens,
+        )
+
+        llm_result = await self._llm.complete(messages, completion_config)
+        if llm_result.is_err:
+            return Result.err(llm_result.error)
+
+        response = llm_result.value
+
+        # Parse response
+        parse_result = parse_semantic_response(response.content)
+        if parse_result.is_err:
+            return Result.err(parse_result.error)
+
+        semantic_result = parse_result.value
+
+        # Emit completion event
+        events.append(
+            create_stage2_completed_event(
+                execution_id=context.execution_id,
+                score=semantic_result.score,
+                ac_compliance=semantic_result.ac_compliance,
+                goal_alignment=semantic_result.goal_alignment,
+                drift_score=semantic_result.drift_score,
+                uncertainty=semantic_result.uncertainty,
+            )
+        )
+
+        return Result.ok((semantic_result, events))
+
+
+async def run_semantic_evaluation(
+    context: EvaluationContext,
+    llm_adapter: LiteLLMAdapter,
+    config: SemanticConfig | None = None,
+) -> Result[tuple[SemanticResult, list[BaseEvent]], ProviderError | ValidationError]:
+    """Convenience function for running semantic evaluation.
+
+    Args:
+        context: Evaluation context
+        llm_adapter: LLM adapter
+        config: Optional configuration
+
+    Returns:
+        Result with SemanticResult and events
+    """
+    evaluator = SemanticEvaluator(llm_adapter, config)
+    return await evaluator.evaluate(context)