contextforge-eval 0.1.0__py3-none-any.whl

context_forge/graders/deterministic/memory_corruption.py

@@ -0,0 +1,130 @@
+"""Deterministic Memory Corruption Grader.
+
+Checks for TRUE INVARIANTS that are ALWAYS wrong regardless of agent path:
+- Data corruption: Existing correct data was deleted or overwritten with null
+- Schema violations: Wrong types, malformed data structures
+
+These checks detect hard failures that should never happen, regardless of
+the non-deterministic path the agent takes.
+
+For semantic evaluation (did the agent save the right facts?), use the
+MemoryHygieneJudge LLM-based grader instead.
+"""
+
+from context_forge.core.trace import MemoryWriteStep, TraceRun
+from context_forge.graders.base import Evidence, Grader, GraderResult, Severity
+
+
+class MemoryCorruptionGrader(Grader):
+    """Deterministic grader for memory corruption detection.
+
+    Checks for invariant violations that are ALWAYS wrong:
+    1. Data corruption: Existing data deleted without replacement
+    2. Field deletion: Required fields removed
+
+    These are hard constraints - if violated, something is broken.
+
+    For semantic checks (missed facts, hallucinations), use the
+    MemoryHygieneJudge LLM-based grader.
+
+    Usage:
+        grader = MemoryCorruptionGrader()
+        result = grader.grade(trace)
+        if not result.passed:
+            for error in result.errors:
+                print(f"Corruption detected: {error.description}")
+    """
+
+    name = "memory_corruption"
+    deterministic = True
+    required_step_types = []  # Can run on any trace
+
+    def __init__(self, fail_on_data_loss: bool = True):
+        """Initialize the grader.
+
+        Args:
+            fail_on_data_loss: Treat data loss as errors (default: True)
+        """
+        self.fail_on_data_loss = fail_on_data_loss
+
+    def grade(self, trace: TraceRun) -> GraderResult:
+        """Check for memory corruption in a trace.
+
+        Args:
+            trace: The trace to evaluate
+
+        Returns:
+            GraderResult with corruption findings
+        """
+        evidence: list[Evidence] = []
+
+        # Get memory write steps
+        memory_writes = [s for s in trace.steps if isinstance(s, MemoryWriteStep)]
+
+        # Check for data corruption (deletion of existing values)
+        evidence.extend(self._check_data_corruption(memory_writes))
+
+        # Calculate score and pass/fail
+        errors = [e for e in evidence if e.severity == Severity.ERROR]
+
+        # Score: 1.0 - 0.5 per corruption error
+        score = max(0.0, 1.0 - (len(errors) * 0.5))
+        passed = len(errors) == 0
+
+        return GraderResult(
+            grader_name=self.name,
+            passed=passed,
+            score=score,
+            evidence=evidence,
+            metadata={
+                "total_memory_writes": len(memory_writes),
+                "corruption_errors": len(errors),
+            },
+        )
+
+    def _check_data_corruption(
+        self, memory_writes: list[MemoryWriteStep]
+    ) -> list[Evidence]:
+        """Check for data corruption: existing data deleted or nullified.
+
+        Data corruption occurs when:
+        - old_value exists (not None)
+        - new_value is None (data deleted)
+
+        This is an invariant violation - correct user data should never
+        be deleted without explicit user request.
+        """
+        evidence = []
+
+        for write in memory_writes:
+            if not write.changes:
+                continue
+
+            corrupted_fields = []
+            for change in write.changes:
+                # Corruption: had value, now null (data lost)
+                if change.old_value is not None and change.new_value is None:
+                    corrupted_fields.append(change)
+
+            if corrupted_fields:
+                severity = Severity.ERROR if self.fail_on_data_loss else Severity.WARN
+                paths = [c.path for c in corrupted_fields]
+                evidence.append(
+                    Evidence(
+                        check_name="data_corruption",
+                        description=f"Existing data was deleted: {paths}",
+                        severity=severity,
+                        step_ids=[write.step_id],
+                        details={
+                            "corrupted_fields": [
+                                {
+                                    "path": c.path,
+                                    "lost_value": c.old_value,
+                                }
+                                for c in corrupted_fields
+                            ],
+                        },
+                    )
+                )
+
+        return evidence

context_forge/graders/hybrid.py

@@ -0,0 +1,190 @@
+"""Hybrid graders that combine deterministic and LLM-based evaluation.
+
+Hybrid graders leverage the strengths of both approaches:
+- Deterministic: Fast, cheap, catches invariant violations (corruption)
+- LLM Judge: Semantic understanding, catches meaning-related issues
+"""
+
+from typing import Optional
+
+from context_forge.core.trace import TraceRun
+from context_forge.graders.base import Evidence, Grader, GraderResult, Severity
+from context_forge.graders.deterministic.memory_corruption import MemoryCorruptionGrader
+from context_forge.graders.judges.base import LLMBackend
+from context_forge.graders.judges.memory_hygiene_judge import MemoryHygieneJudge
+
+
+class HybridMemoryHygieneGrader(Grader):
+    """Hybrid grader combining corruption detection and semantic evaluation.
+
+    Layer 1 - Deterministic (MemoryCorruptionGrader):
+    Checks for INVARIANTS that are always wrong:
+    - Data corruption: Existing data deleted without replacement
+
+    Layer 2 - LLM Judge (MemoryHygieneJudge):
+    Checks for SEMANTIC issues requiring understanding:
+    - Missed facts: User stated something, agent didn't save it
+    - Hallucinations: Agent saved something user never said
+    - Contradictions: Saved data conflicts with user statements
+
+    The deterministic layer catches hard failures (corruption).
+    The LLM layer catches semantic failures (wrong understanding).
+
+    Usage:
+        # With LLM (recommended - full semantic analysis)
+        from context_forge.graders.judges.backends import OllamaBackend
+
+        grader = HybridMemoryHygieneGrader(
+            llm_backend=OllamaBackend(model="llama3.2")
+        )
+        result = grader.grade(trace)
+
+        # Without LLM (only corruption detection)
+        grader = HybridMemoryHygieneGrader()
+        result = grader.grade(trace)  # Only checks for data corruption
+    """
+
+    name = "hybrid_memory_hygiene"
+    deterministic = False  # Because LLM layer is non-deterministic
+
+    def __init__(
+        self,
+        llm_backend: Optional[LLMBackend] = None,
+        skip_llm_on_corruption: bool = True,
+        llm_temperature: float = 0.0,
+    ):
+        """Initialize the hybrid grader.
+
+        Args:
+            llm_backend: Optional LLM backend for semantic checks.
+                         If None, only corruption detection runs.
+            skip_llm_on_corruption: If True, skip LLM when corruption
+                         is detected (saves tokens, corruption is fatal).
+            llm_temperature: Temperature for LLM calls (0.0 recommended).
+        """
+        self.llm_backend = llm_backend
+        self.skip_llm_on_corruption = skip_llm_on_corruption
+
+        # Layer 1: Corruption detection (invariants)
+        self.corruption_grader = MemoryCorruptionGrader()
+
+        # Layer 2: Semantic evaluation (understanding)
+        self.llm_judge: Optional[MemoryHygieneJudge] = None
+        if llm_backend:
+            self.llm_judge = MemoryHygieneJudge(
+                backend=llm_backend,
+                temperature=llm_temperature,
+            )
+
+    def grade(self, trace: TraceRun) -> GraderResult:
+        """Run hybrid evaluation on a trace.
+
+        1. Run corruption detection (always)
+        2. Run LLM judge (if configured and no corruption found)
+        3. Combine results
+
+        Args:
+            trace: The trace to evaluate
+
+        Returns:
+            Combined GraderResult from both layers
+        """
+        all_evidence: list[Evidence] = []
+
+        # Layer 1: Corruption detection
+        corruption_result = self.corruption_grader.grade(trace)
+        all_evidence.extend(corruption_result.evidence)
+
+        # Add layer marker
+        all_evidence.append(
+            Evidence(
+                check_name="layer_1_complete",
+                description=f"Corruption check: {'PASSED' if corruption_result.passed else 'FAILED'} (score: {corruption_result.score:.2f})",
+                severity=Severity.INFO,
+            )
+        )
+
+        # Layer 2: LLM Semantic Judge (if configured)
+        llm_result: Optional[GraderResult] = None
+
+        if self.llm_judge:
+            # Skip LLM if corruption detected (corruption is fatal)
+            if self.skip_llm_on_corruption and not corruption_result.passed:
+                all_evidence.append(
+                    Evidence(
+                        check_name="layer_2_skipped",
+                        description="Semantic evaluation skipped: data corruption detected",
+                        severity=Severity.INFO,
+                    )
+                )
+            else:
+                try:
+                    llm_result = self.llm_judge.grade(trace)
+                    all_evidence.extend(llm_result.evidence)
+
+                    all_evidence.append(
+                        Evidence(
+                            check_name="layer_2_complete",
+                            description=f"Semantic evaluation: {'PASSED' if llm_result.passed else 'FAILED'} (score: {llm_result.score:.2f})",
+                            severity=Severity.INFO,
+                        )
+                    )
+                except Exception as e:
+                    all_evidence.append(
+                        Evidence(
+                            check_name="layer_2_error",
+                            description=f"Semantic evaluation failed: {e}",
+                            severity=Severity.WARN,
+                        )
+                    )
+
+        # Combine results
+        combined_result = self._combine_results(
+            corruption_result, llm_result, all_evidence
+        )
+        return combined_result
+
+    def _combine_results(
+        self,
+        corruption_result: GraderResult,
+        llm_result: Optional[GraderResult],
+        all_evidence: list[Evidence],
+    ) -> GraderResult:
+        """Combine corruption and semantic results into final result.
+
+        Scoring:
+        - If LLM ran: average of both scores
+        - If LLM didn't run: corruption score only
+
+        Passing:
+        - Must pass BOTH layers to pass overall
+        - Corruption failure is fatal (always fails)
+        """
+        if llm_result:
+            # Both layers ran - combine
+            combined_score = (corruption_result.score + llm_result.score) / 2
+            combined_passed = corruption_result.passed and llm_result.passed
+
+            # Merge metadata
+            metadata = {
+                "corruption": corruption_result.metadata,
+                "semantic": llm_result.metadata,
+                "layers_run": ["corruption", "semantic"],
+            }
+        else:
+            # Only corruption check ran
+            combined_score = corruption_result.score
+            combined_passed = corruption_result.passed
+
+            metadata = {
+                "corruption": corruption_result.metadata,
+                "layers_run": ["corruption"],
+            }
+
+        return GraderResult(
+            grader_name=self.name,
+            passed=combined_passed,
+            score=combined_score,
+            evidence=all_evidence,
+            metadata=metadata,
+        )

context_forge/graders/judges/__init__.py

@@ -0,0 +1,11 @@
+"""LLM-based judges for ContextForge.
+
+These judges use language models to evaluate semantic aspects of traces
+that rule-based graders cannot assess. They include full reproducibility
+metadata (prompt, response, model parameters).
+"""
+
+from context_forge.graders.judges.base import LLMJudge
+from context_forge.graders.judges.memory_hygiene_judge import MemoryHygieneJudge
+
+__all__ = ["LLMJudge", "MemoryHygieneJudge"]

context_forge/graders/judges/backends/__init__.py

@@ -0,0 +1,9 @@
+"""LLM backends for ContextForge judges.
+
+Backends provide the connection to LLM providers. Ollama is the
+primary/default backend for local execution.
+"""
+
+from context_forge.graders.judges.backends.ollama import OllamaBackend
+
+__all__ = ["OllamaBackend"]

context_forge/graders/judges/backends/ollama.py

@@ -0,0 +1,173 @@
+"""Ollama backend for LLM judges.
+
+Provides local LLM execution via Ollama. This is the primary backend
+for ContextForge, enabling evaluation without sending data to cloud APIs.
+
+Supports structured output via JSON schema - pass a Pydantic model
+to get validated, typed responses.
+
+Uses the official Ollama Python SDK for cleaner, more maintainable code.
+"""
+
+import logging
+from typing import TypeVar
+
+import ollama
+from pydantic import BaseModel
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class OllamaBackend:
+    """Ollama backend for local LLM execution with structured output support.
+
+    Usage:
+        backend = OllamaBackend(model="llama3.2")
+
+        # Basic completion (returns string)
+        response = backend.complete("Evaluate this trace...")
+
+        # Structured output with Pydantic model (returns validated object)
+        result = backend.complete_structured(
+            prompt="Evaluate this trace...",
+            response_model=MemoryHygieneEvaluation,
+        )
+
+    Requires Ollama to be running at localhost:11434 (default).
+    """
+
+    def __init__(
+        self,
+        model: str = "llama3.2",
+        host: str = "http://localhost:11434",
+        timeout: float = 120.0,
+    ):
+        """Initialize the Ollama backend.
+
+        Args:
+            model: Ollama model to use (e.g., "llama3.2", "mistral")
+            host: Ollama host URL
+            timeout: Request timeout in seconds
+        """
+        self.model = model
+        self.host = host
+        self.timeout = timeout
+        self._client = ollama.Client(host=host, timeout=timeout)
+
+    @property
+    def model_id(self) -> str:
+        """The model identifier."""
+        return f"ollama/{self.model}"
+
+    def complete(
+        self,
+        prompt: str,
+        temperature: float = 0.0,
+        json_mode: bool = False,
+    ) -> str:
+        """Generate a completion using Ollama.
+
+        Args:
+            prompt: The prompt to complete
+            temperature: Sampling temperature (0.0 for deterministic)
+            json_mode: If True, enforce JSON output format
+
+        Returns:
+            The model's response text
+
+        Raises:
+            ollama.ResponseError: If the request fails
+            ValueError: If Ollama is not running
+        """
+        try:
+            response = self._client.generate(
+                model=self.model,
+                prompt=prompt,
+                format="json" if json_mode else None,
+                options={"temperature": temperature},
+            )
+            return response.get("response", "")
+
+        except ollama.ResponseError as e:
+            logger.error(f"Ollama request failed: {e}")
+            raise
+
+        except Exception as e:
+            if "connection" in str(e).lower() or "refused" in str(e).lower():
+                logger.error(f"Failed to connect to Ollama at {self.host}: {e}")
+                raise ValueError(
+                    f"Cannot connect to Ollama at {self.host}. "
+                    "Is Ollama running? Start it with: ollama serve"
+                ) from e
+            raise
+
+    def complete_structured(
+        self,
+        prompt: str,
+        response_model: type[T],
+        temperature: float = 0.0,
+    ) -> T:
+        """Generate a structured completion with Pydantic validation.
+
+        Uses Ollama's structured output feature to enforce a JSON schema,
+        then validates with Pydantic for type safety.
+
+        Args:
+            prompt: The prompt to complete
+            response_model: Pydantic model class for the response
+            temperature: Sampling temperature (0.0 for deterministic)
+
+        Returns:
+            Validated Pydantic model instance
+
+        Raises:
+            ollama.ResponseError: If the request fails
+            pydantic.ValidationError: If response doesn't match schema
+            ValueError: If Ollama connection fails
+        """
+        # Get JSON schema from Pydantic model
+        schema = response_model.model_json_schema()
+
+        try:
+            response = self._client.generate(
+                model=self.model,
+                prompt=prompt,
+                format=schema,  # Ollama enforces this schema
+                options={"temperature": temperature},
+            )
+            response_text = response.get("response", "")
+
+            # Parse and validate with Pydantic
+            return response_model.model_validate_json(response_text)
+
+        except ollama.ResponseError as e:
+            logger.error(f"Ollama request failed: {e}")
+            raise
+
+        except Exception as e:
+            if "connection" in str(e).lower() or "refused" in str(e).lower():
+                logger.error(f"Failed to connect to Ollama at {self.host}: {e}")
+                raise ValueError(
+                    f"Cannot connect to Ollama at {self.host}. "
+                    "Is Ollama running? Start it with: ollama serve"
+                ) from e
+            raise
+
+    def is_available(self) -> bool:
+        """Check if Ollama is running and the model is available.
+
+        Returns:
+            True if Ollama is accessible and model is pulled
+        """
+        try:
+            response = self._client.list()
+            # SDK returns ListResponse with .models attribute containing Model objects
+            model_names = [m.model.split(":")[0] for m in response.models]
+            return self.model.split(":")[0] in model_names
+        except Exception:
+            return False
+
+    def __repr__(self) -> str:
+        return f"OllamaBackend(model={self.model!r}, host={self.host!r})"

context_forge/graders/judges/base.py

@@ -0,0 +1,158 @@
+"""Base class for LLM-based judges.
+
+LLM judges evaluate semantic aspects of traces that require
+natural language understanding. They are marked as non-deterministic
+and include full reproducibility metadata.
+
+Supports structured output with Pydantic models for reliable parsing.
+"""
+
+from abc import abstractmethod
+from typing import Any, Optional, Protocol, TypeVar
+
+from pydantic import BaseModel
+
+from context_forge.core.trace import TraceRun
+from context_forge.graders.base import Grader, GraderResult
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class LLMBackend(Protocol):
+    """Protocol for LLM backends (Ollama, OpenAI, etc.)."""
+
+    @property
+    def model_id(self) -> str:
+        """The model identifier."""
+        ...
+
+    def complete(self, prompt: str, temperature: float = 0.0) -> str:
+        """Generate a completion for the given prompt.
+
+        Args:
+            prompt: The prompt to complete
+            temperature: Sampling temperature (0.0 for deterministic)
+
+        Returns:
+            The model's response text
+        """
+        ...
+
+    def complete_structured(
+        self,
+        prompt: str,
+        response_model: type[T],
+        temperature: float = 0.0,
+    ) -> T:
+        """Generate a structured completion with Pydantic validation.
+
+        Args:
+            prompt: The prompt to complete
+            response_model: Pydantic model class for the response
+            temperature: Sampling temperature (0.0 for deterministic)
+
+        Returns:
+            Validated Pydantic model instance
+        """
+        ...
+
+
+class LLMJudge(Grader):
+    """Base class for LLM-based judges.
+
+    LLM judges use language models to evaluate aspects of traces that
+    require semantic understanding. Unlike deterministic graders, they:
+    - Are marked as non-deterministic
+    - Include full reproducibility metadata (prompt, response, model)
+    - Support configurable backends (Ollama, OpenAI, etc.)
+
+    Subclasses must implement:
+    - _build_prompt(): Construct the evaluation prompt
+    - _parse_response(): Parse the LLM response into a GraderResult
+
+    Usage:
+        class MyJudge(LLMJudge):
+            def _build_prompt(self, trace):
+                return f"Evaluate this trace: {trace}"
+
+            def _parse_response(self, response, trace):
+                return GraderResult(...)
+
+        judge = MyJudge(backend=OllamaBackend(model="llama3.2"))
+        result = judge.grade(trace)
+    """
+
+    name = "llm_judge"
+    deterministic = False  # LLM outputs can vary
+
+    def __init__(
+        self,
+        backend: LLMBackend,
+        temperature: float = 0.0,
+    ):
+        """Initialize the judge with an LLM backend.
+
+        Args:
+            backend: LLM backend to use for evaluation
+            temperature: Sampling temperature (default 0.0 for consistency)
+        """
+        self.backend = backend
+        self.temperature = temperature
+
+    def grade(self, trace: TraceRun) -> GraderResult:
+        """Evaluate a trace using the LLM.
+
+        Args:
+            trace: The trace to evaluate
+
+        Returns:
+            GraderResult with LLM evaluation and reproducibility metadata
+        """
+        # Build prompt
+        prompt = self._build_prompt(trace)
+
+        # Call LLM
+        response = self.backend.complete(prompt, temperature=self.temperature)
+
+        # Parse response into result
+        result = self._parse_response(response, trace)
+
+        # Add reproducibility metadata
+        if result.metadata is None:
+            result.metadata = {}
+
+        result.metadata.update({
+            "llm": {
+                "model_id": self.backend.model_id,
+                "temperature": self.temperature,
+                "prompt": prompt,
+                "raw_response": response,
+            }
+        })
+
+        return result
+
+    @abstractmethod
+    def _build_prompt(self, trace: TraceRun) -> str:
+        """Construct the evaluation prompt for the LLM.
+
+        Args:
+            trace: The trace to evaluate
+
+        Returns:
+            The prompt string to send to the LLM
+        """
+        pass
+
+    @abstractmethod
+    def _parse_response(self, response: str, trace: TraceRun) -> GraderResult:
+        """Parse the LLM response into a GraderResult.
+
+        Args:
+            response: Raw LLM response text
+            trace: The original trace (for context)
+
+        Returns:
+            GraderResult with parsed findings
+        """
+        pass