contextforge-eval 0.1.0__py3-none-any.whl

context_forge/evaluation.py

@@ -0,0 +1,267 @@
+"""Simple evaluation API for ContextForge.
+
+This module provides high-level functions for common evaluation patterns,
+hiding the complexity of instrumentors, adapters, and graders.
+
+Two usage levels:
+
+Level 2 (Simple): Single-turn evaluation with minimal setup
+    from context_forge.evaluation import evaluate_agent
+
+    result = evaluate_agent(
+        graph=my_graph,
+        message="I work from home now",
+        store=my_store,
+    )
+    result.print_report()
+
+Level 3 (Simulation): Multi-turn with personas and scenarios
+    from context_forge import SimulationRunner, LangGraphAdapter, Persona
+    # ... full control over simulation
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+from context_forge.core.trace import TraceRun
+from context_forge.graders import GraderResult, HybridMemoryHygieneGrader
+from context_forge.graders.base import Evidence
+from context_forge.instrumentation import LangGraphInstrumentor
+
+
+@dataclass
+class EvaluationResult:
+    """Result from a simple evaluation run.
+
+    Combines the agent's response with grader results for easy inspection.
+
+    Attributes:
+        response: The agent's final response
+        trace: The captured trace (for debugging)
+        grader_results: Results from each grader that was run
+        passed: True if all graders passed
+    """
+    response: Any
+    trace: TraceRun
+    grader_results: list[GraderResult] = field(default_factory=list)
+
+    @property
+    def passed(self) -> bool:
+        """True if all graders passed."""
+        return all(r.passed for r in self.grader_results)
+
+    @property
+    def score(self) -> float:
+        """Average score across all graders."""
+        if not self.grader_results:
+            return 1.0
+        return sum(r.score for r in self.grader_results) / len(self.grader_results)
+
+    @property
+    def errors(self) -> list[Evidence]:
+        """All errors from all graders."""
+        errors = []
+        for r in self.grader_results:
+            errors.extend(r.errors)
+        return errors
+
+    def print_report(self, verbose: bool = False) -> None:
+        """Print a combined report of all grader results."""
+        print("\n" + "=" * 60)
+        print("EVALUATION REPORT")
+        print("=" * 60)
+
+        status = "PASSED" if self.passed else "FAILED"
+        print(f"\nOverall: {status} (score: {self.score:.2f})")
+        print(f"Response: {str(self.response)[:100]}...")
+
+        for result in self.grader_results:
+            result.print_report(verbose=verbose)
+
+
+def evaluate_agent(
+    graph,
+    message: str,
+    store=None,
+    user_id: str = "eval_user",
+    session_id: str = "eval_session",
+    graders: Optional[list[str]] = None,
+    llm_model: str = "llama3.2",
+    print_result: bool = True,
+) -> EvaluationResult:
+    """Evaluate a LangGraph agent with a single message.
+
+    This is the simplest way to evaluate your agent. It:
+    1. Instruments the agent to capture traces
+    2. Runs your message through the agent
+    3. Grades the trace with specified graders
+    4. Returns a combined result
+
+    Args:
+        graph: Your compiled LangGraph graph
+        message: The user message to send
+        store: Optional LangGraph store (for memory operations)
+        user_id: User ID for the session
+        session_id: Session ID for the conversation
+        graders: List of grader names to run. Default: ["memory_hygiene"]
+                 Available: "memory_hygiene", "memory_corruption"
+        llm_model: Ollama model for LLM-based graders
+        print_result: Whether to print the report automatically
+
+    Returns:
+        EvaluationResult with response, trace, and grader results
+
+    Example:
+        from context_forge.evaluation import evaluate_agent
+        from langgraph.store.memory import InMemoryStore
+        from my_agent import build_graph
+
+        store = InMemoryStore()
+        # ... populate store with user profile ...
+
+        graph = build_graph(store=store)
+        result = evaluate_agent(
+            graph=graph,
+            message="I switched to working from home",
+            store=store,
+        )
+
+        if not result.passed:
+            print("Agent failed evaluation!")
+            for error in result.errors:
+                print(f"  - {error.description}")
+    """
+    graders = graders or ["memory_hygiene"]
+
+    # Set up instrumentation
+    instrumentor = LangGraphInstrumentor(
+        agent_name="evaluated_agent",
+        agent_version="1.0.0",
+    )
+    instrumentor.instrument()
+
+    try:
+        # Build initial state
+        initial_state = {
+            "user_id": user_id,
+            "session_id": session_id,
+            "message": message,
+            "messages": [],
+            "turn_count": 0,
+            "user_profile": None,  # Will be loaded from store
+            "response": None,
+        }
+
+        # Add store config if provided
+        config = {}
+        if store is not None:
+            config["configurable"] = {"store": store}
+
+        # Run the agent
+        result = graph.invoke(initial_state, config=config)
+        response = result.get("response", result)
+
+        # Get the trace
+        traces = instrumentor.get_traces()
+        if not traces:
+            raise RuntimeError("No trace captured. Is the graph using LangChain components?")
+        trace = traces[0]
+
+        # Run graders
+        grader_results = []
+        for grader_name in graders:
+            grader_result = _run_grader(grader_name, trace, llm_model)
+            grader_results.append(grader_result)
+
+        # Build result
+        eval_result = EvaluationResult(
+            response=response,
+            trace=trace,
+            grader_results=grader_results,
+        )
+
+        if print_result:
+            eval_result.print_report()
+
+        return eval_result
+
+    finally:
+        instrumentor.uninstrument()
+
+
+def evaluate_trace(
+    trace: TraceRun,
+    graders: Optional[list[str]] = None,
+    llm_model: str = "llama3.2",
+    print_result: bool = True,
+) -> EvaluationResult:
+    """Evaluate an existing trace.
+
+    Use this when you already have a trace (e.g., loaded from a file)
+    and just want to run graders on it.
+
+    Args:
+        trace: The trace to evaluate
+        graders: List of grader names to run
+        llm_model: Ollama model for LLM-based graders
+        print_result: Whether to print the report automatically
+
+    Returns:
+        EvaluationResult with grader results
+
+    Example:
+        from context_forge.evaluation import evaluate_trace
+        from context_forge import TraceRun
+        import json
+
+        with open("my_trace.json") as f:
+            trace = TraceRun.model_validate(json.load(f))
+
+        result = evaluate_trace(trace)
+    """
+    graders = graders or ["memory_hygiene"]
+
+    grader_results = []
+    for grader_name in graders:
+        grader_result = _run_grader(grader_name, trace, llm_model)
+        grader_results.append(grader_result)
+
+    eval_result = EvaluationResult(
+        response=None,
+        trace=trace,
+        grader_results=grader_results,
+    )
+
+    if print_result:
+        eval_result.print_report()
+
+    return eval_result
+
+
+def _run_grader(grader_name: str, trace: TraceRun, llm_model: str) -> GraderResult:
+    """Run a grader by name."""
+    from context_forge.graders import MemoryCorruptionGrader
+    from context_forge.graders.judges.backends import OllamaBackend
+
+    if grader_name == "memory_hygiene":
+        # Check if Ollama is available
+        try:
+            backend = OllamaBackend(model=llm_model)
+            if backend.is_available():
+                grader = HybridMemoryHygieneGrader(llm_backend=backend)
+            else:
+                # Fall back to deterministic only
+                grader = HybridMemoryHygieneGrader()
+        except Exception:
+            grader = HybridMemoryHygieneGrader()
+        return grader.grade(trace)
+
+    elif grader_name == "memory_corruption":
+        grader = MemoryCorruptionGrader()
+        return grader.grade(trace)
+
+    else:
+        raise ValueError(
+            f"Unknown grader: {grader_name}. "
+            f"Available: memory_hygiene, memory_corruption"
+        )

context_forge/exceptions.py

@@ -0,0 +1,56 @@
+"""Custom exceptions for ContextForge.
+
+This module implements T025: Custom exceptions.
+"""
+
+
+class ContextForgeError(Exception):
+    """Base exception for all ContextForge errors."""
+
+    pass
+
+
+class TraceValidationError(ContextForgeError):
+    """Raised when trace validation fails."""
+
+    def __init__(self, message: str, field: str | None = None):
+        self.field = field
+        super().__init__(message)
+
+
+class InstrumentationError(ContextForgeError):
+    """Raised when instrumentation fails."""
+
+    pass
+
+
+class InstrumentorAlreadyActiveError(InstrumentationError):
+    """Raised when trying to instrument when already instrumented."""
+
+    pass
+
+
+class InstrumentorNotActiveError(InstrumentationError):
+    """Raised when trying to uninstrument when not instrumented."""
+
+    pass
+
+
+class SpanConversionError(ContextForgeError):
+    """Raised when span conversion fails."""
+
+    def __init__(self, message: str, span_id: str | None = None):
+        self.span_id = span_id
+        super().__init__(message)
+
+
+class TracerError(ContextForgeError):
+    """Raised when tracer operations fail."""
+
+    pass
+
+
+class TracerNotActiveError(TracerError):
+    """Raised when trying to record steps without an active tracer."""
+
+    pass

context_forge/graders/__init__.py

@@ -0,0 +1,44 @@
+"""ContextForge Graders - Evaluate agent trajectories.
+
+Graders analyze traces to detect behavioral issues that output-only
+evaluation would miss.
+
+Two types of evaluation:
+- Deterministic (MemoryCorruptionGrader): Checks INVARIANTS that are always wrong
+- LLM Judges (MemoryHygieneJudge): SEMANTIC evaluation requiring understanding
+- Hybrid: Combines both for comprehensive analysis
+
+Usage:
+    from context_forge.graders import HybridMemoryHygieneGrader
+    from context_forge.graders.judges.backends import OllamaBackend
+
+    # Full evaluation (recommended)
+    grader = HybridMemoryHygieneGrader(
+        llm_backend=OllamaBackend(model="llama3.2")
+    )
+    result = grader.grade(trace)
+
+    if not result.passed:
+        for error in result.errors:
+            print(f"Issue: {error.description}")
+"""
+
+from context_forge.graders.base import Evidence, Grader, GraderResult, Severity
+from context_forge.graders.deterministic import MemoryCorruptionGrader
+from context_forge.graders.hybrid import HybridMemoryHygieneGrader
+from context_forge.graders.judges import LLMJudge, MemoryHygieneJudge
+
+__all__ = [
+    # Base classes
+    "Grader",
+    "GraderResult",
+    "Evidence",
+    "Severity",
+    # Deterministic graders (invariant checks)
+    "MemoryCorruptionGrader",
+    # LLM judges (semantic evaluation)
+    "LLMJudge",
+    "MemoryHygieneJudge",
+    # Hybrid graders (recommended)
+    "HybridMemoryHygieneGrader",
+]

context_forge/graders/base.py

@@ -0,0 +1,264 @@
+"""Base classes for ContextForge graders.
+
+This module implements the core grader interface that all graders
+(deterministic and LLM judges) must implement.
+
+Key design principles:
+- Graders operate ONLY on traces, never on framework objects
+- Results include evidence with step_ids for traceability
+- Deterministic graders are stateless and reproducible
+- LLM judges include full reproducibility metadata
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Optional
+
+from context_forge.core.trace import TraceRun
+
+
+class Severity(str, Enum):
+    """Severity level for evidence items."""
+
+    INFO = "info"       # Informational, not a problem
+    WARN = "warn"       # Potential issue, doesn't fail the grader
+    ERROR = "error"     # Definite issue, fails the grader
+
+
+@dataclass
+class Evidence:
+    """Proof of what was evaluated by a grader.
+
+    Every grader result must include evidence explaining what was
+    checked and why the result was pass/fail.
+
+    Attributes:
+        check_name: Name of the specific check (e.g., "redundant_write")
+        description: Human-readable explanation of the finding
+        severity: How serious this finding is
+        step_ids: Which trace steps were examined
+        details: Additional structured data about the finding
+    """
+
+    check_name: str
+    description: str
+    severity: Severity = Severity.INFO
+    step_ids: list[str] = field(default_factory=list)
+    details: Optional[dict[str, Any]] = None
+
+
+@dataclass
+class GraderResult:
+    """Result from any grader (deterministic or LLM judge).
+
+    Attributes:
+        grader_name: Name of the grader that produced this result
+        passed: Whether the trace passed all checks
+        score: Numeric score from 0.0 (worst) to 1.0 (best)
+        evidence: List of evidence items explaining the result
+        timestamp: When the grading was performed
+        metadata: Additional grader-specific data (LLM judges add prompt/response)
+    """
+
+    grader_name: str
+    passed: bool
+    score: float
+    evidence: list[Evidence] = field(default_factory=list)
+    timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    metadata: Optional[dict[str, Any]] = None
+
+    def __post_init__(self):
+        """Validate score is in valid range."""
+        if not 0.0 <= self.score <= 1.0:
+            raise ValueError(f"Score must be between 0.0 and 1.0, got {self.score}")
+
+    @property
+    def errors(self) -> list[Evidence]:
+        """Get all evidence items with ERROR severity."""
+        return [e for e in self.evidence if e.severity == Severity.ERROR]
+
+    @property
+    def warnings(self) -> list[Evidence]:
+        """Get all evidence items with WARN severity."""
+        return [e for e in self.evidence if e.severity == Severity.WARN]
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert result to dictionary for serialization."""
+        return {
+            "grader_name": self.grader_name,
+            "passed": self.passed,
+            "score": self.score,
+            "evidence": [
+                {
+                    "check_name": e.check_name,
+                    "description": e.description,
+                    "severity": e.severity.value,
+                    "step_ids": e.step_ids,
+                    "details": e.details,
+                }
+                for e in self.evidence
+            ],
+            "timestamp": self.timestamp.isoformat(),
+            "metadata": self.metadata,
+        }
+
+    def format_report(self, verbose: bool = False) -> str:
+        """Format the grader result as a human-readable report.
+
+        Args:
+            verbose: Include additional details and metadata
+
+        Returns:
+            Formatted string report
+        """
+        lines = []
+
+        # Header
+        lines.append("")
+        lines.append("=" * 60)
+        lines.append(f"GRADER REPORT: {self.grader_name}")
+        lines.append("=" * 60)
+
+        # Result summary
+        status = "PASSED" if self.passed else "FAILED"
+        status_icon = "[OK]" if self.passed else "[FAIL]"
+        lines.append("")
+        lines.append(f"Result: {status_icon} {status}")
+        lines.append(f"Score:  {self.score:.2f} / 1.00")
+
+        # Errors (always show)
+        errors = self.errors
+        if errors:
+            lines.append("")
+            lines.append(f"ERRORS ({len(errors)}):")
+            for e in errors:
+                lines.append(f"  [ERROR] {e.check_name}")
+                lines.append(f"          {e.description}")
+                if verbose and e.details:
+                    for k, v in e.details.items():
+                        lines.append(f"          {k}: {v}")
+
+        # Warnings (always show)
+        warnings = self.warnings
+        if warnings:
+            lines.append("")
+            lines.append(f"WARNINGS ({len(warnings)}):")
+            for e in warnings:
+                lines.append(f"  [WARN]  {e.check_name}")
+                lines.append(f"          {e.description}")
+
+        # Info items (show summary only, or all if verbose)
+        info_items = [e for e in self.evidence if e.severity == Severity.INFO]
+        if info_items:
+            # Always show the summary if present
+            summary = next(
+                (e for e in info_items if e.check_name == "llm_summary"),
+                None,
+            )
+            if summary:
+                lines.append("")
+                lines.append("SUMMARY:")
+                lines.append(f"  {summary.description}")
+
+            # Show correct saves
+            correct_saves = [e for e in info_items if e.check_name == "correct_save"]
+            if correct_saves:
+                lines.append("")
+                lines.append(f"CORRECTLY SAVED ({len(correct_saves)}):")
+                for e in correct_saves:
+                    lines.append(f"  [OK] {e.description}")
+
+            # Verbose: show all info items
+            if verbose:
+                other_info = [
+                    e for e in info_items
+                    if e.check_name not in ("llm_summary", "correct_save")
+                ]
+                if other_info:
+                    lines.append("")
+                    lines.append("ADDITIONAL INFO:")
+                    for e in other_info:
+                        lines.append(f"  [{e.check_name}] {e.description}")
+
+        lines.append("")
+        lines.append("-" * 60)
+
+        return "\n".join(lines)
+
+    def print_report(self, verbose: bool = False) -> None:
+        """Print the grader result as a human-readable report.
+
+        Args:
+            verbose: Include additional details and metadata
+        """
+        print(self.format_report(verbose=verbose))
+
+    def __str__(self) -> str:
+        """Short string representation."""
+        status = "PASSED" if self.passed else "FAILED"
+        return f"GraderResult({self.grader_name}: {status}, score={self.score:.2f})"
+
+
+class Grader(ABC):
+    """Base class for all graders.
+
+    Graders evaluate traces and return structured results with evidence.
+    All graders must implement the `grade` method.
+
+    Attributes:
+        name: Human-readable name for this grader
+        deterministic: Whether this grader produces identical results on repeated runs
+        required_step_types: Step types this grader needs to function
+    """
+
+    name: str = "base_grader"
+    deterministic: bool = True
+    required_step_types: list[str] = []
+
+    @abstractmethod
+    def grade(self, trace: TraceRun) -> GraderResult:
+        """Evaluate a trace and return a grading result.
+
+        Args:
+            trace: The trace to evaluate
+
+        Returns:
+            GraderResult with pass/fail, score, and evidence
+        """
+        pass
+
+    def validate_trace(self, trace: TraceRun) -> list[str]:
+        """Check if trace has required step types.
+
+        Args:
+            trace: The trace to validate
+
+        Returns:
+            List of missing step types (empty if all present)
+        """
+        present_types = {step.step_type for step in trace.steps}
+        missing = [st for st in self.required_step_types if st not in present_types]
+        return missing
+
+    def check_required_steps(self, trace: TraceRun) -> None:
+        """Validate trace has required step types, raising if not.
+
+        Call this at the start of grade() to fail fast on invalid traces.
+
+        Args:
+            trace: The trace to validate
+
+        Raises:
+            ValueError: If required step types are missing
+        """
+        missing = self.validate_trace(trace)
+        if missing:
+            raise ValueError(
+                f"Grader '{self.name}' requires step types {self.required_step_types}, "
+                f"but trace is missing: {missing}"
+            )
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(name={self.name!r}, deterministic={self.deterministic})"

context_forge/graders/deterministic/__init__.py

@@ -0,0 +1,11 @@
+"""Deterministic graders for ContextForge.
+
+These graders check for INVARIANTS - things that are always wrong
+regardless of the agent's non-deterministic path.
+
+For semantic evaluation (understanding), use LLM judges instead.
+"""
+
+from context_forge.graders.deterministic.memory_corruption import MemoryCorruptionGrader
+
+__all__ = ["MemoryCorruptionGrader"]