selfevals 0.2.2__py3-none-any.whl

selfevals/examples/evals/experiments/example_pingpong.yaml

@@ -0,0 +1,58 @@
+# Example experiment spec.
+#
+# Copy it with:
+#   selfevals examples copy pingpong
+# Then run:
+#   selfevals run evals/experiments/example_pingpong.yaml --no-persist
+
+workspace: ws_01HZZZZZZZZZZZZZZZZZZZZZZZ
+
+experiment:
+  name: pingpong baseline
+  goal: warm up the end-to-end loop with a trivial echo agent
+  mode: handoff
+  taxonomy:
+    target_features:
+      - commerce.product_resolution
+    dataset_types:
+      - capability
+  datasets:
+    optimization: { id: ds_pingpong, version: 1 }
+  target:
+    primary: { name: pass@1, operator: ">=", value: 0.5 }
+  editable:
+    prompt: true
+    model_params: true
+  frozen:
+    fleet: { id: flt_demo }
+    agents:
+      - { id: ag_demo }
+    datasets:
+      - { id: ds_pingpong }
+  proposer:
+    strategy: grid
+  search_space:
+    model_params:
+      level: [0.0, 1.0]
+  run:
+    sandbox: mock
+    max_iterations: 4
+    convergence:
+      min_delta: 1.0e-6
+      patience: 10
+  reliability:
+    metrics:
+      - pass@1
+  error_analysis:
+    enabled: true
+    taxonomy: workspace
+    trigger:
+      when: fail_rate_above
+      threshold: 0.10
+    scope: failed_only
+
+dataset:
+  cases_path: ../datasets/pingpong.jsonl
+
+agent:
+  entrypoint: selfevals.examples.pingpong:run

selfevals/examples/pingpong.py

@@ -0,0 +1,21 @@
+"""Trivial example agent for the pingpong experiment.
+
+It returns "pong" when the proposer cranks `model_params.level >= 0.5`,
+and "miss" otherwise. That lets the grid proposer demonstrate a real
+improvement path from level=0.0 (fail) to level=1.0 (pass).
+
+Real agents will replace this with a function that calls Anthropic /
+OpenAI / their framework of choice. The contract is the same:
+
+    def run(req: AdapterRequest) -> AdapterResponse | str: ...
+"""
+
+from __future__ import annotations
+
+from selfevals.runner.adapters import AdapterRequest, AdapterResponse
+
+
+def run(req: AdapterRequest) -> AdapterResponse:
+    level = req.parameters.get("model_params", {}).get("level", 0.0)
+    content = "pong" if level >= 0.5 else "miss"
+    return AdapterResponse(content=content, tokens_input=4, tokens_output=2)

selfevals/graders/__init__.py

@@ -0,0 +1,46 @@
+"""Graders: score traces against expectations.
+
+A `Grader` reads a Trace + EvalCase and returns a `GradeResult` (label
++ optional score + reason). Two concrete graders ship in MVP:
+
+- `DeterministicGrader` evaluates rule-based expectations declared on
+  `EvalCase.expected` (must_include / forbidden_tools / regex / schema).
+- `LLMJudgeGrader` invokes an `AgentAdapter` as a judge against a rubric
+  prompt; single-judge in MVP, panel infrastructure-ready for post-MVP.
+
+Calibration helpers turn observed predictions + human annotations into
+the metrics tracked on a `GraderCard`.
+"""
+
+from selfevals.graders.base import GradeLabel, Grader, GraderContext, GradeResult
+from selfevals.graders.calibration import (
+    CalibrationReport,
+    HumanLabel,
+    PredictedLabel,
+    compute_classification_metrics,
+)
+from selfevals.graders.deterministic import (
+    DeterministicGrader,
+    DeterministicRuleViolationError,
+)
+from selfevals.graders.llm_judge import (
+    JudgeDecision,
+    LLMJudgeGrader,
+    RubricTemplate,
+)
+
+__all__ = [
+    "CalibrationReport",
+    "DeterministicGrader",
+    "DeterministicRuleViolationError",
+    "GradeLabel",
+    "GradeResult",
+    "Grader",
+    "GraderContext",
+    "HumanLabel",
+    "JudgeDecision",
+    "LLMJudgeGrader",
+    "PredictedLabel",
+    "RubricTemplate",
+    "compute_classification_metrics",
+]

selfevals/graders/base.py

@@ -0,0 +1,54 @@
+"""Grader ABC + shared result types.
+
+The grader contract is intentionally narrow: receive a context bundle
+(case, trace, optional response) and return a `GradeResult`. The case
+and the trace carry everything needed to score; graders never reach
+into storage themselves.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from selfevals.runner.adapters import AdapterResponse
+    from selfevals.schemas.eval_case import EvalCase
+    from selfevals.schemas.trace import Trace
+
+
+class GradeLabel(StrEnum):
+    PASS = "pass"
+    FAIL = "fail"
+    PARTIAL = "partial"
+    ERROR = "error"
+    SKIPPED = "skipped"
+
+
+@dataclass(frozen=True)
+class GradeResult:
+    grader: str
+    label: GradeLabel
+    reason: str
+    score: float | None = None
+    confidence: float | None = None
+    failure_modes: list[str] = field(default_factory=list)
+    details: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class GraderContext:
+    case: EvalCase
+    trace: Trace
+    response: AdapterResponse | None = None
+
+
+class Grader(ABC):
+    name: str
+    """Stable identifier — used as the GradeResult.grader and the
+    Trace.grader_results[i].grader field."""
+
+    @abstractmethod
+    def grade(self, context: GraderContext) -> GradeResult: ...

selfevals/graders/calibration.py

@@ -0,0 +1,145 @@
+"""Calibration helpers: turn predictions + human labels into classification metrics.
+
+These functions live outside any specific grader so they can be reused by
+calibration scripts, dashboards, and the optimizer. They consume two flat
+lists keyed by `case_id` and produce a `CalibrationReport`.
+
+Metrics are computed treating `pass` as the positive class by default;
+`positive_label` is configurable. Macro-F1 is averaged across all observed
+labels.
+
+A class-imbalance guard: if a label appears in only one of the two streams,
+precision/recall for that class are reported as `None` rather than zero —
+this avoids the "100% precision on a class with 0 predictions" trap.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+
+from selfevals.graders.base import GradeLabel
+
+
+@dataclass(frozen=True)
+class PredictedLabel:
+    case_id: str
+    label: GradeLabel
+    confidence: float | None = None
+
+
+@dataclass(frozen=True)
+class HumanLabel:
+    case_id: str
+    label: GradeLabel
+    high_risk: bool = False
+
+
+@dataclass(frozen=True)
+class CalibrationReport:
+    n_pairs: int
+    precision: float | None
+    recall: float | None
+    f1: float | None
+    macro_f1: float | None
+    accuracy: float
+    high_risk_false_negatives: int
+    per_label_precision: dict[str, float | None] = field(default_factory=dict)
+    per_label_recall: dict[str, float | None] = field(default_factory=dict)
+    confusion: dict[tuple[str, str], int] = field(default_factory=dict)
+
+
+def _safe_div(num: float, den: float) -> float | None:
+    if den == 0:
+        return None
+    return num / den
+
+
+def _f1(p: float | None, r: float | None) -> float | None:
+    if p is None or r is None or (p + r) == 0:
+        return None
+    return 2 * p * r / (p + r)
+
+
+def compute_classification_metrics(
+    predictions: list[PredictedLabel],
+    human_labels: list[HumanLabel],
+    *,
+    positive_label: GradeLabel = GradeLabel.PASS,
+) -> CalibrationReport:
+    """Compute precision/recall/F1/macro-F1 + high-risk FNs.
+
+    Pairs are joined on case_id. Cases with a prediction but no human label
+    (or vice versa) are dropped from the metrics but counted via n_pairs=0
+    if there are no pairs at all.
+    """
+    pred_by_case: Mapping[str, PredictedLabel] = {p.case_id: p for p in predictions}
+    human_by_case: Mapping[str, HumanLabel] = {h.case_id: h for h in human_labels}
+    paired_ids = sorted(set(pred_by_case) & set(human_by_case))
+    n = len(paired_ids)
+
+    if n == 0:
+        return CalibrationReport(
+            n_pairs=0,
+            precision=None,
+            recall=None,
+            f1=None,
+            macro_f1=None,
+            accuracy=0.0,
+            high_risk_false_negatives=0,
+        )
+
+    confusion: Counter[tuple[str, str]] = Counter()
+    correct = 0
+    high_risk_fns = 0
+    for case_id in paired_ids:
+        pred = pred_by_case[case_id].label.value
+        human = human_by_case[case_id].label.value
+        confusion[(human, pred)] += 1
+        if pred == human:
+            correct += 1
+        if (
+            human == positive_label.value
+            and pred != positive_label.value
+            and human_by_case[case_id].high_risk
+        ):
+            high_risk_fns += 1
+
+    accuracy = correct / n
+
+    labels = sorted({k[0] for k in confusion} | {k[1] for k in confusion})
+    per_label_precision: dict[str, float | None] = {}
+    per_label_recall: dict[str, float | None] = {}
+    f1_values: list[float] = []
+    for label in labels:
+        tp = confusion[(label, label)]
+        pred_pos = sum(c for (h, p), c in confusion.items() if p == label)
+        actual_pos = sum(c for (h, p), c in confusion.items() if h == label)
+        precision_l = _safe_div(tp, pred_pos)
+        recall_l = _safe_div(tp, actual_pos)
+        per_label_precision[label] = precision_l
+        per_label_recall[label] = recall_l
+        f1_l = _f1(precision_l, recall_l)
+        if f1_l is not None:
+            f1_values.append(f1_l)
+
+    macro_f1 = sum(f1_values) / len(f1_values) if f1_values else None
+
+    pos = positive_label.value
+    precision_pos = per_label_precision.get(pos)
+    recall_pos = per_label_recall.get(pos)
+    f1_pos = _f1(precision_pos, recall_pos)
+
+    return CalibrationReport(
+        n_pairs=n,
+        precision=precision_pos,
+        recall=recall_pos,
+        f1=f1_pos,
+        macro_f1=macro_f1,
+        accuracy=accuracy,
+        high_risk_false_negatives=high_risk_fns,
+        per_label_precision=per_label_precision,
+        per_label_recall=per_label_recall,
+        confusion=dict(confusion),
+    )

selfevals/graders/deterministic.py

@@ -0,0 +1,143 @@
+"""DeterministicGrader: declarative rules from EvalCase.expected.
+
+Rules supported in MVP:
+- must_include: every string must appear in the final response (case-
+  insensitive by default; controllable via constructor flag).
+- must_not_include: none of the strings may appear.
+- required_tools: every tool listed must appear in the trace.
+- forbidden_tools: no tool listed may appear.
+- regex_match: optional regex applied to the final response.
+- structured_output equality: when EvalCase.expected.structured_output
+  is set, the adapter's structured_output must match exactly.
+
+Each rule has a stable failure-mode tag emitted in GradeResult.failure_modes
+so weighted scoring can attribute failures upstream.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from selfevals.graders.base import GradeLabel, Grader, GraderContext, GradeResult
+from selfevals.schemas.trace import LLMCallSpan, ToolCallSpan
+
+if TYPE_CHECKING:
+    from selfevals.schemas.eval_case import Expected
+    from selfevals.schemas.trace import Trace
+
+
+class DeterministicRuleViolationError(RuntimeError):
+    """Raised if the grader is asked to evaluate a contradictory rule set."""
+
+
+@dataclass(frozen=True)
+class _Violation:
+    failure_mode: str
+    detail: str
+
+
+def _final_response_text(ctx: GraderContext) -> str:
+    if ctx.response is not None and ctx.response.content:
+        return ctx.response.content
+    # Fall back to the final structured output's "content" or empty.
+    if ctx.response is not None and ctx.response.structured_output is not None:
+        content = ctx.response.structured_output.get("content")
+        if isinstance(content, str):
+            return content
+    return ""
+
+
+def _tools_invoked(trace: Trace) -> list[str]:
+    return [s.tool_name for s in trace.spans if isinstance(s, ToolCallSpan)]
+
+
+def _llm_call_count(trace: Trace) -> int:
+    return sum(1 for s in trace.spans if isinstance(s, LLMCallSpan))
+
+
+class DeterministicGrader(Grader):
+    def __init__(
+        self,
+        name: str = "deterministic",
+        *,
+        case_sensitive: bool = False,
+        regex_match: re.Pattern[str] | str | None = None,
+    ) -> None:
+        if not name:
+            raise ValueError("grader name must be non-empty")
+        self.name = name
+        self._case_sensitive = case_sensitive
+        if isinstance(regex_match, str):
+            self._regex: re.Pattern[str] | None = re.compile(regex_match)
+        else:
+            self._regex = regex_match
+
+    def grade(self, context: GraderContext) -> GradeResult:
+        expected: Expected = context.case.expected
+        violations: list[_Violation] = []
+
+        text = _final_response_text(context)
+        haystack = text if self._case_sensitive else text.lower()
+        invoked = _tools_invoked(context.trace)
+        invoked_set = set(invoked)
+
+        for needle in expected.must_include:
+            probe = needle if self._case_sensitive else needle.lower()
+            if probe not in haystack:
+                violations.append(
+                    _Violation(failure_mode="missing_required_substring", detail=needle)
+                )
+
+        for needle in expected.must_not_include:
+            probe = needle if self._case_sensitive else needle.lower()
+            if probe in haystack:
+                violations.append(_Violation(failure_mode="forbidden_substring", detail=needle))
+
+        for required in expected.required_tools:
+            if required not in invoked_set:
+                violations.append(_Violation(failure_mode="missing_required_tool", detail=required))
+
+        for forbidden in expected.forbidden_tools:
+            if forbidden in invoked_set:
+                violations.append(
+                    _Violation(failure_mode="forbidden_tool_invoked", detail=forbidden)
+                )
+
+        if self._regex is not None and not self._regex.search(text):
+            violations.append(_Violation(failure_mode="regex_mismatch", detail=self._regex.pattern))
+
+        if expected.structured_output is not None:
+            response_struct = context.response.structured_output if context.response else None
+            if response_struct != expected.structured_output:
+                violations.append(
+                    _Violation(
+                        failure_mode="structured_output_mismatch",
+                        detail="expected != actual",
+                    )
+                )
+
+        # Loose hint metrics that don't fail but are useful for debug.
+        details = {
+            "tools_invoked": invoked,
+            "llm_call_count": _llm_call_count(context.trace),
+        }
+        if not violations:
+            return GradeResult(
+                grader=self.name,
+                label=GradeLabel.PASS,
+                reason="all deterministic rules satisfied",
+                score=1.0,
+                details=details,
+            )
+        modes = sorted({v.failure_mode for v in violations})
+        reason = "; ".join(f"{v.failure_mode}:{v.detail}" for v in violations)
+        return GradeResult(
+            grader=self.name,
+            label=GradeLabel.FAIL,
+            reason=reason,
+            score=0.0,
+            failure_modes=modes,
+            details=details,
+        )

selfevals/graders/llm_judge.py

@@ -0,0 +1,187 @@
+"""LLMJudgeGrader: invoke an AgentAdapter as a judge.
+
+The grader formats a rubric prompt with the case input, the agent's final
+response, and the rubric instructions; the judge adapter returns a JSON
+response with `label`, `reason`, optional `score`, optional `confidence`.
+
+MVP ships single-judge. The constructor accepts an optional `card` (a
+`GraderCard`) so future panel infra can pin behavior to a calibrated
+configuration. When `card.blocking` and calibration metrics are below
+thresholds, the grader degrades to advisory (returns SKIPPED) per the
+operational spec — unless `force=True` is set.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from string import Template
+from typing import TYPE_CHECKING, Any
+
+from selfevals.graders.base import GradeLabel, Grader, GraderContext, GradeResult
+
+if TYPE_CHECKING:
+    from selfevals.runner.adapters import AdapterRequest, AgentAdapter
+    from selfevals.schemas.grader_card import GraderCard
+
+
+_DEFAULT_RUBRIC = """You are an evaluator. Read the case input and the agent's response,
+then decide whether the response meets the rubric.
+
+Rubric:
+$rubric
+
+Case input:
+$case_input
+
+Agent response:
+$agent_response
+
+Return a single JSON object with keys:
+- label: one of "pass", "fail", "partial"
+- reason: short justification
+- score: number in [0, 1] (optional)
+- confidence: number in [0, 1] (optional)
+"""
+
+
+@dataclass(frozen=True)
+class RubricTemplate:
+    rubric: str
+    template: str = _DEFAULT_RUBRIC
+
+    def render(self, *, case_input: Any, agent_response: str) -> str:
+        return Template(self.template).safe_substitute(
+            rubric=self.rubric,
+            case_input=json.dumps(case_input, ensure_ascii=False),
+            agent_response=agent_response,
+        )
+
+
+@dataclass(frozen=True)
+class JudgeDecision:
+    label: GradeLabel
+    reason: str
+    score: float | None = None
+    confidence: float | None = None
+
+
+def _parse_judge_output(text: str) -> JudgeDecision:
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"judge did not return valid JSON: {exc}; text={text!r}") from exc
+    if not isinstance(data, dict):
+        raise ValueError(f"judge JSON must be an object, got {type(data).__name__}")
+    raw_label = str(data.get("label", "")).strip().lower()
+    if raw_label not in {label.value for label in GradeLabel}:
+        raise ValueError(f"judge returned unknown label: {raw_label!r}")
+    label = GradeLabel(raw_label)
+    reason = str(data.get("reason", "")).strip() or "no reason supplied"
+    score = data.get("score")
+    confidence = data.get("confidence")
+    return JudgeDecision(
+        label=label,
+        reason=reason,
+        score=float(score) if score is not None else None,
+        confidence=float(confidence) if confidence is not None else None,
+    )
+
+
+def _is_card_calibrated(card: GraderCard | None) -> bool:
+    if card is None:
+        return True
+    if not card.blocking:
+        return True
+    metrics = card.metrics
+    thresholds = card.thresholds
+    if thresholds.min_precision is not None and (
+        metrics.precision is None or metrics.precision < thresholds.min_precision
+    ):
+        return False
+    if thresholds.min_recall is not None and (
+        metrics.recall is None or metrics.recall < thresholds.min_recall
+    ):
+        return False
+    return not (
+        thresholds.max_high_risk_false_negatives is not None
+        and (
+            metrics.high_risk_false_negatives is None
+            or metrics.high_risk_false_negatives > thresholds.max_high_risk_false_negatives
+        )
+    )
+
+
+class LLMJudgeGrader(Grader):
+    def __init__(
+        self,
+        name: str,
+        *,
+        judge_adapter: AgentAdapter,
+        rubric: RubricTemplate,
+        card: GraderCard | None = None,
+        force: bool = False,
+    ) -> None:
+        if not name:
+            raise ValueError("grader name must be non-empty")
+        self.name = name
+        self._judge = judge_adapter
+        self._rubric = rubric
+        self._card = card
+        self._force = force
+
+    def grade(self, context: GraderContext) -> GradeResult:
+        if not self._force and not _is_card_calibrated(self._card):
+            return GradeResult(
+                grader=self.name,
+                label=GradeLabel.SKIPPED,
+                reason="blocking grader below calibration thresholds; degraded to advisory",
+                score=None,
+                confidence=None,
+                details={"card_state": getattr(self._card, "state", None)},
+            )
+        prompt = self._rubric.render(
+            case_input=context.case.input,
+            agent_response=_extract_response_text(context),
+        )
+        request = _build_judge_request(context, prompt, self.name)
+        try:
+            response = self._judge.invoke(request)
+        except Exception as exc:
+            return GradeResult(
+                grader=self.name,
+                label=GradeLabel.ERROR,
+                reason=f"judge invocation failed: {exc}",
+            )
+        try:
+            decision = _parse_judge_output(response.content or "")
+        except ValueError as exc:
+            return GradeResult(
+                grader=self.name,
+                label=GradeLabel.ERROR,
+                reason=f"could not parse judge output: {exc}",
+            )
+        return GradeResult(
+            grader=self.name,
+            label=decision.label,
+            reason=decision.reason,
+            score=decision.score,
+            confidence=decision.confidence,
+        )
+
+
+def _extract_response_text(context: GraderContext) -> str:
+    if context.response is not None and context.response.content:
+        return context.response.content
+    return ""
+
+
+def _build_judge_request(context: GraderContext, prompt: str, grader_name: str) -> AdapterRequest:
+    from selfevals.runner.adapters import AdapterRequest  # local import to avoid cycle
+
+    return AdapterRequest(
+        workspace_id=context.case.workspace_id,
+        case_id=context.case.id,
+        input={"messages": [{"role": "user", "content": prompt}]},
+        metadata={"grader": grader_name, "judge": True},
+    )