evalproof 0.2.0__py3-none-any.whl

evalproof/compare.py

@@ -0,0 +1,169 @@
+"""Paired comparison of two runs on the cases they share."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from evalproof.models import Run
+from evalproof.stats import mcnemar_exact, paired_bootstrap, percentile, wilson
+
+
+@dataclass
+class MetricComparison:
+    name: str
+    kind: str  # "binary" | "continuous"
+    n_paired: int
+    baseline: float
+    candidate: float
+    delta: float
+    ci_low: float
+    ci_high: float
+    p_value: float
+    test: str
+    regressed: list[str] = field(default_factory=list)  # passed in baseline, failed in candidate
+    improved: list[str] = field(default_factory=list)
+    undecided: int = 0
+    baseline_ci: tuple[float, float] | None = None
+    candidate_ci: tuple[float, float] | None = None
+
+
+@dataclass
+class RunStats:
+    cases: int
+    errors: int
+    p50_latency_ms: float
+    p95_latency_ms: float
+    cost_usd: float
+    cached: int
+
+
+@dataclass
+class Comparison:
+    paired_ids: list[str]
+    only_baseline: list[str]
+    only_candidate: list[str]
+    same_dataset: bool
+    metrics: dict[str, MetricComparison]
+    baseline: RunStats
+    candidate: RunStats
+    baseline_label: str = "baseline"
+    candidate_label: str = "candidate"
+
+
+def run_stats(run: Run) -> RunStats:
+    lat = [r.output.latency_ms for r in run.results if not r.output.error]
+    cost = sum(r.output.cost_usd for r in run.results) + sum(
+        s.judge_cost_usd for r in run.results for s in r.scores.values()
+    )
+    return RunStats(
+        cases=len(run.results),
+        errors=sum(1 for r in run.results if r.output.error),
+        p50_latency_ms=percentile(lat, 50),
+        p95_latency_ms=percentile(lat, 95),
+        cost_usd=cost,
+        cached=sum(1 for r in run.results if r.output.cached),
+    )
+
+
+def compare_metric(
+    name: str,
+    baseline: Run,
+    candidate: Run,
+    ids: list[str],
+    *,
+    alpha: float = 0.05,
+    n_boot: int = 5000,
+    seed: int = 0,
+) -> MetricComparison | None:
+    b_by, c_by = baseline.by_id(), candidate.by_id()
+    pairs = []
+    undecided = 0
+    for cid in ids:
+        bs, cs = b_by[cid].scores.get(name), c_by[cid].scores.get(name)
+        if bs is None or cs is None:
+            continue
+        if bs.value is None or cs.value is None:
+            undecided += 1
+            continue
+        pairs.append((cid, bs, cs))
+    if not pairs:
+        return None
+
+    binary = all(bs.passed is not None and cs.passed is not None for _, bs, cs in pairs)
+    if binary:
+        a = [1.0 if bs.passed else 0.0 for _, bs, _ in pairs]
+        c = [1.0 if cs.passed else 0.0 for _, _, cs in pairs]
+        regressed = [cid for cid, bs, cs in pairs if bs.passed and not cs.passed]
+        improved = [cid for cid, bs, cs in pairs if not bs.passed and cs.passed]
+        boot = paired_bootstrap(a, c, n_boot=n_boot, alpha=alpha, seed=seed)
+        n = len(pairs)
+        return MetricComparison(
+            name=name,
+            kind="binary",
+            n_paired=n,
+            baseline=sum(a) / n,
+            candidate=sum(c) / n,
+            delta=boot.delta,
+            ci_low=boot.ci_low,
+            ci_high=boot.ci_high,
+            p_value=mcnemar_exact(len(regressed), len(improved)),
+            test="exact McNemar",
+            regressed=regressed,
+            improved=improved,
+            undecided=undecided,
+            baseline_ci=wilson(int(sum(a)), n, alpha),
+            candidate_ci=wilson(int(sum(c)), n, alpha),
+        )
+
+    a = [float(bs.value) for _, bs, _ in pairs if bs.value is not None]
+    c = [float(cs.value) for _, _, cs in pairs if cs.value is not None]
+    boot = paired_bootstrap(a, c, n_boot=n_boot, alpha=alpha, seed=seed)
+    regressed = [cid for cid, bs, cs in pairs if (cs.value or 0) < (bs.value or 0)]
+    improved = [cid for cid, bs, cs in pairs if (cs.value or 0) > (bs.value or 0)]
+    return MetricComparison(
+        name=name,
+        kind="continuous",
+        n_paired=len(pairs),
+        baseline=sum(a) / len(a),
+        candidate=sum(c) / len(c),
+        delta=boot.delta,
+        ci_low=boot.ci_low,
+        ci_high=boot.ci_high,
+        p_value=boot.p_value,
+        test="paired bootstrap",
+        regressed=regressed,
+        improved=improved,
+        undecided=undecided,
+    )
+
+
+def compare_runs(
+    baseline: Run,
+    candidate: Run,
+    *,
+    alpha: float = 0.05,
+    n_boot: int = 5000,
+    seed: int = 0,
+    metrics: list[str] | None = None,
+) -> Comparison:
+    b_ids, c_ids = set(baseline.by_id()), set(candidate.by_id())
+    paired = sorted(b_ids & c_ids)
+    names = metrics or [n for n in baseline.scorer_names() if n in candidate.scorer_names()]
+    out: dict[str, MetricComparison] = {}
+    for name in names:
+        mc = compare_metric(
+            name, baseline, candidate, paired, alpha=alpha, n_boot=n_boot, seed=seed
+        )
+        if mc is not None:
+            out[name] = mc
+    return Comparison(
+        paired_ids=paired,
+        only_baseline=sorted(b_ids - c_ids),
+        only_candidate=sorted(c_ids - b_ids),
+        same_dataset=baseline.info.dataset_hash == candidate.info.dataset_hash,
+        metrics=out,
+        baseline=run_stats(baseline),
+        candidate=run_stats(candidate),
+        baseline_label=baseline.info.system,
+        candidate_label=candidate.info.system,
+    )

evalproof/config.py

@@ -0,0 +1,158 @@
+"""Suite configuration: one YAML file describes the data, the system, the scorers and the gate."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+from pydantic import BaseModel, Field, ValidationError, model_validator
+
+ScorerType = Literal[
+    "exact_match",
+    "contains",
+    "regex",
+    "json_schema",
+    "numeric_tolerance",
+    "recall_at_k",
+    "mrr",
+    "ndcg",
+    "llm_judge",
+]
+
+
+class ConfigError(ValueError):
+    pass
+
+
+class Pricing(BaseModel):
+    """USD per million tokens. Local models default to zero."""
+
+    input_per_mtok: float = 0.0
+    output_per_mtok: float = 0.0
+
+    def cost(self, tokens_in: int, tokens_out: int) -> float:
+        return (tokens_in * self.input_per_mtok + tokens_out * self.output_per_mtok) / 1_000_000
+
+
+class GenerationParams(BaseModel):
+    temperature: float = 0.0
+    seed: int | None = 0
+    max_tokens: int = 512
+    think: bool | None = False  # reasoning models: off by default for speed and determinism
+    extra: dict[str, Any] = Field(default_factory=dict)
+
+
+class TargetConfig(BaseModel):
+    """A model or system to call: ``ollama:<model>``, ``anthropic:<model>``, ``openai:<model>``,
+    ``python:<module>:<function>`` or an ``http(s)://`` endpoint."""
+
+    target: str
+    label: str | None = None  # how reports name this system, e.g. "qwen3:1.7b + grounded prompt"
+    system_prompt: str | None = None
+    prompt_template: str = "{input}"
+    params: GenerationParams = Field(default_factory=GenerationParams)
+    timeout_s: float = 120.0
+    base_url: str | None = None
+    pricing: Pricing = Field(default_factory=Pricing)
+
+    @property
+    def kind(self) -> str:
+        if self.target.startswith(("http://", "https://")):
+            return "http"
+        return self.target.split(":", 1)[0]
+
+
+class ScorerConfig(BaseModel):
+    name: str
+    type: ScorerType
+    params: dict[str, Any] = Field(default_factory=dict)
+
+
+class CalibrationConfig(BaseModel):
+    """How to calibrate a judge: against a labels file, or against another (trusted) scorer
+    sampled down to ``gold_sample`` cases to mimic a small human-reviewed set."""
+
+    judge_scorer: str
+    labels: str | None = None
+    gold_scorer: str | None = None
+    gold_sample: int | None = None
+
+    @model_validator(mode="after")
+    def _one_source(self) -> CalibrationConfig:
+        if (self.labels is None) == (self.gold_scorer is None):
+            raise ValueError("calibration needs exactly one of 'labels' or 'gold_scorer'")
+        return self
+
+
+class GateConfig(BaseModel):
+    """Fail only when a drop is both larger than ``max_drop`` and statistically significant."""
+
+    metric: str
+    max_drop: float = 0.02
+    alpha: float = 0.05
+    min_cases: int = 20
+    max_p95_latency_ms: float | None = None
+    max_cost_usd: float | None = None
+    n_boot: int = 5000
+
+
+class SuiteConfig(BaseModel):
+    name: str
+    dataset: str
+    system: TargetConfig
+    scorers: list[ScorerConfig]
+    judge: TargetConfig | None = None
+    calibration: CalibrationConfig | None = None
+    gate: GateConfig | None = None
+    seed: int = 0
+    cache_dir: str = ".evalproof/cache"
+    concurrency: int = 1
+    base_dir: str = "."
+
+    @model_validator(mode="after")
+    def _check(self) -> SuiteConfig:
+        names = [s.name for s in self.scorers]
+        if len(set(names)) != len(names):
+            raise ValueError(f"scorer names must be unique, got {names}")
+        if any(s.type == "llm_judge" for s in self.scorers) and self.judge is None:
+            raise ValueError("an llm_judge scorer needs a 'judge' target")
+        if self.gate and self.gate.metric not in names:
+            raise ValueError(f"gate.metric {self.gate.metric!r} is not a scorer name")
+        if self.calibration:
+            for field in ("judge_scorer", "gold_scorer"):
+                ref = getattr(self.calibration, field)
+                if ref is not None and ref not in names:
+                    raise ValueError(f"calibration.{field} {ref!r} is not a scorer name")
+        if self.concurrency < 1:
+            raise ValueError("concurrency must be >= 1")
+        return self
+
+    def resolve(self, path: str) -> Path:
+        p = Path(path)
+        return p if p.is_absolute() else Path(self.base_dir) / p
+
+
+def load_config(path: str | Path) -> SuiteConfig:
+    path = Path(path)
+    if not path.exists():
+        raise ConfigError(f"suite file not found: {path}")
+    try:
+        raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    except yaml.YAMLError as exc:
+        raise ConfigError(f"{path}: invalid YAML: {exc}") from exc
+    if not isinstance(raw, dict):
+        raise ConfigError(f"{path}: top level must be a mapping")
+    raw.setdefault("base_dir", str(path.parent))
+    # Prompt and rubric text may live in files next to the suite.
+    for section in ("system", "judge"):
+        block = raw.get(section)
+        if isinstance(block, dict):
+            for key in ("system_prompt", "prompt_template"):
+                file_key = f"{key}_file"
+                if file_key in block:
+                    block[key] = (path.parent / block.pop(file_key)).read_text(encoding="utf-8")
+    try:
+        return SuiteConfig.model_validate(raw)
+    except ValidationError as exc:
+        raise ConfigError(f"{path}: {exc}") from exc

evalproof/dataset.py

@@ -0,0 +1,66 @@
+"""Golden sets as JSONL: one case per line, stable ids, and a content hash for provenance."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from pydantic import ValidationError
+
+from evalproof.models import Case
+
+
+class DatasetError(ValueError):
+    pass
+
+
+def canonical_json(obj: Any) -> str:
+    return json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+
+
+def stable_id(record: dict[str, Any]) -> str:
+    """An id derived from the input, so reordering the file never re-keys a case."""
+    digest = hashlib.sha256(canonical_json(record.get("input")).encode()).hexdigest()
+    return f"case-{digest[:12]}"
+
+
+def load_dataset(path: str | Path) -> list[Case]:
+    path = Path(path)
+    if not path.exists():
+        raise DatasetError(f"dataset not found: {path}")
+    cases: list[Case] = []
+    seen: set[str] = set()
+    for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
+        if not line.strip() or line.lstrip().startswith("#"):
+            continue
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError as exc:
+            raise DatasetError(f"{path}:{lineno}: invalid JSON: {exc.msg}") from exc
+        if not isinstance(record, dict) or "input" not in record:
+            raise DatasetError(f"{path}:{lineno}: each line needs an object with an 'input'")
+        record.setdefault("id", stable_id(record))
+        try:
+            case = Case.model_validate(record)
+        except ValidationError as exc:
+            first = exc.errors()[0]
+            where = ".".join(str(p) for p in first["loc"])
+            raise DatasetError(f"{path}:{lineno}: {where}: {first['msg']}") from exc
+        if case.id in seen:
+            raise DatasetError(f"{path}:{lineno}: duplicate case id {case.id!r}")
+        seen.add(case.id)
+        cases.append(case)
+    if not cases:
+        raise DatasetError(f"dataset is empty: {path}")
+    return cases
+
+
+def dataset_hash(cases: list[Case]) -> str:
+    """Hash of the cases as loaded; two runs on different data never compare silently."""
+    h = hashlib.sha256()
+    for case in sorted(cases, key=lambda c: c.id):
+        h.update(canonical_json(case.model_dump(mode="json")).encode())
+        h.update(b"\n")
+    return h.hexdigest()[:16]

evalproof/gate.py

@@ -0,0 +1,107 @@
+"""The CI gate: fail on a regression that is both large enough to matter and unlikely to be noise.
+
+Exit codes are distinct so a pipeline can tell a quality regression from a budget breach
+from a setup problem:
+
+* ``0``  pass (possibly with warnings)
+* ``1``  quality regression: drop >= ``max_drop`` and p < ``alpha``
+* ``3``  budget breach: candidate p95 latency or total cost over the limit
+* ``4``  cannot gate: too few paired cases, metric missing, or the runs used different data
+* ``2``  usage or configuration error (raised by the CLI, not here)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from evalproof.compare import Comparison
+from evalproof.config import GateConfig
+
+EXIT_PASS = 0
+EXIT_REGRESSION = 1
+EXIT_USAGE = 2
+EXIT_BUDGET = 3
+EXIT_CANNOT_GATE = 4
+
+
+@dataclass
+class GateDecision:
+    status: str  # "pass" | "warn" | "fail"
+    exit_code: int
+    reasons: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+
+def decide(cmp: Comparison, gate: GateConfig) -> GateDecision:
+    reasons: list[str] = []
+    warnings: list[str] = []
+
+    if not cmp.same_dataset:
+        return GateDecision(
+            "fail",
+            EXIT_CANNOT_GATE,
+            ["baseline and candidate were run on different datasets (dataset hash differs)"],
+        )
+    mc = cmp.metrics.get(gate.metric)
+    if mc is None:
+        return GateDecision(
+            "fail", EXIT_CANNOT_GATE, [f"metric {gate.metric!r} has no paired scores"]
+        )
+    if mc.n_paired < gate.min_cases:
+        return GateDecision(
+            "fail",
+            EXIT_CANNOT_GATE,
+            [
+                f"only {mc.n_paired} paired cases for {gate.metric!r}; "
+                f"the gate needs {gate.min_cases}"
+            ],
+        )
+    if mc.undecided:
+        warnings.append(f"{mc.undecided} paired case(s) had an undecided {gate.metric!r} verdict")
+
+    exit_code = EXIT_PASS
+    drop_big = mc.delta <= -gate.max_drop
+    significant = mc.p_value < gate.alpha
+    if drop_big and significant:
+        reasons.append(
+            f"{gate.metric}: {mc.baseline:.3f} → {mc.candidate:.3f} (Δ {mc.delta:+.3f}, "
+            f"95% CI [{mc.ci_low:+.3f}, {mc.ci_high:+.3f}], {mc.test} p={mc.p_value:.4f}) — "
+            f"a drop beyond {gate.max_drop:.3f} at α={gate.alpha}"
+        )
+        exit_code = EXIT_REGRESSION
+    elif drop_big:
+        warnings.append(
+            f"{gate.metric} dropped {mc.delta:+.3f}, beyond the {gate.max_drop:.3f} threshold, but "
+            f"p={mc.p_value:.3f} ≥ α={gate.alpha}: not distinguishable from noise "
+            f"at n={mc.n_paired}. "
+            "Add cases before trusting either direction."
+        )
+    elif mc.delta < 0 and significant:
+        warnings.append(
+            f"{gate.metric} dropped {mc.delta:+.3f} (significant, p={mc.p_value:.4f}) but within "
+            f"the {gate.max_drop:.3f} tolerance"
+        )
+
+    if (
+        gate.max_p95_latency_ms is not None
+        and cmp.candidate.p95_latency_ms > gate.max_p95_latency_ms
+    ):
+        reasons.append(
+            f"p95 latency {cmp.candidate.p95_latency_ms:.0f} ms exceeds "
+            f"{gate.max_p95_latency_ms:.0f} ms"
+        )
+        exit_code = exit_code or EXIT_BUDGET
+    if gate.max_cost_usd is not None and cmp.candidate.cost_usd > gate.max_cost_usd:
+        reasons.append(f"cost ${cmp.candidate.cost_usd:.4f} exceeds ${gate.max_cost_usd:.4f}")
+        exit_code = exit_code or EXIT_BUDGET
+
+    if cmp.candidate.errors > cmp.baseline.errors:
+        warnings.append(f"system errors rose from {cmp.baseline.errors} to {cmp.candidate.errors}")
+    if cmp.only_baseline or cmp.only_candidate:
+        warnings.append(
+            f"{len(cmp.only_baseline)} case(s) only in baseline, "
+            f"{len(cmp.only_candidate)} only in candidate — excluded from the comparison"
+        )
+
+    status = "fail" if exit_code else ("warn" if warnings else "pass")
+    return GateDecision(status, exit_code, reasons, warnings)

evalproof/models.py

@@ -0,0 +1,115 @@
+"""Data shared by every stage: a case, what the system said, how it scored, the run."""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class Case(BaseModel):
+    """One golden-set item. Extra fields (answer keys, relevant ids) stay on the case."""
+
+    model_config = {"extra": "allow"}
+
+    id: str
+    input: str | dict[str, Any]
+    reference: str | None = None
+    tags: list[str] = Field(default_factory=list)
+
+    def field(self, name: str) -> Any:
+        """Read a named field, including extras such as ``accept`` or ``relevant``."""
+        if name in type(self).model_fields:
+            return getattr(self, name)
+        extra = self.model_extra or {}
+        return extra.get(name)
+
+    def render_vars(self) -> dict[str, Any]:
+        """Variables available to prompt templates."""
+        out: dict[str, Any] = {"id": self.id, "reference": self.reference or ""}
+        if isinstance(self.input, dict):
+            out.update(self.input)
+            out["input"] = self.input
+        else:
+            out["input"] = self.input
+        out.update(self.model_extra or {})
+        return out
+
+
+class SystemOutput(BaseModel):
+    """What the system under test returned for one case."""
+
+    output: str
+    retrieved: list[str] = Field(default_factory=list)
+    latency_ms: float = 0.0
+    tokens_in: int = 0
+    tokens_out: int = 0
+    cost_usd: float = 0.0
+    cached: bool = False
+    error: str | None = None
+
+
+class Score(BaseModel):
+    """One scorer's verdict on one case.
+
+    ``passed`` is the binary outcome used by pass rates and McNemar;
+    ``value`` is the continuous score used by bootstrap comparisons.
+    Both are None when the scorer could not decide (e.g. an unparseable judge reply);
+    undecided cases are counted and reported, never silently scored as passes.
+    """
+
+    value: float | None
+    passed: bool | None = None
+    reason: str | None = None
+    judge_tokens: int = 0
+    judge_cost_usd: float = 0.0
+
+    @field_validator("value")
+    @classmethod
+    def _finite(cls, v: float | None) -> float | None:
+        if v is not None and (math.isinf(v) or math.isnan(v)):
+            raise ValueError("score value must be finite (use None for undecided)")
+        return v
+
+    @property
+    def decided(self) -> bool:
+        return self.value is not None
+
+
+class CaseResult(BaseModel):
+    case_id: str
+    tags: list[str] = Field(default_factory=list)
+    output: SystemOutput
+    scores: dict[str, Score] = Field(default_factory=dict)
+
+
+class RunInfo(BaseModel):
+    suite: str
+    system: str
+    dataset_path: str
+    dataset_hash: str
+    started_at: str
+    finished_at: str
+    wall_time_s: float
+    evalproof_version: str
+    seed: int
+    environment: dict[str, str] = Field(default_factory=dict)
+
+
+class Run(BaseModel):
+    """A complete, self-describing run artifact. Compare two of these to gate a change."""
+
+    info: RunInfo
+    results: list[CaseResult]
+
+    def by_id(self) -> dict[str, CaseResult]:
+        return {r.case_id: r for r in self.results}
+
+    def scorer_names(self) -> list[str]:
+        names: list[str] = []
+        for r in self.results:
+            for name in r.scores:
+                if name not in names:
+                    names.append(name)
+        return names

evalproof/providers/__init__.py

@@ -0,0 +1,54 @@
+"""Provider registry. A target string ``<provider>:<model>`` picks the implementation."""
+
+from __future__ import annotations
+
+import httpx
+
+from evalproof.config import TargetConfig
+from evalproof.providers.anthropic import AnthropicProvider
+from evalproof.providers.base import (
+    ChatResponse,
+    Message,
+    Provider,
+    ProviderError,
+    with_retries,
+)
+from evalproof.providers.ollama import OllamaProvider
+from evalproof.providers.openai import OpenAIProvider
+
+__all__ = [
+    "AnthropicProvider",
+    "ChatResponse",
+    "Message",
+    "OllamaProvider",
+    "OpenAIProvider",
+    "Provider",
+    "ProviderError",
+    "make_provider",
+    "parse_target",
+    "with_retries",
+]
+
+LLM_PROVIDERS = {"ollama", "anthropic", "openai"}
+
+
+def parse_target(target: str) -> tuple[str, str]:
+    """``"ollama:qwen3:8b"`` → ``("ollama", "qwen3:8b")``. Model names may contain colons."""
+    if ":" not in target:
+        raise ProviderError(f"target {target!r} must look like '<provider>:<model>'")
+    provider, model = target.split(":", 1)
+    if not model:
+        raise ProviderError(f"target {target!r} has no model")
+    return provider, model
+
+
+def make_provider(cfg: TargetConfig, client: httpx.Client | None = None) -> Provider:
+    provider, model = parse_target(cfg.target)
+    kwargs = {"base_url": cfg.base_url, "timeout_s": cfg.timeout_s, "client": client}
+    if provider == "ollama":
+        return OllamaProvider(model, **kwargs)  # type: ignore[arg-type]
+    if provider == "anthropic":
+        return AnthropicProvider(model, **kwargs)  # type: ignore[arg-type]
+    if provider == "openai":
+        return OpenAIProvider(model, **kwargs)  # type: ignore[arg-type]
+    raise ProviderError(f"unknown provider {provider!r}; expected one of {sorted(LLM_PROVIDERS)}")