evalproof 0.2.0__py3-none-any.whl

evalproof/__init__.py

@@ -0,0 +1,69 @@
+"""evalproof: treat LLM evaluation as measurement.
+
+An LLM judge is a classifier with error rates. evalproof calibrates it against a
+small gold-labelled sample, reports prevalence-corrected pass rates with
+confidence intervals, compares runs on paired cases, and gates CI on
+statistically meaningful regressions rather than on noise.
+
+The names below are the supported Python API; everything else is internal and may
+change between minor versions.
+"""
+
+from evalproof._version import __version__
+from evalproof.calibration import CalibrationReport, Confusion, Estimate, calibrate
+from evalproof.compare import Comparison, MetricComparison, RunStats, compare_runs
+from evalproof.config import ConfigError, GateConfig, SuiteConfig, load_config
+from evalproof.dataset import DatasetError, load_dataset
+from evalproof.gate import (
+    EXIT_BUDGET,
+    EXIT_CANNOT_GATE,
+    EXIT_PASS,
+    EXIT_REGRESSION,
+    EXIT_USAGE,
+    GateDecision,
+    decide,
+)
+from evalproof.models import Case, CaseResult, Run, RunInfo, Score, SystemOutput
+from evalproof.report import calibration_markdown, comparison_markdown, run_markdown
+from evalproof.runner import load_run, run_suite, save_run
+from evalproof.stats import mcnemar_exact, paired_bootstrap, wilson
+
+__all__ = [
+    "EXIT_BUDGET",
+    "EXIT_CANNOT_GATE",
+    "EXIT_PASS",
+    "EXIT_REGRESSION",
+    "EXIT_USAGE",
+    "CalibrationReport",
+    "Case",
+    "CaseResult",
+    "Comparison",
+    "ConfigError",
+    "Confusion",
+    "DatasetError",
+    "Estimate",
+    "GateConfig",
+    "GateDecision",
+    "MetricComparison",
+    "Run",
+    "RunInfo",
+    "RunStats",
+    "Score",
+    "SuiteConfig",
+    "SystemOutput",
+    "__version__",
+    "calibrate",
+    "calibration_markdown",
+    "compare_runs",
+    "comparison_markdown",
+    "decide",
+    "load_config",
+    "load_dataset",
+    "load_run",
+    "mcnemar_exact",
+    "paired_bootstrap",
+    "run_markdown",
+    "run_suite",
+    "save_run",
+    "wilson",
+]

evalproof/_version.py

@@ -0,0 +1,3 @@
+"""The package version: the single place it is written. pyproject.toml reads it at build time."""
+
+__version__ = "0.2.0"

evalproof/cache.py

@@ -0,0 +1,89 @@
+"""Content-addressed response cache.
+
+The key is everything that determines a model's reply: provider, model, endpoint,
+messages and generation parameters. Re-running an unchanged suite costs nothing and
+returns byte-identical results; changing one prompt re-runs only the calls it touches.
+
+A hit returns the *original* latency and token counts, flagged ``cached=True``, so a
+re-run reproduces the run artifact; the run's own wall time shows the real speed-up.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+from evalproof.config import GenerationParams
+from evalproof.providers.base import ChatResponse, Message, Provider
+
+
+def cache_key(
+    provider: str,
+    model: str,
+    base_url: str | None,
+    messages: list[Message],
+    params: GenerationParams,
+) -> str:
+    material = {
+        "provider": provider,
+        "model": model,
+        "base_url": base_url,
+        "messages": [m.model_dump() for m in messages],
+        "params": params.model_dump(mode="json"),
+    }
+    blob = json.dumps(material, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+    return hashlib.sha256(blob.encode()).hexdigest()
+
+
+class ResponseCache:
+    def __init__(self, root: str | Path) -> None:
+        self.root = Path(root)
+
+    def _path(self, key: str) -> Path:
+        return self.root / key[:2] / f"{key}.json"
+
+    def get(self, key: str) -> ChatResponse | None:
+        path = self._path(key)
+        if not path.exists():
+            return None
+        try:
+            resp = ChatResponse.model_validate_json(path.read_text(encoding="utf-8"))
+        except ValueError:
+            return None  # a torn write is a miss, not a crash
+        return resp.model_copy(update={"cached": True})
+
+    def put(self, key: str, response: ChatResponse) -> None:
+        path = self._path(key)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(".tmp")
+        tmp.write_text(response.model_copy(update={"cached": False}).model_dump_json(), "utf-8")
+        tmp.replace(path)
+
+    def stats(self) -> dict[str, int]:
+        files = list(self.root.glob("*/*.json")) if self.root.exists() else []
+        return {"entries": len(files), "bytes": sum(f.stat().st_size for f in files)}
+
+
+class CachedProvider:
+    """Wraps any provider; transparent on a miss, free on a hit."""
+
+    def __init__(self, inner: Provider, cache: ResponseCache, base_url: str | None = None) -> None:
+        self.inner = inner
+        self.cache = cache
+        self.name = inner.name
+        self.model = inner.model
+        self.base_url = base_url
+        self.hits = 0
+        self.misses = 0
+
+    def chat(self, messages: list[Message], params: GenerationParams) -> ChatResponse:
+        key = cache_key(self.name, self.model, self.base_url, messages, params)
+        hit = self.cache.get(key)
+        if hit is not None:
+            self.hits += 1
+            return hit
+        self.misses += 1
+        resp = self.inner.chat(messages, params)
+        self.cache.put(key, resp)
+        return resp

evalproof/calibration.py

@@ -0,0 +1,266 @@
+"""Judge calibration: measure the judge against gold labels, then correct its pass rate.
+
+A judge's raw pass rate is classify-and-count. If the judge passes bad answers at
+rate FPR and passes good answers at rate TPR, the raw rate converges to
+``q = FPR + (TPR - FPR) * p``, not to the true pass rate ``p``. Two corrections:
+
+* **ACC** (adjusted classify-and-count, Rogan–Gladen): ``p = (q - FPR) / (TPR - FPR)``,
+  with TPR and FPR estimated on the gold sample. Bootstrap interval.
+* **PPI** (prediction-powered inference, Angelopoulos et al., 2023): the judge's mean on
+  the unlabelled cases plus the mean label-minus-judge residual on the gold sample.
+  Needs no assumption about the judge; unbiased whenever the gold sample is random.
+
+Background: https://basantbhattarai.com.np/writing/count-prevalence-not-classifications/
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass, field
+
+import numpy as np
+
+from evalproof.models import Run
+from evalproof.stats import FloatArray, wilson, z_for
+
+
+@dataclass(frozen=True)
+class Confusion:
+    tp: int  # judge pass, gold pass
+    fp: int  # judge pass, gold fail
+    fn: int  # judge fail, gold pass
+    tn: int  # judge fail, gold fail
+
+    @property
+    def n(self) -> int:
+        return self.tp + self.fp + self.fn + self.tn
+
+    @property
+    def tpr(self) -> float | None:
+        pos = self.tp + self.fn
+        return self.tp / pos if pos else None
+
+    @property
+    def fpr(self) -> float | None:
+        neg = self.fp + self.tn
+        return self.fp / neg if neg else None
+
+    @property
+    def accuracy(self) -> float | None:
+        return (self.tp + self.tn) / self.n if self.n else None
+
+    @property
+    def kappa(self) -> float | None:
+        """Cohen's kappa: agreement beyond what the two marginals give by chance."""
+        n = self.n
+        if not n:
+            return None
+        po = (self.tp + self.tn) / n
+        pe = (
+            (self.tp + self.fp) * (self.tp + self.fn) + (self.fn + self.tn) * (self.fp + self.tn)
+        ) / (n * n)
+        return None if pe == 1 else (po - pe) / (1 - pe)
+
+
+def confusion(judge: list[bool], gold: list[bool]) -> Confusion:
+    if len(judge) != len(gold):
+        raise ValueError("judge and gold must be paired")
+    tp = sum(j and g for j, g in zip(judge, gold, strict=True))
+    fp = sum(j and not g for j, g in zip(judge, gold, strict=True))
+    fn = sum((not j) and g for j, g in zip(judge, gold, strict=True))
+    tn = sum((not j) and (not g) for j, g in zip(judge, gold, strict=True))
+    return Confusion(tp, fp, fn, tn)
+
+
+def acc_point(q: float, tpr: float, fpr: float) -> float | None:
+    """Rogan–Gladen. None when the judge is no better than chance (TPR <= FPR)."""
+    if tpr - fpr <= 1e-12:
+        return None
+    return min(1.0, max(0.0, (q - fpr) / (tpr - fpr)))
+
+
+@dataclass(frozen=True)
+class Estimate:
+    method: str
+    point: float | None
+    low: float | None
+    high: float | None
+    n: int
+    note: str = ""
+
+
+def raw_estimate(judge_all: list[bool], alpha: float = 0.05) -> Estimate:
+    k, n = sum(judge_all), len(judge_all)
+    lo, hi = wilson(k, n, alpha)
+    return Estimate("raw judge pass rate", k / n if n else None, lo, hi, n)
+
+
+def gold_only_estimate(gold: list[bool], alpha: float = 0.05) -> Estimate:
+    k, n = sum(gold), len(gold)
+    lo, hi = wilson(k, n, alpha)
+    return Estimate("gold labels only", k / n if n else None, lo, hi, n)
+
+
+def acc_estimate(
+    judge_all: list[bool],
+    judge_gold: list[bool],
+    gold: list[bool],
+    *,
+    alpha: float = 0.05,
+    n_boot: int = 5000,
+    seed: int = 0,
+) -> Estimate:
+    """ACC point estimate with a bootstrap interval that resamples both the full judged set
+    (uncertainty in q) and the gold sample (uncertainty in TPR and FPR)."""
+    cm = confusion(judge_gold, gold)
+    q = sum(judge_all) / len(judge_all)
+    if cm.tpr is None or cm.fpr is None:
+        return Estimate("ACC", None, None, None, len(judge_all), "gold sample has only one class")
+    point = acc_point(q, cm.tpr, cm.fpr)
+    if point is None:
+        return Estimate("ACC", None, None, None, len(judge_all), "judge no better than chance")
+    rng = np.random.default_rng(seed)
+    ja = np.asarray(judge_all, dtype=np.float64)
+    jg = np.asarray(judge_gold, dtype=bool)
+    gg = np.asarray(gold, dtype=bool)
+    reps: list[float] = []
+    undefined = 0
+    for _ in range(n_boot):
+        qa = float(ja[rng.integers(0, ja.size, ja.size)].mean())
+        idx = rng.integers(0, gg.size, gg.size)
+        j, g = jg[idx], gg[idx]
+        pos, neg = int(g.sum()), int((~g).sum())
+        if pos == 0 or neg == 0:
+            undefined += 1
+            continue
+        est = acc_point(qa, float((j & g).sum() / pos), float((j & ~g).sum() / neg))
+        if est is None:
+            undefined += 1
+            continue
+        reps.append(est)
+    if not reps:
+        return Estimate("ACC", point, None, None, len(judge_all), "no defined bootstrap replicates")
+    lo, hi = np.quantile(np.asarray(reps), [alpha / 2, 1 - alpha / 2])
+    note = f"{undefined}/{n_boot} replicates undefined" if undefined else ""
+    return Estimate("ACC", point, float(lo), float(hi), len(judge_all), note)
+
+
+def ppi_estimate(
+    judge_unlabelled: list[bool],
+    judge_labelled: list[bool],
+    gold: list[bool],
+    *,
+    alpha: float = 0.05,
+) -> Estimate:
+    """Prediction-powered mean: ``mean(f_unlabelled) + mean(y - f_labelled)``, normal CI."""
+    if not judge_labelled or not judge_unlabelled:
+        return Estimate("PPI", None, None, None, 0, "needs labelled and unlabelled cases")
+    fu: FloatArray = np.asarray(judge_unlabelled, dtype=np.float64)
+    fl: FloatArray = np.asarray(judge_labelled, dtype=np.float64)
+    y: FloatArray = np.asarray(gold, dtype=np.float64)
+    resid = y - fl
+    point = float(fu.mean() + resid.mean())
+    var_u = float(fu.var(ddof=1)) if fu.size > 1 else 0.0
+    var_r = float(resid.var(ddof=1)) if resid.size > 1 else 0.0
+    se = (var_u / fu.size + var_r / resid.size) ** 0.5
+    z = z_for(alpha)
+    n = int(fu.size + fl.size)
+    return Estimate(
+        "PPI", min(1.0, max(0.0, point)), max(0.0, point - z * se), min(1.0, point + z * se), n
+    )
+
+
+@dataclass
+class CalibrationReport:
+    judge_scorer: str
+    gold_source: str
+    n_judged: int
+    n_gold: int
+    undecided_judge: int
+    confusion: Confusion
+    estimates: list[Estimate]
+    truth: Estimate | None = None
+    disagreements: list[str] = field(default_factory=list)
+
+
+def calibrate(
+    run: Run,
+    judge_scorer: str,
+    *,
+    labels: dict[str, bool] | None = None,
+    gold_scorer: str | None = None,
+    gold_sample: int | None = None,
+    seed: int = 0,
+    alpha: float = 0.05,
+    n_boot: int = 5000,
+) -> CalibrationReport:
+    """Calibrate ``judge_scorer`` against human ``labels`` or a trusted ``gold_scorer``.
+
+    With ``gold_scorer`` and ``gold_sample``, a random sample of that size stands in for a
+    human-reviewed set; the remaining gold labels are used only to report the truth the
+    estimates are trying to recover.
+    """
+    if (labels is None) == (gold_scorer is None):
+        raise ValueError("pass exactly one of labels or gold_scorer")
+    judged: dict[str, bool] = {}
+    undecided = 0
+    for r in run.results:
+        s = r.scores.get(judge_scorer)
+        if s is None:
+            continue
+        if s.passed is None:
+            undecided += 1
+        else:
+            judged[r.case_id] = s.passed
+    if not judged:
+        raise ValueError(f"no decided verdicts from scorer {judge_scorer!r}")
+
+    truth_all: dict[str, bool] = {}
+    if labels is not None:
+        gold_ids = [cid for cid in labels if cid in judged]
+        gold_map = {cid: labels[cid] for cid in gold_ids}
+        source = "labels file"
+    else:
+        assert gold_scorer is not None
+        for r in run.results:
+            s = r.scores.get(gold_scorer)
+            if s is not None and s.passed is not None and r.case_id in judged:
+                truth_all[r.case_id] = s.passed
+        ids = sorted(truth_all)
+        if gold_sample is not None and gold_sample < len(ids):
+            ids = sorted(random.Random(seed).sample(ids, gold_sample))
+            source = f"{gold_scorer} (random sample of {gold_sample})"
+        else:
+            source = gold_scorer
+        gold_map = {cid: truth_all[cid] for cid in ids}
+
+    if not gold_map:
+        raise ValueError("no overlap between gold labels and judged cases")
+    lab_ids = sorted(gold_map)
+    unl_ids = sorted(set(judged) - set(lab_ids))
+    jl = [judged[c] for c in lab_ids]
+    gl = [gold_map[c] for c in lab_ids]
+    ja = [judged[c] for c in sorted(judged)]
+    estimates = [
+        raw_estimate(ja, alpha),
+        gold_only_estimate(gl, alpha),
+        acc_estimate(ja, jl, gl, alpha=alpha, n_boot=n_boot, seed=seed),
+        ppi_estimate([judged[c] for c in unl_ids], jl, gl, alpha=alpha),
+    ]
+    truth = None
+    if truth_all and len(truth_all) > len(gold_map):
+        k = sum(truth_all.values())
+        lo, hi = wilson(k, len(truth_all), alpha)
+        truth = Estimate("truth (all gold labels)", k / len(truth_all), lo, hi, len(truth_all))
+    disagreements = [c for c in lab_ids if judged[c] != gold_map[c]]
+    return CalibrationReport(
+        judge_scorer=judge_scorer,
+        gold_source=source,
+        n_judged=len(judged),
+        n_gold=len(gold_map),
+        undecided_judge=undecided,
+        confusion=confusion(jl, gl),
+        estimates=estimates,
+        truth=truth,
+        disagreements=disagreements,
+    )

evalproof/cli.py

@@ -0,0 +1,265 @@
+"""Command line: ``evalproof run | compare | calibrate | report | pairwise``."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from evalproof._version import __version__
+from evalproof.cache import CachedProvider, ResponseCache
+from evalproof.calibration import calibrate
+from evalproof.compare import compare_runs
+from evalproof.config import ConfigError, GateConfig, load_config
+from evalproof.dataset import DatasetError, load_dataset
+from evalproof.gate import EXIT_PASS, EXIT_USAGE, decide
+from evalproof.models import CaseResult
+from evalproof.providers import ProviderError, make_provider
+from evalproof.report import (
+    calibration_markdown,
+    comparison_markdown,
+    run_markdown,
+    to_json,
+)
+from evalproof.runner import load_run, run_suite, save_run
+from evalproof.scorers import pairwise
+
+
+def _emit(markdown: str, path: str | None) -> None:
+    sys.stdout.write(markdown)
+    if path:
+        # Append, so it can target $GITHUB_STEP_SUMMARY alongside other steps.
+        with Path(path).open("a", encoding="utf-8") as fh:
+            fh.write(markdown + "\n")
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+    cfg = load_config(args.suite)
+    override = None
+    if args.system or args.system_prompt_file or args.label:
+        override = cfg.system.model_copy(deep=True)
+        if args.system:
+            override.target = args.system
+        if args.label:
+            override.label = args.label
+        if args.system_prompt_file:
+            override.system_prompt = Path(args.system_prompt_file).read_text(encoding="utf-8")
+
+    def progress(i: int, n: int, res: CaseResult) -> None:
+        if not args.quiet:
+            flag = " (cached)" if res.output.cached else ""
+            err = f" ERROR {res.output.error[:60]}" if res.output.error else ""
+            sys.stderr.write(f"\r[{i}/{n}] {res.case_id}{flag}{err}      ")
+            if i == n:
+                sys.stderr.write("\n")
+
+    run = run_suite(
+        cfg,
+        limit=args.limit,
+        use_cache=not args.no_cache,
+        progress=progress,
+        system_override=override,
+    )
+    out = args.out or f"runs/{cfg.name}-{run.info.started_at.replace(':', '')}.json"
+    save_run(run, out)
+    sys.stderr.write(f"wrote {out}\n")
+    _emit(run_markdown(run), args.markdown)
+    return EXIT_PASS
+
+
+def _gate_config(args: argparse.Namespace) -> GateConfig | None:
+    base: GateConfig | None = None
+    if args.suite:
+        base = load_config(args.suite).gate
+    overrides = {
+        k: v
+        for k, v in {
+            "metric": args.metric,
+            "max_drop": args.max_drop,
+            "alpha": args.alpha,
+            "min_cases": args.min_cases,
+            "max_p95_latency_ms": args.max_p95_ms,
+            "max_cost_usd": args.max_cost,
+        }.items()
+        if v is not None
+    }
+    if base is None and "metric" not in overrides:
+        return None
+    if base is None:
+        return GateConfig(**overrides)
+    return base.model_copy(update=overrides)
+
+
+def cmd_compare(args: argparse.Namespace) -> int:
+    baseline, candidate = load_run(args.baseline), load_run(args.candidate)
+    gate = _gate_config(args)
+    alpha = gate.alpha if gate else (args.alpha or 0.05)
+    cmp = compare_runs(baseline, candidate, alpha=alpha, seed=args.seed)
+    decision = decide(cmp, gate) if gate else None
+    if args.gate and decision is None:
+        raise ConfigError("--gate needs a gate: pass --suite with a gate block, or --metric")
+    _emit(comparison_markdown(cmp, decision, gate.metric if gate else None), args.markdown)
+    if args.json:
+        payload = {"comparison": json.loads(to_json(cmp))}
+        if decision:
+            payload["gate"] = json.loads(to_json(decision))
+        Path(args.json).write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    if args.gate and decision is not None:
+        return decision.exit_code
+    return EXIT_PASS
+
+
+def cmd_calibrate(args: argparse.Namespace) -> int:
+    run = load_run(args.run)
+    labels = None
+    if args.labels:
+        labels = {}
+        for line in Path(args.labels).read_text(encoding="utf-8").splitlines():
+            if line.strip():
+                rec = json.loads(line)
+                labels[str(rec["id"])] = bool(rec["label"])
+    rep = calibrate(
+        run,
+        args.judge_scorer,
+        labels=labels,
+        gold_scorer=args.gold_scorer,
+        gold_sample=args.gold_sample,
+        seed=args.seed,
+        alpha=args.alpha,
+    )
+    _emit(calibration_markdown(rep), args.markdown)
+    if args.json:
+        Path(args.json).write_text(to_json(rep), encoding="utf-8")
+    return EXIT_PASS
+
+
+def cmd_report(args: argparse.Namespace) -> int:
+    _emit(run_markdown(load_run(args.run)), args.markdown)
+    return EXIT_PASS
+
+
+def cmd_pairwise(args: argparse.Namespace) -> int:
+    cfg = load_config(args.suite)
+    if cfg.judge is None:
+        raise ConfigError("pairwise needs a 'judge' in the suite")
+    judge = make_provider(cfg.judge)
+    if not args.no_cache:
+        judge = CachedProvider(judge, ResponseCache(cfg.resolve(cfg.cache_dir)), cfg.judge.base_url)
+    cases = {c.id: c for c in load_dataset(cfg.resolve(cfg.dataset))}
+    a_by, b_by = load_run(args.baseline).by_id(), load_run(args.candidate).by_id()
+    ids = [cid for cid in sorted(set(a_by) & set(b_by)) if cid in cases][: args.limit]
+    tally = {"a": 0, "b": 0, "tie": 0}
+    inconsistent = 0
+    rows = []
+    for cid in ids:
+        res = pairwise(
+            judge,
+            cfg.judge.params,
+            cases[cid],
+            a_by[cid].output.output,
+            b_by[cid].output.output,
+            swap=not args.no_swap,
+        )
+        tally[res.winner] += 1
+        inconsistent += 0 if res.consistent else 1
+        rows.append({"id": cid, **res.__dict__})
+    n = len(ids)
+    md = [
+        "## Pairwise preference",
+        "",
+        f"{n} cases · baseline wins {tally['a']} · candidate wins {tally['b']} · "
+        f"ties {tally['tie']}",
+    ]
+    if not args.no_swap:
+        md.append(
+            f"Position-inconsistent verdicts (flipped when the order was swapped): "
+            f"{inconsistent}/{n} ({100 * inconsistent / n if n else 0:.1f}%) — reported as ties"
+        )
+    _emit("\n".join(md) + "\n", args.markdown)
+    if args.json:
+        Path(args.json).write_text(json.dumps(rows, indent=2), encoding="utf-8")
+    return EXIT_PASS
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="evalproof", description=__doc__)
+    p.add_argument("--version", action="version", version=f"evalproof {__version__}")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    r = sub.add_parser("run", help="run a suite and write a run artifact")
+    r.add_argument("suite")
+    r.add_argument("--out")
+    r.add_argument("--limit", type=int)
+    r.add_argument("--no-cache", action="store_true")
+    r.add_argument("--system", help="override the system target")
+    r.add_argument("--system-prompt-file", help="override the system prompt")
+    r.add_argument("--label", help="name this system in reports")
+    r.add_argument("--markdown", help="also append the report to this file")
+    r.add_argument("--quiet", action="store_true")
+    r.set_defaults(func=cmd_run)
+
+    c = sub.add_parser(
+        "compare", help="compare two runs; --gate turns the result into an exit code"
+    )
+    c.add_argument("baseline")
+    c.add_argument("candidate")
+    c.add_argument("--suite", help="take gate settings from this suite")
+    c.add_argument("--metric")
+    c.add_argument("--max-drop", type=float)
+    c.add_argument("--alpha", type=float)
+    c.add_argument("--min-cases", type=int)
+    c.add_argument("--max-p95-ms", type=float)
+    c.add_argument("--max-cost", type=float)
+    c.add_argument("--gate", action="store_true", help="exit non-zero when the gate fails")
+    c.add_argument("--seed", type=int, default=0)
+    c.add_argument("--markdown")
+    c.add_argument("--json")
+    c.set_defaults(func=cmd_compare)
+
+    k = sub.add_parser("calibrate", help="measure a judge against gold labels and correct its rate")
+    k.add_argument("run")
+    k.add_argument("--judge-scorer", required=True)
+    src = k.add_mutually_exclusive_group(required=True)
+    src.add_argument("--labels", help="JSONL of {id, label} human labels")
+    src.add_argument("--gold-scorer", help="a trusted scorer in the same run")
+    k.add_argument(
+        "--gold-sample", type=int, help="use a random sample of this size as the gold set"
+    )
+    k.add_argument("--seed", type=int, default=0)
+    k.add_argument("--alpha", type=float, default=0.05)
+    k.add_argument("--markdown")
+    k.add_argument("--json")
+    k.set_defaults(func=cmd_calibrate)
+
+    rep = sub.add_parser("report", help="summarise one run")
+    rep.add_argument("run")
+    rep.add_argument("--markdown")
+    rep.set_defaults(func=cmd_report)
+
+    pw = sub.add_parser("pairwise", help="judge baseline vs candidate outputs head to head")
+    pw.add_argument("baseline")
+    pw.add_argument("candidate")
+    pw.add_argument("--suite", required=True)
+    pw.add_argument("--limit", type=int)
+    pw.add_argument("--no-swap", action="store_true")
+    pw.add_argument("--no-cache", action="store_true")
+    pw.add_argument("--markdown")
+    pw.add_argument("--json")
+    pw.set_defaults(func=cmd_pairwise)
+    return p
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        code: int = args.func(args)
+    except (ConfigError, DatasetError, ProviderError, FileNotFoundError, ValueError) as exc:
+        sys.stderr.write(f"evalproof: error: {exc}\n")
+        return EXIT_USAGE
+    return code
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())