poorjev 0.1.0__py3-none-any.whl

poorjev/__init__.py

@@ -0,0 +1,33 @@
+"""poorjev: the poor man's Jev.
+
+A local-first "System One" decision layer. Ask typed questions about some state,
+get typed answers with calibrated confidence, in one pass, with no API key by
+default. The one thing we prove: the confidence is honest.
+
+M1 ships the contract (the three primitives + typed answers). Backends (M2/M5)
+and calibration (M4) build on top without ever being able to break schema
+validity, which is structural. See PRD.md.
+"""
+
+from .primitives import (
+    Choice,
+    Score,
+    Noul,
+    ChoiceAnswer,
+    ScoreAnswer,
+    NoulAnswer,
+)
+from .client import Client
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Client",
+    "Choice",
+    "Score",
+    "Noul",
+    "ChoiceAnswer",
+    "ScoreAnswer",
+    "NoulAnswer",
+    "__version__",
+]

poorjev/backends/__init__.py

@@ -0,0 +1,4 @@
+"""Pluggable backends. M1 ships none; the local NLI backend lands in M2 and the
+opt-in LLM backend in M5. A backend's whole job is to turn state + a primitive's
+hypotheses into a vector of raw scores; the primitive does the rest.
+"""

poorjev/backends/local_nli.py

@@ -0,0 +1,108 @@
+"""Local, keyless NLI backend, the hero.
+
+One small natural-language-inference model does all three primitives by scoring
+(state, hypothesis) pairs for entailment:
+
+- Noul   -> P(entailment) of the statement given the state = P(true).
+- Choice -> score each option as a hypothesis, softmax across options.
+- Score  -> same, over the ordered levels.
+
+Why NLI and not a sampled LLM: it is one forward pass (Jev's speed ballpark, not
+seconds), it is fully local and offline after a one-time ~400MB download, and its
+softmax output is a real probability we can calibrate directly, no sampling tax.
+It is not the smartest option; that is the point of the optional [llm] backend.
+Here, calibration + abstention are what make moderate intelligence safe: the
+model knows when it is unsure and escalates.
+
+The heavy imports (torch, transformers) are lazy so importing poorjev, and the
+whole M1 contract, stays dependency-free.
+"""
+
+from __future__ import annotations
+
+DEFAULT_MODEL = "MoritzLaurer/deberta-v3-base-zeroshot-v2.0"
+
+
+class LocalNLIBackend:
+    """Scores (premise, hypothesis) pairs and returns P(entailment) for each.
+
+    This is the only method the Client needs. Everything primitive-specific
+    (templating options, normalising across a Choice) lives in the Client so a
+    future backend can reuse the same orchestration.
+    """
+
+    def __init__(self, model_name: str = DEFAULT_MODEL, device: str | None = None,
+                 batch_size: int = 16, max_length: int = 512):
+        self.model_name = model_name
+        self.batch_size = batch_size
+        self.max_length = max_length
+        self._device = device
+        self._tokenizer = None
+        self._model = None
+        self._entail_idx: int | None = None
+
+    # -- lazy load -------------------------------------------------------- #
+    def _ensure_loaded(self) -> None:
+        if self._model is not None:
+            return
+        try:
+            import torch
+            from transformers import AutoModelForSequenceClassification, AutoTokenizer
+        except ImportError as e:  # pragma: no cover - env dependent
+            raise ImportError(
+                "The local NLI backend needs the 'local' extra. "
+                "Install it with:  pip install 'poorjev[local]'"
+            ) from e
+
+        if self._device is None:
+            if torch.cuda.is_available():
+                self._device = "cuda"
+            elif getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():
+                self._device = "mps"
+            else:
+                self._device = "cpu"
+
+        try:
+            self._tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+        except Exception:  # deberta-v3 fast tokenizer can be finicky; fall back
+            self._tokenizer = AutoTokenizer.from_pretrained(self.model_name, use_fast=False)
+
+        self._model = AutoModelForSequenceClassification.from_pretrained(self.model_name)
+        self._model.to(self._device)
+        self._model.eval()
+        self._entail_idx = self._find_entail_idx(self._model.config)
+
+    @staticmethod
+    def _find_entail_idx(config) -> int:
+        id2label = getattr(config, "id2label", None) or {}
+        for idx, label in id2label.items():
+            name = str(label).lower()
+            if "entail" in name and "not" not in name and "non" not in name:
+                return int(idx)
+        # Fallback: many NLI heads put entailment first or last; default to 0.
+        return 0
+
+    # -- the one method the Client calls ---------------------------------- #
+    def entail_probs(self, pairs: list[tuple[str, str]]) -> list[float]:
+        """Return P(entailment) in [0, 1] for each (premise, hypothesis) pair,
+        computed in batched forward passes (the single-pass promise)."""
+        if not pairs:
+            return []
+        self._ensure_loaded()
+        import torch
+
+        out: list[float] = []
+        for start in range(0, len(pairs), self.batch_size):
+            chunk = pairs[start:start + self.batch_size]
+            premises = [p for p, _ in chunk]
+            hypotheses = [h for _, h in chunk]
+            enc = self._tokenizer(
+                premises, hypotheses,
+                return_tensors="pt", padding=True,
+                truncation=True, max_length=self.max_length,
+            ).to(self._device)
+            with torch.no_grad():
+                logits = self._model(**enc).logits
+            probs = torch.softmax(logits, dim=-1)[:, self._entail_idx]
+            out.extend(probs.detach().cpu().tolist())
+        return out

poorjev/calibration.py

@@ -0,0 +1,164 @@
+"""Calibration: the one thing poorjev gets provably right.
+
+Two pieces:
+
+1. Temperature scaling. A classifier's probabilities are usually wrong as
+   confidences: this model is overconfident by 0.170 ECE raw. Temperature scaling
+   fits a single scalar T that rescales every distribution, softmax(log p / T),
+   so predicted confidence lines up with real accuracy. T is fit by minimising
+   negative log-likelihood on held-out data. It is monotonic, so it never
+   changes which answer wins, only how confident we are in it.
+
+2. Selective prediction. Given a target risk (max error rate you will tolerate
+   on answered items), pick the confidence threshold that holds that risk on a
+   calibration set. Below it, abstain, that is the honest "I don't know,
+   escalate" signal.
+
+To report an honest "after" number we fit T on training folds and measure the
+calibrated ECE on the held-out fold (k-fold CV), so we never grade calibration
+on data it was fit on. Pure stdlib.
+"""
+
+from __future__ import annotations
+
+import math
+import random
+
+from .metrics import DecisionRecord, ece as _ece
+
+
+# --------------------------------------------------------------------------- #
+# Temperature scaling
+# --------------------------------------------------------------------------- #
+
+def apply_temperature(dist, T: float):
+    """Rescale a probability distribution by temperature T (softmax(log p / T)).
+
+    T = 1 is a no-op. T > 1 softens (less confident); T < 1 sharpens. Argmax is
+    preserved, so the picked class never changes.
+    """
+    if T <= 0:
+        raise ValueError("temperature must be > 0")
+    logits = [math.log(max(p, 1e-12)) / T for p in dist]
+    m = max(logits)
+    exps = [math.exp(l - m) for l in logits]
+    s = sum(exps)
+    return [e / s for e in exps]
+
+
+def _nll(dists, gold_idx, T: float) -> float:
+    total = 0.0
+    for d, g in zip(dists, gold_idx):
+        q = apply_temperature(d, T)
+        total += -math.log(max(q[g], 1e-12))
+    return total / len(dists)
+
+
+def fit_temperature(dists, gold_idx, coarse=None) -> float:
+    """Fit T by minimising NLL, coarse grid then a local refine. Robust and
+    transparent, no optimiser dependency."""
+    if not dists:
+        return 1.0
+    coarse = coarse or [0.25, 0.4, 0.55, 0.7, 0.85, 1.0, 1.25, 1.5,
+                        2.0, 2.5, 3.0, 4.0, 5.0, 6.0]
+    best = min(coarse, key=lambda T: _nll(dists, gold_idx, T))
+    # refine in a window around the coarse winner
+    step = 0.05
+    fine = [round(best + k * step, 4) for k in range(-6, 7) if best + k * step > 0.05]
+    best = min(fine + [best], key=lambda T: _nll(dists, gold_idx, T))
+    return best
+
+
+def _records_with_dist(records: list[DecisionRecord]) -> list[DecisionRecord]:
+    return [r for r in records if r.dist and r.gold_index >= 0]
+
+
+def _recalibrate_record(r: DecisionRecord, T: float) -> DecisionRecord:
+    q = apply_temperature(r.dist, T)
+    return DecisionRecord(
+        confidence=max(q),
+        correct=r.correct,                 # monotonic: winner unchanged
+        prob_gold=q[r.gold_index],
+        n_classes=r.n_classes,
+        task=r.task, question=r.question,
+        dist=tuple(q), gold_index=r.gold_index,
+    )
+
+
+def cross_val_calibrate(records: list[DecisionRecord], k: int = 5, seed: int = 0):
+    """Fit T on train folds, apply to the held-out fold, aggregate.
+
+    Returns (calibrated_records, mean_T). The calibrated records are honest
+    held-out predictions: no record was calibrated by a T fit on itself.
+    """
+    usable = _records_with_dist(records)
+    if len(usable) < k:
+        T = fit_temperature([r.dist for r in usable], [r.gold_index for r in usable])
+        return [_recalibrate_record(r, T) for r in usable], T
+
+    rng = random.Random(seed)
+    idx = list(range(len(usable)))
+    rng.shuffle(idx)
+    folds = [idx[i::k] for i in range(k)]
+
+    calibrated: list[DecisionRecord] = []
+    temps: list[float] = []
+    for f in range(k):
+        test_ids = set(folds[f])
+        train = [usable[i] for i in idx if i not in test_ids]
+        test = [usable[i] for i in folds[f]]
+        T = fit_temperature([r.dist for r in train], [r.gold_index for r in train])
+        temps.append(T)
+        calibrated.extend(_recalibrate_record(r, T) for r in test)
+    return calibrated, sum(temps) / len(temps)
+
+
+def fit_global_temperature(records: list[DecisionRecord]) -> float:
+    """Fit one T on all data, for saving into a deployable calibrator."""
+    usable = _records_with_dist(records)
+    return fit_temperature([r.dist for r in usable], [r.gold_index for r in usable])
+
+
+# --------------------------------------------------------------------------- #
+# Selective prediction (conformal-style thresholding)
+# --------------------------------------------------------------------------- #
+
+def fit_abstention_threshold(records: list[DecisionRecord], target_risk: float = 0.1):
+    """Pick the lowest confidence threshold that keeps error rate on accepted
+    decisions at or below ``target_risk`` on this set.
+
+    Returns (threshold, coverage, realized_risk). Accept a decision at inference
+    when its confidence >= threshold; otherwise abstain and escalate.
+    """
+    if not records:
+        return 1.0, 0.0, 0.0
+    ordered = sorted(records, key=lambda r: r.confidence, reverse=True)
+    n = len(ordered)
+    best = None  # (threshold, coverage, risk) with the largest coverage under risk
+    errors = 0
+    for i, r in enumerate(ordered, start=1):
+        if not r.correct:
+            errors += 1
+        risk = errors / i
+        if risk <= target_risk:
+            best = (r.confidence, i / n, risk)
+    if best is None:
+        # even the single most-confident decision is wrong; abstain on everything
+        return 1.01, 0.0, 0.0
+    return best
+
+
+def summarize_calibration(raw: list[DecisionRecord], k: int = 5, target_risk: float = 0.1):
+    """The M4 headline: ECE before vs after (held-out), plus a selective point."""
+    calibrated, mean_T = cross_val_calibrate(raw, k=k)
+    thr, cov, risk = fit_abstention_threshold(calibrated, target_risk)
+    return {
+        "temperature": mean_T,
+        "ece_before": _ece(_records_with_dist(raw)),
+        "ece_after": _ece(calibrated),
+        "target_risk": target_risk,
+        "abstain_threshold": thr,
+        "coverage_at_target": cov,
+        "risk_at_target": risk,
+        "calibrated_records": calibrated,
+    }

poorjev/cli.py

@@ -0,0 +1,184 @@
+"""poorjev command line.
+
+    poorjev eval   --set evalset/tasks.jsonl [--plots] [--verbose]
+    poorjev ask    --state-file ticket.txt --questions questions.yaml   (M4+)
+    poorjev calibrate --set evalset/tasks.jsonl                          (M4)
+
+M3 ships ``eval``: run the set through the local backend and print accuracy +
+ECE + Brier + AURC, overall and per task. ``--plots`` writes the reliability
+diagram (the launch asset) once matplotlib is installed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+
+def _fmt(summary: dict) -> str:
+    return (
+        f"n={summary['n']:>4}  "
+        f"acc={summary['accuracy']:.3f}  "
+        f"ECE={summary['ece']:.3f}  "
+        f"Brier={summary['brier']:.3f}  "
+        f"AURC={summary['aurc']:.3f}"
+    )
+
+
+def cmd_eval(args) -> int:
+    from .evaluate import evaluate, report
+    from .client import Client
+
+    print(f"Running eval set: {args.set}")
+    print("(first run downloads a ~400MB model; then offline)\n")
+    client = Client()
+    records, _ = evaluate(args.set, client=client, verbose=args.verbose)
+    rep = report(records)
+
+    print("\n== overall ==")
+    print("  " + _fmt(rep["overall"]))
+    print("\n== by task ==")
+    for task, s in rep["by_task"].items():
+        print(f"  {task:<16} " + _fmt(s))
+
+    if args.plots:
+        try:
+            from .plots import reliability_diagram
+            out = reliability_diagram(records, title="poorjev, raw (uncalibrated)",
+                                      path=args.plot_path)
+            print(f"\nwrote reliability diagram -> {out}")
+        except ImportError:
+            print("\n[--plots needs the 'plots' extra: pip install 'poorjev[plots]']")
+    return 0
+
+
+def cmd_calibrate(args) -> int:
+    import json
+    from .evaluate import evaluate
+    from .client import Client
+    from .calibration import summarize_calibration, fit_global_temperature
+
+    print(f"Calibrating on: {args.set}")
+    print("(runs the eval set through the local model once)\n")
+    raw, _ = evaluate(args.set, client=Client())
+
+    summ = summarize_calibration(raw, k=args.folds, target_risk=args.target_risk)
+    global_T = fit_global_temperature(raw)  # the deployable scalar
+
+    print("== calibration (5-fold held-out) ==")
+    print(f"  temperature (mean)   : {summ['temperature']:.3f}")
+    print(f"  ECE  before -> after : {summ['ece_before']:.3f} -> {summ['ece_after']:.3f}")
+    print(f"  selective @ risk<={summ['target_risk']:.2f}: "
+          f"coverage={summ['coverage_at_target']:.2f} "
+          f"(threshold {summ['abstain_threshold']:.3f})")
+
+    saved = {
+        "temperature": global_T,
+        "target_risk": summ["target_risk"],
+        "abstain_threshold": summ["abstain_threshold"],
+        "ece_before": summ["ece_before"],
+        "ece_after": summ["ece_after"],
+    }
+    with open(args.out, "w") as f:
+        json.dump(saved, f, indent=2)
+    print(f"\nsaved calibrator -> {args.out}")
+
+    if args.plots:
+        try:
+            from .plots import reliability_pair, risk_coverage_plot
+            from .calibration import cross_val_calibrate
+            cal, _ = cross_val_calibrate(raw, k=args.folds)
+            p1 = reliability_pair(raw, cal, path=args.plot_path)
+            p2 = risk_coverage_plot(cal)
+            print(f"wrote {p1}")
+            print(f"wrote {p2}")
+        except ImportError:
+            print("[--plots needs the 'plots' extra: pip install 'poorjev[plots]']")
+    return 0
+
+
+def cmd_ask(args) -> int:
+    import json
+    from .client import Client
+
+    temperature = 1.0
+    if args.calibrator:
+        try:
+            with open(args.calibrator) as f:
+                temperature = json.load(f).get("temperature", 1.0)
+        except FileNotFoundError:
+            print(f"[no calibrator at {args.calibrator}; using raw confidence]",
+                  file=sys.stderr)
+
+    if args.state_file:
+        with open(args.state_file) as f:
+            state = f.read().strip()
+    else:
+        state = args.state
+    if not state:
+        print("provide --state or --state-file", file=sys.stderr)
+        return 1
+
+    with open(args.questions) as f:
+        specs = json.load(f)   # {name: {"type":..., ...}}
+
+    from .evaluate import _build_primitive
+    questions = {name: _build_primitive(spec) for name, spec in specs.items()}
+    res = Client(temperature=temperature).ask(state, questions)
+
+    print(json.dumps({
+        name: {
+            "value": ans.value,
+            "confidence": round(ans.confidence, 4),
+        } for name, ans in res.items()
+    }, indent=2))
+    return 0
+
+
+def cmd_serve(args) -> int:
+    from .mcp_server import main as serve_main
+    print("Starting poorjev MCP server (stdio). Add it to your MCP client config.",
+          file=sys.stderr)
+    serve_main(calibrator_path=args.calibrator)
+    return 0
+
+
+def main(argv=None) -> int:
+    p = argparse.ArgumentParser(prog="poorjev", description="the poor man's Jev")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    pe = sub.add_parser("eval", help="score the eval set (accuracy + calibration)")
+    pe.add_argument("--set", default="evalset/tasks.jsonl", help="path to tasks.jsonl")
+    pe.add_argument("--plots", action="store_true", help="write the reliability diagram")
+    pe.add_argument("--plot-path", default="reliability.png")
+    pe.add_argument("--verbose", action="store_true", help="print each item's answers")
+    pe.set_defaults(func=cmd_eval)
+
+    pa = sub.add_parser("ask", help="answer typed questions about one state")
+    pa.add_argument("--state", default="", help="the state text")
+    pa.add_argument("--state-file", help="read state from a file")
+    pa.add_argument("--questions", required=True, help="JSON file: {name: spec}")
+    pa.add_argument("--calibrator", default="calibration.json",
+                    help="JSON from `poorjev calibrate` (applies the fitted temperature)")
+    pa.set_defaults(func=cmd_ask)
+
+    pc = sub.add_parser("calibrate", help="fit calibration on the eval set")
+    pc.add_argument("--set", default="evalset/tasks.jsonl")
+    pc.add_argument("--folds", type=int, default=5)
+    pc.add_argument("--target-risk", type=float, default=0.1)
+    pc.add_argument("--plots", action="store_true", help="write before/after diagrams")
+    pc.add_argument("--plot-path", default="docs/reliability_before_after.png")
+    pc.add_argument("--out", default="calibration.json")
+    pc.set_defaults(func=cmd_calibrate)
+
+    ps = sub.add_parser("serve", help="run the MCP server (for Claude Code etc.)")
+    ps.add_argument("--calibrator", default="calibration.json",
+                    help="calibration.json to apply (fitted temperature)")
+    ps.set_defaults(func=cmd_serve)
+
+    args = p.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

poorjev/client.py

@@ -0,0 +1,96 @@
+"""The Client: ask a batch of typed questions about one state, in a single pass.
+
+The Client owns orchestration so backends stay simple. It:
+  1. turns each question into (state, hypothesis) pairs,
+  2. concatenates every pair across every question into ONE batched call,
+  3. hands the raw entailment probabilities back to each primitive to decide.
+
+A backend only has to implement ``entail_probs(pairs) -> list[float]``. The
+default backend is the local NLI model (keyless, offline). Pass your own backend
+(e.g. a fake in tests, or the future [llm] backend) to swap it out.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+from .primitives import Choice, Score, Noul, ChoiceAnswer, ScoreAnswer, NoulAnswer
+
+DEFAULT_TEMPLATE = "This example is {}."
+
+
+class Backend(Protocol):
+    def entail_probs(self, pairs: list[tuple[str, str]]) -> list[float]: ...
+
+
+class Client:
+    def __init__(self, backend: Backend | None = None,
+                 hypothesis_template: str = DEFAULT_TEMPLATE,
+                 temperature: float = 1.0):
+        self._backend = backend
+        self.hypothesis_template = hypothesis_template
+        # A fitted temperature (from `poorjev calibrate`) makes ask()'s
+        # confidences calibrated. 1.0 is a no-op (raw).
+        self.temperature = temperature
+
+    @property
+    def backend(self) -> Backend:
+        if self._backend is None:
+            from .backends.local_nli import LocalNLIBackend
+            self._backend = LocalNLIBackend()
+        return self._backend
+
+    def ask(self, state: str, questions: dict):
+        """Answer every question about ``state`` in one batched pass.
+
+        Returns a dict mapping each question name to its typed answer
+        (ChoiceAnswer / ScoreAnswer / NoulAnswer).
+        """
+        if not isinstance(state, str) or state == "":
+            raise ValueError("state must be a non-empty string")
+        if not questions:
+            raise ValueError("ask() needs at least one question")
+
+        pairs: list[tuple[str, str]] = []
+        plan: list[tuple[str, object, int, int, str]] = []  # name, prim, start, count, mode
+
+        for name, prim in questions.items():
+            if isinstance(prim, Noul):
+                hyps = [prim.statement]
+                mode = "noul"
+            elif isinstance(prim, (Choice, Score)):
+                hyps = [self.hypothesis_template.format(h) for h in prim.hypotheses]
+                mode = "dist"
+            else:
+                raise TypeError(
+                    f"question {name!r} must be a Choice, Score or Noul, got {type(prim).__name__}"
+                )
+            start = len(pairs)
+            pairs.extend((state, h) for h in hyps)
+            plan.append((name, prim, start, len(hyps), mode))
+
+        probs = self.backend.entail_probs(pairs)  # the single pass
+        if len(probs) != len(pairs):
+            raise RuntimeError(
+                f"backend returned {len(probs)} scores for {len(pairs)} pairs"
+            )
+
+        out: dict[str, object] = {}
+        for name, prim, start, count, mode in plan:
+            seg = probs[start:start + count]
+            if mode == "noul":
+                p_true = seg[0]
+                if self.temperature != 1.0:
+                    from .calibration import apply_temperature
+                    p_true = apply_temperature([1.0 - p_true, p_true], self.temperature)[1]
+                out[name] = prim.decide(p_true, kind="prob")
+            else:
+                # entailment probs per option/level, normalised across the set
+                dist = seg
+                if self.temperature != 1.0:
+                    from .calibration import apply_temperature
+                    total = sum(max(0.0, s) for s in seg) or 1.0
+                    norm = [max(0.0, s) / total for s in seg]
+                    dist = apply_temperature(norm, self.temperature)
+                out[name] = prim.decide(dist, kind="probs")
+        return out

poorjev/evaluate.py

@@ -0,0 +1,95 @@
+"""Run the eval set through a backend and score it into DecisionRecords.
+
+This is the bridge between the labelled data and the metrics. It builds the
+right primitive for each question, asks the Client, then compares the picked
+value to gold and records the confidence and the probability mass on the gold
+answer. M4 will reuse the collected records (and their gold prob mass) to fit
+and evaluate calibration.
+"""
+
+from __future__ import annotations
+
+import json
+
+from .primitives import Choice, Score, Noul
+from .client import Client
+from .metrics import DecisionRecord, summarize
+
+
+def load_tasks(path: str) -> list[dict]:
+    items = []
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                items.append(json.loads(line))
+    return items
+
+
+def _build_primitive(spec: dict):
+    t = spec["type"]
+    if t == "choice":
+        return Choice(spec["options"])
+    if t == "score":
+        return Score(spec["levels"])
+    if t == "noul":
+        return Noul(spec["statement"])
+    raise ValueError(f"unknown question type {t!r}")
+
+
+def _score_one(spec: dict, answer) -> DecisionRecord:
+    t = spec["type"]
+    gold = spec["gold"]
+    if t == "noul":
+        correct = bool(answer.value) == bool(gold)
+        prob_gold = answer.prob if gold else (1.0 - answer.prob)
+        # class order: index 0 = False, index 1 = True
+        dist = (1.0 - answer.prob, answer.prob)
+        gold_index = 1 if gold else 0
+        return DecisionRecord(answer.confidence, correct, prob_gold, n_classes=2,
+                              dist=dist, gold_index=gold_index)
+    # choice / score share the same shape
+    order = spec["options"] if t == "choice" else spec["levels"]
+    probs = answer.probs if t == "choice" else answer.distribution
+    correct = answer.value == gold
+    prob_gold = probs.get(gold, 0.0)
+    dist = tuple(probs[k] for k in order)
+    gold_index = order.index(gold)
+    return DecisionRecord(answer.confidence, correct, prob_gold, n_classes=len(probs),
+                          dist=dist, gold_index=gold_index)
+
+
+def evaluate(path: str, client: Client | None = None, verbose: bool = False):
+    """Return (records, per_question_specs) after running the whole set.
+
+    ``records`` is a flat list of DecisionRecord across every question of every
+    item, tagged with task + question so callers can slice per task.
+    """
+    client = client or Client()
+    items = load_tasks(path)
+    records: list[DecisionRecord] = []
+
+    for item in items:
+        questions = {name: _build_primitive(spec) for name, spec in item["questions"].items()}
+        answers = client.ask(item["state"], questions)
+        for name, spec in item["questions"].items():
+            r = _score_one(spec, answers[name])
+            records.append(DecisionRecord(
+                r.confidence, r.correct, r.prob_gold, r.n_classes,
+                task=item["task"], question=name,
+                dist=r.dist, gold_index=r.gold_index,
+            ))
+        if verbose:
+            print(f"  {item['id']}: " + ", ".join(
+                f"{n}={answers[n].value!r}" for n in item["questions"]
+            ))
+    return records, items
+
+
+def report(records: list[DecisionRecord]) -> dict:
+    """Overall summary plus a per-task breakdown."""
+    out = {"overall": summarize(records), "by_task": {}}
+    tasks = sorted({r.task for r in records})
+    for task in tasks:
+        out["by_task"][task] = summarize([r for r in records if r.task == task])
+    return out