eval-builder 0.1.0__py3-none-any.whl

eval_builder/io.py

@@ -0,0 +1,82 @@
+"""Small file helpers shared by every step."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Iterable, Iterator
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def sha256_file(path: str | Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def read_jsonl(path: str | Path) -> Iterator[dict[str, Any]]:
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                yield json.loads(line)
+
+
+def write_jsonl(path: str | Path, rows: Iterable[dict[str, Any]]) -> int:
+    n = 0
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(row, ensure_ascii=False, sort_keys=False) + "\n")
+            n += 1
+    return n
+
+
+def write_json(path: str | Path, data: Any) -> None:
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+        f.write("\n")
+
+
+def read_json(path: str | Path) -> Any:
+    with open(path, encoding="utf-8") as f:
+        return json.load(f)
+
+
+class _LiteralDumper(yaml.SafeDumper):
+    """Writes multi-line strings as YAML block literals so case files stay readable."""
+
+
+def _str_presenter(dumper: yaml.SafeDumper, data: str) -> yaml.ScalarNode:
+    if "\n" in data:
+        # Trailing spaces before newlines break block style; strip them.
+        cleaned = "\n".join(line.rstrip() for line in data.split("\n"))
+        return dumper.represent_scalar("tag:yaml.org,2002:str", cleaned, style="|")
+    return dumper.represent_scalar("tag:yaml.org,2002:str", data)
+
+
+_LiteralDumper.add_representer(str, _str_presenter)
+
+
+def dump_yaml(data: Any) -> str:
+    return yaml.dump(data, Dumper=_LiteralDumper, sort_keys=False, allow_unicode=True, width=100)
+
+
+def write_yaml(path: str | Path, data: Any, header: str = "") -> None:
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        if header:
+            for line in header.strip().splitlines():
+                f.write(f"# {line}\n" if line else "#\n")
+        f.write(dump_yaml(data))
+
+
+def read_yaml(path: str | Path) -> Any:
+    with open(path, encoding="utf-8") as f:
+        return yaml.safe_load(f)

eval_builder/judge/__init__.py

@@ -0,0 +1 @@
+"""Judge reliability: plan repeated judge calls and probes, run them (opt-in), check them."""

eval_builder/judge/check.py

@@ -0,0 +1,420 @@
+"""Measure how far each judge can be trusted.
+
+For every judge: self-consistency over repeated trials (flip rate), agreement with
+human labels (accuracy with a Wilson interval, Cohen's kappa with an approximate
+interval), a position probe (swap answer order) and a verbosity probe (pad with
+irrelevant text). A verdict is derived from explicit, configurable thresholds and
+every verdict lists the numbers behind it.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter, defaultdict
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+from ..io import read_jsonl, write_json
+from ..workspace import Workspace
+from .stats import cohen_kappa, kappa_interval, majority, majority_vote_stability, wilson
+
+NO_MAJORITY = "(no majority)"
+POSITIVE = {"pass", "yes", "true", "good", "correct", "acceptable", "1"}
+VERDICTS = ("trustworthy", "unstable", "biased", "misaligned", "not_enough_data")
+
+
+@dataclass
+class Thresholds:
+    min_trials: int = 3
+    min_cases: int = 10
+    max_flip_rate: float = 0.2
+    min_position_consistency: float = 0.8
+    max_toward_padded_rate: float = 0.1
+    min_kappa: float = 0.4
+    min_labeled: int = 20
+
+
+_PAIR_A = re.compile(r"^\W*(?:\[\[)?\s*(?:answer\s*|assistant\s*|response\s*)?a\b", re.I)
+_PAIR_B = re.compile(r"^\W*(?:\[\[)?\s*(?:answer\s*|assistant\s*|response\s*)?b\b", re.I)
+_PAIR_TIE = re.compile(r"^\W*(?:\[\[)?\s*(?:tie|c|equal|draw|both)\b", re.I)
+
+
+def normalize_verdict(v: Any, mode: str) -> str:
+    """Canonical label for a judge's raw output. Unparseable pairwise outputs become 'invalid'."""
+    if v is None:
+        return "invalid"
+    if isinstance(v, bool):
+        return "pass" if v else "fail"
+    if isinstance(v, (int, float)):
+        return str(int(v)) if float(v).is_integer() else str(v)
+    s = str(v).strip()
+    if mode == "pairwise":
+        if _PAIR_TIE.match(s):
+            return "tie"
+        if _PAIR_A.match(s):
+            return "A"
+        if _PAIR_B.match(s):
+            return "B"
+        if s in ("1", "first"):
+            return "A"
+        if s in ("2", "second"):
+            return "B"
+        return "invalid"
+    s = s.lower().strip(" .!\"'")
+    try:
+        f = float(s)
+        return str(int(f)) if f.is_integer() else str(f)
+    except ValueError:
+        return s or "invalid"
+
+
+def normalize_label(v: Any, mode: str) -> str:
+    if mode == "pairwise" and isinstance(v, str) and v.lower() in ("model_a", "a"):
+        return "A"
+    if mode == "pairwise" and isinstance(v, str) and v.lower() in ("model_b", "b"):
+        return "B"
+    return normalize_verdict(v, mode)
+
+
+def unswap(label: str) -> str:
+    return {"A": "B", "B": "A"}.get(label, label)
+
+
+def _favorable(label: str | None, other: str | None, mode: str, pad_side: str | None) -> bool:
+    """Did the verdict move toward the padded answer?"""
+    if label is None:
+        return False
+    if mode == "pairwise":
+        return label == pad_side and other != pad_side
+    try:
+        return other is not None and float(label) > float(other)
+    except ValueError:
+        return label in POSITIVE and (other not in POSITIVE)
+
+
+def _mean(xs: list[float]) -> float | None:
+    return round(sum(xs) / len(xs), 4) if xs else None
+
+
+def _rate(k: int, n: int) -> dict[str, Any]:
+    ci = wilson(k, n)
+    return {
+        "k": k,
+        "n": n,
+        "rate": round(k / n, 4) if n else None,
+        "ci95": [round(ci[0], 4), round(ci[1], 4)] if ci else None,
+    }
+
+
+def _load_judgments(path: Path, requests_path: Path | None) -> list[dict[str, Any]]:
+    req_index: dict[str, dict[str, Any]] = {}
+    if requests_path and requests_path.exists():
+        req_index = {r["request_id"]: r for r in read_jsonl(requests_path)}
+    rows = []
+    for r in read_jsonl(path):
+        if "case_id" not in r and r.get("request_id") in req_index:
+            base = {
+                k: v
+                for k, v in req_index[r["request_id"]].items()
+                if k not in ("presented", "prompt")
+            }
+            r = {**base, **r}
+        if "judge" not in r or "case_id" not in r:
+            raise ValueError(f"judgment row missing judge/case_id: {r}")
+        rows.append(r)
+    return rows
+
+
+def load_labels(path: Path, mode_by_case: dict[str, str] | None = None) -> dict[str, str]:
+    """Human labels per case. Several rows for one case are resolved by strict majority."""
+    raw: dict[str, list[str]] = defaultdict(list)
+    for r in read_jsonl(path):
+        cid = str(r["case_id"])
+        mode = r.get("mode") or (mode_by_case or {}).get(cid, "pointwise")
+        raw[cid].append(normalize_label(r.get("label"), mode))
+    out = {}
+    for cid, labels in raw.items():
+        lab, _ = majority(labels)
+        if lab is not None:
+            out[cid] = lab
+    return out
+
+
+def check_judge(
+    rows: list[dict[str, Any]], labels: dict[str, str], th: Thresholds
+) -> dict[str, Any]:
+    mode = rows[0].get("mode", "pointwise")
+    by_probe: dict[str, dict[str, list[str]]] = defaultdict(lambda: defaultdict(list))
+    raw_presented: list[str] = []
+    pad_side: dict[str, str | None] = {}
+    invalid = 0
+    for r in rows:
+        label = normalize_verdict(r.get("verdict"), mode)
+        if label == "invalid":
+            invalid += 1
+        probe = r.get("probe") or "none"
+        if mode == "pairwise" and probe in ("none", "swap") and label in ("A", "B"):
+            raw_presented.append(label)
+        if probe == "swap":
+            label = unswap(label)
+        by_probe[probe][str(r["case_id"])].append(label)
+        if probe == "pad":
+            pad_side[str(r["case_id"])] = r.get("pad_side")
+
+    base = by_probe.get("none", {})
+    per_case: dict[str, dict[str, Any]] = {}
+    for cid, labs in base.items():
+        maj, share = majority(labs)
+        per_case[cid] = {
+            "case_id": cid,
+            "trials": len(labs),
+            "counts": dict(Counter(labs)),
+            "majority": maj,
+            "majority_share": round(share, 4),
+            "flipped": len(set(labs)) > 1,
+            "majority_of_3_stability": majority_vote_stability(Counter(labs), 3),
+        }
+    eligible = [c for c in per_case.values() if c["trials"] >= th.min_trials]
+    flipped = sum(1 for c in eligible if c["flipped"])
+    stability = {
+        "cases": len(eligible),
+        "trials_per_case": sorted({c["trials"] for c in per_case.values()}),
+        "flip_rate": _rate(flipped, len(eligible)),
+        "mean_self_agreement": round(sum(c["majority_share"] for c in eligible) / len(eligible), 4)
+        if eligible
+        else None,
+        "no_majority_cases": sum(1 for c in eligible if c["majority"] is None),
+        "majority_of_3_stability": _mean(
+            [
+                c["majority_of_3_stability"]
+                for c in eligible
+                if c["majority_of_3_stability"] is not None
+            ]
+        ),
+        "invalid_outputs": invalid,
+        "calls": len(rows),
+    }
+
+    # agreement with human labels, using each case's majority verdict
+    agreement: dict[str, Any] | None = None
+    labeled = [c for c in eligible if c["case_id"] in labels]
+    if labeled:
+        judge_labels = [c["majority"] or NO_MAJORITY for c in labeled]
+        human = [labels[c["case_id"]] for c in labeled]
+        for c in labeled:
+            c["human"] = labels[c["case_id"]]
+        correct = sum(j == h for j, h in zip(judge_labels, human, strict=True))
+        kappa, po, pe = cohen_kappa(judge_labels, human)
+        kci = kappa_interval(po, pe, len(labeled))
+        single = [(lab, labels[cid]) for cid, labs in base.items() if cid in labels for lab in labs]
+        agreement = {
+            "cases": len(labeled),
+            "accuracy": _rate(correct, len(labeled)),
+            "kappa": round(kappa, 4) if kappa is not None else None,
+            "kappa_ci95": [round(kci[0], 4), round(kci[1], 4)] if kci else None,
+            "chance_agreement": round(pe, 4),
+            "single_call_accuracy": round(sum(a == b for a, b in single) / len(single), 4)
+            if single
+            else None,
+            "human_label_counts": dict(Counter(human)),
+            "judge_label_counts": dict(Counter(judge_labels)),
+        }
+
+    # position probe: majority in swapped order vs original order
+    position: dict[str, Any] | None = None
+    if mode == "pairwise" and by_probe.get("swap"):
+        pairs = []
+        for cid, labs in by_probe["swap"].items():
+            smaj, _ = majority(labs)
+            if cid in per_case:
+                per_case[cid]["swap_majority"] = smaj
+                pairs.append((per_case[cid]["majority"], smaj))
+        consistent = sum(1 for a, b in pairs if a is not None and a == b)
+        first = sum(1 for x in raw_presented if x == "A")
+        position = {
+            "cases": len(pairs),
+            "consistency": _rate(consistent, len(pairs)),
+            "first_position_rate": _rate(first, len(raw_presented)),
+            "note": "consistency = share of cases whose majority verdict survives swapping the "
+            "answer order; first_position_rate = share of A/B verdicts naming whichever "
+            "answer was shown first (0.5 means no position preference)",
+        }
+
+    # verbosity probe: majority with irrelevant padding vs original
+    verbosity: dict[str, Any] | None = None
+    if by_probe.get("pad"):
+        moved = toward = 0
+        n = 0
+        for cid, labs in by_probe["pad"].items():
+            pmaj, _ = majority(labs)
+            if cid not in per_case:
+                continue
+            n += 1
+            orig = per_case[cid]["majority"]
+            per_case[cid]["pad_majority"] = pmaj
+            if pmaj != orig:
+                moved += 1
+                if _favorable(pmaj, orig, mode, pad_side.get(cid)):
+                    toward += 1
+        verbosity = {
+            "cases": n,
+            "changed": _rate(moved, n),
+            "toward_padded": _rate(toward, n),
+            "note": "padding appends an irrelevant paragraph; toward_padded = share of cases "
+            "where the padded answer's verdict improved",
+        }
+
+    verdict, reasons, checks = _verdict(stability, agreement, position, verbosity, th)
+    return {
+        "mode": mode,
+        "verdict": verdict,
+        "reasons": reasons,
+        "checks": checks,
+        "stability": stability,
+        "human_agreement": agreement,
+        "position_probe": position,
+        "verbosity_probe": verbosity,
+        "cases": sorted(per_case.values(), key=lambda c: c["case_id"]),
+    }
+
+
+def _verdict(
+    stability: dict[str, Any],
+    agreement: dict[str, Any] | None,
+    position: dict[str, Any] | None,
+    verbosity: dict[str, Any] | None,
+    th: Thresholds,
+) -> tuple[str, list[str], dict[str, str]]:
+    reasons: list[str] = []
+    checks: dict[str, str] = {}
+    if stability["cases"] < th.min_cases:
+        checks["stability"] = "not enough data"
+        reasons.append(
+            f"only {stability['cases']} case(s) with >= {th.min_trials} trials "
+            f"(need {th.min_cases})"
+        )
+        return "not_enough_data", reasons, checks
+    fr = stability["flip_rate"]["rate"]
+    unstable = fr > th.max_flip_rate
+    checks["stability"] = "fail" if unstable else "pass"
+    if unstable:
+        reasons.append(
+            f"verdict changed across repeated trials on {fr:.0%} of cases "
+            f"(limit {th.max_flip_rate:.0%})"
+        )
+    biased = False
+    if position and position["cases"] >= th.min_cases:
+        pc = position["consistency"]["rate"]
+        if pc < th.min_position_consistency:
+            biased = True
+            checks["position"] = "fail"
+            reasons.append(
+                f"verdict survived swapping answer order on only {pc:.0%} of cases "
+                f"(need {th.min_position_consistency:.0%}); first-shown answer "
+                f"picked {position['first_position_rate']['rate']:.0%} of the time"
+            )
+        else:
+            checks["position"] = "pass"
+    else:
+        checks["position"] = "not run" if not position else "not enough data"
+    if verbosity and verbosity["cases"] >= th.min_cases:
+        tp = verbosity["toward_padded"]["rate"]
+        if tp > th.max_toward_padded_rate:
+            biased = True
+            checks["verbosity"] = "fail"
+            reasons.append(
+                f"irrelevant padding moved the verdict toward the padded answer on "
+                f"{tp:.0%} of cases (limit {th.max_toward_padded_rate:.0%})"
+            )
+        else:
+            checks["verbosity"] = "pass"
+    else:
+        checks["verbosity"] = "not run" if not verbosity else "not enough data"
+    misaligned = False
+    if agreement and agreement["cases"] >= th.min_labeled:
+        k = agreement["kappa"]
+        if k is None or k < th.min_kappa:
+            misaligned = True
+            checks["human_agreement"] = "fail"
+            reasons.append(
+                f"agreement with human labels is low: kappa "
+                f"{'undefined' if k is None else f'{k:.2f}'} (need {th.min_kappa}), "
+                f"accuracy {agreement['accuracy']['rate']:.0%}"
+            )
+        else:
+            checks["human_agreement"] = "pass"
+    else:
+        checks["human_agreement"] = "not enough data"
+    if unstable:
+        return "unstable", reasons, checks
+    if biased:
+        return "biased", reasons, checks
+    if misaligned:
+        return "misaligned", reasons, checks
+    if checks["human_agreement"] != "pass":
+        have = agreement["cases"] if agreement else 0
+        reasons.append(
+            f"stable, but only {have} case(s) have human labels (need "
+            f"{th.min_labeled}); cannot tell whether it measures the right thing"
+        )
+        return "not_enough_data", reasons, checks
+    k = agreement["kappa"] if agreement else None
+    reasons.append(
+        f"stable (flip rate {fr:.0%}), kappa {k:.2f} with humans"
+        + (
+            ""
+            if checks["position"] == "pass" or checks["verbosity"] == "pass"
+            else "; bias probes not run"
+        )
+    )
+    return "trustworthy", reasons, checks
+
+
+def judge_check(
+    workspace: str | Path,
+    judgments: str | Path | None = None,
+    labels: str | Path | None = None,
+    thresholds: Thresholds | None = None,
+) -> dict[str, Any]:
+    ws = Workspace.at(workspace)
+    th = thresholds or Thresholds()
+    jpath = Path(judgments) if judgments else ws.judgments
+    if not jpath.exists():
+        raise FileNotFoundError(f"{jpath} not found; record judge outputs first")
+    rows = _load_judgments(jpath, ws.judge_requests)
+    by_judge: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for r in rows:
+        by_judge[str(r["judge"])].append(r)
+    mode_by_case = {str(r["case_id"]): r.get("mode", "pointwise") for r in rows}
+    lpath = Path(labels) if labels else ws.labels
+    human = load_labels(lpath, mode_by_case) if lpath.exists() else {}
+    judges = {jid: check_judge(jrows, human, th) for jid, jrows in sorted(by_judge.items())}
+    result = {
+        "judgments_file": str(jpath),
+        "labels_file": str(lpath) if lpath.exists() else None,
+        "labeled_cases": len(human),
+        "thresholds": asdict(th),
+        "summary": [
+            {
+                "judge": jid,
+                "mode": r["mode"],
+                "verdict": r["verdict"],
+                "flip_rate": r["stability"]["flip_rate"]["rate"],
+                "kappa": (r["human_agreement"] or {}).get("kappa"),
+                "accuracy": ((r["human_agreement"] or {}).get("accuracy") or {}).get("rate"),
+                "position_consistency": ((r["position_probe"] or {}).get("consistency") or {}).get(
+                    "rate"
+                ),
+                "toward_padded": ((r["verbosity_probe"] or {}).get("toward_padded") or {}).get(
+                    "rate"
+                ),
+                "reasons": r["reasons"],
+            }
+            for jid, r in judges.items()
+        ],
+        "judges": judges,
+        "trustworthy": [j for j, r in judges.items() if r["verdict"] == "trustworthy"],
+    }
+    write_json(ws.judge_check, result)
+    return result

eval_builder/judge/plan.py

@@ -0,0 +1,151 @@
+"""Build the list of judge calls (with probes) that someone has to run."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from ..draft import load_rubric, ready_cases
+from ..io import write_jsonl
+from ..workspace import Workspace
+
+PROBES = ("swap", "pad")
+
+# Deliberately irrelevant filler for the verbosity probe. It adds length without
+# adding anything that answers the question, so a judge's verdict should not move.
+PAD_TEXT = (
+    "Additional background: The metric system was first adopted in France in 1795, and "
+    "the meter was originally defined as one ten-millionth of the distance from the equator "
+    "to the North Pole along a meridian. Over the following two centuries most countries "
+    "adopted it for trade and science. Standard units have been redefined several times, "
+    "most recently in 2019, when the kilogram, ampere, kelvin and mole were tied to fixed "
+    "values of physical constants. Many everyday objects, from paper sizes to bottle volumes, "
+    "follow metric conventions. This background is provided for completeness."
+)
+
+
+def pad(text: str) -> str:
+    return f"{text.rstrip()}\n\n{PAD_TEXT}"
+
+
+def render(template: str | None, fields: dict[str, str]) -> str | None:
+    if not template:
+        return None
+    out = template
+    for k, v in fields.items():
+        out = out.replace("{" + k + "}", v)
+    return out
+
+
+def _presented(case: dict[str, Any], mode: str, probe: str, pad_side: str | None) -> dict[str, str]:
+    inp = str(case.get("input", ""))
+    out = str(case.get("observed_output") or "")
+    if mode == "pointwise":
+        return {"input": inp, "output": pad(out) if probe == "pad" else out}
+    a, b = out, str(case.get("compare_output") or "")
+    if probe == "pad":
+        if pad_side == "A":
+            a = pad(a)
+        else:
+            b = pad(b)
+    if probe == "swap":
+        a, b = b, a
+    return {"input": inp, "answer_a": a, "answer_b": b}
+
+
+def build_requests(
+    cases: list[dict[str, Any]],
+    rubric: dict[str, Any],
+    judges: list[str] | None = None,
+    trials: int = 5,
+    probes: list[str] | None = None,
+    probe_trials: int = 3,
+) -> list[dict[str, Any]]:
+    if trials < 1 or probe_trials < 1:
+        raise ValueError("trials and probe_trials must be >= 1")
+    probes = list(probes or [])
+    for p in probes:
+        if p not in PROBES:
+            raise ValueError(f"unknown probe {p!r}; expected one of {PROBES}")
+    all_judges = {j["id"]: j for j in rubric.get("judges") or []}
+    if not all_judges:
+        raise ValueError("rubric.yaml defines no judges")
+    chosen = judges or list(all_judges)
+    missing = [j for j in chosen if j not in all_judges]
+    if missing:
+        raise ValueError(f"judges not in rubric.yaml: {missing}")
+    crit_desc = {c["id"]: c.get("description", "") for c in rubric.get("criteria") or []}
+    rows: list[dict[str, Any]] = []
+    for jid in chosen:
+        judge = all_judges[jid]
+        mode = judge.get("mode", "pointwise")
+        for ci, case in enumerate(cases):
+            if mode == "pairwise" and not case.get("compare_output"):
+                continue
+            criteria = [c for c in case.get("criteria") or [] if c in crit_desc]
+            criteria_text = "\n".join(f"- {c}: {crit_desc[c]}" for c in criteria)
+            context_text = (
+                "\n\n".join(
+                    f"{m.get('role', 'user')}: {m.get('content', '')}"
+                    for m in case.get("context") or []
+                )
+                or "(none)"
+            )
+            # swap only makes sense for pairwise judges
+            plan = [("none", trials)] + [
+                (p, probe_trials) for p in probes if not (p == "swap" and mode != "pairwise")
+            ]
+            for probe, n in plan:
+                pad_side = None
+                if probe == "pad":
+                    pad_side = "output" if mode == "pointwise" else ("A" if ci % 2 == 0 else "B")
+                shown = _presented(case, mode, probe, pad_side)
+                fields = {
+                    **shown,
+                    "context": context_text,
+                    "expected_behavior": str(case.get("expected_behavior", "")),
+                    "criteria": criteria_text,
+                }
+                for t in range(n):
+                    rows.append(
+                        {
+                            "request_id": f"{jid}:{case['id']}:{probe}:{t}",
+                            "judge": jid,
+                            "case_id": case["id"],
+                            "mode": mode,
+                            "probe": probe,
+                            "trial": t,
+                            "pad_side": pad_side,
+                            "labels": judge.get("labels") or [],
+                            "presented": shown,
+                            "expected_behavior": case.get("expected_behavior"),
+                            "criteria": criteria,
+                            "prompt": render(judge.get("prompt"), fields),
+                        }
+                    )
+    return rows
+
+
+def judge_plan(
+    workspace: str | Path,
+    judges: list[str] | None = None,
+    trials: int = 5,
+    probes: list[str] | None = None,
+    probe_trials: int = 3,
+) -> dict[str, Any]:
+    ws = Workspace.at(workspace)
+    rows = build_requests(
+        ready_cases(ws.root), load_rubric(ws.root), judges, trials, probes, probe_trials
+    )
+    write_jsonl(ws.judge_requests, rows)
+    per_judge: dict[str, int] = {}
+    for r in rows:
+        per_judge[r["judge"]] = per_judge.get(r["judge"], 0) + 1
+    return {
+        "requests_file": str(ws.judge_requests),
+        "requests": len(rows),
+        "per_judge": per_judge,
+        "next": "run every request and append {request_id, verdict} rows (plus the request "
+        "fields) to judgments.jsonl, or use `eval-builder judge-run` with your own "
+        "judge command, then `eval-builder judge-check`",
+    }