agentstats 0.3.0__py3-none-any.whl

agentstats/policy.py

@@ -0,0 +1,105 @@
+"""
+Release-process tiers, from a TOML policy file (after Yesha Shah, "Agentic Evals" Part 1: evals in CI in tiers).
+
+    design = "curated"              # how the eval set was built (see analysis.py)
+
+    [smoke]                         # every change, minutes: a stratified sample + rule checks
+    sample = 20
+    stratify_by = "feature"         # tag prefix: feature:search, feature:checkout, ...
+    always_include = ["critical"]   # tags always in the sample
+    runs = 1
+    require.rules = 1.0             # every trajectory rule check passes
+    require.min_pass_rate = 0.5     # a crash threshold, not a regression test
+
+    [pr]                            # every pull request: fail only on a proven drop
+    runs = 2
+    require.no_significant_drop = 0.0
+
+    [nightly]                       # the whole eval set, k runs per case
+    runs = 3
+    require.no_regression = 0.03    # vs baseline: CI lower bound of the change >= -3 pts
+    require.max_infra = 0.05
+
+    [release]                       # the whole set, against the production baseline, with cost and latency
+    runs = 5
+    require.no_regression = 0.02
+    require.max_cost_ratio = 1.10
+    require.max_p95_latency_ratio = 1.20
+    require.min_pass_hat_k = 0.70
+"""
+from __future__ import annotations
+
+import tomllib
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from agentstats.analysis import DESIGNS
+
+REQUIREMENTS = {
+    "min_pass_rate": "pass rate (point estimate) at least this; a crash threshold",
+    "rules": "share of trajectory rule checks passing, at least this",
+    "min_rule_rate": "every trajectory rule holds at least this often: PASS when the estimate clears it, FAIL when "
+                     "the whole interval is below it, INCONCLUSIVE in between",
+    "no_rule_regression": "vs baseline, per trajectory rule: FAIL if any rule is proven to hold less often than in "
+                          "the baseline by more than this (fixed-cases test, Bonferroni across rules). Needs no labels",
+    "no_regression": "vs baseline: lower CI bound of (candidate - baseline) at least -margin (non-inferiority: "
+                     "proves it isn't worse; needs many runs)",
+    "no_significant_drop": "vs baseline: FAIL only if the whole CI is below -margin (proves it got worse; "
+                           "otherwise PASS, with the smallest drop this run could catch)",
+    "max_infra": "infrastructure failure rate at most this; above it the run is unhealthy",
+    "max_cost_ratio": "cost per attempt, candidate / baseline, at most this (CI-based)",
+    "max_p95_latency_ratio": "p95 latency, candidate / baseline, at most this (CI-based)",
+    "min_pass_hat_k": "pass^k (k = the tier's runs) at least this (CI-based)",
+    "max_no_answer": "share of attempts that ended with no usable answer (step limit, unparseable output) at most "
+                     "this; a jump usually means the harness or loop broke",
+    "max_slice_drop": "vs baseline, per slice (the tier's slice_by tag prefix): FAIL if any slice is proven worse by "
+                      "more than this (Bonferroni-adjusted interval across slices)",
+}
+NEEDS_BASELINE = ("no_regression", "no_significant_drop", "max_cost_ratio", "max_p95_latency_ratio", "max_slice_drop",
+                  "no_rule_regression")
+
+
+@dataclass
+class Tier:
+    name: str
+    runs: int = 1
+    sample: int | None = None
+    stratify_by: str | None = None
+    always_include: list[str] = field(default_factory=list)
+    seed: int = 0
+    slice_by: str | None = None      # tag prefix to compare per slice against the baseline, e.g. "template"
+    require: dict[str, float] = field(default_factory=dict)
+
+
+@dataclass
+class Policy:
+    design: str
+    cluster_by: str | None
+    tiers: dict[str, Tier]
+
+
+def parse(data: dict) -> Policy:
+    design = data.get("design", "curated")
+    if design not in DESIGNS:
+        raise ValueError(f"design must be one of {DESIGNS}, not {design!r}")
+    tiers = {}
+    for name, body in data.items():
+        if not isinstance(body, dict):
+            continue
+        req = dict(body.get("require") or {})
+        unknown = set(req) - set(REQUIREMENTS)
+        if unknown:
+            raise ValueError(f"[{name}] unknown requirement(s) {sorted(unknown)}; known: {sorted(REQUIREMENTS)}")
+        tiers[name] = Tier(name=name, runs=int(body.get("runs", 1)), sample=body.get("sample"),
+                           stratify_by=body.get("stratify_by"), always_include=list(body.get("always_include") or []),
+                           seed=int(body.get("seed", 0)), slice_by=body.get("slice_by"),
+                           require={k: float(v) for k, v in req.items()})
+        if "max_slice_drop" in req and not tiers[name].slice_by:
+            raise ValueError(f"[{name}] max_slice_drop needs slice_by (a tag prefix, e.g. slice_by = \"template\")")
+    if not tiers:
+        raise ValueError("policy defines no tiers")
+    return Policy(design=design, cluster_by=data.get("cluster_by"), tiers=tiers)
+
+
+def load(path: str | Path) -> Policy:
+    return parse(tomllib.loads(Path(path).read_text()))

agentstats/prereg.py

@@ -0,0 +1,139 @@
+"""
+Pre-registration: write the switch rule down before any result exists, and check later that it was.
+
+    agentstats preregister --name triage-haiku-5-5 --margin 0.03 --provenance current-model \\
+        --input labels=app/evals/triage_heldout.jsonl --out decisions/triage-haiku-5-5
+    git add decisions/triage-haiku-5-5 && git commit -m "pre-register" && git push
+    ... run both arms ...
+    agentstats switch --prereg decisions/triage-haiku-5-5/prereg.json --current ... --candidate ... --out ...
+
+Why: the margin, confidence level and slices are easy to pick after seeing the numbers, and an answer chosen that way
+isn't evidence of anything. A decision only counts as pre-registered when it can be shown, so `decide --prereg` checks:
+
+1. prereg.json is committed in git, and unchanged since that commit;
+2. that commit is older than every result file's last modification (a good-faith check: file times can be changed,
+   but this catches the honest mistake of registering after running);
+3. every declared input (e.g. the labels) still has the hash it had when registered.
+
+If any check fails the decision is still made with the registered rule, but recorded as NOT pre-registered, with the
+reasons. The rule itself always comes from the file; conflicting command-line flags are an error.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import subprocess
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+
+import agentstats
+from agentstats.switch import PROVENANCE, Rule
+
+
+@dataclass
+class Registration:
+    name: str
+    rule: Rule
+    provenance: str
+    slice_by: str | None = None
+    strict_slices: bool = False
+    inputs: dict[str, dict] = field(default_factory=dict)      # name -> {"path", "sha256"}
+    note: str = ""
+    created_at: str = ""
+    agentstats_version: str = ""
+
+
+def _sha256(path: Path) -> str:
+    return hashlib.sha256(Path(path).read_bytes()).hexdigest()
+
+
+def create(out_dir: Path, name: str, rule: Rule, provenance: str, *, slice_by: str | None = None,
+           strict_slices: bool = False, inputs: dict[str, Path] | None = None, note: str = "") -> Path:
+    """Write prereg.json and prereg.md. Refuses to overwrite an existing registration."""
+    if provenance not in PROVENANCE:
+        raise ValueError(f"provenance must be one of {PROVENANCE}")
+    out_dir = Path(out_dir)
+    if (out_dir / "prereg.json").exists():
+        raise FileExistsError(f"{out_dir / 'prereg.json'} exists: a registration is never rewritten")
+    reg = Registration(name, rule, provenance, slice_by, strict_slices,
+                       {k: {"path": str(p), "sha256": _sha256(p)} for k, p in (inputs or {}).items()}, note,
+                       datetime.now(UTC).isoformat(timespec="seconds"), agentstats.__version__)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    data = asdict(reg)
+    (out_dir / "prereg.json").write_text(json.dumps(data, indent=1) + "\n")
+    (out_dir / "prereg.md").write_text(render(data))
+    return out_dir / "prereg.json"
+
+
+def load(path: Path) -> Registration:
+    d = json.loads(Path(path).read_text())
+    return Registration(d["name"], Rule(**d["rule"]), d["provenance"], d.get("slice_by"), d.get("strict_slices", False),
+                        d.get("inputs") or {}, d.get("note", ""), d.get("created_at", ""),
+                        d.get("agentstats_version") or d.get("stepdown_version", ""))  # stepdown-era files
+
+
+def _git(cwd: Path, *args: str) -> subprocess.CompletedProcess:
+    return subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=False)
+
+
+def verify(path: Path, results: list[Path]) -> tuple[bool, list[str], dict]:
+    """Was the registration committed before the results, and are the declared inputs unchanged?"""
+    path = Path(path).resolve()
+    reasons: list[str] = []
+    info: dict = {}
+    d = path.parent
+    top = _git(d, "rev-parse", "--show-toplevel")
+    if top.returncode != 0:
+        return False, ["the registration isn't in a git repository, so when it was written can't be shown"], info
+    root = Path(top.stdout.strip())
+    rel = str(path.relative_to(root.resolve()))
+    d = root                                   # git pathspecs below are relative to the repository root
+    log = _git(d, "log", "-1", "--format=%H %ct", "--", rel)
+    if not log.stdout.strip():
+        return False, ["the registration was never committed"], info
+    commit, ts = log.stdout.split()
+    info.update(commit=commit, committed_at=datetime.fromtimestamp(int(ts), UTC).isoformat(timespec="seconds"))
+    if _git(d, "diff", "--quiet", "HEAD", "--", rel).returncode != 0:
+        reasons.append("the registration has uncommitted changes")
+    first = _git(d, "log", "--diff-filter=A", "--format=%ct", "--", rel).stdout.split()
+    registered_at = int(first[-1]) if first else int(ts)
+    if len(_git(d, "log", "--format=%H", "--", rel).stdout.split()) > 1:
+        reasons.append("the registration was changed after it was first committed")
+    later = [str(p) for p in results if Path(p).stat().st_mtime < registered_at]
+    if later:
+        reasons.append(f"result file(s) older than the registration: {', '.join(later)}")
+    reg = load(path)
+    for name, inp in reg.inputs.items():
+        p = Path(inp["path"])                      # as given at registration: relative to where it ran,
+        if not p.exists():                         # else to the repository root
+            p = root / inp["path"]
+        if not p.exists():
+            reasons.append(f"declared input {name} ({inp['path']}) is missing")
+        elif _sha256(p) != inp["sha256"]:
+            reasons.append(f"declared input {name} ({inp['path']}) changed since registration")
+    info["registered_at"] = datetime.fromtimestamp(registered_at, UTC).isoformat(timespec="seconds")
+    return not reasons, reasons, info
+
+
+def render(d: dict) -> str:
+    r = d["rule"]
+    lines = [f"# Pre-registration: {d['name']}", "",
+             (f"Written {d['created_at']} with agentstats {d['agentstats_version']}, before any result. Commit this "
+              "file before running anything; `agentstats switch --prereg` checks that it was."), "",
+             "## Rule", "",
+             (f"- **SWITCH** if the lower {100 * r['conf']:g}% bound of (candidate - current) is at least "
+              f"-{100 * r['margin']:g} points and the cost ratio's upper bound is at most {r['max_cost_ratio']:g}x;"),
+             f"- **KEEP** if the upper bound is below -{100 * r['margin']:g} points, or it costs more;",
+             "- **NOT YET** otherwise, reported as is.", "",
+             (f"Eval set design: {r['design']}. Infrastructure failures capped at {100 * r['max_infra']:g}%. "
+              f"Eval-set provenance: {d['provenance']}. Slices: "
+              + (f"by `{d['slice_by']}`, {'enforced' if d['strict_slices'] else 'reported, not enforced'}."
+                 if d["slice_by"] else "none.")), ""]
+    if d["inputs"]:
+        lines += ["## Declared inputs (must be unchanged at decision time)", ""]
+        lines += [f"- {k}: `{v['path']}` (sha256 {v['sha256'][:16]})" for k, v in d["inputs"].items()]
+        lines.append("")
+    if d.get("note"):
+        lines += ["## Plan", "", d["note"], ""]
+    return "\n".join(lines)

agentstats/records.py

@@ -0,0 +1,150 @@
+"""
+Attempt records: one row per (arm, run, case) attempt, the unit everything else works on.
+
+    {"arm": "baseline", "run": 1, "case": "django-11099", "passed": true, "infra_error": null,
+     "cost_usd": 0.12, "input_tokens": 41000, "output_tokens": 2100, "steps": 14, "tool_calls": 9,
+     "latency_s": 33.0, "tags": ["search"], "checks": {"no_refund_before_lookup": true}}
+
+A *run* is one full pass over the eval set. Run-level statistics (the right ones for a hand-picked eval set)
+need several runs; per-case statistics need several cases. `infra_error` marks attempts that failed for reasons
+outside the agent (timeout, quota, billing, sandbox crash): they are excluded from pass rates and reported apart.
+`no_answer` marks attempts where the agent produced no usable answer at all (it hit its step limit, or its output
+couldn't be parsed): they count as failures, like any wrong answer, but are reported apart, because a jump in them
+usually means the harness or the loop broke, not that the agent's judgement got worse.
+
+Rows are validated on load, with the field and the value in the error. `schema_version` is optional (default 1);
+a row written for a newer format is refused rather than half-read.
+"""
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+
+NUMERIC_FIELDS = ("cost_usd", "input_tokens", "cached_tokens", "output_tokens", "steps", "tool_calls", "latency_s")
+SCHEMA_VERSION = 1
+
+
+class RowError(ValueError):
+    """A row that doesn't match the attempt format; the message names the field and what was wrong."""
+
+
+@dataclass
+class Attempt:
+    case: str
+    passed: bool | None
+    arm: str = "default"
+    run: int = 1
+    infra_error: str | None = None
+    no_answer: str | None = None      # why there was no usable answer, e.g. "step_limit", "parse_error"
+    tags: list[str] = field(default_factory=list)
+    checks: dict[str, bool] = field(default_factory=dict)
+    metrics: dict[str, float] = field(default_factory=dict)
+
+    @property
+    def valid(self) -> bool:
+        """Counts toward pass rates: graded, and not an infrastructure failure."""
+        return self.infra_error is None and self.passed is not None
+
+    def to_json(self) -> dict:
+        row = {"arm": self.arm, "run": self.run, "case": self.case, "passed": self.passed,
+               "infra_error": self.infra_error, "tags": self.tags, "checks": self.checks}
+        if self.no_answer:
+            row["no_answer"] = self.no_answer
+        row.update(self.metrics)
+        return row
+
+
+def _number(row: dict, k: str) -> float:
+    v = row[k]
+    if isinstance(v, bool) or not isinstance(v, (int, float)):
+        hint = " (a list of steps? write its length)" if isinstance(v, list) else ""
+        raise RowError(f"'{k}' must be a number, got {type(v).__name__} {str(v)[:60]!r}{hint}")
+    return float(v)
+
+
+def from_dict(row: dict, *, arm: str | None = None) -> Attempt:
+    if not isinstance(row, dict):
+        raise RowError(f"a row must be a JSON object, got {type(row).__name__}")
+    version = row.get("schema_version", SCHEMA_VERSION)
+    if not isinstance(version, int) or version > SCHEMA_VERSION:
+        raise RowError(f"schema_version {version!r}: this agentstats reads version {SCHEMA_VERSION}; upgrade it")
+    if row.get("case") in (None, ""):
+        raise RowError("'case' is missing")
+    if not isinstance(row["case"], (str, int)):
+        raise RowError(f"'case' must be a string, got {type(row['case']).__name__}")
+    passed = row.get("passed")
+    if isinstance(passed, (int, float)) and not isinstance(passed, bool) and passed in (0, 1):
+        passed = bool(passed)
+    if passed is not None and not isinstance(passed, bool):
+        raise RowError(f"'passed' must be true, false or null, got {passed!r}")
+    run = row.get("run", 1)
+    if isinstance(run, bool) or not isinstance(run, int) and not (isinstance(run, str) and run.isdigit()):
+        raise RowError(f"'run' must be an integer (the trial or run number), got {run!r}")
+    for k in ("infra_error", "no_answer"):
+        if row.get(k) is not None and not isinstance(row[k], str):
+            raise RowError(f"'{k}' must be a string (the reason) or null, got {row[k]!r}")
+    no_answer = row.get("no_answer") or None
+    if no_answer and passed:
+        raise RowError(f"'no_answer' is {no_answer!r} but 'passed' is true: an attempt with no answer can't pass")
+    if no_answer and passed is None and not row.get("infra_error"):
+        passed = False                    # no usable answer is a failed attempt
+    tags = row.get("tags") or []
+    if not isinstance(tags, list) or not all(isinstance(t, str) for t in tags):
+        raise RowError(f"'tags' must be a list of strings, got {tags!r}"[:160])
+    checks = row.get("checks") or {}
+    if not isinstance(checks, dict) or not all(isinstance(v, bool) for v in checks.values()):
+        raise RowError(f"'checks' must map rule names to true/false, got {checks!r}"[:160])
+    metrics = {k: _number(row, k) for k in NUMERIC_FIELDS if row.get(k) is not None}
+    return Attempt(case=str(row["case"]), passed=passed, arm=arm or str(row.get("arm", "default")),
+                   run=int(run), infra_error=row.get("infra_error") or None, no_answer=no_answer,
+                   tags=list(tags), checks=dict(checks), metrics=metrics)
+
+
+def load_jsonl(path: str | Path, *, arm: str | None = None) -> list[Attempt]:
+    out = []
+    for i, line in enumerate(Path(path).read_text().splitlines(), 1):
+        if line.strip():
+            try:
+                out.append(from_dict(json.loads(line), arm=arm))
+            except (json.JSONDecodeError, ValueError) as e:
+                raise ValueError(f"{path}:{i}: {e}") from e
+    return out
+
+
+def write_jsonl(attempts: list[Attempt], path: str | Path) -> None:
+    Path(path).write_text("".join(json.dumps(a.to_json()) + "\n" for a in attempts))
+
+
+def load(path: str | Path, *, arm: str | None = None, run: int | None = None) -> list[Attempt]:
+    """A JSONL file, a Harbor job directory, or a directory of Harbor jobs (one per run)."""
+    from agentstats import adapters
+
+    path = Path(path)
+    if path.is_file():
+        return load_jsonl(path, arm=arm)
+    if adapters.is_harbor_job(path):
+        return adapters.harbor_job(path, arm=arm, run=run or 1)
+    jobs = sorted(p for p in path.iterdir() if p.is_dir() and adapters.is_harbor_job(p))
+    if jobs:
+        return [a for i, job in enumerate(jobs, 1) for a in adapters.harbor_job(job, arm=arm, run=i)]
+    raise ValueError(f"{path}: not a JSONL file, a Harbor job directory, or a directory of Harbor jobs")
+
+
+def by_case(attempts: list[Attempt]) -> dict[str, list[Attempt]]:
+    out: dict[str, list[Attempt]] = defaultdict(list)
+    for a in attempts:
+        out[a.case].append(a)
+    return dict(out)
+
+
+def by_run(attempts: list[Attempt]) -> dict[int, list[Attempt]]:
+    out: dict[int, list[Attempt]] = defaultdict(list)
+    for a in attempts:
+        out[a.run].append(a)
+    return dict(sorted(out.items()))
+
+
+def arms(attempts: list[Attempt]) -> list[str]:
+    return sorted({a.arm for a in attempts})

agentstats/reliability.py

@@ -0,0 +1,58 @@
+"""
+Capability and consistency from repeated attempts per case.
+
+    pass@k  P(at least one of k attempts passes): capability ([Chen et al. 2021], arXiv 2107.03374).
+    pass^k  P(all k attempts pass): consistency, what a user relying on the agent sees (tau-bench, Yao et al.,
+            arXiv 2406.12045).
+
+Unbiased per-case estimates from n attempts with c passes, averaged over cases:
+    pass@k = 1 - C(n - c, k) / C(n, k)        pass^k = C(c, k) / C(n, k)
+
+A large pass@k - pass@1 means retries help; a large pass@1 - pass^k means success depends on luck.
+CIs come from a bootstrap over cases. Use a deterministic grader here: an LLM judge adds its own randomness.
+"""
+from __future__ import annotations
+
+from math import comb
+
+from agentstats.intervals import Interval, bootstrap, mean
+
+
+def pass_at_k(n: int, c: int, k: int) -> float:
+    if n < k:
+        raise ValueError(f"need n >= k attempts (n={n}, k={k})")
+    return 1.0 - comb(n - c, k) / comb(n, k)
+
+
+def pass_hat_k(n: int, c: int, k: int) -> float:
+    if n < k:
+        raise ValueError(f"need n >= k attempts (n={n}, k={k})")
+    return comb(c, k) / comb(n, k)
+
+
+def summarize(counts: dict[str, tuple[int, int]], conf: float = 0.95, seed: int = 0) -> dict:
+    """counts: case -> (attempts n, passes c). pass@k and pass^k for every k up to the smallest n."""
+    if not counts:
+        return {"k_max": 0, "pass_at": {}, "pass_hat": {}, "cases": {}}
+    k_max = min(n for n, _ in counts.values())
+    units = list(counts.values())
+    out: dict = {"k_max": k_max, "pass_at": {}, "pass_hat": {}}
+    for k in range(1, k_max + 1):
+        out["pass_at"][k] = bootstrap(units, lambda us, k=k: mean([pass_at_k(n, c, k) for n, c in us]),
+                                      conf, seed=seed, method="bootstrap over cases").to_json()
+        out["pass_hat"][k] = bootstrap(units, lambda us, k=k: mean([pass_hat_k(n, c, k) for n, c in us]),
+                                       conf, seed=seed, method="bootstrap over cases").to_json()
+    out["cases"] = {
+        "always": sorted(c for c, (n, p) in counts.items() if p == n),
+        "never": sorted(c for c, (n, p) in counts.items() if p == 0),
+        "flaky": sorted(c for c, (n, p) in counts.items() if 0 < p < n),
+    }
+    return out
+
+
+def flake_rate(counts: dict[str, tuple[int, int]]) -> Interval | None:
+    multi = [(n, c) for n, c in counts.values() if n >= 2]
+    if not multi:
+        return None
+    return bootstrap(multi, lambda us: mean([1.0 if 0 < c < n else 0.0 for n, c in us]),
+                     method="bootstrap over cases")

agentstats/report.py

@@ -0,0 +1,152 @@
+"""Plain-text/markdown rendering of summary, compare and judge results."""
+from __future__ import annotations
+
+import math
+
+METHOD_NAME = {
+    "fixed_cases": "fixed cases, noise from repeated trials per case",
+    "run_level": "Student's t over complete runs",
+    "per_case": "t over per-case pass rates",
+    "clustered": "cluster-robust standard error",
+    "wilson": "Wilson interval",
+    "welch": "Welch's t-test over complete runs",
+    "paired": "paired per-case differences",
+    "paired_clustered": "paired differences, cluster-robust",
+    "mcnemar": "exact McNemar test (one trial per case per arm)",
+}
+
+
+WHEN_USED = {
+    "fixed_cases": "for hand-picked sets when runs don't show run-wide shocks",
+    "run_level": "for hand-picked sets with 3+ runs that show run-wide shocks",
+    "per_case": "when the cases were sampled at random from real traffic",
+    "clustered": "for sampled cases in groups that fail together",
+}
+
+
+def _pct(x) -> str:
+    return "n/a" if x is None or (isinstance(x, float) and math.isnan(x)) else f"{100 * x:.1f}%"
+
+
+def _pp(x) -> str:
+    return "n/a" if x is None or (isinstance(x, float) and math.isnan(x)) else f"{100 * x:+.1f} pts"
+
+
+def _ci(iv: dict, fmt=_pct) -> str:
+    return f"{fmt(iv['estimate'])} (95% CI {fmt(iv['low'])} to {fmt(iv['high'])})"
+
+
+def _num(x: float) -> str:
+    if x is None or math.isnan(x):
+        return "n/a"
+    return f"{x:,.4f}" if abs(x) < 1 else f"{x:,.1f}"
+
+
+def summary(s: dict) -> str:
+    pr = s["pass_rate"]
+    lines = [f"# agentstats summary ({s['design']} eval set)", "",
+             f"**Pass rate: {_ci(pr)}**, {s['claim']}.",
+             f"Method: {METHOD_NAME.get(pr['method'], pr['method'])}. {s['cases']} cases, {s['runs']} run(s), "
+             f"{s['attempts']} graded attempts used ({s['attempts_loaded']} loaded)."
+             + (f" Interval {pr['note']}." if "clipped" in pr.get("note", "") else ""), ""]
+    for m, iv in s.get("not_used", {}).items():
+        lines.append(f"- Not used: {METHOD_NAME.get(m, m)} would give {_ci(iv)} ({WHEN_USED.get(m, '')}).")
+    for n in s["notes"]:
+        lines.append(f"- Note: {n}")
+    if s.get("run_pass_rates") and len(s["run_pass_rates"]) > 1:
+        lines.append("- Per run: " + ", ".join(f"run {r} {_pct(v)}" for r, v in s["run_pass_rates"].items()))
+    inf = s["infra"]
+    lines += ["", "## Infrastructure",
+              f"{inf['count']} infrastructure failures ({_pct(inf['rate'])}), excluded from pass rates"
+              + (f": {inf['kinds']}" if inf["kinds"] else "") + "."]
+    na = s.get("no_answer") or {}
+    if na.get("count"):
+        lines.append(f"{na['count']} attempts ended with no usable answer ({_pct(na['rate'])}), counted as failures: "
+                     f"{na['kinds']}.")
+    if "reliability" in s:
+        r = s["reliability"]
+        lines += ["", "## Reliability", "| k | pass@k (capability) | pass^k (consistency) |", "|---|---|---|"]
+        for k in r["pass_at"]:
+            lines.append(f"| {k} | {_ci(r['pass_at'][k])} | {_ci(r['pass_hat'][k])} |")
+        c = r["cases"]
+        lines.append(f"\nAlways pass: {len(c['always'])}, never pass: {len(c['never'])}, "
+                     f"flaky: {len(c['flaky'])}" + (f" ({', '.join(c['flaky'][:10])}"
+                                                     + (", ..." if len(c['flaky']) > 10 else "") + ")"
+                                                     if c["flaky"] else ""))
+    if s["efficiency"]:
+        lines += ["", "## Efficiency (per attempt, mean over cases)", "| metric | mean (95% CI) | p50 | p95 |",
+                  "|---|---|---|---|"]
+        for f, v in s["efficiency"].items():
+            if f == "cost_per_resolved_task":
+                continue
+            lines.append(f"| {f} | {_ci(v['mean'], _num)} | {_num(v['p50'])} | {_num(v['p95'])} |")
+        if "cost_per_resolved_task" in s["efficiency"]:
+            lines.append(f"\nCost per resolved task: ${_ci(s['efficiency']['cost_per_resolved_task'], _num)}")
+    if s.get("slices"):
+        lines += ["", "## Slices", "| tag | cases | pass rate |", "|---|---|---|"]
+        for t, v in s["slices"].items():
+            lines.append(f"| {t} | {v['cases']} | {_ci(v['pass_rate'])} |")
+    if s.get("checks"):
+        lines += ["", "## Trajectory checks", "| rule | pass rate |", "|---|---|"]
+        for r, v in s["checks"].items():
+            lines.append(f"| {r} | {_ci(v)} |")
+    return "\n".join(lines) + "\n"
+
+
+def compare(c: dict, a_name: str = "A", b_name: str = "B") -> str:
+    d = c["delta"]
+    lines = [f"# agentstats compare: {b_name} vs {a_name} ({c['design']} eval set)", "",
+             f"**{c['verdict']}**: {b_name} minus {a_name} = {_ci(d, _pp)}, {c['claim']}.",
+             f"Method: {METHOD_NAME.get(c['method'], c['method'])}. {c['cases']} matched cases; "
+             f"{a_name} {_pct(c['a']['pass_rate'])} over {c['a']['runs']} run(s), "
+             f"{b_name} {_pct(c['b']['pass_rate'])} over {c['b']['runs']} run(s).", ""]
+    if c.get("mcnemar"):
+        m = c["mcnemar"]
+        lines.append(f"- McNemar (exact): {m['b_only']} cases pass only with {b_name}, {m['a_only']} only with "
+                     f"{a_name}; p = {m['p_value']:.3g}.")
+    if c.get("smallest_detectable") is not None:
+        lines.append(f"- Smallest difference this setup can reliably detect (80% power): "
+                     f"about {_pp(c['smallest_detectable'])}.")
+    if c.get("min_effect") is not None:
+        lines.append(f"- Ship rule (CI lower bound > {_pp(c['min_effect'])}): {'PASS' if c['ship'] else 'FAIL'}.")
+    for n in c["notes"]:
+        lines.append(f"- Note: {n}")
+    mv = c["moved"]
+    lines.append(f"- Cases that moved: {len(mv['b_better'])} better with {b_name}, {len(mv['b_worse'])} worse.")
+    if "cost_per_attempt_delta" in c:
+        lines.append(f"- Cost per attempt: {a_name} ${_num(c['cost_per_attempt']['a'])}, "
+                     f"{b_name} ${_num(c['cost_per_attempt']['b'])}; difference "
+                     f"${_ci(c['cost_per_attempt_delta'], _num)}.")
+    return "\n".join(lines) + "\n"
+
+
+def judge(j: dict) -> str:
+    lines = ["# agentstats judge check", "", f"{j['items']} items."]
+    if "human_agreement" in j:
+        h = j["human_agreement"]
+        lines += ["", "## Human agreement",
+                  f"Fleiss' kappa {h['fleiss_kappa']:.2f} over {h['items']} items with {h['raters_per_item']} "
+                  f"raters each; {h['unanimous']} unanimous."]
+    if "alignment" in j:
+        a = j["alignment"]
+
+        def r(key):
+            iv = a["ci"].get(key)
+            return "n/a" if iv is None else _ci(iv)
+        lines += ["", f"## Judge vs human majority ({a['scope']}, n = {a['n']})",
+                  f"- Accuracy {r('accuracy')}",
+                  f"- Strictness, false fails among human passes: {r('strictness')}",
+                  f"- Leniency, false passes among human fails: {r('leniency')}",
+                  f"- Cohen's kappa {a['cohen_kappa']:.2f}",
+                  f"TP {a['tp']}, FN {a['fn']}, FP {a['fp']}, TN {a['tn']}."]
+        if a["disagreements"]:
+            lines.append("Disagreements: " + ", ".join(f"{x['item']} (human {x['human']}, judge {x['judge']})"
+                                                        for x in a["disagreements"][:10]))
+    if "consistency" in j:
+        c = j["consistency"]
+        lines += ["", "## Judge consistency",
+                  f"{c['items']} items judged {c['runs_per_item']}+ times: flip rate {100 * c['flip_rate']:.1f}%"
+                  + (f", mean pairwise disagreement {c['mean_pairwise_disagreement']:.3f}"
+                     if "mean_pairwise_disagreement" in c else
+                     f", mean pairwise |delta| {c['mean_pairwise_delta']:.3f}") + "."]
+    return "\n".join(lines) + "\n"