claimeval 1.0.0__py3-none-any.whl

claimeval/__init__.py

@@ -0,0 +1,95 @@
+"""claimeval -- test the claims made about AI agents against the data behind them.
+
+A claim ("#1", "beats X", "as good at lower cost", "on the cost-accuracy
+frontier") gets the probability that it holds, from a bootstrap over the right
+unit (tasks, or clusters such as repositories), and a verdict. Input is a
+stream of (config, task, seed, success, cost[, group]) runs from any harness,
+or a public leaderboard release read by `claimeval.core.adapters`. Also: frontier
+membership and dominance probabilities, a Hausdorff frontier band, a
+winner's-curse correction, wild cluster bootstrap intervals, and a task-vs-seed
+variance decomposition.
+"""
+
+import sys
+
+# Subpackages
+from . import core
+from . import stats
+from . import analysis
+from . import sim
+
+# Core data structures and loaders
+from .core.schema import Rollout, RolloutSet
+from .core.ingest import FieldMap, guess_map, inspect_path, load
+
+# Statistical methods, frontier geometry & diagnostics
+from .stats.frontier import dominance, frontier_band, frontier_points, hausdorff, log_cost_range, pareto_mask
+from .stats.bootstrap import BootstrapResult, auto_mode, hierarchical_bootstrap, variance_decomposition
+from .stats.wild import wild_intervals
+from .stats.bias import seed_count_curve, split_sample_debias
+from .stats.cost_axis import diagnose_cost_axis, format_diagnosis
+
+# Analysis, claims & reporting
+from .analysis.claims import Claim, evaluate_claims
+from .analysis.report import analyze, format_report, headline
+
+# Simulation & coverage validation
+from .sim.simulate import make_population, population_truth, sample_dataset, simulate
+from .sim.coverage import coverage_study, format_coverage
+
+# Backward compatibility aliases for module paths
+sys.modules["claimeval.schema"] = core.schema
+sys.modules["claimeval.ingest"] = core.ingest
+sys.modules["claimeval.adapters"] = core.adapters
+sys.modules["claimeval.bootstrap"] = stats.bootstrap
+sys.modules["claimeval.wild"] = stats.wild
+sys.modules["claimeval.bca"] = stats.bca
+sys.modules["claimeval.calibrate"] = stats.calibrate
+sys.modules["claimeval.bias"] = stats.bias
+sys.modules["claimeval.cost_axis"] = stats.cost_axis
+sys.modules["claimeval.frontier"] = stats.frontier
+sys.modules["claimeval.claims"] = analysis.claims
+sys.modules["claimeval.compare"] = analysis.compare
+sys.modules["claimeval.baselines"] = analysis.baselines
+sys.modules["claimeval.report"] = analysis.report
+sys.modules["claimeval.simulate"] = sim.simulate
+sys.modules["claimeval.coverage"] = sim.coverage
+
+__version__ = "1.0.0"
+
+__all__ = [
+    "core",
+    "stats",
+    "analysis",
+    "sim",
+    "Claim",
+    "evaluate_claims",
+    "auto_mode",
+    "wild_intervals",
+    "Rollout",
+    "RolloutSet",
+    "FieldMap",
+    "guess_map",
+    "inspect_path",
+    "load",
+    "analyze",
+    "headline",
+    "format_report",
+    "hierarchical_bootstrap",
+    "BootstrapResult",
+    "variance_decomposition",
+    "pareto_mask",
+    "dominance",
+    "frontier_points",
+    "hausdorff",
+    "frontier_band",
+    "log_cost_range",
+    "split_sample_debias",
+    "seed_count_curve",
+    "simulate",
+    "make_population",
+    "sample_dataset",
+    "population_truth",
+    "coverage_study",
+    "format_coverage",
+]

claimeval/__main__.py

@@ -0,0 +1,177 @@
+"""CLI: claimeval {demo,inspect,analyze,claims,seed-curve,coverage}"""
+
+from __future__ import annotations
+
+import argparse
+import glob
+import os
+import json
+import sys
+
+import numpy as np
+
+from claimeval.stats.bias import seed_count_curve
+from claimeval.core.ingest import FieldMap, guess_map, inspect_path, load
+from claimeval.analysis.report import analyze
+from claimeval.core.schema import RolloutSet
+from claimeval.sim.simulate import simulate
+
+
+def _load(args) -> RolloutSet:
+    src = getattr(args, "source", None)
+    if src:
+        from claimeval.core import adapters
+        if src == "tau2":
+            rs, skipped = adapters.tau2(sorted(glob.glob(os.path.join(args.path, "*.json")))
+                                        if os.path.isdir(args.path) else [args.path])
+        elif src == "swebench-cost":
+            rs, skipped = adapters.swebench(args.path, with_cost=True)
+        else:
+            rs, skipped = {"swebench": adapters.swebench,
+                           "terminal-bench": adapters.terminal_bench}[src](args.path)
+        for k, v in skipped.items():
+            print(f"[skipped] {k}: {v}", file=sys.stderr)
+        return rs
+    if args.map:
+        fmap = FieldMap.from_dict(json.loads(open(args.map).read()))
+    else:
+        info = inspect_path(args.path)
+        fmap = info["guess"]
+        print(f"[guessed mapping] {fmap}\n", file=sys.stderr)
+    rs, dropped = load(args.path, fmap)
+    if dropped:
+        print(f"[dropped records] {dropped}\n", file=sys.stderr)
+    return rs
+
+
+def _print_curve(rows: list[dict]) -> None:
+    print(f"\n{'k':>3}{'Jaccard vs full':>18}{'Hausdorff':>12}"
+          f"{'acc inflation':>16}{'cost inflation':>16}{'|front|':>9}")
+    print("-" * 74)
+    for r in rows:
+        print(
+            f"{r['k']:>3}{r['jaccard_vs_full']:>18.3f}{r['hausdorff_vs_full']:>12.4f}"
+            f"{r['acc_inflation']:>+16.4f}{r['cost_inflation']:>+16.4f}"
+            f"{r['frontier_size']:>9.1f}"
+        )
+    print("\nacc inflation = what a k-seed study claims for its own frontier winners,")
+    print("minus what the full data says about those same configs.")
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(prog="claimeval")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    d = sub.add_parser("demo", help="end-to-end run on simulated data")
+    d.add_argument("--seeds", type=int, default=20)
+    d.add_argument("--tasks", type=int, default=50)
+    d.add_argument("--configs", type=int, default=9)
+    d.add_argument("--boot", type=int, default=2000)
+
+    i = sub.add_parser("inspect", help="discover the schema of a results dump")
+    i.add_argument("path")
+
+    a = sub.add_parser("analyze", help="audit frontier claims in a results dump")
+    a.add_argument("path")
+    a.add_argument("--map", help="JSON field mapping; omit to auto-guess")
+    a.add_argument("--from", dest="source", choices=["swebench", "swebench-cost", "terminal-bench", "tau2"],
+                   help="read a public leaderboard release directly (see claimeval.adapters)")
+
+    a.add_argument("--boot", type=int, default=2000)
+    a.add_argument("--threshold", type=float, default=0.90)
+
+    c = sub.add_parser("claims", help="test published claims (top1, pairwise, margin, "
+                                      "cost_dominance, frontier) against a results dump")
+    c.add_argument("path")
+    c.add_argument("claims", help="JSON list of {kind, subject, comparators, margin, id}")
+    c.add_argument("--map")
+    c.add_argument("--from", dest="source", choices=["swebench", "swebench-cost", "terminal-bench", "tau2"],
+                   help="read a public leaderboard release directly (see claimeval.adapters)")
+
+    c.add_argument("--boot", type=int, default=4000)
+    c.add_argument("--threshold", type=float, default=0.90)
+    c.add_argument("--mode", default=None,
+                   choices=["task", "cluster_t"],
+                   help="resampling unit; default: clusters if the data has them, else tasks")
+
+    s = sub.add_parser("seed-curve", help="error of a k-seed frontier vs the full data")
+    s.add_argument("path")
+    s.add_argument("--map")
+    s.add_argument("--ks", default="1,2,3,5,10")
+
+    cv = sub.add_parser("coverage", help="validate the estimator against known truth")
+    cv.add_argument("--reps", type=int, default=200)
+    cv.add_argument("--configs", type=int, default=9)
+    cv.add_argument("--tasks", type=int, default=50)
+    cv.add_argument("--seeds", type=int, default=5)
+    cv.add_argument("--boot", type=int, default=500)
+    cv.add_argument("--level", type=float, default=0.95)
+    cv.add_argument("--threshold", type=float, default=0.90)
+    cv.add_argument("--mode", default="task", choices=["task", "both", "seed"],
+                    help="bootstrap mode; 'both' double-counts within-cell noise")
+
+    args = ap.parse_args(argv)
+    rng = np.random.default_rng(0)
+
+    if args.cmd == "demo":
+        rolls, truth = simulate(args.configs, args.tasks, args.seeds, rng)
+        rs = RolloutSet(rolls)
+        report, head = analyze(rs, n_boot=args.boot, rng=rng)
+        print(report)
+        print("\nGround truth accuracy (simulator):")
+        for c, t in zip(truth["configs"], truth["accuracy"]):
+            print(f"  {c:<26}{t:.3f}")
+        _print_curve(seed_count_curve(rs, (1, 2, 3, 5, 10), n_repeats=30, rng=rng))
+        return 0
+
+    if args.cmd == "coverage":
+        from claimeval.sim.coverage import coverage_study, format_coverage
+
+        res = coverage_study(
+            n_reps=args.reps, n_configs=args.configs, n_tasks=args.tasks,
+            n_seeds=args.seeds, n_boot=args.boot, level=args.level,
+            threshold=args.threshold, boot_mode=args.mode,
+        )
+        print(format_coverage(res))
+        return 0
+
+    if args.cmd == "inspect":
+        info = inspect_path(args.path)
+        print(f"{info['n_records']} records scanned\n")
+        print(f"{'field':<46}{'count':>8}  samples")
+        print("-" * 78)
+        for k, n in info["keys"].most_common(60):
+            ex = ", ".join(str(v)[:18] for v in info["samples"].get(k, []))
+            print(f"{k[:46]:<46}{n:>8}  {ex[:24]}")
+        print(f"\nguessed mapping: {info['guess']}")
+        print("Verify this, then pass it with --map. Wrong success/cost fields")
+        print("produce a confident, meaningless frontier.")
+        return 0
+
+    rs = _load(args)
+    if args.cmd == "claims":
+        from claimeval.analysis.claims import Claim, evaluate_claims
+
+        spec = json.loads(open(args.claims).read())
+        cl = [Claim(x["kind"], x["subject"], x.get("comparators", []), x.get("margin", 0.0),
+                    x.get("id", ""), x.get("k", 1)) for x in spec]
+        res = evaluate_claims(rs, cl, n_boot=args.boot, mode=args.mode, rng=rng, thr=args.threshold)
+        print(f"{'id':<10}{'kind':<16}{'subject':<28}{'P(claim)':>9}  verdict")
+        for r in res:
+            v = "supported" if r["supported"] else (
+                "NOT supported" if r["holds_at_point_estimate"] else "contradicted by point estimate")
+            print(f"{r['id'][:9]:<10}{r['kind']:<16}{r['subject'][:27]:<28}{r['probability']:>9.3f}  {v}")
+        print(f"\nsupported = P >= {args.threshold}; resampling: {res[0]['mode'] if res else '-'}")
+        return 0
+    if args.cmd == "analyze":
+        report, _ = analyze(rs, n_boot=args.boot, threshold=args.threshold, rng=rng)
+        print(report)
+    else:
+        ks = tuple(int(x) for x in args.ks.split(","))
+        print(rs.describe())
+        _print_curve(seed_count_curve(rs, ks, n_repeats=30, rng=rng))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

claimeval/analysis/__init__.py

@@ -0,0 +1,26 @@
+"""Claim evaluation, comparison, and reporting."""
+
+from claimeval.stats.frontier import dominance, frontier_band, frontier_points, hausdorff, log_cost_range, pareto_mask
+from .claims import Claim, evaluate_claims
+from .compare import Scenario, compare, format_rows
+from .baselines import METHODS, run_method
+from .report import analyze, format_report, headline
+
+__all__ = [
+    "Claim",
+    "evaluate_claims",
+    "dominance",
+    "frontier_band",
+    "frontier_points",
+    "hausdorff",
+    "log_cost_range",
+    "pareto_mask",
+    "Scenario",
+    "compare",
+    "format_rows",
+    "METHODS",
+    "run_method",
+    "analyze",
+    "format_report",
+    "headline",
+]

claimeval/analysis/baselines.py

@@ -0,0 +1,135 @@
+"""Existing uncertainty methods, in the same output format as ours.
+
+Every method returns a BootstrapResult: B joint draws of (accuracy, cost) for
+all configurations. Frontier-membership and dominance probabilities, intervals
+and certified claims are then computed by the SAME code for every method, so a
+comparison measures the error model and nothing else.
+
+Methods (paper names in brackets):
+
+  naive          [current practice]  point estimates, no uncertainty.
+  wald           [per-model error bars] independent normal intervals per
+                 configuration, tasks treated as independent. The +/- usually
+                 printed next to a leaderboard score.
+  paired_clt     [Miller 2024, paired] joint normal over all configurations
+                 with the full task-level covariance, so comparisons are
+                 paired. Tasks independent.
+  clustered_clt  [Miller 2024, clustered] as paired_clt with cluster-robust
+                 (sandwich) covariance over task groups, small-sample
+                 corrected. The strongest formula-based competitor.
+  boot_both      [naive bootstrap] resample tasks and runs together.
+  boot_task          [ours, unclustered] task resampling.
+  boot_cluster_t     [ours, clustered] intact clusters + few-cluster widening.
+  boot_group         [our earlier version] two-stage; double-counts within-cluster noise.
+  boot_cluster       [ablation] intact clusters, no widening.
+
+The formula methods have no native notion of P(on frontier). They get the
+fairest extension available: draws from their own sampling distribution,
+pushed through the same frontier code (a parametric bootstrap).
+
+Median cost has no simple standard error. The formula methods use its
+influence function, psi = (1/2 - 1{x <= m}) / f(m), with a Gaussian kernel
+density estimate of f at the median -- the textbook delta-method route.
+"""
+
+from __future__ import annotations
+
+import warnings
+
+import numpy as np
+
+from claimeval.stats.bootstrap import BootstrapResult, hierarchical_bootstrap
+from claimeval.core.schema import RolloutSet
+
+METHODS = ("naive", "wald", "paired_clt", "clustered_clt", "boot_both", "boot_task", "boot_group",
+           "boot_cluster", "boot_cluster_t", "boot_cluster_cr2", "boot_cluster_kish")
+
+
+def _task_means(rs: RolloutSet) -> tuple[np.ndarray, np.ndarray]:
+    """(C, T) per-task mean success and cost, averaged over runs."""
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", RuntimeWarning)
+        return np.nanmean(rs.success, axis=2), np.nanmean(rs.cost, axis=2)
+
+
+def _cost_influence(cost_t: np.ndarray, stat: str) -> tuple[np.ndarray, np.ndarray]:
+    """Point estimate and per-task influence values for the cost statistic.
+
+    Returns (est (C,), psi (C, T)) such that est is approximately the mean of
+    est + psi, i.e. psi plays the role of centred per-task contributions.
+    """
+    if stat == "mean":
+        est = np.nanmean(cost_t, axis=1)
+        return est, cost_t - est[:, None]
+    est = np.nanmedian(cost_t, axis=1)
+    psi = np.zeros_like(cost_t)
+    for c in range(cost_t.shape[0]):
+        x = cost_t[c][~np.isnan(cost_t[c])]
+        n = len(x)
+        sd = np.std(x, ddof=1)
+        iqr = np.subtract(*np.quantile(x, [0.75, 0.25]))
+        h = 0.9 * min(sd, iqr / 1.34 if iqr > 0 else sd) * n ** -0.2  # Silverman
+        h = max(h, 1e-12)
+        f = np.mean(np.exp(-0.5 * ((x - est[c]) / h) ** 2)) / (h * np.sqrt(2 * np.pi))
+        f = max(f, 1e-12)
+        psi[c] = (0.5 - (cost_t[c] <= est[c])) / f
+    return est, psi
+
+
+def _normal_draws(mean: np.ndarray, cov: np.ndarray, B: int, rng) -> np.ndarray:
+    # eigen-decomposition tolerates the rank deficiency of a covariance
+    # estimated from fewer clusters than dimensions
+    w, V = np.linalg.eigh((cov + cov.T) / 2)
+    w = np.clip(w, 0, None)
+    return mean + (rng.standard_normal((B, len(mean))) * np.sqrt(w)) @ V.T
+
+
+def _formula(rs: RolloutSet, B: int, rng, cost_stat: str, kind: str) -> BootstrapResult:
+    acc_t, cost_t = _task_means(rs)
+    C, T = acc_t.shape
+    acc = np.nanmean(acc_t, axis=1)
+    cost_est, psi = _cost_influence(cost_t, cost_stat)
+    # centred per-task contributions for all 2C statistics
+    Z = np.vstack([acc_t - acc[:, None], psi])  # (2C, T)
+    Z = np.nan_to_num(Z)
+    mean = np.concatenate([acc, cost_est])
+
+    if kind == "wald":
+        var = (Z ** 2).sum(axis=1) / (T * (T - 1))
+        cov = np.diag(var)  # every configuration independent
+    elif kind == "paired_clt":
+        cov = (Z @ Z.T) / (T * (T - 1))
+    elif kind == "clustered_clt":
+        G = rs.n_groups
+        S = np.zeros((2 * C, G))
+        for g, idx in enumerate(rs.group_tasks):
+            S[:, g] = Z[:, idx].sum(axis=1)
+        cov = (S @ S.T) / T ** 2 * (G / (G - 1))  # CR1 small-sample correction
+    else:
+        raise ValueError(kind)
+
+    draws = _normal_draws(mean, cov, B, rng)
+    return BootstrapResult(draws[:, :C], draws[:, C:], list(rs.configs), kind)
+
+
+def run_method(rs: RolloutSet, method: str, B: int = 1000, rng=None,
+               cost_stat: str = "median") -> BootstrapResult:
+    rng = rng or np.random.default_rng(0)
+    if method == "naive":
+        acc, cost = rs.point_estimates(cost_stat)
+        return BootstrapResult(acc[None, :], cost[None, :], list(rs.configs), "naive")
+    if method in ("wald", "paired_clt", "clustered_clt"):
+        if method == "clustered_clt" and not rs.is_clustered:
+            method = "paired_clt"  # no groups: the clustered estimator is the paired one
+        return _formula(rs, B, rng, cost_stat, method)
+    if method in ("boot_cluster_cr2", "boot_cluster_kish"):
+        mode = method[5:] if rs.is_clustered else "task"
+        return hierarchical_bootstrap(rs, n_boot=B, mode=mode, rng=rng, cost_stat=cost_stat)
+    if method == "boot_cluster_t":
+        mode = "cluster_t" if rs.is_clustered else "task"
+        return hierarchical_bootstrap(rs, n_boot=B, mode=mode, rng=rng, cost_stat=cost_stat)
+    mode = {"boot_both": "both", "boot_task": "task", "boot_group": "group",
+            "boot_cluster": "cluster"}[method]
+    if mode in ("group", "cluster") and not rs.is_clustered:
+        mode = "task"
+    return hierarchical_bootstrap(rs, n_boot=B, mode=mode, rng=rng, cost_stat=cost_stat)

claimeval/analysis/claims.py

@@ -0,0 +1,97 @@
+"""Test the kinds of claims papers and leaderboards actually make.
+
+A claim names a subject configuration and, usually, comparators:
+
+  top1            subject is the most accurate of {subject} + comparators
+  pairwise        subject is more accurate than each comparator
+  margin          subject beats each comparator by at least `margin` (accuracy, 0-1)
+  cost_dominance  subject is at least as accurate and no more costly than each comparator
+  frontier        subject is on the cost-accuracy Pareto frontier of the given set
+  topk            at most k-1 comparators are more accurate than subject ("#k", "top 5")
+  equivalent      subject is within `margin` of each comparator ("as good as", "on par")
+  cheaper_noninferior  subject costs less than each comparator and is no more than
+                  `margin` less accurate ("as good as, but cheaper")
+
+Each claim gets a bootstrap probability. It is *supported* at level `thr`
+(default 0.9) when that probability reaches it -- the same certification rule
+used throughout the paper. "Not supported" means the data cannot distinguish
+the claim from its negation at that confidence; it does not mean the claim is
+false.
+
+    from claimeval import RolloutSet
+    from claimeval.claims import Claim, evaluate_claims
+    res = evaluate_claims(rs, [Claim("top1", "agentA", ["agentB", "agentC"])])
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+
+from claimeval.stats.bootstrap import Mode, auto_mode, hierarchical_bootstrap
+from claimeval.stats.frontier import pareto_mask
+from claimeval.core.schema import RolloutSet
+
+KINDS = ("top1", "pairwise", "margin", "cost_dominance", "frontier", "topk", "equivalent",
+         "cheaper_noninferior")
+
+
+@dataclass
+class Claim:
+    kind: str
+    subject: str
+    comparators: list[str] = field(default_factory=list)
+    margin: float = 0.0
+    id: str = ""
+    k: int = 1
+
+    def __post_init__(self):
+        if self.kind not in KINDS:
+            raise ValueError(f"unknown claim kind {self.kind!r}; expected one of {KINDS}")
+
+
+def claim_probability(acc: np.ndarray, cost: np.ndarray, claim: Claim, idx: dict) -> np.ndarray:
+    """Per-replicate truth value of the claim. acc, cost: (B, n_configs)."""
+    s = idx[claim.subject]
+    comp = [idx[c] for c in claim.comparators]
+    a, c = acc[:, s], cost[:, s]
+    if claim.kind == "top1":
+        return np.all(a[:, None] > acc[:, comp], axis=1)
+    if claim.kind == "pairwise":
+        return np.all(a[:, None] > acc[:, comp], axis=1)
+    if claim.kind == "margin":
+        return np.all(a[:, None] - acc[:, comp] >= claim.margin, axis=1)
+    if claim.kind == "topk":
+        return (acc[:, comp] > a[:, None]).sum(axis=1) <= claim.k - 1
+    if claim.kind == "equivalent":
+        return np.all(np.abs(a[:, None] - acc[:, comp]) <= claim.margin, axis=1)
+    if claim.kind == "cheaper_noninferior":
+        return np.all((c[:, None] < cost[:, comp]) & (a[:, None] >= acc[:, comp] - claim.margin), axis=1)
+    if claim.kind == "cost_dominance":
+        return np.all((a[:, None] >= acc[:, comp]) & (c[:, None] <= cost[:, comp]), axis=1)
+    keep = [s] + comp
+    return pareto_mask(cost[:, keep], acc[:, keep])[:, 0]
+
+
+def evaluate_claims(rs: RolloutSet, claims: list[Claim], n_boot: int = 4000, mode: Mode | None = None,
+                rng: np.random.Generator | None = None, thr: float = 0.9,
+                cost_stat: str = "median") -> list[dict]:
+    """Bootstrap probability and verdict for each claim, from one shared set of draws."""
+    rng = rng or np.random.default_rng(0)
+    mode = mode or auto_mode(rs)
+    b = hierarchical_bootstrap(rs, n_boot, mode, rng, cost_stat)
+    acc0, cost0 = rs.point_estimates(cost_stat)
+    idx = {k: i for i, k in enumerate(rs.configs)}
+    out = []
+    for cl in claims:
+        missing = [k for k in [cl.subject, *cl.comparators] if k not in idx]
+        if missing:
+            raise KeyError(f"claim {cl.id or cl.subject}: unknown configs {missing}")
+        p = float(claim_probability(b.acc, b.cost, cl, idx).mean())
+        pt = bool(claim_probability(acc0[None], cost0[None], cl, idx)[0])
+        out.append({"id": cl.id, "kind": cl.kind, "subject": cl.subject,
+                    "comparators": list(cl.comparators), "probability": p,
+                    "holds_at_point_estimate": pt, "supported": p >= thr, "mode": mode,
+                    "subject_acc": float(acc0[idx[cl.subject]])})
+    return out

claimeval/analysis/compare.py

@@ -0,0 +1,121 @@
+"""Head-to-head comparison of uncertainty methods on shared simulated data.
+
+Each replication draws one dataset from a scenario with known truth; every
+method sees that same dataset. Reported per method:
+
+  acc_cov / cost_cov   95% interval coverage (target 0.95)
+  acc_w / cost_w       mean interval width (coverage bought by width is no win)
+  gap_cov              coverage of accuracy differences between adjacent configs
+  dom_prec             share of certified dominance claims (P >= 0.9) that are true
+  dom_n                certified dominance claims per replication (power)
+  dom_ece              calibration error of P(i dominates j)
+  front_prec/recall    certified frontier membership vs the true frontier
+
+"naive" has no intervals; its claims are the point-estimate claims, asserted
+with certainty, which is what current practice does.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable
+
+import numpy as np
+
+from .baselines import run_method
+from claimeval.sim.coverage import Calibration
+from claimeval.stats.frontier import dominance, pareto_mask
+from claimeval.core.schema import RolloutSet
+
+
+@dataclass
+class Scenario:
+    name: str
+    draw: Callable[[np.random.Generator], RolloutSet]  # one dataset
+    true_acc: np.ndarray
+    true_cost: np.ndarray
+    cost_stat: str = "median"
+
+
+def compare(scenario: Scenario, methods, reps: int = 200, B: int = 800,
+            thr: float = 0.9, level: float = 0.95, seed: int = 0) -> list[dict]:
+    rng = np.random.default_rng(seed)
+    ta, tc = scenario.true_acc, scenario.true_cost
+    true_dom = dominance(tc, ta)
+    np.fill_diagonal(true_dom, False)
+    true_front = pareto_mask(tc, ta)
+    order = np.argsort(ta)
+    adj = list(zip(order[1:], order[:-1]))  # (better, worse) by true accuracy
+    lo_q, hi_q = (1 - level) / 2, 1 - (1 - level) / 2
+
+    acc_names = {m: {k: [] for k in ("acc_cov", "cost_cov", "acc_w", "cost_w", "gap_cov",
+                                      "dom_true", "front_p", "front_r")} for m in methods}
+    dom_n = {m: 0 for m in methods}
+    cal = {m: Calibration() for m in methods}
+
+    for _ in range(reps):
+        rs = scenario.draw(rng)
+        idx = {k: i for i, k in enumerate(rs.configs)}
+        assert len(idx) == len(ta), "scenario configs must match truth"
+        a_hat, c_hat = rs.point_estimates(scenario.cost_stat)
+        naive_dom = dominance(c_hat, a_hat)
+        np.fill_diagonal(naive_dom, False)
+        naive_front = pareto_mask(c_hat, a_hat)
+        for m in methods:
+            b = run_method(rs, m, B=B, rng=rng, cost_stat=scenario.cost_stat)
+            r = acc_names[m]
+            if m != "naive":
+                alo, ahi = np.quantile(b.acc, [lo_q, hi_q], axis=0)
+                clo, chi = np.quantile(b.cost, [lo_q, hi_q], axis=0)
+                r["acc_cov"].append(np.mean((alo <= ta) & (ta <= ahi)))
+                r["cost_cov"].append(np.mean((clo <= tc) & (tc <= chi)))
+                r["acc_w"].append(np.mean(ahi - alo))
+                r["cost_w"].append(np.mean((chi - clo) / np.maximum(tc, 1e-12)))
+                gd = [b.acc[:, i] - b.acc[:, j] for i, j in adj]
+                r["gap_cov"].append(np.mean([
+                    np.quantile(g, lo_q) <= ta[i] - ta[j] <= np.quantile(g, hi_q)
+                    for g, (i, j) in zip(gd, adj)]))
+                pd = b.dominance_prob()
+                pf = b.frontier_prob()
+                off = ~np.eye(len(ta), dtype=bool)
+                cal[m].pred = np.append(cal[m].pred, pd[off])
+                cal[m].actual = np.append(cal[m].actual, true_dom[off].astype(float))
+                cert_dom = naive_dom & (pd >= thr)
+                cert_front = naive_front & (pf >= thr)
+            else:
+                cert_dom, cert_front = naive_dom, naive_front
+            dom_n[m] += int(cert_dom.sum())
+            r["dom_true"].extend(true_dom[cert_dom].tolist())
+            if cert_front.any():
+                r["front_p"].append(true_front[cert_front].mean())
+            r["front_r"].append(cert_front[true_front].mean() if true_front.any() else np.nan)
+
+    rows = []
+    for m in methods:
+        r = acc_names[m]
+        mean = lambda k: float(np.nanmean(r[k])) if r[k] else None  # noqa: E731
+        rows.append({
+            "scenario": scenario.name, "method": m,
+            "acc_cov": mean("acc_cov"), "cost_cov": mean("cost_cov"),
+            "acc_w": mean("acc_w"), "cost_w": mean("cost_w"), "gap_cov": mean("gap_cov"),
+            "dom_prec": float(np.mean(r["dom_true"])) if r["dom_true"] else None,
+            "dom_n": dom_n[m] / reps,
+            "dom_ece": cal[m].ece() if m != "naive" else None,
+            "front_prec": mean("front_p"), "front_recall": mean("front_r"),
+            "reps": reps,
+        })
+    return rows
+
+
+def format_rows(rows: list[dict]) -> str:
+    def f(v, d=3):
+        return "   --" if v is None else f"{v:.{d}f}"
+    head = (f"{'method':14s}{'acc cov':>8s}{'width':>7s}{'cost cov':>9s}{'gap cov':>8s}"
+            f"{'dom prec':>9s}{'dom/rep':>8s}{'dom ECE':>8s}{'front P':>8s}{'front R':>8s}")
+    out = [head, "-" * len(head)]
+    for r in rows:
+        out.append(f"{r['method']:14s}{f(r['acc_cov']):>8s}{f(r['acc_w']):>7s}"
+                   f"{f(r['cost_cov']):>9s}{f(r['gap_cov']):>8s}{f(r['dom_prec']):>9s}"
+                   f"{r['dom_n']:>8.1f}{f(r['dom_ece']):>8s}{f(r['front_prec']):>8s}"
+                   f"{f(r['front_recall']):>8s}")
+    return "\n".join(out)