abeval 0.1.0__py3-none-any.whl

abeval/__init__.py

@@ -0,0 +1,34 @@
+"""abeval: A/B-test statistics for LLM evals.
+
+Error bars for a single run, paired significance tests for comparing two
+runs, sample-size planning before you spend on inference, and judge-noise
+reliability. Standard library only.
+"""
+
+from .ci import Interval, clustered_mean_ci, mean_ci, proportion_ci
+from .compare import Comparison, mcnemar_exact, paired_compare
+from .data import extract, pair, read_jsonl
+from .power import mde, power_simulated, sample_size, sd_diff_from_rates
+from .reliability import Reliability, judge_reliability
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Comparison",
+    "Interval",
+    "Reliability",
+    "__version__",
+    "clustered_mean_ci",
+    "extract",
+    "judge_reliability",
+    "mcnemar_exact",
+    "mde",
+    "mean_ci",
+    "pair",
+    "paired_compare",
+    "power_simulated",
+    "proportion_ci",
+    "read_jsonl",
+    "sample_size",
+    "sd_diff_from_rates",
+]

abeval/__main__.py

@@ -0,0 +1,191 @@
+"""Command-line interface for abeval."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from . import __version__
+from .ci import clustered_mean_ci, mean_ci, proportion_ci
+from .compare import paired_compare
+from .data import extract, pair, read_jsonl
+from .power import mde, sample_size, sd_diff_from_rates
+from .reliability import judge_reliability
+
+
+def _fmt_pct(x: float) -> str:
+    return f"{100.0 * x:.1f}%"
+
+
+def _is_binary(values: list[float]) -> bool:
+    return all(v in (0.0, 1.0) for v in values)
+
+
+def cmd_ci(args: argparse.Namespace) -> int:
+    records = read_jsonl(args.results)
+    values_by_id = extract(records, args.metric, args.id_key)
+    values = list(values_by_id.values())
+    if _is_binary(values) and not args.bootstrap:
+        interval = proportion_ci(int(sum(values)), len(values), args.level)
+    elif args.cluster_key:
+        clusters = [rec[args.cluster_key] for rec in records]
+        interval = clustered_mean_ci(values, clusters, args.level)
+    elif args.bootstrap:
+        interval = mean_ci(values, args.level, method="bootstrap", reps=args.reps, seed=args.seed)
+    else:
+        interval = mean_ci(values, args.level, method="t")
+    if args.json:
+        print(json.dumps(interval.as_dict(), indent=2))
+        return 0
+    pct = _is_binary(values)
+    fmt = _fmt_pct if pct else lambda v: f"{v:.4g}"
+    print(f"{args.metric}: {fmt(interval.estimate)}  "
+          f"[{fmt(interval.lo)}, {fmt(interval.hi)}]  "
+          f"(n={interval.n}, {int(interval.level * 100)}% CI, {interval.method})")
+    return 0
+
+
+def cmd_compare(args: argparse.Namespace) -> int:
+    rec_a = read_jsonl(args.a)
+    rec_b = read_jsonl(args.b)
+    map_a = extract(rec_a, args.metric, args.id_key)
+    map_b = extract(rec_b, args.metric, args.id_key)
+    values_a, values_b, dropped = pair(map_a, map_b)
+    if not values_a:
+        print("error: no shared item ids between the two runs", file=sys.stderr)
+        return 2
+    if dropped:
+        print(f"note: {dropped} unpaired item(s) dropped", file=sys.stderr)
+    result = paired_compare(values_a, values_b, args.level, args.reps, args.seed)
+    if args.json:
+        print(json.dumps(result.as_dict(), indent=2))
+        return 0
+
+    pct = result.binary
+    fmt = _fmt_pct if pct else lambda v: f"{v:+.4g}"
+    plain = _fmt_pct if pct else lambda v: f"{v:.4g}"
+    sign = "+" if result.diff >= 0 else ""
+    print(f"A: {plain(result.mean_a)}   B: {plain(result.mean_b)}   (n={result.n} paired items)")
+    print(f"B - A: {sign}{fmt(result.diff).lstrip('+') if not pct else fmt(result.diff)}  "
+          f"[{fmt(result.ci_lo)}, {fmt(result.ci_hi)}]  ({int(result.level * 100)}% CI)")
+    print(f"p (sign-flip permutation): {result.p_permutation:.4f}   p (paired t): {result.p_t:.4f}")
+    if result.mcnemar is not None:
+        mc = result.mcnemar
+        print(f"discordant items: B-only wins {mc['b_only']}, A-only wins {mc['a_only']}   "
+              f"p (exact McNemar): {mc['p']:.4f}")
+    verdict = (
+        "significant at the chosen level"
+        if result.p_permutation < 1.0 - args.level
+        else "NOT significant — the difference is within noise at this sample size"
+    )
+    print(f"verdict: {verdict}")
+    return 0
+
+
+def cmd_power(args: argparse.Namespace) -> int:
+    sd = sd_diff_from_rates(args.baseline, args.baseline + args.delta, args.corr)
+    n = sample_size(args.delta, sd, args.level, args.power)
+    out = {
+        "baseline": args.baseline,
+        "delta": args.delta,
+        "corr": args.corr,
+        "sd_diff": sd,
+        "level": args.level,
+        "power": args.power,
+        "n_items": n,
+    }
+    if args.json:
+        print(json.dumps(out, indent=2))
+        return 0
+    print(f"to detect {_fmt_pct(abs(args.delta))} difference from a "
+          f"{_fmt_pct(args.baseline)} baseline (corr={args.corr}):")
+    print(f"  n = {n} paired items  "
+          f"({int(args.power * 100)}% power, {int(args.level * 100)}% confidence)")
+    for n_have in (100, 200, 500, 1000):
+        d = mde(n_have, sd, args.level, args.power)
+        print(f"  with {n_have:>4} items you can detect >= {_fmt_pct(d)}")
+    return 0
+
+
+def cmd_reliability(args: argparse.Namespace) -> int:
+    records = read_jsonl(args.results)
+    groups: dict = {}
+    for rec in records:
+        if args.id_key not in rec or args.metric not in rec:
+            raise ValueError(f"record missing {args.id_key!r} or {args.metric!r}: {rec}")
+        groups.setdefault(rec[args.id_key], []).append(rec[args.metric])
+    rel = judge_reliability(groups)
+    if args.json:
+        print(json.dumps(rel.as_dict(), indent=2))
+        return 0
+    print(f"{rel.n_items} items, {rel.n_judgments} judgments "
+          f"({rel.mean_reps:.1f} per item)")
+    print(f"ICC (signal share of variance): {rel.icc:.3f}")
+    print(f"between-item sd: {rel.sd_between:.4g}   judge noise sd: {rel.sd_within:.4g}")
+    print(f"exact agreement across repeats: {_fmt_pct(rel.exact_agreement)}")
+    reps = rel.repeats_for(0.1)
+    if reps is not None:
+        print(f"repeats per item to push judge noise under 10%: {reps}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="abeval",
+        description="A/B-test statistics for LLM evals.",
+    )
+    parser.add_argument("--version", action="version", version=f"abeval {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("--metric", default="score", help="metric field name (default: score)")
+    common.add_argument("--id-key", default="id", help="item id field name (default: id)")
+    common.add_argument("--level", type=float, default=0.95, help="confidence level (default: 0.95)")
+    common.add_argument("--seed", type=int, default=0, help="RNG seed (default: 0)")
+    common.add_argument("--reps", type=int, default=10_000,
+                        help="bootstrap/permutation resamples (default: 10000)")
+    common.add_argument("--json", action="store_true", help="emit JSON instead of text")
+
+    p_ci = sub.add_parser("ci", parents=[common],
+                          help="confidence interval for one run's metric")
+    p_ci.add_argument("results", help="JSONL file with per-item results")
+    p_ci.add_argument("--bootstrap", action="store_true",
+                      help="percentile bootstrap instead of t/Wilson")
+    p_ci.add_argument("--cluster-key", default=None,
+                      help="field defining dependence clusters (e.g. source document)")
+    p_ci.set_defaults(fn=cmd_ci)
+
+    p_cmp = sub.add_parser("compare", parents=[common],
+                           help="paired comparison of two runs on shared items")
+    p_cmp.add_argument("a", help="baseline run JSONL")
+    p_cmp.add_argument("b", help="candidate run JSONL")
+    p_cmp.set_defaults(fn=cmd_compare)
+
+    p_pow = sub.add_parser("power", parents=[common],
+                           help="how many items you need before you run")
+    p_pow.add_argument("--baseline", type=float, required=True,
+                       help="expected baseline accuracy, e.g. 0.75")
+    p_pow.add_argument("--delta", type=float, required=True,
+                       help="difference you care about detecting, e.g. 0.03")
+    p_pow.add_argument("--corr", type=float, default=0.5,
+                       help="item-level correlation between runs (default: 0.5)")
+    p_pow.add_argument("--power", type=float, default=0.8,
+                       help="target power (default: 0.8)")
+    p_pow.set_defaults(fn=cmd_power)
+
+    p_rel = sub.add_parser("reliability", parents=[common],
+                           help="judge-noise decomposition from repeated judgments")
+    p_rel.add_argument("results", help="JSONL with repeated judgments per item id")
+    p_rel.set_defaults(fn=cmd_reliability)
+
+    args = parser.parse_args(argv)
+    try:
+        return args.fn(args)
+    except (ValueError, OSError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

abeval/_dist.py

@@ -0,0 +1,109 @@
+"""Distribution helpers used by the estimators.
+
+Everything here is standard-library only. The Student-t CDF goes through the
+regularized incomplete beta function (continued-fraction expansion), and the
+quantile inverts it by bisection — slower than a closed form but robust for
+every df, which is what a stats tool should optimize for.
+"""
+
+from __future__ import annotations
+
+import math
+from statistics import NormalDist
+
+_NORMAL = NormalDist()
+
+
+def z_quantile(p: float) -> float:
+    """Standard normal quantile."""
+    if not 0.0 < p < 1.0:
+        raise ValueError("p must be in (0, 1)")
+    return _NORMAL.inv_cdf(p)
+
+
+def normal_cdf(x: float) -> float:
+    return _NORMAL.cdf(x)
+
+
+def _betacf(a: float, b: float, x: float) -> float:
+    """Continued fraction for the incomplete beta function (Lentz's method)."""
+    max_iter = 300
+    eps = 3e-12
+    fpmin = 1e-300
+    qab = a + b
+    qap = a + 1.0
+    qam = a - 1.0
+    c = 1.0
+    d = 1.0 - qab * x / qap
+    if abs(d) < fpmin:
+        d = fpmin
+    d = 1.0 / d
+    h = d
+    for m in range(1, max_iter + 1):
+        m2 = 2 * m
+        aa = m * (b - m) * x / ((qam + m2) * (a + m2))
+        d = 1.0 + aa * d
+        if abs(d) < fpmin:
+            d = fpmin
+        c = 1.0 + aa / c
+        if abs(c) < fpmin:
+            c = fpmin
+        d = 1.0 / d
+        h *= d * c
+        aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2))
+        d = 1.0 + aa * d
+        if abs(d) < fpmin:
+            d = fpmin
+        c = 1.0 + aa / c
+        if abs(c) < fpmin:
+            c = fpmin
+        d = 1.0 / d
+        delta = d * c
+        h *= delta
+        if abs(delta - 1.0) < eps:
+            break
+    return h
+
+
+def reg_inc_beta(a: float, b: float, x: float) -> float:
+    """Regularized incomplete beta I_x(a, b)."""
+    if x <= 0.0:
+        return 0.0
+    if x >= 1.0:
+        return 1.0
+    ln_front = (
+        math.lgamma(a + b)
+        - math.lgamma(a)
+        - math.lgamma(b)
+        + a * math.log(x)
+        + b * math.log1p(-x)
+    )
+    front = math.exp(ln_front)
+    if x < (a + 1.0) / (a + b + 2.0):
+        return front * _betacf(a, b, x) / a
+    return 1.0 - front * _betacf(b, a, 1.0 - x) / b
+
+
+def t_cdf(t: float, df: float) -> float:
+    """CDF of Student's t with `df` degrees of freedom."""
+    if df <= 0:
+        raise ValueError("df must be positive")
+    x = df / (df + t * t)
+    p_tail = 0.5 * reg_inc_beta(df / 2.0, 0.5, x)
+    return 1.0 - p_tail if t > 0 else p_tail
+
+
+def t_quantile(p: float, df: float) -> float:
+    """Quantile of Student's t, by bisection on the CDF."""
+    if not 0.0 < p < 1.0:
+        raise ValueError("p must be in (0, 1)")
+    if df <= 0:
+        raise ValueError("df must be positive")
+    lo, hi = -1e8, 1e8
+    for _ in range(200):
+        mid = (lo + hi) / 2.0
+        if t_cdf(mid, df) < p:
+            lo = mid
+        else:
+            hi = mid
+    return (lo + hi) / 2.0

abeval/ci.py

@@ -0,0 +1,124 @@
+"""Confidence intervals for a single eval run."""
+
+from __future__ import annotations
+
+import math
+import random
+from dataclasses import dataclass
+
+from ._dist import t_quantile, z_quantile
+
+
+@dataclass
+class Interval:
+    estimate: float
+    lo: float
+    hi: float
+    n: int
+    method: str
+    level: float
+    se: float | None = None
+
+    def as_dict(self) -> dict:
+        return {
+            "estimate": self.estimate,
+            "lo": self.lo,
+            "hi": self.hi,
+            "n": self.n,
+            "method": self.method,
+            "level": self.level,
+            "se": self.se,
+        }
+
+
+def proportion_ci(successes: int, n: int, level: float = 0.95) -> Interval:
+    """Wilson score interval for a binomial proportion."""
+    if n <= 0:
+        raise ValueError("n must be positive")
+    if not 0 <= successes <= n:
+        raise ValueError("successes must be in [0, n]")
+    z = z_quantile(1.0 - (1.0 - level) / 2.0)
+    p_hat = successes / n
+    denom = 1.0 + z * z / n
+    center = (p_hat + z * z / (2.0 * n)) / denom
+    half = (z / denom) * math.sqrt(p_hat * (1.0 - p_hat) / n + z * z / (4.0 * n * n))
+    return Interval(
+        estimate=p_hat,
+        lo=max(0.0, center - half),
+        hi=min(1.0, center + half),
+        n=n,
+        method="wilson",
+        level=level,
+    )
+
+
+def _mean(values: list[float]) -> float:
+    return sum(values) / len(values)
+
+
+def _sample_sd(values: list[float]) -> float:
+    n = len(values)
+    m = _mean(values)
+    return math.sqrt(sum((v - m) ** 2 for v in values) / (n - 1))
+
+
+def mean_ci(
+    values: list[float],
+    level: float = 0.95,
+    method: str = "t",
+    reps: int = 10_000,
+    seed: int = 0,
+) -> Interval:
+    """CI for the mean of per-item values.
+
+    method="t" uses the t distribution; method="bootstrap" uses the percentile
+    bootstrap with a seeded RNG so results are reproducible.
+    """
+    n = len(values)
+    if n < 2:
+        raise ValueError("need at least 2 values")
+    m = _mean(values)
+    if method == "t":
+        se = _sample_sd(values) / math.sqrt(n)
+        t = t_quantile(1.0 - (1.0 - level) / 2.0, n - 1)
+        return Interval(m, m - t * se, m + t * se, n, "t", level, se=se)
+    if method == "bootstrap":
+        rng = random.Random(seed)
+        stats = sorted(
+            _mean([values[rng.randrange(n)] for _ in range(n)]) for _ in range(reps)
+        )
+        alpha = (1.0 - level) / 2.0
+        lo = stats[int(alpha * reps)]
+        hi = stats[min(reps - 1, int((1.0 - alpha) * reps))]
+        return Interval(m, lo, hi, n, "bootstrap", level)
+    raise ValueError(f"unknown method {method!r}")
+
+
+def clustered_mean_ci(
+    values: list[float],
+    clusters: list,
+    level: float = 0.95,
+) -> Interval:
+    """CI for a mean when items are not independent within clusters.
+
+    Uses the cluster-robust variance of the mean (sum of squared within-cluster
+    residual sums over n^2) with a t reference on G-1 degrees of freedom, where
+    G is the number of clusters. With few clusters (< ~20) treat the interval
+    as approximate.
+    """
+    if len(values) != len(clusters):
+        raise ValueError("values and clusters must have the same length")
+    n = len(values)
+    if n < 2:
+        raise ValueError("need at least 2 values")
+    m = _mean(values)
+    sums: dict = {}
+    for v, c in zip(values, clusters):
+        sums[c] = sums.get(c, 0.0) + (v - m)
+    n_clusters = len(sums)
+    if n_clusters < 2:
+        raise ValueError("need at least 2 clusters")
+    var = sum(s * s for s in sums.values()) / (n * n)
+    se = math.sqrt(var)
+    t = t_quantile(1.0 - (1.0 - level) / 2.0, n_clusters - 1)
+    return Interval(m, m - t * se, m + t * se, n, f"clustered-t ({n_clusters} clusters)", level, se=se)

abeval/compare.py

@@ -0,0 +1,131 @@
+"""Paired comparison of two eval runs on the same items."""
+
+from __future__ import annotations
+
+import math
+import random
+from dataclasses import dataclass, field
+
+from ._dist import t_cdf
+
+
+@dataclass
+class Comparison:
+    n: int
+    mean_a: float
+    mean_b: float
+    diff: float  # mean(b - a): positive means B is better
+    ci_lo: float
+    ci_hi: float
+    level: float
+    p_permutation: float
+    p_t: float
+    reps: int
+    seed: int
+    binary: bool
+    mcnemar: dict | None = field(default=None)
+
+    def as_dict(self) -> dict:
+        out = {
+            "n": self.n,
+            "mean_a": self.mean_a,
+            "mean_b": self.mean_b,
+            "diff": self.diff,
+            "ci_lo": self.ci_lo,
+            "ci_hi": self.ci_hi,
+            "level": self.level,
+            "p_permutation": self.p_permutation,
+            "p_t": self.p_t,
+            "reps": self.reps,
+            "seed": self.seed,
+            "binary": self.binary,
+        }
+        if self.mcnemar is not None:
+            out["mcnemar"] = self.mcnemar
+        return out
+
+
+def mcnemar_exact(a: list[float], b: list[float]) -> dict:
+    """Exact McNemar test for paired binary outcomes.
+
+    b_only = items B got right and A got wrong; a_only = the reverse.
+    The p-value is the exact two-sided binomial sign test on the discordant
+    pairs (p = 0.5 under the null), clamped to 1.
+    """
+    a_only = sum(1 for x, y in zip(a, b) if x == 1 and y == 0)
+    b_only = sum(1 for x, y in zip(a, b) if x == 0 and y == 1)
+    n_disc = a_only + b_only
+    if n_disc == 0:
+        return {"a_only": 0, "b_only": 0, "p": 1.0}
+    k = min(a_only, b_only)
+    tail = sum(math.comb(n_disc, i) for i in range(k + 1)) / 2.0**n_disc
+    return {"a_only": a_only, "b_only": b_only, "p": min(1.0, 2.0 * tail)}
+
+
+def paired_compare(
+    a: list[float],
+    b: list[float],
+    level: float = 0.95,
+    reps: int = 10_000,
+    seed: int = 0,
+) -> Comparison:
+    """Compare two runs scored on the same items.
+
+    Reports mean(b - a) with a paired percentile-bootstrap CI, a sign-flip
+    permutation p-value (exact under item exchangeability), a paired-t
+    p-value, and — when both runs are 0/1 — an exact McNemar test.
+    """
+    if len(a) != len(b):
+        raise ValueError("runs must have the same length")
+    n = len(a)
+    if n < 2:
+        raise ValueError("need at least 2 paired items")
+    diffs = [y - x for x, y in zip(a, b)]
+    mean_diff = sum(diffs) / n
+
+    rng = random.Random(seed)
+
+    # Percentile bootstrap on the per-item differences.
+    boot = sorted(
+        sum(diffs[rng.randrange(n)] for _ in range(n)) / n for _ in range(reps)
+    )
+    alpha = (1.0 - level) / 2.0
+    ci_lo = boot[int(alpha * reps)]
+    ci_hi = boot[min(reps - 1, int((1.0 - alpha) * reps))]
+
+    # Sign-flip permutation test: under H0 each difference is symmetric
+    # around 0, so flipping signs at random regenerates the null.
+    obs = abs(mean_diff)
+    hits = 0
+    for _ in range(reps):
+        total = 0.0
+        for d in diffs:
+            total += d if rng.random() < 0.5 else -d
+        if abs(total / n) >= obs - 1e-15:
+            hits += 1
+    p_perm = (1 + hits) / (reps + 1)
+
+    # Paired t-test.
+    sd = math.sqrt(sum((d - mean_diff) ** 2 for d in diffs) / (n - 1))
+    if sd == 0.0:
+        p_t = 1.0 if mean_diff == 0.0 else 0.0
+    else:
+        t_stat = mean_diff / (sd / math.sqrt(n))
+        p_t = 2.0 * (1.0 - t_cdf(abs(t_stat), n - 1))
+
+    binary = all(v in (0.0, 1.0) for v in a) and all(v in (0.0, 1.0) for v in b)
+    return Comparison(
+        n=n,
+        mean_a=sum(a) / n,
+        mean_b=sum(b) / n,
+        diff=mean_diff,
+        ci_lo=ci_lo,
+        ci_hi=ci_hi,
+        level=level,
+        p_permutation=p_perm,
+        p_t=p_t,
+        reps=reps,
+        seed=seed,
+        binary=binary,
+        mcnemar=mcnemar_exact(a, b) if binary else None,
+    )

abeval/data.py

@@ -0,0 +1,54 @@
+"""Reading eval results and pairing two runs on a shared item id."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+def read_jsonl(path: str | Path) -> list[dict]:
+    """Read a JSONL file into a list of dicts. Blank lines are skipped."""
+    records = []
+    with open(path, encoding="utf-8") as fh:
+        for line_no, line in enumerate(fh, 1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                records.append(json.loads(line))
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"{path}:{line_no}: invalid JSON: {exc}") from exc
+    return records
+
+
+def extract(records: list[dict], metric: str, id_key: str = "id") -> dict:
+    """Map item id -> metric value. Duplicate ids and missing fields are errors."""
+    out: dict = {}
+    for rec in records:
+        if id_key not in rec:
+            raise ValueError(f"record missing id key {id_key!r}: {rec}")
+        if metric not in rec:
+            raise ValueError(f"record {rec[id_key]!r} missing metric {metric!r}")
+        item_id = rec[id_key]
+        if item_id in out:
+            raise ValueError(f"duplicate id {item_id!r}")
+        value = rec[metric]
+        if isinstance(value, bool):
+            value = int(value)
+        if not isinstance(value, (int, float)):
+            raise ValueError(f"metric {metric!r} for id {item_id!r} is not numeric: {value!r}")
+        out[item_id] = float(value)
+    return out
+
+
+def pair(
+    a: dict, b: dict
+) -> tuple[list[float], list[float], int]:
+    """Inner-join two id->value maps.
+
+    Returns (a_values, b_values, n_dropped) where n_dropped counts ids present
+    in only one of the two runs. Order follows a's insertion order.
+    """
+    shared = [item_id for item_id in a if item_id in b]
+    dropped = (len(a) - len(shared)) + (len(b) - len(shared))
+    return [a[i] for i in shared], [b[i] for i in shared], dropped

abeval/power.py

@@ -0,0 +1,127 @@
+"""Sample-size planning and minimum detectable effects for paired evals."""
+
+from __future__ import annotations
+
+import math
+import random
+
+from ._dist import z_quantile
+
+
+def sd_diff_from_rates(p_a: float, p_b: float, corr: float) -> float:
+    """SD of the per-item difference for paired binary outcomes.
+
+    corr is the item-level correlation between the two runs' outcomes
+    (0 = independent, higher = the models succeed and fail on the same items,
+    which is the typical case and what makes paired designs cheap).
+    """
+    for p in (p_a, p_b):
+        if not 0.0 < p < 1.0:
+            raise ValueError("rates must be in (0, 1)")
+    if not -1.0 < corr < 1.0:
+        raise ValueError("corr must be in (-1, 1)")
+    var = (
+        p_a * (1.0 - p_a)
+        + p_b * (1.0 - p_b)
+        - 2.0 * corr * math.sqrt(p_a * (1.0 - p_a) * p_b * (1.0 - p_b))
+    )
+    if var <= 0.0:
+        raise ValueError("implied variance is not positive; check the inputs")
+    return math.sqrt(var)
+
+
+def sample_size(
+    delta: float,
+    sd_diff: float,
+    level: float = 0.95,
+    power: float = 0.8,
+) -> int:
+    """Paired items needed to detect a true difference `delta`.
+
+    Normal-approximation two-sided test on the mean of per-item differences.
+    """
+    if delta == 0.0:
+        raise ValueError("delta must be nonzero")
+    if sd_diff <= 0.0:
+        raise ValueError("sd_diff must be positive")
+    z_alpha = z_quantile(1.0 - (1.0 - level) / 2.0)
+    z_power = z_quantile(power)
+    n = ((z_alpha + z_power) * sd_diff / abs(delta)) ** 2
+    return max(2, math.ceil(n))
+
+
+def mde(
+    n: int,
+    sd_diff: float,
+    level: float = 0.95,
+    power: float = 0.8,
+) -> float:
+    """Minimum detectable difference at a given number of paired items."""
+    if n < 2:
+        raise ValueError("n must be at least 2")
+    if sd_diff <= 0.0:
+        raise ValueError("sd_diff must be positive")
+    z_alpha = z_quantile(1.0 - (1.0 - level) / 2.0)
+    z_power = z_quantile(power)
+    return (z_alpha + z_power) * sd_diff / math.sqrt(n)
+
+
+def paired_binary_cells(p_a: float, p_b: float, corr: float) -> dict:
+    """Joint distribution of paired 0/1 outcomes with the given margins/corr."""
+    sd_a = math.sqrt(p_a * (1.0 - p_a))
+    sd_b = math.sqrt(p_b * (1.0 - p_b))
+    p_both = p_a * p_b + corr * sd_a * sd_b
+    cells = {
+        "both": p_both,
+        "a_only": p_a - p_both,
+        "b_only": p_b - p_both,
+        "neither": 1.0 - p_a - p_b + p_both,
+    }
+    if any(v < -1e-12 for v in cells.values()):
+        raise ValueError("that correlation is not achievable with these rates")
+    return {k: max(0.0, v) for k, v in cells.items()}
+
+
+def power_simulated(
+    n: int,
+    p_a: float,
+    p_b: float,
+    corr: float,
+    level: float = 0.95,
+    reps: int = 2_000,
+    seed: int = 0,
+) -> float:
+    """Monte Carlo power of the paired z-test for binary outcomes.
+
+    A check on the normal-approximation formulas: simulates `reps` evals of n
+    paired items with the given accuracy margins and item-level correlation,
+    and reports the fraction where the test rejects at the given level.
+    """
+    cells = paired_binary_cells(p_a, p_b, corr)
+    thresholds = []
+    acc = 0.0
+    for key in ("both", "a_only", "b_only", "neither"):
+        acc += cells[key]
+        thresholds.append((acc, key))
+    diff_of = {"both": 0.0, "neither": 0.0, "a_only": -1.0, "b_only": 1.0}
+    z_alpha = z_quantile(1.0 - (1.0 - level) / 2.0)
+    rng = random.Random(seed)
+    rejections = 0
+    for _ in range(reps):
+        diffs = []
+        for _ in range(n):
+            u = rng.random()
+            for cutoff, key in thresholds:
+                if u <= cutoff:
+                    diffs.append(diff_of[key])
+                    break
+            else:
+                diffs.append(0.0)
+        m = sum(diffs) / n
+        var = sum((d - m) ** 2 for d in diffs) / (n - 1)
+        if var == 0.0:
+            continue
+        z_stat = m / math.sqrt(var / n)
+        if abs(z_stat) > z_alpha:
+            rejections += 1
+    return rejections / reps

abeval/reliability.py

@@ -0,0 +1,85 @@
+"""How noisy is the judge? Variance decomposition over repeated judgments."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+
+
+@dataclass
+class Reliability:
+    n_items: int
+    n_judgments: int
+    mean_reps: float
+    icc: float
+    sd_between: float  # spread of true item scores
+    sd_within: float  # judge noise on a single judgment
+    exact_agreement: float  # fraction of items where every repeat matched
+
+    def repeats_for(self, noise_share: float = 0.1) -> int | None:
+        """Repeats per item so judge noise is at most `noise_share` of the
+        variance of an item's averaged score. None when the between-item
+        variance is zero (no signal to protect)."""
+        if not 0.0 < noise_share < 1.0:
+            raise ValueError("noise_share must be in (0, 1)")
+        var_b = self.sd_between**2
+        var_w = self.sd_within**2
+        if var_b <= 0.0:
+            return None
+        if var_w == 0.0:
+            return 1
+        return max(1, math.ceil(var_w * (1.0 - noise_share) / (noise_share * var_b)))
+
+    def as_dict(self) -> dict:
+        return {
+            "n_items": self.n_items,
+            "n_judgments": self.n_judgments,
+            "mean_reps": self.mean_reps,
+            "icc": self.icc,
+            "sd_between": self.sd_between,
+            "sd_within": self.sd_within,
+            "exact_agreement": self.exact_agreement,
+        }
+
+
+def judge_reliability(groups: dict) -> Reliability:
+    """One-way random-effects decomposition of repeated judge scores.
+
+    `groups` maps item id -> list of scores from repeated judgments of the
+    same item. Unbalanced designs are fine (k0 correction). Returns the
+    intraclass correlation (share of variance that is real item signal rather
+    than judge noise) plus both variance components.
+    """
+    groups = {k: [float(v) for v in vals] for k, vals in groups.items() if len(vals) >= 1}
+    if len(groups) < 2:
+        raise ValueError("need at least 2 items")
+    if all(len(v) < 2 for v in groups.values()):
+        raise ValueError("need repeated judgments (at least one item with 2+ scores)")
+
+    n_total = sum(len(v) for v in groups.values())
+    n_groups = len(groups)
+    grand = sum(sum(v) for v in groups.values()) / n_total
+
+    ss_between = sum(len(v) * (sum(v) / len(v) - grand) ** 2 for v in groups.values())
+    ss_within = sum(
+        sum((x - sum(v) / len(v)) ** 2 for x in v) for v in groups.values()
+    )
+    ms_between = ss_between / (n_groups - 1)
+    ms_within = ss_within / (n_total - n_groups) if n_total > n_groups else 0.0
+
+    k0 = (n_total - sum(len(v) ** 2 for v in groups.values()) / n_total) / (n_groups - 1)
+    var_within = ms_within
+    var_between = max(0.0, (ms_between - ms_within) / k0) if k0 > 0 else 0.0
+    total = var_between + var_within
+    icc = var_between / total if total > 0 else 0.0
+
+    exact = sum(1 for v in groups.values() if len(set(v)) == 1) / n_groups
+    return Reliability(
+        n_items=n_groups,
+        n_judgments=n_total,
+        mean_reps=n_total / n_groups,
+        icc=icc,
+        sd_between=math.sqrt(var_between),
+        sd_within=math.sqrt(var_within),
+        exact_agreement=exact,
+    )

abeval-0.1.0.dist-info/METADATA

@@ -0,0 +1,201 @@
+Metadata-Version: 2.4
+Name: abeval
+Version: 0.1.0
+Summary: A/B-test statistics for LLM evals: error bars, paired comparisons, and sample-size planning
+Project-URL: Homepage, https://github.com/mohammadi-hadi/abeval
+Project-URL: Calculator, https://mohammadi.cv/abeval/
+Author: Hadi Mohammadi
+License: MIT
+License-File: LICENSE
+Keywords: ab-testing,evaluation,llm,sample-size,significance,statistics
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# abeval
+
+A/B-test statistics for LLM evals.
+
+Eval scores are sample estimates, but they're routinely reported as exact
+numbers: "our prompt scores 76.5%, the old one 63.5%, ship it." On 200 items
+much smaller gaps than that are pure noise. abeval treats an eval like an A/B
+test — confidence intervals for a single run, paired significance tests for
+comparing two runs on the same items, and sample-size planning *before* you
+spend on inference. Standard library only, no dependencies.
+
+**Sample-size calculator:** https://mohammadi.cv/abeval/
+
+## Install
+
+```
+pip install abeval
+```
+
+Or from source: `git clone https://github.com/mohammadi-hadi/abeval && cd abeval && make install`.
+
+## Is my new prompt actually better?
+
+Point `abeval compare` at two JSONL result files that share item ids:
+
+```
+$ abeval compare examples/run_a.jsonl examples/run_b.jsonl
+A: 63.5%   B: 76.5%   (n=200 paired items)
+B - A: +13.0%  [4.5%, 21.5%]  (95% CI)
+p (sign-flip permutation): 0.0041   p (paired t): 0.0034
+discordant items: B-only wins 53, A-only wins 27   p (exact McNemar): 0.0049
+verdict: significant at the chosen level
+```
+
+The comparison is **paired**: both runs are scored on the same items, so
+item difficulty cancels out and you detect much smaller differences than two
+independent runs would allow. Items present in only one run are dropped (and
+reported).
+
+Every command takes `--json` for machine-readable output, and `--seed` makes
+the resampling reproducible.
+
+## Error bars for one run
+
+```
+$ abeval ci examples/run_a.jsonl
+score: 63.5%  [56.6%, 69.9%]  (n=200, 95% CI, wilson)
+```
+
+Binary metrics get a Wilson score interval, continuous metrics a t interval
+(or `--bootstrap`). If your items aren't independent — 5 questions generated
+from each source document, say — pass `--cluster-key document` and the
+interval widens to account for within-cluster correlation.
+
+## How many items do I need?
+
+Run this *before* the eval, not after:
+
+```
+$ abeval power --baseline 0.75 --delta 0.03
+to detect 3.0% difference from a 75.0% baseline (corr=0.5):
+  n = 1568 paired items  (80% power, 95% confidence)
+  with  100 items you can detect >= 11.9%
+  with  200 items you can detect >= 8.4%
+  with  500 items you can detect >= 5.3%
+  with 1000 items you can detect >= 3.8%
+```
+
+The lesson generalizes: a 200-item eval cannot see a 3-point improvement.
+`--corr` is the item-level correlation between the two runs' outcomes (0.5 is
+a reasonable default for two variants of the same system; higher correlation
+means fewer items needed). There's an interactive version at
+https://mohammadi.cv/abeval/.
+
+## How noisy is my judge?
+
+If an LLM judge scores the same item differently on repeat calls, that noise
+eats your statistical power. Feed repeated judgments to `reliability`:
+
+```
+$ abeval reliability examples/judge_repeats.jsonl
+40 items, 120 judgments (3.0 per item)
+ICC (signal share of variance): 0.764
+between-item sd: 1.708   judge noise sd: 0.9487
+exact agreement across repeats: 15.0%
+repeats per item to push judge noise under 10%: 3
+```
+
+ICC is the share of score variance that is real item signal rather than judge
+noise, from a one-way random-effects decomposition. The last line answers the
+practical question: how many repeat judgments to average per item.
+
+## Data format
+
+JSONL, one item per line, any extra fields ignored:
+
+```json
+{"id": "item-001", "score": 1}
+{"id": "item-002", "score": 0, "category": "reasoning"}
+```
+
+`--metric` and `--id-key` rename the fields; booleans are accepted as 0/1.
+The files in `examples/` are generated by `examples/make_fixtures.py`
+(seeded, so they're reproducible).
+
+## Python API
+
+Everything the CLI does is a plain function:
+
+```python
+from abeval import paired_compare, proportion_ci, sample_size, sd_diff_from_rates
+
+proportion_ci(127, 200)                  # Wilson interval, level=0.95
+result = paired_compare(scores_a, scores_b, seed=0)
+result.diff, result.ci_lo, result.ci_hi, result.p_permutation
+
+sd = sd_diff_from_rates(0.75, 0.78, corr=0.5)
+sample_size(0.03, sd)                    # -> 1568 paired items
+```
+
+## What's inside
+
+| Question | Method |
+|---|---|
+| CI for a pass rate | Wilson score interval |
+| CI for a mean score | Student t, percentile bootstrap, or cluster-robust t |
+| Is B better than A? | Sign-flip permutation test on paired differences (primary), paired t, exact McNemar on discordant pairs (binary) |
+| How many items? | Normal-approximation power for the paired design, with a Monte-Carlo check (`power_simulated`) |
+| Judge noise | One-way random-effects ICC with unbalanced-design correction |
+
+The permutation p-value is the primary test: it's exact under item
+exchangeability and makes no normality assumption. The paired t and McNemar
+values are printed alongside because reviewers ask for them.
+
+The approach follows Miller,
+["Adding Error Bars to Evals"](https://arxiv.org/abs/2411.00640) (2024) —
+report standard errors, use paired designs, plan sample sizes — packaged as a
+tool you can point at result files.
+
+## Honest limitations
+
+- The power formulas use the normal approximation; for very small samples or
+  rates near 0/1 trust `power_simulated` over the closed form.
+- Cluster-robust intervals are approximate with fewer than ~20 clusters.
+- Multiple-comparison correction is on you: if you compare ten prompts and
+  ship the best p-value, it's inflated (a `sweep` command with Bonferroni
+  correction is on the roadmap).
+- This is not a stats library. If you need regressions or GLMs, use
+  statsmodels; abeval covers the eval loop with zero dependencies.
+
+## Sponsoring
+
+abeval is MIT-licensed and dependency-free, and it stays that way. Sponsoring
+funds the roadmap below and the maintenance time to keep the statistics
+trustworthy. Sponsors are credited in release notes and vote on what lands
+next: [GitHub Sponsors](https://github.com/sponsors/mohammadi-hadi).
+
+## Roadmap
+
+- `sweep`: compare N runs against a baseline with multiple-comparison
+  correction.
+- Confidence intervals for pairwise win rates (Bradley-Terry) for
+  head-to-head judge outputs.
+- Variance-reduction guidance: how much power you buy by averaging k judge
+  repeats, given your measured ICC.
+
+## Related projects
+
+- [judgewatch](https://github.com/mohammadi-hadi/judgewatch) — monthly bias
+  audits of LLM judges; abeval tells you whether this month's change is real.
+- [judgekit](https://github.com/mohammadi-hadi/judgekit) — audit your own
+  judge pipelines.
+- [trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge) —
+  evaluating agent trajectories with LLM judges.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

abeval-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+abeval/__init__.py,sha256=9N4TC2HGKp5MR1K2mYDdrne986Vmr0AMg7pA_ndIkqE,902
+abeval/__main__.py,sha256=_gXHiC5klfM8bIZJkggW2pd6sxL5S5hEbWv6XIS5bMg,8118
+abeval/_dist.py,sha256=7xGFRocjFR_vKidhUN_ILMiH9qbsNYjr8HtaABcmQdI,2899
+abeval/ci.py,sha256=RPTAOHTh6lYQMSiC4_4XXQFn9YCMOxYu0bsq3Dt897w,3737
+abeval/compare.py,sha256=zVfDlBVDyXsVFoYiqgEkfL-pIr-9pqJl1V4trACQA4I,3877
+abeval/data.py,sha256=-gnQoxhApCWduPXR1lH3dP8rjCxwD2QaejvE_FNL90s,1955
+abeval/power.py,sha256=SmYBmx9BdF3CIEUd9Dxq2klSvPrKAqY9QqdFGawpfpI,4054
+abeval/reliability.py,sha256=MKJOcWGB8DnxZYd-cJzR8ZXCCrfP7sbLv8wXOl25zu0,3200
+abeval-0.1.0.dist-info/METADATA,sha256=_MgYaOfWv2OiibsmCuXS4SFB9-V7Urb1GYx0UNyhzYI,7530
+abeval-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+abeval-0.1.0.dist-info/entry_points.txt,sha256=7yE5AcyOrFQsKiB0VlKzOhcbt7celHftfkCSYP1OG-g,48
+abeval-0.1.0.dist-info/licenses/LICENSE,sha256=F-4iu05PfGFM6P0UkwKzuusSISVTwYeaC14QMud0b9g,1071
+abeval-0.1.0.dist-info/RECORD,,

abeval-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

abeval-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+abeval = abeval.__main__:main

abeval-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hadi Mohammadi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.