regression-substrate 0.1.0__py3-none-any.whl

regression_substrate/__init__.py

@@ -0,0 +1,36 @@
+"""
+regression_substrate — a statistically rigorous gate for deciding whether a new
+prompt/model version is better, worse, or inconclusive versus the current one.
+
+Public API:
+
+    from regression_substrate import gate, Judge, load_from_jsonl
+
+Offline gate (Day-1):  gate, GateDecision, variance_components
+Ingestion:             load_from_jsonl, load_from_csv, assemble_records,
+                       validate_records, Judge, auto_cluster
+Streaming (Day-2):     SequentialGate, Backend
+Drift (Day-2):         RollingGoldSet, drift_report, sample_for_labeling
+Vendor adapters:       flatten_runs, load_from_langsmith
+"""
+
+from .diff_engine import gate, GateDecision, variance_components
+from .ingest import (
+    load_from_jsonl, load_from_csv, assemble_records, validate_records,
+    Judge, cohen_kappa, auto_cluster, tfidf_embedder, sentence_transformer_embedder,
+)
+from .sequential_gate import SequentialGate, Backend, MartingaleState
+from .gold import RollingGoldSet, drift_report, sample_for_labeling
+from .adapters import flatten_runs, load_from_langsmith, TraceMap, LANGSMITH
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "gate", "GateDecision", "variance_components",
+    "load_from_jsonl", "load_from_csv", "assemble_records", "validate_records",
+    "Judge", "cohen_kappa", "auto_cluster", "tfidf_embedder", "sentence_transformer_embedder",
+    "SequentialGate", "Backend", "MartingaleState",
+    "RollingGoldSet", "drift_report", "sample_for_labeling",
+    "flatten_runs", "load_from_langsmith", "TraceMap", "LANGSMITH",
+    "__version__",
+]

regression_substrate/adapters.py

@@ -0,0 +1,185 @@
+"""
+adapters.py — pull from vendor observability platforms into the 7-field schema.
+
+HONESTY NOTE — READ THIS:
+  * The FLATTENING logic below is tested (see __main__) against a synthetic
+    fixture shaped like a vendor's documented run/feedback model. That transform
+    is proven.
+  * The LIVE FETCH (`load_from_langsmith`) is SDK- and auth-dependent and is NOT
+    exercised here. Vendor schemas drift, so the field paths in the presets are
+    best-effort and MUST be verified against the platform's current docs before
+    you trust them in production.
+
+Design: don't hardwire any one vendor. A `TraceMap` says where each field lives
+inside one run record; `flatten_runs` collapses (possibly nested, multi-step)
+runs + feedback into flat 7-field records that ingest.assemble_records consumes.
+A vendor is then just a TraceMap preset plus a thin fetch wrapper.
+"""
+
+from __future__ import annotations
+from dataclasses import dataclass
+import json
+
+
+def _dig(obj, path: str, default=None):
+    """Read a nested field by dotted path, e.g. 'extra.metadata.version'."""
+    cur = obj
+    for key in path.split("."):
+        if isinstance(cur, dict) and key in cur:
+            cur = cur[key]
+        else:
+            return default
+    return cur
+
+
+def _canon(x) -> str:
+    """Canonicalize an input/output payload to a stable string. The `input`
+    string is what groups replicates and pairs versions, so it must be stable."""
+    if isinstance(x, str):
+        return x
+    if isinstance(x, dict):
+        for k in ("input", "question", "query", "text", "prompt",
+                  "output", "answer", "result", "response"):
+            if isinstance(x.get(k), str):
+                return x[k]
+        return json.dumps(x, sort_keys=True)
+    return str(x)
+
+
+@dataclass
+class TraceMap:
+    """Where the fields live inside one vendor run record."""
+    input_path: str
+    output_path: str
+    version_path: str            # MUST have been logged by the team; no version => unpairable
+    score_key: str               # which feedback key carries the quality score
+    run_type_path: str = "run_type"
+    parent_path: str = "parent_run_id"
+    id_path: str = "id"
+    cluster_path: str | None = None
+    score_scale: tuple = (0.0, 1.0)
+
+
+# Best-effort preset. VERIFY these paths against current LangSmith docs.
+LANGSMITH = TraceMap(
+    input_path="inputs",
+    output_path="outputs",
+    version_path="extra.metadata.version",
+    score_key="quality",
+    run_type_path="run_type",
+    parent_path="parent_run_id",
+    id_path="id",
+    cluster_path="extra.metadata.cluster",
+    score_scale=(0.0, 1.0),
+)
+
+
+def flatten_runs(runs: list[dict], feedback: list[dict], tmap: TraceMap,
+                 unit: str = "root") -> list[dict]:
+    """Collapse runs + feedback into flat 7-field records.
+
+    unit="root"  -> evaluate whole trajectories (input=root input, response=root
+                    output). Child LLM/tool runs are diagnostic and dropped.
+    unit=<type>  -> evaluate a component instead (e.g. "retriever", "llm").
+    """
+    by_run: dict[str, dict] = {}
+    for f in feedback:
+        by_run.setdefault(f["run_id"], {})[f["key"]] = f.get("score")
+
+    lo, hi = tmap.score_scale
+    skipped_no_version = skipped_no_score = 0
+    records = []
+    for r in runs:
+        is_root = _dig(r, tmap.parent_path) is None
+        if unit == "root":
+            if not is_root:
+                continue
+        elif _dig(r, tmap.run_type_path) != unit:
+            continue
+
+        raw = (by_run.get(_dig(r, tmap.id_path)) or {}).get(tmap.score_key)
+        if raw is None:
+            skipped_no_score += 1
+            continue
+        version = _dig(r, tmap.version_path)
+        if version is None:
+            skipped_no_version += 1
+            continue
+
+        score = (float(raw) - lo) / (hi - lo) if hi != lo else float(raw)
+        rec = {
+            "input": _canon(_dig(r, tmap.input_path)),
+            "version": str(version),
+            "response": _canon(_dig(r, tmap.output_path)),
+            "score": max(0.0, min(1.0, score)),
+        }
+        if tmap.cluster_path:
+            c = _dig(r, tmap.cluster_path)
+            if c is not None:
+                rec["cluster"] = c
+        records.append(rec)
+
+    if skipped_no_version:
+        print(f"  [adapter] WARNING: skipped {skipped_no_version} runs with no "
+              f"version tag at '{tmap.version_path}' -- they cannot be paired.")
+    if skipped_no_score:
+        print(f"  [adapter] note: skipped {skipped_no_score} runs with no "
+              f"'{tmap.score_key}' feedback.")
+    return records
+
+
+def load_from_langsmith(project: str, version_a: str, version_b: str,
+                        tmap: TraceMap = LANGSMITH, unit: str = "root"):
+    """LIVE fetch + flatten + assemble. NOT exercised in this repo -- requires
+    the `langsmith` SDK and LANGSMITH_API_KEY, and the SDK call signatures and
+    schema below must be verified against current LangSmith docs."""
+    from langsmith import Client                      # raises if not installed
+    from .ingest import assemble_records
+
+    client = Client()
+    runs = [r.dict() for r in client.list_runs(project_name=project)]
+    run_ids = [r["id"] for r in runs]
+    feedback = [f.dict() for f in client.list_feedback(run_ids=run_ids)]
+    records = flatten_runs(runs, feedback, tmap, unit=unit)
+    return assemble_records(records, version_a, version_b)
+
+
+# --------------------------------------------------------------------------- #
+# Demo: flatten a SYNTHETIC LangSmith-shaped fixture -> gate(). Proves the
+# transform (nested trajectory collapse, version extraction, score scaling,
+# replicate derivation) without touching the live API.
+# --------------------------------------------------------------------------- #
+
+if __name__ == "__main__":
+    import numpy as np
+    from .ingest import assemble_records, validate_records
+    from .diff_engine import gate
+
+    rng = np.random.default_rng(0)
+    inputs = ([("billing", f"billing question {i}") for i in range(3)] +
+              [("general", f"general question {i}") for i in range(3)])
+
+    runs, feedback, rid = [], [], 0
+    for cluster, q in inputs:
+        for ver, base in [("v1", 0.90), ("v2", 0.20 if cluster == "billing" else 0.78)]:
+            for _rep in range(2):                      # two replicates per (input, version)
+                root = f"run-{rid}"; rid += 1
+                runs.append({"id": root, "run_type": "chain", "parent_run_id": None,
+                             "inputs": {"question": q}, "outputs": {"answer": "..."},
+                             "extra": {"metadata": {"version": ver, "cluster": cluster}}})
+                child = f"run-{rid}"; rid += 1          # a nested LLM step (must be dropped)
+                runs.append({"id": child, "run_type": "llm", "parent_run_id": root,
+                             "inputs": {}, "outputs": {}, "extra": {"metadata": {"version": ver}}})
+                score = float(np.clip(base + rng.normal(0, 0.03), 0, 1))
+                feedback.append({"run_id": root, "key": "quality", "score": round(score, 3)})
+
+    records = flatten_runs(runs, feedback, LANGSMITH, unit="root")
+    print(f"FLATTEN: {len(runs)} raw runs -> {len(records)} flat records "
+          f"(child runs dropped under unit='root')")
+    print("  sample:", records[0])
+    print("  validation problems:", validate_records(records) or "none")
+
+    sa, sb, cids, meta = assemble_records(records, "v1", "v2")
+    print("  assembled:", meta)
+    dec = gate(sa, sb, cids, judge_error_sd=0.05, kappa=0.78, alpha=0.05)
+    print("  verdict:", dec.verdict, "| CI:", tuple(round(x, 3) for x in dec.delta_ci))

regression_substrate/cli.py

@@ -0,0 +1,108 @@
+"""
+regression_substrate.cli — the `regsub` command. Reads scored eval data, calibrates
+a judge against a gold set, runs the offline gate, and writes reports. Designed to
+drop into a CI step:
+
+    regsub --data evals.csv --gold gold.jsonl --version-a v1 --version-b v2 --out out/
+    # exit code 0 = SHIP / SHIP_WITH_FLAGS ; 1 = REGRESSION / HOLD ; 2 = JUDGE_INADMISSIBLE
+
+By default the rubric judge below is used so the command runs with no API key. In
+production, import the package and pass your own judge:
+
+    from regression_substrate import Judge, load_from_csv, gate
+"""
+
+from __future__ import annotations
+import argparse
+import csv
+import json
+import os
+import sys
+from collections import defaultdict
+import numpy as np
+
+from .ingest import Judge, validate_records, assemble_records, auto_cluster
+from .diff_engine import gate, variance_components
+
+# A self-contained rubric judge so the CLI runs offline. Replace in production.
+_ACTION = {"refund", "reset", "update", "settings", "24/7", "link"}
+_COURTESY = {"sorry", "happy", "please", "help", "glad", "sure"}
+
+def default_judge(question: str, answer: str) -> float:
+    t = answer.lower()
+    spec = 1.0 if any(x in t for x in _ACTION) else 0.0
+    completeness = min(len(answer.split()) / 12.0, 1.0)
+    courtesy = 1.0 if any(w in t for w in _COURTESY) else 0.7
+    return round(0.55 * spec + 0.35 * completeness + 0.10 * courtesy, 3)
+
+
+_EXIT = {"SHIP": 0, "SHIP_WITH_FLAGS": 0, "REGRESSION": 1, "HOLD": 1, "JUDGE_INADMISSIBLE": 2}
+
+
+def run(data, gold, version_a, version_b, out_dir, alpha, min_n, score_fn=default_judge):
+    os.makedirs(out_dir, exist_ok=True)
+
+    records = []
+    with open(data, newline="") as f:
+        for row in csv.DictReader(f):
+            row["score"] = score_fn(row.get("input", ""), row.get("response", ""))
+            if str(row.get("replicate", "")).strip().isdigit():
+                row["replicate"] = int(row["replicate"])
+            else:
+                row.pop("replicate", None)
+            records.append(row)
+    problems = validate_records(records)
+    if problems:
+        sys.exit("Data problems:\n  " + "\n  ".join(problems))
+    with open(f"{out_dir}/records.jsonl", "w") as f:
+        f.write("\n".join(json.dumps(r) for r in records))
+
+    judge = Judge(score_fn)
+    cal = judge.calibrate([json.loads(l) for l in open(gold) if l.strip()])
+    with open(f"{out_dir}/calibration.json", "w") as f:
+        json.dump(cal, f, indent=2)
+
+    if any(not r.get("cluster") for r in records):
+        uniq = sorted({r["input"] for r in records})
+        cmap = dict(zip(uniq, auto_cluster(uniq, n_clusters=2)))
+        for r in records:
+            r.setdefault("cluster", int(cmap[r["input"]]))
+
+    sa, sb, cids, meta = assemble_records(records, version_a, version_b)
+    dec = gate(sa, sb, cids, judge_error_sd=cal["error_sd"], kappa=cal["kappa"],
+               alpha=alpha, min_n=min_n)
+    comp = variance_components(sa, sb)
+
+    report = {
+        "verdict": dec.verdict,
+        "delta_ci": [round(x, 4) for x in dec.delta_ci],
+        "weighted_delta": round(comp.delta_hat, 4),
+        "flagged_clusters": dec.flagged_clusters,
+        "judge": cal,
+        "n_inputs": meta["n_inputs"],
+        "note": dec.note,
+    }
+    with open(f"{out_dir}/gate_report.json", "w") as f:
+        json.dump(report, f, indent=2)
+
+    print(json.dumps(report, indent=2))
+    return dec.verdict
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog="regsub", description="Offline AI regression gate.")
+    p.add_argument("--data", required=True, help="CSV of scored responses (input,version,replicate,cluster,response)")
+    p.add_argument("--gold", required=True, help="JSONL gold set (input,response,human)")
+    p.add_argument("--version-a", default="v1", help="baseline version label")
+    p.add_argument("--version-b", default="v2", help="candidate version label")
+    p.add_argument("--out", default="out", help="output directory")
+    p.add_argument("--alpha", type=float, default=0.05)
+    p.add_argument("--min-n", type=int, default=30, help="power floor")
+    args = p.parse_args(argv)
+    verdict = run(args.data, args.gold, args.version_a, args.version_b,
+                  args.out, args.alpha, args.min_n)
+    sys.exit(_EXIT.get(verdict, 1))
+
+
+if __name__ == "__main__":
+    main()

regression_substrate/diff_engine.py

@@ -0,0 +1,350 @@
+"""
+diff_engine.py — reference implementation of the statistical diff engine.
+
+This is the core gate for a regression substrate for probabilistic software:
+given evaluation scores for a current version A and a candidate version B over
+the SAME captured inputs (paired design), decide whether B can ship, must be
+held, or is an outright regression — accounting for input difficulty, judge
+measurement noise, and within-version (token) sampling variance — and localize
+regressions to semantic clusters under FDR control.
+
+Data model
+----------
+scores_a, scores_b : np.ndarray of shape (N, k)
+    Judge quality scores in [0, 1]. N inputs, k replicate samples per input per
+    version (k > 1 is what lets us separate token noise from real difference).
+cluster_ids : np.ndarray of shape (N,)
+    Integer semantic-cluster label per input.
+judge_error_sd : float
+    Std-dev of the judge's measurement error, ESTIMATED FROM A HUMAN GOLD SET
+    (not assumed). Propagated through the bootstrap so an unreliable judge
+    widens the interval instead of silently passing the gate.
+kappa : float
+    Judge–human agreement (Cohen's kappa) on the gold set. Used only as an
+    admissibility diagnostic: below kappa_min the judge cannot gate at all.
+
+All statistics are deliberately explicit rather than delegated to a black-box
+fitter, so each variance component is auditable.
+"""
+
+from __future__ import annotations
+from dataclasses import dataclass, field
+import numpy as np
+from scipy import stats
+
+
+# --------------------------------------------------------------------------- #
+# 1. Variance components (mixed-effects view of the paired delta)
+# --------------------------------------------------------------------------- #
+# Model:  score[v, i, r] = mu + tau_v + u_i + (tau*u)_{v,i} + e_{v,i,r}
+#   tau_v        fixed version effect (the thing we want: tau_B - tau_A)
+#   u_i          random input effect    ~ N(0, sigma_input^2)
+#   (tau*u)_{vi} version x input interaction (where regressions live)
+#   e_{v,i,r}    residual token noise    ~ N(0, sigma_resid^2)
+#
+# In a PAIRED design the per-input delta cancels u_i and mu, leaving the
+# interaction plus averaged residuals. That is the estimator we actually gate
+# on; the residual variance is estimated separately so we can report the
+# signal-to-noise ratio explicitly ("variance-components filtering").
+
+@dataclass
+class VarianceComponents:
+    delta_hat: float          # estimated tau_B - tau_A (mean paired delta)
+    se_delta: float           # standard error of delta_hat
+    sigma_resid2: float       # within-(version,input) token-noise variance
+    interaction_var: float    # structural version x input variance (>=0)
+    snr: float                # |delta_hat| / sqrt(interaction_var + tiny)
+
+
+def _as_ragged(scores) -> list[np.ndarray]:
+    """Accept either a balanced (N, k) array or a list of variable-length 1D
+    arrays (one per input, length k_i) and normalize to a list of 1D arrays."""
+    if isinstance(scores, np.ndarray) and scores.ndim == 2:
+        return [scores[i] for i in range(scores.shape[0])]
+    return [np.asarray(s, dtype=float) for s in scores]
+
+
+def _pooled_residual_var(cells: list[np.ndarray]) -> tuple[float, float]:
+    """Token-noise variance pooled across cells, weighted by df = k_i - 1.
+    Cells with k=1 contribute zero df (no internal info) but are not dropped
+    elsewhere. Returns (sigma_resid2, total_df)."""
+    num = den = 0.0
+    for arr in cells:
+        if len(arr) >= 2:
+            num += (len(arr) - 1) * arr.var(ddof=1)
+            den += (len(arr) - 1)
+    return (num / den if den > 0 else 0.0), den
+
+
+def variance_components(scores_a, scores_b) -> VarianceComponents:
+    A, B = _as_ragged(scores_a), _as_ragged(scores_b)
+    N = len(A)
+    kA = np.array([len(a) for a in A], dtype=float)
+    kB = np.array([len(b) for b in B], dtype=float)
+    mA = np.array([a.mean() for a in A])
+    mB = np.array([b.mean() for b in B])
+    d = mB - mA                                        # per-input paired delta
+
+    # (1) df-weighted pooled token-noise variance across all cells of A and B.
+    sigma_resid2, total_df = _pooled_residual_var([*A, *B])
+
+    # (2) per-input measurement variance of d_i from token noise.
+    v = sigma_resid2 * (1.0 / kA + 1.0 / kB)           # (N,)
+
+    if total_df == 0:                                  # no replicate info anywhere
+        delta_hat = float(d.mean())
+        se = float(d.std(ddof=1) / np.sqrt(N))
+        tau2 = float(max(0.0, d.var(ddof=1)))
+        return VarianceComponents(delta_hat, se, sigma_resid2, tau2,
+                                  abs(delta_hat) / np.sqrt(tau2 + 1e-12))
+
+    # (3) DerSimonian-Laird estimate of between-input (interaction) variance.
+    w0 = 1.0 / np.maximum(v, 1e-12)                    # fixed-effect weights
+    delta_fe = np.sum(w0 * d) / np.sum(w0)
+    Q = np.sum(w0 * (d - delta_fe) ** 2)               # Cochran's Q
+    C = np.sum(w0) - np.sum(w0 ** 2) / np.sum(w0)
+    tau2 = max(0.0, (Q - (N - 1)) / C) if C > 0 else 0.0
+
+    # (4) random-effects inverse-variance weights: keep low-k inputs, trust
+    #     them proportionally to their precision (never truncate).
+    w = 1.0 / (tau2 + v)
+    delta_hat = float(np.sum(w * d) / np.sum(w))
+    se_delta = float(np.sqrt(1.0 / np.sum(w)))
+    snr = abs(delta_hat) / np.sqrt(tau2 + 1e-12)
+    return VarianceComponents(delta_hat, se_delta, sigma_resid2, float(tau2), snr)
+
+
+# --------------------------------------------------------------------------- #
+# 2. Bootstrap CI that wraps judge resampling
+# --------------------------------------------------------------------------- #
+# Cluster (input-level) bootstrap so the CI reflects input sampling, AND inject
+# judge measurement noise on every scored output so the CI also reflects judge
+# unreliability. An unreliable judge (large judge_error_sd) widens the interval
+# and prevents a confident ship — by construction, not by a hand-tuned fudge.
+
+def bootstrap_delta_ci(
+    scores_a,
+    scores_b,
+    judge_error_sd: float,
+    alpha: float = 0.05,
+    n_boot: int = 5000,
+    rng: np.random.Generator | None = None,
+) -> tuple[float, float, np.ndarray]:
+    rng = rng or np.random.default_rng(0)
+    A, B = _as_ragged(scores_a), _as_ragged(scores_b)
+    N = len(A)
+    kA = np.array([len(a) for a in A], dtype=float)
+    kB = np.array([len(b) for b in B], dtype=float)
+    mA = np.array([a.mean() for a in A])
+    mB = np.array([b.mean() for b in B])
+
+    # Random-effects weights from the full-data fit (held fixed across boots).
+    comp = variance_components(A, B)
+    v = comp.sigma_resid2 * (1.0 / kA + 1.0 / kB)
+    w = 1.0 / (comp.interaction_var + v + 1e-12)
+
+    # Judge noise on a CELL MEAN is Gaussian with variance sd^2 / k_i, so we
+    # perturb means directly instead of every score: same distribution, fully
+    # vectorizable, and low-k cells correctly carry MORE judge uncertainty.
+    idx = rng.integers(0, N, size=(n_boot, N))
+    seA = judge_error_sd / np.sqrt(kA[idx])
+    seB = judge_error_sd / np.sqrt(kB[idx])
+    da = mA[idx] + rng.normal(0, 1, idx.shape) * seA
+    db = mB[idx] + rng.normal(0, 1, idx.shape) * seB
+    dd = db - da
+    ww = w[idx]
+    boot = np.sum(ww * dd, axis=1) / np.sum(ww, axis=1)
+
+    lo, hi = np.percentile(boot, [100 * alpha / 2, 100 * (1 - alpha / 2)])
+    return float(lo), float(hi), boot
+
+
+# --------------------------------------------------------------------------- #
+# 3. Multiplicity control across clusters: BH (fixed-n) and e-BH (sequential)
+# --------------------------------------------------------------------------- #
+
+def benjamini_hochberg(pvalues: np.ndarray, alpha: float) -> np.ndarray:
+    """Return boolean mask of rejected hypotheses. FDR <= alpha under
+    independence / positive dependence (PRDS)."""
+    m = len(pvalues)
+    order = np.argsort(pvalues)
+    ranked = pvalues[order]
+    thresh = alpha * (np.arange(1, m + 1) / m)
+    passed = ranked <= thresh
+    reject = np.zeros(m, dtype=bool)
+    if passed.any():
+        kmax = np.max(np.where(passed)[0])           # largest rank that passes
+        reject[order[: kmax + 1]] = True
+    return reject
+
+
+def p_to_e(p: np.ndarray, kappa: float = 0.5) -> np.ndarray:
+    """Calibrate p-values to e-values with the kappa-calibrator
+    e = kappa * p**(kappa - 1), valid for any p (integrates to 1 under H0)."""
+    return kappa * np.power(np.clip(p, 1e-12, 1.0), kappa - 1.0)
+
+
+def e_bh(evalues: np.ndarray, alpha: float) -> np.ndarray:
+    """e-BH (Wang & Ramdas 2022). FDR <= alpha under ARBITRARY dependence and
+    valid under optional stopping — the version to use when the gate is hit
+    repeatedly over many deploys."""
+    m = len(evalues)
+    order = np.argsort(-evalues)                      # decreasing
+    ranked = evalues[order]
+    crit = m / (alpha * np.arange(1, m + 1))          # threshold per rank
+    passed = ranked >= crit
+    reject = np.zeros(m, dtype=bool)
+    if passed.any():
+        kmax = np.max(np.where(passed)[0])
+        reject[order[: kmax + 1]] = True
+    return reject
+
+
+# --------------------------------------------------------------------------- #
+# 4. Cluster regression scan
+# --------------------------------------------------------------------------- #
+# For each cluster, one-sided test of H0: delta >= 0 (B not worse) vs
+# H1: delta < 0 (regression). Small p = strong regression evidence. Then apply
+# BH (fixed-n) or e-BH (sequential) across clusters to control false flags.
+
+def paired_permutation_p(
+    d: np.ndarray, n_perm: int = 10000, rng: np.random.Generator | None = None
+) -> float:
+    """One-sided sign-flip permutation p-value for H1: mean(d) < 0 (regression).
+    Under H0 the paired delta is symmetric about 0, so each sign is exchangeable.
+    Distribution-free -- the correct test for small clusters where the normal
+    approximation throws false positives. (+1 correction keeps it valid.)"""
+    rng = rng or np.random.default_rng(0)
+    obs = d.mean()
+    signs = rng.choice(np.array([-1.0, 1.0]), size=(n_perm, len(d)))
+    perm_means = (signs * d).mean(axis=1)
+    return float((1 + np.sum(perm_means <= obs)) / (1 + n_perm))
+
+
+@dataclass
+class ClusterResult:
+    cluster_id: int
+    n: int
+    delta: float
+    p_value: float
+    e_value: float
+    flagged_regression: bool = False
+
+
+def cluster_scan(
+    scores_a: np.ndarray,
+    scores_b: np.ndarray,
+    cluster_ids: np.ndarray,
+    alpha: float = 0.10,
+    mode: str = "fixed",          # "fixed" -> BH ; "sequential" -> e-BH
+    e_kappa: float = 0.5,
+    small_n: int = 30,            # below this, use the permutation test
+    n_perm: int = 10000,
+    rng: np.random.Generator | None = None,
+) -> list[ClusterResult]:
+    rng = rng or np.random.default_rng(0)
+    A, B = _as_ragged(scores_a), _as_ragged(scores_b)
+    d_i = np.array([b.mean() - a.mean() for a, b in zip(A, B)])
+    results: list[ClusterResult] = []
+
+    for c in np.unique(cluster_ids):
+        d = d_i[cluster_ids == c]
+        n = len(d)
+        if n < 3:                                     # too small to test at all
+            results.append(ClusterResult(int(c), n, float(d.mean()) if n else 0.0,
+                                         1.0, p_to_e(np.array([1.0]), e_kappa)[0]))
+            continue
+        if n < small_n:                               # distribution-free fallback
+            p = paired_permutation_p(d, n_perm=n_perm, rng=rng)
+        else:                                         # normal approximation
+            se = d.std(ddof=1) / np.sqrt(n)
+            p = 0.0 if (se == 0 and d.mean() < 0) else (
+                1.0 if se == 0 else float(stats.norm.cdf(d.mean() / se)))
+        results.append(ClusterResult(int(c), n, float(d.mean()), p,
+                                     float(p_to_e(np.array([p]), e_kappa)[0])))
+
+    pvals = np.array([r.p_value for r in results])
+    if mode == "sequential":
+        reject = e_bh(np.array([r.e_value for r in results]), alpha)
+    else:
+        reject = benjamini_hochberg(pvals, alpha)
+    for r, rej in zip(results, reject):
+        r.flagged_regression = bool(rej)
+    return results
+
+
+# --------------------------------------------------------------------------- #
+# 5. Top-level gate
+# --------------------------------------------------------------------------- #
+
+@dataclass
+class GateDecision:
+    verdict: str                              # SHIP | HOLD | REGRESSION | JUDGE_INADMISSIBLE
+    delta_ci: tuple[float, float]
+    components: VarianceComponents | None
+    flagged_clusters: list[int] = field(default_factory=list)
+    note: str = ""
+
+
+def gate(
+    scores_a: np.ndarray,
+    scores_b: np.ndarray,
+    cluster_ids: np.ndarray,
+    judge_error_sd: float = 0.0,
+    kappa: float = 1.0,
+    alpha: float = 0.05,
+    margin: float = 0.0,            # superiority margin on the delta
+    kappa_min: float = 0.4,
+    min_n: int = 30,                # below this, no SHIP/REGRESSION -- insufficient power
+    mode: str = "fixed",
+    rng: np.random.Generator | None = None,
+) -> GateDecision:
+    # (a) Judge must be admissible before it is allowed to gate anything.
+    if kappa < kappa_min:
+        return GateDecision("JUDGE_INADMISSIBLE", (float("nan"), float("nan")), None,
+                            note=f"kappa={kappa:.2f} < kappa_min={kappa_min}; recalibrate judge.")
+
+    # (b) Power floor, handled ASYMMETRICALLY. At small N the bootstrap
+    #     underestimates variance, so we never trust it to SHIP. But the danger
+    #     is one-sided: a consistent, large negative delta is a real catastrophe
+    #     even at small N, and the EXACT sign-flip permutation test (which does
+    #     not depend on the bootstrap) can certify it. A significant *improvement*
+    #     at small N is NOT shippable -- a tiny sample can't represent the input
+    #     distribution (an external-validity failure, not a power failure). So:
+    #     regression -> flag it; everything else -> HOLD. (Below ~N=5 the
+    #     permutation p floor of 1/2^N exceeds alpha, so this correctly falls
+    #     through to HOLD on its own.)
+    n_inputs = len(_as_ragged(scores_a))
+    if n_inputs < min_n:
+        comp = variance_components(scores_a, scores_b)
+        lo, hi, _ = bootstrap_delta_ci(scores_a, scores_b, judge_error_sd, alpha, rng=rng)
+        A, B = _as_ragged(scores_a), _as_ragged(scores_b)
+        d = np.array([b.mean() - a.mean() for a, b in zip(A, B)])
+        p_reg = paired_permutation_p(d, rng=rng)        # H1: mean(d) < 0
+        if d.mean() < 0 and p_reg < alpha:
+            return GateDecision("REGRESSION", (lo, hi), comp,
+                                note=f"small-N catastrophic regression caught by exact "
+                                     f"permutation test: N={n_inputs}, p={p_reg:.3f} < alpha={alpha}.")
+        return GateDecision("HOLD", (lo, hi), comp,
+                            note=f"insufficient power: N={n_inputs} < min_n={min_n}; "
+                                 f"permutation p={p_reg:.3f} is not a clear regression and a "
+                                 f"small sample cannot certify a ship -- collect more paired inputs.")
+
+    comp = variance_components(scores_a, scores_b)
+    lo, hi, _ = bootstrap_delta_ci(scores_a, scores_b, judge_error_sd, alpha, rng=rng)
+
+    if lo > margin:
+        verdict = "SHIP"
+    elif hi < 0:
+        verdict = "REGRESSION"
+    else:
+        verdict = "HOLD"
+
+    # (b) Localized regressions are reported even when the overall verdict ships.
+    clusters = cluster_scan(scores_a, scores_b, cluster_ids, alpha=2 * alpha, mode=mode, rng=rng)
+    flagged = [r.cluster_id for r in clusters if r.flagged_regression]
+    if flagged and verdict == "SHIP":
+        verdict = "SHIP_WITH_FLAGS"
+
+    return GateDecision(verdict, (lo, hi), comp, flagged,
+                        note=f"snr={comp.snr:.2f}, token-noise var={comp.sigma_resid2:.4f}")