shellde 0.2.0__py3-none-any.whl

shellde/__init__.py

@@ -0,0 +1,36 @@
+"""ShellDE: low-N, acquisition-driven mutation recommendation for directed evolution.
+
+The name states the scope. `shell` is the active-site contact shell that `resolve-site`
+computes, and the second-shell ring that `--exclude-catalytic` designs over. `DE` is
+directed evolution. Casing is fixed and does not vary: the distribution, the import name
+and the CLI command are always lowercase `shellde`, while prose and citations use
+`ShellDE`. Never `Shellde`, `shellDE` or `SHELLDE`.
+
+The shipped default is calibrated-additive. On `recommend`, --auto-gate-pairwise and
+--if-logprobs are CV-gated on your own held-out data (--no-auto-signals forces the latter
+on), while --contacts-pdb opens the contact-restricted pairwise block unconditionally.
+`campaign` and `round` offer neither pairwise flag and add --if-logprobs UNCONDITIONALLY,
+with no gate. --plm and --naturalness are never gated on any command."""
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _pkg_version
+
+try:
+    __version__ = _pkg_version("shellde")
+except PackageNotFoundError:  # not installed (e.g. run from a source tree without install)
+    __version__ = "0.0.0+unknown"
+
+from shellde.funclib import (
+    FuncLibLibrary,
+    combine_tolerances,
+    design_library,
+    tolerance_from_logprob_table,
+)
+
+__all__ = [
+    "FuncLibLibrary",
+    "combine_tolerances",
+    "design_library",
+    "tolerance_from_logprob_table",
+]

shellde/acquisition.py

@@ -0,0 +1,135 @@
+"""Acquisition layer: the experimental-design lever (the tool's organizing axis).
+
+Score-based acquisitions (Greedy / UCB / MaxVariance) rank candidates by a scalar;
+``select`` returns the top-k. DiverseBatch wraps a score acquisition and applies
+KURO greedy-maximin so a batch is both high-value and spread out (no near-duplicate
+wells). All satisfy ``protocols.Acquisition`` and compose freely:
+
+    DiverseBatch(UCB(beta=1.0))          # diverse, uncertainty-aware exploitation
+    DiverseBatch(MaxVariance())          # diverse pure exploration (resolve uncertainty)
+    Greedy()                             # pure exploitation (the additive-greedy baseline)
+
+UCB / MaxVariance are only meaningful because the surrogate std is CALIBRATED (P1).
+"""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Sequence
+
+import numpy as np
+
+from shellde.design_space import DesignSpace, to_assignment
+from shellde.types import Prediction
+
+
+class _ScoreAcquisition(ABC):
+    """Acquisitions that rank candidates by a per-candidate scalar score."""
+
+    @abstractmethod
+    def score(self, pred: Prediction) -> np.ndarray:
+        """Per-candidate acquisition value (higher = more desirable)."""
+
+    def select(
+        self,
+        pred: Prediction,
+        k: int,
+        *,
+        candidates: Sequence[str] | None = None,
+        space: DesignSpace | None = None,
+    ) -> list[int]:
+        s = self.score(pred)
+        if k <= 0 or s.size == 0:
+            return []
+        k = min(k, s.size)
+        return np.argsort(s)[::-1][:k].astype(int).tolist()
+
+
+class Greedy(_ScoreAcquisition):
+    """Pure exploitation: score = predicted mean (the additive-greedy lever)."""
+
+    def score(self, pred: Prediction) -> np.ndarray:
+        return np.asarray(pred.mean, dtype=float)
+
+
+class UCB(_ScoreAcquisition):
+    """Upper-confidence bound: score = mean + beta * calibrated std."""
+
+    def __init__(self, beta: float = 1.0) -> None:
+        self.beta = beta
+
+    def score(self, pred: Prediction) -> np.ndarray:
+        return np.asarray(pred.mean, dtype=float) + self.beta * np.asarray(pred.std, dtype=float)
+
+
+class MaxVariance(_ScoreAcquisition):
+    """Information-targeted (BALD-style) pure exploration: score = calibrated std.
+
+    Acquires where the model is most uncertain, i.e. where a measurement most reduces
+    predictive uncertainty. With calibrated std this is a principled exploration lever
+    (and the seed of resolving epistasis once interaction uncertainty is exposed).
+    """
+
+    def score(self, pred: Prediction) -> np.ndarray:
+        return np.asarray(pred.std, dtype=float)
+
+
+def _combo_vectors(candidates: Sequence[str], space: DesignSpace) -> list[tuple[str, ...]]:
+    """Per-position residue tuples for fast Hamming distance over the design space."""
+    out: list[tuple[str, ...]] = []
+    for v in candidates:
+        a = to_assignment(v, space)
+        out.append(tuple(a[p] for p in space.positions))
+    return out
+
+
+def _hamming(a: tuple[str, ...], b: tuple[str, ...]) -> int:
+    return sum(1 for x, y in zip(a, b, strict=True) if x != y)
+
+
+class DiverseBatch:
+    """KURO greedy-maximin diversity over a top-scored pool of a base acquisition.
+
+    Ranks all candidates by the base acquisition, keeps the top ``pool_factor * k``,
+    then greedily builds a batch that maximizes the minimum design-space (Hamming)
+    distance to the already-chosen set, seeded by the single best-scoring candidate.
+    The result is a high-value batch with no near-duplicate wells.
+    """
+
+    def __init__(self, base: _ScoreAcquisition, pool_factor: float = 4.0) -> None:
+        self.base = base
+        self.pool_factor = pool_factor
+
+    def select(
+        self,
+        pred: Prediction,
+        k: int,
+        *,
+        candidates: Sequence[str],
+        space: DesignSpace,
+    ) -> list[int]:
+        n = len(candidates)
+        if n == 0 or k <= 0:
+            return []
+        k = min(k, n)
+        scores = self.base.score(pred)
+        order = np.argsort(scores)[::-1]
+        pool_size = min(n, max(k, int(self.pool_factor * k)))
+        pool: list[int] = [int(j) for j in order[:pool_size]]
+        vecs = _combo_vectors([candidates[i] for i in pool], space)
+
+        chosen = [0]  # local index into pool; pool[0] is the highest base score
+        mind = [_hamming(vecs[j], vecs[0]) for j in range(len(pool))]
+        mind[0] = -1
+        while len(chosen) < k:
+            j = int(np.argmax(mind))  # max-min distance; ties -> lowest index = higher base score
+            if mind[j] < 0:  # pool exhausted of distinct points
+                break
+            chosen.append(j)
+            mind[j] = -1
+            vj = vecs[j]
+            for t in range(len(pool)):
+                if mind[t] >= 0:
+                    d = _hamming(vecs[t], vj)
+                    if d < mind[t]:
+                        mind[t] = d
+        return [pool[j] for j in chosen]

shellde/advisor.py

@@ -0,0 +1,158 @@
+"""Campaign readiness advisor: per-round signals turned into a next-step recommendation.
+
+In the N=95 (one plate per round) regime a PI needs to know, after each round, whether
+to keep gathering data (model not yet trustworthy), open the explicit-epistasis arm
+(enough data has accumulated), exploit and finish (learning has plateaued), or simply
+continue. This reads the metrics ``run_campaign`` already records (held-out Spearman
+trend, best-fitness trajectory, per-round active blocks) plus an on-demand pairwise-gate
+evidence check, and returns thresholded guidance. It forecasts nothing it cannot
+measure: "how many more plates" is intentionally NOT claimed (low-N extrapolation is
+unreliable); only the current state and a discrete recommendation are returned.
+"""
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import numpy as np
+from sklearn.model_selection import KFold
+
+from shellde.conformal import conformal_quantile
+from shellde.design_space import DesignSpace
+from shellde.features import FeatureMatrix, OneHotBlock
+from shellde.gating import pairwise_gate_evidence
+from shellde.protocols import Surrogate
+from shellde.surrogate import RidgeSurrogate
+from shellde.types import CampaignResult
+
+SPEARMAN_TRUST = 0.4
+PLATEAU_EPS = 0.03
+GATE_MIN_IMPROVE = 0.02
+
+
+def campaign_readiness(
+    result: CampaignResult,
+    *,
+    space: DesignSpace | None = None,
+    spearman_trust: float = SPEARMAN_TRUST,
+    plateau_eps: float = PLATEAU_EPS,
+    gate_min_improve: float = GATE_MIN_IMPROVE,
+) -> dict:
+    """Return a next-step recommendation plus the signals it is based on.
+
+    ``recommendation`` is one of: ``gather_more`` (predictions not yet reliable),
+    ``open_epistasis`` (the pairwise block now earns its place), ``exploit_and_finish``
+    (learning and best-fitness have plateaued), or ``continue`` (trustworthy and still
+    improving). Pass ``space`` to enable the epistasis-readiness check.
+    """
+    ho = list(result.heldout_spearman)
+    traj = list(result.trajectory)
+    latest = ho[-1] if ho else None
+    spear_delta = (ho[-1] - ho[-2]) if len(ho) >= 2 else None
+    best_gain = None
+    if len(traj) >= 2:
+        span = max(max(traj) - min(traj), 1e-9)
+        best_gain = (traj[-1] - traj[-2]) / span
+
+    trustworthy = latest is not None and latest >= spearman_trust
+    spear_plateau = spear_delta is not None and abs(spear_delta) <= plateau_eps
+    fitness_plateau = best_gain is not None and best_gain <= plateau_eps
+    plateaued = bool(spear_plateau and fitness_plateau)
+
+    last_blocks = result.rounds[-1].notes.get("active_blocks", []) if result.rounds else []
+    pairwise_open = "pairwise" in last_blocks
+    epistasis_ready = False
+    gate_improve: float | None = None
+    if space is not None and result.measured:
+        variants = list(result.measured)
+        fitness = np.asarray([result.measured[v] for v in variants], dtype=float)
+        gate_improve = float(pairwise_gate_evidence(space, variants, fitness)["improvement"])
+        epistasis_ready = gate_improve >= gate_min_improve
+
+    if not trustworthy:
+        rec, why = "gather_more", "held-out Spearman below trust threshold; predictions not yet reliable"
+    elif epistasis_ready and not pairwise_open:
+        rec, why = "open_epistasis", "enough data: the pairwise block now improves held-out prediction"
+    elif plateaued:
+        rec, why = "exploit_and_finish", "learning and best-fitness have plateaued; exploit the remaining budget"
+    else:
+        rec, why = "continue", "model is trustworthy and still improving; run another round"
+
+    return {
+        "recommendation": rec,
+        "why": why,
+        "latest_heldout_spearman": latest,
+        "spearman_delta": spear_delta,
+        "best_fitness_gain_rel": best_gain,
+        "model_trustworthy": bool(trustworthy),
+        "learning_plateaued": plateaued,
+        "epistasis_ready": bool(epistasis_ready),
+        "pairwise_open": bool(pairwise_open),
+        "gate_improvement": gate_improve,
+        "n_rounds": result.n_rounds,
+    }
+
+
+def _spearman(a: np.ndarray, b: np.ndarray) -> float:
+    ra = np.argsort(np.argsort(a)).astype(float)
+    rb = np.argsort(np.argsort(b)).astype(float)
+    if np.std(ra) < 1e-12 or np.std(rb) < 1e-12:
+        return 0.0
+    return float(np.corrcoef(ra, rb)[0, 1])
+
+
+def data_readiness(
+    space: DesignSpace,
+    variants: "list[str]",
+    fitness: np.ndarray,
+    *,
+    model_factory: "Callable[[], Surrogate]" = lambda: RidgeSurrogate(random_state=0),
+    spearman_trust: float = SPEARMAN_TRUST,
+    gate_min_improve: float = GATE_MIN_IMPROVE,
+    k: int = 4,
+    seed: int = 0,
+) -> dict:
+    """Readiness from a measured set alone (no simulation) for the real per-round workflow.
+
+    Computes k-fold held-out Spearman of an additive (one-hot) model on the accumulated
+    data plus the pairwise-gate evidence, and returns a discrete recommendation:
+    ``gather_more`` (predictions not reliable yet), ``open_epistasis`` (pairwise now
+    earns its place), or ``model_reliable`` (additive/PLM model is trustworthy, proceed).
+    """
+    y = np.asarray(fitness, dtype=float)
+    n = len(variants)
+    x = FeatureMatrix(space, [OneHotBlock(space)]).encode(variants)
+    heldout: float | None = None
+    conformal_halfwidth: float | None = None
+    if n >= 8 and float(np.std(y)) > 1e-12:
+        kf = KFold(n_splits=min(k, n), shuffle=True, random_state=seed)
+        pred = np.zeros(n)
+        for tr, te in kf.split(x):
+            if len(tr) < 2 or float(np.std(y[tr])) < 1e-12:
+                pred[te] = float(np.mean(y[tr])) if len(tr) else 0.0
+                continue
+            pred[te] = model_factory().fit(x[tr], y[tr]).predict(x[te]).mean
+        heldout = _spearman(pred, y)
+        # Pooled out-of-fold residuals = leave-one-fold-out cross-conformal scores;
+        # their conformal quantile is a distribution-free ~90% interval half-width.
+        conformal_halfwidth = conformal_quantile(y - pred)
+    gate_improve = float(pairwise_gate_evidence(space, variants, fitness, k=k, seed=seed)["improvement"])
+    epistasis_ready = gate_improve >= gate_min_improve
+    trustworthy = heldout is not None and heldout >= spearman_trust
+
+    if not trustworthy:
+        rec, why = "gather_more", "held-out Spearman below trust threshold; measure more before trusting recommendations"
+    elif epistasis_ready:
+        rec, why = "open_epistasis", "pairwise block now improves held-out prediction; enable --auto-gate-pairwise"
+    else:
+        rec, why = "model_reliable", "additive/PLM model is reliable on current data; proceed with recommend"
+
+    return {
+        "recommendation": rec,
+        "why": why,
+        "n_measured": n,
+        "heldout_spearman": heldout,
+        "conformal_halfwidth_90": conformal_halfwidth,
+        "model_trustworthy": bool(trustworthy),
+        "epistasis_ready": bool(epistasis_ready),
+        "gate_improvement": gate_improve,
+    }

shellde/bench/__init__.py

@@ -0,0 +1,17 @@
+"""Benchmark harness: tool-comparative campaigns + paired statistics."""
+from __future__ import annotations
+
+from shellde.bench.falsification import additive_null_control, epistatic_residuals
+from shellde.bench.harness import Method, default_methods, run_spectrum
+from shellde.bench.stats import benjamini_hochberg, bootstrap_p_value, paired_bootstrap_ci
+
+__all__ = [
+    "Method",
+    "additive_null_control",
+    "benjamini_hochberg",
+    "bootstrap_p_value",
+    "default_methods",
+    "epistatic_residuals",
+    "paired_bootstrap_ci",
+    "run_spectrum",
+]

shellde/bench/falsification.py

@@ -0,0 +1,95 @@
+"""Visani additive-null control: the mandatory floor before claiming epistasis learning.
+
+Per Visani, Verma & DeWitt 2026 (and rec 3 of the tool synthesis): a flexible model
+only has EVIDENCE of learning epistasis if its predictions DECORRELATE from a fixed
+additive null AND it beats that null out of sample. If a nonlinear model's held-out
+predictions correlate with the additive null at r > ~0.999, its extra capacity is
+unused and any "learned epistasis / synergy" claim is unsupported (the gain, if any,
+is a statistical artifact). This module makes that control a first-class bench output.
+"""
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+
+import numpy as np
+from sklearn.model_selection import KFold
+
+from shellde.design_space import DesignSpace
+from shellde.features import FeatureMatrix, OneHotBlock
+from shellde.protocols import Surrogate
+from shellde.surrogate import RFSurrogate, RidgeSurrogate
+
+R_THRESHOLD = 0.999
+
+
+def _pearson(a: np.ndarray, b: np.ndarray) -> float:
+    if np.std(a) < 1e-12 or np.std(b) < 1e-12:
+        return 1.0
+    return float(np.corrcoef(a, b)[0, 1])
+
+
+def _rmse(pred: np.ndarray, y: np.ndarray) -> float:
+    return float(np.sqrt(np.mean((pred - y) ** 2)))
+
+
+def additive_null_control(
+    space: DesignSpace,
+    variants: Sequence[str],
+    fitness: np.ndarray,
+    *,
+    model_factory: Callable[[], Surrogate] = lambda: RFSurrogate(random_state=0),
+    k: int = 5,
+    seed: int = 0,
+    r_threshold: float = R_THRESHOLD,
+) -> dict:
+    """k-fold held-out predictions of a flexible model vs a fixed additive (ridge) null.
+
+    Both use the SAME one-hot encoding so only model capacity differs. Returns the
+    Pearson correlation of the two prediction vectors, each held-out RMSE, and a
+    ``learned_epistasis`` verdict (decorrelated AND beats the null out of sample).
+    """
+    y = np.asarray(fitness, dtype=float)
+    mat = FeatureMatrix(space, [OneHotBlock(space)])
+    x = mat.encode(variants)
+    n = len(variants)
+    if n < 8 or float(np.std(y)) < 1e-12:
+        return {
+            "pearson_r": 1.0, "additive_rmse": float("nan"), "model_rmse": float("nan"),
+            "decorrelated": False, "beats_additive": False, "learned_epistasis": False,
+        }
+    add_pred = np.zeros(n)
+    model_pred = np.zeros(n)
+    kf = KFold(n_splits=min(k, n), shuffle=True, random_state=seed)
+    for tr, te in kf.split(x):
+        if len(tr) < 2 or float(np.std(y[tr])) < 1e-12:
+            add_pred[te] = float(np.mean(y[tr])) if len(tr) else 0.0
+            model_pred[te] = add_pred[te]
+            continue
+        add_pred[te] = RidgeSurrogate(random_state=seed).fit(x[tr], y[tr]).predict(x[te]).mean
+        model_pred[te] = model_factory().fit(x[tr], y[tr]).predict(x[te]).mean
+    r = _pearson(model_pred, add_pred)
+    a_rmse, m_rmse = _rmse(add_pred, y), _rmse(model_pred, y)
+    decorrelated = r < r_threshold
+    beats = m_rmse < a_rmse
+    return {
+        "pearson_r": r, "additive_rmse": a_rmse, "model_rmse": m_rmse,
+        "decorrelated": bool(decorrelated), "beats_additive": bool(beats),
+        "learned_epistasis": bool(decorrelated and beats),
+    }
+
+
+def epistatic_residuals(
+    space: DesignSpace, variants: Sequence[str], fitness: np.ndarray, *, seed: int = 0
+) -> dict:
+    """Residuals of a full-data additive (ridge) fit; their spread relative to y.
+
+    Under pure additivity the residuals carry no structure (small relative spread);
+    a large ``residual_fraction`` is necessary (not sufficient) for genuine epistasis.
+    """
+    y = np.asarray(fitness, dtype=float)
+    mat = FeatureMatrix(space, [OneHotBlock(space)])
+    x = mat.encode(variants)
+    add = RidgeSurrogate(random_state=seed).fit(x, y)
+    resid = y - add.predict(x).mean
+    ystd = float(np.std(y)) or 1.0
+    return {"residual_std": float(np.std(resid)), "residual_fraction": float(np.std(resid) / ystd)}

shellde/bench/harness.py

@@ -0,0 +1,134 @@
+"""Tool-comparative benchmark over a synthetic epistasis spectrum.
+
+Runs each method's active-learning campaign at IDENTICAL budget on a SyntheticOracle
+across an alpha spectrum and many seeds, scores normalised simple regret against the
+known optimum, and reports paired regret reductions of ``ours`` vs each baseline with
+bootstrap CIs and BH-FDR. Methods are (surrogate, acquisition) configs:
+
+  ours            BayesLinear + default_blocks (onehot) + UCB  (THE shipped additive default)
+  ours_pairwise   BayesLinear + [onehot+gated-pairwise] + UCB  (explicit opt-in arm, not default)
+  additive_greedy RidgeSurrogate    + Greedy              (the additive-greedy baseline)
+  rf_greedy       RFSurrogate       + Greedy              (EVOLVEpro-style RF greedy)
+  ucb_bo          EnsembleSurrogate + UCB                 (UCB Bayesian-optimisation style)
+
+`ours` is sourced from `features.default_blocks`, the SAME factory `cli.recommend` ships, so the
+head-to-head always measures what ships; explicit gated-pairwise is the separate `ours_pairwise` arm.
+"""
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+
+import numpy as np
+
+from shellde.acquisition import Greedy, UCB
+from shellde.bench.stats import benjamini_hochberg, bootstrap_p_value, paired_bootstrap_ci
+from shellde.design_space import DesignSpace
+from shellde.features import FeatureBlock, FeatureMatrix, GatedPairwiseBlock, OneHotBlock, default_blocks
+from shellde.loop import run_campaign
+from shellde.oracle import SyntheticOracle
+from shellde.protocols import Acquisition, Surrogate
+from shellde.surrogate import BayesLinearSurrogate, EnsembleSurrogate, RFSurrogate, RidgeSurrogate
+
+
+def _onehot_blocks(space: DesignSpace) -> list[FeatureBlock]:
+    return [OneHotBlock(space)]
+
+
+def _ours_blocks(space: DesignSpace) -> list[FeatureBlock]:
+    # ours == THE shipped additive default (single source of truth), so the head-to-head
+    # measures what `cli.recommend` actually ships, not an always-on pairwise overfit.
+    return default_blocks(space)
+
+
+def _ours_pairwise_blocks(space: DesignSpace) -> list[FeatureBlock]:
+    # explicit opt-in arm: additive one-hot + always-on gated explicit-pairwise epistasis
+    # (the OLD ours config, now a separate arm that has not earned default status).
+    return [OneHotBlock(space), GatedPairwiseBlock(space)]
+
+
+@dataclass
+class Method:
+    name: str
+    make_surrogate: Callable[[], Surrogate]
+    acquisition: Acquisition
+    make_blocks: Callable[[DesignSpace], list[FeatureBlock]] = _onehot_blocks
+
+
+def default_methods() -> list[Method]:
+    return [
+        Method("ours", lambda: BayesLinearSurrogate(random_state=0), UCB(1.0), _ours_blocks),
+        Method("ours_pairwise", lambda: BayesLinearSurrogate(random_state=0), UCB(1.0), _ours_pairwise_blocks),
+        Method("additive_greedy", lambda: RidgeSurrogate(random_state=0), Greedy()),
+        Method("rf_greedy", lambda: RFSurrogate(random_state=0), Greedy()),
+        Method("ucb_bo", lambda: EnsembleSurrogate(random_state=0), UCB(1.0)),
+    ]
+
+
+def run_spectrum(
+    space: DesignSpace,
+    alphas: Sequence[float],
+    seeds: Sequence[int],
+    methods: Sequence[Method] | None = None,
+    *,
+    n_init: int = 95,
+    batch_size: int = 95,
+    n_rounds: int = 3,
+) -> dict:
+    """Sweep alpha x seed x method; return per-alpha regret + ours-vs-baseline stats."""
+    methods = list(methods) if methods is not None else default_methods()
+    names = [m.name for m in methods]
+    if "ours" not in names:
+        raise ValueError("methods must include 'ours' for the head-to-head comparison")
+
+
+    per_alpha: dict[str, dict] = {}
+    all_pvals: list[float] = []
+    pval_index: list[tuple[float, str]] = []
+
+    for alpha in alphas:
+        regret: dict[str, list[float]] = {m.name: [] for m in methods}
+        ho_spear: dict[str, list[float]] = {m.name: [] for m in methods}
+        for seed in seeds:
+            oracle = SyntheticOracle(space, alpha=alpha, seed=seed)
+            universe = oracle.universe
+            assert universe is not None
+            f = oracle.evaluate(universe)
+            fmax, fmin = float(f.max()), float(f.min())
+            span = max(fmax - fmin, 1e-9)
+            for m in methods:
+                res = run_campaign(
+                    oracle, space,
+                    make_matrix=lambda mm=m: FeatureMatrix(space, mm.make_blocks(space)),
+                    make_surrogate=m.make_surrogate,
+                    acquisition=m.acquisition, n_init=n_init, batch_size=batch_size,
+                    n_rounds=n_rounds, seed=seed,
+                )
+                regret[m.name].append((fmax - res.best_fitness) / span)
+                if res.heldout_spearman:
+                    ho_spear[m.name].append(float(np.mean(res.heldout_spearman)))
+
+        ours = np.asarray(regret["ours"], dtype=float)
+        comparisons: dict[str, dict] = {}
+        for name in names:
+            if name == "ours":
+                continue
+            base = np.asarray(regret[name], dtype=float)
+            delta = base - ours  # positive => ours has lower regret (better)
+            mean, lo, hi = paired_bootstrap_ci(delta)
+            p = bootstrap_p_value(delta)
+            comparisons[name] = {"regret_reduction_mean": mean, "ci_lo": lo, "ci_hi": hi, "p": p}
+            all_pvals.append(p)
+            pval_index.append((alpha, name))
+
+        per_alpha[str(alpha)] = {
+            "mean_regret": {k: float(np.mean(v)) for k, v in regret.items()},
+            "heldout_spearman": {k: (float(np.mean(v)) if v else None) for k, v in ho_spear.items()},
+            "ours_vs": comparisons,
+        }
+
+    sig = benjamini_hochberg(all_pvals)
+    for (alpha, name), s in zip(pval_index, sig, strict=True):
+        per_alpha[str(alpha)]["ours_vs"][name]["fdr_significant"] = bool(s)
+
+    return {"methods": names, "alphas": list(alphas), "n_seeds": len(seeds), "per_alpha": per_alpha}

shellde/bench/stats.py

@@ -0,0 +1,46 @@
+"""Paired statistics for benchmark comparisons: bootstrap CI, bootstrap p, BH-FDR."""
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import numpy as np
+
+
+def benjamini_hochberg(pvals: Sequence[float], alpha: float = 0.05) -> np.ndarray:
+    """Return a boolean array: which hypotheses are rejected at FDR ``alpha``."""
+    p = np.asarray(pvals, dtype=float)
+    m = p.size
+    if m == 0:
+        return np.zeros(0, dtype=bool)
+    order = np.argsort(p)
+    thresh = 0.0
+    for rank, idx in enumerate(order, start=1):
+        if p[idx] <= rank / m * alpha:
+            thresh = rank / m * alpha
+    return p <= thresh
+
+
+def paired_bootstrap_ci(
+    deltas: Sequence[float] | np.ndarray, *, n_boot: int = 2000, seed: int = 0, ci: float = 0.95
+) -> tuple[float, float, float]:
+    """Mean paired delta + bootstrap CI (mean, lo, hi)."""
+    d = np.asarray(deltas, dtype=float)
+    n = d.size
+    if n == 0:
+        return 0.0, 0.0, 0.0
+    rng = np.random.default_rng(seed)
+    means = np.array([d[rng.integers(0, n, n)].mean() for _ in range(n_boot)])
+    lo, hi = np.percentile(means, [(1 - ci) / 2 * 100, (1 + ci) / 2 * 100])
+    return float(d.mean()), float(lo), float(hi)
+
+
+def bootstrap_p_value(deltas: Sequence[float] | np.ndarray, *, n_boot: int = 2000, seed: int = 0) -> float:
+    """Two-sided bootstrap p-value that the mean paired delta differs from zero."""
+    d = np.asarray(deltas, dtype=float)
+    n = d.size
+    if n == 0:
+        return 1.0
+    rng = np.random.default_rng(seed)
+    means = np.array([d[rng.integers(0, n, n)].mean() for _ in range(n_boot)])
+    p = 2.0 * min(float((means <= 0).mean()), float((means >= 0).mean()))
+    return float(min(1.0, p))