sys1bench 0.3.0__py3-none-any.whl

sys1bench/adapters/majority_prior.py

@@ -0,0 +1,53 @@
+"""Class-prior baseline. Priors are fitted from a manifest (or uniform if none given)."""
+
+from __future__ import annotations
+
+import time
+from collections import Counter
+
+from ..schemas import (
+    DecisionRequest,
+    DecisionResponse,
+    LatencyRecord,
+    ModelCapabilities,
+    ProviderRecord,
+    TaskItem,
+)
+from .base import BaseAdapter, finalize_answer, register
+
+
+@register("majority_prior")
+class MajorityPriorAdapter(BaseAdapter):
+    def __init__(self, model_id: str = "majority_prior", **tunables) -> None:
+        super().__init__(model_id, **tunables)
+        self.priors: dict[str, Counter] = {}
+
+    @property
+    def capabilities(self) -> ModelCapabilities:
+        return ModelCapabilities(name=self.model_id, deployment="local", supports_batching=True)
+
+    def fit(self, items: list[TaskItem]) -> None:
+        for it in items:
+            for k, q in it.questions.items():
+                if q.ground_truth is None:
+                    continue
+                gt = str(q.ground_truth).lower() if q.type == "noul" else str(q.ground_truth)
+                self.priors.setdefault(q.framing_group or k, Counter())[gt] += 1
+
+    def decide(self, request: DecisionRequest) -> DecisionResponse:
+        t0 = self._now_ms()
+        answers = {}
+        for k, q in request.questions.items():
+            keys = q.option_keys
+            counts = self.priors.get(q.framing_group or k)
+            if counts:
+                total = sum(counts.get(x, 0) for x in keys) + len(keys) * 1e-3
+                probs = [(counts.get(x, 0) + 1e-3) / total for x in keys]
+            else:
+                probs = [1.0 / len(keys)] * len(keys)
+            answers[k] = finalize_answer(q, probs)
+        return DecisionResponse(
+            answers=answers,
+            latency=LatencyRecord(client_ms=self._now_ms() - t0, timestamp=time.time()),
+            provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id, cost_usd=0.0),
+        )

sys1bench/adapters/mock.py

@@ -0,0 +1,77 @@
+"""Deterministic synthetic model for offline tests and CI.
+
+Behaviour is controlled so metric kernels can be validated end to end:
+`skill` sets P(argmax == truth); `temperature` scales logits (>1 under-confident,
+<1 over-confident); `position_bias` adds mass to index 0; `quantise` rounds probs.
+It reads `ground_truth` only through a hash of the state so it behaves like a real
+model that is right some fraction of the time, not like an oracle.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import random
+import time
+
+import numpy as np
+
+from ..schemas import DecisionRequest, DecisionResponse, LatencyRecord, ModelCapabilities, ProviderRecord
+from .base import BaseAdapter, finalize_answer, register
+
+
+@register("mock")
+class MockAdapter(BaseAdapter):
+    def __init__(self, model_id: str = "mock-v1", skill: float = 0.8, temperature: float = 1.0,
+                 position_bias: float = 0.0, quantise: float | None = None, framing_sensitivity: float = 0.0,
+                 latency_ms: float = 5.0, **tunables) -> None:
+        super().__init__(model_id, **tunables)
+        self.skill, self.temperature, self.position_bias = skill, temperature, position_bias
+        self.quantise, self.framing_sensitivity, self.latency_ms = quantise, framing_sensitivity, latency_ms
+
+    @property
+    def capabilities(self) -> ModelCapabilities:
+        return ModelCapabilities(name=self.model_id, deployment="local", max_options=255, emits_confidence=True,
+                                 supports_native_abstain=True, supports_batching=True)
+
+    def _rng(self, request: DecisionRequest, key: str) -> random.Random:
+        seed_src = f"{request.state_text()}|{key}|{self.model_id}"
+        if self.framing_sensitivity > 0:
+            seed_src += "|" + request.questions[key].instructions
+        return random.Random(int(hashlib.sha256(seed_src.encode()).hexdigest()[:16], 16))
+
+    def decide(self, request: DecisionRequest) -> DecisionResponse:
+        t0 = self._now_ms()
+        answers = {}
+        for key, q in request.questions.items():
+            rng = self._rng(request, key)
+            n = q.cardinality
+            keys = q.option_keys
+            truth = q.ground_truth
+            truth_key = None
+            if truth is not None:
+                truth_key = str(truth).lower() if q.type == "noul" else str(truth)
+            skill = self.skill
+            if self.framing_sensitivity > 0:
+                skill = max(0.0, min(1.0, skill + self.framing_sensitivity * (rng.random() - 0.5)))
+            if truth_key in keys and rng.random() < skill:
+                target = keys.index(truth_key)
+            else:
+                candidates = [i for i in range(n) if keys[i] != truth_key] or list(range(n))
+                target = rng.choice(candidates)
+            logits = np.array([rng.gauss(0, 1) for _ in range(n)])
+            logits[target] += 3.0
+            logits[0] += self.position_bias
+            logits = logits / self.temperature
+            probs = np.exp(logits - logits.max())
+            probs /= probs.sum()
+            if self.quantise:
+                probs = np.round(probs / self.quantise) * self.quantise
+                probs[int(probs.argmax())] += 1.0 - probs.sum()  # keep sum exactly 1 like a vendor would
+            answers[key] = finalize_answer(q, list(probs), confidence=float(probs.max()))
+        return DecisionResponse(
+            answers=answers,
+            latency=LatencyRecord(client_ms=self.latency_ms + (self._now_ms() - t0), compute_ms=self.latency_ms,
+                                  timestamp=time.time()),
+            provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id,
+                                    model_id_returned=self.model_id, version_hash="mock", cost_usd=0.0),
+        )

sys1bench/adapters/nli_zeroshot.py

@@ -0,0 +1,50 @@
+"""Zero-shot NLI baseline (BART-large-MNLI / DeBERTa-v3-large-MNLI). Hypothesis = label description."""
+
+from __future__ import annotations
+
+import time
+
+import numpy as np
+
+from ..schemas import DecisionRequest, DecisionResponse, LatencyRecord, ModelCapabilities, ProviderRecord
+from .base import BaseAdapter, finalize_answer, register
+
+
+@register("nli_zeroshot")
+class NLIZeroShotAdapter(BaseAdapter):
+    def __init__(self, model_id: str = "facebook/bart-large-mnli", device: int | str = -1,
+                 hypothesis_template: str = "This text is about {}.", **tunables) -> None:
+        super().__init__(model_id, **tunables)
+        self.device, self.template = device, hypothesis_template
+        self._pipe = None
+
+    @property
+    def capabilities(self) -> ModelCapabilities:
+        return ModelCapabilities(name=f"nli:{self.model_id}", deployment="local", supports_batching=True)
+
+    def _p(self):
+        if self._pipe is None:
+            from transformers import pipeline  # type: ignore
+
+            self._pipe = pipeline("zero-shot-classification", model=self.model_id, device=self.device)
+        return self._pipe
+
+    def decide(self, request: DecisionRequest) -> DecisionResponse:
+        t0 = time.perf_counter()
+        text = request.state_text()
+        answers = {}
+        for k, q in request.questions.items():
+            if q.type == "noul":
+                labels = [f"{q.instructions} yes", f"{q.instructions} no"]
+            elif q.type == "choice":
+                labels = [c.description or c.key for c in q.criteria]  # type: ignore[union-attr]
+            else:
+                labels = [c.description or f"level {c.level}" for c in q.criteria]  # type: ignore[union-attr]
+            out = self._p()(text, candidate_labels=labels, hypothesis_template=self.template, multi_label=False)
+            score = dict(zip(out["labels"], out["scores"]))
+            p = np.array([score[lab] for lab in labels])
+            p /= p.sum()
+            answers[k] = finalize_answer(q, [float(x) for x in p], confidence=float(p.max()))
+        ms = (time.perf_counter() - t0) * 1000
+        return DecisionResponse(answers=answers, latency=LatencyRecord(client_ms=ms, compute_ms=ms, timestamp=time.time()),
+                                provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id, cost_usd=0.0))

sys1bench/adapters/regex_keyword.py

@@ -0,0 +1,70 @@
+"""Hand-written rule baseline. Rules come from a YAML file per domain:
+
+    queue:
+      billing: ["refund", "charged", "invoice"]
+      shipping: ["deliver", "tracking", "package"]
+    is_angry: ["unacceptable", "furious", "ridiculous"]      # noul: any match -> true
+    priority: {"3": ["urgent", "asap"], "4": ["outage", "down"]}  # score: highest matching level wins
+
+Confidence is a fixed soft score (match -> 0.9 on the winner) so the baseline gets calibration
+metrics too, which shows what a rule system's probabilities are worth.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from pathlib import Path
+
+import yaml
+
+from ..schemas import DecisionRequest, DecisionResponse, LatencyRecord, ModelCapabilities, ProviderRecord
+from .base import BaseAdapter, finalize_answer, register
+
+
+@register("regex_keyword")
+class RegexKeywordAdapter(BaseAdapter):
+    def __init__(self, model_id: str = "regex_keyword", rules_path: str | None = None, rules: dict | None = None,
+                 hit_confidence: float = 0.9, **tunables) -> None:
+        super().__init__(model_id, **tunables)
+        if rules is None and rules_path:
+            rules = yaml.safe_load(Path(rules_path).read_text())
+        self.rules = rules or {}
+        self.hit_confidence = hit_confidence
+        self._compiled: dict[str, dict[str, list[re.Pattern]]] = {}
+        for qkey, spec in self.rules.items():
+            if isinstance(spec, list):
+                self._compiled[qkey] = {"true": [re.compile(p, re.IGNORECASE) for p in spec]}
+            else:
+                self._compiled[qkey] = {str(k): [re.compile(p, re.IGNORECASE) for p in v] for k, v in spec.items()}
+
+    @property
+    def capabilities(self) -> ModelCapabilities:
+        return ModelCapabilities(name=self.model_id, deployment="local", emits_confidence=True, supports_batching=True)
+
+    def decide(self, request: DecisionRequest) -> DecisionResponse:
+        t0 = self._now_ms()
+        text = request.state_text()
+        answers = {}
+        for k, q in request.questions.items():
+            keys = q.option_keys
+            rules = self._compiled.get(q.framing_group or k) or self._compiled.get(k) or {}
+            hits = {opt: sum(1 for p in pats if p.search(text)) for opt, pats in rules.items()}
+            hits = {o: h for o, h in hits.items() if h > 0 and (o in keys or q.type == "noul")}
+            n = len(keys)
+            if not hits:
+                probs = [1.0 / n] * n
+            elif q.type == "noul":
+                probs = [self.hit_confidence, 1.0 - self.hit_confidence]
+            elif q.type == "score":
+                winner = str(max(int(o) for o in hits))
+                probs = [(self.hit_confidence if o == winner else (1 - self.hit_confidence) / (n - 1)) for o in keys]
+            else:
+                winner = max(hits, key=hits.get)
+                probs = [(self.hit_confidence if o == winner else (1 - self.hit_confidence) / (n - 1)) for o in keys]
+            answers[k] = finalize_answer(q, probs, confidence=max(probs))
+        return DecisionResponse(
+            answers=answers,
+            latency=LatencyRecord(client_ms=self._now_ms() - t0, timestamp=time.time()),
+            provider=ProviderRecord(adapter_id=self.adapter_id, model_id_requested=self.model_id, cost_usd=0.0),
+        )

sys1bench/analysis/__init__.py

@@ -0,0 +1,4 @@
+from .arms import arm_accuracy, label_noise_report, unknowable_report  # noqa: F401
+from .decomposition import decomposition_report  # noqa: F401
+from .meta_eval import label_order_report, leakage_report, short_circuit_report  # noqa: F401
+from .stats import cluster_bootstrap, holm, mcnemar, paired_bootstrap  # noqa: F401

sys1bench/analysis/arms.py

@@ -0,0 +1,46 @@
+"""Scoring for the control arms: unknowable (calibration ceiling), label-noise, corruption, short-circuit."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from ..schemas import PredictionRow, TaskItem
+
+
+def unknowable_report(rows: list[PredictionRow], question_key: str = "priority") -> dict:
+    """On items whose label depends on removed information, a calibrated model should spread mass. For the ticket
+    priority arm the tier is removed, so the truth is one of two adjacent levels: report mass on the two candidates,
+    max-probability, and how often confidence exceeds 0.7."""
+    rs = [r for r in rows if r.question_key == question_key and not r.error and r.controls.unknowable]
+    if not rs:
+        return {"n": 0}
+    conf = np.array([max(r.probs) for r in rs])
+    return {"n": len(rs), "mean_max_prob": float(conf.mean()), "frac_conf_over_0.7": float((conf > 0.7).mean()),
+            "frac_conf_over_0.9": float((conf > 0.9).mean()),
+            "accuracy_vs_hidden_truth": float(np.mean([r.correct for r in rs if r.correct is not None])),
+            "mean_entropy_bits": float(np.mean([-(np.array(r.probs) + 1e-12).dot(np.log2(np.array(r.probs) + 1e-12)) for r in rs]))}
+
+
+def label_noise_report(rows: list[PredictionRow], items: list["TaskItem"]) -> dict:
+    """Manifest with a corrupted-label fraction: accuracy on corrupted rows should be near zero (the label is wrong, the
+    model should still answer the true thing) and confidence on them should not exceed confidence on clean rows.
+    Joined on task_id against the manifest so it works for rows produced by any runner version."""
+    noisy_ids = {it.task_id for it in items if it.label_provenance.noise_injected}
+    clean = [r for r in rows if not r.error and r.task_id not in noisy_ids]
+    noisy = [r for r in rows if not r.error and r.task_id in noisy_ids]
+
+    def f(rs):
+        acc = float(np.mean([r.correct for r in rs if r.correct is not None])) if rs else float("nan")
+        conf = float(np.mean([max(r.probs) for r in rs])) if rs else float("nan")
+        return acc, conf
+
+    ac, cc = f(clean)
+    an, cn = f(noisy)
+    return {"n_clean": len(clean), "n_noisy": len(noisy), "accuracy_clean": ac, "accuracy_on_corrupted_labels": an,
+            "confidence_clean": cc, "confidence_on_corrupted": cn,
+            "flag_more_confident_on_corrupted": bool(cn > cc + 0.02) if noisy else False}
+
+
+def arm_accuracy(rows: list[PredictionRow], arm: str, question_key: str | None = None) -> float:
+    rs = [r for r in rows if r.arm == arm and r.correct is not None and (question_key is None or r.question_key == question_key)]
+    return float(np.mean([r.correct for r in rs])) if rs else float("nan")

sys1bench/analysis/decision_value.py

@@ -0,0 +1,43 @@
+"""Suite I: downstream decision value from the cost matrices shipped in every Tier G manifest."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+
+from ..metrics.decision_value import expected_cost, value_of_calibration
+from ..schemas import PredictionRow, TaskItem
+
+
+def decision_value_report(rows: list[PredictionRow], items: list[TaskItem], escalate_cost: float | None = None,
+                          thresholds: tuple[float, ...] = (0.5, 0.7, 0.9)) -> dict[str, Any]:
+    """Per question with a cost matrix: mean realised cost per 10k decisions under argmax, Bayes (uses probabilities)
+    and act-or-escalate at each threshold, plus regret vs oracle and the value of calibration (argmax minus Bayes)."""
+    by_task = {it.task_id: it for it in items}
+    out: dict[str, Any] = {}
+    keys = sorted({r.question_key for r in rows})
+    for k in keys:
+        cm = next((it.cost_matrix.get(k) for it in items if it.cost_matrix and k in it.cost_matrix), None)
+        if not cm:
+            continue
+        rs = [r for r in rows if r.question_key == k and not r.error and r.ground_truth in r.option_keys and r.task_id in by_task
+              and r.permutation_id == "p0" and r.framing_id == "f0"]
+        if len(rs) < 20:
+            continue
+        labels = rs[0].option_keys
+        C = np.array([[float(cm.get(str(t), {}).get(str(p), 1.0 if t != p else 0.0)) for p in labels] for t in labels])
+        P = np.array([r.probs for r in rs])
+        y = np.array([labels.index(r.ground_truth) for r in rs])
+        esc = escalate_cost if escalate_cost is not None else float(np.median(C[C > 0])) if (C > 0).any() else 1.0
+        res = {"n": len(rs), "labels": labels, "escalate_cost": esc,
+               "argmax": expected_cost(P, y, C, "argmax"), "bayes": expected_cost(P, y, C, "bayes"),
+               "value_of_calibration_per_10k": value_of_calibration(P, y, C) * 10_000}
+        for t in thresholds:
+            res[f"escalate@{t}"] = expected_cost(P, y, C, "threshold", escalate_cost=esc, threshold=t)
+        # trivial policies for reference
+        prior = np.bincount(y, minlength=len(labels)) / len(y)
+        Pprior = np.tile(prior, (len(y), 1))
+        res["majority_prior_bayes"] = expected_cost(Pprior, y, C, "bayes")
+        out[k] = res
+    return out

sys1bench/analysis/decomposition.py

@@ -0,0 +1,47 @@
+"""Suite B decomposition analysis: holistic question vs sub-questions combined by a fixed rule vs fitted weights.
+
+Rows must contain, per task_id, one holistic noul row (question_key == target) and one row per sub-question
+(metadata.decomposition_of == target). Weights are fitted on the first half of task_ids and evaluated on the second.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+from ..framing.expand import apply_decomposition_weights, decompose_fixed_rule, fit_decomposition_weights
+from ..metrics.calibration import brier, ece
+from ..schemas import PredictionRow
+
+
+def decomposition_report(rows: list[PredictionRow], target: str, rule: str = "any") -> dict:
+    hol = {r.task_id: r for r in rows if r.question_key == target and not r.error and r.framing_id == "f0" and r.permutation_id == "p0"}
+    subs: dict[str, dict[str, PredictionRow]] = {}
+    for r in rows:
+        if r.metadata.get("decomposition_of") == target and not r.error and r.framing_id == "f0" and r.permutation_id == "p0":
+            subs.setdefault(r.task_id, {})[r.question_key] = r
+    tids = sorted(t for t in hol if t in subs)
+    if len(tids) < 20:
+        return {"n": len(tids), "note": "too few paired items"}
+    sub_keys = sorted({k for t in tids for k in subs[t]})
+    X = np.array([[subs[t][k].probs[0] if k in subs[t] else 0.5 for k in sub_keys] for t in tids])
+    y = np.array([1.0 if hol[t].ground_truth == "true" else 0.0 for t in tids])
+    p_hol = np.array([hol[t].probs[0] for t in tids])
+    p_rule = np.array([decompose_fixed_rule(dict(zip(sub_keys, x)), rule) for x in X])
+    half = len(tids) // 2
+    w = fit_decomposition_weights(X[:half], y[:half])
+    p_fit = apply_decomposition_weights(X[half:], w)
+
+    def summarise(p: np.ndarray, yy: np.ndarray) -> dict:
+        pred = p >= 0.5
+        P2 = np.stack([p, 1 - p], 1)
+        yi = (1 - yy).astype(int)  # index 0 == "true"
+        return {"accuracy": float((pred == yy.astype(bool)).mean()), "brier": brier(P2, yi),
+                "ece15": ece(P2.max(1), (P2.argmax(1) == yi).astype(float))}
+
+    out = {"n": len(tids), "sub_questions": sub_keys, "rule": rule,
+           "holistic": summarise(p_hol, y), "fixed_rule": summarise(p_rule, y),
+           "holistic_eval_half": summarise(p_hol[half:], y[half:]), "fitted_weights_eval_half": summarise(p_fit, y[half:]),
+           "weights": dict(zip(sub_keys + ["bias"], [float(v) for v in w]))}
+    out["decomposition_gain_fixed"] = out["fixed_rule"]["accuracy"] - out["holistic"]["accuracy"]
+    out["decomposition_gain_fitted"] = out["fitted_weights_eval_half"]["accuracy"] - out["holistic_eval_half"]["accuracy"]
+    return out

sys1bench/analysis/meta_eval.py

@@ -0,0 +1,67 @@
+"""Benchmark audits: short-circuit, leakage, label-order bias, label-noise control."""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+
+import numpy as np
+
+from ..metrics.consistency import positional_bias_chi2
+from ..schemas import PredictionRow, TaskItem
+
+
+def _acc(rows: list[PredictionRow]) -> float:
+    c = [r.correct for r in rows if r.correct is not None]
+    return float(np.mean(c)) if c else float("nan")
+
+
+def short_circuit_report(full: list[PredictionRow], state_only: list[PredictionRow], options_only: list[PredictionRow],
+                         prior_acc: float, margin: float = 0.10) -> dict:
+    a_full, a_state, a_opts = _acc(full), _acc(state_only), _acc(options_only)
+    return {"accuracy_full": a_full, "accuracy_state_only": a_state, "accuracy_options_only": a_opts,
+            "accuracy_prior": prior_acc,
+            "flag_state_only": bool(a_state > prior_acc + margin), "flag_options_only": bool(a_opts > prior_acc + margin)}
+
+
+def _ngrams(text: str, n: int) -> set[tuple[str, ...]]:
+    toks = re.findall(r"[a-z0-9]+", text.lower())
+    return {tuple(toks[i:i + n]) for i in range(len(toks) - n + 1)}
+
+
+def leakage_report(items: list[TaskItem], n: int = 1) -> dict:
+    """Fraction of items where the true option's key or description n-grams appear in the state."""
+    hits, total, per_q = 0, 0, Counter()
+    for it in items:
+        state = it.state_text()
+        sg = _ngrams(state, n)
+        for k, q in it.questions.items():
+            if q.type != "choice" or q.ground_truth is None:
+                continue
+            total += 1
+            opt = next((o for o in q.criteria if o.key == q.ground_truth), None)  # type: ignore[union-attr]
+            if opt is None:
+                continue
+            key_words = set(re.findall(r"[a-z0-9]+", opt.key.lower()))
+            leak = bool(sg & {(w,) for w in key_words}) if n == 1 else bool(sg & _ngrams(opt.description, n))
+            if leak:
+                hits += 1
+                per_q[k] += 1
+    return {"n_choice_questions": total, "leak_rate": hits / total if total else float("nan"), "per_question": dict(per_q)}
+
+
+def label_order_report(rows: list[PredictionRow]) -> dict:
+    """Over permuted rows, test whether argmax index prefers a position."""
+    by_k: dict[int, list[int]] = {}
+    for r in rows:
+        if r.primitive != "choice" or r.error or not r.argmax:
+            continue
+        by_k.setdefault(r.cardinality, []).append(r.option_keys.index(r.argmax))
+    return {k: positional_bias_chi2(np.array(v), k) for k, v in by_k.items()}
+
+
+def label_noise_control(rows_clean: list[PredictionRow], rows_noisy: list[PredictionRow]) -> dict:
+    """On corrupted labels a model should be ~wrong at the corruption rate and NOT more confident."""
+    conf = lambda rs: float(np.mean([max(r.probs) for r in rs if not r.error])) if rs else float("nan")
+    return {"accuracy_clean": _acc(rows_clean), "accuracy_noisy_labels": _acc(rows_noisy),
+            "confidence_clean": conf(rows_clean), "confidence_noisy_labels": conf(rows_noisy)}

sys1bench/analysis/stats.py

@@ -0,0 +1,76 @@
+"""Item-level statistics. The item (or framing group) is the unit; seeds are not."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import numpy as np
+from scipy.stats import binomtest
+
+
+def paired_bootstrap(a: np.ndarray, b: np.ndarray, stat: Callable[[np.ndarray], float] = np.mean,
+                     n_boot: int = 10_000, seed: int = 0, ci: float = 0.95) -> dict[str, float]:
+    """Bootstrap CI for stat(a) - stat(b) over paired items."""
+    a, b = np.asarray(a, float), np.asarray(b, float)
+    assert a.shape == b.shape
+    rng = np.random.default_rng(seed)
+    n = len(a)
+    idx = rng.integers(0, n, size=(n_boot, n))
+    diffs = np.array([stat(a[i]) - stat(b[i]) for i in idx])
+    lo, hi = np.quantile(diffs, [(1 - ci) / 2, 1 - (1 - ci) / 2])
+    point = float(stat(a) - stat(b))
+    return {"diff": point, "ci_low": float(lo), "ci_high": float(hi),
+            "frac_sign_holds": float((np.sign(diffs) == np.sign(point)).mean()) if point != 0 else 0.5,
+            "n": int(n)}
+
+
+def bootstrap_ci(x: np.ndarray, stat: Callable[[np.ndarray], float] = np.mean, n_boot: int = 10_000, seed: int = 0,
+                 ci: float = 0.95) -> tuple[float, float, float]:
+    x = np.asarray(x, float)
+    rng = np.random.default_rng(seed)
+    vals = np.array([stat(x[rng.integers(0, len(x), len(x))]) for _ in range(n_boot)])
+    lo, hi = np.quantile(vals, [(1 - ci) / 2, 1 - (1 - ci) / 2])
+    return float(stat(x)), float(lo), float(hi)
+
+
+def mcnemar(correct_a: np.ndarray, correct_b: np.ndarray) -> dict[str, float]:
+    a, b = np.asarray(correct_a, bool), np.asarray(correct_b, bool)
+    n01 = int((a & ~b).sum())
+    n10 = int((~a & b).sum())
+    if n01 + n10 == 0:
+        return {"n_a_only": n01, "n_b_only": n10, "p": 1.0}
+    p = binomtest(n01, n01 + n10, 0.5).pvalue
+    return {"n_a_only": n01, "n_b_only": n10, "p": float(p)}
+
+
+def cluster_bootstrap(values: np.ndarray, clusters: np.ndarray, stat: Callable[[np.ndarray], float] = np.mean,
+                      n_boot: int = 5_000, seed: int = 0) -> tuple[float, float, float]:
+    """Resample clusters (e.g. task_id when paraphrases are pooled) rather than rows."""
+    values, clusters = np.asarray(values, float), np.asarray(clusters)
+    uniq = np.unique(clusters)
+    groups = [values[clusters == c] for c in uniq]
+    rng = np.random.default_rng(seed)
+    vals = []
+    for _ in range(n_boot):
+        pick = rng.integers(0, len(groups), len(groups))
+        vals.append(stat(np.concatenate([groups[i] for i in pick])))
+    lo, hi = np.quantile(vals, [0.025, 0.975])
+    return float(stat(values)), float(lo), float(hi)
+
+
+def holm(pvals: dict[str, float], alpha: float = 0.05) -> dict[str, bool]:
+    items = sorted(pvals.items(), key=lambda kv: kv[1])
+    m = len(items)
+    out, stop = {}, False
+    for i, (k, p) in enumerate(items):
+        if not stop and p <= alpha / (m - i):
+            out[k] = True
+        else:
+            stop = True
+            out[k] = False
+    return out
+
+
+def power_note(n: int) -> str:
+    half_width = 1.96 * np.sqrt(0.25 / n) * 100
+    return f"n={n}: accuracy 95% CI half-width about ±{half_width:.1f} points"