peeksafe 0.0.0-stage → 0.1.0

package/dist/gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gate.d.ts","sourceRoot":"","sources":["../src/gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAQlD,OAAO,EAEL,KAAK,QAAQ,EAId,MAAM,eAAe,CAAC;AACvB,OAAO,EAIL,KAAK,YAAY,EAElB,MAAM,YAAY,CAAC;AAEpB;;;GAGG;AACH,eAAO,MAAM,YAAY,aACb;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,OAC1C,MAAM,KACV,OAAsD,CAAC;AAE1D,4EAA4E;AAC5E,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC;IACf,gFAAgF;IAChF,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,MAAM,CAAC,EAAE,SAAS,CAAC;CACpB;AAED,8EAA8E;AAC9E,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,qDAAqD;IACrD,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,iFAAiF;IACjF,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED,eAAO,MAAM,oBAAoB;aAC/B,GAAG,EAAE,IAAI;aACT,GAAG,EAAE,IAAI;aACT,gBAAgB,EAAE,CAAC;aACnB,QAAQ,EAAE,OAAO;CACT,CAAC;AAEX,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,0EAA0E;IAC1E,SAAS,EAAE,OAAO,CAAC;IACnB,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC;IACf,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;IACb;;;;;OAKG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,YAAY,EAAE,OAAO,CAAC;IACtB;;;;OAIG;IACH,UAAU,EAAE,OAAO,CAAC;IACpB,qEAAqE;IACrE,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,MAAM,EAAE,UAAU,GAAG,QAAQ,CAAC;IAC9B,yCAAyC;IACzC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAED,MAAM,WAAW,UAAU;IACzB,oDAAoD;IACpD,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,wDAAwD;IACxD,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB;;;OAGG;IACH,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,4EAA4E;IAC5E,SAAS,EAAE,MAAM,CAAC;IAClB,4EAA4E;IAC5E,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B;;;;;;;OAOG;IACH,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,IAAI,CAClB,KAAK,EAAE,SAAS,CAAC,QAAQ,GAAG,cAAc,CAAC,EAAE,EAC7C,OAAO,GAAE,WAAgB,GACxB,UAAU,CAiKZ"}

package/dist/gate.js

@@ -0,0 +1,196 @@
+/**
+ * gate.ts: the decision.
+ *
+ * You bring counts. This decides which cases regressed, at a false discovery
+ * rate you name, in a way that stays valid however many times you looked at the
+ * data on the way here.
+ *
+ * ## Why this is safe to peek at
+ *
+ * The usual construction is: run the suite, stop a case when it looks
+ * significant, compute a p-value from the counts, hand the p-values to
+ * Benjamini-Hochberg. That is invalid in two independent places.
+ *
+ * A p-value computed at a stopping boundary is not a p-value. `P(p_n <= a) <= a`
+ * holds for each fixed `n`; a procedure that adds runs and re-tests until the
+ * result looks significant reports `min` over the peeks, and that minimum is not
+ * bounded by `a`. Benjamini-Hochberg takes `P(p <= t) <= t` as its input
+ * assumption, so it inherits nothing and controls nothing.
+ *
+ * This module never computes a p-value. It computes an e-value, which is a
+ * non-negative statistic with mean at most 1 under the null. Ville's inequality
+ * bounds `P(sup_n E_n >= 1/a) <= a` over the whole trajectory, so an e-value is
+ * valid at any stopping time, including one chosen after looking. (The mean is
+ * exactly 1 averaged over the shared rate, not at every fixed rate; the README
+ * section "What that guarantee rests on" says where the bound is checked.) The
+ * multiplicity correction is e-BH (Wang and Ramdas), which takes e-values
+ * directly and controls FDR under arbitrary dependence between cases.
+ *
+ * The practical consequence: you may stop a case whenever you like, for any
+ * reason, including "it already looks bad". Nothing here cares when you stopped.
+ *
+ * ## What it will refuse to do
+ *
+ * Gate a case with no baseline. There is no null to test against, and the
+ * failure mode of inventing one is not a wrong number, it is a confident wrong
+ * number: a missing baseline read as `0/0` has a Beta(1,1) posterior median of
+ * 0.5, which gates the case against "it passes half the time" and reports a
+ * large improvement against nothing at all. Cases without a baseline come back
+ * in `newCases` and take no part in the verdict or the e-BH family.
+ *
+ * ## Paired cases
+ *
+ * A case may instead carry `paired` counts from re-running the baseline next to
+ * the candidate, one baseline run per candidate run. Its e-value is
+ * `pairedLogE` on the discordant pairs, which is valid at every rate by
+ * construction and has no ceiling, whichever `evidence` the unpaired cases use.
+ * The pairs need not share a seed: independent runs paired in order are still
+ * valid, and a shared seed only makes discordances more informative.
+ */
+import { PeeksafeError, requireAltConcentration, requireCounts, requireOpenProbability, requirePairedCounts, } from "./errors.js";
+import { ceilingLogEvidence, logEvidence, requireEvidence, toEvalue, } from "./evidence.js";
+import { discordant, ebhCorrect, ebhSoloThreshold, pairedLogE, } from "./stats.js";
+/**
+ * True when the baseline's observed rate leaves no room for an `mde`-sized drop.
+ * `makePlan` calls the same cases IMPOSSIBLE, by the same rule.
+ */
+export const cannotDropBy = (baseline, mde) => baseline.successes / baseline.trials <= mde;
+export const DEFAULT_GATE_OPTIONS = {
+    mde: 0.15,
+    fdr: 0.05,
+    altConcentration: 8,
+    evidence: "bayes",
+};
+/**
+ * Decide which cases regressed.
+ *
+ * Valid at any stopping time. Run every case to a fixed count, or stop each one
+ * the moment it looks decided, or anything in between; the guarantee does not
+ * depend on the rule you used.
+ */
+export function gate(cases, options = {}) {
+    const opts = { ...DEFAULT_GATE_OPTIONS, ...options };
+    requireOpenProbability(opts.mde, "mde", "gate");
+    requireOpenProbability(opts.fdr, "fdr", "gate");
+    requireEvidence(opts.evidence, "gate");
+    requireAltConcentration(opts.altConcentration, opts.evidence, "gate");
+    const seen = new Set();
+    for (const c of cases) {
+        if (seen.has(c.id)) {
+            throw new PeeksafeError("PEEKSAFE_E_CASE_DUPLICATE", `gate: duplicate case id ${c.id}`, {
+                detail: { id: c.id },
+                hint: "two cases sharing an id would be merged into one baseline entry and counted once",
+            });
+        }
+        seen.add(c.id);
+        if (c.paired) {
+            requirePairedCounts(c.paired, `gate(${c.id}).paired`);
+            continue;
+        }
+        requireCounts(c.successes, c.trials, `gate(${c.id})`);
+        if (c.baseline)
+            requireCounts(c.baseline.successes, c.baseline.trials, `gate(${c.id}).baseline`);
+    }
+    const hasBaseline = (c) => c.paired !== undefined || (c.baseline !== undefined && c.baseline.trials > 0);
+    const newCases = cases.filter((c) => !hasBaseline(c)).map((c) => c.id);
+    const gated = cases.filter(hasBaseline);
+    if (gated.length === 0) {
+        throw new PeeksafeError("PEEKSAFE_E_BASELINE_MISSING", "gate: no case has a baseline, so there is nothing to test against", {
+            detail: { cases: cases.length, newCases: newCases.length },
+            hint: 'record a baseline on the reference revision first; gating without one compares against a Beta(1,1) prior, which reads as "this case passes half the time"',
+        });
+    }
+    const m = gated.length;
+    // Testing these anyway clamps the alternative onto a rate near zero, which is
+    // where the case already sits, and the false positive rate then exceeds `fdr`.
+    // A paired case has an exact null and never needs this.
+    const impossible = gated.map((c) => !c.paired && cannotDropBy(c.baseline, opts.mde));
+    const logEs = gated.map((c, i) => {
+        if (c.paired)
+            return pairedLogE(c.paired.worse, discordant(c.paired));
+        return impossible[i]
+            ? 0
+            : logEvidence(opts.evidence, c.successes, c.trials, c.baseline.successes, c.baseline.trials, opts.mde, opts.altConcentration);
+    });
+    const evalues = logEs.map(toEvalue);
+    const ebh = ebhCorrect(evalues, opts.fdr);
+    const solo = ebhSoloThreshold(m, opts.fdr);
+    const verdicts = gated.map((c, i) => {
+        const shared = { id: c.id, regressed: ebh.rejected[i], evalue: evalues[i], logE: logEs[i] };
+        if (c.paired) {
+            const p = c.paired;
+            const pairs = p.bothPass + p.worse + p.better + p.bothFail;
+            return {
+                ...shared,
+                ceiling: Infinity,
+                undetectable: false,
+                impossible: false,
+                observed: { successes: p.bothPass + p.better, trials: pairs },
+                baseline: { successes: p.bothPass + p.worse, trials: pairs },
+                design: "paired",
+                paired: { ...p },
+            };
+        }
+        // The ceiling is evaluated at the rate a regression would actually sit at,
+        // one MDE below the baseline's own rate. That is the alternative the gate is
+        // trying to detect, so it is the right place to ask "could this ever clear
+        // the bar". Matches how plan.ts prices the same case.
+        const bRate = c.baseline.successes / c.baseline.trials;
+        const pAlt = Math.max(1e-6, Math.min(1 - 1e-6, bRate - opts.mde));
+        const ceiling = impossible[i]
+            ? 1
+            : Math.exp(ceilingLogEvidence(opts.evidence, pAlt, c.baseline.successes, c.baseline.trials, opts.mde, opts.altConcentration));
+        return {
+            ...shared,
+            ceiling,
+            undetectable: !impossible[i] && ceiling < solo,
+            impossible: impossible[i],
+            observed: { successes: c.successes, trials: c.trials },
+            baseline: { successes: c.baseline.successes, trials: c.baseline.trials },
+            design: "unpaired",
+        };
+    });
+    const regressed = verdicts.filter((v) => v.regressed);
+    const blind = verdicts.filter((v) => v.undetectable && !v.regressed);
+    const untested = verdicts.filter((v) => v.impossible);
+    const caveats = [];
+    if (blind.length > 0) {
+        caveats.push(`${blind.length} of ${m} gated case(s) have a baseline too thin to certify a ` +
+            `${(opts.mde * 100).toFixed(0)}pt drop at any candidate budget (${blind
+                .slice(0, 3)
+                .map((v) => v.id)
+                .join(", ")}` +
+            `${blind.length > 3 ? ", …" : ""}), which needs more baseline runs, not more candidate runs`);
+    }
+    if (untested.length > 0) {
+        caveats.push(`${untested.length} case(s) pass at or below ${(opts.mde * 100).toFixed(0)}% and cannot drop by ` +
+            `${(opts.mde * 100).toFixed(0)}pts, so they were not tested (${untested
+                .slice(0, 3)
+                .map((v) => v.id)
+                .join(", ")}` +
+            `${untested.length > 3 ? ", …" : ""})`);
+    }
+    if (newCases.length > 0) {
+        caveats.push(`${newCases.length} case(s) have no baseline and were not gated ` +
+            `(${newCases.slice(0, 3).join(", ")}${newCases.length > 3 ? ", …" : ""})`);
+    }
+    const headline = (regressed.length === 0
+        ? `PASS: no case cleared the bar of ${solo.toFixed(0)} at ${(opts.fdr * 100).toFixed(0)}% FDR over ${m} gated case(s)`
+        : `FAIL: ${regressed.length} of ${m} gated case(s) certified as regressed at ${(opts.fdr * 100).toFixed(0)}% FDR ` +
+            `(${regressed
+                .slice(0, 3)
+                .map((v) => v.id)
+                .join(", ")}${regressed.length > 3 ? ", …" : ""})`) +
+        (caveats.length > 0 ? `. Read with care: ${caveats.join("; ")}.` : "");
+    return {
+        verdict: regressed.length === 0 ? "PASS" : "FAIL",
+        cases: verdicts,
+        regressed,
+        newCases,
+        threshold: ebh.threshold,
+        soloThreshold: solo,
+        options: opts,
+        headline,
+    };
+}
+//# sourceMappingURL=gate.js.map

package/dist/gate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gate.js","sourceRoot":"","sources":["../src/gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAGH,OAAO,EACL,aAAa,EACb,uBAAuB,EACvB,aAAa,EACb,sBAAsB,EACtB,mBAAmB,GACpB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,kBAAkB,EAElB,WAAW,EACX,eAAe,EACf,QAAQ,GACT,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,UAAU,EACV,UAAU,EACV,gBAAgB,EAEhB,UAAU,GACX,MAAM,YAAY,CAAC;AAEpB;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,QAA+C,EAC/C,GAAW,EACF,EAAE,CAAC,QAAQ,CAAC,SAAS,GAAG,QAAQ,CAAC,MAAM,IAAI,GAAG,CAAC;AAyC1D,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,GAAG,EAAE,IAAI;IACT,GAAG,EAAE,IAAI;IACT,gBAAgB,EAAE,CAAC;IACnB,QAAQ,EAAE,OAAO;CACT,CAAC;AA4DX;;;;;;GAMG;AACH,MAAM,UAAU,IAAI,CAClB,KAA6C,EAC7C,OAAO,GAAgB,EAAE;IAEzB,MAAM,IAAI,GAA0B,EAAE,GAAG,oBAAoB,EAAE,GAAG,OAAO,EAAE,CAAC;IAC5E,sBAAsB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAChD,sBAAsB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAChD,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACvC,uBAAuB,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEtE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,aAAa,CAAC,2BAA2B,EAAE,2BAA2B,CAAC,CAAC,EAAE,EAAE,EAAE;gBACtF,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE;gBACpB,IAAI,EAAE,kFAAkF;aACzF,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACf,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACb,mBAAmB,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;YACtD,SAAS;QACX,CAAC;QACD,aAAa,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACtD,IAAI,CAAC,CAAC,QAAQ;YACZ,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACrF,CAAC;IAED,MAAM,WAAW,GAAG,CAAC,CAA4B,EAAW,EAAE,CAC5D,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAChF,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAExC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,aAAa,CACrB,6BAA6B,EAC7B,mEAAmE,EACnE;YACE,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE;YAC1D,IAAI,EAAE,2JAA2J;SAClK,CACF,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,8EAA8E;IAC9E,+EAA+E;IAC/E,wDAAwD;IACxD,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,YAAY,CAAC,CAAC,CAAC,QAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtF,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC/B,IAAI,CAAC,CAAC,MAAM;YAAE,OAAO,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QACtE,OAAO,UAAU,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,WAAW,CACT,IAAI,CAAC,QAAQ,EACb,CAAC,CAAC,SAAS,EACX,CAAC,CAAC,MAAM,EACR,CAAC,CAAC,QAAS,CAAC,SAAS,EACrB,CAAC,CAAC,QAAS,CAAC,MAAM,EAClB,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,gBAAgB,CACtB,CAAC;IACR,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAE3C,MAAM,QAAQ,GAAkB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACjD,MAAM,MAAM,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAE,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAE,EAAE,CAAC;QAC/F,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;YACnB,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC3D,OAAO;gBACL,GAAG,MAAM;gBACT,OAAO,EAAE,QAAQ;gBACjB,YAAY,EAAE,KAAK;gBACnB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE;gBAC7D,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;gBAC5D,MAAM,EAAE,QAAQ;gBAChB,MAAM,EAAE,EAAE,GAAG,CAAC,EAAE;aACjB,CAAC;QACJ,CAAC;QACD,2EAA2E;QAC3E,6EAA6E;QAC7E,2EAA2E;QAC3E,sDAAsD;QACtD,MAAM,KAAK,GAAG,CAAC,CAAC,QAAS,CAAC,SAAS,GAAG,CAAC,CAAC,QAAS,CAAC,MAAM,CAAC;QACzD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC;YAC3B,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,IAAI,CAAC,GAAG,CACN,kBAAkB,CAChB,IAAI,CAAC,QAAQ,EACb,IAAI,EACJ,CAAC,CAAC,QAAS,CAAC,SAAS,EACrB,CAAC,CAAC,QAAS,CAAC,MAAM,EAClB,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,gBAAgB,CACtB,CACF,CAAC;QACN,OAAO;YACL,GAAG,MAAM;YACT,OAAO;YACP,YAAY,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,OAAO,GAAG,IAAI;YAC9C,UAAU,EAAE,UAAU,CAAC,CAAC,CAAE;YAC1B,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;YACtD,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,QAAS,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,QAAS,CAAC,MAAM,EAAE;YAC1E,MAAM,EAAE,UAAU;SACnB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACrE,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAEtD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CACV,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,uDAAuD;YAC5E,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,oCAAoC,KAAK;iBACpE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;iBACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAChB,IAAI,CAAC,IAAI,CAAC,EAAE;YACf,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,4DAA4D,CAC/F,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CACV,GAAG,QAAQ,CAAC,MAAM,6BAA6B,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB;YAC/F,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,iCAAiC,QAAQ;iBACpE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;iBACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAChB,IAAI,CAAC,IAAI,CAAC,EAAE;YACf,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CACzC,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CACV,GAAG,QAAQ,CAAC,MAAM,+CAA+C;YAC/D,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GACZ,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;QACrB,CAAC,CAAC,oCAAoC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,gBAAgB;QACtH,CAAC,CAAC,SAAS,SAAS,CAAC,MAAM,OAAO,CAAC,4CAA4C,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ;YAChH,IAAI,SAAS;iBACV,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;iBACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAChB,IAAI,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;QACzD,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEzE,OAAO;QACL,OAAO,EAAE,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QACjD,KAAK,EAAE,QAAQ;QACf,SAAS;QACT,QAAQ;QACR,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE,IAAI;QACb,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,60 @@
+/**
+ * peeksafe: statistically valid gates for non-deterministic eval suites.
+ *
+ * The problem this exists for: you run an eval suite, watch results arrive, stop
+ * a case when it looks decided, then correct for multiplicity with
+ * Benjamini-Hochberg. That construction has no error control. A p-value computed
+ * at a stopping boundary is not a p-value, and BH assumes it is.
+ *
+ * The library is organised in four layers, and most callers only need the first.
+ *
+ * ## 1. The decision
+ *
+ *   gate(cases, options)   which cases regressed, valid at any stopping time
+ *
+ * You bring counts. It brings e-values and e-BH. Nothing here knows or cares how
+ * you ran the evals, and nothing here will gate a case that has no baseline.
+ *
+ * ## 2. The budget
+ *
+ *   makePlan(...)          how many runs each case needs, and which are impossible
+ *   computeFrontier(...)   what is certifiable inside a budget
+ *   affordabilityGrid(...) the same question as a grid over two axes
+ *
+ * These answer "how many runs do I need" before you spend anything, including
+ * the case where the honest answer is that no candidate budget will do and only
+ * more baseline runs will.
+ *
+ * ## 3. The check
+ *
+ *   typeOneError(...)              the error rate the gate delivers at rates you name
+ *   certifyProbability(...)        the same computation with separate baseline and candidate rates
+ *   pairedCertifyProbability(...)  the power of a paired design
+ *
+ * These are exact, not simulated, so a caller can check the gate on their own
+ * rates before trusting a verdict.
+ *
+ * ## 4. The statistics
+ *
+ * Everything the first two layers are built from, exported so a caller can check
+ * the arithmetic, build a different gate, or use one piece on its own: special
+ * functions, intervals, the SPRT, e-values, e-BH, power, paired designs, and the
+ * cluster-robust suite summary.
+ *
+ * Cases in a suite are not independent draws. Several cases generated from one
+ * source file share a failure mode, so a change that breaks the file moves all
+ * of them together. `clusteredEffect` and the rest of `cluster` are what stop
+ * the suite-level interval from being too narrow for that reason.
+ */
+export { type BaselineStat, baselineNullRate, type CaseRef, toBaselineMap, } from "./baseline.js";
+export { type CertifyProbabilityOptions, certifyProbability, type PairedCertifyProbabilityOptions, pairedCertifyProbability, type TypeOneErrorOptions, typeOneError, } from "./check.js";
+export { type ClusteredEffect, type ClusteredMean, type ClusterKeyDiagnostic, type ClusterObservation, caseFamily, clusteredEffect, clusterKey, clusterKeyDiagnostic, clusterRobustMean, compareEstimators, type FamilyEffect, type GroupingSummary, groupByCluster, iidMean, MIN_TRUSTWORTHY_CLUSTERS, type RandomEffectsMean, randomEffectsMean, } from "./cluster.js";
+export { MAX_ALT_CONCENTRATION, PeeksafeError, type PeeksafeErrorCode, type PeeksafeErrorOptions, } from "./errors.js";
+export type { Evidence } from "./evidence.js";
+export { computeFrontier, costShares, DEFAULT_AXES, DEFAULT_FRONTIER, enumerateFrontier, evaluatePoint, type FrontierAxes, type FrontierBasis, type FrontierCell, type FrontierConfig, type FrontierCost, type FrontierDesign, type FrontierPoint, type FrontierResult, type FrontierScreen, fmtRate, typicalObservationsPerCase, } from "./frontier.js";
+export { type CaseVerdict, DEFAULT_GATE_OPTIONS, type GateCase, type GateOptions, type GateResult, gate, type PairedGateCase, } from "./gate.js";
+export { type AffordabilityCell, affordabilityGrid, DEFAULT_PLAN, type Detectability, expectedSequentialSamples, makePlan, type Plan, type PlanCase, type PlanConfig, type PlanTotals, planCase, } from "./plan.js";
+export { hash32, makeRand, type Rand, streamFor } from "./rand.js";
+export { type BetaPosterior, type BhResult, bernoulliEntropy, betaCredibleInterval, betaMassBetween, betaPosterior, betaQuantile, bhCorrect, cohensH, diffInterval, discordant, type EbhResult, ebhCorrect, ebhSoloThreshold, erf, evidenceCeilingAsymptotic, evidenceCeilingLogE, evidenceCeilingSlope, expectedLogE, expectedLogEExact, fisherExact2x2, gammaP, type Interval, ibeta, logBeta, logBetaPdf, logGamma, logMarginalBetaBinomial, mcnemarSamplesForEvidence, normalCdf, normalQuantile, type PairedCounts, pairedDiscordance, pairedLogE, pairPhi, probabilityMoved, type SprtDecision, type SprtOpts, type SprtResult, sampleSizeTwoProportion, samplesForEvidence, sprtDecision, sprtExpectedN, type TestResult, twoProportionZTest, twoSampleLogE, twoSamplePriors, universalCeilingLogE, universalTwoSampleLogE, wilsonInterval, } from "./stats.js";
+export { DEFAULT_STOP_OPTIONS, type PairedStopDecision, type PairedStopOptions, type PairedStopReason, type StopDecision, type StopOptions, type StopReason, shouldStop, shouldStopPaired, } from "./stop.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EACL,KAAK,YAAY,EACjB,gBAAgB,EAChB,KAAK,OAAO,EACZ,aAAa,GACd,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,KAAK,yBAAyB,EAC9B,kBAAkB,EAClB,KAAK,+BAA+B,EACpC,wBAAwB,EACxB,KAAK,mBAAmB,EACxB,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,UAAU,EACV,eAAe,EACf,UAAU,EACV,oBAAoB,EACpB,iBAAiB,EACjB,iBAAiB,EACjB,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,cAAc,EACd,OAAO,EACP,wBAAwB,EACxB,KAAK,iBAAiB,EACtB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,qBAAqB,EACrB,aAAa,EACb,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,GAC1B,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EACL,eAAe,EACf,UAAU,EACV,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,OAAO,EACP,0BAA0B,GAC3B,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,KAAK,WAAW,EAChB,oBAAoB,EACpB,KAAK,QAAQ,EACb,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,IAAI,EACJ,KAAK,cAAc,GACpB,MAAM,WAAW,CAAC;AAEnB,OAAO,EACL,KAAK,iBAAiB,EACtB,iBAAiB,EACjB,YAAY,EACZ,KAAK,aAAa,EAClB,yBAAyB,EACzB,QAAQ,EACR,KAAK,IAAI,EACT,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,UAAU,EACf,QAAQ,GACT,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEnE,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,QAAQ,EACb,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,EACf,aAAa,EACb,YAAY,EAEZ,SAAS,EACT,OAAO,EACP,YAAY,EACZ,UAAU,EACV,KAAK,SAAS,EACd,UAAU,EACV,gBAAgB,EAChB,GAAG,EACH,yBAAyB,EACzB,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,MAAM,EACN,KAAK,QAAQ,EACb,KAAK,EACL,OAAO,EACP,UAAU,EAEV,QAAQ,EAER,uBAAuB,EACvB,yBAAyB,EACzB,SAAS,EACT,cAAc,EACd,KAAK,YAAY,EACjB,iBAAiB,EAEjB,UAAU,EACV,OAAO,EACP,gBAAgB,EAChB,KAAK,YAAY,EACjB,KAAK,QAAQ,EACb,KAAK,UAAU,EAEf,uBAAuB,EACvB,kBAAkB,EAElB,YAAY,EACZ,aAAa,EACb,KAAK,UAAU,EAEf,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,oBAAoB,EACpB,sBAAsB,EAEtB,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,oBAAoB,EACpB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,UAAU,EACV,gBAAgB,GACjB,MAAM,WAAW,CAAC"}

package/dist/index.js

@@ -0,0 +1,79 @@
+/**
+ * peeksafe: statistically valid gates for non-deterministic eval suites.
+ *
+ * The problem this exists for: you run an eval suite, watch results arrive, stop
+ * a case when it looks decided, then correct for multiplicity with
+ * Benjamini-Hochberg. That construction has no error control. A p-value computed
+ * at a stopping boundary is not a p-value, and BH assumes it is.
+ *
+ * The library is organised in four layers, and most callers only need the first.
+ *
+ * ## 1. The decision
+ *
+ *   gate(cases, options)   which cases regressed, valid at any stopping time
+ *
+ * You bring counts. It brings e-values and e-BH. Nothing here knows or cares how
+ * you ran the evals, and nothing here will gate a case that has no baseline.
+ *
+ * ## 2. The budget
+ *
+ *   makePlan(...)          how many runs each case needs, and which are impossible
+ *   computeFrontier(...)   what is certifiable inside a budget
+ *   affordabilityGrid(...) the same question as a grid over two axes
+ *
+ * These answer "how many runs do I need" before you spend anything, including
+ * the case where the honest answer is that no candidate budget will do and only
+ * more baseline runs will.
+ *
+ * ## 3. The check
+ *
+ *   typeOneError(...)              the error rate the gate delivers at rates you name
+ *   certifyProbability(...)        the same computation with separate baseline and candidate rates
+ *   pairedCertifyProbability(...)  the power of a paired design
+ *
+ * These are exact, not simulated, so a caller can check the gate on their own
+ * rates before trusting a verdict.
+ *
+ * ## 4. The statistics
+ *
+ * Everything the first two layers are built from, exported so a caller can check
+ * the arithmetic, build a different gate, or use one piece on its own: special
+ * functions, intervals, the SPRT, e-values, e-BH, power, paired designs, and the
+ * cluster-robust suite summary.
+ *
+ * Cases in a suite are not independent draws. Several cases generated from one
+ * source file share a failure mode, so a change that breaks the file moves all
+ * of them together. `clusteredEffect` and the rest of `cluster` are what stop
+ * the suite-level interval from being too narrow for that reason.
+ */
+export { baselineNullRate, toBaselineMap, } from "./baseline.js";
+export { certifyProbability, pairedCertifyProbability, typeOneError, } from "./check.js";
+export { caseFamily, clusteredEffect, clusterKey, clusterKeyDiagnostic, clusterRobustMean, compareEstimators, groupByCluster, iidMean, MIN_TRUSTWORTHY_CLUSTERS, randomEffectsMean, } from "./cluster.js";
+/* ── errors and determinism ──────────────────────────────────────────────── */
+export { MAX_ALT_CONCENTRATION, PeeksafeError, } from "./errors.js";
+export { computeFrontier, costShares, DEFAULT_AXES, DEFAULT_FRONTIER, enumerateFrontier, evaluatePoint, fmtRate, typicalObservationsPerCase, } from "./frontier.js";
+/* ── 1. the decision ─────────────────────────────────────────────────────── */
+export { DEFAULT_GATE_OPTIONS, gate, } from "./gate.js";
+/* ── 2. the budget ───────────────────────────────────────────────────────── */
+export { affordabilityGrid, DEFAULT_PLAN, expectedSequentialSamples, makePlan, planCase, } from "./plan.js";
+export { hash32, makeRand, streamFor } from "./rand.js";
+/* ── 3. the statistics ───────────────────────────────────────────────────── */
+export { bernoulliEntropy, betaCredibleInterval, betaMassBetween, betaPosterior, betaQuantile, 
+/* multiplicity */
+bhCorrect, cohensH, diffInterval, discordant, ebhCorrect, ebhSoloThreshold, erf, evidenceCeilingAsymptotic, evidenceCeilingLogE, evidenceCeilingSlope, expectedLogE, expectedLogEExact, fisherExact2x2, gammaP, ibeta, logBeta, logBetaPdf, 
+/* special functions */
+logGamma, 
+/* e-values */
+logMarginalBetaBinomial, mcnemarSamplesForEvidence, normalCdf, normalQuantile, pairedDiscordance, 
+/* paired designs */
+pairedLogE, pairPhi, probabilityMoved, 
+/* power */
+sampleSizeTwoProportion, samplesForEvidence, 
+/* sequential */
+sprtDecision, sprtExpectedN, 
+/* fixed-sample tests, for comparison rather than for gating */
+twoProportionZTest, twoSampleLogE, twoSamplePriors, universalCeilingLogE, universalTwoSampleLogE, 
+/* intervals and posteriors */
+wilsonInterval, } from "./stats.js";
+export { DEFAULT_STOP_OPTIONS, shouldStop, shouldStopPaired, } from "./stop.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAEL,gBAAgB,EAEhB,aAAa,GACd,MAAM,eAAe,CAAC;AACvB,OAAO,EAEL,kBAAkB,EAElB,wBAAwB,EAExB,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,OAAO,EAKL,UAAU,EACV,eAAe,EACf,UAAU,EACV,oBAAoB,EACpB,iBAAiB,EACjB,iBAAiB,EAGjB,cAAc,EACd,OAAO,EACP,wBAAwB,EAExB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,gFAAgF;AAChF,OAAO,EACL,qBAAqB,EACrB,aAAa,GAGd,MAAM,aAAa,CAAC;AAErB,OAAO,EACL,eAAe,EACf,UAAU,EACV,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EAUb,OAAO,EACP,0BAA0B,GAC3B,MAAM,eAAe,CAAC;AACvB,gFAAgF;AAChF,OAAO,EAEL,oBAAoB,EAIpB,IAAI,GAEL,MAAM,WAAW,CAAC;AACnB,gFAAgF;AAChF,OAAO,EAEL,iBAAiB,EACjB,YAAY,EAEZ,yBAAyB,EACzB,QAAQ,EAKR,QAAQ,GACT,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAa,SAAS,EAAE,MAAM,WAAW,CAAC;AACnE,gFAAgF;AAChF,OAAO,EAGL,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,EACf,aAAa,EACb,YAAY;AACZ,kBAAkB;AAClB,SAAS,EACT,OAAO,EACP,YAAY,EACZ,UAAU,EAEV,UAAU,EACV,gBAAgB,EAChB,GAAG,EACH,yBAAyB,EACzB,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,MAAM,EAEN,KAAK,EACL,OAAO,EACP,UAAU;AACV,uBAAuB;AACvB,QAAQ;AACR,cAAc;AACd,uBAAuB,EACvB,yBAAyB,EACzB,SAAS,EACT,cAAc,EAEd,iBAAiB;AACjB,oBAAoB;AACpB,UAAU,EACV,OAAO,EACP,gBAAgB;AAIhB,WAAW;AACX,uBAAuB,EACvB,kBAAkB;AAClB,gBAAgB;AAChB,YAAY,EACZ,aAAa;AAEb,+DAA+D;AAC/D,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,oBAAoB,EACpB,sBAAsB;AACtB,8BAA8B;AAC9B,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,oBAAoB,EAOpB,UAAU,EACV,gBAAgB,GACjB,MAAM,WAAW,CAAC"}

package/dist/plan.d.ts

@@ -0,0 +1,248 @@
+/**
+ * plan.ts: power economics. What will this cost, and can we even detect it?
+ *
+ * Without a plan, the first sign that a suite cannot be decided is a large bill
+ * and no certified case. This module tells you *before* you spend anything, and it will tell you that the honest answer is
+ * sometimes "you cannot detect this at any price".
+ *
+ * The three numbers that matter, per case:
+ *
+ *  1. **The bar.** e-BH at FDR q over m cases certifies a case on its own only
+ *     once its e-value clears m/q. That is the target the evidence has to hit.
+ *
+ *  2. **The ceiling.** `evidenceCeilingLogE`, the limit of the unpaired
+ *     two-sample e-value as candidate runs go to infinity. Both marginals are
+ *     the same binomial likelihood under different priors, so everything that
+ *     grows with n cancels and the Bayes factor converges to the prior density
+ *     ratio at the observed rate. If the ceiling is under the bar, no number of
+ *     candidate runs will certify the case. This is not a budget problem and no
+ *     budget fixes it: **more baseline runs** fix it, by sharpening the null.
+ *
+ *  3. **The paired alternative.** Re-run the baseline alongside the candidate
+ *     on the same seed and the null becomes an exact point mass at ½ (a
+ *     discordant pair points either way with probability exactly one half under
+ *     "nothing changed"). Nothing is estimated, so nothing caps the evidence:
+ *     the paired e-value grows linearly in discordant pairs. It costs two runs
+ *     per observation and it is still, routinely, the cheaper design, and it
+ *     is the only one that works at all for a case whose ceiling is under the
+ *     bar.
+ *
+ * Everything here is a *plan*: it evaluates the mean trajectory (the candidate
+ * scores exactly its expected successes). Realised sample counts vary around
+ * it.
+ */
+import { type BaselineStat, type CaseRef } from "./baseline.js";
+import { type PairedCounts } from "./stats.js";
+export interface PlanConfig {
+    /** the drop we want to be able to detect, in probability units (0.15 = 15 points) */
+    mde: number;
+    /** suite-level false discovery rate */
+    fdr: number;
+    /** USD per candidate run */
+    costPerRunUsd: number;
+    /** milliseconds per run, wall clock, at concurrency 1 */
+    msPerRun: number;
+    /** how many runs the screening pass spends per case */
+    screenRuns: number;
+    /**
+     * Measured coupling between the paired arms, in [0,1]. 0 means the seeds
+     * bought nothing; 1 means perfect common-random-numbers coupling. Measure it
+     * with `pairPhi` on a pilot rather than assuming it, and plan at 0 as well
+     * so the sensitivity is visible.
+     */
+    pairCoupling: number;
+    /** how many runs to fan out in parallel, only affects wall time */
+    concurrency: number;
+    /** beyond this many runs for one case we stop calling it "affordable" */
+    runsPerCaseCeiling: number;
+    /**
+     * Which design the *expected* bill is priced for. A paired observation is two
+     * runs, so this doubles `typicalRuns`.
+     *
+     * `undefined` means "price the design this plan recommends", which is the
+     * only self-consistent answer: `makePlan` recommends pairing whenever more
+     * than a quarter of the suite has an unpaired evidence ceiling under the bar,
+     * and quoting the *unpaired* bill underneath that recommendation would price
+     * a design nobody runs.
+     */
+    design?: "unpaired" | "paired";
+    alpha: number;
+    beta: number;
+    minTrials: number;
+    maxTrials: number;
+    batchSize: number;
+    nullQuantile: number;
+}
+export declare const DEFAULT_PLAN: PlanConfig;
+export type Detectability = 
+/** the unpaired sequential test can certify this case within the run ceiling */
+"UNPAIRED"
+/** only the paired design can certify it, the unpaired evidence ceiling is under the bar */
+ | "PAIRED_ONLY"
+/** the run count is finite but past the ceiling: technically possible, practically not */
+ | "TOO_EXPENSIVE"
+/** the case cannot lose `mde` points at all, its baseline rate is not that high */
+ | "IMPOSSIBLE";
+export interface PlanCase {
+    caseId: string;
+    suite: string;
+    baseline: {
+        successes: number;
+        trials: number;
+        rate: number;
+    };
+    /** the rate the alternative sits at: baseline − mde, floored at 0 */
+    alternativeRate: number;
+    /** log of the e-value a case must reach to be certified on its own evidence */
+    barLogE: number;
+    /** the limit of the unpaired log e-value as runs → ∞ */
+    ceilingLogE: number;
+    unpaired: {
+        runs: number;
+        costUsd: number;
+        wallMs: number;
+    };
+    paired: {
+        pairs: number;
+        runs: number;
+        costUsd: number;
+        wallMs: number;
+    };
+    /**
+     * The coupling `paired` is priced at: `pairPhi` of this case's pilot when
+     * `makePlan` was given one, floored at 0, otherwise `config.pairCoupling`.
+     */
+    pairCoupling: number;
+    detectability: Detectability;
+    /**
+     * The design to run this case with: of the designs that decide it within
+     * `runsPerCaseCeiling`, the one that needs fewer runs, unpaired on a tie.
+     * Null when neither does.
+     */
+    design: "unpaired" | "paired" | null;
+    /**
+     * How many baseline runs this case would need for the *unpaired* design to
+     * work, or null if it already works. This is the actionable half of an
+     * "undetectable" verdict.
+     */
+    baselineRunsNeeded: number | null;
+    /** what a fixed-N two-proportion design would spend per arm, for comparison */
+    fixedNPerArm: number;
+    reason: string;
+}
+export interface PlanTotals {
+    cases: number;
+    /** how many cases each design can actually certify */
+    decidableUnpaired: number;
+    decidableBest: number;
+    /** every case run to certification, unpaired */
+    unpairedRuns: number;
+    unpairedCostUsd: number;
+    /** every case run to certification, paired */
+    pairedRuns: number;
+    pairedCostUsd: number;
+    /** per case, the cheaper of the two designs that can decide it */
+    bestRuns: number;
+    bestCostUsd: number;
+    /** screen everything cheaply, then pay only for what the screen keeps */
+    screenedRuns: number;
+    screenedCostUsd: number;
+    /** the fixed-N design with the same nominal guarantees */
+    fixedNRuns: number;
+    fixedNCostUsd: number;
+    wallMsUnpaired: number;
+    wallMsPaired: number;
+    wallMsBest: number;
+    wallMsScreened: number;
+    /** fraction of cases the screen is budgeted to carry through to the full test */
+    screenContinueFraction: number;
+    /** 1 unpaired, 2 paired, what one observation costs in runs */
+    runsPerObservation: number;
+    /** expected *runs* (not observations) for a pull request in which nothing moved */
+    typicalRuns: number;
+    typicalCostUsd: number;
+    wallMsTypical: number;
+    /** the same, with a screening pass in front */
+    typicalScreenedRuns: number;
+    typicalScreenedCostUsd: number;
+}
+export interface Plan {
+    config: PlanConfig;
+    cases: PlanCase[];
+    totals: PlanTotals;
+    /** cases no budget can decide with the current baseline */
+    undecidable: PlanCase[];
+    /** cases only the paired design can reach */
+    pairedOnly: PlanCase[];
+    /** cases whose baseline rate is too low to lose `mde` points at all */
+    impossible: PlanCase[];
+    /** the design the recommendation names, and the one `expectedCostUsd` prices */
+    recommendedDesign: "unpaired" | "paired";
+    /** the cheapest design that decides every decidable case */
+    recommendation: string;
+    /** honest one-liner about affordability at the configured cost per run */
+    verdict: string;
+    /**
+     * What a *typical* pull request costs, one in which nothing moved, so most
+     * cases stop at the H₀ wall. This is the headline number.
+     */
+    expectedCostUsd: number;
+    /**
+     * What a pull request in which *every* case regressed would cost.
+     *
+     * Not an upper bound on `expectedCostUsd`, despite running every case to
+     * certification: the sequential test stops a case as soon as it clears the
+     * bar, so an all-regressed suite can finish cheaper than one that did not
+     * move and ran its full schedule. `verdict` says which way round they came
+     * out rather than presenting this as the worst case.
+     */
+    ceilingCostUsd: number;
+}
+export declare function planCase(c: CaseRef, b: BaselineStat | undefined, cfg: PlanConfig, m: number): PlanCase;
+/**
+ * Expected total samples for the sequential design, priced under Wald's
+ * expected sample number at a supplied "true" rate per case, and clamped into
+ * the gate's own [minTrials, maxTrials] batch grid.
+ *
+ * This is the number that predicts what the gate will actually spend.
+ * `samplesForEvidence`, the one behind `PlanCase.unpaired.runs`, answers a
+ * different question: how many runs it takes to *certify a regression*. A case
+ * that did not regress never gets there; it hits the H₀ wall in a few batches
+ * and stops. Pricing every case at the certification cost over-predicts the
+ * bill.
+ *
+ * `trueRate` defaults to the case's own baseline rate, i.e. "nothing moved", * which is what almost every pull request looks like.
+ */
+export declare function expectedSequentialSamples(cases: Array<CaseRef>, baseline: Map<string, BaselineStat>, cfg: PlanConfig, trueRate?: (caseId: string, b: BaselineStat) => number): {
+    samples: number;
+    perCase: Map<string, number>;
+};
+/**
+ * Plan a suite. `pilots` holds a pair table per case from a pilot run of the
+ * paired design, ideally with both arms on the same revision; a case with one
+ * prices pairing at its measured `pairPhi` instead of `cfg.pairCoupling`, so
+ * its `design` is chosen from a coupling the runner actually delivers.
+ */
+export declare function makePlan(cases: Array<CaseRef>, baseline: Map<string, BaselineStat>, cfg?: PlanConfig, pilots?: Map<string, PairedCounts>): Plan;
+/**
+ * The suite size / MDE combinations that *are* affordable at a given per-run
+ * cost and budget. This is the honest answer to "is this usable for a
+ * coding-agent benchmark at $1 a run?", usually "not at 200 cases and 15
+ * points, but yes at 20 cases and 25 points".
+ */
+export interface AffordabilityCell {
+    cases: number;
+    mde: number;
+    /** runs to certify every case, the ceiling */
+    runs: number;
+    costUsd: number;
+    /** runs a pull request that moved nothing actually spends, the expected bill */
+    typicalRuns: number;
+    typicalCostUsd: number;
+    /** does the *ceiling* fit the budget? */
+    affordable: boolean;
+    /** does the *expected* cost fit the budget? */
+    typicallyAffordable: boolean;
+}
+export declare function affordabilityGrid(baselineRate: number, baselineTrials: number, cfg: PlanConfig, budgetUsd: number, caseCounts?: number[], mdes?: number[]): AffordabilityCell[];
+//# sourceMappingURL=plan.d.ts.map

package/dist/plan.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"plan.d.ts","sourceRoot":"","sources":["../src/plan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,EAAE,KAAK,YAAY,EAAoB,KAAK,OAAO,EAAE,MAAM,eAAe,CAAC;AAOlF,OAAO,EAKL,KAAK,YAAY,EAKlB,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,UAAU;IACzB,qFAAqF;IACrF,GAAG,EAAE,MAAM,CAAC;IACZ,uCAAuC;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,4BAA4B;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;;;;;;;OASG;IACH,MAAM,CAAC,EAAE,UAAU,GAAG,QAAQ,CAAC;IAO/B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,eAAO,MAAM,YAAY,EAAE,UAiB1B,CAAC;AAEF,MAAM,MAAM,aAAa;AACvB,gFAAgF;AAC9E,UAAU;AACZ,4FAA4F;GAC1F,aAAa;AACf,0FAA0F;GACxF,eAAe;AACjB,mFAAmF;GACjF,YAAY,CAAC;AAEjB,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9D,qEAAqE;IACrE,eAAe,EAAE,MAAM,CAAC;IACxB,+EAA+E;IAC/E,OAAO,EAAE,MAAM,CAAC;IAChB,wDAAwD;IACxD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5D,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACzE;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,aAAa,CAAC;IAC7B;;;;OAIG;IACH,MAAM,EAAE,UAAU,GAAG,QAAQ,GAAG,IAAI,CAAC;IACrC;;;;OAIG;IACH,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,+EAA+E;IAC/E,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,gDAAgD;IAChD,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,0DAA0D;IAC1D,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,iFAAiF;IACjF,sBAAsB,EAAE,MAAM,CAAC;IAU/B,+DAA+D;IAC/D,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mFAAmF;IACnF,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,+CAA+C;IAC/C,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,IAAI;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,MAAM,EAAE,UAAU,CAAC;IACnB,2DAA2D;IAC3D,WAAW,EAAE,QAAQ,EAAE,CAAC;IACxB,6CAA6C;IAC7C,UAAU,EAAE,QAAQ,EAAE,CAAC;IACvB,uEAAuE;IACvE,UAAU,EAAE,QAAQ,EAAE,CAAC;IACvB,gFAAgF;IAChF,iBAAiB,EAAE,UAAU,GAAG,QAAQ,CAAC;IACzC,4DAA4D;IAC5D,cAAc,EAAE,MAAM,CAAC;IACvB,0EAA0E;IAC1E,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;;OAQG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AA+CD,wBAAgB,QAAQ,CACtB,CAAC,EAAE,OAAO,EACV,CAAC,EAAE,YAAY,GAAG,SAAS,EAC3B,GAAG,EAAE,UAAU,EACf,CAAC,EAAE,MAAM,GACR,QAAQ,CAsHV;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,EACrB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACnC,GAAG,EAAE,UAAU,EACf,QAAQ,GAAE,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,YAAY,KAAK,MACJ,GAC1C;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAuBnD;AAED;;;;;GAKG;AACH,wBAAgB,QAAQ,CACtB,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,EACrB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACnC,GAAG,GAAE,UAAyB,EAC9B,MAAM,GAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAa,GAC5C,IAAI,CAuMN;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,8CAA8C;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,yCAAyC;IACzC,UAAU,EAAE,OAAO,CAAC;IACpB,+CAA+C;IAC/C,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAED,wBAAgB,iBAAiB,CAC/B,YAAY,EAAE,MAAM,EACpB,cAAc,EAAE,MAAM,EACtB,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,UAAU,GAAE,MAAM,EAAuB,EACzC,IAAI,GAAE,MAAM,EAA4B,GACvC,iBAAiB,EAAE,CA+ErB"}