peeksafe 0.0.0-stage → 0.2.0

package/dist/gate.d.ts

@@ -0,0 +1,175 @@
+/**
+ * gate.ts: the decision.
+ *
+ * You bring counts. This decides which cases regressed, at a false discovery
+ * rate you name, in a way that stays valid however many times you looked at the
+ * data on the way here.
+ *
+ * ## Why this is safe to peek at
+ *
+ * The usual construction is: run the suite, stop a case when it looks
+ * significant, compute a p-value from the counts, hand the p-values to
+ * Benjamini-Hochberg. That is invalid in two independent places.
+ *
+ * A p-value computed at a stopping boundary is not a p-value. `P(p_n <= a) <= a`
+ * holds for each fixed `n`; a procedure that adds runs and re-tests until the
+ * result looks significant reports `min` over the peeks, and that minimum is not
+ * bounded by `a`. Benjamini-Hochberg takes `P(p <= t) <= t` as its input
+ * assumption, so it inherits nothing and controls nothing.
+ *
+ * This module never computes a p-value. It computes an e-value, which is a
+ * non-negative statistic with mean at most 1 under the null. Ville's inequality
+ * bounds `P(sup_n E_n >= 1/a) <= a` over the whole trajectory, so an e-value is
+ * valid at any stopping time, including one chosen after looking. (The mean is
+ * exactly 1 averaged over the shared rate, not at every fixed rate; the README
+ * section "What that guarantee rests on" says where the bound is checked.) The
+ * multiplicity correction is e-BH (Wang and Ramdas), which takes e-values
+ * directly and controls FDR under arbitrary dependence between cases.
+ *
+ * The practical consequence: you may stop a case whenever you like, for any
+ * reason, including "it already looks bad". Nothing here cares when you stopped.
+ *
+ * ## What it will refuse to do
+ *
+ * Gate a case with no baseline. There is no null to test against, and the
+ * failure mode of inventing one is not a wrong number, it is a confident wrong
+ * number: a missing baseline read as `0/0` has a Beta(1,1) posterior median of
+ * 0.5, which gates the case against "it passes half the time" and reports a
+ * large improvement against nothing at all. Cases without a baseline come back
+ * in `newCases` and take no part in the verdict or the e-BH family.
+ *
+ * ## Paired cases
+ *
+ * A case may instead carry `paired` counts from re-running the baseline next to
+ * the candidate, one baseline run per candidate run. Its e-value is
+ * `pairedLogE` on the discordant pairs, which is valid at every rate by
+ * construction and has no ceiling, whichever `evidence` the unpaired cases use.
+ * The pairs need not share a seed: independent runs paired in order are still
+ * valid, and a shared seed only makes discordances more informative.
+ */
+import type { BaselineStat } from "./baseline.js";
+import { type Evidence } from "./evidence.js";
+import { type PairedCounts } from "./stats.js";
+/**
+ * True when the baseline's observed rate leaves no room for an `mde`-sized drop.
+ * `makePlan` calls the same cases IMPOSSIBLE, by the same rule.
+ */
+export declare const cannotDropBy: (baseline: {
+    successes: number;
+    trials: number;
+}, mde: number) => boolean;
+/** `options` without its `undefined` entries, so a spread over the defaults leaves those defaults in place. */
+export declare function definedOptions<T extends object>(options: T): T;
+/** One case's candidate result, and the baseline it is measured against. */
+export interface GateCase {
+    id: string;
+    /** Successes observed for the revision under test. */
+    successes: number;
+    /** Trials run for the revision under test. Stop whenever you like. */
+    trials: number;
+    /** The reference revision's recorded counts, or undefined if there are none. */
+    baseline?: BaselineStat;
+    paired?: undefined;
+}
+/** One case run as pairs: each candidate run next to a fresh baseline run. */
+export interface PairedGateCase {
+    id: string;
+    /** The pair table so far. Stop whenever you like. */
+    paired: PairedCounts;
+}
+export interface GateOptions {
+    /** Minimum detectable effect: the drop, in rate, that counts as a regression. */
+    mde?: number;
+    /** Target false discovery rate across the suite. */
+    fdr?: number;
+    /**
+     * Concentration of the alternative prior. Lower spreads the alternative over
+     * "anything materially worse", which is closer to how a broken case behaves
+     * than a point mass exactly `mde` below the baseline. At most
+     * `MAX_ALT_CONCENTRATION` (16) with the default `bayes` statistic.
+     */
+    altConcentration?: number;
+    /**
+     * Which e-value decides. `bayes` has more power and error control checked
+     * numerically; `universal` is valid at every rate by construction and needs
+     * more runs. See the README section "What that guarantee rests on".
+     */
+    evidence?: Evidence;
+}
+export declare const DEFAULT_GATE_OPTIONS: {
+    readonly mde: 0.15;
+    readonly fdr: 0.05;
+    readonly altConcentration: 8;
+    readonly evidence: "bayes";
+};
+export interface CaseVerdict {
+    id: string;
+    /** True when e-BH certifies this case as regressed at the suite's FDR. */
+    regressed: boolean;
+    /** The e-value. Larger is more evidence against the null. */
+    evalue: number;
+    /** log of the above, which is what the arithmetic actually uses. */
+    logE: number;
+    /**
+     * The largest e-value this case could ever reach against its current
+     * baseline, however many candidate runs you add. When this is below
+     * `threshold`, the case is undetectable at any candidate budget and the only
+     * remedy is more baseline runs.
+     */
+    ceiling: number;
+    /** True when `ceiling` is below the bar the case has to clear. */
+    undetectable: boolean;
+    /**
+     * True when the baseline rate is at or below `mde`, so an `mde`-sized drop
+     * cannot happen. The case is not tested: its e-value is fixed at 1, which
+     * keeps it in the e-BH family without it ever being certified.
+     */
+    impossible: boolean;
+    /** For a paired case, each arm's passes over the number of pairs. */
+    observed: {
+        successes: number;
+        trials: number;
+    };
+    baseline: {
+        successes: number;
+        trials: number;
+    };
+    design: "unpaired" | "paired";
+    /** The pair table, for a paired case. */
+    paired?: PairedCounts;
+}
+export interface GateResult {
+    /** PASS when nothing was certified as regressed. */
+    verdict: "PASS" | "FAIL";
+    cases: CaseVerdict[];
+    /** Cases certified as regressed, in the order given. */
+    regressed: CaseVerdict[];
+    /**
+     * Cases with no baseline. These are reported, not gated: they have no null to
+     * be tested against and they are excluded from the e-BH family.
+     */
+    newCases: string[];
+    /** The e-value a case had to beat, given how many discoveries were made. */
+    threshold: number;
+    /** The e-value a case must reach to be certified whatever the others do. */
+    soloThreshold: number;
+    options: Required<GateOptions>;
+    /**
+     * One line an operator can read without knowing what an e-value is.
+     *
+     * Names the two things a passing suite can hide: cases with no baseline,
+     * which were not gated at all, and cases whose baseline is too thin to
+     * certify an `mde`-sized drop however many runs you add. A PASS that is
+     * really "nothing was checked" should not read like a PASS.
+     */
+    headline: string;
+}
+/**
+ * Decide which cases regressed.
+ *
+ * Valid at any stopping time. Run every case to a fixed count, or stop each one
+ * the moment it looks decided, or anything in between; the guarantee does not
+ * depend on the rule you used.
+ */
+export declare function gate(cases: readonly (GateCase | PairedGateCase)[], options?: GateOptions): GateResult;
+//# sourceMappingURL=gate.d.ts.map

package/dist/gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gate.d.ts","sourceRoot":"","sources":["../src/gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAQlD,OAAO,EAEL,KAAK,QAAQ,EAId,MAAM,eAAe,CAAC;AACvB,OAAO,EAIL,KAAK,YAAY,EAElB,MAAM,YAAY,CAAC;AAEpB;;;GAGG;AACH,eAAO,MAAM,YAAY,aACb;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,OAC1C,MAAM,KACV,OAAsD,CAAC;AAE1D,+GAA+G;AAC/G,wBAAgB,cAAc,CAAC,CAAC,SAAS,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,CAE9D;AAED,4EAA4E;AAC5E,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC;IACf,gFAAgF;IAChF,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,MAAM,CAAC,EAAE,SAAS,CAAC;CACpB;AAED,8EAA8E;AAC9E,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,qDAAqD;IACrD,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,iFAAiF;IACjF,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED,eAAO,MAAM,oBAAoB;aAC/B,GAAG,EAAE,IAAI;aACT,GAAG,EAAE,IAAI;aACT,gBAAgB,EAAE,CAAC;aACnB,QAAQ,EAAE,OAAO;CACT,CAAC;AAEX,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,0EAA0E;IAC1E,SAAS,EAAE,OAAO,CAAC;IACnB,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC;IACf,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;IACb;;;;;OAKG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,YAAY,EAAE,OAAO,CAAC;IACtB;;;;OAIG;IACH,UAAU,EAAE,OAAO,CAAC;IACpB,qEAAqE;IACrE,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,MAAM,EAAE,UAAU,GAAG,QAAQ,CAAC;IAC9B,yCAAyC;IACzC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAED,MAAM,WAAW,UAAU;IACzB,oDAAoD;IACpD,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,wDAAwD;IACxD,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB;;;OAGG;IACH,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,4EAA4E;IAC5E,SAAS,EAAE,MAAM,CAAC;IAClB,4EAA4E;IAC5E,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B;;;;;;;OAOG;IACH,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,IAAI,CAClB,KAAK,EAAE,SAAS,CAAC,QAAQ,GAAG,cAAc,CAAC,EAAE,EAC7C,OAAO,GAAE,WAAgB,GACxB,UAAU,CAiKZ"}

package/dist/gate.js

@@ -0,0 +1,200 @@
+/**
+ * gate.ts: the decision.
+ *
+ * You bring counts. This decides which cases regressed, at a false discovery
+ * rate you name, in a way that stays valid however many times you looked at the
+ * data on the way here.
+ *
+ * ## Why this is safe to peek at
+ *
+ * The usual construction is: run the suite, stop a case when it looks
+ * significant, compute a p-value from the counts, hand the p-values to
+ * Benjamini-Hochberg. That is invalid in two independent places.
+ *
+ * A p-value computed at a stopping boundary is not a p-value. `P(p_n <= a) <= a`
+ * holds for each fixed `n`; a procedure that adds runs and re-tests until the
+ * result looks significant reports `min` over the peeks, and that minimum is not
+ * bounded by `a`. Benjamini-Hochberg takes `P(p <= t) <= t` as its input
+ * assumption, so it inherits nothing and controls nothing.
+ *
+ * This module never computes a p-value. It computes an e-value, which is a
+ * non-negative statistic with mean at most 1 under the null. Ville's inequality
+ * bounds `P(sup_n E_n >= 1/a) <= a` over the whole trajectory, so an e-value is
+ * valid at any stopping time, including one chosen after looking. (The mean is
+ * exactly 1 averaged over the shared rate, not at every fixed rate; the README
+ * section "What that guarantee rests on" says where the bound is checked.) The
+ * multiplicity correction is e-BH (Wang and Ramdas), which takes e-values
+ * directly and controls FDR under arbitrary dependence between cases.
+ *
+ * The practical consequence: you may stop a case whenever you like, for any
+ * reason, including "it already looks bad". Nothing here cares when you stopped.
+ *
+ * ## What it will refuse to do
+ *
+ * Gate a case with no baseline. There is no null to test against, and the
+ * failure mode of inventing one is not a wrong number, it is a confident wrong
+ * number: a missing baseline read as `0/0` has a Beta(1,1) posterior median of
+ * 0.5, which gates the case against "it passes half the time" and reports a
+ * large improvement against nothing at all. Cases without a baseline come back
+ * in `newCases` and take no part in the verdict or the e-BH family.
+ *
+ * ## Paired cases
+ *
+ * A case may instead carry `paired` counts from re-running the baseline next to
+ * the candidate, one baseline run per candidate run. Its e-value is
+ * `pairedLogE` on the discordant pairs, which is valid at every rate by
+ * construction and has no ceiling, whichever `evidence` the unpaired cases use.
+ * The pairs need not share a seed: independent runs paired in order are still
+ * valid, and a shared seed only makes discordances more informative.
+ */
+import { PeeksafeError, requireAltConcentration, requireCounts, requireOpenProbability, requirePairedCounts, } from "./errors.js";
+import { ceilingLogEvidence, logEvidence, requireEvidence, toEvalue, } from "./evidence.js";
+import { discordant, ebhCorrect, ebhSoloThreshold, pairedLogE, } from "./stats.js";
+/**
+ * True when the baseline's observed rate leaves no room for an `mde`-sized drop.
+ * `makePlan` calls the same cases IMPOSSIBLE, by the same rule.
+ */
+export const cannotDropBy = (baseline, mde) => baseline.successes / baseline.trials <= mde;
+/** `options` without its `undefined` entries, so a spread over the defaults leaves those defaults in place. */
+export function definedOptions(options) {
+    return Object.fromEntries(Object.entries(options ?? {}).filter(([, v]) => v !== undefined));
+}
+export const DEFAULT_GATE_OPTIONS = {
+    mde: 0.15,
+    fdr: 0.05,
+    altConcentration: 8,
+    evidence: "bayes",
+};
+/**
+ * Decide which cases regressed.
+ *
+ * Valid at any stopping time. Run every case to a fixed count, or stop each one
+ * the moment it looks decided, or anything in between; the guarantee does not
+ * depend on the rule you used.
+ */
+export function gate(cases, options = {}) {
+    const opts = { ...DEFAULT_GATE_OPTIONS, ...definedOptions(options) };
+    requireOpenProbability(opts.mde, "mde", "gate");
+    requireOpenProbability(opts.fdr, "fdr", "gate");
+    requireEvidence(opts.evidence, "gate");
+    requireAltConcentration(opts.altConcentration, opts.evidence, "gate");
+    const seen = new Set();
+    for (const c of cases) {
+        if (seen.has(c.id)) {
+            throw new PeeksafeError("PEEKSAFE_E_CASE_DUPLICATE", `gate: duplicate case id ${c.id}`, {
+                detail: { id: c.id },
+                hint: "two cases sharing an id would be merged into one baseline entry and counted once",
+            });
+        }
+        seen.add(c.id);
+        if (c.paired) {
+            requirePairedCounts(c.paired, `gate(${c.id}).paired`);
+            continue;
+        }
+        requireCounts(c.successes, c.trials, `gate(${c.id})`);
+        if (c.baseline)
+            requireCounts(c.baseline.successes, c.baseline.trials, `gate(${c.id}).baseline`);
+    }
+    const hasBaseline = (c) => c.paired !== undefined || (c.baseline !== undefined && c.baseline.trials > 0);
+    const newCases = cases.filter((c) => !hasBaseline(c)).map((c) => c.id);
+    const gated = cases.filter(hasBaseline);
+    if (gated.length === 0) {
+        throw new PeeksafeError("PEEKSAFE_E_BASELINE_MISSING", "gate: no case has a baseline, so there is nothing to test against", {
+            detail: { cases: cases.length, newCases: newCases.length },
+            hint: 'record a baseline on the reference revision first; gating without one compares against a Beta(1,1) prior, which reads as "this case passes half the time"',
+        });
+    }
+    const m = gated.length;
+    // Testing these anyway clamps the alternative onto a rate near zero, which is
+    // where the case already sits, and the false positive rate then exceeds `fdr`.
+    // A paired case has an exact null and never needs this.
+    const impossible = gated.map((c) => !c.paired && cannotDropBy(c.baseline, opts.mde));
+    const logEs = gated.map((c, i) => {
+        if (c.paired)
+            return pairedLogE(c.paired.worse, discordant(c.paired));
+        return impossible[i]
+            ? 0
+            : logEvidence(opts.evidence, c.successes, c.trials, c.baseline.successes, c.baseline.trials, opts.mde, opts.altConcentration);
+    });
+    const evalues = logEs.map(toEvalue);
+    const ebh = ebhCorrect(evalues, opts.fdr);
+    const solo = ebhSoloThreshold(m, opts.fdr);
+    const verdicts = gated.map((c, i) => {
+        const shared = { id: c.id, regressed: ebh.rejected[i], evalue: evalues[i], logE: logEs[i] };
+        if (c.paired) {
+            const p = c.paired;
+            const pairs = p.bothPass + p.worse + p.better + p.bothFail;
+            return {
+                ...shared,
+                ceiling: Infinity,
+                undetectable: false,
+                impossible: false,
+                observed: { successes: p.bothPass + p.better, trials: pairs },
+                baseline: { successes: p.bothPass + p.worse, trials: pairs },
+                design: "paired",
+                paired: { ...p },
+            };
+        }
+        // The ceiling is evaluated at the rate a regression would actually sit at,
+        // one MDE below the baseline's own rate. That is the alternative the gate is
+        // trying to detect, so it is the right place to ask "could this ever clear
+        // the bar". Matches how plan.ts prices the same case.
+        const bRate = c.baseline.successes / c.baseline.trials;
+        const pAlt = Math.max(1e-6, Math.min(1 - 1e-6, bRate - opts.mde));
+        const ceiling = impossible[i]
+            ? 1
+            : Math.exp(ceilingLogEvidence(opts.evidence, pAlt, c.baseline.successes, c.baseline.trials, opts.mde, opts.altConcentration));
+        return {
+            ...shared,
+            ceiling,
+            undetectable: !impossible[i] && ceiling < solo,
+            impossible: impossible[i],
+            observed: { successes: c.successes, trials: c.trials },
+            baseline: { successes: c.baseline.successes, trials: c.baseline.trials },
+            design: "unpaired",
+        };
+    });
+    const regressed = verdicts.filter((v) => v.regressed);
+    const blind = verdicts.filter((v) => v.undetectable && !v.regressed);
+    const untested = verdicts.filter((v) => v.impossible);
+    const caveats = [];
+    if (blind.length > 0) {
+        caveats.push(`${blind.length} of ${m} gated case(s) have a baseline too thin to certify a ` +
+            `${(opts.mde * 100).toFixed(0)}pt drop at any candidate budget (${blind
+                .slice(0, 3)
+                .map((v) => v.id)
+                .join(", ")}` +
+            `${blind.length > 3 ? ", …" : ""}), which needs more baseline runs, not more candidate runs`);
+    }
+    if (untested.length > 0) {
+        caveats.push(`${untested.length} case(s) pass at or below ${(opts.mde * 100).toFixed(0)}% and cannot drop by ` +
+            `${(opts.mde * 100).toFixed(0)}pts, so they were not tested (${untested
+                .slice(0, 3)
+                .map((v) => v.id)
+                .join(", ")}` +
+            `${untested.length > 3 ? ", …" : ""})`);
+    }
+    if (newCases.length > 0) {
+        caveats.push(`${newCases.length} case(s) have no baseline and were not gated ` +
+            `(${newCases.slice(0, 3).join(", ")}${newCases.length > 3 ? ", …" : ""})`);
+    }
+    const headline = (regressed.length === 0
+        ? `PASS: no case cleared the bar of ${solo.toFixed(0)} at ${(opts.fdr * 100).toFixed(0)}% FDR over ${m} gated case(s)`
+        : `FAIL: ${regressed.length} of ${m} gated case(s) certified as regressed at ${(opts.fdr * 100).toFixed(0)}% FDR ` +
+            `(${regressed
+                .slice(0, 3)
+                .map((v) => v.id)
+                .join(", ")}${regressed.length > 3 ? ", …" : ""})`) +
+        (caveats.length > 0 ? `. Read with care: ${caveats.join("; ")}.` : "");
+    return {
+        verdict: regressed.length === 0 ? "PASS" : "FAIL",
+        cases: verdicts,
+        regressed,
+        newCases,
+        threshold: ebh.threshold,
+        soloThreshold: solo,
+        options: opts,
+        headline,
+    };
+}
+//# sourceMappingURL=gate.js.map

package/dist/gate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gate.js","sourceRoot":"","sources":["../src/gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAGH,OAAO,EACL,aAAa,EACb,uBAAuB,EACvB,aAAa,EACb,sBAAsB,EACtB,mBAAmB,GACpB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,kBAAkB,EAElB,WAAW,EACX,eAAe,EACf,QAAQ,GACT,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,UAAU,EACV,UAAU,EACV,gBAAgB,EAEhB,UAAU,GACX,MAAM,YAAY,CAAC;AAEpB;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,QAA+C,EAC/C,GAAW,EACF,EAAE,CAAC,QAAQ,CAAC,SAAS,GAAG,QAAQ,CAAC,MAAM,IAAI,GAAG,CAAC;AAE1D,+GAA+G;AAC/G,MAAM,UAAU,cAAc,CAAmB,OAAU;IACzD,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAM,CAAC;AACnG,CAAC;AAyCD,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,GAAG,EAAE,IAAI;IACT,GAAG,EAAE,IAAI;IACT,gBAAgB,EAAE,CAAC;IACnB,QAAQ,EAAE,OAAO;CACT,CAAC;AA4DX;;;;;;GAMG;AACH,MAAM,UAAU,IAAI,CAClB,KAA6C,EAC7C,OAAO,GAAgB,EAAE;IAEzB,MAAM,IAAI,GAA0B,EAAE,GAAG,oBAAoB,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;IAC5F,sBAAsB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAChD,sBAAsB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAChD,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACvC,uBAAuB,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEtE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,aAAa,CAAC,2BAA2B,EAAE,2BAA2B,CAAC,CAAC,EAAE,EAAE,EAAE;gBACtF,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE;gBACpB,IAAI,EAAE,kFAAkF;aACzF,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACf,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACb,mBAAmB,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;YACtD,SAAS;QACX,CAAC;QACD,aAAa,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACtD,IAAI,CAAC,CAAC,QAAQ;YACZ,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACrF,CAAC;IAED,MAAM,WAAW,GAAG,CAAC,CAA4B,EAAW,EAAE,CAC5D,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAChF,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAExC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,aAAa,CACrB,6BAA6B,EAC7B,mEAAmE,EACnE;YACE,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE;YAC1D,IAAI,EAAE,2JAA2J;SAClK,CACF,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,8EAA8E;IAC9E,+EAA+E;IAC/E,wDAAwD;IACxD,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,YAAY,CAAC,CAAC,CAAC,QAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACtF,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC/B,IAAI,CAAC,CAAC,MAAM;YAAE,OAAO,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QACtE,OAAO,UAAU,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,WAAW,CACT,IAAI,CAAC,QAAQ,EACb,CAAC,CAAC,SAAS,EACX,CAAC,CAAC,MAAM,EACR,CAAC,CAAC,QAAS,CAAC,SAAS,EACrB,CAAC,CAAC,QAAS,CAAC,MAAM,EAClB,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,gBAAgB,CACtB,CAAC;IACR,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAE3C,MAAM,QAAQ,GAAkB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACjD,MAAM,MAAM,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAE,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAE,EAAE,CAAC;QAC/F,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;YACnB,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC3D,OAAO;gBACL,GAAG,MAAM;gBACT,OAAO,EAAE,QAAQ;gBACjB,YAAY,EAAE,KAAK;gBACnB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE;gBAC7D,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;gBAC5D,MAAM,EAAE,QAAQ;gBAChB,MAAM,EAAE,EAAE,GAAG,CAAC,EAAE;aACjB,CAAC;QACJ,CAAC;QACD,2EAA2E;QAC3E,6EAA6E;QAC7E,2EAA2E;QAC3E,sDAAsD;QACtD,MAAM,KAAK,GAAG,CAAC,CAAC,QAAS,CAAC,SAAS,GAAG,CAAC,CAAC,QAAS,CAAC,MAAM,CAAC;QACzD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC;YAC3B,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,IAAI,CAAC,GAAG,CACN,kBAAkB,CAChB,IAAI,CAAC,QAAQ,EACb,IAAI,EACJ,CAAC,CAAC,QAAS,CAAC,SAAS,EACrB,CAAC,CAAC,QAAS,CAAC,MAAM,EAClB,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,gBAAgB,CACtB,CACF,CAAC;QACN,OAAO;YACL,GAAG,MAAM;YACT,OAAO;YACP,YAAY,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,OAAO,GAAG,IAAI;YAC9C,UAAU,EAAE,UAAU,CAAC,CAAC,CAAE;YAC1B,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;YACtD,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,QAAS,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,QAAS,CAAC,MAAM,EAAE;YAC1E,MAAM,EAAE,UAAU;SACnB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACrE,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAEtD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CACV,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,uDAAuD;YAC5E,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,oCAAoC,KAAK;iBACpE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;iBACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAChB,IAAI,CAAC,IAAI,CAAC,EAAE;YACf,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,4DAA4D,CAC/F,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CACV,GAAG,QAAQ,CAAC,MAAM,6BAA6B,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB;YAC/F,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,iCAAiC,QAAQ;iBACpE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;iBACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAChB,IAAI,CAAC,IAAI,CAAC,EAAE;YACf,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CACzC,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CACV,GAAG,QAAQ,CAAC,MAAM,+CAA+C;YAC/D,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GACZ,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;QACrB,CAAC,CAAC,oCAAoC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,gBAAgB;QACtH,CAAC,CAAC,SAAS,SAAS,CAAC,MAAM,OAAO,CAAC,4CAA4C,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ;YAChH,IAAI,SAAS;iBACV,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;iBACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAChB,IAAI,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;QACzD,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEzE,OAAO;QACL,OAAO,EAAE,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QACjD,KAAK,EAAE,QAAQ;QACf,SAAS;QACT,QAAQ;QACR,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE,IAAI;QACb,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,60 @@
+/**
+ * peeksafe: statistically valid gates for non-deterministic eval suites.
+ *
+ * The problem this exists for: you run an eval suite, watch results arrive, stop
+ * a case when it looks decided, then correct for multiplicity with
+ * Benjamini-Hochberg. That construction has no error control. A p-value computed
+ * at a stopping boundary is not a p-value, and BH assumes it is.
+ *
+ * The library is organised in four layers, and most callers only need the first.
+ *
+ * ## 1. The decision
+ *
+ *   gate(cases, options)   which cases regressed, valid at any stopping time
+ *
+ * You bring counts. It brings e-values and e-BH. Nothing here knows or cares how
+ * you ran the evals, and nothing here will gate a case that has no baseline.
+ *
+ * ## 2. The budget
+ *
+ *   makePlan(...)          how many runs each case needs, and which are impossible
+ *   computeFrontier(...)   what is certifiable inside a budget
+ *   affordabilityGrid(...) the same question as a grid over two axes
+ *
+ * These answer "how many runs do I need" before you spend anything, including
+ * the case where the honest answer is that no candidate budget will do and only
+ * more baseline runs will.
+ *
+ * ## 3. The check
+ *
+ *   typeOneError(...)              the error rate the gate delivers at rates you name
+ *   certifyProbability(...)        the same computation with separate baseline and candidate rates
+ *   pairedCertifyProbability(...)  the power of a paired design
+ *
+ * These are exact, not simulated, so a caller can check the gate on their own
+ * rates before trusting a verdict.
+ *
+ * ## 4. The statistics
+ *
+ * Everything the first two layers are built from, exported so a caller can check
+ * the arithmetic, build a different gate, or use one piece on its own: special
+ * functions, intervals, the SPRT, e-values, e-BH, power, paired designs, and the
+ * cluster-robust suite summary.
+ *
+ * Cases in a suite are not independent draws. Several cases generated from one
+ * source file share a failure mode, so a change that breaks the file moves all
+ * of them together. `clusteredEffect` and the rest of `cluster` are what stop
+ * the suite-level interval from being too narrow for that reason.
+ */
+export { type BaselineStat, baselineNullRate, type CaseRef, toBaselineMap, } from "./baseline.js";
+export { type CertifyProbabilityOptions, certifyProbability, type PairedCertifyProbabilityOptions, pairedCertifyProbability, type TypeOneErrorOptions, typeOneError, } from "./check.js";
+export { type ClusteredEffect, type ClusteredMean, type ClusterKeyDiagnostic, type ClusterObservation, caseFamily, clusteredEffect, clusterKey, clusterKeyDiagnostic, clusterRobustMean, compareEstimators, type FamilyEffect, type GroupingSummary, groupByCluster, iidMean, MIN_TRUSTWORTHY_CLUSTERS, type RandomEffectsMean, randomEffectsMean, } from "./cluster.js";
+export { MAX_ALT_CONCENTRATION, PeeksafeError, type PeeksafeErrorCode, type PeeksafeErrorOptions, } from "./errors.js";
+export type { Evidence } from "./evidence.js";
+export { computeFrontier, costShares, DEFAULT_AXES, DEFAULT_FRONTIER, enumerateFrontier, evaluatePoint, type FrontierAxes, type FrontierBasis, type FrontierCell, type FrontierConfig, type FrontierCost, type FrontierDesign, type FrontierPoint, type FrontierResult, type FrontierScreen, fmtRate, typicalObservationsPerCase, } from "./frontier.js";
+export { type CaseVerdict, DEFAULT_GATE_OPTIONS, type GateCase, type GateOptions, type GateResult, gate, type PairedGateCase, } from "./gate.js";
+export { type AffordabilityCell, affordabilityGrid, DEFAULT_PLAN, type Detectability, expectedSequentialSamples, makePlan, type Plan, type PlanCase, type PlanConfig, type PlanTotals, planCase, } from "./plan.js";
+export { hash32, makeRand, type Rand, streamFor } from "./rand.js";
+export { type BetaPosterior, type BhResult, bernoulliEntropy, betaCredibleInterval, betaMassBetween, betaPosterior, betaQuantile, bhCorrect, cohensH, diffInterval, discordant, type EbhResult, ebhCorrect, ebhSoloThreshold, erf, evidenceCeilingAsymptotic, evidenceCeilingLogE, evidenceCeilingSlope, expectedLogE, expectedLogEExact, fisherExact2x2, gammaP, type Interval, ibeta, logBeta, logBetaPdf, logGamma, logMarginalBetaBinomial, mcnemarSamplesForEvidence, normalCdf, normalQuantile, type PairedCounts, pairedDiscordance, pairedLogE, pairPhi, probabilityMoved, type SprtDecision, type SprtOpts, type SprtResult, sampleSizeTwoProportion, samplesForEvidence, sprtDecision, sprtExpectedN, type TestResult, twoProportionZTest, twoSampleLogE, twoSamplePriors, universalCeilingLogE, universalTwoSampleLogE, wilsonInterval, } from "./stats.js";
+export { DEFAULT_STOP_OPTIONS, type PairedStopDecision, type PairedStopOptions, type PairedStopReason, type StopDecision, type StopOptions, type StopReason, shouldStop, shouldStopPaired, } from "./stop.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EACL,KAAK,YAAY,EACjB,gBAAgB,EAChB,KAAK,OAAO,EACZ,aAAa,GACd,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,KAAK,yBAAyB,EAC9B,kBAAkB,EAClB,KAAK,+BAA+B,EACpC,wBAAwB,EACxB,KAAK,mBAAmB,EACxB,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,UAAU,EACV,eAAe,EACf,UAAU,EACV,oBAAoB,EACpB,iBAAiB,EACjB,iBAAiB,EACjB,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,cAAc,EACd,OAAO,EACP,wBAAwB,EACxB,KAAK,iBAAiB,EACtB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,qBAAqB,EACrB,aAAa,EACb,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,GAC1B,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EACL,eAAe,EACf,UAAU,EACV,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,OAAO,EACP,0BAA0B,GAC3B,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,KAAK,WAAW,EAChB,oBAAoB,EACpB,KAAK,QAAQ,EACb,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,IAAI,EACJ,KAAK,cAAc,GACpB,MAAM,WAAW,CAAC;AAEnB,OAAO,EACL,KAAK,iBAAiB,EACtB,iBAAiB,EACjB,YAAY,EACZ,KAAK,aAAa,EAClB,yBAAyB,EACzB,QAAQ,EACR,KAAK,IAAI,EACT,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,UAAU,EACf,QAAQ,GACT,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEnE,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,QAAQ,EACb,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,EACf,aAAa,EACb,YAAY,EAEZ,SAAS,EACT,OAAO,EACP,YAAY,EACZ,UAAU,EACV,KAAK,SAAS,EACd,UAAU,EACV,gBAAgB,EAChB,GAAG,EACH,yBAAyB,EACzB,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,MAAM,EACN,KAAK,QAAQ,EACb,KAAK,EACL,OAAO,EACP,UAAU,EAEV,QAAQ,EAER,uBAAuB,EACvB,yBAAyB,EACzB,SAAS,EACT,cAAc,EACd,KAAK,YAAY,EACjB,iBAAiB,EAEjB,UAAU,EACV,OAAO,EACP,gBAAgB,EAChB,KAAK,YAAY,EACjB,KAAK,QAAQ,EACb,KAAK,UAAU,EAEf,uBAAuB,EACvB,kBAAkB,EAElB,YAAY,EACZ,aAAa,EACb,KAAK,UAAU,EAEf,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,oBAAoB,EACpB,sBAAsB,EAEtB,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,oBAAoB,EACpB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,UAAU,EACV,gBAAgB,GACjB,MAAM,WAAW,CAAC"}

package/dist/index.js

@@ -0,0 +1,79 @@
+/**
+ * peeksafe: statistically valid gates for non-deterministic eval suites.
+ *
+ * The problem this exists for: you run an eval suite, watch results arrive, stop
+ * a case when it looks decided, then correct for multiplicity with
+ * Benjamini-Hochberg. That construction has no error control. A p-value computed
+ * at a stopping boundary is not a p-value, and BH assumes it is.
+ *
+ * The library is organised in four layers, and most callers only need the first.
+ *
+ * ## 1. The decision
+ *
+ *   gate(cases, options)   which cases regressed, valid at any stopping time
+ *
+ * You bring counts. It brings e-values and e-BH. Nothing here knows or cares how
+ * you ran the evals, and nothing here will gate a case that has no baseline.
+ *
+ * ## 2. The budget
+ *
+ *   makePlan(...)          how many runs each case needs, and which are impossible
+ *   computeFrontier(...)   what is certifiable inside a budget
+ *   affordabilityGrid(...) the same question as a grid over two axes
+ *
+ * These answer "how many runs do I need" before you spend anything, including
+ * the case where the honest answer is that no candidate budget will do and only
+ * more baseline runs will.
+ *
+ * ## 3. The check
+ *
+ *   typeOneError(...)              the error rate the gate delivers at rates you name
+ *   certifyProbability(...)        the same computation with separate baseline and candidate rates
+ *   pairedCertifyProbability(...)  the power of a paired design
+ *
+ * These are exact, not simulated, so a caller can check the gate on their own
+ * rates before trusting a verdict.
+ *
+ * ## 4. The statistics
+ *
+ * Everything the first two layers are built from, exported so a caller can check
+ * the arithmetic, build a different gate, or use one piece on its own: special
+ * functions, intervals, the SPRT, e-values, e-BH, power, paired designs, and the
+ * cluster-robust suite summary.
+ *
+ * Cases in a suite are not independent draws. Several cases generated from one
+ * source file share a failure mode, so a change that breaks the file moves all
+ * of them together. `clusteredEffect` and the rest of `cluster` are what stop
+ * the suite-level interval from being too narrow for that reason.
+ */
+export { baselineNullRate, toBaselineMap, } from "./baseline.js";
+export { certifyProbability, pairedCertifyProbability, typeOneError, } from "./check.js";
+export { caseFamily, clusteredEffect, clusterKey, clusterKeyDiagnostic, clusterRobustMean, compareEstimators, groupByCluster, iidMean, MIN_TRUSTWORTHY_CLUSTERS, randomEffectsMean, } from "./cluster.js";
+/* ── errors and determinism ──────────────────────────────────────────────── */
+export { MAX_ALT_CONCENTRATION, PeeksafeError, } from "./errors.js";
+export { computeFrontier, costShares, DEFAULT_AXES, DEFAULT_FRONTIER, enumerateFrontier, evaluatePoint, fmtRate, typicalObservationsPerCase, } from "./frontier.js";
+/* ── 1. the decision ─────────────────────────────────────────────────────── */
+export { DEFAULT_GATE_OPTIONS, gate, } from "./gate.js";
+/* ── 2. the budget ───────────────────────────────────────────────────────── */
+export { affordabilityGrid, DEFAULT_PLAN, expectedSequentialSamples, makePlan, planCase, } from "./plan.js";
+export { hash32, makeRand, streamFor } from "./rand.js";
+/* ── 3. the statistics ───────────────────────────────────────────────────── */
+export { bernoulliEntropy, betaCredibleInterval, betaMassBetween, betaPosterior, betaQuantile, 
+/* multiplicity */
+bhCorrect, cohensH, diffInterval, discordant, ebhCorrect, ebhSoloThreshold, erf, evidenceCeilingAsymptotic, evidenceCeilingLogE, evidenceCeilingSlope, expectedLogE, expectedLogEExact, fisherExact2x2, gammaP, ibeta, logBeta, logBetaPdf, 
+/* special functions */
+logGamma, 
+/* e-values */
+logMarginalBetaBinomial, mcnemarSamplesForEvidence, normalCdf, normalQuantile, pairedDiscordance, 
+/* paired designs */
+pairedLogE, pairPhi, probabilityMoved, 
+/* power */
+sampleSizeTwoProportion, samplesForEvidence, 
+/* sequential */
+sprtDecision, sprtExpectedN, 
+/* fixed-sample tests, for comparison rather than for gating */
+twoProportionZTest, twoSampleLogE, twoSamplePriors, universalCeilingLogE, universalTwoSampleLogE, 
+/* intervals and posteriors */
+wilsonInterval, } from "./stats.js";
+export { DEFAULT_STOP_OPTIONS, shouldStop, shouldStopPaired, } from "./stop.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAEL,gBAAgB,EAEhB,aAAa,GACd,MAAM,eAAe,CAAC;AACvB,OAAO,EAEL,kBAAkB,EAElB,wBAAwB,EAExB,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,OAAO,EAKL,UAAU,EACV,eAAe,EACf,UAAU,EACV,oBAAoB,EACpB,iBAAiB,EACjB,iBAAiB,EAGjB,cAAc,EACd,OAAO,EACP,wBAAwB,EAExB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,gFAAgF;AAChF,OAAO,EACL,qBAAqB,EACrB,aAAa,GAGd,MAAM,aAAa,CAAC;AAErB,OAAO,EACL,eAAe,EACf,UAAU,EACV,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EAUb,OAAO,EACP,0BAA0B,GAC3B,MAAM,eAAe,CAAC;AACvB,gFAAgF;AAChF,OAAO,EAEL,oBAAoB,EAIpB,IAAI,GAEL,MAAM,WAAW,CAAC;AACnB,gFAAgF;AAChF,OAAO,EAEL,iBAAiB,EACjB,YAAY,EAEZ,yBAAyB,EACzB,QAAQ,EAKR,QAAQ,GACT,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAa,SAAS,EAAE,MAAM,WAAW,CAAC;AACnE,gFAAgF;AAChF,OAAO,EAGL,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,EACf,aAAa,EACb,YAAY;AACZ,kBAAkB;AAClB,SAAS,EACT,OAAO,EACP,YAAY,EACZ,UAAU,EAEV,UAAU,EACV,gBAAgB,EAChB,GAAG,EACH,yBAAyB,EACzB,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,MAAM,EAEN,KAAK,EACL,OAAO,EACP,UAAU;AACV,uBAAuB;AACvB,QAAQ;AACR,cAAc;AACd,uBAAuB,EACvB,yBAAyB,EACzB,SAAS,EACT,cAAc,EAEd,iBAAiB;AACjB,oBAAoB;AACpB,UAAU,EACV,OAAO,EACP,gBAAgB;AAIhB,WAAW;AACX,uBAAuB,EACvB,kBAAkB;AAClB,gBAAgB;AAChB,YAAY,EACZ,aAAa;AAEb,+DAA+D;AAC/D,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,oBAAoB,EACpB,sBAAsB;AACtB,8BAA8B;AAC9B,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,oBAAoB,EAOpB,UAAU,EACV,gBAAgB,GACjB,MAAM,WAAW,CAAC"}