peeksafe 0.0.0-stage → 0.1.0

package/src/gate.ts

@@ -0,0 +1,358 @@
+/**
+ * gate.ts: the decision.
+ *
+ * You bring counts. This decides which cases regressed, at a false discovery
+ * rate you name, in a way that stays valid however many times you looked at the
+ * data on the way here.
+ *
+ * ## Why this is safe to peek at
+ *
+ * The usual construction is: run the suite, stop a case when it looks
+ * significant, compute a p-value from the counts, hand the p-values to
+ * Benjamini-Hochberg. That is invalid in two independent places.
+ *
+ * A p-value computed at a stopping boundary is not a p-value. `P(p_n <= a) <= a`
+ * holds for each fixed `n`; a procedure that adds runs and re-tests until the
+ * result looks significant reports `min` over the peeks, and that minimum is not
+ * bounded by `a`. Benjamini-Hochberg takes `P(p <= t) <= t` as its input
+ * assumption, so it inherits nothing and controls nothing.
+ *
+ * This module never computes a p-value. It computes an e-value, which is a
+ * non-negative statistic with mean at most 1 under the null. Ville's inequality
+ * bounds `P(sup_n E_n >= 1/a) <= a` over the whole trajectory, so an e-value is
+ * valid at any stopping time, including one chosen after looking. (The mean is
+ * exactly 1 averaged over the shared rate, not at every fixed rate; the README
+ * section "What that guarantee rests on" says where the bound is checked.) The
+ * multiplicity correction is e-BH (Wang and Ramdas), which takes e-values
+ * directly and controls FDR under arbitrary dependence between cases.
+ *
+ * The practical consequence: you may stop a case whenever you like, for any
+ * reason, including "it already looks bad". Nothing here cares when you stopped.
+ *
+ * ## What it will refuse to do
+ *
+ * Gate a case with no baseline. There is no null to test against, and the
+ * failure mode of inventing one is not a wrong number, it is a confident wrong
+ * number: a missing baseline read as `0/0` has a Beta(1,1) posterior median of
+ * 0.5, which gates the case against "it passes half the time" and reports a
+ * large improvement against nothing at all. Cases without a baseline come back
+ * in `newCases` and take no part in the verdict or the e-BH family.
+ *
+ * ## Paired cases
+ *
+ * A case may instead carry `paired` counts from re-running the baseline next to
+ * the candidate, one baseline run per candidate run. Its e-value is
+ * `pairedLogE` on the discordant pairs, which is valid at every rate by
+ * construction and has no ceiling, whichever `evidence` the unpaired cases use.
+ * The pairs need not share a seed: independent runs paired in order are still
+ * valid, and a shared seed only makes discordances more informative.
+ */
+
+import type { BaselineStat } from "./baseline.js";
+import {
+  PeeksafeError,
+  requireAltConcentration,
+  requireCounts,
+  requireOpenProbability,
+  requirePairedCounts,
+} from "./errors.js";
+import {
+  ceilingLogEvidence,
+  type Evidence,
+  logEvidence,
+  requireEvidence,
+  toEvalue,
+} from "./evidence.js";
+import {
+  discordant,
+  ebhCorrect,
+  ebhSoloThreshold,
+  type PairedCounts,
+  pairedLogE,
+} from "./stats.js";
+
+/**
+ * True when the baseline's observed rate leaves no room for an `mde`-sized drop.
+ * `makePlan` calls the same cases IMPOSSIBLE, by the same rule.
+ */
+export const cannotDropBy = (
+  baseline: { successes: number; trials: number },
+  mde: number,
+): boolean => baseline.successes / baseline.trials <= mde;
+
+/** One case's candidate result, and the baseline it is measured against. */
+export interface GateCase {
+  id: string;
+  /** Successes observed for the revision under test. */
+  successes: number;
+  /** Trials run for the revision under test. Stop whenever you like. */
+  trials: number;
+  /** The reference revision's recorded counts, or undefined if there are none. */
+  baseline?: BaselineStat;
+  paired?: undefined;
+}
+
+/** One case run as pairs: each candidate run next to a fresh baseline run. */
+export interface PairedGateCase {
+  id: string;
+  /** The pair table so far. Stop whenever you like. */
+  paired: PairedCounts;
+}
+
+export interface GateOptions {
+  /** Minimum detectable effect: the drop, in rate, that counts as a regression. */
+  mde?: number;
+  /** Target false discovery rate across the suite. */
+  fdr?: number;
+  /**
+   * Concentration of the alternative prior. Lower spreads the alternative over
+   * "anything materially worse", which is closer to how a broken case behaves
+   * than a point mass exactly `mde` below the baseline. At most
+   * `MAX_ALT_CONCENTRATION` (16) with the default `bayes` statistic.
+   */
+  altConcentration?: number;
+  /**
+   * Which e-value decides. `bayes` has more power and error control checked
+   * numerically; `universal` is valid at every rate by construction and needs
+   * more runs. See the README section "What that guarantee rests on".
+   */
+  evidence?: Evidence;
+}
+
+export const DEFAULT_GATE_OPTIONS = {
+  mde: 0.15,
+  fdr: 0.05,
+  altConcentration: 8,
+  evidence: "bayes",
+} as const;
+
+export interface CaseVerdict {
+  id: string;
+  /** True when e-BH certifies this case as regressed at the suite's FDR. */
+  regressed: boolean;
+  /** The e-value. Larger is more evidence against the null. */
+  evalue: number;
+  /** log of the above, which is what the arithmetic actually uses. */
+  logE: number;
+  /**
+   * The largest e-value this case could ever reach against its current
+   * baseline, however many candidate runs you add. When this is below
+   * `threshold`, the case is undetectable at any candidate budget and the only
+   * remedy is more baseline runs.
+   */
+  ceiling: number;
+  /** True when `ceiling` is below the bar the case has to clear. */
+  undetectable: boolean;
+  /**
+   * True when the baseline rate is at or below `mde`, so an `mde`-sized drop
+   * cannot happen. The case is not tested: its e-value is fixed at 1, which
+   * keeps it in the e-BH family without it ever being certified.
+   */
+  impossible: boolean;
+  /** For a paired case, each arm's passes over the number of pairs. */
+  observed: { successes: number; trials: number };
+  baseline: { successes: number; trials: number };
+  design: "unpaired" | "paired";
+  /** The pair table, for a paired case. */
+  paired?: PairedCounts;
+}
+
+export interface GateResult {
+  /** PASS when nothing was certified as regressed. */
+  verdict: "PASS" | "FAIL";
+  cases: CaseVerdict[];
+  /** Cases certified as regressed, in the order given. */
+  regressed: CaseVerdict[];
+  /**
+   * Cases with no baseline. These are reported, not gated: they have no null to
+   * be tested against and they are excluded from the e-BH family.
+   */
+  newCases: string[];
+  /** The e-value a case had to beat, given how many discoveries were made. */
+  threshold: number;
+  /** The e-value a case must reach to be certified whatever the others do. */
+  soloThreshold: number;
+  options: Required<GateOptions>;
+  /**
+   * One line an operator can read without knowing what an e-value is.
+   *
+   * Names the two things a passing suite can hide: cases with no baseline,
+   * which were not gated at all, and cases whose baseline is too thin to
+   * certify an `mde`-sized drop however many runs you add. A PASS that is
+   * really "nothing was checked" should not read like a PASS.
+   */
+  headline: string;
+}
+
+/**
+ * Decide which cases regressed.
+ *
+ * Valid at any stopping time. Run every case to a fixed count, or stop each one
+ * the moment it looks decided, or anything in between; the guarantee does not
+ * depend on the rule you used.
+ */
+export function gate(
+  cases: readonly (GateCase | PairedGateCase)[],
+  options: GateOptions = {},
+): GateResult {
+  const opts: Required<GateOptions> = { ...DEFAULT_GATE_OPTIONS, ...options };
+  requireOpenProbability(opts.mde, "mde", "gate");
+  requireOpenProbability(opts.fdr, "fdr", "gate");
+  requireEvidence(opts.evidence, "gate");
+  requireAltConcentration(opts.altConcentration, opts.evidence, "gate");
+
+  const seen = new Set<string>();
+  for (const c of cases) {
+    if (seen.has(c.id)) {
+      throw new PeeksafeError("PEEKSAFE_E_CASE_DUPLICATE", `gate: duplicate case id ${c.id}`, {
+        detail: { id: c.id },
+        hint: "two cases sharing an id would be merged into one baseline entry and counted once",
+      });
+    }
+    seen.add(c.id);
+    if (c.paired) {
+      requirePairedCounts(c.paired, `gate(${c.id}).paired`);
+      continue;
+    }
+    requireCounts(c.successes, c.trials, `gate(${c.id})`);
+    if (c.baseline)
+      requireCounts(c.baseline.successes, c.baseline.trials, `gate(${c.id}).baseline`);
+  }
+
+  const hasBaseline = (c: GateCase | PairedGateCase): boolean =>
+    c.paired !== undefined || (c.baseline !== undefined && c.baseline.trials > 0);
+  const newCases = cases.filter((c) => !hasBaseline(c)).map((c) => c.id);
+  const gated = cases.filter(hasBaseline);
+
+  if (gated.length === 0) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_BASELINE_MISSING",
+      "gate: no case has a baseline, so there is nothing to test against",
+      {
+        detail: { cases: cases.length, newCases: newCases.length },
+        hint: 'record a baseline on the reference revision first; gating without one compares against a Beta(1,1) prior, which reads as "this case passes half the time"',
+      },
+    );
+  }
+
+  const m = gated.length;
+  // Testing these anyway clamps the alternative onto a rate near zero, which is
+  // where the case already sits, and the false positive rate then exceeds `fdr`.
+  // A paired case has an exact null and never needs this.
+  const impossible = gated.map((c) => !c.paired && cannotDropBy(c.baseline!, opts.mde));
+  const logEs = gated.map((c, i) => {
+    if (c.paired) return pairedLogE(c.paired.worse, discordant(c.paired));
+    return impossible[i]
+      ? 0
+      : logEvidence(
+          opts.evidence,
+          c.successes,
+          c.trials,
+          c.baseline!.successes,
+          c.baseline!.trials,
+          opts.mde,
+          opts.altConcentration,
+        );
+  });
+  const evalues = logEs.map(toEvalue);
+  const ebh = ebhCorrect(evalues, opts.fdr);
+  const solo = ebhSoloThreshold(m, opts.fdr);
+
+  const verdicts: CaseVerdict[] = gated.map((c, i) => {
+    const shared = { id: c.id, regressed: ebh.rejected[i]!, evalue: evalues[i]!, logE: logEs[i]! };
+    if (c.paired) {
+      const p = c.paired;
+      const pairs = p.bothPass + p.worse + p.better + p.bothFail;
+      return {
+        ...shared,
+        ceiling: Infinity,
+        undetectable: false,
+        impossible: false,
+        observed: { successes: p.bothPass + p.better, trials: pairs },
+        baseline: { successes: p.bothPass + p.worse, trials: pairs },
+        design: "paired",
+        paired: { ...p },
+      };
+    }
+    // The ceiling is evaluated at the rate a regression would actually sit at,
+    // one MDE below the baseline's own rate. That is the alternative the gate is
+    // trying to detect, so it is the right place to ask "could this ever clear
+    // the bar". Matches how plan.ts prices the same case.
+    const bRate = c.baseline!.successes / c.baseline!.trials;
+    const pAlt = Math.max(1e-6, Math.min(1 - 1e-6, bRate - opts.mde));
+    const ceiling = impossible[i]
+      ? 1
+      : Math.exp(
+          ceilingLogEvidence(
+            opts.evidence,
+            pAlt,
+            c.baseline!.successes,
+            c.baseline!.trials,
+            opts.mde,
+            opts.altConcentration,
+          ),
+        );
+    return {
+      ...shared,
+      ceiling,
+      undetectable: !impossible[i] && ceiling < solo,
+      impossible: impossible[i]!,
+      observed: { successes: c.successes, trials: c.trials },
+      baseline: { successes: c.baseline!.successes, trials: c.baseline!.trials },
+      design: "unpaired",
+    };
+  });
+
+  const regressed = verdicts.filter((v) => v.regressed);
+  const blind = verdicts.filter((v) => v.undetectable && !v.regressed);
+  const untested = verdicts.filter((v) => v.impossible);
+
+  const caveats: string[] = [];
+  if (blind.length > 0) {
+    caveats.push(
+      `${blind.length} of ${m} gated case(s) have a baseline too thin to certify a ` +
+        `${(opts.mde * 100).toFixed(0)}pt drop at any candidate budget (${blind
+          .slice(0, 3)
+          .map((v) => v.id)
+          .join(", ")}` +
+        `${blind.length > 3 ? ", …" : ""}), which needs more baseline runs, not more candidate runs`,
+    );
+  }
+  if (untested.length > 0) {
+    caveats.push(
+      `${untested.length} case(s) pass at or below ${(opts.mde * 100).toFixed(0)}% and cannot drop by ` +
+        `${(opts.mde * 100).toFixed(0)}pts, so they were not tested (${untested
+          .slice(0, 3)
+          .map((v) => v.id)
+          .join(", ")}` +
+        `${untested.length > 3 ? ", …" : ""})`,
+    );
+  }
+  if (newCases.length > 0) {
+    caveats.push(
+      `${newCases.length} case(s) have no baseline and were not gated ` +
+        `(${newCases.slice(0, 3).join(", ")}${newCases.length > 3 ? ", …" : ""})`,
+    );
+  }
+
+  const headline =
+    (regressed.length === 0
+      ? `PASS: no case cleared the bar of ${solo.toFixed(0)} at ${(opts.fdr * 100).toFixed(0)}% FDR over ${m} gated case(s)`
+      : `FAIL: ${regressed.length} of ${m} gated case(s) certified as regressed at ${(opts.fdr * 100).toFixed(0)}% FDR ` +
+        `(${regressed
+          .slice(0, 3)
+          .map((v) => v.id)
+          .join(", ")}${regressed.length > 3 ? ", …" : ""})`) +
+    (caveats.length > 0 ? `. Read with care: ${caveats.join("; ")}.` : "");
+
+  return {
+    verdict: regressed.length === 0 ? "PASS" : "FAIL",
+    cases: verdicts,
+    regressed,
+    newCases,
+    threshold: ebh.threshold,
+    soloThreshold: solo,
+    options: opts,
+    headline,
+  };
+}

package/src/index.ts

@@ -0,0 +1,206 @@
+/**
+ * peeksafe: statistically valid gates for non-deterministic eval suites.
+ *
+ * The problem this exists for: you run an eval suite, watch results arrive, stop
+ * a case when it looks decided, then correct for multiplicity with
+ * Benjamini-Hochberg. That construction has no error control. A p-value computed
+ * at a stopping boundary is not a p-value, and BH assumes it is.
+ *
+ * The library is organised in four layers, and most callers only need the first.
+ *
+ * ## 1. The decision
+ *
+ *   gate(cases, options)   which cases regressed, valid at any stopping time
+ *
+ * You bring counts. It brings e-values and e-BH. Nothing here knows or cares how
+ * you ran the evals, and nothing here will gate a case that has no baseline.
+ *
+ * ## 2. The budget
+ *
+ *   makePlan(...)          how many runs each case needs, and which are impossible
+ *   computeFrontier(...)   what is certifiable inside a budget
+ *   affordabilityGrid(...) the same question as a grid over two axes
+ *
+ * These answer "how many runs do I need" before you spend anything, including
+ * the case where the honest answer is that no candidate budget will do and only
+ * more baseline runs will.
+ *
+ * ## 3. The check
+ *
+ *   typeOneError(...)              the error rate the gate delivers at rates you name
+ *   certifyProbability(...)        the same computation with separate baseline and candidate rates
+ *   pairedCertifyProbability(...)  the power of a paired design
+ *
+ * These are exact, not simulated, so a caller can check the gate on their own
+ * rates before trusting a verdict.
+ *
+ * ## 4. The statistics
+ *
+ * Everything the first two layers are built from, exported so a caller can check
+ * the arithmetic, build a different gate, or use one piece on its own: special
+ * functions, intervals, the SPRT, e-values, e-BH, power, paired designs, and the
+ * cluster-robust suite summary.
+ *
+ * Cases in a suite are not independent draws. Several cases generated from one
+ * source file share a failure mode, so a change that breaks the file moves all
+ * of them together. `clusteredEffect` and the rest of `cluster` are what stop
+ * the suite-level interval from being too narrow for that reason.
+ */
+
+export {
+  type BaselineStat,
+  baselineNullRate,
+  type CaseRef,
+  toBaselineMap,
+} from "./baseline.js";
+export {
+  type CertifyProbabilityOptions,
+  certifyProbability,
+  type PairedCertifyProbabilityOptions,
+  pairedCertifyProbability,
+  type TypeOneErrorOptions,
+  typeOneError,
+} from "./check.js";
+export {
+  type ClusteredEffect,
+  type ClusteredMean,
+  type ClusterKeyDiagnostic,
+  type ClusterObservation,
+  caseFamily,
+  clusteredEffect,
+  clusterKey,
+  clusterKeyDiagnostic,
+  clusterRobustMean,
+  compareEstimators,
+  type FamilyEffect,
+  type GroupingSummary,
+  groupByCluster,
+  iidMean,
+  MIN_TRUSTWORTHY_CLUSTERS,
+  type RandomEffectsMean,
+  randomEffectsMean,
+} from "./cluster.js";
+/* ── errors and determinism ──────────────────────────────────────────────── */
+export {
+  MAX_ALT_CONCENTRATION,
+  PeeksafeError,
+  type PeeksafeErrorCode,
+  type PeeksafeErrorOptions,
+} from "./errors.js";
+export type { Evidence } from "./evidence.js";
+export {
+  computeFrontier,
+  costShares,
+  DEFAULT_AXES,
+  DEFAULT_FRONTIER,
+  enumerateFrontier,
+  evaluatePoint,
+  type FrontierAxes,
+  type FrontierBasis,
+  type FrontierCell,
+  type FrontierConfig,
+  type FrontierCost,
+  type FrontierDesign,
+  type FrontierPoint,
+  type FrontierResult,
+  type FrontierScreen,
+  fmtRate,
+  typicalObservationsPerCase,
+} from "./frontier.js";
+/* ── 1. the decision ─────────────────────────────────────────────────────── */
+export {
+  type CaseVerdict,
+  DEFAULT_GATE_OPTIONS,
+  type GateCase,
+  type GateOptions,
+  type GateResult,
+  gate,
+  type PairedGateCase,
+} from "./gate.js";
+/* ── 2. the budget ───────────────────────────────────────────────────────── */
+export {
+  type AffordabilityCell,
+  affordabilityGrid,
+  DEFAULT_PLAN,
+  type Detectability,
+  expectedSequentialSamples,
+  makePlan,
+  type Plan,
+  type PlanCase,
+  type PlanConfig,
+  type PlanTotals,
+  planCase,
+} from "./plan.js";
+export { hash32, makeRand, type Rand, streamFor } from "./rand.js";
+/* ── 3. the statistics ───────────────────────────────────────────────────── */
+export {
+  type BetaPosterior,
+  type BhResult,
+  bernoulliEntropy,
+  betaCredibleInterval,
+  betaMassBetween,
+  betaPosterior,
+  betaQuantile,
+  /* multiplicity */
+  bhCorrect,
+  cohensH,
+  diffInterval,
+  discordant,
+  type EbhResult,
+  ebhCorrect,
+  ebhSoloThreshold,
+  erf,
+  evidenceCeilingAsymptotic,
+  evidenceCeilingLogE,
+  evidenceCeilingSlope,
+  expectedLogE,
+  expectedLogEExact,
+  fisherExact2x2,
+  gammaP,
+  type Interval,
+  ibeta,
+  logBeta,
+  logBetaPdf,
+  /* special functions */
+  logGamma,
+  /* e-values */
+  logMarginalBetaBinomial,
+  mcnemarSamplesForEvidence,
+  normalCdf,
+  normalQuantile,
+  type PairedCounts,
+  pairedDiscordance,
+  /* paired designs */
+  pairedLogE,
+  pairPhi,
+  probabilityMoved,
+  type SprtDecision,
+  type SprtOpts,
+  type SprtResult,
+  /* power */
+  sampleSizeTwoProportion,
+  samplesForEvidence,
+  /* sequential */
+  sprtDecision,
+  sprtExpectedN,
+  type TestResult,
+  /* fixed-sample tests, for comparison rather than for gating */
+  twoProportionZTest,
+  twoSampleLogE,
+  twoSamplePriors,
+  universalCeilingLogE,
+  universalTwoSampleLogE,
+  /* intervals and posteriors */
+  wilsonInterval,
+} from "./stats.js";
+export {
+  DEFAULT_STOP_OPTIONS,
+  type PairedStopDecision,
+  type PairedStopOptions,
+  type PairedStopReason,
+  type StopDecision,
+  type StopOptions,
+  type StopReason,
+  shouldStop,
+  shouldStopPaired,
+} from "./stop.js";