peeksafe 0.1.0 → 0.2.0

package/src/plan.ts

@@ -35,9 +35,13 @@
 import { type BaselineStat, baselineNullRate, type CaseRef } from "./baseline.js";
 import {
   PeeksafeError,
+  requireCounts,
   requireErrorRates,
   requireOpenProbability,
   requireProbability,
+  requireRunCeiling,
+  requireTrialGrid,
+  requireUniqueCaseIds,
 } from "./errors.js";
 import {
   betaQuantile,
@@ -50,6 +54,7 @@ import {
   samplesForEvidence,
   sprtExpectedN,
 } from "./stats.js";
+import { DEFAULT_STOP_OPTIONS, settleLogE, shouldStop } from "./stop.js";
 
 export interface PlanConfig {
   /** the drop we want to be able to detect, in probability units (0.15 = 15 points) */
@@ -74,14 +79,14 @@ export interface PlanConfig {
   /** beyond this many runs for one case we stop calling it "affordable" */
   runsPerCaseCeiling: number;
   /**
-   * Which design the *expected* bill is priced for. A paired observation is two
-   * runs, so this doubles `typicalRuns`.
+   * Which design the *expected* bill and the screening pass are priced for. A
+   * paired observation is two runs, and a paired case's observation count is
+   * the pairs `shouldStopPaired` takes to settle it at its coupling; an
+   * unpaired case's is the runs `shouldStop` takes to stop it.
    *
-   * `undefined` means "price the design this plan recommends", which is the
-   * only self-consistent answer: `makePlan` recommends pairing whenever more
-   * than a quarter of the suite has an unpaired evidence ceiling under the bar,
-   * and quoting the *unpaired* bill underneath that recommendation would price
-   * a design nobody runs.
+   * `undefined` prices each case at its own `PlanCase.design`, and a case with
+   * none at the suite's `recommendedDesign`. Setting it prices every case at
+   * that one design.
    */
   design?: "unpaired" | "paired";
 
@@ -96,6 +101,13 @@ export interface PlanConfig {
   maxTrials: number;
   batchSize: number;
   nullQuantile: number;
+  /**
+   * The `futilityConfidence` you run `shouldStop` and `shouldStopPaired` at.
+   * The expected bill counts the runs or pairs a case takes to stop at this
+   * level, so a higher one prices more. `undefined` takes the stopping rules'
+   * own default.
+   */
+  futilityConfidence?: number;
 }
 
 export const DEFAULT_PLAN: PlanConfig = {
@@ -197,7 +209,11 @@ export interface PlanTotals {
    * over-predicts what the gate actually spends by 5 to 8×, and the numbers below
    * predict it to within about 10% when checked against real runs.
    */
-  /** 1 unpaired, 2 paired, what one observation costs in runs */
+  /**
+   * What one observation of the expected bill costs in runs, averaged over its
+   * cases: 1 when every case runs unpaired, 2 when every case is paired, and in
+   * between for a suite that mixes the two.
+   */
   runsPerObservation: number;
   /** expected *runs* (not observations) for a pull request in which nothing moved */
   typicalRuns: number;
@@ -218,7 +234,13 @@ export interface Plan {
   pairedOnly: PlanCase[];
   /** cases whose baseline rate is too low to lose `mde` points at all */
   impossible: PlanCase[];
-  /** the design the recommendation names, and the one `expectedCostUsd` prices */
+  /**
+   * The design the recommendation names: paired when more than a quarter of
+   * the suite is `PAIRED_ONLY` or most decidable cases' own `design` is
+   * paired. `expectedCostUsd` prices each case at
+   * its own `design` and falls back to this one only for a case with none,
+   * unless `config.design` fixes one design for the whole suite.
+   */
   recommendedDesign: "unpaired" | "paired";
   /** the cheapest design that decides every decidable case */
   recommendation: string;
@@ -286,12 +308,28 @@ function baselineRunsToClear(
   return hi;
 }
 
+function requirePrices(cfg: PlanConfig, where: string): void {
+  if (!(cfg.costPerRunUsd >= 0) || !(cfg.msPerRun >= 0)) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_CONFIG",
+      `${where}: costPerRunUsd and msPerRun must be ≥ 0`,
+      {
+        detail: { costPerRunUsd: cfg.costPerRunUsd, msPerRun: cfg.msPerRun },
+      },
+    );
+  }
+}
+
 export function planCase(
   c: CaseRef,
   b: BaselineStat | undefined,
   cfg: PlanConfig,
   m: number,
 ): PlanCase {
+  // Exported on its own, so it refuses what makePlan refuses before calling it: unchecked, a -5/60 baseline was IMPOSSIBLE and a price of -1 billed -$152.
+  if (b) requireCounts(b.successes, b.trials, `planCase(${c.id}).baseline`);
+  requireErrorRates(cfg.alpha, cfg.beta, "planCase");
+  requirePrices(cfg, "planCase");
   const barLogE = Math.log(ebhSoloThreshold(m, cfg.fdr));
   const trials = b?.trials ?? 0;
   const successes = b?.successes ?? 0;
@@ -416,7 +454,8 @@ export function planCase(
  * expected sample number at a supplied "true" rate per case, and clamped into
  * the gate's own [minTrials, maxTrials] batch grid.
  *
- * This is the number that predicts what the gate will actually spend.
+ * `makePlan` no longer bills from it: an unpaired case there is counted in
+ * the runs `shouldStop` takes, which this figure misses either way (#105).
  * `samplesForEvidence`, the one behind `PlanCase.unpaired.runs`, answers a
  * different question: how many runs it takes to *certify a regression*. A case
  * that did not regress never gets there; it hits the H₀ wall in a few batches
@@ -432,6 +471,9 @@ export function expectedSequentialSamples(
   trueRate: (caseId: string, b: BaselineStat) => number = (_id, b) =>
     b.trials > 0 ? b.successes / b.trials : 0,
 ): { samples: number; perCase: Map<string, number> } {
+  requireTrialGrid(cfg, "expectedSequentialSamples");
+  // A repeated id was added to `samples` once per listing but kept once in `perCase`, so the two disagreed.
+  requireUniqueCaseIds(cases, "expectedSequentialSamples");
   const perCase = new Map<string, number>();
   let total = 0;
   for (const c of cases) {
@@ -441,21 +483,112 @@ export function expectedSequentialSamples(
     const p1 = Math.max(0.005, p0 - cfg.mde);
     const p = Math.max(0, Math.min(1, trueRate(c.id, b)));
     const raw = sprtExpectedN(p, p0, p1, cfg.alpha, cfg.beta);
-    const n =
-      Number.isFinite(raw) && raw > 0
-        ? Math.min(cfg.maxTrials, Math.max(cfg.minTrials, Math.ceil(raw)))
-        : cfg.maxTrials;
-    // The gate only ever adds whole batches past the minimum, so round the
-    // prediction onto the same grid the gate samples on.
-    const batched =
-      cfg.minTrials + Math.ceil(Math.max(0, n - cfg.minTrials) / cfg.batchSize) * cfg.batchSize;
-    const capped = Math.min(cfg.maxTrials, batched);
+    const capped = onBatchGrid(
+      Number.isFinite(raw) && raw > 0 ? Math.ceil(raw) : cfg.maxTrials,
+      cfg,
+    );
     perCase.set(c.id, capped);
     total += capped;
   }
   return { samples: total, perCase };
 }
 
+type TrialGrid = Pick<PlanConfig, "minTrials" | "maxTrials" | "batchSize">;
+
+/** Refuses a set `futilityConfidence` outside (0, 1), where the settle bar is 0 or Infinity. */
+export function requireSettleConfidence(
+  cfg: Pick<PlanConfig, "futilityConfidence">,
+  where: string,
+): void {
+  if (cfg.futilityConfidence !== undefined) {
+    requireOpenProbability(cfg.futilityConfidence, "futilityConfidence", where);
+  }
+}
+
+// The gate only ever adds whole batches past the minimum, so a prediction is rounded onto the grid it samples on.
+function onBatchGrid(n: number, cfg: TrialGrid): number {
+  const clamped = Math.min(cfg.maxTrials, Math.max(cfg.minTrials, n));
+  const batched =
+    cfg.minTrials + Math.ceil(Math.max(0, clamped - cfg.minTrials) / cfg.batchSize) * cfg.batchSize;
+  return Math.min(cfg.maxTrials, batched);
+}
+
+/**
+ * Expected pairs for a paired case that did not move: the first pair count at
+ * which `shouldStopPaired`'s settle e-value, on the mean trajectory at coupling
+ * `rho` and `cfg.futilityConfidence` (the stopping rule's default when unset),
+ * rules out a drop of `mde`, on the same batch grid as
+ * `expectedSequentialSamples`, and `maxTrials` when it never does. The
+ * unpaired SPRT figure knows nothing of this rule or of the coupling, which
+ * sets how often an unmoved pair disagrees.
+ */
+export function expectedSettlePairs(
+  rate: number,
+  rho: number,
+  mde: number,
+  cfg: TrialGrid & Pick<PlanConfig, "futilityConfidence">,
+): number {
+  // Unmoved arms disagree each way only when uncoupled: (1 - rho) p (1 - p).
+  const each = (1 - rho) * rate * (1 - rate);
+  const confidence = cfg.futilityConfidence ?? DEFAULT_STOP_OPTIONS.futilityConfidence;
+  const target = -Math.log(1 - confidence);
+  for (let n = 1; n <= cfg.maxTrials; n++) {
+    const counts = { bothPass: n * (1 - 2 * each), worse: n * each, better: n * each, bothFail: 0 };
+    if (settleLogE(counts, mde) >= target) return onBatchGrid(n, cfg);
+  }
+  return cfg.maxTrials;
+}
+
+/**
+ * Expected runs `shouldStop` takes to stop an unpaired case whose candidate
+ * passes at `rate`, checked at `minTrials` and then every `batchSize` runs up
+ * to `maxTrials`, in a suite of `suiteSize`. Exact rather than simulated:
+ * the decision depends on the candidate only through its counts, so a dynamic
+ * program carries the distribution of the success count from one check to the
+ * next. Rounded up to a whole run; an expectation is not a grid point.
+ */
+function expectedStopRuns(
+  baseline: BaselineStat,
+  rate: number,
+  suiteSize: number,
+  cfg: TrialGrid & Pick<PlanConfig, "mde" | "fdr" | "futilityConfidence">,
+): number {
+  const options = {
+    suiteSize,
+    mde: cfg.mde,
+    fdr: cfg.fdr,
+    futilityConfidence: cfg.futilityConfidence,
+  };
+  // live[k]: probability of k successes in n runs with the case still running.
+  let live = new Float64Array([1]);
+  let n = 0;
+  let expected = 0;
+  for (let step = cfg.minTrials; ; step = Math.min(cfg.batchSize, cfg.maxTrials - n)) {
+    for (let i = 0; i < step; i++) {
+      const next = new Float64Array(live.length + 1);
+      for (let k = 0; k < live.length; k++) {
+        next[k]! += live[k]! * (1 - rate);
+        next[k + 1]! += live[k]! * rate;
+      }
+      live = next;
+    }
+    n += step;
+    let running = 0;
+    for (let k = 0; k <= n; k++) {
+      if (live[k] === 0) continue;
+      if (n >= cfg.maxTrials || shouldStop({ successes: k, trials: n }, baseline, options).stop) {
+        expected += live[k]! * n;
+        live[k] = 0;
+      } else {
+        running += live[k]!;
+      }
+    }
+    if (running === 0) break;
+  }
+  // Float residue must not push an exact whole number up a run.
+  return Math.ceil(expected - 1e-9);
+}
+
 /**
  * Plan a suite. `pilots` holds a pair table per case from a pilot run of the
  * paired design, ideally with both arms on the same revision; a case with one
@@ -485,20 +618,17 @@ export function makePlan(
   // reaches it, and under that function's name for it, `q`. Checked here so
   // every planner entry point refuses it the same way and names it.
   requireOpenProbability(cfg.nullQuantile, "nullQuantile", "makePlan");
+  requireSettleConfidence(cfg, "makePlan");
+  requireRunCeiling(cfg.runsPerCaseCeiling, "makePlan");
+  requireTrialGrid(cfg, "makePlan");
   if (cases.length === 0) {
     throw new PeeksafeError("PEEKSAFE_E_SUITE_EMPTY", "makePlan: no cases to plan for", {
       hint: "pass at least one case; makePlan prices a suite, not an empty list",
     });
   }
-  if (!(cfg.costPerRunUsd >= 0) || !(cfg.msPerRun >= 0)) {
-    throw new PeeksafeError(
-      "PEEKSAFE_E_CONFIG",
-      "makePlan: costPerRunUsd and msPerRun must be ≥ 0",
-      {
-        detail: { costPerRunUsd: cfg.costPerRunUsd, msPerRun: cfg.msPerRun },
-      },
-    );
-  }
+  // `gate` refuses a repeated id; planned, it was priced twice and counted twice in the suite size the bar is set from.
+  requireUniqueCaseIds(cases, "makePlan");
+  requirePrices(cfg, "makePlan");
 
   const m = cases.length;
   const planned = cases.map((c) => {
@@ -529,13 +659,18 @@ export function makePlan(
   );
 
   const pairedOnlyCount = planned.filter((p) => p.detectability === "PAIRED_ONLY").length;
-  // The design the recommendation will name, computed here because the
-  // *expected* bill has to be priced for it. `pairedOnly > 25%` is the same
-  // condition the recommendation string uses; they cannot drift apart.
+  const pairedDesignCount = decidableAny.filter((p) => p.design === "paired").length;
+  // The recommendation string reads these same two flags, so it cannot drift from the design it names.
+  const manyPairedOnly = pairedOnlyCount > cases.length * 0.25;
+  const mostlyPaired = pairedDesignCount * 2 > decidableAny.length;
   const recommendedDesign: "unpaired" | "paired" =
     cfg.design ??
-    (decidableAny.length > 0 && pairedOnlyCount > cases.length * 0.25 ? "paired" : "unpaired");
-  const runsPerObservation = recommendedDesign === "paired" ? 2 : 1;
+    (decidableAny.length > 0 && (manyPairedOnly || mostlyPaired) ? "paired" : "unpaired");
+  // Per case, like `bestRuns`: one suite-wide design billed a mixed suite as if every case ran it.
+  const runsPerObs = (p: PlanCase) =>
+    (cfg.design ?? p.design ?? recommendedDesign) === "paired" ? 2 : 1;
+  // `screenRuns` counts observations, so each case's pass costs its own design's runs.
+  const screenPassRuns = planned.reduce((a, p) => a + cfg.screenRuns * runsPerObs(p), 0);
 
   // Screening: everything gets `screenRuns`; only the cases the screen ranks as
   // plausibly moved go on to the sequential test. A clean PR moves nothing, so
@@ -550,21 +685,45 @@ export function makePlan(
   // which is `pairs * 2`. The expected total below reads `screenRuns` the same
   // way, and when this one did not, the two screened figures in `totals`
   // disagreed about the price of the same pass.
-  const screenedRuns = m * cfg.screenRuns * runsPerObservation + screenSelected * meanBest;
+  const screenedRuns = screenPassRuns + screenSelected * meanBest;
 
   const fixedNRuns = planned.reduce((a, p) => a + finite(p.fixedNPerArm), 0);
 
   // The expected cost of a pull request that changed nothing, which is what a
   // pull request almost always is, and therefore the number a budget should be
   // set from.
-  // `expectedSequentialSamples` counts *observations*. A paired observation is
-  // two runs, the candidate and the reference revision on the same seed, and
-  // so is a paired screening run.
-  const typical = expectedSequentialSamples(cases, baseline, cfg);
-  const typicalRuns = typical.samples * runsPerObservation;
-  const meanTypical = typical.perCase.size > 0 ? typical.samples / typical.perCase.size : 0;
+  // Both counts are *observations*. A paired observation is two runs, the
+  // candidate and the reference revision on the same seed, and so is a paired
+  // screening run. Each case is counted under the rule that stops it: a paired
+  // one in pairs to settle, an unpaired one in the runs `shouldStop` takes.
+  // Wald's SPRT figure, used for both before, runs a different rule, ignores
+  // the coupling and the suite size, and mispriced cases either way.
+  const stopRuns = new Map<string, number>();
+  const observations = (p: PlanCase) => {
+    const b = baseline.get(p.caseId);
+    if (!b || b.trials === 0) return 0;
+    if (runsPerObs(p) === 2) {
+      return expectedSettlePairs(p.baseline.rate, p.pairCoupling, cfg.mde, cfg);
+    }
+    const key = `${b.successes}/${b.trials}`;
+    const known = stopRuns.get(key);
+    if (known !== undefined) return known;
+    const runs = expectedStopRuns(b, b.successes / b.trials, m, cfg);
+    stopRuns.set(key, runs);
+    return runs;
+  };
+  const priced = planned.filter((p) => (baseline.get(p.caseId)?.trials ?? 0) > 0).length;
+  const typicalObservations = planned.reduce((a, p) => a + observations(p), 0);
+  const typicalRuns = planned.reduce((a, p) => a + observations(p) * runsPerObs(p), 0);
+  const runsPerObservation =
+    typicalObservations > 0
+      ? typicalRuns / typicalObservations
+      : recommendedDesign === "paired"
+        ? 2
+        : 1;
+  const meanTypical = priced > 0 ? typicalRuns / priced : 0;
   const typicalScreenedRuns = Math.ceil(
-    (m * cfg.screenRuns + Math.ceil(m * screenContinueFraction) * meanTypical) * runsPerObservation,
+    screenPassRuns + Math.ceil(m * screenContinueFraction) * meanTypical,
   );
 
   const totals: PlanTotals = {
@@ -604,8 +763,12 @@ export function makePlan(
     decidableAny.length === 0
       ? `nothing is decidable at ${(cfg.mde * 100).toFixed(0)} points against this baseline, record more baseline runs or raise the MDE`
       : recommendedDesign === "paired"
-        ? `pair the runs: ${pairedOnly.length}/${m} cases have an unpaired evidence ceiling under the bar, so no candidate budget can decide them` +
-          ` (the costs below are priced at two runs an observation, which is what pairing costs)`
+        ? (manyPairedOnly
+            ? `pair the runs: ${pairedOnly.length}/${m} cases have an unpaired evidence ceiling under the bar, so no candidate budget can decide them`
+            : `pair the runs: ${pairedDesignCount}/${decidableAny.length} decidable cases are cheaper paired than against the stored baseline`) +
+          (cfg.design === undefined
+            ? ` (the costs below price each case at its own design, two runs an observation where it is paired)`
+            : ` (the costs below are priced at two runs an observation, which is what pairing costs)`)
         : totals.screenedCostUsd < totals.bestCostUsd
           ? `screen first, then run the sequential test on what the screen keeps ` +
             `($${totals.screenedCostUsd.toFixed(2)} vs $${totals.bestCostUsd.toFixed(2)})`
@@ -703,6 +866,30 @@ export function affordabilityGrid(
   // straight from `betaQuantile`, which answers 0 or 1 for a quantile outside
   // (0,1) and about 0 for NaN instead of refusing. `makePlan` refuses it.
   requireOpenProbability(cfg.nullQuantile, "nullQuantile", "affordabilityGrid");
+  // makePlan refuses both; unchecked, an fdr of 2 priced a grid cheaper than one at 0.05.
+  requireOpenProbability(cfg.fdr, "fdr", "affordabilityGrid");
+  requireSettleConfidence(cfg, "affordabilityGrid");
+  requireTrialGrid(cfg, "affordabilityGrid");
+  if (!(cfg.costPerRunUsd >= 0)) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_CONFIG",
+      `affordabilityGrid: costPerRunUsd must be ≥ 0, got ${cfg.costPerRunUsd}`,
+      {
+        detail: { costPerRunUsd: cfg.costPerRunUsd },
+      },
+    );
+  }
+  for (const m of caseCounts) {
+    if (!Number.isInteger(m) || m < 1) {
+      throw new PeeksafeError(
+        "PEEKSAFE_E_CONFIG",
+        `affordabilityGrid: every case count must be a positive integer, got ${m}`,
+        {
+          detail: { cases: m },
+        },
+      );
+    }
+  }
   // Every other planner entry point refuses a case with no baseline: `planCase`
   // returns IMPOSSIBLE with "no baseline", `evaluatePoint` throws on
   // `baselineRuns < 1`, and `gate` throws PEEKSAFE_E_BASELINE_MISSING. This one
@@ -752,10 +939,13 @@ export function affordabilityGrid(
       const p0 = betaQuantile(1 + s, 1 + baselineTrials - s, cfg.nullQuantile);
       const p1 = Math.max(0.005, p0 - mde);
       const rawN = sprtExpectedN(baselineRate, p0, p1, cfg.alpha, cfg.beta);
+      // A paired cell counts the pairs `shouldStopPaired` takes to settle, as `makePlan` does; the SPRT figure ignores the coupling.
       const perCaseTypical =
-        Number.isFinite(rawN) && rawN > 0
-          ? Math.min(cfg.maxTrials, Math.max(cfg.minTrials, Math.ceil(rawN)))
-          : cfg.maxTrials;
+        runsPerObs === 2
+          ? expectedSettlePairs(baselineRate, cfg.pairCoupling, mde, cfg)
+          : Number.isFinite(rawN) && rawN > 0
+            ? Math.min(cfg.maxTrials, Math.max(cfg.minTrials, Math.ceil(rawN)))
+            : cfg.maxTrials;
       // Priced for the same design as the ceiling above (`runsPerObs`), or the
       // cell quotes an unpaired price for a plan only pairing can execute.
       const typicalRuns = Math.ceil(

package/src/rand.ts

@@ -20,7 +20,7 @@
  * It is not a fine thing to do by accident, which is why the constants are
  * spelled out here rather than imported.
  */
-import { PeeksafeError } from "./errors.js";
+import { PeeksafeError, requireProbability } from "./errors.js";
 
 export interface Rand {
   (): number;
@@ -75,11 +75,27 @@ export function hash32(s: string): number {
 }
 
 export function makeRand(seed: number | string): Rand {
-  const s = typeof seed === "string" ? hash32(seed) : seed >>> 0;
-  const base = lcg(s === 0 ? 1 : s);
+  // `>>> 0` folded NaN, Infinity, 1.5 and 2^32 onto 0, and the old 0-to-1 remap put seed 1 there too.
+  if (typeof seed === "number" && !(Number.isInteger(seed) && seed >= 0 && seed < 0x100000000)) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_STAT_DOMAIN",
+      `makeRand: a numeric seed must be an integer in [0, 2^32), got ${seed}`,
+      {
+        detail: { seed },
+        hint: "pass a string to key a stream by name; it is hashed with hash32",
+      },
+    );
+  }
+  const s = typeof seed === "string" ? hash32(seed) : seed;
+  // Hull-Dobell: an odd increment and a multiplier of 1 mod 4 give a full period from every seed, 0 included.
+  const base = lcg(s);
   const next = (() => mix32(Math.floor(base() * 0x100000000)) / 0x100000000) as Rand;
 
-  next.bernoulli = (p: number) => next() < p;
+  // `next() < p` is false for every draw at NaN, so a rate that failed to compute simulated a case that never passes.
+  next.bernoulli = (p: number) => {
+    requireProbability(p, "p", "rand.bernoulli");
+    return next() < p;
+  };
 
   next.int = (n: number) => {
     if (!Number.isInteger(n) || n < 1) {