peeksafe 0.0.0-stage → 0.2.0

package/src/plan.ts

@@ -0,0 +1,968 @@
+/**
+ * plan.ts: power economics. What will this cost, and can we even detect it?
+ *
+ * Without a plan, the first sign that a suite cannot be decided is a large bill
+ * and no certified case. This module tells you *before* you spend anything, and it will tell you that the honest answer is
+ * sometimes "you cannot detect this at any price".
+ *
+ * The three numbers that matter, per case:
+ *
+ *  1. **The bar.** e-BH at FDR q over m cases certifies a case on its own only
+ *     once its e-value clears m/q. That is the target the evidence has to hit.
+ *
+ *  2. **The ceiling.** `evidenceCeilingLogE`, the limit of the unpaired
+ *     two-sample e-value as candidate runs go to infinity. Both marginals are
+ *     the same binomial likelihood under different priors, so everything that
+ *     grows with n cancels and the Bayes factor converges to the prior density
+ *     ratio at the observed rate. If the ceiling is under the bar, no number of
+ *     candidate runs will certify the case. This is not a budget problem and no
+ *     budget fixes it: **more baseline runs** fix it, by sharpening the null.
+ *
+ *  3. **The paired alternative.** Re-run the baseline alongside the candidate
+ *     on the same seed and the null becomes an exact point mass at ½ (a
+ *     discordant pair points either way with probability exactly one half under
+ *     "nothing changed"). Nothing is estimated, so nothing caps the evidence:
+ *     the paired e-value grows linearly in discordant pairs. It costs two runs
+ *     per observation and it is still, routinely, the cheaper design, and it
+ *     is the only one that works at all for a case whose ceiling is under the
+ *     bar.
+ *
+ * Everything here is a *plan*: it evaluates the mean trajectory (the candidate
+ * scores exactly its expected successes). Realised sample counts vary around
+ * it.
+ */
+
+import { type BaselineStat, baselineNullRate, type CaseRef } from "./baseline.js";
+import {
+  PeeksafeError,
+  requireCounts,
+  requireErrorRates,
+  requireOpenProbability,
+  requireProbability,
+  requireRunCeiling,
+  requireTrialGrid,
+  requireUniqueCaseIds,
+} from "./errors.js";
+import {
+  betaQuantile,
+  ebhSoloThreshold,
+  evidenceCeilingLogE,
+  mcnemarSamplesForEvidence,
+  type PairedCounts,
+  pairPhi,
+  sampleSizeTwoProportion,
+  samplesForEvidence,
+  sprtExpectedN,
+} from "./stats.js";
+import { DEFAULT_STOP_OPTIONS, settleLogE, shouldStop } from "./stop.js";
+
+export interface PlanConfig {
+  /** the drop we want to be able to detect, in probability units (0.15 = 15 points) */
+  mde: number;
+  /** suite-level false discovery rate */
+  fdr: number;
+  /** USD per candidate run */
+  costPerRunUsd: number;
+  /** milliseconds per run, wall clock, at concurrency 1 */
+  msPerRun: number;
+  /** how many runs the screening pass spends per case */
+  screenRuns: number;
+  /**
+   * Measured coupling between the paired arms, in [0,1]. 0 means the seeds
+   * bought nothing; 1 means perfect common-random-numbers coupling. Measure it
+   * with `pairPhi` on a pilot rather than assuming it, and plan at 0 as well
+   * so the sensitivity is visible.
+   */
+  pairCoupling: number;
+  /** how many runs to fan out in parallel, only affects wall time */
+  concurrency: number;
+  /** beyond this many runs for one case we stop calling it "affordable" */
+  runsPerCaseCeiling: number;
+  /**
+   * Which design the *expected* bill and the screening pass are priced for. A
+   * paired observation is two runs, and a paired case's observation count is
+   * the pairs `shouldStopPaired` takes to settle it at its coupling; an
+   * unpaired case's is the runs `shouldStop` takes to stop it.
+   *
+   * `undefined` prices each case at its own `PlanCase.design`, and a case with
+   * none at the suite's `recommendedDesign`. Setting it prices every case at
+   * that one design.
+   */
+  design?: "unpaired" | "paired";
+
+  /* ── the sequential design being priced ────────────────────────────────
+   * Only the *expected* cost depends on these; the certify-all ceiling does
+   * not. Keep them in step with the stopping rule you actually run, or the plan prices a design
+   * nobody is going to execute.
+   */
+  alpha: number;
+  beta: number;
+  minTrials: number;
+  maxTrials: number;
+  batchSize: number;
+  nullQuantile: number;
+  /**
+   * The `futilityConfidence` you run `shouldStop` and `shouldStopPaired` at.
+   * The expected bill counts the runs or pairs a case takes to stop at this
+   * level, so a higher one prices more. `undefined` takes the stopping rules'
+   * own default.
+   */
+  futilityConfidence?: number;
+}
+
+export const DEFAULT_PLAN: PlanConfig = {
+  mde: 0.15,
+  fdr: 0.05,
+  // These three are *assumptions* until you measure them on a real run: cost
+  // and latency from your runner, coupling with `pairPhi` on a pilot.
+  costPerRunUsd: 0.0021,
+  msPerRun: 780,
+  pairCoupling: 0.9,
+  screenRuns: 4,
+  concurrency: 8,
+  runsPerCaseCeiling: 2000,
+  alpha: 0.05,
+  beta: 0.1,
+  minTrials: 8,
+  maxTrials: 96,
+  batchSize: 4,
+  nullQuantile: 0.5,
+};
+
+export type Detectability =
+  /** the unpaired sequential test can certify this case within the run ceiling */
+  | "UNPAIRED"
+  /** only the paired design can certify it, the unpaired evidence ceiling is under the bar */
+  | "PAIRED_ONLY"
+  /** the run count is finite but past the ceiling: technically possible, practically not */
+  | "TOO_EXPENSIVE"
+  /** the case cannot lose `mde` points at all, its baseline rate is not that high */
+  | "IMPOSSIBLE";
+
+export interface PlanCase {
+  caseId: string;
+  suite: string;
+  baseline: { successes: number; trials: number; rate: number };
+  /** the rate the alternative sits at: baseline − mde, floored at 0 */
+  alternativeRate: number;
+  /** log of the e-value a case must reach to be certified on its own evidence */
+  barLogE: number;
+  /** the limit of the unpaired log e-value as runs → ∞ */
+  ceilingLogE: number;
+  unpaired: { runs: number; costUsd: number; wallMs: number };
+  paired: { pairs: number; runs: number; costUsd: number; wallMs: number };
+  /**
+   * The coupling `paired` is priced at: `pairPhi` of this case's pilot when
+   * `makePlan` was given one, floored at 0, otherwise `config.pairCoupling`.
+   */
+  pairCoupling: number;
+  detectability: Detectability;
+  /**
+   * The design to run this case with: of the designs that decide it within
+   * `runsPerCaseCeiling`, the one that needs fewer runs, unpaired on a tie.
+   * Null when neither does.
+   */
+  design: "unpaired" | "paired" | null;
+  /**
+   * How many baseline runs this case would need for the *unpaired* design to
+   * work, or null if it already works. This is the actionable half of an
+   * "undetectable" verdict.
+   */
+  baselineRunsNeeded: number | null;
+  /** what a fixed-N two-proportion design would spend per arm, for comparison */
+  fixedNPerArm: number;
+  reason: string;
+}
+
+export interface PlanTotals {
+  cases: number;
+  /** how many cases each design can actually certify */
+  decidableUnpaired: number;
+  decidableBest: number;
+  /** every case run to certification, unpaired */
+  unpairedRuns: number;
+  unpairedCostUsd: number;
+  /** every case run to certification, paired */
+  pairedRuns: number;
+  pairedCostUsd: number;
+  /** per case, the cheaper of the two designs that can decide it */
+  bestRuns: number;
+  bestCostUsd: number;
+  /** screen everything cheaply, then pay only for what the screen keeps */
+  screenedRuns: number;
+  screenedCostUsd: number;
+  /** the fixed-N design with the same nominal guarantees */
+  fixedNRuns: number;
+  fixedNCostUsd: number;
+  wallMsUnpaired: number;
+  wallMsPaired: number;
+  wallMsBest: number;
+  wallMsScreened: number;
+  /** fraction of cases the screen is budgeted to carry through to the full test */
+  screenContinueFraction: number;
+
+  /* ── the expected cost, as opposed to the worst case ───────────────────
+   * `bestRuns` above is what it costs to certify *every* case, i.e. a pull
+   * request in which everything regressed by exactly the MDE. Real pull
+   * requests move nothing, cases hit the H₀ wall early, and they stop costing
+   * money. Measured against three real runs of this suite, `bestRuns`
+   * over-predicts what the gate actually spends by 5 to 8×, and the numbers below
+   * predict it to within about 10% when checked against real runs.
+   */
+  /**
+   * What one observation of the expected bill costs in runs, averaged over its
+   * cases: 1 when every case runs unpaired, 2 when every case is paired, and in
+   * between for a suite that mixes the two.
+   */
+  runsPerObservation: number;
+  /** expected *runs* (not observations) for a pull request in which nothing moved */
+  typicalRuns: number;
+  typicalCostUsd: number;
+  wallMsTypical: number;
+  /** the same, with a screening pass in front */
+  typicalScreenedRuns: number;
+  typicalScreenedCostUsd: number;
+}
+
+export interface Plan {
+  config: PlanConfig;
+  cases: PlanCase[];
+  totals: PlanTotals;
+  /** cases no budget can decide with the current baseline */
+  undecidable: PlanCase[];
+  /** cases only the paired design can reach */
+  pairedOnly: PlanCase[];
+  /** cases whose baseline rate is too low to lose `mde` points at all */
+  impossible: PlanCase[];
+  /**
+   * The design the recommendation names: paired when more than a quarter of
+   * the suite is `PAIRED_ONLY` or most decidable cases' own `design` is
+   * paired. `expectedCostUsd` prices each case at
+   * its own `design` and falls back to this one only for a case with none,
+   * unless `config.design` fixes one design for the whole suite.
+   */
+  recommendedDesign: "unpaired" | "paired";
+  /** the cheapest design that decides every decidable case */
+  recommendation: string;
+  /** honest one-liner about affordability at the configured cost per run */
+  verdict: string;
+  /**
+   * What a *typical* pull request costs, one in which nothing moved, so most
+   * cases stop at the H₀ wall. This is the headline number.
+   */
+  expectedCostUsd: number;
+  /**
+   * What a pull request in which *every* case regressed would cost.
+   *
+   * Not an upper bound on `expectedCostUsd`, despite running every case to
+   * certification: the sequential test stops a case as soon as it clears the
+   * bar, so an all-regressed suite can finish cheaper than one that did not
+   * move and ran its full schedule. `verdict` says which way round they came
+   * out rather than presenting this as the worst case.
+   */
+  ceilingCostUsd: number;
+}
+
+const round = (x: number) => (Number.isFinite(x) ? Math.ceil(x) : Infinity);
+
+const medianTrials = (ps: PlanCase[]): number => {
+  const xs = ps.map((p) => p.baseline.trials).sort((a, b) => a - b);
+  return xs.length === 0 ? 0 : xs[Math.floor(xs.length / 2)]!;
+};
+
+/**
+ * How many baseline runs would lift this case's evidence ceiling over the bar?
+ * The *smallest* such number, holding the baseline *rate* fixed, "if I keep
+ * measuring main at the same quality, how long until this case becomes
+ * decidable at all?"
+ *
+ * Smallest, and not a level the case stays decidable above: the ceiling is a
+ * sawtooth in the baseline size (see the comment in the body), so a baseline a
+ * few runs longer than this can fall back under the bar. Treat it as the floor
+ * it is rather than a promise about a bigger baseline.
+ */
+function baselineRunsToClear(
+  rate: number,
+  mde: number,
+  barLogE: number,
+  cap = 100_000,
+): number | null {
+  const at = (n: number) => {
+    const s = Math.round(rate * n);
+    return evidenceCeilingLogE(Math.max(1e-6, Math.min(1 - 1e-6, rate - mde)), s, n, mde);
+  };
+  // `at` is not monotone in `n`, so this cannot bisect the way
+  // `samplesForEvidence` does. Holding the rate fixed still forces an integer
+  // success count, so the implied rate jitters around `rate` and the ceiling
+  // drops hard every time the count ticks over: at 99% losing 10 points it
+  // clears the bar at 49 runs, falls back under it at 51, and does not clear
+  // again until 71. Bisecting that returns whichever crossing the doubling
+  // happened to bracket, which is no particular quantity at all; it answered 81
+  // for the README's own case where 75 is enough, and 5 for every case that
+  // needed fewer, because it bisected (4, 8]. Doubling now only bounds the
+  // scan, and never steps past `cap`, so the answer is one this search checked.
+  let hi = 1;
+  while (hi < cap && at(hi) < barLogE) hi = Math.min(cap, hi * 2);
+  if (at(hi) < barLogE) return null;
+  for (let n = 1; n < hi; n++) if (at(n) >= barLogE) return n;
+  return hi;
+}
+
+function requirePrices(cfg: PlanConfig, where: string): void {
+  if (!(cfg.costPerRunUsd >= 0) || !(cfg.msPerRun >= 0)) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_CONFIG",
+      `${where}: costPerRunUsd and msPerRun must be ≥ 0`,
+      {
+        detail: { costPerRunUsd: cfg.costPerRunUsd, msPerRun: cfg.msPerRun },
+      },
+    );
+  }
+}
+
+export function planCase(
+  c: CaseRef,
+  b: BaselineStat | undefined,
+  cfg: PlanConfig,
+  m: number,
+): PlanCase {
+  // Exported on its own, so it refuses what makePlan refuses before calling it: unchecked, a -5/60 baseline was IMPOSSIBLE and a price of -1 billed -$152.
+  if (b) requireCounts(b.successes, b.trials, `planCase(${c.id}).baseline`);
+  requireErrorRates(cfg.alpha, cfg.beta, "planCase");
+  requirePrices(cfg, "planCase");
+  const barLogE = Math.log(ebhSoloThreshold(m, cfg.fdr));
+  const trials = b?.trials ?? 0;
+  const successes = b?.successes ?? 0;
+  const rate = trials > 0 ? successes / trials : 0;
+  const alt = Math.max(0, rate - cfg.mde);
+
+  const base = {
+    caseId: c.id,
+    // `suite` groups cases into correlation families. A caller who does not
+    // group them gets one family per case, which is the conservative reading.
+    suite: c.suite ?? c.id,
+    baseline: { successes, trials, rate },
+    alternativeRate: alt,
+    barLogE,
+    fixedNPerArm: sampleSizeTwoProportion(Math.max(1e-6, rate), cfg.mde, cfg.alpha, cfg.beta),
+  };
+
+  if (trials === 0) {
+    return {
+      ...base,
+      ceilingLogE: 0,
+      unpaired: { runs: Infinity, costUsd: Infinity, wallMs: Infinity },
+      paired: { pairs: Infinity, runs: Infinity, costUsd: Infinity, wallMs: Infinity },
+      pairCoupling: cfg.pairCoupling,
+      detectability: "IMPOSSIBLE",
+      design: null,
+      baselineRunsNeeded: null,
+      reason: "no baseline: nothing to compare against, record one first",
+    };
+  }
+  if (rate <= cfg.mde) {
+    return {
+      ...base,
+      ceilingLogE: 0,
+      unpaired: { runs: Infinity, costUsd: Infinity, wallMs: Infinity },
+      paired: { pairs: Infinity, runs: Infinity, costUsd: Infinity, wallMs: Infinity },
+      pairCoupling: cfg.pairCoupling,
+      detectability: "IMPOSSIBLE",
+      design: null,
+      baselineRunsNeeded: null,
+      reason:
+        `baseline rate ${(rate * 100).toFixed(0)}% cannot lose ${(cfg.mde * 100).toFixed(0)} points, ` +
+        `the effect is arithmetically impossible, not merely expensive`,
+    };
+  }
+
+  const pTrue = Math.max(1e-6, Math.min(1 - 1e-6, alt));
+  const ceilingLogE = evidenceCeilingLogE(pTrue, successes, trials, cfg.mde);
+  const unpairedRuns = samplesForEvidence(
+    pTrue,
+    successes,
+    trials,
+    cfg.mde,
+    barLogE,
+    cfg.runsPerCaseCeiling * 8,
+  );
+  const pairs = mcnemarSamplesForEvidence(
+    rate,
+    cfg.mde,
+    cfg.pairCoupling,
+    barLogE,
+    cfg.runsPerCaseCeiling * 8,
+  );
+
+  const unpaired = {
+    runs: unpairedRuns,
+    costUsd: unpairedRuns * cfg.costPerRunUsd,
+    wallMs: unpairedRuns * cfg.msPerRun,
+  };
+  const paired = {
+    pairs,
+    runs: pairs * 2,
+    costUsd: pairs * 2 * cfg.costPerRunUsd,
+    wallMs: pairs * 2 * cfg.msPerRun,
+  };
+
+  let detectability: Detectability;
+  let reason: string;
+  if (Number.isFinite(unpairedRuns) && unpairedRuns <= cfg.runsPerCaseCeiling) {
+    detectability = "UNPAIRED";
+    reason = `${unpairedRuns} candidate runs against the stored baseline`;
+  } else if (Number.isFinite(pairs) && pairs * 2 <= cfg.runsPerCaseCeiling) {
+    detectability = "PAIRED_ONLY";
+    reason = Number.isFinite(unpairedRuns)
+      ? `unpaired needs ${unpairedRuns} runs (past the ${cfg.runsPerCaseCeiling} ceiling); paired needs ${pairs * 2}`
+      : `unpaired evidence ceiling is e^${ceilingLogE.toFixed(1)}, under the bar e^${barLogE.toFixed(1)}, ` +
+        `no number of candidate runs certifies it. Paired: ${pairs * 2} runs.`;
+  } else if (Number.isFinite(unpairedRuns) || Number.isFinite(pairs)) {
+    detectability = "TOO_EXPENSIVE";
+    reason = `cheapest design needs ${Math.min(unpairedRuns, pairs * 2)} runs, past the ${cfg.runsPerCaseCeiling} ceiling`;
+  } else {
+    detectability = "IMPOSSIBLE";
+    reason = `neither design reaches the bar within ${cfg.runsPerCaseCeiling * 8} runs`;
+  }
+
+  const baselineRunsNeeded =
+    detectability === "UNPAIRED" ? null : baselineRunsToClear(rate, cfg.mde, barLogE);
+  const design =
+    detectability === "PAIRED_ONLY"
+      ? "paired"
+      : detectability !== "UNPAIRED"
+        ? null
+        : paired.runs < unpaired.runs
+          ? "paired"
+          : "unpaired";
+
+  return {
+    ...base,
+    ceilingLogE,
+    unpaired,
+    paired,
+    pairCoupling: cfg.pairCoupling,
+    detectability,
+    design,
+    baselineRunsNeeded,
+    reason,
+  };
+}
+
+/**
+ * Expected total samples for the sequential design, priced under Wald's
+ * expected sample number at a supplied "true" rate per case, and clamped into
+ * the gate's own [minTrials, maxTrials] batch grid.
+ *
+ * `makePlan` no longer bills from it: an unpaired case there is counted in
+ * the runs `shouldStop` takes, which this figure misses either way (#105).
+ * `samplesForEvidence`, the one behind `PlanCase.unpaired.runs`, answers a
+ * different question: how many runs it takes to *certify a regression*. A case
+ * that did not regress never gets there; it hits the H₀ wall in a few batches
+ * and stops. Pricing every case at the certification cost over-predicts the
+ * bill.
+ *
+ * `trueRate` defaults to the case's own baseline rate, i.e. "nothing moved", * which is what almost every pull request looks like.
+ */
+export function expectedSequentialSamples(
+  cases: Array<CaseRef>,
+  baseline: Map<string, BaselineStat>,
+  cfg: PlanConfig,
+  trueRate: (caseId: string, b: BaselineStat) => number = (_id, b) =>
+    b.trials > 0 ? b.successes / b.trials : 0,
+): { samples: number; perCase: Map<string, number> } {
+  requireTrialGrid(cfg, "expectedSequentialSamples");
+  // A repeated id was added to `samples` once per listing but kept once in `perCase`, so the two disagreed.
+  requireUniqueCaseIds(cases, "expectedSequentialSamples");
+  const perCase = new Map<string, number>();
+  let total = 0;
+  for (const c of cases) {
+    const b = baseline.get(c.id);
+    if (!b || b.trials === 0) continue;
+    const p0 = baselineNullRate(b, cfg.nullQuantile);
+    const p1 = Math.max(0.005, p0 - cfg.mde);
+    const p = Math.max(0, Math.min(1, trueRate(c.id, b)));
+    const raw = sprtExpectedN(p, p0, p1, cfg.alpha, cfg.beta);
+    const capped = onBatchGrid(
+      Number.isFinite(raw) && raw > 0 ? Math.ceil(raw) : cfg.maxTrials,
+      cfg,
+    );
+    perCase.set(c.id, capped);
+    total += capped;
+  }
+  return { samples: total, perCase };
+}
+
+type TrialGrid = Pick<PlanConfig, "minTrials" | "maxTrials" | "batchSize">;
+
+/** Refuses a set `futilityConfidence` outside (0, 1), where the settle bar is 0 or Infinity. */
+export function requireSettleConfidence(
+  cfg: Pick<PlanConfig, "futilityConfidence">,
+  where: string,
+): void {
+  if (cfg.futilityConfidence !== undefined) {
+    requireOpenProbability(cfg.futilityConfidence, "futilityConfidence", where);
+  }
+}
+
+// The gate only ever adds whole batches past the minimum, so a prediction is rounded onto the grid it samples on.
+function onBatchGrid(n: number, cfg: TrialGrid): number {
+  const clamped = Math.min(cfg.maxTrials, Math.max(cfg.minTrials, n));
+  const batched =
+    cfg.minTrials + Math.ceil(Math.max(0, clamped - cfg.minTrials) / cfg.batchSize) * cfg.batchSize;
+  return Math.min(cfg.maxTrials, batched);
+}
+
+/**
+ * Expected pairs for a paired case that did not move: the first pair count at
+ * which `shouldStopPaired`'s settle e-value, on the mean trajectory at coupling
+ * `rho` and `cfg.futilityConfidence` (the stopping rule's default when unset),
+ * rules out a drop of `mde`, on the same batch grid as
+ * `expectedSequentialSamples`, and `maxTrials` when it never does. The
+ * unpaired SPRT figure knows nothing of this rule or of the coupling, which
+ * sets how often an unmoved pair disagrees.
+ */
+export function expectedSettlePairs(
+  rate: number,
+  rho: number,
+  mde: number,
+  cfg: TrialGrid & Pick<PlanConfig, "futilityConfidence">,
+): number {
+  // Unmoved arms disagree each way only when uncoupled: (1 - rho) p (1 - p).
+  const each = (1 - rho) * rate * (1 - rate);
+  const confidence = cfg.futilityConfidence ?? DEFAULT_STOP_OPTIONS.futilityConfidence;
+  const target = -Math.log(1 - confidence);
+  for (let n = 1; n <= cfg.maxTrials; n++) {
+    const counts = { bothPass: n * (1 - 2 * each), worse: n * each, better: n * each, bothFail: 0 };
+    if (settleLogE(counts, mde) >= target) return onBatchGrid(n, cfg);
+  }
+  return cfg.maxTrials;
+}
+
+/**
+ * Expected runs `shouldStop` takes to stop an unpaired case whose candidate
+ * passes at `rate`, checked at `minTrials` and then every `batchSize` runs up
+ * to `maxTrials`, in a suite of `suiteSize`. Exact rather than simulated:
+ * the decision depends on the candidate only through its counts, so a dynamic
+ * program carries the distribution of the success count from one check to the
+ * next. Rounded up to a whole run; an expectation is not a grid point.
+ */
+function expectedStopRuns(
+  baseline: BaselineStat,
+  rate: number,
+  suiteSize: number,
+  cfg: TrialGrid & Pick<PlanConfig, "mde" | "fdr" | "futilityConfidence">,
+): number {
+  const options = {
+    suiteSize,
+    mde: cfg.mde,
+    fdr: cfg.fdr,
+    futilityConfidence: cfg.futilityConfidence,
+  };
+  // live[k]: probability of k successes in n runs with the case still running.
+  let live = new Float64Array([1]);
+  let n = 0;
+  let expected = 0;
+  for (let step = cfg.minTrials; ; step = Math.min(cfg.batchSize, cfg.maxTrials - n)) {
+    for (let i = 0; i < step; i++) {
+      const next = new Float64Array(live.length + 1);
+      for (let k = 0; k < live.length; k++) {
+        next[k]! += live[k]! * (1 - rate);
+        next[k + 1]! += live[k]! * rate;
+      }
+      live = next;
+    }
+    n += step;
+    let running = 0;
+    for (let k = 0; k <= n; k++) {
+      if (live[k] === 0) continue;
+      if (n >= cfg.maxTrials || shouldStop({ successes: k, trials: n }, baseline, options).stop) {
+        expected += live[k]! * n;
+        live[k] = 0;
+      } else {
+        running += live[k]!;
+      }
+    }
+    if (running === 0) break;
+  }
+  // Float residue must not push an exact whole number up a run.
+  return Math.ceil(expected - 1e-9);
+}
+
+/**
+ * Plan a suite. `pilots` holds a pair table per case from a pilot run of the
+ * paired design, ideally with both arms on the same revision; a case with one
+ * prices pairing at its measured `pairPhi` instead of `cfg.pairCoupling`, so
+ * its `design` is chosen from a coupling the runner actually delivers.
+ */
+export function makePlan(
+  cases: Array<CaseRef>,
+  baseline: Map<string, BaselineStat>,
+  cfg: PlanConfig = DEFAULT_PLAN,
+  pilots: Map<string, PairedCounts> = new Map(),
+): Plan {
+  // Open interval: mde = 0 asks for a zero-point drop and fdr = 0 asks for a
+  // bar of Infinity. Both used to be accepted and produce a plan in which
+  // nothing is decidable, for a reason the printed message blamed on the
+  // baseline.
+  requireOpenProbability(cfg.mde, "mde", "makePlan");
+  requireOpenProbability(cfg.fdr, "fdr", "makePlan");
+  // The two error rates the SPRT costing runs on. Left unchecked they reached
+  // `sprtExpectedN`, which answers NaN or Infinity outside (0,1), and both
+  // callers of it here gate on `Number.isFinite(raw) && raw > 0` and quote
+  // `maxTrials` when that fails. So an alpha of 2 produced a complete plan with
+  // every case at the per-case cap and no error anywhere.
+  requireErrorRates(cfg.alpha, cfg.beta, "makePlan");
+  requireProbability(cfg.pairCoupling, "pairCoupling", "makePlan");
+  // Refused by `baselineNullRate` already, but only once a case with a baseline
+  // reaches it, and under that function's name for it, `q`. Checked here so
+  // every planner entry point refuses it the same way and names it.
+  requireOpenProbability(cfg.nullQuantile, "nullQuantile", "makePlan");
+  requireSettleConfidence(cfg, "makePlan");
+  requireRunCeiling(cfg.runsPerCaseCeiling, "makePlan");
+  requireTrialGrid(cfg, "makePlan");
+  if (cases.length === 0) {
+    throw new PeeksafeError("PEEKSAFE_E_SUITE_EMPTY", "makePlan: no cases to plan for", {
+      hint: "pass at least one case; makePlan prices a suite, not an empty list",
+    });
+  }
+  // `gate` refuses a repeated id; planned, it was priced twice and counted twice in the suite size the bar is set from.
+  requireUniqueCaseIds(cases, "makePlan");
+  requirePrices(cfg, "makePlan");
+
+  const m = cases.length;
+  const planned = cases.map((c) => {
+    const pilot = pilots.get(c.id);
+    // pairPhi validates the counts, so a malformed pilot is refused even when it sums to 0.
+    const phi = pilot ? pairPhi(pilot) : 0;
+    // An empty pilot measured nothing, so it leaves the configured coupling in place.
+    const measured = pilot && pilot.bothPass + pilot.worse + pilot.better + pilot.bothFail > 0;
+    // A negative phi is noise around independence for a design that couples by construction.
+    const caseCfg = measured ? { ...cfg, pairCoupling: Math.max(0, phi) } : cfg;
+    return planCase(c, baseline.get(c.id), caseCfg, m);
+  });
+
+  const finite = (x: number) => (Number.isFinite(x) ? x : 0);
+  const decidableUnpaired = planned.filter((p) => p.detectability === "UNPAIRED");
+  const decidableAny = planned.filter(
+    (p) => p.detectability === "UNPAIRED" || p.detectability === "PAIRED_ONLY",
+  );
+
+  const unpairedRuns = decidableUnpaired.reduce((a, p) => a + finite(p.unpaired.runs), 0);
+  const pairedRuns = decidableAny.reduce((a, p) => a + finite(p.paired.runs), 0);
+  // Per case, take the cheaper design that can actually decide it. This is the
+  // number the recommendation and the affordability verdict are built on: a
+  // design that is cheap because it decides nothing is not cheap.
+  const bestRuns = decidableAny.reduce(
+    (a, p) => a + Math.min(finite(p.unpaired.runs) || Infinity, finite(p.paired.runs) || Infinity),
+    0,
+  );
+
+  const pairedOnlyCount = planned.filter((p) => p.detectability === "PAIRED_ONLY").length;
+  const pairedDesignCount = decidableAny.filter((p) => p.design === "paired").length;
+  // The recommendation string reads these same two flags, so it cannot drift from the design it names.
+  const manyPairedOnly = pairedOnlyCount > cases.length * 0.25;
+  const mostlyPaired = pairedDesignCount * 2 > decidableAny.length;
+  const recommendedDesign: "unpaired" | "paired" =
+    cfg.design ??
+    (decidableAny.length > 0 && (manyPairedOnly || mostlyPaired) ? "paired" : "unpaired");
+  // Per case, like `bestRuns`: one suite-wide design billed a mixed suite as if every case ran it.
+  const runsPerObs = (p: PlanCase) =>
+    (cfg.design ?? p.design ?? recommendedDesign) === "paired" ? 2 : 1;
+  // `screenRuns` counts observations, so each case's pass costs its own design's runs.
+  const screenPassRuns = planned.reduce((a, p) => a + cfg.screenRuns * runsPerObs(p), 0);
+
+  // Screening: everything gets `screenRuns`; only the cases the screen ranks as
+  // plausibly moved go on to the sequential test. A clean PR moves nothing, so
+  // in expectation only the FDR-sized tail continues, but a plan must be
+  // conservative, so we budget for the worst realistic case: 10% continue.
+  const screenContinueFraction = 0.1;
+  const screenSelected = Math.ceil(decidableAny.length * screenContinueFraction);
+  const meanBest = decidableAny.length > 0 ? bestRuns / decidableAny.length : 0;
+  // `screenRuns` counts *observations*, and a paired observation is two runs,
+  // so a paired screening pass costs twice what an unpaired one does. Only the
+  // screen term is scaled here: `meanBest` already comes from `paired.runs`,
+  // which is `pairs * 2`. The expected total below reads `screenRuns` the same
+  // way, and when this one did not, the two screened figures in `totals`
+  // disagreed about the price of the same pass.
+  const screenedRuns = screenPassRuns + screenSelected * meanBest;
+
+  const fixedNRuns = planned.reduce((a, p) => a + finite(p.fixedNPerArm), 0);
+
+  // The expected cost of a pull request that changed nothing, which is what a
+  // pull request almost always is, and therefore the number a budget should be
+  // set from.
+  // Both counts are *observations*. A paired observation is two runs, the
+  // candidate and the reference revision on the same seed, and so is a paired
+  // screening run. Each case is counted under the rule that stops it: a paired
+  // one in pairs to settle, an unpaired one in the runs `shouldStop` takes.
+  // Wald's SPRT figure, used for both before, runs a different rule, ignores
+  // the coupling and the suite size, and mispriced cases either way.
+  const stopRuns = new Map<string, number>();
+  const observations = (p: PlanCase) => {
+    const b = baseline.get(p.caseId);
+    if (!b || b.trials === 0) return 0;
+    if (runsPerObs(p) === 2) {
+      return expectedSettlePairs(p.baseline.rate, p.pairCoupling, cfg.mde, cfg);
+    }
+    const key = `${b.successes}/${b.trials}`;
+    const known = stopRuns.get(key);
+    if (known !== undefined) return known;
+    const runs = expectedStopRuns(b, b.successes / b.trials, m, cfg);
+    stopRuns.set(key, runs);
+    return runs;
+  };
+  const priced = planned.filter((p) => (baseline.get(p.caseId)?.trials ?? 0) > 0).length;
+  const typicalObservations = planned.reduce((a, p) => a + observations(p), 0);
+  const typicalRuns = planned.reduce((a, p) => a + observations(p) * runsPerObs(p), 0);
+  const runsPerObservation =
+    typicalObservations > 0
+      ? typicalRuns / typicalObservations
+      : recommendedDesign === "paired"
+        ? 2
+        : 1;
+  const meanTypical = priced > 0 ? typicalRuns / priced : 0;
+  const typicalScreenedRuns = Math.ceil(
+    screenPassRuns + Math.ceil(m * screenContinueFraction) * meanTypical,
+  );
+
+  const totals: PlanTotals = {
+    cases: m,
+    decidableUnpaired: decidableUnpaired.length,
+    decidableBest: decidableAny.length,
+    unpairedRuns,
+    unpairedCostUsd: unpairedRuns * cfg.costPerRunUsd,
+    pairedRuns,
+    pairedCostUsd: pairedRuns * cfg.costPerRunUsd,
+    bestRuns: round(bestRuns),
+    bestCostUsd: round(bestRuns) * cfg.costPerRunUsd,
+    screenedRuns: round(screenedRuns),
+    screenedCostUsd: round(screenedRuns) * cfg.costPerRunUsd,
+    fixedNRuns,
+    fixedNCostUsd: fixedNRuns * cfg.costPerRunUsd,
+    wallMsUnpaired: (unpairedRuns * cfg.msPerRun) / Math.max(1, cfg.concurrency),
+    wallMsPaired: (pairedRuns * cfg.msPerRun) / Math.max(1, cfg.concurrency),
+    wallMsBest: (round(bestRuns) * cfg.msPerRun) / Math.max(1, cfg.concurrency),
+    wallMsScreened: (round(screenedRuns) * cfg.msPerRun) / Math.max(1, cfg.concurrency),
+    screenContinueFraction,
+    runsPerObservation,
+    typicalRuns,
+    typicalCostUsd: typicalRuns * cfg.costPerRunUsd,
+    wallMsTypical: (typicalRuns * cfg.msPerRun) / Math.max(1, cfg.concurrency),
+    typicalScreenedRuns,
+    typicalScreenedCostUsd: typicalScreenedRuns * cfg.costPerRunUsd,
+  };
+
+  const undecidable = planned.filter(
+    (p) => p.detectability === "TOO_EXPENSIVE" || p.detectability === "IMPOSSIBLE",
+  );
+  const pairedOnly = planned.filter((p) => p.detectability === "PAIRED_ONLY");
+  const impossible = planned.filter((p) => p.detectability === "IMPOSSIBLE");
+
+  const recommendation =
+    decidableAny.length === 0
+      ? `nothing is decidable at ${(cfg.mde * 100).toFixed(0)} points against this baseline, record more baseline runs or raise the MDE`
+      : recommendedDesign === "paired"
+        ? (manyPairedOnly
+            ? `pair the runs: ${pairedOnly.length}/${m} cases have an unpaired evidence ceiling under the bar, so no candidate budget can decide them`
+            : `pair the runs: ${pairedDesignCount}/${decidableAny.length} decidable cases are cheaper paired than against the stored baseline`) +
+          (cfg.design === undefined
+            ? ` (the costs below price each case at its own design, two runs an observation where it is paired)`
+            : ` (the costs below are priced at two runs an observation, which is what pairing costs)`)
+        : totals.screenedCostUsd < totals.bestCostUsd
+          ? `screen first, then run the sequential test on what the screen keeps ` +
+            `($${totals.screenedCostUsd.toFixed(2)} vs $${totals.bestCostUsd.toFixed(2)})`
+          : `run the sequential test directly`;
+
+  // The honest cost is the cheapest plan that still decides something. A design
+  // that is free because it certifies nothing is not a cheap design.
+  const ceilingCost =
+    decidableAny.length === 0 ? Infinity : Math.min(totals.screenedCostUsd, totals.bestCostUsd);
+  // The headline is the expected cost; leading with the ceiling over-states the bill.
+  // Mirrors the ceiling's structure, the cheapest plan of its kind, so the two
+  // numbers are roughly comparable. Comparing an unscreened expected cost
+  // with a screened ceiling was the first version of this and it made the
+  // "expected" figure look *larger* than the worst case.
+  const expectedCost =
+    decidableAny.length === 0
+      ? Infinity
+      : Math.min(totals.typicalCostUsd, totals.typicalScreenedCostUsd);
+  const coverage = `${decidableAny.length}/${m} cases`;
+  // "Every case regressed" is not the expensive pull request, and calling it
+  // the worst case put a number below the headline that an operator would
+  // budget from: a 200-case suite on a 12-run baseline read "about $11.42 per
+  // pull request. Worst case ... is $9.16". The sequential test stops a case
+  // the moment it clears the bar, so a suite that really did regress finishes
+  // early, while one that did not move runs its full schedule. When the
+  // all-regressed cost lands under the expected one, say which it is.
+  //
+  // The sentence stops short of naming early stopping as *the* cause, because
+  // there is a second one: both figures are a `min` over two designs, and the
+  // two minima need not select the same design, so the cheaper figure may be
+  // pricing a plan the dearer one is not. Until both sides are made to select
+  // together, the honest claim is only that neither bounds the other.
+  const ceilingNote = !Number.isFinite(ceilingCost)
+    ? ""
+    : ceilingCost >= expectedCost
+      ? ` Worst case, every case regressing at once, is $${ceilingCost.toFixed(2)}.`
+      : ` A pull request in which every case regressed costs less, $${ceilingCost.toFixed(2)}: ` +
+        `the two do not price the same schedule, so neither bounds the other.`;
+  const verdict = !Number.isFinite(expectedCost)
+    ? `undecidable: no design reaches the bar for any case at ${(cfg.mde * 100).toFixed(0)} points with a ${medianTrials(planned)}-run baseline`
+    : expectedCost <= 5
+      ? `affordable: about $${expectedCost.toFixed(2)} per pull request, covering ${coverage}.${ceilingNote}`
+      : expectedCost <= 100
+        ? `viable but not free: about $${expectedCost.toFixed(2)} per pull request covering ${coverage}, worth gating a release on, not every commit.${ceilingNote}`
+        : `not affordable per-PR at $${cfg.costPerRunUsd.toFixed(4)}/run: about $${expectedCost.toFixed(0)} a time for ${coverage}. ` +
+          `Cut the suite, raise the MDE, or gate nightly instead of per-PR.${ceilingNote}`;
+
+  return {
+    config: cfg,
+    cases: planned,
+    totals,
+    undecidable,
+    pairedOnly,
+    impossible,
+    recommendedDesign,
+    recommendation,
+    verdict,
+    expectedCostUsd: expectedCost,
+    ceilingCostUsd: ceilingCost,
+  };
+}
+
+/**
+ * The suite size / MDE combinations that *are* affordable at a given per-run
+ * cost and budget. This is the honest answer to "is this usable for a
+ * coding-agent benchmark at $1 a run?", usually "not at 200 cases and 15
+ * points, but yes at 20 cases and 25 points".
+ */
+export interface AffordabilityCell {
+  cases: number;
+  mde: number;
+  /** runs to certify every case, the ceiling */
+  runs: number;
+  costUsd: number;
+  /** runs a pull request that moved nothing actually spends, the expected bill */
+  typicalRuns: number;
+  typicalCostUsd: number;
+  /** does the *ceiling* fit the budget? */
+  affordable: boolean;
+  /** does the *expected* cost fit the budget? */
+  typicallyAffordable: boolean;
+}
+
+export function affordabilityGrid(
+  baselineRate: number,
+  baselineTrials: number,
+  cfg: PlanConfig,
+  budgetUsd: number,
+  caseCounts: number[] = [20, 50, 100, 200],
+  mdes: number[] = [0.1, 0.15, 0.25, 0.35],
+): AffordabilityCell[] {
+  requireProbability(baselineRate, "baselineRate", "affordabilityGrid");
+  requireErrorRates(cfg.alpha, cfg.beta, "affordabilityGrid");
+  // The typical bill below takes this quantile of the baseline posterior
+  // straight from `betaQuantile`, which answers 0 or 1 for a quantile outside
+  // (0,1) and about 0 for NaN instead of refusing. `makePlan` refuses it.
+  requireOpenProbability(cfg.nullQuantile, "nullQuantile", "affordabilityGrid");
+  // makePlan refuses both; unchecked, an fdr of 2 priced a grid cheaper than one at 0.05.
+  requireOpenProbability(cfg.fdr, "fdr", "affordabilityGrid");
+  requireSettleConfidence(cfg, "affordabilityGrid");
+  requireTrialGrid(cfg, "affordabilityGrid");
+  if (!(cfg.costPerRunUsd >= 0)) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_CONFIG",
+      `affordabilityGrid: costPerRunUsd must be ≥ 0, got ${cfg.costPerRunUsd}`,
+      {
+        detail: { costPerRunUsd: cfg.costPerRunUsd },
+      },
+    );
+  }
+  for (const m of caseCounts) {
+    if (!Number.isInteger(m) || m < 1) {
+      throw new PeeksafeError(
+        "PEEKSAFE_E_CONFIG",
+        `affordabilityGrid: every case count must be a positive integer, got ${m}`,
+        {
+          detail: { cases: m },
+        },
+      );
+    }
+  }
+  // Every other planner entry point refuses a case with no baseline: `planCase`
+  // returns IMPOSSIBLE with "no baseline", `evaluatePoint` throws on
+  // `baselineRuns < 1`, and `gate` throws PEEKSAFE_E_BASELINE_MISSING. This one
+  // checked only the rate, so `baselineTrials: 0` reached `twoSamplePriors` as
+  // 0 successes in 0 trials, whose null prior is Beta(1, 1), and the grid
+  // priced the suite against "every case passes half the time" and reported it
+  // affordable, on a cheaper typical bill than the same call at a real
+  // baseline. A fractional or negative count did reach a refusal, but from
+  // `requireCounts` three frames down, naming a success count the caller never
+  // passed. `test/budget.test.ts` has the figures.
+  if (!Number.isInteger(baselineTrials) || baselineTrials < 1) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_CONFIG",
+      `affordabilityGrid: baselineTrials must be a positive integer, got ${baselineTrials}`,
+      {
+        detail: { baselineTrials },
+        hint: 'this grid prices a suite against a recorded baseline; with none the null prior is Beta(1, 1), which reads as "every case passes half the time"',
+      },
+    );
+  }
+  const out: AffordabilityCell[] = [];
+  for (const m of caseCounts) {
+    for (const mde of mdes) {
+      const bar = Math.log(ebhSoloThreshold(m, cfg.fdr));
+      const pTrue = Math.max(1e-6, baselineRate - mde);
+      const s = Math.round(baselineRate * baselineTrials);
+      const perCase = samplesForEvidence(pTrue, s, baselineTrials, mde, bar, 200_000);
+      const pairs = mcnemarSamplesForEvidence(baselineRate, mde, cfg.pairCoupling, bar, 200_000);
+      // the cheaper of the two designs, screened down to a 10% continue rate
+      const best = Math.min(Number.isFinite(perCase) ? perCase : Infinity, pairs * 2);
+      // Which design `best` just picked, needed by both totals below: they have
+      // to price the same screening pass as each other, and `screenRuns` counts
+      // observations, so a paired pass costs two runs an observation. Only the
+      // screen term is scaled, because `best` is already in runs (`pairs * 2`).
+      // When this total left the screen term unscaled it understated a paired
+      // cell by `m * screenRuns` runs, which made `affordable` below answer for
+      // a cheaper ceiling than the one it reports, and made this grid disagree
+      // with `makePlan`'s `screenedRuns` about the price of the same pass.
+      const runsPerObs = Number.isFinite(perCase) && perCase <= pairs * 2 ? 1 : 2;
+      const runs = Number.isFinite(best)
+        ? Math.ceil(m * cfg.screenRuns * runsPerObs + Math.ceil(m * 0.1) * best)
+        : Infinity;
+      const costUsd = runs * cfg.costPerRunUsd;
+      // The ceiling above prices a pull request in which every case regressed.
+      // This prices one in which nothing did, which is what a pull request
+      // almost always is.
+      const p0 = betaQuantile(1 + s, 1 + baselineTrials - s, cfg.nullQuantile);
+      const p1 = Math.max(0.005, p0 - mde);
+      const rawN = sprtExpectedN(baselineRate, p0, p1, cfg.alpha, cfg.beta);
+      // A paired cell counts the pairs `shouldStopPaired` takes to settle, as `makePlan` does; the SPRT figure ignores the coupling.
+      const perCaseTypical =
+        runsPerObs === 2
+          ? expectedSettlePairs(baselineRate, cfg.pairCoupling, mde, cfg)
+          : Number.isFinite(rawN) && rawN > 0
+            ? Math.min(cfg.maxTrials, Math.max(cfg.minTrials, Math.ceil(rawN)))
+            : cfg.maxTrials;
+      // Priced for the same design as the ceiling above (`runsPerObs`), or the
+      // cell quotes an unpaired price for a plan only pairing can execute.
+      const typicalRuns = Math.ceil(
+        (m * cfg.screenRuns + Math.ceil(m * 0.1) * perCaseTypical) * runsPerObs,
+      );
+      const typicalCostUsd = typicalRuns * cfg.costPerRunUsd;
+      out.push({
+        cases: m,
+        mde,
+        runs,
+        costUsd,
+        typicalRuns,
+        typicalCostUsd,
+        affordable: costUsd <= budgetUsd,
+        typicallyAffordable: typicalCostUsd <= budgetUsd,
+      });
+    }
+  }
+  return out;
+}