peeksafe 0.0.0-stage → 0.2.0

package/dist/plan.js

@@ -0,0 +1,626 @@
+/**
+ * plan.ts: power economics. What will this cost, and can we even detect it?
+ *
+ * Without a plan, the first sign that a suite cannot be decided is a large bill
+ * and no certified case. This module tells you *before* you spend anything, and it will tell you that the honest answer is
+ * sometimes "you cannot detect this at any price".
+ *
+ * The three numbers that matter, per case:
+ *
+ *  1. **The bar.** e-BH at FDR q over m cases certifies a case on its own only
+ *     once its e-value clears m/q. That is the target the evidence has to hit.
+ *
+ *  2. **The ceiling.** `evidenceCeilingLogE`, the limit of the unpaired
+ *     two-sample e-value as candidate runs go to infinity. Both marginals are
+ *     the same binomial likelihood under different priors, so everything that
+ *     grows with n cancels and the Bayes factor converges to the prior density
+ *     ratio at the observed rate. If the ceiling is under the bar, no number of
+ *     candidate runs will certify the case. This is not a budget problem and no
+ *     budget fixes it: **more baseline runs** fix it, by sharpening the null.
+ *
+ *  3. **The paired alternative.** Re-run the baseline alongside the candidate
+ *     on the same seed and the null becomes an exact point mass at ½ (a
+ *     discordant pair points either way with probability exactly one half under
+ *     "nothing changed"). Nothing is estimated, so nothing caps the evidence:
+ *     the paired e-value grows linearly in discordant pairs. It costs two runs
+ *     per observation and it is still, routinely, the cheaper design, and it
+ *     is the only one that works at all for a case whose ceiling is under the
+ *     bar.
+ *
+ * Everything here is a *plan*: it evaluates the mean trajectory (the candidate
+ * scores exactly its expected successes). Realised sample counts vary around
+ * it.
+ */
+import { baselineNullRate } from "./baseline.js";
+import { PeeksafeError, requireCounts, requireErrorRates, requireOpenProbability, requireProbability, requireRunCeiling, requireTrialGrid, requireUniqueCaseIds, } from "./errors.js";
+import { betaQuantile, ebhSoloThreshold, evidenceCeilingLogE, mcnemarSamplesForEvidence, pairPhi, sampleSizeTwoProportion, samplesForEvidence, sprtExpectedN, } from "./stats.js";
+import { DEFAULT_STOP_OPTIONS, settleLogE, shouldStop } from "./stop.js";
+export const DEFAULT_PLAN = {
+    mde: 0.15,
+    fdr: 0.05,
+    // These three are *assumptions* until you measure them on a real run: cost
+    // and latency from your runner, coupling with `pairPhi` on a pilot.
+    costPerRunUsd: 0.0021,
+    msPerRun: 780,
+    pairCoupling: 0.9,
+    screenRuns: 4,
+    concurrency: 8,
+    runsPerCaseCeiling: 2000,
+    alpha: 0.05,
+    beta: 0.1,
+    minTrials: 8,
+    maxTrials: 96,
+    batchSize: 4,
+    nullQuantile: 0.5,
+};
+const round = (x) => (Number.isFinite(x) ? Math.ceil(x) : Infinity);
+const medianTrials = (ps) => {
+    const xs = ps.map((p) => p.baseline.trials).sort((a, b) => a - b);
+    return xs.length === 0 ? 0 : xs[Math.floor(xs.length / 2)];
+};
+/**
+ * How many baseline runs would lift this case's evidence ceiling over the bar?
+ * The *smallest* such number, holding the baseline *rate* fixed, "if I keep
+ * measuring main at the same quality, how long until this case becomes
+ * decidable at all?"
+ *
+ * Smallest, and not a level the case stays decidable above: the ceiling is a
+ * sawtooth in the baseline size (see the comment in the body), so a baseline a
+ * few runs longer than this can fall back under the bar. Treat it as the floor
+ * it is rather than a promise about a bigger baseline.
+ */
+function baselineRunsToClear(rate, mde, barLogE, cap = 100_000) {
+    const at = (n) => {
+        const s = Math.round(rate * n);
+        return evidenceCeilingLogE(Math.max(1e-6, Math.min(1 - 1e-6, rate - mde)), s, n, mde);
+    };
+    // `at` is not monotone in `n`, so this cannot bisect the way
+    // `samplesForEvidence` does. Holding the rate fixed still forces an integer
+    // success count, so the implied rate jitters around `rate` and the ceiling
+    // drops hard every time the count ticks over: at 99% losing 10 points it
+    // clears the bar at 49 runs, falls back under it at 51, and does not clear
+    // again until 71. Bisecting that returns whichever crossing the doubling
+    // happened to bracket, which is no particular quantity at all; it answered 81
+    // for the README's own case where 75 is enough, and 5 for every case that
+    // needed fewer, because it bisected (4, 8]. Doubling now only bounds the
+    // scan, and never steps past `cap`, so the answer is one this search checked.
+    let hi = 1;
+    while (hi < cap && at(hi) < barLogE)
+        hi = Math.min(cap, hi * 2);
+    if (at(hi) < barLogE)
+        return null;
+    for (let n = 1; n < hi; n++)
+        if (at(n) >= barLogE)
+            return n;
+    return hi;
+}
+function requirePrices(cfg, where) {
+    if (!(cfg.costPerRunUsd >= 0) || !(cfg.msPerRun >= 0)) {
+        throw new PeeksafeError("PEEKSAFE_E_CONFIG", `${where}: costPerRunUsd and msPerRun must be ≥ 0`, {
+            detail: { costPerRunUsd: cfg.costPerRunUsd, msPerRun: cfg.msPerRun },
+        });
+    }
+}
+export function planCase(c, b, cfg, m) {
+    // Exported on its own, so it refuses what makePlan refuses before calling it: unchecked, a -5/60 baseline was IMPOSSIBLE and a price of -1 billed -$152.
+    if (b)
+        requireCounts(b.successes, b.trials, `planCase(${c.id}).baseline`);
+    requireErrorRates(cfg.alpha, cfg.beta, "planCase");
+    requirePrices(cfg, "planCase");
+    const barLogE = Math.log(ebhSoloThreshold(m, cfg.fdr));
+    const trials = b?.trials ?? 0;
+    const successes = b?.successes ?? 0;
+    const rate = trials > 0 ? successes / trials : 0;
+    const alt = Math.max(0, rate - cfg.mde);
+    const base = {
+        caseId: c.id,
+        // `suite` groups cases into correlation families. A caller who does not
+        // group them gets one family per case, which is the conservative reading.
+        suite: c.suite ?? c.id,
+        baseline: { successes, trials, rate },
+        alternativeRate: alt,
+        barLogE,
+        fixedNPerArm: sampleSizeTwoProportion(Math.max(1e-6, rate), cfg.mde, cfg.alpha, cfg.beta),
+    };
+    if (trials === 0) {
+        return {
+            ...base,
+            ceilingLogE: 0,
+            unpaired: { runs: Infinity, costUsd: Infinity, wallMs: Infinity },
+            paired: { pairs: Infinity, runs: Infinity, costUsd: Infinity, wallMs: Infinity },
+            pairCoupling: cfg.pairCoupling,
+            detectability: "IMPOSSIBLE",
+            design: null,
+            baselineRunsNeeded: null,
+            reason: "no baseline: nothing to compare against, record one first",
+        };
+    }
+    if (rate <= cfg.mde) {
+        return {
+            ...base,
+            ceilingLogE: 0,
+            unpaired: { runs: Infinity, costUsd: Infinity, wallMs: Infinity },
+            paired: { pairs: Infinity, runs: Infinity, costUsd: Infinity, wallMs: Infinity },
+            pairCoupling: cfg.pairCoupling,
+            detectability: "IMPOSSIBLE",
+            design: null,
+            baselineRunsNeeded: null,
+            reason: `baseline rate ${(rate * 100).toFixed(0)}% cannot lose ${(cfg.mde * 100).toFixed(0)} points, ` +
+                `the effect is arithmetically impossible, not merely expensive`,
+        };
+    }
+    const pTrue = Math.max(1e-6, Math.min(1 - 1e-6, alt));
+    const ceilingLogE = evidenceCeilingLogE(pTrue, successes, trials, cfg.mde);
+    const unpairedRuns = samplesForEvidence(pTrue, successes, trials, cfg.mde, barLogE, cfg.runsPerCaseCeiling * 8);
+    const pairs = mcnemarSamplesForEvidence(rate, cfg.mde, cfg.pairCoupling, barLogE, cfg.runsPerCaseCeiling * 8);
+    const unpaired = {
+        runs: unpairedRuns,
+        costUsd: unpairedRuns * cfg.costPerRunUsd,
+        wallMs: unpairedRuns * cfg.msPerRun,
+    };
+    const paired = {
+        pairs,
+        runs: pairs * 2,
+        costUsd: pairs * 2 * cfg.costPerRunUsd,
+        wallMs: pairs * 2 * cfg.msPerRun,
+    };
+    let detectability;
+    let reason;
+    if (Number.isFinite(unpairedRuns) && unpairedRuns <= cfg.runsPerCaseCeiling) {
+        detectability = "UNPAIRED";
+        reason = `${unpairedRuns} candidate runs against the stored baseline`;
+    }
+    else if (Number.isFinite(pairs) && pairs * 2 <= cfg.runsPerCaseCeiling) {
+        detectability = "PAIRED_ONLY";
+        reason = Number.isFinite(unpairedRuns)
+            ? `unpaired needs ${unpairedRuns} runs (past the ${cfg.runsPerCaseCeiling} ceiling); paired needs ${pairs * 2}`
+            : `unpaired evidence ceiling is e^${ceilingLogE.toFixed(1)}, under the bar e^${barLogE.toFixed(1)}, ` +
+                `no number of candidate runs certifies it. Paired: ${pairs * 2} runs.`;
+    }
+    else if (Number.isFinite(unpairedRuns) || Number.isFinite(pairs)) {
+        detectability = "TOO_EXPENSIVE";
+        reason = `cheapest design needs ${Math.min(unpairedRuns, pairs * 2)} runs, past the ${cfg.runsPerCaseCeiling} ceiling`;
+    }
+    else {
+        detectability = "IMPOSSIBLE";
+        reason = `neither design reaches the bar within ${cfg.runsPerCaseCeiling * 8} runs`;
+    }
+    const baselineRunsNeeded = detectability === "UNPAIRED" ? null : baselineRunsToClear(rate, cfg.mde, barLogE);
+    const design = detectability === "PAIRED_ONLY"
+        ? "paired"
+        : detectability !== "UNPAIRED"
+            ? null
+            : paired.runs < unpaired.runs
+                ? "paired"
+                : "unpaired";
+    return {
+        ...base,
+        ceilingLogE,
+        unpaired,
+        paired,
+        pairCoupling: cfg.pairCoupling,
+        detectability,
+        design,
+        baselineRunsNeeded,
+        reason,
+    };
+}
+/**
+ * Expected total samples for the sequential design, priced under Wald's
+ * expected sample number at a supplied "true" rate per case, and clamped into
+ * the gate's own [minTrials, maxTrials] batch grid.
+ *
+ * `makePlan` no longer bills from it: an unpaired case there is counted in
+ * the runs `shouldStop` takes, which this figure misses either way (#105).
+ * `samplesForEvidence`, the one behind `PlanCase.unpaired.runs`, answers a
+ * different question: how many runs it takes to *certify a regression*. A case
+ * that did not regress never gets there; it hits the H₀ wall in a few batches
+ * and stops. Pricing every case at the certification cost over-predicts the
+ * bill.
+ *
+ * `trueRate` defaults to the case's own baseline rate, i.e. "nothing moved", * which is what almost every pull request looks like.
+ */
+export function expectedSequentialSamples(cases, baseline, cfg, trueRate = (_id, b) => b.trials > 0 ? b.successes / b.trials : 0) {
+    requireTrialGrid(cfg, "expectedSequentialSamples");
+    // A repeated id was added to `samples` once per listing but kept once in `perCase`, so the two disagreed.
+    requireUniqueCaseIds(cases, "expectedSequentialSamples");
+    const perCase = new Map();
+    let total = 0;
+    for (const c of cases) {
+        const b = baseline.get(c.id);
+        if (!b || b.trials === 0)
+            continue;
+        const p0 = baselineNullRate(b, cfg.nullQuantile);
+        const p1 = Math.max(0.005, p0 - cfg.mde);
+        const p = Math.max(0, Math.min(1, trueRate(c.id, b)));
+        const raw = sprtExpectedN(p, p0, p1, cfg.alpha, cfg.beta);
+        const capped = onBatchGrid(Number.isFinite(raw) && raw > 0 ? Math.ceil(raw) : cfg.maxTrials, cfg);
+        perCase.set(c.id, capped);
+        total += capped;
+    }
+    return { samples: total, perCase };
+}
+/** Refuses a set `futilityConfidence` outside (0, 1), where the settle bar is 0 or Infinity. */
+export function requireSettleConfidence(cfg, where) {
+    if (cfg.futilityConfidence !== undefined) {
+        requireOpenProbability(cfg.futilityConfidence, "futilityConfidence", where);
+    }
+}
+// The gate only ever adds whole batches past the minimum, so a prediction is rounded onto the grid it samples on.
+function onBatchGrid(n, cfg) {
+    const clamped = Math.min(cfg.maxTrials, Math.max(cfg.minTrials, n));
+    const batched = cfg.minTrials + Math.ceil(Math.max(0, clamped - cfg.minTrials) / cfg.batchSize) * cfg.batchSize;
+    return Math.min(cfg.maxTrials, batched);
+}
+/**
+ * Expected pairs for a paired case that did not move: the first pair count at
+ * which `shouldStopPaired`'s settle e-value, on the mean trajectory at coupling
+ * `rho` and `cfg.futilityConfidence` (the stopping rule's default when unset),
+ * rules out a drop of `mde`, on the same batch grid as
+ * `expectedSequentialSamples`, and `maxTrials` when it never does. The
+ * unpaired SPRT figure knows nothing of this rule or of the coupling, which
+ * sets how often an unmoved pair disagrees.
+ */
+export function expectedSettlePairs(rate, rho, mde, cfg) {
+    // Unmoved arms disagree each way only when uncoupled: (1 - rho) p (1 - p).
+    const each = (1 - rho) * rate * (1 - rate);
+    const confidence = cfg.futilityConfidence ?? DEFAULT_STOP_OPTIONS.futilityConfidence;
+    const target = -Math.log(1 - confidence);
+    for (let n = 1; n <= cfg.maxTrials; n++) {
+        const counts = { bothPass: n * (1 - 2 * each), worse: n * each, better: n * each, bothFail: 0 };
+        if (settleLogE(counts, mde) >= target)
+            return onBatchGrid(n, cfg);
+    }
+    return cfg.maxTrials;
+}
+/**
+ * Expected runs `shouldStop` takes to stop an unpaired case whose candidate
+ * passes at `rate`, checked at `minTrials` and then every `batchSize` runs up
+ * to `maxTrials`, in a suite of `suiteSize`. Exact rather than simulated:
+ * the decision depends on the candidate only through its counts, so a dynamic
+ * program carries the distribution of the success count from one check to the
+ * next. Rounded up to a whole run; an expectation is not a grid point.
+ */
+function expectedStopRuns(baseline, rate, suiteSize, cfg) {
+    const options = {
+        suiteSize,
+        mde: cfg.mde,
+        fdr: cfg.fdr,
+        futilityConfidence: cfg.futilityConfidence,
+    };
+    // live[k]: probability of k successes in n runs with the case still running.
+    let live = new Float64Array([1]);
+    let n = 0;
+    let expected = 0;
+    for (let step = cfg.minTrials;; step = Math.min(cfg.batchSize, cfg.maxTrials - n)) {
+        for (let i = 0; i < step; i++) {
+            const next = new Float64Array(live.length + 1);
+            for (let k = 0; k < live.length; k++) {
+                next[k] += live[k] * (1 - rate);
+                next[k + 1] += live[k] * rate;
+            }
+            live = next;
+        }
+        n += step;
+        let running = 0;
+        for (let k = 0; k <= n; k++) {
+            if (live[k] === 0)
+                continue;
+            if (n >= cfg.maxTrials || shouldStop({ successes: k, trials: n }, baseline, options).stop) {
+                expected += live[k] * n;
+                live[k] = 0;
+            }
+            else {
+                running += live[k];
+            }
+        }
+        if (running === 0)
+            break;
+    }
+    // Float residue must not push an exact whole number up a run.
+    return Math.ceil(expected - 1e-9);
+}
+/**
+ * Plan a suite. `pilots` holds a pair table per case from a pilot run of the
+ * paired design, ideally with both arms on the same revision; a case with one
+ * prices pairing at its measured `pairPhi` instead of `cfg.pairCoupling`, so
+ * its `design` is chosen from a coupling the runner actually delivers.
+ */
+export function makePlan(cases, baseline, cfg = DEFAULT_PLAN, pilots = new Map()) {
+    // Open interval: mde = 0 asks for a zero-point drop and fdr = 0 asks for a
+    // bar of Infinity. Both used to be accepted and produce a plan in which
+    // nothing is decidable, for a reason the printed message blamed on the
+    // baseline.
+    requireOpenProbability(cfg.mde, "mde", "makePlan");
+    requireOpenProbability(cfg.fdr, "fdr", "makePlan");
+    // The two error rates the SPRT costing runs on. Left unchecked they reached
+    // `sprtExpectedN`, which answers NaN or Infinity outside (0,1), and both
+    // callers of it here gate on `Number.isFinite(raw) && raw > 0` and quote
+    // `maxTrials` when that fails. So an alpha of 2 produced a complete plan with
+    // every case at the per-case cap and no error anywhere.
+    requireErrorRates(cfg.alpha, cfg.beta, "makePlan");
+    requireProbability(cfg.pairCoupling, "pairCoupling", "makePlan");
+    // Refused by `baselineNullRate` already, but only once a case with a baseline
+    // reaches it, and under that function's name for it, `q`. Checked here so
+    // every planner entry point refuses it the same way and names it.
+    requireOpenProbability(cfg.nullQuantile, "nullQuantile", "makePlan");
+    requireSettleConfidence(cfg, "makePlan");
+    requireRunCeiling(cfg.runsPerCaseCeiling, "makePlan");
+    requireTrialGrid(cfg, "makePlan");
+    if (cases.length === 0) {
+        throw new PeeksafeError("PEEKSAFE_E_SUITE_EMPTY", "makePlan: no cases to plan for", {
+            hint: "pass at least one case; makePlan prices a suite, not an empty list",
+        });
+    }
+    // `gate` refuses a repeated id; planned, it was priced twice and counted twice in the suite size the bar is set from.
+    requireUniqueCaseIds(cases, "makePlan");
+    requirePrices(cfg, "makePlan");
+    const m = cases.length;
+    const planned = cases.map((c) => {
+        const pilot = pilots.get(c.id);
+        // pairPhi validates the counts, so a malformed pilot is refused even when it sums to 0.
+        const phi = pilot ? pairPhi(pilot) : 0;
+        // An empty pilot measured nothing, so it leaves the configured coupling in place.
+        const measured = pilot && pilot.bothPass + pilot.worse + pilot.better + pilot.bothFail > 0;
+        // A negative phi is noise around independence for a design that couples by construction.
+        const caseCfg = measured ? { ...cfg, pairCoupling: Math.max(0, phi) } : cfg;
+        return planCase(c, baseline.get(c.id), caseCfg, m);
+    });
+    const finite = (x) => (Number.isFinite(x) ? x : 0);
+    const decidableUnpaired = planned.filter((p) => p.detectability === "UNPAIRED");
+    const decidableAny = planned.filter((p) => p.detectability === "UNPAIRED" || p.detectability === "PAIRED_ONLY");
+    const unpairedRuns = decidableUnpaired.reduce((a, p) => a + finite(p.unpaired.runs), 0);
+    const pairedRuns = decidableAny.reduce((a, p) => a + finite(p.paired.runs), 0);
+    // Per case, take the cheaper design that can actually decide it. This is the
+    // number the recommendation and the affordability verdict are built on: a
+    // design that is cheap because it decides nothing is not cheap.
+    const bestRuns = decidableAny.reduce((a, p) => a + Math.min(finite(p.unpaired.runs) || Infinity, finite(p.paired.runs) || Infinity), 0);
+    const pairedOnlyCount = planned.filter((p) => p.detectability === "PAIRED_ONLY").length;
+    const pairedDesignCount = decidableAny.filter((p) => p.design === "paired").length;
+    // The recommendation string reads these same two flags, so it cannot drift from the design it names.
+    const manyPairedOnly = pairedOnlyCount > cases.length * 0.25;
+    const mostlyPaired = pairedDesignCount * 2 > decidableAny.length;
+    const recommendedDesign = cfg.design ??
+        (decidableAny.length > 0 && (manyPairedOnly || mostlyPaired) ? "paired" : "unpaired");
+    // Per case, like `bestRuns`: one suite-wide design billed a mixed suite as if every case ran it.
+    const runsPerObs = (p) => (cfg.design ?? p.design ?? recommendedDesign) === "paired" ? 2 : 1;
+    // `screenRuns` counts observations, so each case's pass costs its own design's runs.
+    const screenPassRuns = planned.reduce((a, p) => a + cfg.screenRuns * runsPerObs(p), 0);
+    // Screening: everything gets `screenRuns`; only the cases the screen ranks as
+    // plausibly moved go on to the sequential test. A clean PR moves nothing, so
+    // in expectation only the FDR-sized tail continues, but a plan must be
+    // conservative, so we budget for the worst realistic case: 10% continue.
+    const screenContinueFraction = 0.1;
+    const screenSelected = Math.ceil(decidableAny.length * screenContinueFraction);
+    const meanBest = decidableAny.length > 0 ? bestRuns / decidableAny.length : 0;
+    // `screenRuns` counts *observations*, and a paired observation is two runs,
+    // so a paired screening pass costs twice what an unpaired one does. Only the
+    // screen term is scaled here: `meanBest` already comes from `paired.runs`,
+    // which is `pairs * 2`. The expected total below reads `screenRuns` the same
+    // way, and when this one did not, the two screened figures in `totals`
+    // disagreed about the price of the same pass.
+    const screenedRuns = screenPassRuns + screenSelected * meanBest;
+    const fixedNRuns = planned.reduce((a, p) => a + finite(p.fixedNPerArm), 0);
+    // The expected cost of a pull request that changed nothing, which is what a
+    // pull request almost always is, and therefore the number a budget should be
+    // set from.
+    // Both counts are *observations*. A paired observation is two runs, the
+    // candidate and the reference revision on the same seed, and so is a paired
+    // screening run. Each case is counted under the rule that stops it: a paired
+    // one in pairs to settle, an unpaired one in the runs `shouldStop` takes.
+    // Wald's SPRT figure, used for both before, runs a different rule, ignores
+    // the coupling and the suite size, and mispriced cases either way.
+    const stopRuns = new Map();
+    const observations = (p) => {
+        const b = baseline.get(p.caseId);
+        if (!b || b.trials === 0)
+            return 0;
+        if (runsPerObs(p) === 2) {
+            return expectedSettlePairs(p.baseline.rate, p.pairCoupling, cfg.mde, cfg);
+        }
+        const key = `${b.successes}/${b.trials}`;
+        const known = stopRuns.get(key);
+        if (known !== undefined)
+            return known;
+        const runs = expectedStopRuns(b, b.successes / b.trials, m, cfg);
+        stopRuns.set(key, runs);
+        return runs;
+    };
+    const priced = planned.filter((p) => (baseline.get(p.caseId)?.trials ?? 0) > 0).length;
+    const typicalObservations = planned.reduce((a, p) => a + observations(p), 0);
+    const typicalRuns = planned.reduce((a, p) => a + observations(p) * runsPerObs(p), 0);
+    const runsPerObservation = typicalObservations > 0
+        ? typicalRuns / typicalObservations
+        : recommendedDesign === "paired"
+            ? 2
+            : 1;
+    const meanTypical = priced > 0 ? typicalRuns / priced : 0;
+    const typicalScreenedRuns = Math.ceil(screenPassRuns + Math.ceil(m * screenContinueFraction) * meanTypical);
+    const totals = {
+        cases: m,
+        decidableUnpaired: decidableUnpaired.length,
+        decidableBest: decidableAny.length,
+        unpairedRuns,
+        unpairedCostUsd: unpairedRuns * cfg.costPerRunUsd,
+        pairedRuns,
+        pairedCostUsd: pairedRuns * cfg.costPerRunUsd,
+        bestRuns: round(bestRuns),
+        bestCostUsd: round(bestRuns) * cfg.costPerRunUsd,
+        screenedRuns: round(screenedRuns),
+        screenedCostUsd: round(screenedRuns) * cfg.costPerRunUsd,
+        fixedNRuns,
+        fixedNCostUsd: fixedNRuns * cfg.costPerRunUsd,
+        wallMsUnpaired: (unpairedRuns * cfg.msPerRun) / Math.max(1, cfg.concurrency),
+        wallMsPaired: (pairedRuns * cfg.msPerRun) / Math.max(1, cfg.concurrency),
+        wallMsBest: (round(bestRuns) * cfg.msPerRun) / Math.max(1, cfg.concurrency),
+        wallMsScreened: (round(screenedRuns) * cfg.msPerRun) / Math.max(1, cfg.concurrency),
+        screenContinueFraction,
+        runsPerObservation,
+        typicalRuns,
+        typicalCostUsd: typicalRuns * cfg.costPerRunUsd,
+        wallMsTypical: (typicalRuns * cfg.msPerRun) / Math.max(1, cfg.concurrency),
+        typicalScreenedRuns,
+        typicalScreenedCostUsd: typicalScreenedRuns * cfg.costPerRunUsd,
+    };
+    const undecidable = planned.filter((p) => p.detectability === "TOO_EXPENSIVE" || p.detectability === "IMPOSSIBLE");
+    const pairedOnly = planned.filter((p) => p.detectability === "PAIRED_ONLY");
+    const impossible = planned.filter((p) => p.detectability === "IMPOSSIBLE");
+    const recommendation = decidableAny.length === 0
+        ? `nothing is decidable at ${(cfg.mde * 100).toFixed(0)} points against this baseline, record more baseline runs or raise the MDE`
+        : recommendedDesign === "paired"
+            ? (manyPairedOnly
+                ? `pair the runs: ${pairedOnly.length}/${m} cases have an unpaired evidence ceiling under the bar, so no candidate budget can decide them`
+                : `pair the runs: ${pairedDesignCount}/${decidableAny.length} decidable cases are cheaper paired than against the stored baseline`) +
+                (cfg.design === undefined
+                    ? ` (the costs below price each case at its own design, two runs an observation where it is paired)`
+                    : ` (the costs below are priced at two runs an observation, which is what pairing costs)`)
+            : totals.screenedCostUsd < totals.bestCostUsd
+                ? `screen first, then run the sequential test on what the screen keeps ` +
+                    `($${totals.screenedCostUsd.toFixed(2)} vs $${totals.bestCostUsd.toFixed(2)})`
+                : `run the sequential test directly`;
+    // The honest cost is the cheapest plan that still decides something. A design
+    // that is free because it certifies nothing is not a cheap design.
+    const ceilingCost = decidableAny.length === 0 ? Infinity : Math.min(totals.screenedCostUsd, totals.bestCostUsd);
+    // The headline is the expected cost; leading with the ceiling over-states the bill.
+    // Mirrors the ceiling's structure, the cheapest plan of its kind, so the two
+    // numbers are roughly comparable. Comparing an unscreened expected cost
+    // with a screened ceiling was the first version of this and it made the
+    // "expected" figure look *larger* than the worst case.
+    const expectedCost = decidableAny.length === 0
+        ? Infinity
+        : Math.min(totals.typicalCostUsd, totals.typicalScreenedCostUsd);
+    const coverage = `${decidableAny.length}/${m} cases`;
+    // "Every case regressed" is not the expensive pull request, and calling it
+    // the worst case put a number below the headline that an operator would
+    // budget from: a 200-case suite on a 12-run baseline read "about $11.42 per
+    // pull request. Worst case ... is $9.16". The sequential test stops a case
+    // the moment it clears the bar, so a suite that really did regress finishes
+    // early, while one that did not move runs its full schedule. When the
+    // all-regressed cost lands under the expected one, say which it is.
+    //
+    // The sentence stops short of naming early stopping as *the* cause, because
+    // there is a second one: both figures are a `min` over two designs, and the
+    // two minima need not select the same design, so the cheaper figure may be
+    // pricing a plan the dearer one is not. Until both sides are made to select
+    // together, the honest claim is only that neither bounds the other.
+    const ceilingNote = !Number.isFinite(ceilingCost)
+        ? ""
+        : ceilingCost >= expectedCost
+            ? ` Worst case, every case regressing at once, is $${ceilingCost.toFixed(2)}.`
+            : ` A pull request in which every case regressed costs less, $${ceilingCost.toFixed(2)}: ` +
+                `the two do not price the same schedule, so neither bounds the other.`;
+    const verdict = !Number.isFinite(expectedCost)
+        ? `undecidable: no design reaches the bar for any case at ${(cfg.mde * 100).toFixed(0)} points with a ${medianTrials(planned)}-run baseline`
+        : expectedCost <= 5
+            ? `affordable: about $${expectedCost.toFixed(2)} per pull request, covering ${coverage}.${ceilingNote}`
+            : expectedCost <= 100
+                ? `viable but not free: about $${expectedCost.toFixed(2)} per pull request covering ${coverage}, worth gating a release on, not every commit.${ceilingNote}`
+                : `not affordable per-PR at $${cfg.costPerRunUsd.toFixed(4)}/run: about $${expectedCost.toFixed(0)} a time for ${coverage}. ` +
+                    `Cut the suite, raise the MDE, or gate nightly instead of per-PR.${ceilingNote}`;
+    return {
+        config: cfg,
+        cases: planned,
+        totals,
+        undecidable,
+        pairedOnly,
+        impossible,
+        recommendedDesign,
+        recommendation,
+        verdict,
+        expectedCostUsd: expectedCost,
+        ceilingCostUsd: ceilingCost,
+    };
+}
+export function affordabilityGrid(baselineRate, baselineTrials, cfg, budgetUsd, caseCounts = [20, 50, 100, 200], mdes = [0.1, 0.15, 0.25, 0.35]) {
+    requireProbability(baselineRate, "baselineRate", "affordabilityGrid");
+    requireErrorRates(cfg.alpha, cfg.beta, "affordabilityGrid");
+    // The typical bill below takes this quantile of the baseline posterior
+    // straight from `betaQuantile`, which answers 0 or 1 for a quantile outside
+    // (0,1) and about 0 for NaN instead of refusing. `makePlan` refuses it.
+    requireOpenProbability(cfg.nullQuantile, "nullQuantile", "affordabilityGrid");
+    // makePlan refuses both; unchecked, an fdr of 2 priced a grid cheaper than one at 0.05.
+    requireOpenProbability(cfg.fdr, "fdr", "affordabilityGrid");
+    requireSettleConfidence(cfg, "affordabilityGrid");
+    requireTrialGrid(cfg, "affordabilityGrid");
+    if (!(cfg.costPerRunUsd >= 0)) {
+        throw new PeeksafeError("PEEKSAFE_E_CONFIG", `affordabilityGrid: costPerRunUsd must be ≥ 0, got ${cfg.costPerRunUsd}`, {
+            detail: { costPerRunUsd: cfg.costPerRunUsd },
+        });
+    }
+    for (const m of caseCounts) {
+        if (!Number.isInteger(m) || m < 1) {
+            throw new PeeksafeError("PEEKSAFE_E_CONFIG", `affordabilityGrid: every case count must be a positive integer, got ${m}`, {
+                detail: { cases: m },
+            });
+        }
+    }
+    // Every other planner entry point refuses a case with no baseline: `planCase`
+    // returns IMPOSSIBLE with "no baseline", `evaluatePoint` throws on
+    // `baselineRuns < 1`, and `gate` throws PEEKSAFE_E_BASELINE_MISSING. This one
+    // checked only the rate, so `baselineTrials: 0` reached `twoSamplePriors` as
+    // 0 successes in 0 trials, whose null prior is Beta(1, 1), and the grid
+    // priced the suite against "every case passes half the time" and reported it
+    // affordable, on a cheaper typical bill than the same call at a real
+    // baseline. A fractional or negative count did reach a refusal, but from
+    // `requireCounts` three frames down, naming a success count the caller never
+    // passed. `test/budget.test.ts` has the figures.
+    if (!Number.isInteger(baselineTrials) || baselineTrials < 1) {
+        throw new PeeksafeError("PEEKSAFE_E_CONFIG", `affordabilityGrid: baselineTrials must be a positive integer, got ${baselineTrials}`, {
+            detail: { baselineTrials },
+            hint: 'this grid prices a suite against a recorded baseline; with none the null prior is Beta(1, 1), which reads as "every case passes half the time"',
+        });
+    }
+    const out = [];
+    for (const m of caseCounts) {
+        for (const mde of mdes) {
+            const bar = Math.log(ebhSoloThreshold(m, cfg.fdr));
+            const pTrue = Math.max(1e-6, baselineRate - mde);
+            const s = Math.round(baselineRate * baselineTrials);
+            const perCase = samplesForEvidence(pTrue, s, baselineTrials, mde, bar, 200_000);
+            const pairs = mcnemarSamplesForEvidence(baselineRate, mde, cfg.pairCoupling, bar, 200_000);
+            // the cheaper of the two designs, screened down to a 10% continue rate
+            const best = Math.min(Number.isFinite(perCase) ? perCase : Infinity, pairs * 2);
+            // Which design `best` just picked, needed by both totals below: they have
+            // to price the same screening pass as each other, and `screenRuns` counts
+            // observations, so a paired pass costs two runs an observation. Only the
+            // screen term is scaled, because `best` is already in runs (`pairs * 2`).
+            // When this total left the screen term unscaled it understated a paired
+            // cell by `m * screenRuns` runs, which made `affordable` below answer for
+            // a cheaper ceiling than the one it reports, and made this grid disagree
+            // with `makePlan`'s `screenedRuns` about the price of the same pass.
+            const runsPerObs = Number.isFinite(perCase) && perCase <= pairs * 2 ? 1 : 2;
+            const runs = Number.isFinite(best)
+                ? Math.ceil(m * cfg.screenRuns * runsPerObs + Math.ceil(m * 0.1) * best)
+                : Infinity;
+            const costUsd = runs * cfg.costPerRunUsd;
+            // The ceiling above prices a pull request in which every case regressed.
+            // This prices one in which nothing did, which is what a pull request
+            // almost always is.
+            const p0 = betaQuantile(1 + s, 1 + baselineTrials - s, cfg.nullQuantile);
+            const p1 = Math.max(0.005, p0 - mde);
+            const rawN = sprtExpectedN(baselineRate, p0, p1, cfg.alpha, cfg.beta);
+            // A paired cell counts the pairs `shouldStopPaired` takes to settle, as `makePlan` does; the SPRT figure ignores the coupling.
+            const perCaseTypical = runsPerObs === 2
+                ? expectedSettlePairs(baselineRate, cfg.pairCoupling, mde, cfg)
+                : Number.isFinite(rawN) && rawN > 0
+                    ? Math.min(cfg.maxTrials, Math.max(cfg.minTrials, Math.ceil(rawN)))
+                    : cfg.maxTrials;
+            // Priced for the same design as the ceiling above (`runsPerObs`), or the
+            // cell quotes an unpaired price for a plan only pairing can execute.
+            const typicalRuns = Math.ceil((m * cfg.screenRuns + Math.ceil(m * 0.1) * perCaseTypical) * runsPerObs);
+            const typicalCostUsd = typicalRuns * cfg.costPerRunUsd;
+            out.push({
+                cases: m,
+                mde,
+                runs,
+                costUsd,
+                typicalRuns,
+                typicalCostUsd,
+                affordable: costUsd <= budgetUsd,
+                typicallyAffordable: typicalCostUsd <= budgetUsd,
+            });
+        }
+    }
+    return out;
+}
+//# sourceMappingURL=plan.js.map