peeksafe 0.0.0-stage → 0.2.0

package/dist/plan.d.ts

@@ -0,0 +1,280 @@
+/**
+ * plan.ts: power economics. What will this cost, and can we even detect it?
+ *
+ * Without a plan, the first sign that a suite cannot be decided is a large bill
+ * and no certified case. This module tells you *before* you spend anything, and it will tell you that the honest answer is
+ * sometimes "you cannot detect this at any price".
+ *
+ * The three numbers that matter, per case:
+ *
+ *  1. **The bar.** e-BH at FDR q over m cases certifies a case on its own only
+ *     once its e-value clears m/q. That is the target the evidence has to hit.
+ *
+ *  2. **The ceiling.** `evidenceCeilingLogE`, the limit of the unpaired
+ *     two-sample e-value as candidate runs go to infinity. Both marginals are
+ *     the same binomial likelihood under different priors, so everything that
+ *     grows with n cancels and the Bayes factor converges to the prior density
+ *     ratio at the observed rate. If the ceiling is under the bar, no number of
+ *     candidate runs will certify the case. This is not a budget problem and no
+ *     budget fixes it: **more baseline runs** fix it, by sharpening the null.
+ *
+ *  3. **The paired alternative.** Re-run the baseline alongside the candidate
+ *     on the same seed and the null becomes an exact point mass at ½ (a
+ *     discordant pair points either way with probability exactly one half under
+ *     "nothing changed"). Nothing is estimated, so nothing caps the evidence:
+ *     the paired e-value grows linearly in discordant pairs. It costs two runs
+ *     per observation and it is still, routinely, the cheaper design, and it
+ *     is the only one that works at all for a case whose ceiling is under the
+ *     bar.
+ *
+ * Everything here is a *plan*: it evaluates the mean trajectory (the candidate
+ * scores exactly its expected successes). Realised sample counts vary around
+ * it.
+ */
+import { type BaselineStat, type CaseRef } from "./baseline.js";
+import { type PairedCounts } from "./stats.js";
+export interface PlanConfig {
+    /** the drop we want to be able to detect, in probability units (0.15 = 15 points) */
+    mde: number;
+    /** suite-level false discovery rate */
+    fdr: number;
+    /** USD per candidate run */
+    costPerRunUsd: number;
+    /** milliseconds per run, wall clock, at concurrency 1 */
+    msPerRun: number;
+    /** how many runs the screening pass spends per case */
+    screenRuns: number;
+    /**
+     * Measured coupling between the paired arms, in [0,1]. 0 means the seeds
+     * bought nothing; 1 means perfect common-random-numbers coupling. Measure it
+     * with `pairPhi` on a pilot rather than assuming it, and plan at 0 as well
+     * so the sensitivity is visible.
+     */
+    pairCoupling: number;
+    /** how many runs to fan out in parallel, only affects wall time */
+    concurrency: number;
+    /** beyond this many runs for one case we stop calling it "affordable" */
+    runsPerCaseCeiling: number;
+    /**
+     * Which design the *expected* bill and the screening pass are priced for. A
+     * paired observation is two runs, and a paired case's observation count is
+     * the pairs `shouldStopPaired` takes to settle it at its coupling; an
+     * unpaired case's is the runs `shouldStop` takes to stop it.
+     *
+     * `undefined` prices each case at its own `PlanCase.design`, and a case with
+     * none at the suite's `recommendedDesign`. Setting it prices every case at
+     * that one design.
+     */
+    design?: "unpaired" | "paired";
+    alpha: number;
+    beta: number;
+    minTrials: number;
+    maxTrials: number;
+    batchSize: number;
+    nullQuantile: number;
+    /**
+     * The `futilityConfidence` you run `shouldStop` and `shouldStopPaired` at.
+     * The expected bill counts the runs or pairs a case takes to stop at this
+     * level, so a higher one prices more. `undefined` takes the stopping rules'
+     * own default.
+     */
+    futilityConfidence?: number;
+}
+export declare const DEFAULT_PLAN: PlanConfig;
+export type Detectability = 
+/** the unpaired sequential test can certify this case within the run ceiling */
+"UNPAIRED"
+/** only the paired design can certify it, the unpaired evidence ceiling is under the bar */
+ | "PAIRED_ONLY"
+/** the run count is finite but past the ceiling: technically possible, practically not */
+ | "TOO_EXPENSIVE"
+/** the case cannot lose `mde` points at all, its baseline rate is not that high */
+ | "IMPOSSIBLE";
+export interface PlanCase {
+    caseId: string;
+    suite: string;
+    baseline: {
+        successes: number;
+        trials: number;
+        rate: number;
+    };
+    /** the rate the alternative sits at: baseline − mde, floored at 0 */
+    alternativeRate: number;
+    /** log of the e-value a case must reach to be certified on its own evidence */
+    barLogE: number;
+    /** the limit of the unpaired log e-value as runs → ∞ */
+    ceilingLogE: number;
+    unpaired: {
+        runs: number;
+        costUsd: number;
+        wallMs: number;
+    };
+    paired: {
+        pairs: number;
+        runs: number;
+        costUsd: number;
+        wallMs: number;
+    };
+    /**
+     * The coupling `paired` is priced at: `pairPhi` of this case's pilot when
+     * `makePlan` was given one, floored at 0, otherwise `config.pairCoupling`.
+     */
+    pairCoupling: number;
+    detectability: Detectability;
+    /**
+     * The design to run this case with: of the designs that decide it within
+     * `runsPerCaseCeiling`, the one that needs fewer runs, unpaired on a tie.
+     * Null when neither does.
+     */
+    design: "unpaired" | "paired" | null;
+    /**
+     * How many baseline runs this case would need for the *unpaired* design to
+     * work, or null if it already works. This is the actionable half of an
+     * "undetectable" verdict.
+     */
+    baselineRunsNeeded: number | null;
+    /** what a fixed-N two-proportion design would spend per arm, for comparison */
+    fixedNPerArm: number;
+    reason: string;
+}
+export interface PlanTotals {
+    cases: number;
+    /** how many cases each design can actually certify */
+    decidableUnpaired: number;
+    decidableBest: number;
+    /** every case run to certification, unpaired */
+    unpairedRuns: number;
+    unpairedCostUsd: number;
+    /** every case run to certification, paired */
+    pairedRuns: number;
+    pairedCostUsd: number;
+    /** per case, the cheaper of the two designs that can decide it */
+    bestRuns: number;
+    bestCostUsd: number;
+    /** screen everything cheaply, then pay only for what the screen keeps */
+    screenedRuns: number;
+    screenedCostUsd: number;
+    /** the fixed-N design with the same nominal guarantees */
+    fixedNRuns: number;
+    fixedNCostUsd: number;
+    wallMsUnpaired: number;
+    wallMsPaired: number;
+    wallMsBest: number;
+    wallMsScreened: number;
+    /** fraction of cases the screen is budgeted to carry through to the full test */
+    screenContinueFraction: number;
+    /**
+     * What one observation of the expected bill costs in runs, averaged over its
+     * cases: 1 when every case runs unpaired, 2 when every case is paired, and in
+     * between for a suite that mixes the two.
+     */
+    runsPerObservation: number;
+    /** expected *runs* (not observations) for a pull request in which nothing moved */
+    typicalRuns: number;
+    typicalCostUsd: number;
+    wallMsTypical: number;
+    /** the same, with a screening pass in front */
+    typicalScreenedRuns: number;
+    typicalScreenedCostUsd: number;
+}
+export interface Plan {
+    config: PlanConfig;
+    cases: PlanCase[];
+    totals: PlanTotals;
+    /** cases no budget can decide with the current baseline */
+    undecidable: PlanCase[];
+    /** cases only the paired design can reach */
+    pairedOnly: PlanCase[];
+    /** cases whose baseline rate is too low to lose `mde` points at all */
+    impossible: PlanCase[];
+    /**
+     * The design the recommendation names: paired when more than a quarter of
+     * the suite is `PAIRED_ONLY` or most decidable cases' own `design` is
+     * paired. `expectedCostUsd` prices each case at
+     * its own `design` and falls back to this one only for a case with none,
+     * unless `config.design` fixes one design for the whole suite.
+     */
+    recommendedDesign: "unpaired" | "paired";
+    /** the cheapest design that decides every decidable case */
+    recommendation: string;
+    /** honest one-liner about affordability at the configured cost per run */
+    verdict: string;
+    /**
+     * What a *typical* pull request costs, one in which nothing moved, so most
+     * cases stop at the H₀ wall. This is the headline number.
+     */
+    expectedCostUsd: number;
+    /**
+     * What a pull request in which *every* case regressed would cost.
+     *
+     * Not an upper bound on `expectedCostUsd`, despite running every case to
+     * certification: the sequential test stops a case as soon as it clears the
+     * bar, so an all-regressed suite can finish cheaper than one that did not
+     * move and ran its full schedule. `verdict` says which way round they came
+     * out rather than presenting this as the worst case.
+     */
+    ceilingCostUsd: number;
+}
+export declare function planCase(c: CaseRef, b: BaselineStat | undefined, cfg: PlanConfig, m: number): PlanCase;
+/**
+ * Expected total samples for the sequential design, priced under Wald's
+ * expected sample number at a supplied "true" rate per case, and clamped into
+ * the gate's own [minTrials, maxTrials] batch grid.
+ *
+ * `makePlan` no longer bills from it: an unpaired case there is counted in
+ * the runs `shouldStop` takes, which this figure misses either way (#105).
+ * `samplesForEvidence`, the one behind `PlanCase.unpaired.runs`, answers a
+ * different question: how many runs it takes to *certify a regression*. A case
+ * that did not regress never gets there; it hits the H₀ wall in a few batches
+ * and stops. Pricing every case at the certification cost over-predicts the
+ * bill.
+ *
+ * `trueRate` defaults to the case's own baseline rate, i.e. "nothing moved", * which is what almost every pull request looks like.
+ */
+export declare function expectedSequentialSamples(cases: Array<CaseRef>, baseline: Map<string, BaselineStat>, cfg: PlanConfig, trueRate?: (caseId: string, b: BaselineStat) => number): {
+    samples: number;
+    perCase: Map<string, number>;
+};
+type TrialGrid = Pick<PlanConfig, "minTrials" | "maxTrials" | "batchSize">;
+/** Refuses a set `futilityConfidence` outside (0, 1), where the settle bar is 0 or Infinity. */
+export declare function requireSettleConfidence(cfg: Pick<PlanConfig, "futilityConfidence">, where: string): void;
+/**
+ * Expected pairs for a paired case that did not move: the first pair count at
+ * which `shouldStopPaired`'s settle e-value, on the mean trajectory at coupling
+ * `rho` and `cfg.futilityConfidence` (the stopping rule's default when unset),
+ * rules out a drop of `mde`, on the same batch grid as
+ * `expectedSequentialSamples`, and `maxTrials` when it never does. The
+ * unpaired SPRT figure knows nothing of this rule or of the coupling, which
+ * sets how often an unmoved pair disagrees.
+ */
+export declare function expectedSettlePairs(rate: number, rho: number, mde: number, cfg: TrialGrid & Pick<PlanConfig, "futilityConfidence">): number;
+/**
+ * Plan a suite. `pilots` holds a pair table per case from a pilot run of the
+ * paired design, ideally with both arms on the same revision; a case with one
+ * prices pairing at its measured `pairPhi` instead of `cfg.pairCoupling`, so
+ * its `design` is chosen from a coupling the runner actually delivers.
+ */
+export declare function makePlan(cases: Array<CaseRef>, baseline: Map<string, BaselineStat>, cfg?: PlanConfig, pilots?: Map<string, PairedCounts>): Plan;
+/**
+ * The suite size / MDE combinations that *are* affordable at a given per-run
+ * cost and budget. This is the honest answer to "is this usable for a
+ * coding-agent benchmark at $1 a run?", usually "not at 200 cases and 15
+ * points, but yes at 20 cases and 25 points".
+ */
+export interface AffordabilityCell {
+    cases: number;
+    mde: number;
+    /** runs to certify every case, the ceiling */
+    runs: number;
+    costUsd: number;
+    /** runs a pull request that moved nothing actually spends, the expected bill */
+    typicalRuns: number;
+    typicalCostUsd: number;
+    /** does the *ceiling* fit the budget? */
+    affordable: boolean;
+    /** does the *expected* cost fit the budget? */
+    typicallyAffordable: boolean;
+}
+export declare function affordabilityGrid(baselineRate: number, baselineTrials: number, cfg: PlanConfig, budgetUsd: number, caseCounts?: number[], mdes?: number[]): AffordabilityCell[];
+export {};
+//# sourceMappingURL=plan.d.ts.map

package/dist/plan.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"plan.d.ts","sourceRoot":"","sources":["../src/plan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,EAAE,KAAK,YAAY,EAAoB,KAAK,OAAO,EAAE,MAAM,eAAe,CAAC;AAWlF,OAAO,EAKL,KAAK,YAAY,EAKlB,MAAM,YAAY,CAAC;AAGpB,MAAM,WAAW,UAAU;IACzB,qFAAqF;IACrF,GAAG,EAAE,MAAM,CAAC;IACZ,uCAAuC;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,4BAA4B;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;;;;;;;OASG;IACH,MAAM,CAAC,EAAE,UAAU,GAAG,QAAQ,CAAC;IAO/B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,eAAO,MAAM,YAAY,EAAE,UAiB1B,CAAC;AAEF,MAAM,MAAM,aAAa;AACvB,gFAAgF;AAC9E,UAAU;AACZ,4FAA4F;GAC1F,aAAa;AACf,0FAA0F;GACxF,eAAe;AACjB,mFAAmF;GACjF,YAAY,CAAC;AAEjB,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9D,qEAAqE;IACrE,eAAe,EAAE,MAAM,CAAC;IACxB,+EAA+E;IAC/E,OAAO,EAAE,MAAM,CAAC;IAChB,wDAAwD;IACxD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5D,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACzE;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,aAAa,CAAC;IAC7B;;;;OAIG;IACH,MAAM,EAAE,UAAU,GAAG,QAAQ,GAAG,IAAI,CAAC;IACrC;;;;OAIG;IACH,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,+EAA+E;IAC/E,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,gDAAgD;IAChD,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,0DAA0D;IAC1D,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,iFAAiF;IACjF,sBAAsB,EAAE,MAAM,CAAC;IAU/B;;;;OAIG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mFAAmF;IACnF,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,+CAA+C;IAC/C,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,IAAI;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,MAAM,EAAE,UAAU,CAAC;IACnB,2DAA2D;IAC3D,WAAW,EAAE,QAAQ,EAAE,CAAC;IACxB,6CAA6C;IAC7C,UAAU,EAAE,QAAQ,EAAE,CAAC;IACvB,uEAAuE;IACvE,UAAU,EAAE,QAAQ,EAAE,CAAC;IACvB;;;;;;OAMG;IACH,iBAAiB,EAAE,UAAU,GAAG,QAAQ,CAAC;IACzC,4DAA4D;IAC5D,cAAc,EAAE,MAAM,CAAC;IACvB,0EAA0E;IAC1E,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;;OAQG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AA2DD,wBAAgB,QAAQ,CACtB,CAAC,EAAE,OAAO,EACV,CAAC,EAAE,YAAY,GAAG,SAAS,EAC3B,GAAG,EAAE,UAAU,EACf,CAAC,EAAE,MAAM,GACR,QAAQ,CA0HV;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,EACrB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACnC,GAAG,EAAE,UAAU,EACf,QAAQ,GAAE,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,YAAY,KAAK,MACJ,GAC1C;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAqBnD;AAED,KAAK,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,WAAW,GAAG,WAAW,GAAG,WAAW,CAAC,CAAC;AAE3E,gGAAgG;AAChG,wBAAgB,uBAAuB,CACrC,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,oBAAoB,CAAC,EAC3C,KAAK,EAAE,MAAM,GACZ,IAAI,CAIN;AAUD;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,oBAAoB,CAAC,GACtD,MAAM,CAUR;AAoDD;;;;;GAKG;AACH,wBAAgB,QAAQ,CACtB,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,EACrB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACnC,GAAG,GAAE,UAAyB,EAC9B,MAAM,GAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAa,GAC5C,IAAI,CAqON;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,8CAA8C;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,yCAAyC;IACzC,UAAU,EAAE,OAAO,CAAC;IACpB,+CAA+C;IAC/C,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAED,wBAAgB,iBAAiB,CAC/B,YAAY,EAAE,MAAM,EACpB,cAAc,EAAE,MAAM,EACtB,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,UAAU,GAAE,MAAM,EAAuB,EACzC,IAAI,GAAE,MAAM,EAA4B,GACvC,iBAAiB,EAAE,CA0GrB"}