peeksafe 0.1.0 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "peeksafe",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Statistically valid gates for non-deterministic eval suites. Stop early without losing error control.",
   "keywords": [
     "evals",
@@ -43,6 +43,14 @@
   "engines": {
     "node": ">=22"
   },
+  "devDependencies": {
+    "@biomejs/biome": "2.5.15",
+    "@types/node": "22.20.5",
+    "knip": "6.39.0",
+    "lefthook": "2.1.16",
+    "typescript": "7.0.2",
+    "vitest": "5.0.3"
+  },
   "scripts": {
     "build": "tsc -p tsconfig.build.json",
     "clean": "rm -rf dist",
@@ -52,15 +60,6 @@
     "knip": "knip --treat-config-hints-as-errors",
     "test": "vitest run",
     "test:watch": "vitest",
-    "prepack": "pnpm run clean && pnpm run build",
     "version": "node scripts/release-changelog.mjs && git add CHANGELOG.md"
-  },
-  "devDependencies": {
-    "@biomejs/biome": "catalog:",
-    "@types/node": "catalog:",
-    "knip": "catalog:",
-    "lefthook": "catalog:",
-    "typescript": "catalog:",
-    "vitest": "catalog:"
   }
-}
+}

package/src/cluster.ts

@@ -188,6 +188,18 @@ export interface ClusteredMean extends Interval {
 /** The naive iid interval, kept so the two can be printed side by side. */
 export function iidMean(values: number[], level = 0.95): Interval {
   requireLevel(level, "iidMean");
+  // groupByCluster refused these for the other estimators, so the same data was an error there and a NaN interval here.
+  for (let i = 0; i < values.length; i++) {
+    if (!Number.isFinite(values[i])) {
+      throw new PeeksafeError(
+        "PEEKSAFE_E_STAT_DOMAIN",
+        `iidMean: value at index ${i} is not finite`,
+        {
+          detail: { index: i, value: values[i] },
+        },
+      );
+    }
+  }
   const n = values.length;
   if (n === 0) return { low: -1, high: 1, point: 0, observations: 0 };
   const mean = values.reduce((a, b) => a + b, 0) / n;

package/src/errors.ts

@@ -153,11 +153,11 @@ export function requireAltConcentration(v: number, evidence: string, where: stri
 }
 
 /**
- * The error rates of the sequential test a planner prices: each strictly
- * between 0 and 1, and summing to less than 1.
+ * The error rates of a Wald SPRT, run by `sprtDecision` or priced by a
+ * planner: each strictly between 0 and 1, and summing to less than 1.
  *
  * At `alpha + beta >= 1` Wald's walls meet or cross, so the test decides
- * before it has seen a run. `sprtExpectedN` answers a small positive number
+ * before it has seen a run. `sprtExpectedN` answered a small positive number
  * there, and a plan at `alpha: 0.6, beta: 0.5` priced every case at
  * `minTrials` with nothing to say the design had no error control left.
  */
@@ -176,6 +176,59 @@ export function requireErrorRates(alpha: number, beta: number, where: string): v
   }
 }
 
+/**
+ * `runsPerCaseCeiling`: finite and at least one run. The planners search up to
+ * eight times it, and at NaN or 0 that search never ran, so every case came
+ * back undecidable with a reason that blamed the evidence bar.
+ */
+export function requireRunCeiling(v: number, where: string): void {
+  if (!(v >= 1) || !Number.isFinite(v)) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_CONFIG",
+      `${where}: runsPerCaseCeiling must be finite and at least 1, got ${v}`,
+      {
+        detail: { runsPerCaseCeiling: v, where },
+      },
+    );
+  }
+}
+
+/**
+ * The batch grid a plan prices the gate on: `minTrials` and `batchSize`
+ * positive integers, `maxTrials` an integer at least `minTrials`. A NaN bound
+ * or a zero batch made the typical bill NaN, and a cap under the minimum
+ * priced every case at the cap.
+ */
+export function requireTrialGrid(
+  cfg: { minTrials: number; maxTrials: number; batchSize: number },
+  where: string,
+): void {
+  const { minTrials, maxTrials, batchSize } = cfg;
+  const detail = { minTrials, maxTrials, batchSize, where };
+  for (const [name, v] of [
+    ["minTrials", minTrials],
+    ["batchSize", batchSize],
+  ] as const) {
+    if (!Number.isInteger(v) || v < 1) {
+      throw new PeeksafeError(
+        "PEEKSAFE_E_CONFIG",
+        `${where}: ${name} must be a positive integer, got ${v}`,
+        { detail },
+      );
+    }
+  }
+  if (!Number.isInteger(maxTrials) || maxTrials < minTrials) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_CONFIG",
+      `${where}: maxTrials must be an integer ≥ minTrials, got ${maxTrials} vs ${minTrials}`,
+      {
+        detail,
+        hint: "the typical bill is clamped to [minTrials, maxTrials]; a cap under the minimum prices a gate that stops before it starts",
+      },
+    );
+  }
+}
+
 /** The four cells of a pair table, each a non-negative integer. */
 export function requirePairedCounts(
   p: { bothPass: number; worse: number; better: number; bothFail: number },
@@ -206,3 +259,17 @@ export function requireProbability(p: number, name: string, where: string): void
     );
   }
 }
+
+/** Case ids, each at most once: a repeated id is planned twice and inflates the suite size the bar is set from. */
+export function requireUniqueCaseIds(cases: readonly { id: string }[], where: string): void {
+  const seen = new Set<string>();
+  for (const c of cases) {
+    if (seen.has(c.id)) {
+      throw new PeeksafeError("PEEKSAFE_E_CASE_DUPLICATE", `${where}: duplicate case id ${c.id}`, {
+        detail: { id: c.id, where },
+        hint: "a case listed twice is priced twice and counted twice in the suite size; dedupe the case list before calling",
+      });
+    }
+    seen.add(c.id);
+  }
+}

package/src/frontier.ts

@@ -62,8 +62,10 @@ import {
   requireErrorRates,
   requireOpenProbability,
   requireProbability,
+  requireRunCeiling,
+  requireTrialGrid,
 } from "./errors.js";
-import { DEFAULT_PLAN } from "./plan.js";
+import { DEFAULT_PLAN, expectedSettlePairs, requireSettleConfidence } from "./plan.js";
 import {
   betaQuantile,
   ebhSoloThreshold,
@@ -161,9 +163,9 @@ export interface FrontierConfig {
   /**
    * Baseline runs per case the *paired* design needs. The paired e-value's null
    * is a point mass, so the stored baseline is not part of its validity and
-   * does not set a ceiling, it only feeds the SPRT stopping rule and the
-   * screen. A short one is enough, and that asymmetry is most of why the paired
-   * design wins at these prices.
+   * does not set a ceiling; here it is only the baseline bill, since a paired
+   * typical bill is priced in pairs to settle. A short one is enough, and that
+   * asymmetry is most of why the paired design wins at these prices.
    */
   pairedBaselineRuns: number;
   /** fraction of cases a screen carries through to the expensive test */
@@ -185,6 +187,8 @@ export interface FrontierConfig {
   maxTrials: number;
   batchSize: number;
   nullQuantile: number;
+  /** the `futilityConfidence` `shouldStopPaired` runs at; unset takes its default */
+  futilityConfidence?: number;
 }
 
 export const DEFAULT_FRONTIER: FrontierConfig = {
@@ -268,6 +272,7 @@ export interface FrontierPoint {
 }
 
 const RUNS_PER_OBS: Record<FrontierDesign, number> = { unpaired: 1, paired: 2 };
+const SCREENS: readonly FrontierScreen[] = ["none", "expensive", "proxy"];
 
 /** $1.00, but $0.0021 rather than $0.00, a per-run price rounds to nothing. */
 export const fmtRate = (x: number): string =>
@@ -283,6 +288,62 @@ export const fmtRate = (x: number): string =>
 const graded = (runs: number, cfg: FrontierConfig): number =>
   runs * (cfg.costPerRunUsd + cfg.costPerGradeUsd);
 
+function requireBaselineRuns(v: number, name: string, where: string): void {
+  if (!Number.isInteger(v) || v < 1) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_CONFIG",
+      `${where}: ${name} must be a positive integer, got ${v}`,
+      {
+        detail: { [name]: v, where },
+      },
+    );
+  }
+}
+
+/**
+ * The budget, the prices, the screen's keep fraction and `amortisePrs`.
+ * `enumerateFrontier` checked these and `evaluatePoint` did not, so a direct
+ * call recommended a point at a negative bill and priced a screen that kept
+ * twice the suite.
+ */
+function requirePricing(cfg: FrontierConfig, where: string): void {
+  const finiteCost = (v: number) => Number.isFinite(v) && v >= 0;
+  if (!(cfg.budgetUsd >= 0) || !finiteCost(cfg.costPerRunUsd) || !finiteCost(cfg.costPerGradeUsd)) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_CONFIG",
+      `${where}: budget must be ≥ 0 and costs finite and ≥ 0`,
+      {
+        detail: {
+          budgetUsd: cfg.budgetUsd,
+          costPerRunUsd: cfg.costPerRunUsd,
+          costPerGradeUsd: cfg.costPerGradeUsd,
+          where,
+        },
+        hint: "an infinite cost fits an infinite budget, so the frontier recommended a point on an infinite bill",
+      },
+    );
+  }
+  if (!(cfg.screenKeepFraction > 0) || !(cfg.screenKeepFraction <= 1)) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_CONFIG",
+      `${where}: screenKeepFraction must be in (0,1], got ${cfg.screenKeepFraction}`,
+      {
+        detail: { screenKeepFraction: cfg.screenKeepFraction, where },
+      },
+    );
+  }
+  if (!Number.isInteger(cfg.amortisePrs) || cfg.amortisePrs < 1) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_CONFIG",
+      `${where}: amortisePrs must be a positive integer, got ${cfg.amortisePrs}`,
+      {
+        detail: { amortisePrs: cfg.amortisePrs, where },
+        hint: 'a baseline spread over "infinity" pull requests is how the baseline cost disappears from a plan',
+      },
+    );
+  }
+}
+
 /**
  * Observations per case on a pull request where nothing moved, clamped onto the
  * gate's own batch grid.
@@ -298,6 +359,11 @@ export function typicalObservationsPerCase(
   // 0 for a quantile at or below 0, 1 at or above 1, and about 0 for NaN, so the
   // frontier priced the typical bill against a null rate nobody chose.
   requireOpenProbability(cfg.nullQuantile, "nullQuantile", "typicalObservationsPerCase");
+  requireTrialGrid(cfg, "typicalObservationsPerCase");
+  // Unchecked, an mde of 0 or below priced a bill for an effect nobody can detect, and a bad rate or run count was refused under betaQuantile's name.
+  requireOpenProbability(rate, "rate", "typicalObservationsPerCase");
+  requireOpenProbability(mde, "mde", "typicalObservationsPerCase");
+  requireBaselineRuns(baselineRuns, "baselineRuns", "typicalObservationsPerCase");
   const s = Math.round(rate * baselineRuns);
   const p0 = betaQuantile(1 + s, 1 + baselineRuns - s, cfg.nullQuantile);
   const p1 = Math.max(0.005, p0 - mde);
@@ -334,6 +400,11 @@ export function evaluatePoint(
   requireErrorRates(cfg.alpha, cfg.beta, "evaluatePoint");
   requireProbability(cfg.pairCoupling, "pairCoupling", "evaluatePoint");
   requireOpenProbability(cfg.nullQuantile, "nullQuantile", "evaluatePoint");
+  requireSettleConfidence(cfg, "evaluatePoint");
+  // makePlan refuses it; unchecked, an fdr of 2 cut the bar and 0 was blamed on the ceiling.
+  requireOpenProbability(cfg.fdr, "fdr", "evaluatePoint");
+  requireRunCeiling(cfg.runsPerCaseCeiling, "evaluatePoint");
+  requirePricing(cfg, "evaluatePoint");
   if (!Number.isInteger(cases) || cases < 1) {
     throw new PeeksafeError(
       "PEEKSAFE_E_CONFIG",
@@ -343,25 +414,35 @@ export function evaluatePoint(
       },
     );
   }
-  if (!Number.isInteger(baselineRuns) || baselineRuns < 1) {
+  // The paired design prices its baseline from this one, and 0 or 2.5 runs priced a cheaper frontier than any real baseline.
+  requireBaselineRuns(cfg.pairedBaselineRuns, "pairedBaselineRuns", "evaluatePoint");
+  requireBaselineRuns(baselineRuns, "baselineRuns", "evaluatePoint");
+  requireTrialGrid(cfg, "evaluatePoint");
+  // An unknown design fell through to the paired branch with no runs per observation and came back feasible on an infinite bill.
+  if (!Object.hasOwn(RUNS_PER_OBS, design)) {
     throw new PeeksafeError(
       "PEEKSAFE_E_CONFIG",
-      `frontier: baselineRuns must be a positive integer, got ${baselineRuns}`,
+      `frontier: design must be one of ${Object.keys(RUNS_PER_OBS).join(", ")}, got "${String(design)}"`,
       {
-        detail: { baselineRuns },
+        detail: { design },
       },
     );
   }
-  if (!Number.isInteger(cfg.maxTrials) || cfg.maxTrials < 1 || cfg.maxTrials < cfg.minTrials) {
+  // An unknown screen was priced and recommended as the expensive one.
+  if (!SCREENS.includes(screen)) {
     throw new PeeksafeError(
       "PEEKSAFE_E_CONFIG",
-      `frontier: maxTrials must be a positive integer ≥ minTrials, got ${cfg.maxTrials} vs ${cfg.minTrials}`,
+      `frontier: screen must be one of ${SCREENS.join(", ")}, got "${String(screen)}"`,
       {
-        detail: { maxTrials: cfg.maxTrials, minTrials: cfg.minTrials },
-        hint: "the ceiling bill is priced at this cap; a cap of zero prices a gate that never runs",
+        detail: { screen },
       },
     );
   }
+  // Only null withholds a recommendation, so a NaN recall (0 of 0 regressions caught) recommended the screen.
+  for (const kind of ["expensive", "proxy"] as const) {
+    const recall = cfg.screenRecall[kind];
+    if (recall !== null) requireProbability(recall, `screenRecall.${kind}`, "evaluatePoint");
+  }
   if (screen !== "none" && (!Number.isInteger(screenRunsPerCase) || screenRunsPerCase < 1)) {
     throw new PeeksafeError(
       "PEEKSAFE_E_CONFIG",
@@ -428,7 +509,11 @@ export function evaluatePoint(
   }
   const feasible = infeasibleBecause === null;
 
-  const typicalObservations = typicalObservationsPerCase(rate, mde, cfg, effectiveBaselineRuns);
+  // A paired case counts the pairs `shouldStopPaired` takes to settle, as `makePlan` does; the SPRT figure ignores the coupling.
+  const typicalObservations =
+    design === "paired"
+      ? expectedSettlePairs(rate, cfg.pairCoupling, mde, cfg)
+      : typicalObservationsPerCase(rate, mde, cfg, effectiveBaselineRuns);
   const keep = screen === "none" ? cases : Math.max(1, Math.ceil(cases * cfg.screenKeepFraction));
 
   // Screening runs are graded by whichever grader the screen uses. A proxy
@@ -449,7 +534,7 @@ export function evaluatePoint(
 
   // The baseline. Unpaired buys its ceiling here; paired does not.
   const baselineRunsTotal = cases * effectiveBaselineRuns;
-  const baselineRunsAmortised = baselineRunsTotal / Math.max(1, cfg.amortisePrs);
+  const baselineRunsAmortised = baselineRunsTotal / cfg.amortisePrs;
   const baselineUsd = graded(baselineRunsAmortised, cfg);
 
   const costFor = (obsPerCase: number): FrontierCost => {
@@ -495,8 +580,8 @@ export function evaluatePoint(
     "certify-all": certifyAll,
     ceiling,
   };
-  const basisCost = BILLS[cfg.basis];
-  if (basisCost === undefined) {
+  // `hasOwn`, as in `costShares`: `BILLS.toString` is inherited, so "toString" passed an `undefined` check.
+  if (!Object.hasOwn(BILLS, cfg.basis)) {
     throw new PeeksafeError(
       "PEEKSAFE_E_CONFIG",
       `frontier: basis must be one of ${Object.keys(BILLS).join(", ")}, got "${String(cfg.basis)}"`,
@@ -506,6 +591,7 @@ export function evaluatePoint(
       },
     );
   }
+  const basisCost = BILLS[cfg.basis];
   const withinBudget = feasible && basisCost.totalUsd <= cfg.budgetUsd;
 
   // Reportable and recommendable are different. A screen whose recall nobody
@@ -551,38 +637,7 @@ export function enumerateFrontier(
   cfg: FrontierConfig,
   axes: FrontierAxes = DEFAULT_AXES,
 ): FrontierPoint[] {
-  if (!(cfg.budgetUsd >= 0) || !(cfg.costPerRunUsd >= 0) || !(cfg.costPerGradeUsd >= 0)) {
-    throw new PeeksafeError(
-      "PEEKSAFE_E_CONFIG",
-      "frontier: budget and costs must be ≥ 0 and not NaN",
-      {
-        detail: {
-          budgetUsd: cfg.budgetUsd,
-          costPerRunUsd: cfg.costPerRunUsd,
-          costPerGradeUsd: cfg.costPerGradeUsd,
-        },
-      },
-    );
-  }
-  if (!(cfg.screenKeepFraction > 0) || !(cfg.screenKeepFraction <= 1)) {
-    throw new PeeksafeError(
-      "PEEKSAFE_E_CONFIG",
-      `frontier: screenKeepFraction must be in (0,1], got ${cfg.screenKeepFraction}`,
-      {
-        detail: { screenKeepFraction: cfg.screenKeepFraction },
-      },
-    );
-  }
-  if (!Number.isInteger(cfg.amortisePrs) || cfg.amortisePrs < 1) {
-    throw new PeeksafeError(
-      "PEEKSAFE_E_CONFIG",
-      `frontier: amortisePrs must be a positive integer, got ${cfg.amortisePrs}`,
-      {
-        detail: { amortisePrs: cfg.amortisePrs },
-        hint: 'a baseline spread over "infinity" pull requests is how the baseline cost disappears from a plan',
-      },
-    );
-  }
+  requirePricing(cfg, "frontier");
   for (const [name, xs] of Object.entries(axes) as Array<[string, unknown[]]>) {
     if (!Array.isArray(xs) || xs.length === 0) {
       throw new PeeksafeError(

package/src/gate.ts

@@ -80,6 +80,11 @@ export const cannotDropBy = (
   mde: number,
 ): boolean => baseline.successes / baseline.trials <= mde;
 
+/** `options` without its `undefined` entries, so a spread over the defaults leaves those defaults in place. */
+export function definedOptions<T extends object>(options: T): T {
+  return Object.fromEntries(Object.entries(options ?? {}).filter(([, v]) => v !== undefined)) as T;
+}
+
 /** One case's candidate result, and the baseline it is measured against. */
 export interface GateCase {
   id: string;
@@ -195,7 +200,7 @@ export function gate(
   cases: readonly (GateCase | PairedGateCase)[],
   options: GateOptions = {},
 ): GateResult {
-  const opts: Required<GateOptions> = { ...DEFAULT_GATE_OPTIONS, ...options };
+  const opts: Required<GateOptions> = { ...DEFAULT_GATE_OPTIONS, ...definedOptions(options) };
   requireOpenProbability(opts.mde, "mde", "gate");
   requireOpenProbability(opts.fdr, "fdr", "gate");
   requireEvidence(opts.evidence, "gate");