peeksafe 0.3.0 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "peeksafe",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Statistically valid gates for non-deterministic eval suites. Stop early without losing error control.",
   "keywords": [
     "evals",
@@ -10,7 +10,16 @@
     "false-discovery-rate",
     "llm",
     "statistics",
-    "ci"
+    "ci",
+    "eval",
+    "llm-evals",
+    "regression-testing",
+    "flaky-tests",
+    "ab-testing",
+    "benjamini-hochberg",
+    "e-bh",
+    "early-stopping",
+    "significance"
   ],
   "license": "Apache-2.0",
   "author": "Alexis Rico",

package/src/check.ts

@@ -50,16 +50,20 @@ export interface CertifyProbabilityOptions extends Omit<TypeOneErrorOptions, "ra
  * baseline at a horizon of 600 takes tens of milliseconds.
  */
 export function certifyProbability(options: CertifyProbabilityOptions): number {
+  return certify(options, "certifyProbability");
+}
+
+function certify(options: CertifyProbabilityOptions, where: string): number {
   const { baselineRate, candidateRate: pc, baselineTrials: nb, alpha, horizon } = options;
   const mde = options.mde ?? DEFAULT_GATE_OPTIONS.mde;
   const concentration = options.altConcentration ?? DEFAULT_GATE_OPTIONS.altConcentration;
   const evidence = options.evidence ?? DEFAULT_GATE_OPTIONS.evidence;
-  requireEvidence(evidence, "certifyProbability");
-  requireOpenProbability(baselineRate, "baselineRate", "certifyProbability");
-  requireOpenProbability(pc, "candidateRate", "certifyProbability");
-  requireOpenProbability(alpha, "alpha", "certifyProbability");
-  requireOpenProbability(mde, "mde", "certifyProbability");
-  requirePositiveConfig(concentration, "altConcentration", "certifyProbability");
+  requireEvidence(evidence, where);
+  requireOpenProbability(baselineRate, "baselineRate", where);
+  requireOpenProbability(pc, "candidateRate", where);
+  requireOpenProbability(alpha, "alpha", where);
+  requireOpenProbability(mde, "mde", where);
+  requirePositiveConfig(concentration, "altConcentration", where);
   for (const [name, v] of [
     ["baselineTrials", nb],
     ["horizon", horizon],
@@ -67,7 +71,7 @@ export function certifyProbability(options: CertifyProbabilityOptions): number {
     if (!Number.isInteger(v) || v < 1) {
       throw new PeeksafeError(
         "PEEKSAFE_E_CONFIG",
-        `certifyProbability: ${name} must be a positive integer, got ${v}`,
+        `${where}: ${name} must be a positive integer, got ${v}`,
         {
           detail: { [name]: v },
         },
@@ -164,7 +168,7 @@ function universalFromBayes(rawLog: number, s: number, n: number, sb: number, nb
 export function typeOneError(options: TypeOneErrorOptions): number {
   const { rate, ...rest } = options;
   requireOpenProbability(rate, "rate", "typeOneError");
-  return certifyProbability({ ...rest, baselineRate: rate, candidateRate: rate });
+  return certify({ ...rest, baselineRate: rate, candidateRate: rate }, "typeOneError");
 }
 
 export interface PairedCertifyProbabilityOptions {

package/src/cluster.ts

@@ -135,12 +135,16 @@ function requireLevel(level: number, where: string): void {
 
 /** Group observations by cluster, preserving first-seen order. */
 export function groupByCluster(obs: ClusterObservation[]): Map<string, number[]> {
+  return group(obs, "groupByCluster");
+}
+
+function group(obs: ClusterObservation[], where: string): Map<string, number[]> {
   const m = new Map<string, number[]>();
   for (const o of obs) {
     if (!Number.isFinite(o.value)) {
       throw new PeeksafeError(
         "PEEKSAFE_E_STAT_DOMAIN",
-        `clustered mean: value for cluster "${o.cluster}" is not finite`,
+        `${where}: value for cluster "${o.cluster}" is not finite`,
         {
           detail: { cluster: o.cluster, value: o.value },
         },
@@ -220,7 +224,7 @@ export function iidMean(values: number[], level = 0.95): Interval {
  */
 export function clusterRobustMean(obs: ClusterObservation[], level = 0.95): ClusteredMean {
   requireLevel(level, "clusterRobustMean");
-  const groups = groupByCluster(obs);
+  const groups = group(obs, "clusterRobustMean");
   const n = obs.length;
   const G = groups.size;
   const sizes = [...groups.values()].map((xs) => xs.length);
@@ -355,7 +359,7 @@ export interface RandomEffectsMean extends Interval {
  */
 export function randomEffectsMean(obs: ClusterObservation[], level = 0.95): RandomEffectsMean {
   requireLevel(level, "randomEffectsMean");
-  const groups = groupByCluster(obs);
+  const groups = group(obs, "randomEffectsMean");
   const n = obs.length;
   const G = groups.size;
   const grand = n > 0 ? obs.reduce((a, o) => a + o.value, 0) / n : 0;
@@ -529,6 +533,9 @@ const fmtPts = (x: number) => `${x >= 0 ? "+" : ""}${(x * 100).toFixed(1)}`;
  * a reader would conclude.
  */
 export function clusteredEffect(obs: ClusterObservation[], level = 0.95): ClusteredEffect {
+  // Checked before the estimators below, which would refuse the same input in their own names.
+  requireLevel(level, "clusteredEffect");
+  const groups = group(obs, "clusteredEffect");
   const cr2 = clusterRobustMean(obs, level);
   const re = randomEffectsMean(obs, level);
   const naive = iidMean(
@@ -536,7 +543,6 @@ export function clusteredEffect(obs: ClusterObservation[], level = 0.95): Cluste
     level,
   );
 
-  const groups = groupByCluster(obs);
   const families: FamilyEffect[] = [...groups.entries()]
     .map(([family, xs]) => {
       const m = xs.reduce((a, x) => a + x, 0) / xs.length;
@@ -741,6 +747,8 @@ export function clusterKeyDiagnostic(
       },
     );
   }
+  group(declared, "clusterKeyDiagnostic");
+  group(fallback, "clusterKeyDiagnostic");
   const d = summarise("declared", declared);
   const f = summarise("case-id", fallback);
 

package/src/gate.ts

@@ -189,6 +189,39 @@ export interface GateResult {
   headline: string;
 }
 
+/** A number as JSON can hold it: finite as is, the rest as the string `String` gives, which `Number` reads back. */
+export type JsonNumber = number | "Infinity" | "-Infinity" | "NaN";
+
+/** A `CaseVerdict` as `gateReport` writes it. A paired case's `ceiling` is `"Infinity"`. */
+export type CaseReport = Omit<CaseVerdict, "ceiling" | "logE"> & {
+  ceiling: JsonNumber;
+  logE: JsonNumber;
+};
+
+/** A `GateResult` as `gateReport` writes it, and as `JSON.stringify(result)` does. */
+export type GateReport = Omit<GateResult, "cases" | "regressed"> & {
+  cases: CaseReport[];
+  regressed: CaseReport[];
+};
+
+const jsonSafe = (v: unknown): unknown => {
+  if (typeof v === "number") return Number.isFinite(v) ? v : String(v);
+  if (Array.isArray(v)) return v.map(jsonSafe);
+  if (v !== null && typeof v === "object") {
+    return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, jsonSafe(x)]));
+  }
+  return v;
+};
+
+/**
+ * A copy of `result` that survives JSON: `JSON.stringify` writes `Infinity` as
+ * `null`, which a reader of a CI artifact takes for "no ceiling" where the truth
+ * is "no limit". Every non-finite number becomes a string instead.
+ */
+export function gateReport(result: GateResult): GateReport {
+  return jsonSafe(result) as GateReport;
+}
+
 /**
  * Decide which cases regressed.
  *
@@ -220,8 +253,19 @@ export function gate(
       continue;
     }
     requireCounts(c.successes, c.trials, `gate(${c.id})`);
-    if (c.baseline)
-      requireCounts(c.baseline.successes, c.baseline.trials, `gate(${c.id}).baseline`);
+    if (!c.baseline) continue;
+    // Only the counts are read, so a baseline joined to the wrong case was gated against without a word.
+    if (c.baseline.caseId !== c.id) {
+      throw new PeeksafeError(
+        "PEEKSAFE_E_CONFIG",
+        `gate: case ${c.id} carries the baseline of case ${c.baseline.caseId}`,
+        {
+          detail: { id: c.id, baselineCaseId: c.baseline.caseId },
+          hint: "join baselines to cases by id, e.g. toBaselineMap(baselines).get(c.id)",
+        },
+      );
+    }
+    requireCounts(c.baseline.successes, c.baseline.trials, `gate(${c.id}).baseline`);
   }
 
   const hasBaseline = (c: GateCase | PairedGateCase): boolean =>
@@ -350,7 +394,7 @@ export function gate(
           .join(", ")}${regressed.length > 3 ? ", …" : ""})`) +
     (caveats.length > 0 ? `. Read with care: ${caveats.join("; ")}.` : "");
 
-  return {
+  const result: GateResult = {
     verdict: regressed.length === 0 ? "PASS" : "FAIL",
     cases: verdicts,
     regressed,
@@ -360,4 +404,7 @@ export function gate(
     options: opts,
     headline,
   };
+  // Not enumerable, so equality checks and spreads see only the fields.
+  Object.defineProperty(result, "toJSON", { value: () => gateReport(result) });
+  return result;
 }

package/src/index.ts

@@ -11,6 +11,7 @@
  * ## 1. The decision
  *
  *   gate(cases, options)   which cases regressed, valid at any stopping time
+ *   gateReport(result)     the same verdict with Infinity kept through JSON
  *
  * You bring counts. It brings e-values and e-BH. Nothing here knows or cares how
  * you ran the evals, and nothing here will gate a case that has no baseline.
@@ -109,12 +110,16 @@ export {
 } from "./frontier.js";
 /* ── 1. the decision ─────────────────────────────────────────────────────── */
 export {
+  type CaseReport,
   type CaseVerdict,
   DEFAULT_GATE_OPTIONS,
   type GateCase,
   type GateOptions,
+  type GateReport,
   type GateResult,
   gate,
+  gateReport,
+  type JsonNumber,
   type PairedGateCase,
 } from "./gate.js";
 /* ── 2. the budget ───────────────────────────────────────────────────────── */

package/src/plan.ts

@@ -162,9 +162,10 @@ export interface PlanCase {
    */
   design: "unpaired" | "paired" | null;
   /**
-   * How many baseline runs this case would need for the *unpaired* design to
-   * work, or null if it already works. This is the actionable half of an
-   * "undetectable" verdict.
+   * The smallest total baseline size (not an increment over the current one)
+   * at which the *unpaired* design would work, or null if it already works.
+   * A floor, not a threshold: see `baselineRunsToClear`. This is the
+   * actionable half of an "undetectable" verdict.
    */
   baselineRunsNeeded: number | null;
   /**

package/src/stats.ts

@@ -115,13 +115,30 @@ export function gammaP(a: number, x: number): number {
  * approximation is visible in a p-value and we compare p-values to thresholds.
  */
 export function erf(x: number): number {
+  requireNumber(x, "x", "erf");
   if (x === 0) return 0;
   const p = gammaP(0.5, x * x);
   return x > 0 ? p : -p;
 }
 
 /** Φ(x): standard normal CDF. */
-export const normalCdf = (x: number): number => 0.5 * (1 + erf(x / Math.SQRT2));
+export function normalCdf(x: number): number {
+  requireNumber(x, "x", "normalCdf");
+  return 0.5 * (1 + erf(x / Math.SQRT2));
+}
+
+/** Refuses NaN only: erf and normalCdf answer the infinities as their limits. */
+function requireNumber(x: number, name: string, where: string): void {
+  if (Number.isNaN(x)) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_STAT_DOMAIN",
+      `${where}: ${name} must be a number, got NaN`,
+      {
+        detail: { [name]: x },
+      },
+    );
+  }
+}
 
 /** Φ⁻¹(p): Acklam's inverse normal CDF, refined by one Halley step. */
 export function normalQuantile(p: number): number {
@@ -330,10 +347,16 @@ export function betaPosterior(
   prior: { alpha: number; beta: number } = { alpha: 1, beta: 1 },
 ): BetaPosterior {
   requireCounts(successes, successes + failures, "betaPosterior");
-  if (!(prior.alpha > 0) || !(prior.beta > 0)) {
+  // An infinite shape passed `> 0` and came back as NaN moments.
+  if (
+    !(prior.alpha > 0) ||
+    !(prior.beta > 0) ||
+    !Number.isFinite(prior.alpha) ||
+    !Number.isFinite(prior.beta)
+  ) {
     throw new PeeksafeError(
       "PEEKSAFE_E_STAT_DOMAIN",
-      "betaPosterior: prior parameters must be positive",
+      "betaPosterior: prior parameters must be finite and positive",
       {
         detail: { prior },
       },
@@ -424,8 +447,27 @@ export function tQuantile(p: number, df: number): number {
   return Math.sqrt((df * (1 - x)) / x);
 }
 
+// A posterior built by hand reached betaQuantile or ibeta unchecked and was refused in their name.
+function requirePosteriorShape(post: BetaPosterior, where: string): void {
+  if (
+    !(post.alpha > 0) ||
+    !(post.beta > 0) ||
+    !Number.isFinite(post.alpha) ||
+    !Number.isFinite(post.beta)
+  ) {
+    throw new PeeksafeError(
+      "PEEKSAFE_E_STAT_DOMAIN",
+      `${where}: posterior alpha and beta must be finite and positive`,
+      {
+        detail: { alpha: post.alpha, beta: post.beta },
+      },
+    );
+  }
+}
+
 /** Equal-tailed credible interval of a Beta posterior. */
 export function betaCredibleInterval(post: BetaPosterior, level = 0.95): Interval {
+  requirePosteriorShape(post, "betaCredibleInterval");
   // betaQuantile clamps q to [0, 1], so a level of 95 or -1 came back as [0, 1] or [1, 0].
   requireOpenProbability(level, "level", "betaCredibleInterval");
   const tail = (1 - level) / 2;
@@ -438,6 +480,7 @@ export function betaCredibleInterval(post: BetaPosterior, level = 0.95): Interva
 
 /** P(a < θ < b) under a Beta posterior. */
 export function betaMassBetween(post: BetaPosterior, a: number, b: number): number {
+  requirePosteriorShape(post, "betaMassBetween");
   requireProbability(a, "a", "betaMassBetween");
   requireProbability(b, "b", "betaMassBetween");
   if (a > b) {
@@ -1070,6 +1113,8 @@ export function bernoulliEntropy(p: number): number {
 /** log of the Beta(a,b) density at p. */
 export function logBetaPdf(p: number, a: number, b: number): number {
   requireProbability(p, "p", "logBetaPdf");
+  requireConcentration(a, "a", "logBetaPdf");
+  requireConcentration(b, "b", "logBetaPdf");
   if (p <= 0 || p >= 1) {
     // density is 0 or ∞ at the boundary depending on the shape; the planner
     // only ever evaluates strictly inside, so clamp rather than return ±∞.
@@ -1145,6 +1190,9 @@ export function evidenceCeilingLogE(
   altConcentration = 8,
 ): number {
   requireProbability(pTrue, "pTrue", "evidenceCeilingLogE");
+  requireCounts(baselineSuccesses, baselineTrials, "evidenceCeilingLogE");
+  requireConcentration(altConcentration, "altConcentration", "evidenceCeilingLogE");
+  requireOpenProbability(mde, "mde", "evidenceCeilingLogE");
   const { nullPrior, altPrior } = twoSamplePriors(
     baselineSuccesses,
     baselineTrials,
@@ -1212,6 +1260,8 @@ export function evidenceCeilingAsymptotic(
 ): number {
   requireCounts(baselineSuccesses, baselineTrials, "evidenceCeilingAsymptotic");
   requireProbability(pTrue, "pTrue", "evidenceCeilingAsymptotic");
+  requireConcentration(altConcentration, "altConcentration", "evidenceCeilingAsymptotic");
+  requireOpenProbability(mde, "mde", "evidenceCeilingAsymptotic");
   if (baselineTrials === 0) {
     throw new PeeksafeError(
       "PEEKSAFE_E_STAT_DOMAIN",
@@ -1273,6 +1323,9 @@ export function expectedLogE(
       detail: { n },
     });
   }
+  requireCounts(baselineSuccesses, baselineTrials, "expectedLogE");
+  requireConcentration(altConcentration, "altConcentration", "expectedLogE");
+  requireOpenProbability(mde, "mde", "expectedLogE");
   const { nullPrior: n0, altPrior: a1 } = twoSamplePriors(
     baselineSuccesses,
     baselineTrials,
@@ -1306,6 +1359,9 @@ export function expectedLogEExact(
       { detail: { n } },
     );
   }
+  requireCounts(baselineSuccesses, baselineTrials, "expectedLogEExact");
+  requireConcentration(altConcentration, "altConcentration", "expectedLogEExact");
+  requireOpenProbability(mde, "mde", "expectedLogEExact");
   let acc = 0;
   for (let s = 0; s <= n; s++) {
     const logW =
@@ -1338,6 +1394,10 @@ export function samplesForEvidence(
 ): number {
   requireSearchCap(maxN, "maxN", "samplesForEvidence");
   requireLogThreshold(logThreshold, "samplesForEvidence");
+  requireProbability(pTrue, "pTrue", "samplesForEvidence");
+  requireCounts(baselineSuccesses, baselineTrials, "samplesForEvidence");
+  requireConcentration(altConcentration, "altConcentration", "samplesForEvidence");
+  requireOpenProbability(mde, "mde", "samplesForEvidence");
   const ceiling = evidenceCeilingLogE(
     pTrue,
     baselineSuccesses,
@@ -1406,6 +1466,8 @@ export function universalTwoSampleLogE(
 ): number {
   requireCounts(candidateSuccesses, candidateTrials, "universalTwoSampleLogE(candidate)");
   requireCounts(baselineSuccesses, baselineTrials, "universalTwoSampleLogE(baseline)");
+  requireConcentration(altConcentration, "altConcentration", "universalTwoSampleLogE");
+  requireOpenProbability(mde, "mde", "universalTwoSampleLogE");
   if (baselineTrials === 0) {
     throw new PeeksafeError(
       "PEEKSAFE_E_STAT_DOMAIN",
@@ -1586,6 +1648,9 @@ export function mcnemarSamplesForEvidence(
   requireConcentration(concentration, "concentration", "mcnemarSamplesForEvidence");
   requireSearchCap(maxPairs, "maxPairs", "mcnemarSamplesForEvidence");
   requireLogThreshold(logThreshold, "mcnemarSamplesForEvidence");
+  requireProbability(pBaseline, "pBaseline", "mcnemarSamplesForEvidence");
+  requireProbability(rho, "rho", "mcnemarSamplesForEvidence");
+  requireOpenProbability(mde, "mde", "mcnemarSamplesForEvidence");
   const { rate, theta } = pairedDiscordance(pBaseline, mde, rho);
   if (rate <= 0 || theta <= 0.5) return Infinity;
   const a1 = Math.max(0.35, concentration * theta);