textopt 0.0.0 → 0.1.0

package/dist/index.d.mts

@@ -1,8 +1,8 @@
-import { a as EvaluationContext, c as RolloutUsage, d as UsageTotals, f as componentNames, i as EvaluationBatch, l as ScoreResult, n as Candidate, o as EvaluationPhase, r as EvaluateArgs, s as EvaluationSplit, t as Adapter, u as TextModel } from "./types-CWv4IQFF.mjs";
+import { _ as TextModel, a as RunFinished, c as Adapter, d as EvaluationBatch, f as EvaluationContext, g as ScoreResult, h as RolloutUsage, i as Reporter, l as Candidate, m as EvaluationSplit, n as OptimizerEvent, o as isCandidateAccepted, p as EvaluationPhase, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, u as EvaluateArgs, v as UsageTotals, y as componentNames } from "./reporting-DQbAohc9.mjs";
 import { n as EvaluationCache, r as createMemoryCache, t as CachedScore } from "./cache-CuSo0NJ8.mjs";
 import { t as Rng } from "./rng-BR5MOedA.mjs";
-import { a as formatDemos, i as bootstrapDemos, n as Demo, o as parseDemos, r as DemoRenderer, t as BootstrapResult } from "./demos-B0pVQjYC.mjs";
-import { a as EvaluateBatchArgs, c as Evaluator, d as costExhausted, f as createEvaluator, g as createBudget, h as Budget, i as BudgetExhausted, l as RetryPolicy, m as requireMeasuredMean, n as OptimizerResult, o as EvaluateTracedArgs, p as measuredMean, r as OptimizerTask, s as EvaluationEvent, t as Optimizer, u as ScoredBatch } from "./optimizer-DqCoth_w.mjs";
+import { a as harvestFewShotExamples, i as formatDemos, n as Demo, o as parseDemos, r as DemoRenderer, t as BootstrapResult } from "./demos-1b7JiUue.mjs";
+import { a as EvaluateBatchArgs, c as Evaluator, d as costExhausted, f as createEvaluator, g as createBudget, h as Budget, i as BudgetExhausted, l as RetryPolicy, m as requireMeasuredMean, n as OptimizerResult, o as EvaluateTracedArgs, p as measuredMean, r as OptimizerTask, s as EvaluationEvent, t as Optimizer, u as ScoredBatch } from "./optimizer-Ck6-e_8o.mjs";
 import { t as BatchSampler } from "./sampling-DFo_7RNJ.mjs";
 //#region src/checkpoint.d.ts
 /**
@@ -46,6 +46,10 @@ interface ComparisonRun {
   bestScore: number;
   testScore?: number;
   metricCalls: number;
+  /** Rollouts this run got from the cache instead of paying for. */
+  cacheHits: number;
+  /** Calls to a proposal or reflection model, outside the metric budget. */
+  reflectionCalls: number;
   costUsd: number;
   stopReason: string;
 }
@@ -58,12 +62,33 @@ interface ComparisonSummary {
   maxScore: number;
   meanMetricCalls: number;
   meanCostUsd: number;
+  meanCacheHits: number;
+  meanReflectionCalls: number;
+  /**
+   * How many distinct values this entrant's score took across its seeds.
+   * 1 means every seed landed on the same number — the seed changed nothing
+   * about the outcome, whatever the search did internally with it.
+   */
+  distinctScores: number;
   /**
    * How often the winner's margin over this entrant would arise if the two
-   * were equally good and each seed's outcome were a coin flip. Absent for the
-   * winner itself.
+   * were equally good and each seed's outcome were a coin flip. Absent for
+   * the winner itself, and also absent when every seed produced the exact
+   * same margin: a sign-flip test over n seeds is answering a question about
+   * n independent trials, and identical margins mean the seed never actually
+   * put that to the test — there was one realization, repeated. Reporting a
+   * p-value there would state a precision (as fine as 2^-n) that the run
+   * never earned, so it is withheld rather than printed misleadingly small.
    */
   pValueVsWinner?: number;
+  /**
+   * `pValueVsWinner` after Holm-Bonferroni step-down across the other
+   * entrants in this same `compare()` call — the family the raw p-value
+   * would otherwise be read against in isolation. Absent wherever the raw
+   * p-value is: a withheld comparison has nothing to adjust, but it still
+   * occupies a slot in the family the other comparisons are corrected for.
+   */
+  pValueVsWinnerHolm?: number;
 }
 interface Comparison {
   /** Highest mean score. Read `pValueVsWinner` before believing it. */
@@ -85,7 +110,7 @@ interface Comparison {
  * seed is constructor config and every optimizer here is deterministic given
  * one — comparing two entrants at a single seed compares two anecdotes. It also
  * leaves the task where the caller builds it, which is the only place the
- * optimizer-specific parts of it (`reflect`, `cache`, `onEvent`) are known.
+ * optimizer-specific parts of it (`reflect`, `cache`, `reporters`) are known.
  */
 declare function compare<K extends string, Output = unknown>(args: {
   entrants: Record<string, (args: {
@@ -136,6 +161,119 @@ declare function createDeadline(args: {
   now?: () => number;
 }): Deadline;
 //#endregion
+//#region src/harvest.d.ts
+/**
+ * One rollout worth keeping: what went in, what the system produced, and how
+ * that output scored. Harvested, never written — the value of the pair is that
+ * the system actually produced the output and the metric actually rewarded it.
+ */
+interface Rollout<Datum = unknown, Output = unknown> {
+  input: Datum;
+  output: Output;
+  score: number;
+}
+interface HarvestResult<Datum, Output> {
+  rollouts: Rollout<Datum, Output>[];
+  /** Rollouts this cost. Harvesting is cheap, not free. */
+  metricCalls: number;
+  /**
+   * Tokens and dollars this cost. Harvesting runs on its own evaluator, so a
+   * caller that bounds spend has to fold these into its own totals.
+   */
+  usage: UsageTotals;
+  /** Instances run, including the ones the metric did not reward. */
+  attempted: number;
+}
+/**
+ * Run a candidate over data and keep the rollouts the metric rewarded.
+ *
+ * The library's one paid collection primitive, with two consumers: a few-shot
+ * block wants four of these, and a distillation set wants thousands. Both are
+ * the same pass — run the candidate, score it, keep what cleared the bar — so
+ * both share the budget, retry and transient-failure handling that pass needs.
+ *
+ * Which data to sweep is the caller's decision and the consequential one. A
+ * validation set is the wrong choice: it is the set that selected the candidate,
+ * so the rollouts it yields are enriched for the candidate's fit to those
+ * instances rather than to the task. Prefer the training set, or a pool held
+ * out of the run entirely.
+ */
+declare function harvestRollouts<Datum, Trajectory, Output, K extends string = string>(args: {
+  adapter: Adapter<Datum, Trajectory, Output, K>;
+  /** The candidate to run. Usually a run's winner, sometimes the seed. */
+  candidate: Candidate<K>;
+  data: readonly Datum[];
+  /**
+   * Score a rollout must reach to be kept. Unset keeps every rollout the
+   * metric rewarded at all, which is what MIPROv2's bootstrapper does without
+   * a `metric_threshold`: it keeps a trace on any truthy score and only
+   * compares against a number once one is configured.
+   *
+   * Demanding a perfect score instead is the right call for a boolean metric
+   * and the wrong one for a graded metric, where it throws away every rollout
+   * that was most of the way there — which on a hard task is all of them.
+   */
+  minScore?: number;
+  /** Rollouts to collect before stopping. Unset sweeps the whole pool. */
+  maxRollouts?: number;
+  /**
+   * Instances per rollout batch. Smaller batches stop closer to the moment
+   * enough rollouts exist, at the cost of less concurrency inside the adapter.
+   */
+  batchSize?: number;
+  /** Ceiling on rollouts run. Defaults to one pass over `data`. */
+  maxMetricCalls?: number;
+  /**
+   * Ceiling on dollars this pass may spend, checked between batches. Rollout
+   * counts are a poor proxy for spend, and a caller bounding dollars cannot
+   * bound this pass from outside: it runs on its own evaluator.
+   */
+  maxCostUsd?: number;
+  /** Shuffles `data` first, so rollouts are not all drawn from its head. */
+  rng?: Rng;
+  signal?: AbortSignal;
+}): Promise<HarvestResult<Datum, Output>>;
+//#endregion
+//#region src/distillation.d.ts
+/** A turn in a training example, in the shape trainers read. */
+interface ChatMessage {
+  role: "system" | "user" | "assistant";
+  content: string;
+}
+interface TrainingExample {
+  messages: ChatMessage[];
+}
+/**
+ * Turns one harvested rollout into the example to train on, or `null` to skip
+ * it. The callback exists because only the caller knows how to render a
+ * `Datum` — the library never sees inside one.
+ *
+ * It is also where the consequential decision lives: how much of the optimized
+ * candidate to leave in the input. Dropping it entirely moves the whole prompt
+ * into weights and gives up the ability to steer the model with text
+ * afterwards; keeping a short task statement distills away only the tokens the
+ * search accreted. See `docs/distillation.md`.
+ */
+type TrainingExampleRenderer<Datum, Output> = (args: {
+  rollout: Rollout<Datum, Output>;
+  index: number;
+}) => TrainingExample | null;
+/**
+ * Serialize harvested rollouts as JSONL, one training example per line.
+ *
+ * The chat-messages shape rather than any one vendor's: it is what Axolotl,
+ * Together, Fireworks and the Hugging Face trainers all ingest, and the
+ * providers that read it outlive the ones that do not.
+ *
+ * Returns the text rather than writing it. Only `file-cache` touches the
+ * filesystem, and a caller uploading this straight to a provider should not
+ * have to round-trip it through a file to do so.
+ */
+declare function toTrainingJsonl<Datum, Output>(args: {
+  rollouts: readonly Rollout<Datum, Output>[];
+  render: TrainingExampleRenderer<Datum, Output>;
+}): string;
+//#endregion
 //#region src/judge.d.ts
 /** One thing the judge grades, and what a perfect answer looks like for it. */
 interface JudgeCriterion {
@@ -218,4 +356,4 @@ declare function priceUsage(args: {
   pricing?: TokenPricing;
 }): RolloutUsage;
 //#endregion
-export { type Adapter, type BatchSampler, type BootstrapResult, type Budget, BudgetExhausted, type CachedScore, type Candidate, type Comparison, type ComparisonRun, type ComparisonSummary, type Deadline, type Demo, type DemoRenderer, type EvaluateArgs, type EvaluateBatchArgs, type EvaluateTracedArgs, type EvaluationBatch, type EvaluationCache, type EvaluationContext, type EvaluationEvent, type EvaluationPhase, type EvaluationSplit, type Evaluator, type Judge, type JudgeCriterion, type JudgePromptBuilder, type Optimizer, type OptimizerResult, type OptimizerTask, type RetryPolicy, type Rng, type RolloutUsage, type ScoreResult, type ScoredBatch, type TextModel, type TokenPricing, type UsageTotals, assertResumable, bootstrapDemos, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, runFingerprint };
+export { type Adapter, type BatchSampler, type BootstrapResult, type Budget, BudgetExhausted, type CachedScore, type Candidate, type CandidateAccepted, type ChatMessage, type Comparison, type ComparisonRun, type ComparisonSummary, type Deadline, type Demo, type DemoRenderer, type EvaluateArgs, type EvaluateBatchArgs, type EvaluateTracedArgs, type EvaluationBatch, type EvaluationCache, type EvaluationContext, type EvaluationEvent, type EvaluationPhase, type EvaluationSplit, type Evaluator, type HarvestResult, type Judge, type JudgeCriterion, type JudgePromptBuilder, type Optimizer, type OptimizerEvent, type OptimizerResult, type OptimizerTask, type ReportableEvent, type Reporter, type RetryPolicy, type Rng, type Rollout, type RolloutUsage, type RunFinished, type ScoreResult, type ScoredBatch, type TextModel, type TokenPricing, type TrainingExample, type TrainingExampleRenderer, type UsageTotals, assertResumable, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, harvestFewShotExamples, harvestRollouts, isCandidateAccepted, isRunFinished, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, runFingerprint, toTrainingJsonl };

package/dist/index.mjs

@@ -1,10 +1,12 @@
-import { a as requireMeasuredMean, c as candidateFingerprint, d as createMemoryCache, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "./evaluation-BV0nSZVx.mjs";
+import { _ as createMemoryCache, a as isRunFinished, b as createBudget, c as createEvaluator, d as createDeadline, f as assertResumable, h as componentNames, i as isCandidateAccepted, l as measuredMean, m as runFingerprint, o as BudgetExhausted, p as candidateFingerprint, s as costExhausted, u as requireMeasuredMean } from "./reporting-DY-DC4HG.mjs";
 import { t as mapWithConcurrency } from "./concurrency-D58PWeSk.mjs";
-import { r as signFlipPValue } from "./math-DhrDmpFS.mjs";
-import { n as formatDemos, r as parseDemos, t as bootstrapDemos } from "./demos-Degx6UmP.mjs";
+import { i as signFlipPValue, n as holmAdjust } from "./math-Dqme4rYz.mjs";
+import { a as harvestRollouts, n as harvestFewShotExamples, r as parseDemos, t as formatDemos } from "./demos-DE2oxNWX.mjs";
 import { t as parseProposedText } from "./text--v4Ffbus.mjs";
 //#region src/compare.ts
-const EXACT_LIMIT = 16;
+const EXACT_LIMIT = 20;
+/** Well above float subtraction noise (~1e-16), well below a real margin. */
+const DEGENERACY_TOLERANCE = 1e-9;
 /**
 * Run several optimizers over the same seeds and report which one actually won.
 *
@@ -19,7 +21,7 @@ const EXACT_LIMIT = 16;
 * seed is constructor config and every optimizer here is deterministic given
 * one — comparing two entrants at a single seed compares two anecdotes. It also
 * leaves the task where the caller builds it, which is the only place the
-* optimizer-specific parts of it (`reflect`, `cache`, `onEvent`) are known.
+* optimizer-specific parts of it (`reflect`, `cache`, `reporters`) are known.
 */
 async function compare(args) {
 	const { entrants, seeds, concurrency = 1 } = args;
@@ -42,6 +44,8 @@ async function compare(args) {
 				bestScore: result.bestScore,
 				...result.testScore === void 0 ? {} : { testScore: result.testScore },
 				metricCalls: result.metricCalls,
+				cacheHits: result.cacheHits,
+				reflectionCalls: result.reflectionCalls ?? 0,
 				costUsd: result.usage.costUsd,
 				stopReason: result.stopReason
 			};
@@ -52,17 +56,25 @@ async function compare(args) {
 		runs: runs.filter((run) => run.entrant === entrant)
 	}));
 	const winner = summaries.reduce((best, summary) => summary.meanScore > best.meanScore ? summary : best);
+	const rawPValues = summaries.map((summary) => summary.entrant === winner.entrant ? void 0 : margin({
+		winner: winner.entrant,
+		entrant: summary.entrant,
+		runs,
+		seeds
+	}));
+	const holmAdjusted = holmAdjustSparse({
+		pValues: rawPValues,
+		familySize: names.length - 1
+	});
 	return {
 		winner: winner.entrant,
-		summaries: summaries.map((summary) => summary.entrant === winner.entrant ? summary : {
+		summaries: summaries.map((summary, index) => ({
 			...summary,
-			pValueVsWinner: margin({
-				winner: winner.entrant,
-				entrant: summary.entrant,
-				runs,
-				seeds
-			})
-		}),
+			...rawPValues[index] === void 0 ? {} : {
+				pValueVsWinner: rawPValues[index],
+				pValueVsWinnerHolm: holmAdjusted[index]
+			}
+		})),
 		runs
 	};
 }
@@ -77,13 +89,28 @@ function summarize(args) {
 		minScore: Math.min(...scores),
 		maxScore: Math.max(...scores),
 		meanMetricCalls: mean(runs.map((run) => run.metricCalls)),
-		meanCostUsd: mean(runs.map((run) => run.costUsd))
+		meanCostUsd: mean(runs.map((run) => run.costUsd)),
+		meanCacheHits: mean(runs.map((run) => run.cacheHits)),
+		meanReflectionCalls: mean(runs.map((run) => run.reflectionCalls)),
+		distinctScores: new Set(scores).size
 	};
 }
 /**
 * Paired across seeds rather than pooled: the same seed puts both entrants on
 * the same sampling order, so the difference at a seed is a comparison and the
 * spread between seeds is not.
+*
+* Returns `undefined` when every seed produced the exact same nonzero margin.
+* That is not evidence of an n-seed-strong result — it is one realization the
+* seed never varied, and a sign-flip p-value would report a precision from n
+* independent trials that never happened. A margin of exactly zero every seed
+* is not this case: `signFlipPValue` already reports that honestly as 1, no
+* significance claimed either way, which is not a fabricated number.
+*
+* "Exact same" is judged within `DEGENERACY_TOLERANCE`, not `===`: subtracting
+* two scores that are equal in substance can still land a few ULPs apart
+* (0.95 - 0.55 and 0.9 - 0.5 differ at the 16th digit), and treating that as
+* n real trials would be the same fabrication this check exists to prevent.
 */
 function margin(args) {
 	const { winner, entrant, runs, seeds } = args;
@@ -98,6 +125,10 @@ function margin(args) {
 			seed
 		});
 	});
+	const [first] = differences;
+	if (first !== void 0) {
+		if (differences.reduce((widest, difference) => Math.max(widest, Math.abs(difference - first)), 0) < DEGENERACY_TOLERANCE && Math.abs(first) > DEGENERACY_TOLERANCE) return;
+	}
 	return signFlipPValue({
 		differences,
 		observed: differences.reduce((total, value) => total + value, 0),
@@ -108,6 +139,27 @@ function scoreOf(args) {
 	const { runs, entrant, seed } = args;
 	return runs.find((candidate) => candidate.entrant === entrant && candidate.seed === seed)?.score ?? 0;
 }
+/**
+* `holmAdjust` over the raw p-values that exist, skipping the slots a
+* withheld comparison left `undefined` — those still count toward
+* `familySize`, they just have nothing of their own to adjust.
+*/
+function holmAdjustSparse(args) {
+	const { pValues, familySize } = args;
+	const present = pValues.map((pValue, index) => ({
+		pValue,
+		index
+	})).filter((entry) => entry.pValue !== void 0);
+	const adjusted = holmAdjust({
+		pValues: present.map((entry) => entry.pValue),
+		familySize
+	});
+	const result = new Array(pValues.length).fill(void 0);
+	present.forEach((entry, rank) => {
+		result[entry.index] = adjusted[rank];
+	});
+	return result;
+}
 function mean(values) {
 	return values.reduce((total, value) => total + value, 0) / values.length;
 }
@@ -118,6 +170,51 @@ function standardDeviation(values) {
 	return Math.sqrt(variance);
 }
 //#endregion
+//#region src/distillation.ts
+const ROLES = /* @__PURE__ */ new Set([
+	"system",
+	"user",
+	"assistant"
+]);
+/**
+* Serialize harvested rollouts as JSONL, one training example per line.
+*
+* The chat-messages shape rather than any one vendor's: it is what Axolotl,
+* Together, Fireworks and the Hugging Face trainers all ingest, and the
+* providers that read it outlive the ones that do not.
+*
+* Returns the text rather than writing it. Only `file-cache` touches the
+* filesystem, and a caller uploading this straight to a provider should not
+* have to round-trip it through a file to do so.
+*/
+function toTrainingJsonl(args) {
+	const { rollouts, render } = args;
+	const lines = [];
+	for (const [index, rollout] of rollouts.entries()) {
+		const example = render({
+			rollout,
+			index
+		});
+		if (example === null) continue;
+		assertUsable(example, index);
+		lines.push(JSON.stringify(example));
+	}
+	return lines.join("\n");
+}
+/**
+* Checked here rather than left to the provider. A malformed line surfaces as
+* a rejected upload hours later, naming a line number in a file the caller
+* never wrote by hand; naming the rollout at the point it was rendered is the
+* same error while it is still fixable.
+*/
+function assertUsable(example, index) {
+	if (!Array.isArray(example.messages) || example.messages.length === 0) throw new Error(`rollout ${index} rendered no messages`);
+	for (const message of example.messages) {
+		if (!ROLES.has(message.role)) throw new Error(`rollout ${index} rendered the unknown role ${message.role}`);
+		if (typeof message.content !== "string") throw new Error(`rollout ${index} rendered non-text content`);
+	}
+}
+//#endregion
 //#region src/judge.ts
 const DEFAULT_SCALE = 5;
 const SCORE = /<score\s+name="([^"]+)"\s*>\s*([\d.]+)\s*<\/score>/g;
@@ -140,6 +237,7 @@ const FEEDBACK = /<feedback>([\s\S]*?)<\/feedback>/;
 function createJudge(args) {
 	const { model, criteria, scale = DEFAULT_SCALE, renderInput = stringify, renderOutput = stringify, buildPrompt = buildJudgePrompt } = args;
 	if (criteria.length === 0) throw new Error("createJudge requires at least one criterion");
+	if (!Number.isFinite(scale) || scale <= 0) throw new Error(`scale must be a positive number, received ${scale}`);
 	return async ({ input, output, expected, signal }) => {
 		return readVerdict({
 			response: await model({
@@ -242,4 +340,4 @@ function priceUsage(args) {
 	};
 }
 //#endregion
-export { BudgetExhausted, assertResumable, bootstrapDemos, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, runFingerprint };
+export { BudgetExhausted, assertResumable, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, harvestFewShotExamples, harvestRollouts, isCandidateAccepted, isRunFinished, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, runFingerprint, toTrainingJsonl };

package/dist/{math-COOofUyv.cjs → math-BhlziRPc.cjs}

@@ -44,17 +44,62 @@ function signFlipPValue(args) {
 		differences: moved,
 		observed
 	});
-	const assignments = 2 ** moved.length;
-	let atLeastAsExtreme = 0;
-	for (let mask = 0; mask < assignments; mask += 1) {
-		let total = 0;
-		for (let index = 0; index < moved.length; index += 1) {
-			const sign = mask >> index & 1 ? -1 : 1;
-			total += sign * moved[index];
+	const sums = achievableSums(moved);
+	return sums.filter((total) => total >= observed).length / sums.length;
+}
+/**
+* Holm-Bonferroni step-down adjustment: the p-value each comparison would need
+* to survive on its own if the whole family were held to one error rate,
+* rather than letting the smallest of several tests read as significant by
+* volume alone. Sorted ascending, each rank is scaled by how many comparisons
+* are still in contention at that rank, and the running maximum keeps a later,
+* less-scaled rank from reporting looser than an earlier one already has.
+*
+* `familySize` may exceed `pValues.length`: a comparison `signFlipPValue`
+* could not test at all (every paired difference identical, so no p reflects
+* a real margin) is still a member of the family being controlled for, and
+* excluding it from the denominator would understate the correction owed to
+* the comparisons that could be tested.
+*/
+function holmAdjust(args) {
+	const { pValues, familySize } = args;
+	const ranked = pValues.map((pValue, index) => ({
+		pValue,
+		index
+	})).sort((a, b) => a.pValue - b.pValue);
+	const adjusted = new Array(pValues.length);
+	let runningMax = 0;
+	ranked.forEach(({ pValue, index }, rank) => {
+		runningMax = Math.max(runningMax, Math.min(1, pValue * (familySize - rank)));
+		adjusted[index] = runningMax;
+	});
+	return adjusted;
+}
+/**
+* Every total reachable by flipping some subset of `differences`' signs,
+* built by doubling rather than by scoring each of the 2^n sign masks
+* independently: after `k` differences there are 2^k sums, and folding in
+* difference `k+1` only ever adds or subtracts it from each of them, so the
+* whole enumeration costs O(2^n) instead of the O(2^n * n) a per-mask loop
+* pays for re-summing n terms every time. Measured at n=20, twenty seeds
+* being the bench's ceiling: about 63ms scoring masks one at a time against
+* about 16ms building sums this way — the difference between affording exact
+* enumeration through a full twenty-seed run and falling back to the normal
+* approximation, which is what raising `EXACT_LIMIT` in compare.ts to 20
+* relies on.
+*/
+function achievableSums(differences) {
+	let sums = [0];
+	for (const difference of differences) {
+		const next = new Array(sums.length * 2);
+		for (let index = 0; index < sums.length; index += 1) {
+			const total = sums[index];
+			next[index] = total + difference;
+			next[index + sums.length] = total - difference;
 		}
-		if (total >= observed) atLeastAsExtreme += 1;
+		sums = next;
 	}
-	return atLeastAsExtreme / assignments;
+	return sums;
 }
 /**
 * The same tail probability from a normal approximation, for batches too large
@@ -81,6 +126,12 @@ Object.defineProperty(exports, "argmax", {
 		return argmax;
 	}
 });
+Object.defineProperty(exports, "holmAdjust", {
+	enumerable: true,
+	get: function() {
+		return holmAdjust;
+	}
+});
 Object.defineProperty(exports, "mean", {
 	enumerable: true,
 	get: function() {

package/dist/math-Dqme4rYz.mjs

@@ -0,0 +1,123 @@
+//#region src/math.ts
+function argmax(values) {
+	let bestIndex = 0;
+	let bestValue = Number.NEGATIVE_INFINITY;
+	for (let index = 0; index < values.length; index += 1) {
+		const value = values[index];
+		if (value > bestValue) {
+			bestValue = value;
+			bestIndex = index;
+		}
+	}
+	return bestIndex;
+}
+/** Mean over the values that exist; unscored instances are not zeros. */
+function mean(values) {
+	let total = 0;
+	let count = 0;
+	for (const value of values) if (value !== void 0) {
+		total += value;
+		count += 1;
+	}
+	return count === 0 ? 0 : total / count;
+}
+function sum(values) {
+	let total = 0;
+	for (const value of values) total += value;
+	return total;
+}
+/**
+* One-sided p-value from a paired sign-flip (permutation) test: how often a
+* total this large arises when each difference is equally likely to have come
+* out the other way. Exact by enumeration for small samples, normal-approximated
+* past `maxExact` where 2^n stops being cheap.
+*
+* Paired and distribution-free, which is what a comparison of two runs over the
+* same instances needs — the scores are not independent draws and are not
+* normal, so a t-test on them is answering a different question.
+*/
+function signFlipPValue(args) {
+	const { differences, observed, maxExact } = args;
+	const moved = differences.filter((difference) => difference !== 0);
+	if (moved.length === 0) return 1;
+	if (moved.length > maxExact) return normalTailProbability({
+		differences: moved,
+		observed
+	});
+	const sums = achievableSums(moved);
+	return sums.filter((total) => total >= observed).length / sums.length;
+}
+/**
+* Holm-Bonferroni step-down adjustment: the p-value each comparison would need
+* to survive on its own if the whole family were held to one error rate,
+* rather than letting the smallest of several tests read as significant by
+* volume alone. Sorted ascending, each rank is scaled by how many comparisons
+* are still in contention at that rank, and the running maximum keeps a later,
+* less-scaled rank from reporting looser than an earlier one already has.
+*
+* `familySize` may exceed `pValues.length`: a comparison `signFlipPValue`
+* could not test at all (every paired difference identical, so no p reflects
+* a real margin) is still a member of the family being controlled for, and
+* excluding it from the denominator would understate the correction owed to
+* the comparisons that could be tested.
+*/
+function holmAdjust(args) {
+	const { pValues, familySize } = args;
+	const ranked = pValues.map((pValue, index) => ({
+		pValue,
+		index
+	})).sort((a, b) => a.pValue - b.pValue);
+	const adjusted = new Array(pValues.length);
+	let runningMax = 0;
+	ranked.forEach(({ pValue, index }, rank) => {
+		runningMax = Math.max(runningMax, Math.min(1, pValue * (familySize - rank)));
+		adjusted[index] = runningMax;
+	});
+	return adjusted;
+}
+/**
+* Every total reachable by flipping some subset of `differences`' signs,
+* built by doubling rather than by scoring each of the 2^n sign masks
+* independently: after `k` differences there are 2^k sums, and folding in
+* difference `k+1` only ever adds or subtracts it from each of them, so the
+* whole enumeration costs O(2^n) instead of the O(2^n * n) a per-mask loop
+* pays for re-summing n terms every time. Measured at n=20, twenty seeds
+* being the bench's ceiling: about 63ms scoring masks one at a time against
+* about 16ms building sums this way — the difference between affording exact
+* enumeration through a full twenty-seed run and falling back to the normal
+* approximation, which is what raising `EXACT_LIMIT` in compare.ts to 20
+* relies on.
+*/
+function achievableSums(differences) {
+	let sums = [0];
+	for (const difference of differences) {
+		const next = new Array(sums.length * 2);
+		for (let index = 0; index < sums.length; index += 1) {
+			const total = sums[index];
+			next[index] = total + difference;
+			next[index + sums.length] = total - difference;
+		}
+		sums = next;
+	}
+	return sums;
+}
+/**
+* The same tail probability from a normal approximation, for batches too large
+* to enumerate. Under the sign-flip null each difference has mean zero and
+* variance equal to its square, so the total's variance is their sum.
+*/
+function normalTailProbability(args) {
+	const { differences, observed } = args;
+	const variance = differences.reduce((total, difference) => total + difference * difference, 0);
+	if (variance === 0) return observed > 0 ? 0 : 1;
+	return 1 - standardNormalCdf(observed / Math.sqrt(variance));
+}
+/** Abramowitz and Stegun 7.1.26, which is accurate to about 1e-7. */
+function standardNormalCdf(value) {
+	const sign = value < 0 ? -1 : 1;
+	const scaled = Math.abs(value) / Math.SQRT2;
+	const t = 1 / (1 + .3275911 * scaled);
+	return .5 * (1 + sign * (1 - t * (.254829592 + t * (-.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429)))) * Math.exp(-scaled * scaled)));
+}
+//#endregion
+export { sum as a, signFlipPValue as i, holmAdjust as n, mean as r, argmax as t };