textopt 0.0.0 → 0.1.0

package/dist/gepa/index.mjs

@@ -1,7 +1,7 @@
-import { c as candidateFingerprint, d as createMemoryCache, f as stableHash, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "../evaluation-BV0nSZVx.mjs";
+import { _ as createMemoryCache, a as isRunFinished, b as createBudget, c as createEvaluator, d as createDeadline, f as assertResumable, h as componentNames, i as isCandidateAccepted, l as measuredMean, m as runFingerprint, n as flushReporters, o as BudgetExhausted, p as candidateFingerprint, r as instanceRow, s as costExhausted, t as createEmitter, v as defaultInstanceId } from "../reporting-DY-DC4HG.mjs";
 import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
-import { i as sum, n as mean, r as signFlipPValue, t as argmax } from "../math-DhrDmpFS.mjs";
-import { n as formatDemos, r as parseDemos } from "../demos-Degx6UmP.mjs";
+import { a as sum, i as signFlipPValue, r as mean, t as argmax } from "../math-Dqme4rYz.mjs";
+import { r as parseDemos, t as formatDemos } from "../demos-DE2oxNWX.mjs";
 import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
 import { a as createDefaultProposer, i as buildSimplifyPrompt, n as buildReflectionPrompt, o as diverseReflectionStrategies, r as buildRewritePrompt, t as buildGeneralizePrompt } from "../reflection-mwMhrjs_.mjs";
 import { t as createEpochShuffledSampler } from "../sampling-CfHt7Gue.mjs";
@@ -738,16 +738,20 @@ var GepaOptimizer = class {
 		this.#config = config;
 	}
 	async optimize(task) {
-		return runGepa({
-			config: this.#config,
-			task
-		});
+		try {
+			return await runGepa({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await flushReporters(task.reporters ?? []);
+		}
 	}
 };
 async function runGepa(args) {
 	const { config, task } = args;
 	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, seed = 0, candidateSelector = paretoSelector(), acceptance = improvementAcceptance(), merge, skipPerfectScore = true, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals, reflection, checkpointCache = true, trackBestOutputs = false, raiseOnError = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, componentSelector = roundRobinComponentSelector(), batchSampler = createEpochShuffledSampler({ minibatchSize }), valEvaluationPolicy = fullEvaluationPolicy(), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
+	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, componentSelector = roundRobinComponentSelector(), batchSampler = createEpochShuffledSampler({ minibatchSize }), valEvaluationPolicy = fullEvaluationPolicy(), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
 	const deadline = createDeadline({ maxWallClockMs });
 	const seedComponents = componentNames(seedCandidate);
 	const mergeConfig = {
@@ -827,8 +831,26 @@ async function runGepa(args) {
 	let mergesDue = resumeFrom?.merge.due ?? 0;
 	let totalMergesTested = resumeFrom?.merge.tested ?? 0;
 	let lastIterationAccepted = resumeFrom?.merge.lastIterationAccepted ?? false;
-	function emit(event) {
-		onEvent?.(event);
+	const emit = createEmitter(reporters);
+	/**
+	* Everything an acceptance means, in one event: the text, the aggregate, and
+	* the row it put on the frontier. Emitted from one place because the merge
+	* path and the mutation path accept candidates separately, and a payload
+	* assembled twice is a payload that drifts.
+	*/
+	function emitAccepted(record) {
+		const outputs = outputsByCandidate.get(record.id);
+		emit({
+			type: "candidateAccepted",
+			iteration,
+			candidateId: record.id,
+			parentIds: record.parentIds,
+			aggregateScore: record.aggregateScore,
+			source: record.source,
+			candidate: record.candidate,
+			instanceScores: record.instanceScores,
+			...outputs === void 0 ? {} : { outputs }
+		});
 	}
 	/**
 	* Copies everything mutable: a snapshot handed to `onCheckpoint` is a record
@@ -845,6 +867,7 @@ async function runGepa(args) {
 			metricCalls: budget.spent(),
 			reflectionCalls,
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
 			...samplerState === void 0 ? {} : { sampler: samplerState },
 			rejectedProposals: snapshotRejections({
 				rejections: rejectedProposals,
@@ -873,6 +896,7 @@ async function runGepa(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
 		onEvaluation: (event) => emit({
 			type: "evaluation",
@@ -992,7 +1016,7 @@ async function runGepa(args) {
 	if (records.length === 0) {
 		const seedInstances = selectValInstances(seedCandidate);
 		if (!budget.canAfford(seedInstances.length)) throw new Error(`maxMetricCalls (${maxMetricCalls}) is smaller than the ${seedInstances.length} validation instances selected for scoring; the seed candidate cannot be scored`);
-		addCandidate({
+		emitAccepted(addCandidate({
 			candidate: seedCandidate,
 			parentIds: [],
 			evaluation: await evaluateValidation({
@@ -1003,7 +1027,7 @@ async function runGepa(args) {
 			}),
 			source: "seed",
 			updatedComponents: []
-		});
+		}));
 		lastIterationAccepted = false;
 		mergesDue = 0;
 		await checkpoint();
@@ -1094,14 +1118,7 @@ async function runGepa(args) {
 		});
 		mergesDue -= 1;
 		totalMergesTested += 1;
-		emit({
-			type: "candidateAccepted",
-			iteration,
-			candidateId: record.id,
-			parentIds: record.parentIds,
-			aggregateScore: record.aggregateScore,
-			source: "merge"
-		});
+		emitAccepted(record);
 		return "attempted";
 	}
 	/**
@@ -1346,21 +1363,13 @@ async function runGepa(args) {
 				stop ??= "budgetExhausted";
 				break;
 			}
-			const record = addCandidate({
+			emitAccepted(addCandidate({
 				candidate: item.outcome.child,
 				parentIds: [item.outcome.plan.parent.id],
 				evaluation,
 				source: "mutation",
 				updatedComponents: componentNames(item.outcome.proposed)
-			});
-			emit({
-				type: "candidateAccepted",
-				iteration,
-				candidateId: record.id,
-				parentIds: record.parentIds,
-				aggregateScore: record.aggregateScore,
-				source: "mutation"
-			});
+			}));
 		}
 		return stop;
 	}
@@ -1444,7 +1453,7 @@ async function runGepa(args) {
 	}
 	const bestCandidateId = valEvaluationPolicy.bestCandidate(records);
 	const best = records[bestCandidateId];
-	const testScore = testSet === void 0 ? void 0 : measuredMean(await evaluateCached({
+	const heldOut = testSet === void 0 ? void 0 : await evaluateCached({
 		candidate: best.candidate,
 		batch: testSet,
 		ids: testIds,
@@ -1452,13 +1461,17 @@ async function runGepa(args) {
 		phase: "test",
 		candidateId: bestCandidateId,
 		charge: false
-	}));
+	});
+	const testScore = heldOut === void 0 ? void 0 : measuredMean(heldOut);
 	emit({
 		type: "finish",
 		reason: stopReason,
 		bestCandidateId,
+		bestScore: best.aggregateScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	const perObjectiveBest = collectPerObjectiveBest(records);
 	const bestOutputs = outputsByCandidate.get(bestCandidateId);
@@ -1469,7 +1482,8 @@ async function runGepa(args) {
 		bestCandidateId,
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: evaluator.unchargedCalls()
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		...bestOutputs === void 0 ? {} : { bestOutputs },
 		candidates: records,
@@ -1617,17 +1631,6 @@ function collectDominatorIds(records) {
 	return [...ids].sort((a, b) => a - b);
 }
 /**
-* Names an instance by a hash of its content rather than by the content
-* itself: the id ends up inside every cache key and inside the checkpoint
-* fingerprint, and embedding whole examples there costs memory proportional to
-* the dataset for no benefit. Data that will not serialize falls back to its
-* position, which is stable for as long as the dataset order is.
-*/
-function defaultInstanceId(args) {
-	const hash = stableHash(args.datum);
-	return hash === "" ? String(args.index) : hash;
-}
-/**
 * The two rollout sets restricted to the instances both of them measured.
 *
 * Screening is a paired comparison over one minibatch: a transient row is a
@@ -1649,4 +1652,4 @@ function pairMeasured(args) {
 	};
 }
 //#endregion
-export { GepaOptimizer, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };
+export { GepaOptimizer, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, isCandidateAccepted, isRunFinished, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };

package/dist/index.cjs

@@ -1,11 +1,13 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_evaluation = require("./evaluation-OZOp6TB7.cjs");
+const require_reporting = require("./reporting-CNHzbJC-.cjs");
 const require_concurrency = require("./concurrency-C-cFzWW2.cjs");
-const require_math = require("./math-COOofUyv.cjs");
-const require_demos = require("./demos-B9BJiNKz.cjs");
+const require_math = require("./math-BhlziRPc.cjs");
+const require_demos = require("./demos-D2o0qSSo.cjs");
 const require_text = require("./text-CK_HB3su.cjs");
 //#region src/compare.ts
-const EXACT_LIMIT = 16;
+const EXACT_LIMIT = 20;
+/** Well above float subtraction noise (~1e-16), well below a real margin. */
+const DEGENERACY_TOLERANCE = 1e-9;
 /**
 * Run several optimizers over the same seeds and report which one actually won.
 *
@@ -20,7 +22,7 @@ const EXACT_LIMIT = 16;
 * seed is constructor config and every optimizer here is deterministic given
 * one — comparing two entrants at a single seed compares two anecdotes. It also
 * leaves the task where the caller builds it, which is the only place the
-* optimizer-specific parts of it (`reflect`, `cache`, `onEvent`) are known.
+* optimizer-specific parts of it (`reflect`, `cache`, `reporters`) are known.
 */
 async function compare(args) {
 	const { entrants, seeds, concurrency = 1 } = args;
@@ -43,6 +45,8 @@ async function compare(args) {
 				bestScore: result.bestScore,
 				...result.testScore === void 0 ? {} : { testScore: result.testScore },
 				metricCalls: result.metricCalls,
+				cacheHits: result.cacheHits,
+				reflectionCalls: result.reflectionCalls ?? 0,
 				costUsd: result.usage.costUsd,
 				stopReason: result.stopReason
 			};
@@ -53,17 +57,25 @@ async function compare(args) {
 		runs: runs.filter((run) => run.entrant === entrant)
 	}));
 	const winner = summaries.reduce((best, summary) => summary.meanScore > best.meanScore ? summary : best);
+	const rawPValues = summaries.map((summary) => summary.entrant === winner.entrant ? void 0 : margin({
+		winner: winner.entrant,
+		entrant: summary.entrant,
+		runs,
+		seeds
+	}));
+	const holmAdjusted = holmAdjustSparse({
+		pValues: rawPValues,
+		familySize: names.length - 1
+	});
 	return {
 		winner: winner.entrant,
-		summaries: summaries.map((summary) => summary.entrant === winner.entrant ? summary : {
+		summaries: summaries.map((summary, index) => ({
 			...summary,
-			pValueVsWinner: margin({
-				winner: winner.entrant,
-				entrant: summary.entrant,
-				runs,
-				seeds
-			})
-		}),
+			...rawPValues[index] === void 0 ? {} : {
+				pValueVsWinner: rawPValues[index],
+				pValueVsWinnerHolm: holmAdjusted[index]
+			}
+		})),
 		runs
 	};
 }
@@ -78,13 +90,28 @@ function summarize(args) {
 		minScore: Math.min(...scores),
 		maxScore: Math.max(...scores),
 		meanMetricCalls: mean(runs.map((run) => run.metricCalls)),
-		meanCostUsd: mean(runs.map((run) => run.costUsd))
+		meanCostUsd: mean(runs.map((run) => run.costUsd)),
+		meanCacheHits: mean(runs.map((run) => run.cacheHits)),
+		meanReflectionCalls: mean(runs.map((run) => run.reflectionCalls)),
+		distinctScores: new Set(scores).size
 	};
 }
 /**
 * Paired across seeds rather than pooled: the same seed puts both entrants on
 * the same sampling order, so the difference at a seed is a comparison and the
 * spread between seeds is not.
+*
+* Returns `undefined` when every seed produced the exact same nonzero margin.
+* That is not evidence of an n-seed-strong result — it is one realization the
+* seed never varied, and a sign-flip p-value would report a precision from n
+* independent trials that never happened. A margin of exactly zero every seed
+* is not this case: `signFlipPValue` already reports that honestly as 1, no
+* significance claimed either way, which is not a fabricated number.
+*
+* "Exact same" is judged within `DEGENERACY_TOLERANCE`, not `===`: subtracting
+* two scores that are equal in substance can still land a few ULPs apart
+* (0.95 - 0.55 and 0.9 - 0.5 differ at the 16th digit), and treating that as
+* n real trials would be the same fabrication this check exists to prevent.
 */
 function margin(args) {
 	const { winner, entrant, runs, seeds } = args;
@@ -99,6 +126,10 @@ function margin(args) {
 			seed
 		});
 	});
+	const [first] = differences;
+	if (first !== void 0) {
+		if (differences.reduce((widest, difference) => Math.max(widest, Math.abs(difference - first)), 0) < DEGENERACY_TOLERANCE && Math.abs(first) > DEGENERACY_TOLERANCE) return;
+	}
 	return require_math.signFlipPValue({
 		differences,
 		observed: differences.reduce((total, value) => total + value, 0),
@@ -109,6 +140,27 @@ function scoreOf(args) {
 	const { runs, entrant, seed } = args;
 	return runs.find((candidate) => candidate.entrant === entrant && candidate.seed === seed)?.score ?? 0;
 }
+/**
+* `holmAdjust` over the raw p-values that exist, skipping the slots a
+* withheld comparison left `undefined` — those still count toward
+* `familySize`, they just have nothing of their own to adjust.
+*/
+function holmAdjustSparse(args) {
+	const { pValues, familySize } = args;
+	const present = pValues.map((pValue, index) => ({
+		pValue,
+		index
+	})).filter((entry) => entry.pValue !== void 0);
+	const adjusted = require_math.holmAdjust({
+		pValues: present.map((entry) => entry.pValue),
+		familySize
+	});
+	const result = new Array(pValues.length).fill(void 0);
+	present.forEach((entry, rank) => {
+		result[entry.index] = adjusted[rank];
+	});
+	return result;
+}
 function mean(values) {
 	return values.reduce((total, value) => total + value, 0) / values.length;
 }
@@ -119,6 +171,51 @@ function standardDeviation(values) {
 	return Math.sqrt(variance);
 }
 //#endregion
+//#region src/distillation.ts
+const ROLES = /* @__PURE__ */ new Set([
+	"system",
+	"user",
+	"assistant"
+]);
+/**
+* Serialize harvested rollouts as JSONL, one training example per line.
+*
+* The chat-messages shape rather than any one vendor's: it is what Axolotl,
+* Together, Fireworks and the Hugging Face trainers all ingest, and the
+* providers that read it outlive the ones that do not.
+*
+* Returns the text rather than writing it. Only `file-cache` touches the
+* filesystem, and a caller uploading this straight to a provider should not
+* have to round-trip it through a file to do so.
+*/
+function toTrainingJsonl(args) {
+	const { rollouts, render } = args;
+	const lines = [];
+	for (const [index, rollout] of rollouts.entries()) {
+		const example = render({
+			rollout,
+			index
+		});
+		if (example === null) continue;
+		assertUsable(example, index);
+		lines.push(JSON.stringify(example));
+	}
+	return lines.join("\n");
+}
+/**
+* Checked here rather than left to the provider. A malformed line surfaces as
+* a rejected upload hours later, naming a line number in a file the caller
+* never wrote by hand; naming the rollout at the point it was rendered is the
+* same error while it is still fixable.
+*/
+function assertUsable(example, index) {
+	if (!Array.isArray(example.messages) || example.messages.length === 0) throw new Error(`rollout ${index} rendered no messages`);
+	for (const message of example.messages) {
+		if (!ROLES.has(message.role)) throw new Error(`rollout ${index} rendered the unknown role ${message.role}`);
+		if (typeof message.content !== "string") throw new Error(`rollout ${index} rendered non-text content`);
+	}
+}
+//#endregion
 //#region src/judge.ts
 const DEFAULT_SCALE = 5;
 const SCORE = /<score\s+name="([^"]+)"\s*>\s*([\d.]+)\s*<\/score>/g;
@@ -141,6 +238,7 @@ const FEEDBACK = /<feedback>([\s\S]*?)<\/feedback>/;
 function createJudge(args) {
 	const { model, criteria, scale = DEFAULT_SCALE, renderInput = stringify, renderOutput = stringify, buildPrompt = buildJudgePrompt } = args;
 	if (criteria.length === 0) throw new Error("createJudge requires at least one criterion");
+	if (!Number.isFinite(scale) || scale <= 0) throw new Error(`scale must be a positive number, received ${scale}`);
 	return async ({ input, output, expected, signal }) => {
 		return readVerdict({
 			response: await model({
@@ -243,24 +341,28 @@ function priceUsage(args) {
 	};
 }
 //#endregion
-exports.BudgetExhausted = require_evaluation.BudgetExhausted;
-exports.assertResumable = require_evaluation.assertResumable;
-exports.bootstrapDemos = require_demos.bootstrapDemos;
+exports.BudgetExhausted = require_reporting.BudgetExhausted;
+exports.assertResumable = require_reporting.assertResumable;
 exports.buildJudgePrompt = buildJudgePrompt;
-exports.candidateFingerprint = require_evaluation.candidateFingerprint;
+exports.candidateFingerprint = require_reporting.candidateFingerprint;
 exports.compare = compare;
-exports.componentNames = require_evaluation.componentNames;
-exports.costExhausted = require_evaluation.costExhausted;
-exports.createBudget = require_evaluation.createBudget;
-exports.createDeadline = require_evaluation.createDeadline;
-exports.createEvaluator = require_evaluation.createEvaluator;
+exports.componentNames = require_reporting.componentNames;
+exports.costExhausted = require_reporting.costExhausted;
+exports.createBudget = require_reporting.createBudget;
+exports.createDeadline = require_reporting.createDeadline;
+exports.createEvaluator = require_reporting.createEvaluator;
 exports.createJudge = createJudge;
-exports.createMemoryCache = require_evaluation.createMemoryCache;
+exports.createMemoryCache = require_reporting.createMemoryCache;
 exports.formatDemos = require_demos.formatDemos;
+exports.harvestFewShotExamples = require_demos.harvestFewShotExamples;
+exports.harvestRollouts = require_demos.harvestRollouts;
+exports.isCandidateAccepted = require_reporting.isCandidateAccepted;
+exports.isRunFinished = require_reporting.isRunFinished;
 exports.mapWithConcurrency = require_concurrency.mapWithConcurrency;
-exports.measuredMean = require_evaluation.measuredMean;
+exports.measuredMean = require_reporting.measuredMean;
 exports.parseDemos = require_demos.parseDemos;
 exports.parseProposedText = require_text.parseProposedText;
 exports.priceUsage = priceUsage;
-exports.requireMeasuredMean = require_evaluation.requireMeasuredMean;
-exports.runFingerprint = require_evaluation.runFingerprint;
+exports.requireMeasuredMean = require_reporting.requireMeasuredMean;
+exports.runFingerprint = require_reporting.runFingerprint;
+exports.toTrainingJsonl = toTrainingJsonl;

package/dist/index.d.cts

@@ -1,8 +1,8 @@
-import { a as EvaluationContext, c as RolloutUsage, d as UsageTotals, f as componentNames, i as EvaluationBatch, l as ScoreResult, n as Candidate, o as EvaluationPhase, r as EvaluateArgs, s as EvaluationSplit, t as Adapter, u as TextModel } from "./types-CWv4IQFF.cjs";
+import { _ as TextModel, a as RunFinished, c as Adapter, d as EvaluationBatch, f as EvaluationContext, g as ScoreResult, h as RolloutUsage, i as Reporter, l as Candidate, m as EvaluationSplit, n as OptimizerEvent, o as isCandidateAccepted, p as EvaluationPhase, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, u as EvaluateArgs, v as UsageTotals, y as componentNames } from "./reporting-DQbAohc9.cjs";
 import { n as EvaluationCache, r as createMemoryCache, t as CachedScore } from "./cache-CuSo0NJ8.cjs";
 import { t as Rng } from "./rng-BR5MOedA.cjs";
-import { a as formatDemos, i as bootstrapDemos, n as Demo, o as parseDemos, r as DemoRenderer, t as BootstrapResult } from "./demos-BTuzFNsp.cjs";
-import { a as EvaluateBatchArgs, c as Evaluator, d as costExhausted, f as createEvaluator, g as createBudget, h as Budget, i as BudgetExhausted, l as RetryPolicy, m as requireMeasuredMean, n as OptimizerResult, o as EvaluateTracedArgs, p as measuredMean, r as OptimizerTask, s as EvaluationEvent, t as Optimizer, u as ScoredBatch } from "./optimizer-B7SpRwl7.cjs";
+import { a as harvestFewShotExamples, i as formatDemos, n as Demo, o as parseDemos, r as DemoRenderer, t as BootstrapResult } from "./demos-CU9dy2oT.cjs";
+import { a as EvaluateBatchArgs, c as Evaluator, d as costExhausted, f as createEvaluator, g as createBudget, h as Budget, i as BudgetExhausted, l as RetryPolicy, m as requireMeasuredMean, n as OptimizerResult, o as EvaluateTracedArgs, p as measuredMean, r as OptimizerTask, s as EvaluationEvent, t as Optimizer, u as ScoredBatch } from "./optimizer-Bh5DPRMH.cjs";
 import { t as BatchSampler } from "./sampling-axOwfZf5.cjs";
 //#region src/checkpoint.d.ts
 /**
@@ -46,6 +46,10 @@ interface ComparisonRun {
   bestScore: number;
   testScore?: number;
   metricCalls: number;
+  /** Rollouts this run got from the cache instead of paying for. */
+  cacheHits: number;
+  /** Calls to a proposal or reflection model, outside the metric budget. */
+  reflectionCalls: number;
   costUsd: number;
   stopReason: string;
 }
@@ -58,12 +62,33 @@ interface ComparisonSummary {
   maxScore: number;
   meanMetricCalls: number;
   meanCostUsd: number;
+  meanCacheHits: number;
+  meanReflectionCalls: number;
+  /**
+   * How many distinct values this entrant's score took across its seeds.
+   * 1 means every seed landed on the same number — the seed changed nothing
+   * about the outcome, whatever the search did internally with it.
+   */
+  distinctScores: number;
   /**
    * How often the winner's margin over this entrant would arise if the two
-   * were equally good and each seed's outcome were a coin flip. Absent for the
-   * winner itself.
+   * were equally good and each seed's outcome were a coin flip. Absent for
+   * the winner itself, and also absent when every seed produced the exact
+   * same margin: a sign-flip test over n seeds is answering a question about
+   * n independent trials, and identical margins mean the seed never actually
+   * put that to the test — there was one realization, repeated. Reporting a
+   * p-value there would state a precision (as fine as 2^-n) that the run
+   * never earned, so it is withheld rather than printed misleadingly small.
    */
   pValueVsWinner?: number;
+  /**
+   * `pValueVsWinner` after Holm-Bonferroni step-down across the other
+   * entrants in this same `compare()` call — the family the raw p-value
+   * would otherwise be read against in isolation. Absent wherever the raw
+   * p-value is: a withheld comparison has nothing to adjust, but it still
+   * occupies a slot in the family the other comparisons are corrected for.
+   */
+  pValueVsWinnerHolm?: number;
 }
 interface Comparison {
   /** Highest mean score. Read `pValueVsWinner` before believing it. */
@@ -85,7 +110,7 @@ interface Comparison {
  * seed is constructor config and every optimizer here is deterministic given
  * one — comparing two entrants at a single seed compares two anecdotes. It also
  * leaves the task where the caller builds it, which is the only place the
- * optimizer-specific parts of it (`reflect`, `cache`, `onEvent`) are known.
+ * optimizer-specific parts of it (`reflect`, `cache`, `reporters`) are known.
  */
 declare function compare<K extends string, Output = unknown>(args: {
   entrants: Record<string, (args: {
@@ -136,6 +161,119 @@ declare function createDeadline(args: {
   now?: () => number;
 }): Deadline;
 //#endregion
+//#region src/harvest.d.ts
+/**
+ * One rollout worth keeping: what went in, what the system produced, and how
+ * that output scored. Harvested, never written — the value of the pair is that
+ * the system actually produced the output and the metric actually rewarded it.
+ */
+interface Rollout<Datum = unknown, Output = unknown> {
+  input: Datum;
+  output: Output;
+  score: number;
+}
+interface HarvestResult<Datum, Output> {
+  rollouts: Rollout<Datum, Output>[];
+  /** Rollouts this cost. Harvesting is cheap, not free. */
+  metricCalls: number;
+  /**
+   * Tokens and dollars this cost. Harvesting runs on its own evaluator, so a
+   * caller that bounds spend has to fold these into its own totals.
+   */
+  usage: UsageTotals;
+  /** Instances run, including the ones the metric did not reward. */
+  attempted: number;
+}
+/**
+ * Run a candidate over data and keep the rollouts the metric rewarded.
+ *
+ * The library's one paid collection primitive, with two consumers: a few-shot
+ * block wants four of these, and a distillation set wants thousands. Both are
+ * the same pass — run the candidate, score it, keep what cleared the bar — so
+ * both share the budget, retry and transient-failure handling that pass needs.
+ *
+ * Which data to sweep is the caller's decision and the consequential one. A
+ * validation set is the wrong choice: it is the set that selected the candidate,
+ * so the rollouts it yields are enriched for the candidate's fit to those
+ * instances rather than to the task. Prefer the training set, or a pool held
+ * out of the run entirely.
+ */
+declare function harvestRollouts<Datum, Trajectory, Output, K extends string = string>(args: {
+  adapter: Adapter<Datum, Trajectory, Output, K>;
+  /** The candidate to run. Usually a run's winner, sometimes the seed. */
+  candidate: Candidate<K>;
+  data: readonly Datum[];
+  /**
+   * Score a rollout must reach to be kept. Unset keeps every rollout the
+   * metric rewarded at all, which is what MIPROv2's bootstrapper does without
+   * a `metric_threshold`: it keeps a trace on any truthy score and only
+   * compares against a number once one is configured.
+   *
+   * Demanding a perfect score instead is the right call for a boolean metric
+   * and the wrong one for a graded metric, where it throws away every rollout
+   * that was most of the way there — which on a hard task is all of them.
+   */
+  minScore?: number;
+  /** Rollouts to collect before stopping. Unset sweeps the whole pool. */
+  maxRollouts?: number;
+  /**
+   * Instances per rollout batch. Smaller batches stop closer to the moment
+   * enough rollouts exist, at the cost of less concurrency inside the adapter.
+   */
+  batchSize?: number;
+  /** Ceiling on rollouts run. Defaults to one pass over `data`. */
+  maxMetricCalls?: number;
+  /**
+   * Ceiling on dollars this pass may spend, checked between batches. Rollout
+   * counts are a poor proxy for spend, and a caller bounding dollars cannot
+   * bound this pass from outside: it runs on its own evaluator.
+   */
+  maxCostUsd?: number;
+  /** Shuffles `data` first, so rollouts are not all drawn from its head. */
+  rng?: Rng;
+  signal?: AbortSignal;
+}): Promise<HarvestResult<Datum, Output>>;
+//#endregion
+//#region src/distillation.d.ts
+/** A turn in a training example, in the shape trainers read. */
+interface ChatMessage {
+  role: "system" | "user" | "assistant";
+  content: string;
+}
+interface TrainingExample {
+  messages: ChatMessage[];
+}
+/**
+ * Turns one harvested rollout into the example to train on, or `null` to skip
+ * it. The callback exists because only the caller knows how to render a
+ * `Datum` — the library never sees inside one.
+ *
+ * It is also where the consequential decision lives: how much of the optimized
+ * candidate to leave in the input. Dropping it entirely moves the whole prompt
+ * into weights and gives up the ability to steer the model with text
+ * afterwards; keeping a short task statement distills away only the tokens the
+ * search accreted. See `docs/distillation.md`.
+ */
+type TrainingExampleRenderer<Datum, Output> = (args: {
+  rollout: Rollout<Datum, Output>;
+  index: number;
+}) => TrainingExample | null;
+/**
+ * Serialize harvested rollouts as JSONL, one training example per line.
+ *
+ * The chat-messages shape rather than any one vendor's: it is what Axolotl,
+ * Together, Fireworks and the Hugging Face trainers all ingest, and the
+ * providers that read it outlive the ones that do not.
+ *
+ * Returns the text rather than writing it. Only `file-cache` touches the
+ * filesystem, and a caller uploading this straight to a provider should not
+ * have to round-trip it through a file to do so.
+ */
+declare function toTrainingJsonl<Datum, Output>(args: {
+  rollouts: readonly Rollout<Datum, Output>[];
+  render: TrainingExampleRenderer<Datum, Output>;
+}): string;
+//#endregion
 //#region src/judge.d.ts
 /** One thing the judge grades, and what a perfect answer looks like for it. */
 interface JudgeCriterion {
@@ -218,4 +356,4 @@ declare function priceUsage(args: {
   pricing?: TokenPricing;
 }): RolloutUsage;
 //#endregion
-export { type Adapter, type BatchSampler, type BootstrapResult, type Budget, BudgetExhausted, type CachedScore, type Candidate, type Comparison, type ComparisonRun, type ComparisonSummary, type Deadline, type Demo, type DemoRenderer, type EvaluateArgs, type EvaluateBatchArgs, type EvaluateTracedArgs, type EvaluationBatch, type EvaluationCache, type EvaluationContext, type EvaluationEvent, type EvaluationPhase, type EvaluationSplit, type Evaluator, type Judge, type JudgeCriterion, type JudgePromptBuilder, type Optimizer, type OptimizerResult, type OptimizerTask, type RetryPolicy, type Rng, type RolloutUsage, type ScoreResult, type ScoredBatch, type TextModel, type TokenPricing, type UsageTotals, assertResumable, bootstrapDemos, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, runFingerprint };
+export { type Adapter, type BatchSampler, type BootstrapResult, type Budget, BudgetExhausted, type CachedScore, type Candidate, type CandidateAccepted, type ChatMessage, type Comparison, type ComparisonRun, type ComparisonSummary, type Deadline, type Demo, type DemoRenderer, type EvaluateArgs, type EvaluateBatchArgs, type EvaluateTracedArgs, type EvaluationBatch, type EvaluationCache, type EvaluationContext, type EvaluationEvent, type EvaluationPhase, type EvaluationSplit, type Evaluator, type HarvestResult, type Judge, type JudgeCriterion, type JudgePromptBuilder, type Optimizer, type OptimizerEvent, type OptimizerResult, type OptimizerTask, type ReportableEvent, type Reporter, type RetryPolicy, type Rng, type Rollout, type RolloutUsage, type RunFinished, type ScoreResult, type ScoredBatch, type TextModel, type TokenPricing, type TrainingExample, type TrainingExampleRenderer, type UsageTotals, assertResumable, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, harvestFewShotExamples, harvestRollouts, isCandidateAccepted, isRunFinished, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, runFingerprint, toTrainingJsonl };