textopt 0.0.0 → 0.1.0

package/dist/mipro/index.mjs

@@ -1,7 +1,7 @@
-import { a as requireMeasuredMean, d as createMemoryCache, f as stableHash, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "../evaluation-BV0nSZVx.mjs";
+import { _ as createMemoryCache, a as isRunFinished, b as createBudget, c as createEvaluator, d as createDeadline, f as assertResumable, h as componentNames, i as isCandidateAccepted, l as measuredMean, m as runFingerprint, n as flushReporters, o as BudgetExhausted, r as instanceRow, s as costExhausted, t as createEmitter, u as requireMeasuredMean, v as defaultInstanceId } from "../reporting-DY-DC4HG.mjs";
 import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
-import { n as mean } from "../math-DhrDmpFS.mjs";
-import { n as formatDemos, t as bootstrapDemos } from "../demos-Degx6UmP.mjs";
+import { r as mean } from "../math-Dqme4rYz.mjs";
+import { n as harvestFewShotExamples, t as formatDemos } from "../demos-DE2oxNWX.mjs";
 import { t as parseProposedText } from "../text--v4Ffbus.mjs";
 import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
 import { t as createEpochShuffledSampler } from "../sampling-CfHt7Gue.mjs";
@@ -211,10 +211,14 @@ var MiproOptimizer = class {
 		this.#config = config;
 	}
 	async optimize(task) {
-		return runMipro({
-			config: this.#config,
-			task
-		});
+		try {
+			return await runMipro({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await flushReporters(task.reporters ?? []);
+		}
 	}
 };
 function buildMiproPrompt(args) {
@@ -273,7 +277,8 @@ function buildDatasetSummaryPrompt(examples) {
 async function runMipro(args) {
 	const { config, task } = args;
 	const { instructionsPerComponent = DEFAULT_INSTRUCTIONS, minibatchSize = DEFAULT_MINIBATCH_SIZE, maxTrials = DEFAULT_MAX_TRIALS, startupTrials, gamma, surrogateSamples, multivariate, fullEvalInterval = DEFAULT_FULL_EVAL_INTERVAL, demoSets = DEFAULT_DEMO_SETS, maxDemos = DEFAULT_MAX_DEMOS, demoMinScore, exemplars = DEFAULT_EXEMPLARS, datasetSummary = true, summaryExamples = DEFAULT_SUMMARY_EXAMPLES, concurrency = 1, seed = 0, buildPrompt = buildMiproPrompt, tips = DEFAULT_TIPS, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, componentOptions, demoComponents, renderDemo, goldOutput, maxMetricCalls, renderDatum = renderDefault, batchSampler = createEpochShuffledSampler({ minibatchSize }), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
+	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, componentOptions, demoComponents, renderDemo, goldOutput, maxMetricCalls, renderDatum = renderDefault, batchSampler = createEpochShuffledSampler({ minibatchSize }), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const emit = createEmitter(reporters);
 	const deadline = createDeadline({ maxWallClockMs });
 	const components = componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
@@ -318,15 +323,16 @@ async function runMipro(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
-		onEvaluation: (event) => onEvent?.({
+		onEvaluation: (event) => emit({
 			type: "evaluation",
 			...event
 		})
 	});
 	evaluator.restore(resumeFrom?.cache ?? []);
 	if (resumeFrom?.sampler !== void 0) batchSampler.restore?.(resumeFrom.sampler);
-	onEvent?.({
+	emit({
 		type: "start",
 		components,
 		validationSetSize: validationSet.length
@@ -373,20 +379,26 @@ async function runMipro(args) {
 		for (let index = 0; index < demoSets; index += 1) {
 			const affordable = Math.min(trainingSet.length, budget.remaining() - validationSet.length);
 			if (affordable < 1) break;
+			if (costExhausted({
+				usage: evaluator.usage(),
+				maxCostUsd
+			})) break;
 			const requested = demoSets === 1 ? maxDemos : Math.round(1 + index * (maxDemos - 1) / (demoSets - 1));
-			const harvest = await bootstrapDemos({
+			const harvest = await harvestFewShotExamples({
 				adapter,
 				candidate: seedCandidate,
 				trainingSet,
 				...demoMinScore === void 0 ? {} : { minScore: demoMinScore },
 				maxDemos: requested,
 				maxMetricCalls: affordable,
+				...maxCostUsd === void 0 ? {} : { maxCostUsd: maxCostUsd - evaluator.usage().costUsd },
 				rng,
 				...renderDemo === void 0 ? {} : { renderDemo },
 				...signal === void 0 ? {} : { signal }
 			});
 			bootstrapMetricCalls += harvest.metricCalls;
 			budget.reserve(harvest.metricCalls);
+			evaluator.absorbUsage(harvest.usage);
 			if (harvest.demos.length > 0) blocks.push(formatDemos(harvest.demos, renderDemo === void 0 ? {} : { render: renderDemo }));
 		}
 		return [.../* @__PURE__ */ new Set([
@@ -431,7 +443,7 @@ async function runMipro(args) {
 		});
 		menu[name] = [seedCandidate[name], ...drawn.filter((text) => text.length > 0)];
 	}
-	onEvent?.({
+	emit({
 		type: "menu",
 		menu,
 		reflectionCalls
@@ -466,6 +478,15 @@ async function runMipro(args) {
 		phase: "seed"
 	});
 	if (seedEvaluation !== void 0) fullEvaluations += 1;
+	if (seedEvaluation !== void 0) emit({
+		type: "candidateAccepted",
+		trial: 0,
+		candidateId: 0,
+		candidate: seedCandidate,
+		aggregateScore: seedScore,
+		instanceScores: instanceRow(seedEvaluation),
+		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
+	});
 	if (resumeFrom === void 0) surrogateInput.push({
 		choices: menuSizes.map(() => 0),
 		score: seedScore
@@ -473,6 +494,7 @@ async function runMipro(args) {
 	let best = resumeFrom?.best ?? seedCandidate;
 	let bestScore = resumeFrom?.bestScore ?? seedScore;
 	/** Absent on a resumed run until a sweep wins: outputs are not checkpointed. */
+	let acceptedCandidates = resumeFrom?.acceptedCandidates ?? 0;
 	let bestOutputs = seedEvaluation?.outputs;
 	const readings = new Map(resumeFrom?.readings ?? []);
 	const swept = new Set(resumeFrom?.swept ?? []);
@@ -492,6 +514,8 @@ async function runMipro(args) {
 			bootstrapMetricCalls,
 			metricCalls: budget.spent(),
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
+			acceptedCandidates,
 			rngState: rng.state(),
 			observations: [...observations],
 			surrogateInput: surrogateInput.map((entry) => ({
@@ -553,11 +577,15 @@ async function runMipro(args) {
 			best = candidate;
 			bestScore = score;
 			bestOutputs = evaluation.outputs;
-			onEvent?.({
-				type: "incumbent",
+			acceptedCandidates += 1;
+			emit({
+				type: "candidateAccepted",
 				trial,
-				choices,
-				score
+				candidateId: acceptedCandidates,
+				candidate,
+				aggregateScore: score,
+				instanceScores: instanceRow(evaluation),
+				...trackBestOutputs ? { outputs: evaluation.outputs } : {}
 			});
 		}
 		return "swept";
@@ -624,47 +652,44 @@ async function runMipro(args) {
 			}
 			throw err;
 		}
-		if (minibatchScore === void 0) {
-			trial += 1;
-			continue;
+		if (minibatchScore !== void 0) {
+			surrogateInput.push({
+				choices,
+				score: minibatchScore
+			});
+			const key = choices.join(",");
+			readings.set(key, [...readings.get(key) ?? [], minibatchScore]);
+			const observation = {
+				trial,
+				choices,
+				minibatchScore,
+				promoted: false
+			};
+			observations.push(observation);
+			emit({
+				type: "trial",
+				trial,
+				choices,
+				minibatchScore,
+				promoted: false
+			});
 		}
-		surrogateInput.push({
-			choices,
-			score: minibatchScore
-		});
-		const key = choices.join(",");
-		readings.set(key, [...readings.get(key) ?? [], minibatchScore]);
-		const observation = {
-			trial,
-			choices,
-			minibatchScore,
-			promoted: false
-		};
-		observations.push(observation);
-		onEvent?.({
-			type: "trial",
-			trial,
-			choices,
-			minibatchScore,
-			promoted: false
-		});
 		trial += 1;
-		await checkpoint();
+		let cadenceStop;
 		if (trial % fullEvalInterval === 0) {
 			const outcome = await sweepBestUnswept();
-			if (outcome === "budgetExhausted" || outcome === "aborted") {
-				stopReason = outcome;
-				break;
-			}
-			if (outcome === "unaffordable") {
-				stopReason = "budgetExhausted";
-				break;
-			}
+			if (outcome === "budgetExhausted" || outcome === "aborted") cadenceStop = outcome;
+			if (outcome === "unaffordable") cadenceStop = "budgetExhausted";
+		}
+		await checkpoint();
+		if (cadenceStop !== void 0) {
+			stopReason = cadenceStop;
+			break;
 		}
 	}
 	if (stopReason === "maxTrials" && !signal?.aborted) await sweepBestUnswept();
 	if (signal?.aborted) stopReason = "aborted";
-	const testScore = testSet === void 0 ? void 0 : measuredMean(await evaluator.evaluate({
+	const heldOut = testSet === void 0 ? void 0 : await evaluator.evaluate({
 		candidate: best,
 		batch: testSet,
 		ids: testIds,
@@ -673,13 +698,17 @@ async function runMipro(args) {
 		candidateId: null,
 		iteration: trial,
 		charge: false
-	}));
-	onEvent?.({
+	});
+	const testScore = heldOut === void 0 ? void 0 : measuredMean(heldOut);
+	emit({
 		type: "finish",
 		reason: stopReason,
+		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
 		snapshot: takeSnapshot(),
@@ -690,7 +719,8 @@ async function runMipro(args) {
 		...trackBestOutputs ? { bestOutputs } : {},
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: evaluator.unchargedCalls()
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		trials: trial,
 		menu,
@@ -728,9 +758,5 @@ function assertConfig(config) {
 	if (config.gamma !== void 0 && (!Number.isFinite(config.gamma) || config.gamma <= 0 || config.gamma > 1)) throw new Error(`gamma must be within (0, 1], received ${config.gamma}`);
 	if (config.tips !== void 0 && config.tips.length === 0) throw new Error("tips must not be empty");
 }
-function defaultInstanceId(args) {
-	const hash = stableHash(args.datum);
-	return hash === "" ? String(args.index) : hash;
-}
 //#endregion
-export { MiproOptimizer, buildMiproPrompt, proposeConfiguration };
+export { MiproOptimizer, buildMiproPrompt, isCandidateAccepted, isRunFinished, proposeConfiguration };

package/dist/opro/index.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_evaluation = require("../evaluation-OZOp6TB7.cjs");
+const require_reporting = require("../reporting-CNHzbJC-.cjs");
 const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
 const require_text = require("../text-CK_HB3su.cjs");
 const require_rng = require("../rng-DbA_rPIo.cjs");
@@ -43,10 +43,14 @@ var OproOptimizer = class {
 		this.#config = config;
 	}
 	async optimize(task) {
-		return runOpro({
-			config: this.#config,
-			task
-		});
+		try {
+			return await runOpro({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await require_reporting.flushReporters(task.reporters ?? []);
+		}
 	}
 };
 /**
@@ -91,9 +95,10 @@ function buildOproPrompt(args) {
 async function runOpro(args) {
 	const { config, task } = args;
 	const { proposalsPerRound = DEFAULT_PROPOSALS_PER_ROUND, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, maxReflectionCalls = Number.POSITIVE_INFINITY, seed = 0, historySize = DEFAULT_HISTORY_SIZE, exemplars = DEFAULT_EXEMPLARS, scoringSetSize, fullEvalInterval = DEFAULT_FULL_EVAL_INTERVAL, scoreScale = DEFAULT_SCORE_SCALE, buildPrompt = buildOproPrompt, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, renderDatum = renderDefault, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
-	const deadline = require_evaluation.createDeadline({ maxWallClockMs });
-	const components = require_evaluation.componentNames(seedCandidate);
+	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, renderDatum = renderDefault, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_reporting.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const emit = require_reporting.createEmitter(reporters);
+	const deadline = require_reporting.createDeadline({ maxWallClockMs });
+	const components = require_reporting.componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
 	if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
 	if (components.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
@@ -106,7 +111,7 @@ async function runOpro(args) {
 		datum,
 		index
 	})) ?? [];
-	const fingerprint = require_evaluation.runFingerprint({
+	const fingerprint = require_reporting.runFingerprint({
 		seedCandidate,
 		trainingIds: trainingSet.map((datum, index) => instanceId({
 			datum,
@@ -116,17 +121,17 @@ async function runOpro(args) {
 		seed,
 		...cacheNamespace === void 0 ? {} : { cacheNamespace }
 	});
-	require_evaluation.assertResumable({
+	require_reporting.assertResumable({
 		fingerprint,
 		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
 	});
 	const rng = require_rng.createSeededRng(seed, resumeFrom?.rngState);
-	const budget = require_evaluation.createBudget({
+	const budget = require_reporting.createBudget({
 		maxMetricCalls,
 		spent: resumeFrom?.metricCalls ?? 0
 	});
-	const evaluationCache = cache === false ? void 0 : cache ?? require_evaluation.createMemoryCache();
-	const evaluator = require_evaluation.createEvaluator({
+	const evaluationCache = cache === false ? void 0 : cache ?? require_reporting.createMemoryCache();
+	const evaluator = require_reporting.createEvaluator({
 		adapter,
 		budget,
 		...retry === void 0 ? {} : { retry },
@@ -134,8 +139,9 @@ async function runOpro(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
-		onEvaluation: (event) => onEvent?.({
+		onEvaluation: (event) => emit({
 			type: "evaluation",
 			...event
 		})
@@ -154,13 +160,13 @@ async function runOpro(args) {
 	function contextOf(candidate, component) {
 		const rest = {};
 		for (const name of components) if (name !== component) rest[name] = candidate[name];
-		return require_evaluation.stableHash(rest);
+		return require_reporting.stableHash(rest);
 	}
 	const trajectory = [];
 	let round = resumeFrom?.round ?? 0;
 	let reflectionCalls = resumeFrom?.reflectionCalls ?? 0;
 	let stopReason = "maxRounds";
-	onEvent?.({
+	emit({
 		type: "start",
 		components,
 		validationSetSize: validationSet.length
@@ -193,19 +199,41 @@ async function runOpro(args) {
 			iteration: round
 		});
 	}
+	/** What screening a candidate would cost, in rollouts nothing has cached. */
+	function screenCost(candidate) {
+		return scoringSet === void 0 ? evaluator.countUncached({
+			candidate,
+			ids: validationIds,
+			split: "val"
+		}) : evaluator.countUncached({
+			candidate,
+			ids: scoringIds,
+			split: "train"
+		});
+	}
 	const seedEvaluation = resumeFrom === void 0 ? await sweep(seedCandidate, "seed") : void 0;
-	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_evaluation.requireMeasuredMean({
+	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_reporting.requireMeasuredMean({
 		batch: seedEvaluation,
 		phase: "seed"
 	});
+	if (seedEvaluation !== void 0) emit({
+		type: "candidateAccepted",
+		round: 0,
+		candidateId: 0,
+		candidate: seedCandidate,
+		aggregateScore: seedScore,
+		instanceScores: require_reporting.instanceRow(seedEvaluation),
+		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
+	});
 	let best = resumeFrom?.best ?? seedCandidate;
 	let bestScore = resumeFrom?.bestScore ?? seedScore;
 	/** Absent on a resumed run until a sweep wins: outputs are not checkpointed. */
+	let acceptedCandidates = resumeFrom?.acceptedCandidates ?? 0;
 	let bestOutputs = seedEvaluation?.outputs;
 	let reported = resumeFrom?.reported ?? seedCandidate;
 	let lastSwept = resumeFrom === void 0 ? seedCandidate : resumeFrom.incumbentSwept ? best : {};
 	let bestSearchScore = resumeFrom?.bestSearchScore ?? seedScore;
-	if (resumeFrom === void 0 && scoringSet !== void 0) bestSearchScore = require_evaluation.requireMeasuredMean({
+	if (resumeFrom === void 0 && scoringSet !== void 0) bestSearchScore = require_reporting.requireMeasuredMean({
 		batch: await screen(seedCandidate, "seed"),
 		phase: "seed"
 	});
@@ -215,19 +243,42 @@ async function runOpro(args) {
 	* what gets reported is the best candidate a full sweep has actually seen —
 	* never a subset number wearing a validation set label.
 	*/
+	/**
+	* The incumbent moved and a full sweep measured it. Emitted from the two
+	* places that can be true — a screening run with no scoring set, where the
+	* attempt's own evaluation is the sweep, and the cadence that confirms an
+	* incumbent later — because a payload assembled twice is one that drifts.
+	*/
+	function emitAccepted(args) {
+		acceptedCandidates += 1;
+		emit({
+			type: "candidateAccepted",
+			round,
+			candidateId: acceptedCandidates,
+			candidate: args.candidate,
+			aggregateScore: args.score,
+			instanceScores: require_reporting.instanceRow(args.evaluation),
+			...trackBestOutputs ? { outputs: args.evaluation.outputs } : {}
+		});
+	}
 	async function refreshIncumbent() {
 		if (best === lastSwept || !budget.canAfford(validationSet.length)) return "ok";
 		try {
 			const evaluation = await sweep(best, "validation");
-			const full = require_evaluation.measuredMean(evaluation);
+			const full = require_reporting.measuredMean(evaluation);
 			lastSwept = best;
 			if (full !== void 0 && full > bestScore) {
 				reported = best;
 				bestScore = full;
 				bestOutputs = evaluation.outputs;
+				emitAccepted({
+					candidate: best,
+					evaluation,
+					score: full
+				});
 			}
 		} catch (err) {
-			if (err instanceof require_evaluation.BudgetExhausted || signal?.aborted) return "stop";
+			if (err instanceof require_reporting.BudgetExhausted || signal?.aborted) return "stop";
 			throw err;
 		}
 		return "ok";
@@ -260,6 +311,8 @@ async function runOpro(args) {
 			reflectionCalls,
 			metricCalls: budget.spent(),
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
+			acceptedCandidates,
 			rngState: rng.state(),
 			histories: Object.fromEntries([...histories].map(([name, attempts]) => [name, [...attempts]])),
 			...scoringIndices === void 0 ? {} : { scoringIndices },
@@ -281,7 +334,7 @@ async function runOpro(args) {
 			stopReason = "proposalsExhausted";
 			break;
 		}
-		if (require_evaluation.costExhausted({
+		if (require_reporting.costExhausted({
 			usage: evaluator.usage(),
 			maxCostUsd
 		})) {
@@ -304,7 +357,7 @@ async function runOpro(args) {
 		const history = histories.get(component);
 		const context = contextOf(best, component);
 		const comparable = history.filter((attempt) => attempt.context === context);
-		onEvent?.({
+		emit({
 			type: "roundStart",
 			round,
 			component,
@@ -333,26 +386,44 @@ async function runOpro(args) {
 		const unique = [...new Set(drawn)].filter((text) => text.length > 0 && !tried.has(text));
 		barrenRounds = unique.length === 0 ? barrenRounds + 1 : 0;
 		let roundStop;
+		const scheduled = [];
+		let owed = 0;
 		for (const text of unique) {
 			const candidate = {
 				...best,
 				[component]: text
 			};
-			let evaluation;
-			try {
-				evaluation = await screen(candidate, "validation");
-			} catch (err) {
-				if (err instanceof require_evaluation.BudgetExhausted) {
-					roundStop = "budgetExhausted";
-					break;
-				}
-				if (signal?.aborted) {
-					roundStop = "aborted";
-					break;
+			const cost = screenCost(candidate);
+			if (!budget.canAfford(owed + cost)) {
+				roundStop = "budgetExhausted";
+				break;
+			}
+			owed += cost;
+			scheduled.push(candidate);
+		}
+		const screened = await require_concurrency.mapWithConcurrency({
+			items: scheduled,
+			limit: concurrency,
+			task: async (candidate) => {
+				if (signal?.aborted) return { stop: "aborted" };
+				try {
+					return { evaluation: await screen(candidate, "validation") };
+				} catch (err) {
+					if (err instanceof require_reporting.BudgetExhausted) return { stop: "budgetExhausted" };
+					if (signal?.aborted) return { stop: "aborted" };
+					throw err;
 				}
-				throw err;
 			}
-			const score = require_evaluation.measuredMean(evaluation);
+		});
+		for (const [index, outcome] of screened.entries()) {
+			if (outcome.stop !== void 0) {
+				roundStop = outcome.stop;
+				break;
+			}
+			const candidate = scheduled[index];
+			const text = candidate[component];
+			const evaluation = outcome.evaluation;
+			const score = require_reporting.measuredMean(evaluation);
 			if (score === void 0) continue;
 			const accepted = score > bestSearchScore;
 			history.push({
@@ -366,7 +437,7 @@ async function runOpro(args) {
 				candidate,
 				score
 			});
-			onEvent?.({
+			emit({
 				type: "attempt",
 				round,
 				component,
@@ -381,6 +452,11 @@ async function runOpro(args) {
 					lastSwept = candidate;
 					bestScore = score;
 					bestOutputs = evaluation.outputs;
+					emitAccepted({
+						candidate,
+						evaluation,
+						score
+					});
 				}
 				const scaled = scaleScore(score, scoreScale);
 				for (const name of components) {
@@ -394,22 +470,22 @@ async function runOpro(args) {
 			}
 		}
 		round += 1;
+		let cadenceStop;
+		if (roundStop === void 0 && scoringSet !== void 0 && round % fullEvalInterval === 0 && await refreshIncumbent() === "stop") cadenceStop = signal?.aborted ? "aborted" : "budgetExhausted";
 		await checkpoint();
 		if (roundStop !== void 0) {
 			stopReason = roundStop;
 			break;
 		}
-		if (scoringSet !== void 0 && round % fullEvalInterval === 0) {
-			if (await refreshIncumbent() === "stop") {
-				stopReason = signal?.aborted ? "aborted" : "budgetExhausted";
-				break;
-			}
+		if (cadenceStop !== void 0) {
+			stopReason = cadenceStop;
+			break;
 		}
 	}
 	if (scoringSet !== void 0 && !signal?.aborted) await refreshIncumbent();
 	if (signal?.aborted) stopReason = "aborted";
-	const testScore = testSet === void 0 ? void 0 : require_evaluation.measuredMean(await evaluator.evaluate({
-		candidate: best,
+	const heldOut = testSet === void 0 ? void 0 : await evaluator.evaluate({
+		candidate: reported,
 		batch: testSet,
 		ids: testIds,
 		split: "test",
@@ -417,13 +493,17 @@ async function runOpro(args) {
 		candidateId: null,
 		iteration: round,
 		charge: false
-	}));
-	onEvent?.({
+	});
+	const testScore = heldOut === void 0 ? void 0 : require_reporting.measuredMean(heldOut);
+	emit({
 		type: "finish",
 		reason: stopReason,
+		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: require_reporting.instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
 		snapshot: takeSnapshot(),
@@ -434,7 +514,8 @@ async function runOpro(args) {
 		...trackBestOutputs ? { bestOutputs } : {},
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: evaluator.unchargedCalls()
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		rounds: round,
 		trajectory,
@@ -478,10 +559,8 @@ function assertConfig(config) {
 	for (const [name, value] of positive) if (value !== void 0 && (!Number.isInteger(value) || value < 1)) throw new Error(`${name} must be a positive integer, received ${value}`);
 	if (config.exemplars !== void 0 && (!Number.isInteger(config.exemplars) || config.exemplars < 0)) throw new Error(`exemplars must be a non-negative integer, received ${config.exemplars}`);
 }
-function defaultInstanceId(args) {
-	const hash = require_evaluation.stableHash(args.datum);
-	return hash === "" ? String(args.index) : hash;
-}
 //#endregion
 exports.OproOptimizer = OproOptimizer;
 exports.buildOproPrompt = buildOproPrompt;
+exports.isCandidateAccepted = require_reporting.isCandidateAccepted;
+exports.isRunFinished = require_reporting.isRunFinished;

package/dist/opro/index.d.cts

@@ -1,6 +1,6 @@
-import { n as Candidate, t as Adapter, u as TextModel } from "../types-CWv4IQFF.cjs";
+import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.cjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.cjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-B7SpRwl7.cjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Bh5DPRMH.cjs";
 //#region src/opro/optimize.d.ts
 /** One instruction that was tried, and what it scored. */
 /** A history entry plus the system state its score was measured in. */
@@ -111,6 +111,13 @@ interface OproSnapshot {
   reflectionCalls: number;
   metricCalls: number;
   cacheHits: number;
+  /** Usage already spent, so a resumed run reports totals and honours ceilings. */
+  usage?: UsageTotals;
+  /**
+   * Candidates accepted so far. Reporters key rows by this id, so restarting it
+   * at zero makes a resumed run collide with the run it continues.
+   */
+  acceptedCandidates?: number;
   rngState: number;
   /** Component name -> every text tried for it, with what it scored. */
   histories: Record<string, RecordedAttempt[]>;
@@ -133,7 +140,8 @@ interface OproTask<Datum, Trajectory = unknown, Output = unknown, K extends stri
   }) => string;
   /** Pass `false` to disable caching entirely. */
   cache?: EvaluationCache | false;
-  onEvent?: (event: OproEvent<NoInfer<K>>) => void;
+  /** Observers of the run. Every one sees every event; none can fail it. */
+  reporters?: readonly Reporter<OproEvent<NoInfer<K>>>[];
   /**
    * Called with a resumable snapshot after the seed is scored and after every
    * round. Persist it and a killed run costs the last round, not all of them.
@@ -161,13 +169,13 @@ type OproEvent<K extends string = string> = {
   score: number;
   /** True when this attempt became the new incumbent. */
   accepted: boolean;
-} | {
+} | ({
+  type: "candidateAccepted";
+  round: number;
+} & CandidateAccepted<K>) | ({
   type: "finish";
   reason: OproStopReason;
-  bestScore: number;
-  metricCalls: number;
-  testScore?: number;
-};
+} & RunFinished);
 interface OproAttempt<K extends string = string> {
   round: number;
   component: K;
@@ -227,4 +235,4 @@ declare function buildOproPrompt(args: {
   exemplars: readonly string[];
 }): string;
 //#endregion
-export { type OproAttempt, type OproConfig, type OproEvent, OproOptimizer, type OproPromptBuilder, type OproResult, type OproSnapshot, type OproStopReason, type OproTask, type ScoredAttempt, buildOproPrompt };
+export { type CandidateAccepted, type OproAttempt, type OproConfig, type OproEvent, OproOptimizer, type OproPromptBuilder, type OproResult, type OproSnapshot, type OproStopReason, type OproTask, type OptimizerEvent, type ReportableEvent, type Reporter, type RunFinished, type ScoredAttempt, buildOproPrompt, isCandidateAccepted, isRunFinished };