textopt 0.0.0 → 0.2.0

package/dist/mipro/index.mjs

@@ -1,7 +1,7 @@
-import { a as requireMeasuredMean, d as createMemoryCache, f as stableHash, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "../evaluation-BV0nSZVx.mjs";
+import { S as createBudget, _ as componentNames, a as instanceRow, b as defaultInstanceId, c as BudgetExhausted, d as measuredMean, f as requireMeasuredMean, g as runFingerprint, i as flushReporters, l as costExhausted, m as assertResumable, o as isCandidateAccepted, p as createDeadline, r as createEmitter, s as isRunFinished, t as resolveValidationSet, u as createEvaluator, y as createMemoryCache } from "../warnings-OxvDi9kN.mjs";
 import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
-import { n as mean } from "../math-DhrDmpFS.mjs";
-import { n as formatDemos, t as bootstrapDemos } from "../demos-Degx6UmP.mjs";
+import { r as mean } from "../math-Dqme4rYz.mjs";
+import { n as harvestFewShotExamples, t as formatDemos } from "../demos-Brobjfuc.mjs";
 import { t as parseProposedText } from "../text--v4Ffbus.mjs";
 import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
 import { t as createEpochShuffledSampler } from "../sampling-CfHt7Gue.mjs";
@@ -211,10 +211,14 @@ var MiproOptimizer = class {
 		this.#config = config;
 	}
 	async optimize(task) {
-		return runMipro({
-			config: this.#config,
-			task
-		});
+		try {
+			return await runMipro({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await flushReporters(task.reporters ?? []);
+		}
 	}
 };
 function buildMiproPrompt(args) {
@@ -273,7 +277,12 @@ function buildDatasetSummaryPrompt(examples) {
 async function runMipro(args) {
 	const { config, task } = args;
 	const { instructionsPerComponent = DEFAULT_INSTRUCTIONS, minibatchSize = DEFAULT_MINIBATCH_SIZE, maxTrials = DEFAULT_MAX_TRIALS, startupTrials, gamma, surrogateSamples, multivariate, fullEvalInterval = DEFAULT_FULL_EVAL_INTERVAL, demoSets = DEFAULT_DEMO_SETS, maxDemos = DEFAULT_MAX_DEMOS, demoMinScore, exemplars = DEFAULT_EXEMPLARS, datasetSummary = true, summaryExamples = DEFAULT_SUMMARY_EXAMPLES, concurrency = 1, seed = 0, buildPrompt = buildMiproPrompt, tips = DEFAULT_TIPS, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, componentOptions, demoComponents, renderDemo, goldOutput, maxMetricCalls, renderDatum = renderDefault, batchSampler = createEpochShuffledSampler({ minibatchSize }), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, componentOptions, demoComponents, renderDemo, goldOutput, maxMetricCalls, renderDatum = renderDefault, batchSampler = createEpochShuffledSampler({ minibatchSize }), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
+	const emit = createEmitter(reporters);
 	const deadline = createDeadline({ maxWallClockMs });
 	const components = componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
@@ -318,15 +327,16 @@ async function runMipro(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
-		onEvaluation: (event) => onEvent?.({
+		onEvaluation: (event) => emit({
 			type: "evaluation",
 			...event
 		})
 	});
 	evaluator.restore(resumeFrom?.cache ?? []);
 	if (resumeFrom?.sampler !== void 0) batchSampler.restore?.(resumeFrom.sampler);
-	onEvent?.({
+	emit({
 		type: "start",
 		components,
 		validationSetSize: validationSet.length
@@ -373,20 +383,26 @@ async function runMipro(args) {
 		for (let index = 0; index < demoSets; index += 1) {
 			const affordable = Math.min(trainingSet.length, budget.remaining() - validationSet.length);
 			if (affordable < 1) break;
+			if (costExhausted({
+				usage: evaluator.usage(),
+				maxCostUsd
+			})) break;
 			const requested = demoSets === 1 ? maxDemos : Math.round(1 + index * (maxDemos - 1) / (demoSets - 1));
-			const harvest = await bootstrapDemos({
+			const harvest = await harvestFewShotExamples({
 				adapter,
 				candidate: seedCandidate,
 				trainingSet,
 				...demoMinScore === void 0 ? {} : { minScore: demoMinScore },
 				maxDemos: requested,
 				maxMetricCalls: affordable,
+				...maxCostUsd === void 0 ? {} : { maxCostUsd: maxCostUsd - evaluator.usage().costUsd },
 				rng,
 				...renderDemo === void 0 ? {} : { renderDemo },
 				...signal === void 0 ? {} : { signal }
 			});
 			bootstrapMetricCalls += harvest.metricCalls;
 			budget.reserve(harvest.metricCalls);
+			evaluator.absorbUsage(harvest.usage);
 			if (harvest.demos.length > 0) blocks.push(formatDemos(harvest.demos, renderDemo === void 0 ? {} : { render: renderDemo }));
 		}
 		return [.../* @__PURE__ */ new Set([
@@ -431,7 +447,7 @@ async function runMipro(args) {
 		});
 		menu[name] = [seedCandidate[name], ...drawn.filter((text) => text.length > 0)];
 	}
-	onEvent?.({
+	emit({
 		type: "menu",
 		menu,
 		reflectionCalls
@@ -466,6 +482,15 @@ async function runMipro(args) {
 		phase: "seed"
 	});
 	if (seedEvaluation !== void 0) fullEvaluations += 1;
+	if (seedEvaluation !== void 0) emit({
+		type: "candidateAccepted",
+		trial: 0,
+		candidateId: 0,
+		candidate: seedCandidate,
+		aggregateScore: seedScore,
+		instanceScores: instanceRow(seedEvaluation),
+		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
+	});
 	if (resumeFrom === void 0) surrogateInput.push({
 		choices: menuSizes.map(() => 0),
 		score: seedScore
@@ -473,6 +498,7 @@ async function runMipro(args) {
 	let best = resumeFrom?.best ?? seedCandidate;
 	let bestScore = resumeFrom?.bestScore ?? seedScore;
 	/** Absent on a resumed run until a sweep wins: outputs are not checkpointed. */
+	let acceptedCandidates = resumeFrom?.acceptedCandidates ?? 0;
 	let bestOutputs = seedEvaluation?.outputs;
 	const readings = new Map(resumeFrom?.readings ?? []);
 	const swept = new Set(resumeFrom?.swept ?? []);
@@ -492,6 +518,8 @@ async function runMipro(args) {
 			bootstrapMetricCalls,
 			metricCalls: budget.spent(),
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
+			acceptedCandidates,
 			rngState: rng.state(),
 			observations: [...observations],
 			surrogateInput: surrogateInput.map((entry) => ({
@@ -553,11 +581,15 @@ async function runMipro(args) {
 			best = candidate;
 			bestScore = score;
 			bestOutputs = evaluation.outputs;
-			onEvent?.({
-				type: "incumbent",
+			acceptedCandidates += 1;
+			emit({
+				type: "candidateAccepted",
 				trial,
-				choices,
-				score
+				candidateId: acceptedCandidates,
+				candidate,
+				aggregateScore: score,
+				instanceScores: instanceRow(evaluation),
+				...trackBestOutputs ? { outputs: evaluation.outputs } : {}
 			});
 		}
 		return "swept";
@@ -624,47 +656,44 @@ async function runMipro(args) {
 			}
 			throw err;
 		}
-		if (minibatchScore === void 0) {
-			trial += 1;
-			continue;
+		if (minibatchScore !== void 0) {
+			surrogateInput.push({
+				choices,
+				score: minibatchScore
+			});
+			const key = choices.join(",");
+			readings.set(key, [...readings.get(key) ?? [], minibatchScore]);
+			const observation = {
+				trial,
+				choices,
+				minibatchScore,
+				promoted: false
+			};
+			observations.push(observation);
+			emit({
+				type: "trial",
+				trial,
+				choices,
+				minibatchScore,
+				promoted: false
+			});
 		}
-		surrogateInput.push({
-			choices,
-			score: minibatchScore
-		});
-		const key = choices.join(",");
-		readings.set(key, [...readings.get(key) ?? [], minibatchScore]);
-		const observation = {
-			trial,
-			choices,
-			minibatchScore,
-			promoted: false
-		};
-		observations.push(observation);
-		onEvent?.({
-			type: "trial",
-			trial,
-			choices,
-			minibatchScore,
-			promoted: false
-		});
 		trial += 1;
-		await checkpoint();
+		let cadenceStop;
 		if (trial % fullEvalInterval === 0) {
 			const outcome = await sweepBestUnswept();
-			if (outcome === "budgetExhausted" || outcome === "aborted") {
-				stopReason = outcome;
-				break;
-			}
-			if (outcome === "unaffordable") {
-				stopReason = "budgetExhausted";
-				break;
-			}
+			if (outcome === "budgetExhausted" || outcome === "aborted") cadenceStop = outcome;
+			if (outcome === "unaffordable") cadenceStop = "budgetExhausted";
+		}
+		await checkpoint();
+		if (cadenceStop !== void 0) {
+			stopReason = cadenceStop;
+			break;
 		}
 	}
 	if (stopReason === "maxTrials" && !signal?.aborted) await sweepBestUnswept();
 	if (signal?.aborted) stopReason = "aborted";
-	const testScore = testSet === void 0 ? void 0 : measuredMean(await evaluator.evaluate({
+	const heldOut = testSet === void 0 ? void 0 : await evaluator.evaluate({
 		candidate: best,
 		batch: testSet,
 		ids: testIds,
@@ -673,13 +702,18 @@ async function runMipro(args) {
 		candidateId: null,
 		iteration: trial,
 		charge: false
-	}));
-	onEvent?.({
+	});
+	const testScore = heldOut === void 0 ? void 0 : measuredMean(heldOut);
+	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
+		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
 		snapshot: takeSnapshot(),
@@ -690,7 +724,8 @@ async function runMipro(args) {
 		...trackBestOutputs ? { bestOutputs } : {},
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: evaluator.unchargedCalls()
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		trials: trial,
 		menu,
@@ -700,6 +735,7 @@ async function runMipro(args) {
 		metricCalls: budget.spent(),
 		reflectionCalls,
 		cacheHits: evaluator.cacheHits(),
+		warnings,
 		stopReason
 	};
 }
@@ -728,9 +764,5 @@ function assertConfig(config) {
 	if (config.gamma !== void 0 && (!Number.isFinite(config.gamma) || config.gamma <= 0 || config.gamma > 1)) throw new Error(`gamma must be within (0, 1], received ${config.gamma}`);
 	if (config.tips !== void 0 && config.tips.length === 0) throw new Error("tips must not be empty");
 }
-function defaultInstanceId(args) {
-	const hash = stableHash(args.datum);
-	return hash === "" ? String(args.index) : hash;
-}
 //#endregion
-export { MiproOptimizer, buildMiproPrompt, proposeConfiguration };
+export { MiproOptimizer, buildMiproPrompt, isCandidateAccepted, isRunFinished, proposeConfiguration };

package/dist/opro/index.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_evaluation = require("../evaluation-OZOp6TB7.cjs");
+const require_warnings = require("../warnings-CWRJF-jA.cjs");
 const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
 const require_text = require("../text-CK_HB3su.cjs");
 const require_rng = require("../rng-DbA_rPIo.cjs");
@@ -43,10 +43,14 @@ var OproOptimizer = class {
 		this.#config = config;
 	}
 	async optimize(task) {
-		return runOpro({
-			config: this.#config,
-			task
-		});
+		try {
+			return await runOpro({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await require_warnings.flushReporters(task.reporters ?? []);
+		}
 	}
 };
 /**
@@ -91,9 +95,14 @@ function buildOproPrompt(args) {
 async function runOpro(args) {
 	const { config, task } = args;
 	const { proposalsPerRound = DEFAULT_PROPOSALS_PER_ROUND, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, maxReflectionCalls = Number.POSITIVE_INFINITY, seed = 0, historySize = DEFAULT_HISTORY_SIZE, exemplars = DEFAULT_EXEMPLARS, scoringSetSize, fullEvalInterval = DEFAULT_FULL_EVAL_INTERVAL, scoreScale = DEFAULT_SCORE_SCALE, buildPrompt = buildOproPrompt, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, renderDatum = renderDefault, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
-	const deadline = require_evaluation.createDeadline({ maxWallClockMs });
-	const components = require_evaluation.componentNames(seedCandidate);
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, maxMetricCalls, renderDatum = renderDefault, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_warnings.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = require_warnings.resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
+	const emit = require_warnings.createEmitter(reporters);
+	const deadline = require_warnings.createDeadline({ maxWallClockMs });
+	const components = require_warnings.componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
 	if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
 	if (components.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
@@ -106,7 +115,7 @@ async function runOpro(args) {
 		datum,
 		index
 	})) ?? [];
-	const fingerprint = require_evaluation.runFingerprint({
+	const fingerprint = require_warnings.runFingerprint({
 		seedCandidate,
 		trainingIds: trainingSet.map((datum, index) => instanceId({
 			datum,
@@ -116,17 +125,17 @@ async function runOpro(args) {
 		seed,
 		...cacheNamespace === void 0 ? {} : { cacheNamespace }
 	});
-	require_evaluation.assertResumable({
+	require_warnings.assertResumable({
 		fingerprint,
 		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
 	});
 	const rng = require_rng.createSeededRng(seed, resumeFrom?.rngState);
-	const budget = require_evaluation.createBudget({
+	const budget = require_warnings.createBudget({
 		maxMetricCalls,
 		spent: resumeFrom?.metricCalls ?? 0
 	});
-	const evaluationCache = cache === false ? void 0 : cache ?? require_evaluation.createMemoryCache();
-	const evaluator = require_evaluation.createEvaluator({
+	const evaluationCache = cache === false ? void 0 : cache ?? require_warnings.createMemoryCache();
+	const evaluator = require_warnings.createEvaluator({
 		adapter,
 		budget,
 		...retry === void 0 ? {} : { retry },
@@ -134,8 +143,9 @@ async function runOpro(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
-		onEvaluation: (event) => onEvent?.({
+		onEvaluation: (event) => emit({
 			type: "evaluation",
 			...event
 		})
@@ -154,13 +164,13 @@ async function runOpro(args) {
 	function contextOf(candidate, component) {
 		const rest = {};
 		for (const name of components) if (name !== component) rest[name] = candidate[name];
-		return require_evaluation.stableHash(rest);
+		return require_warnings.stableHash(rest);
 	}
 	const trajectory = [];
 	let round = resumeFrom?.round ?? 0;
 	let reflectionCalls = resumeFrom?.reflectionCalls ?? 0;
 	let stopReason = "maxRounds";
-	onEvent?.({
+	emit({
 		type: "start",
 		components,
 		validationSetSize: validationSet.length
@@ -193,19 +203,41 @@ async function runOpro(args) {
 			iteration: round
 		});
 	}
+	/** What screening a candidate would cost, in rollouts nothing has cached. */
+	function screenCost(candidate) {
+		return scoringSet === void 0 ? evaluator.countUncached({
+			candidate,
+			ids: validationIds,
+			split: "val"
+		}) : evaluator.countUncached({
+			candidate,
+			ids: scoringIds,
+			split: "train"
+		});
+	}
 	const seedEvaluation = resumeFrom === void 0 ? await sweep(seedCandidate, "seed") : void 0;
-	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_evaluation.requireMeasuredMean({
+	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_warnings.requireMeasuredMean({
 		batch: seedEvaluation,
 		phase: "seed"
 	});
+	if (seedEvaluation !== void 0) emit({
+		type: "candidateAccepted",
+		round: 0,
+		candidateId: 0,
+		candidate: seedCandidate,
+		aggregateScore: seedScore,
+		instanceScores: require_warnings.instanceRow(seedEvaluation),
+		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
+	});
 	let best = resumeFrom?.best ?? seedCandidate;
 	let bestScore = resumeFrom?.bestScore ?? seedScore;
 	/** Absent on a resumed run until a sweep wins: outputs are not checkpointed. */
+	let acceptedCandidates = resumeFrom?.acceptedCandidates ?? 0;
 	let bestOutputs = seedEvaluation?.outputs;
 	let reported = resumeFrom?.reported ?? seedCandidate;
 	let lastSwept = resumeFrom === void 0 ? seedCandidate : resumeFrom.incumbentSwept ? best : {};
 	let bestSearchScore = resumeFrom?.bestSearchScore ?? seedScore;
-	if (resumeFrom === void 0 && scoringSet !== void 0) bestSearchScore = require_evaluation.requireMeasuredMean({
+	if (resumeFrom === void 0 && scoringSet !== void 0) bestSearchScore = require_warnings.requireMeasuredMean({
 		batch: await screen(seedCandidate, "seed"),
 		phase: "seed"
 	});
@@ -215,19 +247,42 @@ async function runOpro(args) {
 	* what gets reported is the best candidate a full sweep has actually seen —
 	* never a subset number wearing a validation set label.
 	*/
+	/**
+	* The incumbent moved and a full sweep measured it. Emitted from the two
+	* places that can be true — a screening run with no scoring set, where the
+	* attempt's own evaluation is the sweep, and the cadence that confirms an
+	* incumbent later — because a payload assembled twice is one that drifts.
+	*/
+	function emitAccepted(args) {
+		acceptedCandidates += 1;
+		emit({
+			type: "candidateAccepted",
+			round,
+			candidateId: acceptedCandidates,
+			candidate: args.candidate,
+			aggregateScore: args.score,
+			instanceScores: require_warnings.instanceRow(args.evaluation),
+			...trackBestOutputs ? { outputs: args.evaluation.outputs } : {}
+		});
+	}
 	async function refreshIncumbent() {
 		if (best === lastSwept || !budget.canAfford(validationSet.length)) return "ok";
 		try {
 			const evaluation = await sweep(best, "validation");
-			const full = require_evaluation.measuredMean(evaluation);
+			const full = require_warnings.measuredMean(evaluation);
 			lastSwept = best;
 			if (full !== void 0 && full > bestScore) {
 				reported = best;
 				bestScore = full;
 				bestOutputs = evaluation.outputs;
+				emitAccepted({
+					candidate: best,
+					evaluation,
+					score: full
+				});
 			}
 		} catch (err) {
-			if (err instanceof require_evaluation.BudgetExhausted || signal?.aborted) return "stop";
+			if (err instanceof require_warnings.BudgetExhausted || signal?.aborted) return "stop";
 			throw err;
 		}
 		return "ok";
@@ -260,6 +315,8 @@ async function runOpro(args) {
 			reflectionCalls,
 			metricCalls: budget.spent(),
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
+			acceptedCandidates,
 			rngState: rng.state(),
 			histories: Object.fromEntries([...histories].map(([name, attempts]) => [name, [...attempts]])),
 			...scoringIndices === void 0 ? {} : { scoringIndices },
@@ -281,7 +338,7 @@ async function runOpro(args) {
 			stopReason = "proposalsExhausted";
 			break;
 		}
-		if (require_evaluation.costExhausted({
+		if (require_warnings.costExhausted({
 			usage: evaluator.usage(),
 			maxCostUsd
 		})) {
@@ -304,7 +361,7 @@ async function runOpro(args) {
 		const history = histories.get(component);
 		const context = contextOf(best, component);
 		const comparable = history.filter((attempt) => attempt.context === context);
-		onEvent?.({
+		emit({
 			type: "roundStart",
 			round,
 			component,
@@ -333,26 +390,44 @@ async function runOpro(args) {
 		const unique = [...new Set(drawn)].filter((text) => text.length > 0 && !tried.has(text));
 		barrenRounds = unique.length === 0 ? barrenRounds + 1 : 0;
 		let roundStop;
+		const scheduled = [];
+		let owed = 0;
 		for (const text of unique) {
 			const candidate = {
 				...best,
 				[component]: text
 			};
-			let evaluation;
-			try {
-				evaluation = await screen(candidate, "validation");
-			} catch (err) {
-				if (err instanceof require_evaluation.BudgetExhausted) {
-					roundStop = "budgetExhausted";
-					break;
-				}
-				if (signal?.aborted) {
-					roundStop = "aborted";
-					break;
+			const cost = screenCost(candidate);
+			if (!budget.canAfford(owed + cost)) {
+				roundStop = "budgetExhausted";
+				break;
+			}
+			owed += cost;
+			scheduled.push(candidate);
+		}
+		const screened = await require_concurrency.mapWithConcurrency({
+			items: scheduled,
+			limit: concurrency,
+			task: async (candidate) => {
+				if (signal?.aborted) return { stop: "aborted" };
+				try {
+					return { evaluation: await screen(candidate, "validation") };
+				} catch (err) {
+					if (err instanceof require_warnings.BudgetExhausted) return { stop: "budgetExhausted" };
+					if (signal?.aborted) return { stop: "aborted" };
+					throw err;
 				}
-				throw err;
 			}
-			const score = require_evaluation.measuredMean(evaluation);
+		});
+		for (const [index, outcome] of screened.entries()) {
+			if (outcome.stop !== void 0) {
+				roundStop = outcome.stop;
+				break;
+			}
+			const candidate = scheduled[index];
+			const text = candidate[component];
+			const evaluation = outcome.evaluation;
+			const score = require_warnings.measuredMean(evaluation);
 			if (score === void 0) continue;
 			const accepted = score > bestSearchScore;
 			history.push({
@@ -366,7 +441,7 @@ async function runOpro(args) {
 				candidate,
 				score
 			});
-			onEvent?.({
+			emit({
 				type: "attempt",
 				round,
 				component,
@@ -381,6 +456,11 @@ async function runOpro(args) {
 					lastSwept = candidate;
 					bestScore = score;
 					bestOutputs = evaluation.outputs;
+					emitAccepted({
+						candidate,
+						evaluation,
+						score
+					});
 				}
 				const scaled = scaleScore(score, scoreScale);
 				for (const name of components) {
@@ -394,22 +474,22 @@ async function runOpro(args) {
 			}
 		}
 		round += 1;
+		let cadenceStop;
+		if (roundStop === void 0 && scoringSet !== void 0 && round % fullEvalInterval === 0 && await refreshIncumbent() === "stop") cadenceStop = signal?.aborted ? "aborted" : "budgetExhausted";
 		await checkpoint();
 		if (roundStop !== void 0) {
 			stopReason = roundStop;
 			break;
 		}
-		if (scoringSet !== void 0 && round % fullEvalInterval === 0) {
-			if (await refreshIncumbent() === "stop") {
-				stopReason = signal?.aborted ? "aborted" : "budgetExhausted";
-				break;
-			}
+		if (cadenceStop !== void 0) {
+			stopReason = cadenceStop;
+			break;
 		}
 	}
 	if (scoringSet !== void 0 && !signal?.aborted) await refreshIncumbent();
 	if (signal?.aborted) stopReason = "aborted";
-	const testScore = testSet === void 0 ? void 0 : require_evaluation.measuredMean(await evaluator.evaluate({
-		candidate: best,
+	const heldOut = testSet === void 0 ? void 0 : await evaluator.evaluate({
+		candidate: reported,
 		batch: testSet,
 		ids: testIds,
 		split: "test",
@@ -417,13 +497,18 @@ async function runOpro(args) {
 		candidateId: null,
 		iteration: round,
 		charge: false
-	}));
-	onEvent?.({
+	});
+	const testScore = heldOut === void 0 ? void 0 : require_warnings.measuredMean(heldOut);
+	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
+		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: require_warnings.instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
 		snapshot: takeSnapshot(),
@@ -434,13 +519,15 @@ async function runOpro(args) {
 		...trackBestOutputs ? { bestOutputs } : {},
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: evaluator.unchargedCalls()
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		rounds: round,
 		trajectory,
 		metricCalls: budget.spent(),
 		reflectionCalls,
 		cacheHits: evaluator.cacheHits(),
+		warnings,
 		stopReason
 	};
 }
@@ -478,10 +565,8 @@ function assertConfig(config) {
 	for (const [name, value] of positive) if (value !== void 0 && (!Number.isInteger(value) || value < 1)) throw new Error(`${name} must be a positive integer, received ${value}`);
 	if (config.exemplars !== void 0 && (!Number.isInteger(config.exemplars) || config.exemplars < 0)) throw new Error(`exemplars must be a non-negative integer, received ${config.exemplars}`);
 }
-function defaultInstanceId(args) {
-	const hash = require_evaluation.stableHash(args.datum);
-	return hash === "" ? String(args.index) : hash;
-}
 //#endregion
 exports.OproOptimizer = OproOptimizer;
 exports.buildOproPrompt = buildOproPrompt;
+exports.isCandidateAccepted = require_warnings.isCandidateAccepted;
+exports.isRunFinished = require_warnings.isRunFinished;