textopt 0.0.0 → 0.1.0

package/dist/random-search/index.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_evaluation = require("../evaluation-OZOp6TB7.cjs");
+const require_reporting = require("../reporting-CNHzbJC-.cjs");
 const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
 const require_text = require("../text-CK_HB3su.cjs");
 //#region src/random-search/optimize.ts
@@ -33,10 +33,14 @@ var RandomSearchOptimizer = class {
 		this.#config = config;
 	}
 	async optimize(task) {
-		return runRandomSearch({
-			config: this.#config,
-			task
-		});
+		try {
+			return await runRandomSearch({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await require_reporting.flushReporters(task.reporters ?? []);
+		}
 	}
 };
 /**
@@ -63,9 +67,10 @@ function buildParaphrasePrompt(args) {
 async function runRandomSearch(args) {
 	const { config, task } = args;
 	const { variants = DEFAULT_VARIANTS, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, buildPrompt = buildParaphrasePrompt, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
-	const deadline = require_evaluation.createDeadline({ maxWallClockMs });
-	const components = require_evaluation.componentNames(seedCandidate);
+	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_reporting.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const emit = require_reporting.createEmitter(reporters);
+	const deadline = require_reporting.createDeadline({ maxWallClockMs });
+	const components = require_reporting.componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
 	if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
 	if (components.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
@@ -78,7 +83,7 @@ async function runRandomSearch(args) {
 		datum,
 		index
 	})) ?? [];
-	const fingerprint = require_evaluation.runFingerprint({
+	const fingerprint = require_reporting.runFingerprint({
 		seedCandidate,
 		trainingIds: trainingSet.map((datum, index) => instanceId({
 			datum,
@@ -87,16 +92,16 @@ async function runRandomSearch(args) {
 		validationIds,
 		...cacheNamespace === void 0 ? {} : { cacheNamespace }
 	});
-	require_evaluation.assertResumable({
+	require_reporting.assertResumable({
 		fingerprint,
 		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
 	});
-	const budget = require_evaluation.createBudget({
+	const budget = require_reporting.createBudget({
 		maxMetricCalls,
 		spent: resumeFrom?.metricCalls ?? 0
 	});
-	const evaluationCache = cache === false ? void 0 : cache ?? require_evaluation.createMemoryCache();
-	const evaluator = require_evaluation.createEvaluator({
+	const evaluationCache = cache === false ? void 0 : cache ?? require_reporting.createMemoryCache();
+	const evaluator = require_reporting.createEvaluator({
 		adapter,
 		budget,
 		...retry === void 0 ? {} : { retry },
@@ -104,8 +109,9 @@ async function runRandomSearch(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
-		onEvaluation: (event) => onEvent?.({
+		onEvaluation: (event) => emit({
 			type: "evaluation",
 			...event
 		})
@@ -123,7 +129,7 @@ async function runRandomSearch(args) {
 	* no budget here bounds.
 	*/
 	let stalledRounds = 0;
-	onEvent?.({
+	emit({
 		type: "start",
 		components,
 		validationSetSize: validationSet.length
@@ -152,6 +158,8 @@ async function runRandomSearch(args) {
 			reflectionCalls,
 			metricCalls: budget.spent(),
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
+			acceptedCandidates,
 			...cached === void 0 ? {} : { cache: cached }
 		};
 	}
@@ -163,21 +171,31 @@ async function runRandomSearch(args) {
 		candidate: seedCandidate,
 		phase: "seed"
 	}) : void 0;
-	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_evaluation.requireMeasuredMean({
+	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_reporting.requireMeasuredMean({
 		batch: seedEvaluation,
 		phase: "seed"
 	});
+	if (seedEvaluation !== void 0) emit({
+		type: "candidateAccepted",
+		round: 0,
+		candidateId: 0,
+		candidate: seedCandidate,
+		aggregateScore: seedScore,
+		instanceScores: require_reporting.instanceRow(seedEvaluation),
+		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
+	});
 	let best = resumeFrom?.best ?? seedCandidate;
 	let bestScore = resumeFrom?.bestScore ?? seedScore;
 	/** Absent on a resumed run until a variant wins: outputs are not checkpointed. */
 	let bestOutputs = seedEvaluation?.outputs;
+	let acceptedCandidates = resumeFrom?.acceptedCandidates ?? 0;
 	await checkpoint();
 	while (round < maxRounds) {
 		if (signal?.aborted) {
 			stopReason = "aborted";
 			break;
 		}
-		if (require_evaluation.costExhausted({
+		if (require_reporting.costExhausted({
 			usage: evaluator.usage(),
 			maxCostUsd
 		})) {
@@ -188,12 +206,12 @@ async function runRandomSearch(args) {
 			stopReason = "deadlineReached";
 			break;
 		}
-		if (!budget.canAfford(variants * validationSet.length)) {
+		if (!budget.canAfford(validationSet.length)) {
 			stopReason = "budgetExhausted";
 			break;
 		}
 		const component = components[round % components.length];
-		onEvent?.({
+		emit({
 			type: "roundStart",
 			round,
 			component
@@ -220,41 +238,64 @@ async function runRandomSearch(args) {
 		});
 		const unique = [...new Set(drawn)].filter((text) => text.length > 0 && text !== currentText);
 		let roundStop;
+		const scheduled = [];
+		let owed = 0;
 		for (const text of unique) {
 			const candidate = {
 				...best,
 				[component]: text
 			};
-			let evaluation;
-			try {
-				evaluation = await sweep({
-					candidate,
-					phase: "validation"
-				});
-			} catch (err) {
-				if (err instanceof require_evaluation.BudgetExhausted) {
-					roundStop = "budgetExhausted";
-					break;
-				}
-				if (signal?.aborted) {
-					roundStop = "aborted";
-					break;
+			const uncached = evaluator.countUncached({
+				candidate,
+				ids: validationIds,
+				split: "val"
+			});
+			if (!budget.canAfford(owed + uncached)) {
+				roundStop = "budgetExhausted";
+				break;
+			}
+			owed += uncached;
+			scheduled.push(candidate);
+		}
+		const swept = await require_concurrency.mapWithConcurrency({
+			items: scheduled,
+			limit: concurrency,
+			task: async (candidate) => {
+				if (signal?.aborted) return { stop: "aborted" };
+				try {
+					return { evaluation: await sweep({
+						candidate,
+						phase: "validation"
+					}) };
+				} catch (err) {
+					if (err instanceof require_reporting.BudgetExhausted) return { stop: "budgetExhausted" };
+					if (signal?.aborted) return { stop: "aborted" };
+					throw err;
 				}
-				throw err;
+			}
+		});
+		for (const [index, outcome] of swept.entries()) {
+			if (outcome.stop !== void 0) {
+				roundStop = outcome.stop;
+				break;
 			}
 			variantsEvaluated += 1;
-			const score = require_evaluation.measuredMean(evaluation);
+			const score = require_reporting.measuredMean(outcome.evaluation);
 			if (score !== void 0 && score > bestScore) {
-				onEvent?.({
+				const candidate = scheduled[index];
+				acceptedCandidates += 1;
+				emit({
 					type: "candidateAccepted",
 					round,
-					component,
-					score,
-					previousScore: bestScore
+					candidateId: acceptedCandidates,
+					candidate,
+					aggregateScore: score,
+					instanceScores: require_reporting.instanceRow(outcome.evaluation),
+					...trackBestOutputs ? { outputs: outcome.evaluation.outputs } : {}
 				});
 				best = candidate;
 				bestScore = score;
-				bestOutputs = evaluation.outputs;
+				bestOutputs = outcome.evaluation.outputs;
 			}
 		}
 		round += 1;
@@ -270,7 +311,7 @@ async function runRandomSearch(args) {
 		}
 	}
 	if (signal?.aborted) stopReason = "aborted";
-	const testScore = testSet === void 0 ? void 0 : require_evaluation.measuredMean(await evaluator.evaluate({
+	const heldOut = testSet === void 0 ? void 0 : await evaluator.evaluate({
 		candidate: best,
 		batch: testSet,
 		ids: testIds,
@@ -279,13 +320,17 @@ async function runRandomSearch(args) {
 		candidateId: null,
 		iteration: round,
 		charge: false
-	}));
-	onEvent?.({
+	});
+	const testScore = heldOut === void 0 ? void 0 : require_reporting.measuredMean(heldOut);
+	emit({
 		type: "finish",
 		reason: stopReason,
+		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: require_reporting.instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
 		bestCandidate: best,
@@ -296,7 +341,8 @@ async function runRandomSearch(args) {
 		...trackBestOutputs ? { bestOutputs } : {},
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: evaluator.unchargedCalls()
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		rounds: round,
 		variantsEvaluated,
@@ -312,10 +358,8 @@ function assertConfig(config) {
 	if (!Number.isInteger(concurrency) || concurrency < 1) throw new Error(`concurrency must be a positive integer, received ${concurrency}`);
 	if (maxRounds !== void 0 && (!Number.isInteger(maxRounds) || maxRounds < 1)) throw new Error(`maxRounds must be a positive integer, received ${maxRounds}`);
 }
-function defaultInstanceId(args) {
-	const hash = require_evaluation.stableHash(args.datum);
-	return hash === "" ? String(args.index) : hash;
-}
 //#endregion
 exports.RandomSearchOptimizer = RandomSearchOptimizer;
 exports.buildParaphrasePrompt = buildParaphrasePrompt;
+exports.isCandidateAccepted = require_reporting.isCandidateAccepted;
+exports.isRunFinished = require_reporting.isRunFinished;

package/dist/random-search/index.d.cts

@@ -1,6 +1,6 @@
-import { n as Candidate, t as Adapter, u as TextModel } from "../types-CWv4IQFF.cjs";
+import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.cjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.cjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-B7SpRwl7.cjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Bh5DPRMH.cjs";
 //#region src/random-search/optimize.d.ts
 /** Builds the prompt one variant is drawn from. */
 type ParaphrasePromptBuilder = (args: {
@@ -54,6 +54,13 @@ interface RandomSearchSnapshot {
   reflectionCalls: number;
   metricCalls: number;
   cacheHits: number;
+  /** Usage already spent, so a resumed run reports totals and honours ceilings. */
+  usage?: UsageTotals;
+  /**
+   * Candidates accepted so far. Reporters key rows by this id, so restarting it
+   * at zero makes a resumed run collide with the run it continues.
+   */
+  acceptedCandidates?: number;
   /** Cached instance scores, when the cache can enumerate them. */
   cache?: [string, CachedScore][];
 }
@@ -71,7 +78,8 @@ interface RandomSearchTask<Datum, Trajectory = unknown, Output = unknown, K exte
   }) => string;
   /** Pass `false` to disable caching entirely. */
   cache?: EvaluationCache | false;
-  onEvent?: (event: RandomSearchEvent<NoInfer<K>>) => void;
+  /** Observers of the run. Every one sees every event; none can fail it. */
+  reporters?: readonly Reporter<RandomSearchEvent<NoInfer<K>>>[];
   /**
    * Called with a resumable snapshot after the seed is scored and after every
    * round. Persist it and a killed run costs the last round, not all of them.
@@ -91,19 +99,13 @@ type RandomSearchEvent<K extends string = string> = {
   component: K;
 } | ({
   type: "evaluation";
-} & EvaluationEvent) | {
+} & EvaluationEvent) | ({
   type: "candidateAccepted";
   round: number;
-  component: K;
-  score: number;
-  previousScore: number;
-} | {
+} & CandidateAccepted<K>) | ({
   type: "finish";
   reason: RandomSearchStopReason;
-  bestScore: number;
-  metricCalls: number;
-  testScore?: number;
-};
+} & RunFinished);
 interface RandomSearchResult<K extends string = string, Output = unknown> extends OptimizerResult<K, RandomSearchStopReason, Output> {
   /** The seed's score, so the lift the search bought is readable directly. */
   seedScore: number;
@@ -153,4 +155,4 @@ declare function buildParaphrasePrompt(args: {
   attempt: number;
 }): string;
 //#endregion
-export { type ParaphrasePromptBuilder, type RandomSearchConfig, type RandomSearchEvent, RandomSearchOptimizer, type RandomSearchResult, type RandomSearchSnapshot, type RandomSearchStopReason, type RandomSearchTask, buildParaphrasePrompt };
+export { type CandidateAccepted, type OptimizerEvent, type ParaphrasePromptBuilder, type RandomSearchConfig, type RandomSearchEvent, RandomSearchOptimizer, type RandomSearchResult, type RandomSearchSnapshot, type RandomSearchStopReason, type RandomSearchTask, type ReportableEvent, type Reporter, type RunFinished, buildParaphrasePrompt, isCandidateAccepted, isRunFinished };

package/dist/random-search/index.d.mts

@@ -1,6 +1,6 @@
-import { n as Candidate, t as Adapter, u as TextModel } from "../types-CWv4IQFF.mjs";
+import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.mjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.mjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-DqCoth_w.mjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Ck6-e_8o.mjs";
 //#region src/random-search/optimize.d.ts
 /** Builds the prompt one variant is drawn from. */
 type ParaphrasePromptBuilder = (args: {
@@ -54,6 +54,13 @@ interface RandomSearchSnapshot {
   reflectionCalls: number;
   metricCalls: number;
   cacheHits: number;
+  /** Usage already spent, so a resumed run reports totals and honours ceilings. */
+  usage?: UsageTotals;
+  /**
+   * Candidates accepted so far. Reporters key rows by this id, so restarting it
+   * at zero makes a resumed run collide with the run it continues.
+   */
+  acceptedCandidates?: number;
   /** Cached instance scores, when the cache can enumerate them. */
   cache?: [string, CachedScore][];
 }
@@ -71,7 +78,8 @@ interface RandomSearchTask<Datum, Trajectory = unknown, Output = unknown, K exte
   }) => string;
   /** Pass `false` to disable caching entirely. */
   cache?: EvaluationCache | false;
-  onEvent?: (event: RandomSearchEvent<NoInfer<K>>) => void;
+  /** Observers of the run. Every one sees every event; none can fail it. */
+  reporters?: readonly Reporter<RandomSearchEvent<NoInfer<K>>>[];
   /**
    * Called with a resumable snapshot after the seed is scored and after every
    * round. Persist it and a killed run costs the last round, not all of them.
@@ -91,19 +99,13 @@ type RandomSearchEvent<K extends string = string> = {
   component: K;
 } | ({
   type: "evaluation";
-} & EvaluationEvent) | {
+} & EvaluationEvent) | ({
   type: "candidateAccepted";
   round: number;
-  component: K;
-  score: number;
-  previousScore: number;
-} | {
+} & CandidateAccepted<K>) | ({
   type: "finish";
   reason: RandomSearchStopReason;
-  bestScore: number;
-  metricCalls: number;
-  testScore?: number;
-};
+} & RunFinished);
 interface RandomSearchResult<K extends string = string, Output = unknown> extends OptimizerResult<K, RandomSearchStopReason, Output> {
   /** The seed's score, so the lift the search bought is readable directly. */
   seedScore: number;
@@ -153,4 +155,4 @@ declare function buildParaphrasePrompt(args: {
   attempt: number;
 }): string;
 //#endregion
-export { type ParaphrasePromptBuilder, type RandomSearchConfig, type RandomSearchEvent, RandomSearchOptimizer, type RandomSearchResult, type RandomSearchSnapshot, type RandomSearchStopReason, type RandomSearchTask, buildParaphrasePrompt };
+export { type CandidateAccepted, type OptimizerEvent, type ParaphrasePromptBuilder, type RandomSearchConfig, type RandomSearchEvent, RandomSearchOptimizer, type RandomSearchResult, type RandomSearchSnapshot, type RandomSearchStopReason, type RandomSearchTask, type ReportableEvent, type Reporter, type RunFinished, buildParaphrasePrompt, isCandidateAccepted, isRunFinished };

package/dist/random-search/index.mjs

@@ -1,4 +1,4 @@
-import { a as requireMeasuredMean, d as createMemoryCache, f as stableHash, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "../evaluation-BV0nSZVx.mjs";
+import { _ as createMemoryCache, a as isRunFinished, b as createBudget, c as createEvaluator, d as createDeadline, f as assertResumable, h as componentNames, i as isCandidateAccepted, l as measuredMean, m as runFingerprint, n as flushReporters, o as BudgetExhausted, r as instanceRow, s as costExhausted, t as createEmitter, u as requireMeasuredMean, v as defaultInstanceId } from "../reporting-DY-DC4HG.mjs";
 import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
 import { t as parseProposedText } from "../text--v4Ffbus.mjs";
 //#region src/random-search/optimize.ts
@@ -32,10 +32,14 @@ var RandomSearchOptimizer = class {
 		this.#config = config;
 	}
 	async optimize(task) {
-		return runRandomSearch({
-			config: this.#config,
-			task
-		});
+		try {
+			return await runRandomSearch({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await flushReporters(task.reporters ?? []);
+		}
 	}
 };
 /**
@@ -62,7 +66,8 @@ function buildParaphrasePrompt(args) {
 async function runRandomSearch(args) {
 	const { config, task } = args;
 	const { variants = DEFAULT_VARIANTS, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, buildPrompt = buildParaphrasePrompt, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
+	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const emit = createEmitter(reporters);
 	const deadline = createDeadline({ maxWallClockMs });
 	const components = componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
@@ -103,8 +108,9 @@ async function runRandomSearch(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
-		onEvaluation: (event) => onEvent?.({
+		onEvaluation: (event) => emit({
 			type: "evaluation",
 			...event
 		})
@@ -122,7 +128,7 @@ async function runRandomSearch(args) {
 	* no budget here bounds.
 	*/
 	let stalledRounds = 0;
-	onEvent?.({
+	emit({
 		type: "start",
 		components,
 		validationSetSize: validationSet.length
@@ -151,6 +157,8 @@ async function runRandomSearch(args) {
 			reflectionCalls,
 			metricCalls: budget.spent(),
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
+			acceptedCandidates,
 			...cached === void 0 ? {} : { cache: cached }
 		};
 	}
@@ -166,10 +174,20 @@ async function runRandomSearch(args) {
 		batch: seedEvaluation,
 		phase: "seed"
 	});
+	if (seedEvaluation !== void 0) emit({
+		type: "candidateAccepted",
+		round: 0,
+		candidateId: 0,
+		candidate: seedCandidate,
+		aggregateScore: seedScore,
+		instanceScores: instanceRow(seedEvaluation),
+		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
+	});
 	let best = resumeFrom?.best ?? seedCandidate;
 	let bestScore = resumeFrom?.bestScore ?? seedScore;
 	/** Absent on a resumed run until a variant wins: outputs are not checkpointed. */
 	let bestOutputs = seedEvaluation?.outputs;
+	let acceptedCandidates = resumeFrom?.acceptedCandidates ?? 0;
 	await checkpoint();
 	while (round < maxRounds) {
 		if (signal?.aborted) {
@@ -187,12 +205,12 @@ async function runRandomSearch(args) {
 			stopReason = "deadlineReached";
 			break;
 		}
-		if (!budget.canAfford(variants * validationSet.length)) {
+		if (!budget.canAfford(validationSet.length)) {
 			stopReason = "budgetExhausted";
 			break;
 		}
 		const component = components[round % components.length];
-		onEvent?.({
+		emit({
 			type: "roundStart",
 			round,
 			component
@@ -219,41 +237,64 @@ async function runRandomSearch(args) {
 		});
 		const unique = [...new Set(drawn)].filter((text) => text.length > 0 && text !== currentText);
 		let roundStop;
+		const scheduled = [];
+		let owed = 0;
 		for (const text of unique) {
 			const candidate = {
 				...best,
 				[component]: text
 			};
-			let evaluation;
-			try {
-				evaluation = await sweep({
-					candidate,
-					phase: "validation"
-				});
-			} catch (err) {
-				if (err instanceof BudgetExhausted) {
-					roundStop = "budgetExhausted";
-					break;
-				}
-				if (signal?.aborted) {
-					roundStop = "aborted";
-					break;
+			const uncached = evaluator.countUncached({
+				candidate,
+				ids: validationIds,
+				split: "val"
+			});
+			if (!budget.canAfford(owed + uncached)) {
+				roundStop = "budgetExhausted";
+				break;
+			}
+			owed += uncached;
+			scheduled.push(candidate);
+		}
+		const swept = await mapWithConcurrency({
+			items: scheduled,
+			limit: concurrency,
+			task: async (candidate) => {
+				if (signal?.aborted) return { stop: "aborted" };
+				try {
+					return { evaluation: await sweep({
+						candidate,
+						phase: "validation"
+					}) };
+				} catch (err) {
+					if (err instanceof BudgetExhausted) return { stop: "budgetExhausted" };
+					if (signal?.aborted) return { stop: "aborted" };
+					throw err;
 				}
-				throw err;
+			}
+		});
+		for (const [index, outcome] of swept.entries()) {
+			if (outcome.stop !== void 0) {
+				roundStop = outcome.stop;
+				break;
 			}
 			variantsEvaluated += 1;
-			const score = measuredMean(evaluation);
+			const score = measuredMean(outcome.evaluation);
 			if (score !== void 0 && score > bestScore) {
-				onEvent?.({
+				const candidate = scheduled[index];
+				acceptedCandidates += 1;
+				emit({
 					type: "candidateAccepted",
 					round,
-					component,
-					score,
-					previousScore: bestScore
+					candidateId: acceptedCandidates,
+					candidate,
+					aggregateScore: score,
+					instanceScores: instanceRow(outcome.evaluation),
+					...trackBestOutputs ? { outputs: outcome.evaluation.outputs } : {}
 				});
 				best = candidate;
 				bestScore = score;
-				bestOutputs = evaluation.outputs;
+				bestOutputs = outcome.evaluation.outputs;
 			}
 		}
 		round += 1;
@@ -269,7 +310,7 @@ async function runRandomSearch(args) {
 		}
 	}
 	if (signal?.aborted) stopReason = "aborted";
-	const testScore = testSet === void 0 ? void 0 : measuredMean(await evaluator.evaluate({
+	const heldOut = testSet === void 0 ? void 0 : await evaluator.evaluate({
 		candidate: best,
 		batch: testSet,
 		ids: testIds,
@@ -278,13 +319,17 @@ async function runRandomSearch(args) {
 		candidateId: null,
 		iteration: round,
 		charge: false
-	}));
-	onEvent?.({
+	});
+	const testScore = heldOut === void 0 ? void 0 : measuredMean(heldOut);
+	emit({
 		type: "finish",
 		reason: stopReason,
+		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
 		bestCandidate: best,
@@ -295,7 +340,8 @@ async function runRandomSearch(args) {
 		...trackBestOutputs ? { bestOutputs } : {},
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: evaluator.unchargedCalls()
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		rounds: round,
 		variantsEvaluated,
@@ -311,9 +357,5 @@ function assertConfig(config) {
 	if (!Number.isInteger(concurrency) || concurrency < 1) throw new Error(`concurrency must be a positive integer, received ${concurrency}`);
 	if (maxRounds !== void 0 && (!Number.isInteger(maxRounds) || maxRounds < 1)) throw new Error(`maxRounds must be a positive integer, received ${maxRounds}`);
 }
-function defaultInstanceId(args) {
-	const hash = stableHash(args.datum);
-	return hash === "" ? String(args.index) : hash;
-}
 //#endregion
-export { RandomSearchOptimizer, buildParaphrasePrompt };
+export { RandomSearchOptimizer, buildParaphrasePrompt, isCandidateAccepted, isRunFinished };