textopt 0.0.0 → 0.1.0

package/dist/opro/index.d.mts

@@ -1,6 +1,6 @@
-import { n as Candidate, t as Adapter, u as TextModel } from "../types-CWv4IQFF.mjs";
+import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.mjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.mjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-DqCoth_w.mjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Ck6-e_8o.mjs";
 //#region src/opro/optimize.d.ts
 /** One instruction that was tried, and what it scored. */
 /** A history entry plus the system state its score was measured in. */
@@ -111,6 +111,13 @@ interface OproSnapshot {
   reflectionCalls: number;
   metricCalls: number;
   cacheHits: number;
+  /** Usage already spent, so a resumed run reports totals and honours ceilings. */
+  usage?: UsageTotals;
+  /**
+   * Candidates accepted so far. Reporters key rows by this id, so restarting it
+   * at zero makes a resumed run collide with the run it continues.
+   */
+  acceptedCandidates?: number;
   rngState: number;
   /** Component name -> every text tried for it, with what it scored. */
   histories: Record<string, RecordedAttempt[]>;
@@ -133,7 +140,8 @@ interface OproTask<Datum, Trajectory = unknown, Output = unknown, K extends stri
   }) => string;
   /** Pass `false` to disable caching entirely. */
   cache?: EvaluationCache | false;
-  onEvent?: (event: OproEvent<NoInfer<K>>) => void;
+  /** Observers of the run. Every one sees every event; none can fail it. */
+  reporters?: readonly Reporter<OproEvent<NoInfer<K>>>[];
   /**
    * Called with a resumable snapshot after the seed is scored and after every
    * round. Persist it and a killed run costs the last round, not all of them.
@@ -161,13 +169,13 @@ type OproEvent<K extends string = string> = {
   score: number;
   /** True when this attempt became the new incumbent. */
   accepted: boolean;
-} | {
+} | ({
+  type: "candidateAccepted";
+  round: number;
+} & CandidateAccepted<K>) | ({
   type: "finish";
   reason: OproStopReason;
-  bestScore: number;
-  metricCalls: number;
-  testScore?: number;
-};
+} & RunFinished);
 interface OproAttempt<K extends string = string> {
   round: number;
   component: K;
@@ -227,4 +235,4 @@ declare function buildOproPrompt(args: {
   exemplars: readonly string[];
 }): string;
 //#endregion
-export { type OproAttempt, type OproConfig, type OproEvent, OproOptimizer, type OproPromptBuilder, type OproResult, type OproSnapshot, type OproStopReason, type OproTask, type ScoredAttempt, buildOproPrompt };
+export { type CandidateAccepted, type OproAttempt, type OproConfig, type OproEvent, OproOptimizer, type OproPromptBuilder, type OproResult, type OproSnapshot, type OproStopReason, type OproTask, type OptimizerEvent, type ReportableEvent, type Reporter, type RunFinished, type ScoredAttempt, buildOproPrompt, isCandidateAccepted, isRunFinished };

package/dist/opro/index.mjs

@@ -1,4 +1,4 @@
-import { a as requireMeasuredMean, d as createMemoryCache, f as stableHash, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "../evaluation-BV0nSZVx.mjs";
+import { _ as createMemoryCache, a as isRunFinished, b as createBudget, c as createEvaluator, d as createDeadline, f as assertResumable, h as componentNames, i as isCandidateAccepted, l as measuredMean, m as runFingerprint, n as flushReporters, o as BudgetExhausted, r as instanceRow, s as costExhausted, t as createEmitter, u as requireMeasuredMean, v as defaultInstanceId, y as stableHash } from "../reporting-DY-DC4HG.mjs";
 import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
 import { t as parseProposedText } from "../text--v4Ffbus.mjs";
 import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
@@ -42,10 +42,14 @@ var OproOptimizer = class {
 		this.#config = config;
 	}
 	async optimize(task) {
-		return runOpro({
-			config: this.#config,
-			task
-		});
+		try {
+			return await runOpro({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await flushReporters(task.reporters ?? []);
+		}
 	}
 };
 /**
@@ -90,7 +94,8 @@ function buildOproPrompt(args) {
 async function runOpro(args) {
 	const { config, task } = args;
 	const { proposalsPerRound = DEFAULT_PROPOSALS_PER_ROUND, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, maxReflectionCalls = Number.POSITIVE_INFINITY, seed = 0, historySize = DEFAULT_HISTORY_SIZE, exemplars = DEFAULT_EXEMPLARS, scoringSetSize, fullEvalInterval = DEFAULT_FULL_EVAL_INTERVAL, scoreScale = DEFAULT_SCORE_SCALE, buildPrompt = buildOproPrompt, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, renderDatum = renderDefault, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
+	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, renderDatum = renderDefault, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const emit = createEmitter(reporters);
 	const deadline = createDeadline({ maxWallClockMs });
 	const components = componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
@@ -133,8 +138,9 @@ async function runOpro(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
-		onEvaluation: (event) => onEvent?.({
+		onEvaluation: (event) => emit({
 			type: "evaluation",
 			...event
 		})
@@ -159,7 +165,7 @@ async function runOpro(args) {
 	let round = resumeFrom?.round ?? 0;
 	let reflectionCalls = resumeFrom?.reflectionCalls ?? 0;
 	let stopReason = "maxRounds";
-	onEvent?.({
+	emit({
 		type: "start",
 		components,
 		validationSetSize: validationSet.length
@@ -192,14 +198,36 @@ async function runOpro(args) {
 			iteration: round
 		});
 	}
+	/** What screening a candidate would cost, in rollouts nothing has cached. */
+	function screenCost(candidate) {
+		return scoringSet === void 0 ? evaluator.countUncached({
+			candidate,
+			ids: validationIds,
+			split: "val"
+		}) : evaluator.countUncached({
+			candidate,
+			ids: scoringIds,
+			split: "train"
+		});
+	}
 	const seedEvaluation = resumeFrom === void 0 ? await sweep(seedCandidate, "seed") : void 0;
 	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : requireMeasuredMean({
 		batch: seedEvaluation,
 		phase: "seed"
 	});
+	if (seedEvaluation !== void 0) emit({
+		type: "candidateAccepted",
+		round: 0,
+		candidateId: 0,
+		candidate: seedCandidate,
+		aggregateScore: seedScore,
+		instanceScores: instanceRow(seedEvaluation),
+		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
+	});
 	let best = resumeFrom?.best ?? seedCandidate;
 	let bestScore = resumeFrom?.bestScore ?? seedScore;
 	/** Absent on a resumed run until a sweep wins: outputs are not checkpointed. */
+	let acceptedCandidates = resumeFrom?.acceptedCandidates ?? 0;
 	let bestOutputs = seedEvaluation?.outputs;
 	let reported = resumeFrom?.reported ?? seedCandidate;
 	let lastSwept = resumeFrom === void 0 ? seedCandidate : resumeFrom.incumbentSwept ? best : {};
@@ -214,6 +242,24 @@ async function runOpro(args) {
 	* what gets reported is the best candidate a full sweep has actually seen —
 	* never a subset number wearing a validation set label.
 	*/
+	/**
+	* The incumbent moved and a full sweep measured it. Emitted from the two
+	* places that can be true — a screening run with no scoring set, where the
+	* attempt's own evaluation is the sweep, and the cadence that confirms an
+	* incumbent later — because a payload assembled twice is one that drifts.
+	*/
+	function emitAccepted(args) {
+		acceptedCandidates += 1;
+		emit({
+			type: "candidateAccepted",
+			round,
+			candidateId: acceptedCandidates,
+			candidate: args.candidate,
+			aggregateScore: args.score,
+			instanceScores: instanceRow(args.evaluation),
+			...trackBestOutputs ? { outputs: args.evaluation.outputs } : {}
+		});
+	}
 	async function refreshIncumbent() {
 		if (best === lastSwept || !budget.canAfford(validationSet.length)) return "ok";
 		try {
@@ -224,6 +270,11 @@ async function runOpro(args) {
 				reported = best;
 				bestScore = full;
 				bestOutputs = evaluation.outputs;
+				emitAccepted({
+					candidate: best,
+					evaluation,
+					score: full
+				});
 			}
 		} catch (err) {
 			if (err instanceof BudgetExhausted || signal?.aborted) return "stop";
@@ -259,6 +310,8 @@ async function runOpro(args) {
 			reflectionCalls,
 			metricCalls: budget.spent(),
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
+			acceptedCandidates,
 			rngState: rng.state(),
 			histories: Object.fromEntries([...histories].map(([name, attempts]) => [name, [...attempts]])),
 			...scoringIndices === void 0 ? {} : { scoringIndices },
@@ -303,7 +356,7 @@ async function runOpro(args) {
 		const history = histories.get(component);
 		const context = contextOf(best, component);
 		const comparable = history.filter((attempt) => attempt.context === context);
-		onEvent?.({
+		emit({
 			type: "roundStart",
 			round,
 			component,
@@ -332,25 +385,43 @@ async function runOpro(args) {
 		const unique = [...new Set(drawn)].filter((text) => text.length > 0 && !tried.has(text));
 		barrenRounds = unique.length === 0 ? barrenRounds + 1 : 0;
 		let roundStop;
+		const scheduled = [];
+		let owed = 0;
 		for (const text of unique) {
 			const candidate = {
 				...best,
 				[component]: text
 			};
-			let evaluation;
-			try {
-				evaluation = await screen(candidate, "validation");
-			} catch (err) {
-				if (err instanceof BudgetExhausted) {
-					roundStop = "budgetExhausted";
-					break;
-				}
-				if (signal?.aborted) {
-					roundStop = "aborted";
-					break;
+			const cost = screenCost(candidate);
+			if (!budget.canAfford(owed + cost)) {
+				roundStop = "budgetExhausted";
+				break;
+			}
+			owed += cost;
+			scheduled.push(candidate);
+		}
+		const screened = await mapWithConcurrency({
+			items: scheduled,
+			limit: concurrency,
+			task: async (candidate) => {
+				if (signal?.aborted) return { stop: "aborted" };
+				try {
+					return { evaluation: await screen(candidate, "validation") };
+				} catch (err) {
+					if (err instanceof BudgetExhausted) return { stop: "budgetExhausted" };
+					if (signal?.aborted) return { stop: "aborted" };
+					throw err;
 				}
-				throw err;
 			}
+		});
+		for (const [index, outcome] of screened.entries()) {
+			if (outcome.stop !== void 0) {
+				roundStop = outcome.stop;
+				break;
+			}
+			const candidate = scheduled[index];
+			const text = candidate[component];
+			const evaluation = outcome.evaluation;
 			const score = measuredMean(evaluation);
 			if (score === void 0) continue;
 			const accepted = score > bestSearchScore;
@@ -365,7 +436,7 @@ async function runOpro(args) {
 				candidate,
 				score
 			});
-			onEvent?.({
+			emit({
 				type: "attempt",
 				round,
 				component,
@@ -380,6 +451,11 @@ async function runOpro(args) {
 					lastSwept = candidate;
 					bestScore = score;
 					bestOutputs = evaluation.outputs;
+					emitAccepted({
+						candidate,
+						evaluation,
+						score
+					});
 				}
 				const scaled = scaleScore(score, scoreScale);
 				for (const name of components) {
@@ -393,22 +469,22 @@ async function runOpro(args) {
 			}
 		}
 		round += 1;
+		let cadenceStop;
+		if (roundStop === void 0 && scoringSet !== void 0 && round % fullEvalInterval === 0 && await refreshIncumbent() === "stop") cadenceStop = signal?.aborted ? "aborted" : "budgetExhausted";
 		await checkpoint();
 		if (roundStop !== void 0) {
 			stopReason = roundStop;
 			break;
 		}
-		if (scoringSet !== void 0 && round % fullEvalInterval === 0) {
-			if (await refreshIncumbent() === "stop") {
-				stopReason = signal?.aborted ? "aborted" : "budgetExhausted";
-				break;
-			}
+		if (cadenceStop !== void 0) {
+			stopReason = cadenceStop;
+			break;
 		}
 	}
 	if (scoringSet !== void 0 && !signal?.aborted) await refreshIncumbent();
 	if (signal?.aborted) stopReason = "aborted";
-	const testScore = testSet === void 0 ? void 0 : measuredMean(await evaluator.evaluate({
-		candidate: best,
+	const heldOut = testSet === void 0 ? void 0 : await evaluator.evaluate({
+		candidate: reported,
 		batch: testSet,
 		ids: testIds,
 		split: "test",
@@ -416,13 +492,17 @@ async function runOpro(args) {
 		candidateId: null,
 		iteration: round,
 		charge: false
-	}));
-	onEvent?.({
+	});
+	const testScore = heldOut === void 0 ? void 0 : measuredMean(heldOut);
+	emit({
 		type: "finish",
 		reason: stopReason,
+		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
 		snapshot: takeSnapshot(),
@@ -433,7 +513,8 @@ async function runOpro(args) {
 		...trackBestOutputs ? { bestOutputs } : {},
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: evaluator.unchargedCalls()
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		rounds: round,
 		trajectory,
@@ -477,9 +558,5 @@ function assertConfig(config) {
 	for (const [name, value] of positive) if (value !== void 0 && (!Number.isInteger(value) || value < 1)) throw new Error(`${name} must be a positive integer, received ${value}`);
 	if (config.exemplars !== void 0 && (!Number.isInteger(config.exemplars) || config.exemplars < 0)) throw new Error(`exemplars must be a non-negative integer, received ${config.exemplars}`);
 }
-function defaultInstanceId(args) {
-	const hash = stableHash(args.datum);
-	return hash === "" ? String(args.index) : hash;
-}
 //#endregion
-export { OproOptimizer, buildOproPrompt };
+export { OproOptimizer, buildOproPrompt, isCandidateAccepted, isRunFinished };

package/dist/{optimizer-B7SpRwl7.d.cts → optimizer-Bh5DPRMH.d.cts}

@@ -1,4 +1,4 @@
-import { d as UsageTotals, i as EvaluationBatch, n as Candidate, o as EvaluationPhase, s as EvaluationSplit, t as Adapter } from "./types-CWv4IQFF.cjs";
+import { c as Adapter, d as EvaluationBatch, l as Candidate, m as EvaluationSplit, p as EvaluationPhase, v as UsageTotals } from "./reporting-DQbAohc9.cjs";
 import { n as EvaluationCache, t as CachedScore } from "./cache-CuSo0NJ8.cjs";
 //#region src/budget.d.ts
 interface Budget {
@@ -106,8 +106,24 @@ interface Evaluator<Datum, Trajectory, Output, K extends string> {
   cacheHits(): number;
   /** Rollouts made with `charge: false`, tracked apart from the budget. */
   unchargedCalls(): number;
-  /** Tokens and money the run has spent, as far as adapters have reported it. */
+  /**
+   * Tokens and money the search has spent, as far as adapters have reported
+   * it. `maxCostUsd` is checked against this, so what the ceiling does not
+   * bound is not counted in it: rollouts made with `charge: false` are in
+   * `unchargedUsage` instead.
+   */
   usage(): UsageTotals;
+  /**
+   * What the rollouts made with `charge: false` cost. Held-out measurement is
+   * taken after the search has stopped, so no ceiling bounds it and reporting
+   * it inside `usage` would describe a run as having overrun one.
+   */
+  unchargedUsage(): UsageTotals;
+  /**
+   * Folds in usage spent outside this evaluator — harvesting runs its own, and
+   * a cost ceiling that cannot see it bounds only part of the run.
+   */
+  absorbUsage(spent: UsageTotals): void;
   /** Cache contents for checkpointing, when the cache can enumerate them. */
   entries(): [string, CachedScore][] | undefined;
   restore(entries: Iterable<readonly [string, CachedScore]>): void;
@@ -140,6 +156,12 @@ declare function createEvaluator<Datum, Trajectory, Output, K extends string>(ar
   signal?: AbortSignal;
   /** Resumed counters, so a continued run reports totals rather than deltas. */
   cacheHits?: number;
+  /**
+   * Usage the interrupted run had already spent. Without it `maxCostUsd` is a
+   * ceiling on the segment rather than on the run, and resuming repeatedly
+   * spends it again each time.
+   */
+  usage?: UsageTotals;
   /**
    * Rate limits and 5xx responses are the common case in a long run, and a
    * transient row costs the instance whichever optimizer is driving: it is
@@ -252,8 +274,25 @@ interface OptimizerResult<K extends string, Stop extends string, Output = unknow
   bestOutputs?: (Output | undefined)[];
   metricCalls: number;
   /**
-   * Tokens and dollars the run spent, summed from what the adapter reported.
-   * Zero throughout when the adapter reports no usage.
+   * Rollouts served from the cache rather than charged to `metricCalls`. Every
+   * optimizer here caches by default, so the same `maxMetricCalls` can buy a
+   * search that revisits scored candidates a longer effective run than one
+   * that never does — a comparison over `metricCalls` alone hides that. Zero
+   * for a run with caching disabled.
+   */
+  cacheHits: number;
+  /**
+   * Calls made to a proposal or reflection model, which no metric budget
+   * covers — see each optimizer's own accounting for what a call costs there.
+   * Absent from a search that proposes nothing of its own: bootstrap search
+   * only accepts or rejects rollouts the metric already scored.
+   */
+  reflectionCalls?: number;
+  /**
+   * Tokens and dollars the search spent, summed from what the adapter reported.
+   * Zero throughout when the adapter reports no usage. `maxCostUsd` is checked
+   * against this, so the held-out sweep — which runs after the search has
+   * stopped, under no ceiling — is reported apart from it, in `testUsage`.
    */
   usage: UsageTotals;
   /**
@@ -267,6 +306,13 @@ interface OptimizerResult<K extends string, Stop extends string, Output = unknow
    * measurement rather than search, and so is not charged to `maxMetricCalls`.
    */
   testMetricCalls?: number;
+  /**
+   * Tokens and dollars the held-out sweep cost, for the same reason: no ceiling
+   * bounds it, so a caller adding up what a run spent has to see it as its own
+   * number rather than find it folded into one `maxCostUsd` was supposed to
+   * hold. Absent when no testSet was given.
+   */
+  testUsage?: UsageTotals;
   stopReason: Stop;
 }
 /**

package/dist/{optimizer-DqCoth_w.d.mts → optimizer-Ck6-e_8o.d.mts}

@@ -1,4 +1,4 @@
-import { d as UsageTotals, i as EvaluationBatch, n as Candidate, o as EvaluationPhase, s as EvaluationSplit, t as Adapter } from "./types-CWv4IQFF.mjs";
+import { c as Adapter, d as EvaluationBatch, l as Candidate, m as EvaluationSplit, p as EvaluationPhase, v as UsageTotals } from "./reporting-DQbAohc9.mjs";
 import { n as EvaluationCache, t as CachedScore } from "./cache-CuSo0NJ8.mjs";
 //#region src/budget.d.ts
 interface Budget {
@@ -106,8 +106,24 @@ interface Evaluator<Datum, Trajectory, Output, K extends string> {
   cacheHits(): number;
   /** Rollouts made with `charge: false`, tracked apart from the budget. */
   unchargedCalls(): number;
-  /** Tokens and money the run has spent, as far as adapters have reported it. */
+  /**
+   * Tokens and money the search has spent, as far as adapters have reported
+   * it. `maxCostUsd` is checked against this, so what the ceiling does not
+   * bound is not counted in it: rollouts made with `charge: false` are in
+   * `unchargedUsage` instead.
+   */
   usage(): UsageTotals;
+  /**
+   * What the rollouts made with `charge: false` cost. Held-out measurement is
+   * taken after the search has stopped, so no ceiling bounds it and reporting
+   * it inside `usage` would describe a run as having overrun one.
+   */
+  unchargedUsage(): UsageTotals;
+  /**
+   * Folds in usage spent outside this evaluator — harvesting runs its own, and
+   * a cost ceiling that cannot see it bounds only part of the run.
+   */
+  absorbUsage(spent: UsageTotals): void;
   /** Cache contents for checkpointing, when the cache can enumerate them. */
   entries(): [string, CachedScore][] | undefined;
   restore(entries: Iterable<readonly [string, CachedScore]>): void;
@@ -140,6 +156,12 @@ declare function createEvaluator<Datum, Trajectory, Output, K extends string>(ar
   signal?: AbortSignal;
   /** Resumed counters, so a continued run reports totals rather than deltas. */
   cacheHits?: number;
+  /**
+   * Usage the interrupted run had already spent. Without it `maxCostUsd` is a
+   * ceiling on the segment rather than on the run, and resuming repeatedly
+   * spends it again each time.
+   */
+  usage?: UsageTotals;
   /**
    * Rate limits and 5xx responses are the common case in a long run, and a
    * transient row costs the instance whichever optimizer is driving: it is
@@ -252,8 +274,25 @@ interface OptimizerResult<K extends string, Stop extends string, Output = unknow
   bestOutputs?: (Output | undefined)[];
   metricCalls: number;
   /**
-   * Tokens and dollars the run spent, summed from what the adapter reported.
-   * Zero throughout when the adapter reports no usage.
+   * Rollouts served from the cache rather than charged to `metricCalls`. Every
+   * optimizer here caches by default, so the same `maxMetricCalls` can buy a
+   * search that revisits scored candidates a longer effective run than one
+   * that never does — a comparison over `metricCalls` alone hides that. Zero
+   * for a run with caching disabled.
+   */
+  cacheHits: number;
+  /**
+   * Calls made to a proposal or reflection model, which no metric budget
+   * covers — see each optimizer's own accounting for what a call costs there.
+   * Absent from a search that proposes nothing of its own: bootstrap search
+   * only accepts or rejects rollouts the metric already scored.
+   */
+  reflectionCalls?: number;
+  /**
+   * Tokens and dollars the search spent, summed from what the adapter reported.
+   * Zero throughout when the adapter reports no usage. `maxCostUsd` is checked
+   * against this, so the held-out sweep — which runs after the search has
+   * stopped, under no ceiling — is reported apart from it, in `testUsage`.
    */
   usage: UsageTotals;
   /**
@@ -267,6 +306,13 @@ interface OptimizerResult<K extends string, Stop extends string, Output = unknow
    * measurement rather than search, and so is not charged to `maxMetricCalls`.
    */
   testMetricCalls?: number;
+  /**
+   * Tokens and dollars the held-out sweep cost, for the same reason: no ceiling
+   * bounds it, so a caller adding up what a run spent has to see it as its own
+   * number rather than find it folded into one `maxCostUsd` was supposed to
+   * hold. Absent when no testSet was given.
+   */
+  testUsage?: UsageTotals;
   stopReason: Stop;
 }
 /**