textopt 0.0.0 → 0.2.0

package/dist/simba/index.cjs

@@ -1,6 +1,7 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_evaluation = require("../evaluation-OZOp6TB7.cjs");
-const require_demos = require("../demos-B9BJiNKz.cjs");
+const require_warnings = require("../warnings-CWRJF-jA.cjs");
+const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
+const require_demos = require("../demos-9v5ts7F3.cjs");
 const require_rng = require("../rng-DbA_rPIo.cjs");
 const require_sampling = require("../sampling-Dars7ctR.cjs");
 //#region src/simba/advice.ts
@@ -15,7 +16,7 @@ const ADVICE_BLOCK = /<advice\s+component="([^"]+)"\s*>([\s\S]*?)<\/advice>/g;
 * behaviour rather than to the instance being easier.
 */
 function buildAdvicePrompt(args) {
-	const { components, input, better, worse } = args;
+	const { components, current, input, better, worse } = args;
 	return [
 		"Two runs of the same system on the same input are shown below, with the reward each earned.",
 		"Your job is to write advice that would make the system behave like the higher scoring run the next time it sees a similar input.",
@@ -26,12 +27,18 @@ function buildAdvicePrompt(args) {
 		...worse === void 0 ? [] : trajectoryBlock("worse", worse),
 		...better === void 0 ? [] : trajectoryBlock("better", better),
 		"",
-		"Write advice for each of these components:",
+		"Write advice for each of these components, shown with what it says now:",
 		"",
 		"<components>",
-		components.join("\n"),
+		components.map((component) => [
+			`<component name="${component}">`,
+			current[component] ?? "",
+			"</component>"
+		].join("\n")).join("\n"),
 		"</components>",
 		"",
+		"Your advice is appended to what the component already says. Do not restate advice it already carries; add what is missing or correct what is wrong.",
+		"",
 		"The component will not have access to this example, so advice that only covers this input is wasted. State the general behaviour it should adopt, and be concrete about when it applies.",
 		"Address each component's own sub-task rather than the system as a whole.",
 		"Base the advice on what actually differed between the two runs. If nothing useful can be said for a component, leave it out.",
@@ -229,21 +236,32 @@ const DEMO_DROP_SCALE = 3;
 var SimbaOptimizer = class {
 	#config;
 	constructor(config = {}) {
+		assertSimbaConfig(config);
 		this.#config = config;
 	}
-	optimize(task) {
-		return run({
-			config: this.#config,
-			task
-		});
+	async optimize(task) {
+		try {
+			return await run({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await require_warnings.flushReporters(task.reporters ?? []);
+		}
 	}
 };
 async function run(args) {
 	const { config, task } = args;
-	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, candidates: candidateCount = DEFAULT_CANDIDATES, maxSteps = DEFAULT_MAX_STEPS, maxDemos = DEFAULT_MAX_DEMOS, samplingTemperature = DEFAULT_TEMPERATURE, candidateTemperature = DEFAULT_TEMPERATURE, strategies, maxReflectionCalls, seed = 0, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, demoComponents = [], instructionComponents, renderDemo, buildAdvicePrompt: buildPrompt = buildAdvicePrompt, sampler = require_sampling.createEpochShuffledSampler({ minibatchSize }), maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
-	const components = require_evaluation.componentNames(seedCandidate);
-	const ruleComponents = instructionComponents ?? components.filter((name) => !demoComponents.includes(name));
+	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, candidates: candidateCount = DEFAULT_CANDIDATES, concurrency = 1, maxSteps = DEFAULT_MAX_STEPS, maxDemos = DEFAULT_MAX_DEMOS, samplingTemperature = DEFAULT_TEMPERATURE, candidateTemperature = DEFAULT_TEMPERATURE, strategies, maxReflectionCalls, seed = 0, trackBestOutputs = false, checkpointCache = true } = config;
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, demoComponents = [], instructionComponents, renderDemo, buildAdvicePrompt: buildPrompt = buildAdvicePrompt, sampler = require_sampling.createEpochShuffledSampler({ minibatchSize }), maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_warnings.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = require_warnings.resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
+	const emit = require_warnings.createEmitter(reporters);
+	const components = require_warnings.componentNames(seedCandidate);
+	const nonDemo = components.filter((name) => !demoComponents.includes(name));
+	const ruleComponents = instructionComponents ?? (nonDemo.length > 0 ? nonDemo : components);
 	const enabled = strategies ?? (demoComponents.length > 0 ? ["appendDemo", "appendRule"] : ["appendRule"]);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
 	if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
@@ -252,6 +270,10 @@ async function run(args) {
 	if (enabled.includes("appendDemo") && demoComponents.length === 0) throw new Error("the appendDemo strategy requires at least one entry in demoComponents");
 	if (enabled.includes("appendRule") && ruleComponents.length === 0) throw new Error("the appendRule strategy requires at least one instruction component to write into");
 	if (testSet !== void 0 && testSet.length === 0) throw new Error("optimize requires a non-empty testSet when one is given; omit it to skip held-out evaluation");
+	const trainingIds = trainingSet.map((datum, index) => instanceId({
+		datum,
+		index
+	}));
 	const validationIds = validationSet.map((datum, index) => instanceId({
 		datum,
 		index
@@ -260,28 +282,25 @@ async function run(args) {
 		datum,
 		index
 	})) ?? [];
-	const fingerprint = require_evaluation.runFingerprint({
+	const fingerprint = require_warnings.runFingerprint({
 		seedCandidate,
-		trainingIds: trainingSet.map((datum, index) => instanceId({
-			datum,
-			index
-		})),
+		trainingIds,
 		validationIds,
 		seed,
 		...cacheNamespace === void 0 ? {} : { cacheNamespace }
 	});
-	require_evaluation.assertResumable({
+	require_warnings.assertResumable({
 		fingerprint,
 		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
 	});
-	const deadline = require_evaluation.createDeadline({ maxWallClockMs });
+	const deadline = require_warnings.createDeadline({ maxWallClockMs });
 	const rng = require_rng.createSeededRng(seed, resumeFrom?.rngState);
-	const budget = require_evaluation.createBudget({
+	const budget = require_warnings.createBudget({
 		maxMetricCalls,
 		spent: resumeFrom?.metricCalls ?? 0
 	});
-	const evaluationCache = cache === false ? void 0 : cache ?? require_evaluation.createMemoryCache();
-	const evaluator = require_evaluation.createEvaluator({
+	const evaluationCache = cache === false ? void 0 : cache ?? require_warnings.createMemoryCache();
+	const evaluator = require_warnings.createEvaluator({
 		adapter,
 		budget,
 		...retry === void 0 ? {} : { retry },
@@ -289,8 +308,9 @@ async function run(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: true,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
-		onEvaluation: (event) => onEvent?.({
+		onEvaluation: (event) => emit({
 			type: "evaluation",
 			...event
 		})
@@ -306,7 +326,7 @@ async function run(args) {
 	let step = resumeFrom?.step ?? 0;
 	let reflectionCalls = resumeFrom?.reflectionCalls ?? 0;
 	let stopReason = "maxSteps";
-	onEvent?.({
+	emit({
 		type: "start",
 		components,
 		validationSetSize: validationSet.length
@@ -318,7 +338,7 @@ async function run(args) {
 			stopReason = "aborted";
 			break;
 		}
-		if (require_evaluation.costExhausted({
+		if (require_warnings.costExhausted({
 			usage: evaluator.usage(),
 			maxCostUsd
 		})) {
@@ -341,16 +361,17 @@ async function run(args) {
 			scores: programs.map((_, index) => averageScore(index)),
 			k: candidateCount
 		});
-		onEvent?.({
+		emit({
 			type: "stepStart",
 			step,
 			poolSize: pool.length
 		});
-		const batch = sampler({
+		const batchIndices = sampler({
 			trainingSet,
 			iteration: step,
 			rng
-		}).map((index) => trainingSet[index]);
+		});
+		const batch = batchIndices.map((index) => trainingSet[index]);
 		const samples = [];
 		for (let slot = 0; slot < candidateCount; slot += 1) {
 			const programIndex = softmaxSample({
@@ -372,7 +393,7 @@ async function run(args) {
 				outputs: evaluation.outputs,
 				...evaluation.feedback === void 0 ? {} : { feedback: evaluation.feedback }
 			});
-			programScores[programIndex].push(require_evaluation.measuredMean(evaluation) ?? 0);
+			programScores[programIndex].push(require_warnings.measuredMean(evaluation) ?? 0);
 		}
 		if (samples.length === 0) {
 			stopReason = "budgetExhausted";
@@ -412,7 +433,7 @@ async function run(args) {
 					high
 				});
 			} catch (err) {
-				onEvent?.({
+				emit({
 					type: "error",
 					step,
 					err
@@ -427,36 +448,54 @@ async function run(args) {
 			});
 		}
 		let stepBest;
+		const batchIds = batchIndices.map((index) => trainingIds[index]);
+		const scheduled = [];
+		let owed = 0;
 		for (const entry of built) {
-			if (!budget.canAfford(batch.length)) {
+			if (!budget.canAfford(owed + batch.length)) {
 				stopReason = "budgetExhausted";
 				break;
 			}
-			let score;
-			try {
-				score = require_evaluation.measuredMean(await evaluator.evaluate({
-					candidate: entry.candidate,
-					batch,
-					ids: batch.map((datum, index) => instanceId({
-						datum,
-						index
-					})),
-					split: "train",
-					phase: "minibatch",
-					candidateId: programs.length,
-					iteration: step
-				}));
-			} catch (err) {
-				if (err instanceof require_evaluation.BudgetExhausted) {
-					stopReason = "budgetExhausted";
-					break;
+			owed += evaluator.countUncached({
+				candidate: entry.candidate,
+				ids: batchIds,
+				split: "train"
+			});
+			scheduled.push(entry);
+		}
+		const poolBase = programs.length;
+		const scored = await mapDistinct({
+			items: scheduled,
+			limit: concurrency,
+			key: (entry) => require_warnings.candidateHash(entry.candidate),
+			task: async (entry, index) => {
+				try {
+					return { score: require_warnings.measuredMean(await evaluator.evaluate({
+						candidate: entry.candidate,
+						batch,
+						ids: batchIds,
+						split: "train",
+						phase: "minibatch",
+						candidateId: poolBase + index,
+						iteration: step
+					})) };
+				} catch (err) {
+					if (err instanceof require_warnings.BudgetExhausted) return { stop: true };
+					throw err;
 				}
-				throw err;
 			}
+		});
+		for (const [index, outcome] of scored.entries()) {
+			if (outcome.stop === true) {
+				stopReason = "budgetExhausted";
+				break;
+			}
+			const { score } = outcome;
 			if (score === void 0) continue;
+			const entry = scheduled[index];
 			programs.push(entry.candidate);
 			programScores.push([score]);
-			onEvent?.({
+			emit({
 				type: "candidate",
 				step,
 				strategy: entry.strategy,
@@ -479,38 +518,78 @@ async function run(args) {
 		}
 	}
 	if (signal?.aborted) stopReason = "aborted";
-	const finalists = [];
+	const contenders = [];
+	let owedForFinalists = 0;
 	for (const index of evenlySpacedIndices({
 		length: winners.length,
 		count: candidateCount + 1
 	})) {
 		const winner = winners[index];
-		if (!budget.canAfford(evaluator.countUncached({
+		const uncached = evaluator.countUncached({
 			candidate: winner.candidate,
 			ids: validationIds,
 			split: "val"
-		}))) break;
-		let score;
-		try {
-			score = require_evaluation.measuredMean(await evaluator.evaluate({
-				candidate: winner.candidate,
-				batch: validationSet,
-				ids: validationIds,
-				split: "val",
-				phase: index === 0 ? "seed" : "validation",
-				candidateId: index,
-				iteration: step
-			}));
-		} catch (err) {
-			if (err instanceof require_evaluation.BudgetExhausted || signal?.aborted) break;
-			throw err;
-		}
-		if (score !== void 0) finalists.push({
+		});
+		if (!budget.canAfford(owedForFinalists + uncached)) break;
+		owedForFinalists += uncached;
+		contenders.push({
+			index,
 			candidate: winner.candidate,
-			score,
 			step: winner.step
 		});
 	}
+	const sweeps = await mapDistinct({
+		items: contenders,
+		limit: concurrency,
+		key: (contender) => require_warnings.candidateHash(contender.candidate),
+		task: async (contender) => {
+			if (signal?.aborted) return { stop: true };
+			try {
+				const evaluation = await evaluator.evaluate({
+					candidate: contender.candidate,
+					batch: validationSet,
+					ids: validationIds,
+					split: "val",
+					phase: contender.index === 0 ? "seed" : "validation",
+					candidateId: contender.index,
+					iteration: step
+				});
+				return {
+					score: require_warnings.measuredMean(evaluation),
+					evaluation
+				};
+			} catch (err) {
+				if (err instanceof require_warnings.BudgetExhausted || signal?.aborted) return { stop: true };
+				throw err;
+			}
+		}
+	});
+	const finalists = [];
+	let acceptedCandidates = -1;
+	let acceptedScore = Number.NEGATIVE_INFINITY;
+	for (const [position, outcome] of sweeps.entries()) {
+		if (outcome.stop === true) break;
+		if (outcome.score === void 0) continue;
+		const contender = contenders[position];
+		finalists.push({
+			candidate: contender.candidate,
+			score: outcome.score,
+			step: contender.step
+		});
+		if (outcome.score > acceptedScore) {
+			acceptedScore = outcome.score;
+			acceptedCandidates += 1;
+			emit({
+				type: "candidateAccepted",
+				step: contender.step,
+				candidateId: acceptedCandidates,
+				candidate: contender.candidate,
+				aggregateScore: outcome.score,
+				instanceScores: require_warnings.instanceRow(outcome.evaluation),
+				...trackBestOutputs ? { outputs: outcome.evaluation.outputs } : {}
+			});
+		}
+	}
 	const seedScore = finalists[0]?.score ?? 0;
 	finalists.sort((a, b) => b.score - a.score);
 	const best = finalists[0] ?? {
@@ -526,7 +605,7 @@ async function run(args) {
 		candidateId: null,
 		iteration: step
 	}))?.outputs;
-	const testScore = testSet === void 0 ? void 0 : require_evaluation.measuredMean(await evaluator.evaluate({
+	const heldOut = testSet === void 0 ? void 0 : await evaluator.evaluate({
 		candidate: best.candidate,
 		batch: testSet,
 		ids: testIds,
@@ -535,13 +614,18 @@ async function run(args) {
 		candidateId: null,
 		iteration: step,
 		charge: false
-	}));
-	onEvent?.({
+	});
+	const testScore = heldOut === void 0 ? void 0 : require_warnings.measuredMean(heldOut);
+	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
+		bestCandidateId: acceptedCandidates,
 		bestScore: best.score,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: require_warnings.instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
 		bestCandidate: best.candidate,
@@ -557,8 +641,10 @@ async function run(args) {
 		...bestOutputs === void 0 ? {} : { bestOutputs },
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: testSet?.length ?? 0
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
+		warnings,
 		stopReason
 	};
 	/**
@@ -594,7 +680,11 @@ async function run(args) {
 		const drops = /* @__PURE__ */ new Set();
 		for (let draw = 0; draw < Math.min(wanted, held); draw += 1) drops.add(rng.nextInt(held));
 		const next = { ...candidate };
-		for (const [name, demos] of blocks) next[name] = require_demos.formatDemos(demos.filter((_, index) => !drops.has(index)), renderDemo === void 0 ? {} : { render: renderDemo });
+		for (const [name, demos] of blocks) next[name] = require_demos.replaceDemos({
+			text: next[name] ?? "",
+			demos: demos.filter((_, index) => !drops.has(index)),
+			...renderDemo === void 0 ? {} : { render: renderDemo }
+		});
 		return next;
 	}
 	/**
@@ -615,8 +705,12 @@ async function run(args) {
 		};
 		const next = { ...candidate };
 		for (const name of demoComponents) {
-			const kept = require_demos.parseDemos(next[name] ?? "");
-			next[name] = require_demos.formatDemos([...kept, demo], renderDemo === void 0 ? {} : { render: renderDemo });
+			const text = next[name] ?? "";
+			next[name] = require_demos.replaceDemos({
+				text,
+				demos: [...require_demos.parseDemos(text), demo],
+				...renderDemo === void 0 ? {} : { render: renderDemo }
+			});
 		}
 		return next;
 	}
@@ -644,6 +738,7 @@ async function run(args) {
 		const response = await reflect({
 			prompt: buildPrompt({
 				components: ruleComponents,
+				current: Object.fromEntries(ruleComponents.map((name) => [name, candidate[name] ?? ""])),
 				input: bucket.datum,
 				...better === void 0 ? {} : { better },
 				...worse === void 0 ? {} : { worse }
@@ -684,6 +779,7 @@ async function run(args) {
 			metricCalls: budget.spent(),
 			reflectionCalls,
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
 			rngState: rng.state(),
 			...sampler.state === void 0 ? {} : { sampler: sampler.state() },
 			...cached === void 0 ? {} : { cache: cached }
@@ -694,14 +790,51 @@ async function run(args) {
 		await onCheckpoint(takeSnapshot(completed));
 	}
 }
-function defaultInstanceId(args) {
-	return String(args.index);
+/**
+* Runs `items` concurrently, except that items sharing a key run one after the
+* other.
+*
+* Two identical candidates cost one evaluation and one cache hit when they are
+* scored in sequence, and two evaluations when they overlap: the cache is only
+* written once a rollout returns. Serializing the duplicates is what keeps a
+* fan-out from buying a second copy of a score the run has already paid for.
+*/
+async function mapDistinct(args) {
+	const { items, limit, key, task } = args;
+	const groups = /* @__PURE__ */ new Map();
+	items.forEach((item, index) => {
+		const group = groups.get(key(item));
+		if (group === void 0) {
+			groups.set(key(item), [index]);
+			return;
+		}
+		group.push(index);
+	});
+	const results = new Array(items.length);
+	await require_concurrency.mapWithConcurrency({
+		items: [...groups.values()],
+		limit,
+		task: async (indices) => {
+			for (const index of indices) results[index] = await task(items[index], index);
+		}
+	});
+	return results;
+}
+/**
+* Range checks on the search knobs, run at construction so a configuration
+* that could never terminate is refused before a task is ever handed to it.
+*/
+function assertSimbaConfig(config) {
+	const { concurrency = 1 } = config;
+	if (!Number.isInteger(concurrency) || concurrency < 1) throw new Error(`concurrency must be a positive integer, received ${concurrency}`);
 }
 //#endregion
 exports.SimbaOptimizer = SimbaOptimizer;
 exports.buildAdvicePrompt = buildAdvicePrompt;
 exports.buildBuckets = buildBuckets;
 exports.evenlySpacedIndices = evenlySpacedIndices;
+exports.isCandidateAccepted = require_warnings.isCandidateAccepted;
+exports.isRunFinished = require_warnings.isRunFinished;
 exports.parseAdvice = parseAdvice;
 exports.percentile = percentile;
 exports.samplePoisson = samplePoisson;

package/dist/simba/index.d.cts

@@ -1,8 +1,8 @@
-import { n as Candidate, t as Adapter, u as TextModel } from "../types-CWv4IQFF.cjs";
+import { S as UsageTotals, a as RunFinished, f as Adapter, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, p as Candidate, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, x as TextModel } from "../reporting-bq007_2z.cjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.cjs";
 import { t as Rng } from "../rng-BR5MOedA.cjs";
-import { r as DemoRenderer } from "../demos-BTuzFNsp.cjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-B7SpRwl7.cjs";
+import { r as DemoRenderer } from "../demos-ByaLZy-Z.cjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-4Zv-Zt2t.cjs";
 import { t as BatchSampler } from "../sampling-axOwfZf5.cjs";
 //#region src/simba/advice.d.ts
 interface AdviceTrajectory<Output = unknown> {
@@ -13,6 +13,12 @@ interface AdviceTrajectory<Output = unknown> {
 interface AdvicePromptArgs<Datum = unknown, Output = unknown> {
   /** Components the advice is wanted for, named so each can be addressed. */
   components: readonly string[];
+  /**
+   * What each of those components currently says. Advice is appended to this
+   * text rather than replacing it, so a proposer that cannot see it writes
+   * blind: it restates what is already there, and cannot correct it.
+   */
+  current: Record<string, string>;
   input: Datum;
   /** The higher scoring run of this instance, when there is one to contrast. */
   better?: AdviceTrajectory<Output>;
@@ -52,11 +58,41 @@ interface SimbaFinalist<K extends string = string> {
   step: number;
 }
 interface SimbaConfig {
-  /** Instances per step. Default 32. */
+  /**
+   * Instances per step. Default 32.
+   *
+   * Minibatch defaults do not transfer between optimizers — GEPA's 3 and
+   * MIPRO's 35 mean different things, and SIMBA's ranks instances within a
+   * batch by how much its programs disagreed.
+   *
+   * @see docs/tuning.md
+   */
   minibatchSize?: number;
   /** Programs sampled per step, and candidates built from them. Default 6. */
   candidates?: number;
-  /** Steps to run. Default 8. */
+  /**
+   * How many evaluations may be in flight at once. Default 1.
+   *
+   * Covers the two places a step's work is independent — scoring the
+   * candidates a step built, and sweeping the finalists at the end of the run
+   * — and nothing else. The trajectory samples and the mutations that read
+   * them are deliberately left in sequence: each of those reads state the one
+   * before it wrote, so overlapping them would make a seeded run depend on
+   * which call returned first.
+   */
+  concurrency?: number;
+  /**
+   * Steps to run. Default 8.
+   *
+   * A run has to be funded past the finalist reserve before any step happens:
+   * `min(candidates + 1, maxSteps + 1) * validationSet.length` rollouts are
+   * held back to sweep the finalists, and each step then costs
+   * `candidates * minibatchSize + minibatchSize`. Fund only the steps and the
+   * run stops after one, reporting `budgetExhausted` like any other exhausted
+   * budget.
+   *
+   * @see docs/tuning.md
+   */
   maxSteps?: number;
   /** Demos a candidate may hold before the loop starts dropping them. Default 4. */
   maxDemos?: number;
@@ -97,6 +133,8 @@ interface SimbaSnapshot {
   metricCalls: number;
   reflectionCalls: number;
   cacheHits: number;
+  /** Usage already spent, so a resumed run reports totals and honours ceilings. */
+  usage?: UsageTotals;
   rngState: number;
   sampler?: unknown;
   cache?: [string, CachedScore][];
@@ -121,7 +159,8 @@ interface SimbaTask<Datum, Trajectory = unknown, Output = unknown, K extends str
     index: number;
   }) => string;
   cache?: EvaluationCache | false;
-  onEvent?: (event: SimbaEvent<NoInfer<K>>) => void;
+  /** Observers of the run. Every one sees every event; none can fail it. */
+  reporters?: readonly Reporter<SimbaEvent<NoInfer<K>>>[];
   onCheckpoint?: (snapshot: SimbaSnapshot) => void | Promise<void>;
   resumeFrom?: SimbaSnapshot;
 }
@@ -143,17 +182,18 @@ type SimbaEvent<K extends string = string> = {
   /** The program it was mutated from; 0 is the seed. */
   sourceProgram: number;
   minibatchScore: number;
-} | {
+} | ({
+  type: "candidateAccepted";
+  /** The step whose winner this was. */
+  step: number;
+} & CandidateAccepted<K>) | {
   type: "error";
   step: number;
   err: unknown;
-} | {
+} | ({
   type: "finish";
   reason: SimbaStopReason;
-  bestScore: number;
-  metricCalls: number;
-  testScore?: number;
-};
+} & RunFinished);
 interface SimbaResult<K extends string = string, Output = unknown> extends OptimizerResult<K, SimbaStopReason, Output> {
   /** The seed's own validation score, so the lift the run bought is readable. */
   seedScore: number;
@@ -286,4 +326,4 @@ declare function evenlySpacedIndices(args: {
   count: number;
 }): number[];
 //#endregion
-export { type AdvicePromptArgs, type AdvicePromptBuilder, type AdviceTrajectory, type SimbaBucket, type SimbaConfig, type SimbaEvent, type SimbaFinalist, SimbaOptimizer, type SimbaResult, type SimbaRollout, type SimbaSample, type SimbaSnapshot, type SimbaStopReason, type SimbaStrategy, type SimbaTask, buildAdvicePrompt, buildBuckets, evenlySpacedIndices, parseAdvice, percentile, samplePoisson, softmaxWeights, topKPlusBaseline };
+export { type AdvicePromptArgs, type AdvicePromptBuilder, type AdviceTrajectory, type CandidateAccepted, type OptimizerEvent, type ReportableEvent, type Reporter, type RunFinished, type SimbaBucket, type SimbaConfig, type SimbaEvent, type SimbaFinalist, SimbaOptimizer, type SimbaResult, type SimbaRollout, type SimbaSample, type SimbaSnapshot, type SimbaStopReason, type SimbaStrategy, type SimbaTask, buildAdvicePrompt, buildBuckets, evenlySpacedIndices, isCandidateAccepted, isRunFinished, parseAdvice, percentile, samplePoisson, softmaxWeights, topKPlusBaseline };