textopt 0.0.0 → 0.1.0

package/README.md

@@ -23,7 +23,7 @@ This package has no runtime dependencies. For an overview of the algorithms and
 ```ts
 import {
   assertResumable,
-  bootstrapDemos,
+  harvestFewShotExamples,
   buildJudgePrompt,
   compare,
   componentNames,
@@ -32,11 +32,13 @@ import {
   createJudge,
   createMemoryCache,
   formatDemos,
+  harvestRollouts,
   mapWithConcurrency,
   parseDemos,
   parseProposedText,
   priceUsage,
   runFingerprint,
+  toTrainingJsonl,
 } from "textopt";
 import type {
   Adapter,
@@ -93,9 +95,9 @@ evaluate(args: EvaluateArgs<Datum, K>): Promise<EvaluationBatch<Trajectory, Outp
 
 **`transient`** marks scores caused by infrastructure failures such as rate limits, 5xx responses, or network errors. Transient scores are not cached.
 
-**`Optimizer<Stop extends string>`** defines `optimize(task: OptimizerTask) => Promise<OptimizerResult>`. `OptimizerTask` contains the shared run inputs: `seedCandidate`, `trainingSet`, `validationSet`, `testSet`, `adapter`, `maxMetricCalls`, `maxCostUsd`, `maxWallClockMs`, `cacheNamespace`, `retry`, and `signal`. `OptimizerResult` contains `bestCandidate`, `bestScore`, `bestOutputs`, `metricCalls`, `usage`, `testScore`, `testMetricCalls`, and `stopReason`. Optimizer-specific task and result types extend these interfaces.
+**`Optimizer<Stop extends string>`** defines `optimize(task: OptimizerTask) => Promise<OptimizerResult>`. `OptimizerTask` contains the shared run inputs: `seedCandidate`, `trainingSet`, `validationSet`, `testSet`, `adapter`, `maxMetricCalls`, `maxCostUsd`, `maxWallClockMs`, `cacheNamespace`, `retry`, and `signal`. `OptimizerResult` contains `bestCandidate`, `bestScore`, `bestOutputs`, `metricCalls`, `usage`, `testScore`, `testMetricCalls`, `testUsage`, and `stopReason`. Optimizer-specific task and result types extend these interfaces.
 
-**`maxCostUsd`** and **`maxWallClockMs`** are checked between evaluations, so a run overruns by at most one of them. Neither follows from `maxMetricCalls`: reflective search grows the text it optimizes, so late rollouts cost more than early ones, and a run behind a rate limit spends almost nothing while taking as long as the provider makes it take.
+**`maxCostUsd`** and **`maxWallClockMs`** are checked between evaluations, so a run overruns by whatever it had in flight when the ceiling was reached — one evaluation at the default concurrency, and up to `concurrency` of them above it. Neither bounds the held-out sweep, which runs once the search has already stopped: it is reported apart from the search as `testMetricCalls` and `testUsage`, and has to be budgeted for separately. Neither follows from `maxMetricCalls`: reflective search grows the text it optimizes, so late rollouts cost more than early ones, and a run behind a rate limit spends almost nothing while taking as long as the provider makes it take.
 
 **`cacheNamespace`** scopes every cache key to the system the rollouts were measured under — model id, decoding settings, scorer version. Change it whenever anything outside the candidate text changes.
 
@@ -103,7 +105,7 @@ evaluate(args: EvaluateArgs<Datum, K>): Promise<EvaluationBatch<Trajectory, Outp
 
 **`UsageTotals`** (`inputTokens`, `outputTokens`, `totalTokens`, `costUsd`, `rollouts`) is summed from the `RolloutUsage` entries an adapter reports. Zero throughout when the adapter reports none.
 
-**`testSet`** is excluded from search and evaluated once against the winner. Because candidates are selected on `validationSet`, `bestScore` may be fitted to it. `testScore` measures held-out performance. Test rollouts are reported as `testMetricCalls` and do not count against `maxMetricCalls`.
+**`testSet`** is excluded from search and evaluated once against the winner. Because candidates are selected on `validationSet`, `bestScore` may be fitted to it. `testScore` measures held-out performance. Test rollouts are reported as `testMetricCalls` costing `testUsage`, outside `maxMetricCalls` and `maxCostUsd` both.
 
 **`TextModel`** is the provider-independent interface `({ prompt, signal }) => Promise<string>`.
 
@@ -117,9 +119,13 @@ For Redis, SQLite, or file-backed caching, implement **`EvaluationCache`** with
 
 **`componentNames(candidate)`** returns `Object.keys(candidate)` while preserving the component-name union.
 
-**`createEvaluator({ adapter, budget, cache, cacheNamespace, retry, trackOutputs, onEvaluation, signal, cacheHits })`** handles adapter calls, caching, budget accounting, transient scores, and evaluation events. `evaluate` returns a `ScoredBatch`. `evaluateTraced` returns an `EvaluationBatch`, or `null` when the remaining budget cannot cover the batch. A batch that exceeds the charged budget throws `BudgetExhausted`. All included optimizers use this evaluator.
+**`createEvaluator({ adapter, budget, cache, cacheNamespace, retry, trackOutputs, onEvaluation, signal, cacheHits, usage })`** handles adapter calls, caching, budget accounting, transient scores, and evaluation events. `cacheHits` and `usage` seed the counters from a checkpoint, so a resumed run reports totals rather than deltas. `usage()` covers the charged rollouts a ceiling is checked against; `unchargedUsage()` covers what `charge: false` bought. `evaluate` returns a `ScoredBatch`. `evaluateTraced` returns an `EvaluationBatch`, or `null` when the remaining budget cannot cover the batch. A batch that exceeds the charged budget throws `BudgetExhausted`. All included optimizers use this evaluator.
 
-**`bootstrapDemos({ adapter, candidate, trainingSet, minScore, maxDemos, batchSize, maxMetricCalls, rng, renderDemo, signal })`** evaluates a candidate on `trainingSet` and keeps the rollouts the metric rewarded. Omit `minScore` to keep any rollout scoring above zero, as MIPROv2 does without a `metric_threshold`; pass a number to require at least that score. It returns the selected `demos`, a formatted `block`, and the metric calls used. It does not use the score cache because it needs rollout outputs.
+**`harvestRollouts({ adapter, candidate, data, minScore, maxRollouts, batchSize, maxMetricCalls, maxCostUsd, rng, signal })`** runs a candidate over `data` and returns the `Rollout`s the metric rewarded, alongside `metricCalls` and `attempted`. Omit `minScore` to keep any rollout scoring above zero; omit `maxRollouts` to sweep the whole pool. It carries its own budget and does not use the score cache, because it needs the outputs a cache hit cannot return. `maxCostUsd` bounds its dollars, checked between batches — a caller bounding spend cannot bound this pass from outside, since it runs on its own evaluator. Sweeping a validation set is the mistake to avoid — see [Distilling a run](../../docs/distillation.md).
+
+**`toTrainingJsonl({ rollouts, render })`** serializes harvested rollouts as one chat-messages example per line. `render` turns a rollout into `{ messages }` or returns `null` to skip it, and decides how much of the optimized candidate stays in the training input. Returns the text; writing it is the caller's job.
+
+**`harvestFewShotExamples({ adapter, candidate, trainingSet, minScore, maxDemos, batchSize, maxMetricCalls, maxCostUsd, rng, renderDemo, signal })`** evaluates a candidate on `trainingSet` and keeps the rollouts the metric rewarded. Omit `minScore` to keep any rollout scoring above zero, as MIPROv2 does without a `metric_threshold`; pass a number to require at least that score. It returns the selected `demos`, a formatted `block`, and the metric calls and usage it spent. It does not use the score cache because it needs rollout outputs.
 
 **`formatDemos(demos, { render })`** and **`parseDemos(text)`** write and read the `<demo>`, `<input>`, and `<output>` block format.
 
@@ -137,6 +143,18 @@ For Redis, SQLite, or file-backed caching, implement **`EvaluationCache`** with
 
 **`BatchSampler<Datum>`** and **`Rng`** are type-only exports. Their default implementations are internal.
 
+### Reporting
+
+Every optimizer takes **`reporters`**, an array of **`Reporter<Event>`** with an optional `onEvent` and an optional `flush`. `onEvent` is called synchronously on the search's hot path, so a reporter that ships anywhere over a network buffers there and uploads in `flush`, which is awaited once as the run ends — including when it ends by throwing. A reporter that throws is warned about and skipped: observability never fails a run. Persisting a run so it can be resumed is `onCheckpoint`, which is durability and a separate concern.
+
+Each search emits its own discriminated union, but two members are shared. **`CandidateAccepted`** carries `candidateId`, the `candidate` text, its `aggregateScore`, its `instanceScores` over the validation set, and its `outputs` under `trackBestOutputs`. **`RunFinished`** carries `bestCandidateId`, `bestScore`, `metricCalls`, and the winner's held-out `testScore`, `testInstanceScores` and `testOutputs`. In both rows, `undefined` marks an instance nothing measured — the evaluation policy skipped it, or an infrastructure failure lost it — and reporting one as a zero shows as a regression that never happened.
+
+`candidateAccepted` fires only when the incumbent moves and a full validation sweep measured it, so `instanceScores` never means "a subset, and you work out which". SIMBA accepts on a minibatch, which is too small a sample to name a row against; it reports those in its own `candidate` event and emits `candidateAccepted` when the sweep that confirms a step winner lands.
+
+Every optimizer reports the seed as `candidateId` 0 once its own sweep lands, so a report opens with the baseline every later candidate is read against rather than with the first improvement. A resumed run does not re-emit it. Each search adds its own fields to the event — GEPA its `iteration`, `parentIds` and `source`, MIPRO its `trial`, SIMBA its `step`, OPRO and random search their `round`, bootstrap search its `source` and `demos` — and drops anything the seed could not honestly carry, since the events that name a component or a menu selection already carry it.
+
+A reporter that reads only those two takes **`OptimizerEvent`** — `{ type: string }` — and narrows with **`isCandidateAccepted`** and **`isRunFinished`**. Because a literal tag is assignable to `string` and the parameter position is contravariant, one such reporter drops into any optimizer's `reporters` array. **`ReportableEvent`** names the union it reads. `@textopt/langsmith` is written this way.
+
 ## `textopt/gepa`
 
 ```ts
@@ -185,17 +203,17 @@ These options control search behavior and can be reused across runs.
 
 Required: `seedCandidate`, `trainingSet`, `adapter` (a `GepaAdapter`), `reflect` (a `TextModel`), and `maxMetricCalls`.
 
-| Option                | Default                                                                              |
-| --------------------- | ------------------------------------------------------------------------------------ |
-| `validationSet`       | the trainingSet                                                                      |
-| `testSet`             | none. Held out of the search and scored once, on the winner                          |
-| `componentSelector`   | `roundRobinComponentSelector()`                                                      |
-| `batchSampler`        | an epoch-shuffled sampler over `minibatchSize`                                       |
-| `valEvaluationPolicy` | `fullEvaluationPolicy()`                                                             |
-| `instanceId`          | a content hash of the datum, falling back to its position when it will not serialize |
-| `cache`               | a per-run memory cache. Pass `false` to disable                                      |
+| Option                | Default                                                                            |
+| --------------------- | ---------------------------------------------------------------------------------- |
+| `validationSet`       | the trainingSet                                                                    |
+| `testSet`             | none. Held out of the search and scored once, on the winner                        |
+| `componentSelector`   | `roundRobinComponentSelector()`                                                    |
+| `batchSampler`        | an epoch-shuffled sampler over `minibatchSize`                                     |
+| `valEvaluationPolicy` | `fullEvaluationPolicy()`                                                           |
+| `instanceId`          | a content hash of the datum, falling back to its position when JSON cannot read it |
+| `cache`               | a per-run memory cache. Pass `false` to disable                                    |
 
-`onEvent`, `onCheckpoint`, `resumeFrom`, and `signal` have no defaults.
+`reporters`, `onCheckpoint`, `resumeFrom`, and `signal` have no defaults.
 
 TypeScript infers component names and the datum type from `seedCandidate` and `trainingSet`. Other fields use `NoInfer` and are checked against those inferred types.
 
@@ -230,7 +248,7 @@ When `proposeNewTexts` is implemented, the adapter generates proposals without c
 
 Each export is a factory. Selector and acceptance interfaces accept custom functions. A `ValEvaluationPolicy` is an object with `selectInstances` and `bestCandidate` methods.
 
-`pairedPermutationAcceptance` and `lowerBoundEvaluationPolicy` exist for metrics whose readings vary between runs of the same text. Both are strictly more conservative than the defaults, and on a metric that does not vary that is pure cost: in the twenty-seed benchmark the pair drops GEPA from 0.729 to 0.175 on the noiseless task and ties it on the noisy one. A sign-flip test also needs a wide enough minibatch to say anything — over three instances the smallest p-value it can produce is 0.125, so at the default `minibatchSize` no proposal clears an `alpha` below that.
+`pairedPermutationAcceptance` and `lowerBoundEvaluationPolicy` exist for metrics whose readings vary between runs of the same text. Both are strictly more conservative than the defaults. In the twenty-seed benchmark the pair scores 0.931 against plain GEPA's 0.920 on the noisy task and 0.945 against 0.947 on the noiseless one, neither gap significant over those seeds — conservative enough to be worth having on a metric that varies, cheap enough not to hurt on one that does not. A sign-flip test also needs a wide enough minibatch to say anything — over three instances the smallest p-value it can produce is 0.125, so at the default `minibatchSize` no proposal clears an `alpha` below that.
 
 ### Reflection prompts
 
@@ -270,7 +288,11 @@ Feedback is end-to-end and every module receives the same string. A metric score
 
 **`stopReason`** is one of `"budgetExhausted"`, `"costExhausted"`, `"deadlineReached"`, `"reflectionBudgetExhausted"`, `"aborted"`, or `"maxIterations"`.
 
-**`onEvent`** receives a discriminated `GepaEvent`: `start`, `iterationStart`, `evaluation`, `proposal`, `candidateAccepted`, `candidateRejected` (with `reason: "worse" | "notSelected"`), `error`, and `finish`.
+**`reporters`** is an array of `Reporter<GepaEvent>`, each with an optional `onEvent` and an optional `flush`. `onEvent` receives a discriminated `GepaEvent`: `start`, `iterationStart`, `evaluation`, `proposal`, `candidateAccepted`, `candidateRejected` (with `reason: "worse" | "notSelected"`), `error`, and `finish`. It is called synchronously, on the search's hot path, so a reporter that ships anywhere over a network buffers in `onEvent` and uploads in `flush`, which is awaited once as the run ends — including when it ends by throwing. A reporter that throws is warned about and skipped: observability never fails a run.
+
+`candidateAccepted` fires for the seed too, and carries the `candidate` text, its `instanceScores` over the validation set, and its `outputs` under `trackBestOutputs` — the row a candidate put on the frontier, not just the mean. `finish` carries the same for the held-out sweep in `testInstanceScores` and `testOutputs`. In both, `undefined` marks an instance nothing measured — the evaluation policy skipped it, or an infrastructure failure lost it — and reporting one as a zero shows as a regression that never happened.
+
+Reporting is observability; persisting a run so it can be resumed is `onCheckpoint`, below.
 
 **`onCheckpoint`** runs after seed evaluation and each iteration with a JSON-serializable `GepaSnapshot`. Pass it as `resumeFrom` to continue. A fingerprint prevents resuming with a different seed candidate, instance set, or random seed. Every optimizer here has the same three: `onCheckpoint`, `resumeFrom`, and a `snapshot` on the result. A snapshot handed back as `resumeFrom` is copied, never mutated by the run that continues from it.
 
@@ -280,7 +302,7 @@ Feedback is end-to-end and every module receives the same string. A metric score
 import { SimbaOptimizer, buildAdvicePrompt, parseAdvice } from "textopt/simba";
 ```
 
-SIMBA uses the base `Adapter`: it reads outputs, scores, and feedback and builds its own evidence, so it needs no `makeReflectiveDataset`. `SimbaTask` adds `reflect`, `demoComponents`, `instructionComponents`, `renderDemo`, `buildAdvicePrompt`, `sampler`, `instanceId`, `cache`, `onEvent`, `onCheckpoint`, and `resumeFrom`.
+SIMBA uses the base `Adapter`: it reads outputs, scores, and feedback and builds its own evidence, so it needs no `makeReflectiveDataset`. `SimbaTask` adds `reflect`, `demoComponents`, `instructionComponents`, `renderDemo`, `buildAdvicePrompt`, `sampler`, `instanceId`, `cache`, `reporters`, `onCheckpoint`, and `resumeFrom`.
 
 | Option                 | Default              | Effect                                                                     |
 | ---------------------- | -------------------- | -------------------------------------------------------------------------- |
@@ -320,7 +342,7 @@ Ported from DSPy's SIMBA with two deliberate changes. A trajectory sample runs o
 import { BootstrapSearchOptimizer } from "textopt/bootstrap-search";
 ```
 
-DSPy's `BootstrapFewShotWithRandomSearch`. It uses the base `Adapter` and no reflection model at all: every candidate is assembled from outputs the system itself produced, so the search costs rollouts and nothing else. `BootstrapSearchTask` adds `demoComponents` (required), `renderDemo`, `goldOutput`, `instanceId`, `cache`, `onEvent`, `onCheckpoint`, and `resumeFrom`.
+DSPy's `BootstrapFewShotWithRandomSearch`. It uses the base `Adapter` and no reflection model at all: every candidate is assembled from outputs the system itself produced, so the search costs rollouts and nothing else. `BootstrapSearchTask` adds `demoComponents` (required), `renderDemo`, `goldOutput`, `instanceId`, `cache`, `reporters`, `onCheckpoint`, and `resumeFrom`.
 
 | Option             | Default | Effect                                                               |
 | ------------------ | ------- | -------------------------------------------------------------------- |
@@ -352,7 +374,7 @@ Unlike DSPy, which bootstraps each predictor separately from the traces of one p
 import { OproOptimizer, buildOproPrompt } from "textopt/opro";
 ```
 
-OPRO uses the base `Adapter`. `OproTask` adds `reflect` and optional `renderDatum`, `instanceId`, `cache`, and `onEvent` fields.
+OPRO uses the base `Adapter`. `OproTask` adds `reflect` and optional `renderDatum`, `instanceId`, `cache`, and `reporters` fields.
 
 | Option               | Default           | Effect                                                     |
 | -------------------- | ----------------- | ---------------------------------------------------------- |
@@ -399,7 +421,7 @@ For multi-component candidates, each attempt records the other components presen
 import { MiproOptimizer, proposeConfiguration } from "textopt/mipro";
 ```
 
-MIPRO uses the base `Adapter`. `MiproTask` adds `reflect`, `componentOptions`, `renderDatum`, `batchSampler`, `instanceId`, `cache`, and `onEvent`.
+MIPRO uses the base `Adapter`. `MiproTask` adds `reflect`, `componentOptions`, `renderDatum`, `batchSampler`, `instanceId`, `cache`, and `reporters`.
 
 | Option                     | Default            | Effect                                                                             |
 | -------------------------- | ------------------ | ---------------------------------------------------------------------------------- |

package/dist/bootstrap-search/index.cjs

@@ -1,6 +1,6 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_evaluation = require("../evaluation-OZOp6TB7.cjs");
-const require_demos = require("../demos-B9BJiNKz.cjs");
+const require_reporting = require("../reporting-CNHzbJC-.cjs");
+const require_demos = require("../demos-D2o0qSSo.cjs");
 const require_rng = require("../rng-DbA_rPIo.cjs");
 //#region src/bootstrap-search/optimize.ts
 const DEFAULT_CANDIDATES = 16;
@@ -34,21 +34,27 @@ const DEFAULT_MAX_LABELED_DEMOS = 16;
 var BootstrapSearchOptimizer = class {
 	#config;
 	constructor(config = {}) {
+		assertBootstrapSearchConfig(config);
 		this.#config = config;
 	}
-	optimize(task) {
-		return run({
-			config: this.#config,
-			task
-		});
+	async optimize(task) {
+		try {
+			return await run({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await require_reporting.flushReporters(task.reporters ?? []);
+		}
 	}
 };
 async function run(args) {
 	const { config, task } = args;
-	const { candidates: shuffledHarvests = DEFAULT_CANDIDATES, maxDemos = DEFAULT_MAX_DEMOS, minDemos = DEFAULT_MIN_DEMOS, maxLabeledDemos = DEFAULT_MAX_LABELED_DEMOS, demoMinScore, stopAtScore, seed = 0, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, demoComponents, renderDemo, goldOutput, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
-	const deadline = require_evaluation.createDeadline({ maxWallClockMs });
-	const components = require_evaluation.componentNames(seedCandidate);
+	const { candidates: shuffledHarvests = DEFAULT_CANDIDATES, maxDemos = DEFAULT_MAX_DEMOS, minDemos = DEFAULT_MIN_DEMOS, maxLabeledDemos = DEFAULT_MAX_LABELED_DEMOS, demoMinScore, stopAtScore, concurrency = 1, seed = 0, trackBestOutputs = false, checkpointCache = true } = config;
+	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, demoComponents, renderDemo, goldOutput, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_reporting.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const emit = require_reporting.createEmitter(reporters);
+	const deadline = require_reporting.createDeadline({ maxWallClockMs });
+	const components = require_reporting.componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
 	if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
 	if (demoComponents.length === 0) throw new Error("optimize requires at least one demoComponent: this search has nothing to put demonstrations in otherwise");
@@ -61,7 +67,7 @@ async function run(args) {
 		datum,
 		index
 	})) ?? [];
-	const fingerprint = require_evaluation.runFingerprint({
+	const fingerprint = require_reporting.runFingerprint({
 		seedCandidate,
 		trainingIds: trainingSet.map((datum, index) => instanceId({
 			datum,
@@ -71,17 +77,17 @@ async function run(args) {
 		seed,
 		...cacheNamespace === void 0 ? {} : { cacheNamespace }
 	});
-	require_evaluation.assertResumable({
+	require_reporting.assertResumable({
 		fingerprint,
 		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
 	});
 	const rng = require_rng.createSeededRng(seed, resumeFrom?.rngState);
-	const budget = require_evaluation.createBudget({
+	const budget = require_reporting.createBudget({
 		maxMetricCalls,
 		spent: resumeFrom?.metricCalls ?? 0
 	});
-	const evaluationCache = cache === false ? void 0 : cache ?? require_evaluation.createMemoryCache();
-	const evaluator = require_evaluation.createEvaluator({
+	const evaluationCache = cache === false ? void 0 : cache ?? require_reporting.createMemoryCache();
+	const evaluator = require_reporting.createEvaluator({
 		adapter,
 		budget,
 		...retry === void 0 ? {} : { retry },
@@ -89,8 +95,9 @@ async function run(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
-		onEvaluation: (event) => onEvent?.({
+		onEvaluation: (event) => emit({
 			type: "evaluation",
 			...event
 		})
@@ -100,7 +107,7 @@ async function run(args) {
 	let drawn = resumeFrom?.drawn ?? 0;
 	let bootstrapMetricCalls = resumeFrom?.bootstrapMetricCalls ?? 0;
 	let stopReason = "candidatesExhausted";
-	onEvent?.({
+	emit({
 		type: "start",
 		components,
 		validationSetSize: validationSet.length
@@ -116,13 +123,25 @@ async function run(args) {
 			iteration: evaluated.length
 		});
 	}
-	const seedScore = resumeFrom?.seedScore ?? require_evaluation.requireMeasuredMean({
-		batch: await sweep(seedCandidate, "seed"),
+	const seedEvaluation = resumeFrom === void 0 ? await sweep(seedCandidate, "seed") : void 0;
+	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_reporting.requireMeasuredMean({
+		batch: seedEvaluation,
 		phase: "seed"
 	});
 	let best = resumeFrom?.best ?? seedCandidate;
 	let bestScore = resumeFrom?.bestScore ?? seedScore;
 	let bestOutputs;
+	let acceptedCandidates = resumeFrom?.acceptedCandidates ?? 0;
+	if (seedEvaluation !== void 0) emit({
+		type: "candidateAccepted",
+		source: "zeroShot",
+		demos: 0,
+		candidateId: 0,
+		candidate: seedCandidate,
+		aggregateScore: seedScore,
+		instanceScores: require_reporting.instanceRow(seedEvaluation),
+		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
+	});
 	function takeSnapshot() {
 		const cached = checkpointCache ? evaluationCache?.entries?.() : void 0;
 		return {
@@ -136,6 +155,8 @@ async function run(args) {
 			metricCalls: budget.spent(),
 			bootstrapMetricCalls,
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
+			acceptedCandidates,
 			rngState: rng.state(),
 			...cached === void 0 ? {} : { cache: cached }
 		};
@@ -148,12 +169,13 @@ async function run(args) {
 		shuffledHarvests,
 		labeled: goldOutput !== void 0
 	});
-	for (; drawn < plan.length; drawn += 1) {
+	const waveSize = stopAtScore === void 0 ? concurrency : 1;
+	while (drawn < plan.length) {
 		if (signal?.aborted) {
 			stopReason = "aborted";
 			break;
 		}
-		if (require_evaluation.costExhausted({
+		if (require_reporting.costExhausted({
 			usage: evaluator.usage(),
 			maxCostUsd
 		})) {
@@ -164,58 +186,95 @@ async function run(args) {
 			stopReason = "deadlineReached";
 			break;
 		}
-		if (!budget.canAfford(validationSet.length + 1)) {
-			stopReason = "budgetExhausted";
-			break;
-		}
-		const source = plan[drawn];
-		const block = await buildBlock(source);
-		const candidate = withDemos(block);
-		let evaluation;
-		try {
-			evaluation = await sweep(candidate, "validation");
-		} catch (err) {
-			if (err instanceof require_evaluation.BudgetExhausted) {
-				stopReason = "budgetExhausted";
-				break;
-			}
-			if (signal?.aborted) {
-				stopReason = "aborted";
+		const wave = [];
+		const inFlight = /* @__PURE__ */ new Map();
+		let waveStop;
+		while (wave.length < waveSize && drawn + wave.length < plan.length) {
+			if (!budget.canAfford(validationSet.length + 1)) {
+				waveStop = "budgetExhausted";
 				break;
 			}
-			throw err;
+			const source = plan[drawn + wave.length];
+			const block = await buildBlock(source);
+			const candidate = withDemos(block);
+			const key = require_reporting.candidateHash(candidate);
+			const prior = inFlight.get(key);
+			const sweeping = prior === void 0 ? settled(sweep(candidate, "validation")) : prior.then(() => settled(sweep(candidate, "validation")));
+			inFlight.set(key, sweeping);
+			wave.push({
+				source,
+				candidate,
+				block,
+				sweep: sweeping
+			});
 		}
-		const score = require_evaluation.measuredMean(evaluation);
-		if (score === void 0) continue;
-		const accepted = score > bestScore;
-		evaluated.push({
-			candidate,
-			source,
-			demos: countDemos(block),
-			score
-		});
-		onEvent?.({
-			type: "candidate",
-			index: evaluated.length - 1,
-			source,
-			demos: countDemos(block),
-			score,
-			accepted
-		});
-		if (accepted) {
-			best = candidate;
-			bestScore = score;
-			bestOutputs = evaluation.outputs;
+		let waveError;
+		let draining = false;
+		for (const entry of wave) {
+			const outcome = await entry.sweep;
+			if (draining) continue;
+			drawn += 1;
+			if (outcome.failed === true) {
+				draining = true;
+				if (outcome.err instanceof require_reporting.BudgetExhausted) {
+					waveStop = "budgetExhausted";
+					continue;
+				}
+				if (signal?.aborted) {
+					waveStop = "aborted";
+					continue;
+				}
+				waveError = outcome.err;
+				continue;
+			}
+			const evaluation = outcome.evaluation;
+			const score = require_reporting.measuredMean(evaluation);
+			if (score === void 0) continue;
+			const accepted = score > bestScore;
+			evaluated.push({
+				candidate: entry.candidate,
+				source: entry.source,
+				demos: countDemos(entry.block),
+				score
+			});
+			emit({
+				type: "candidate",
+				index: evaluated.length - 1,
+				source: entry.source,
+				demos: countDemos(entry.block),
+				score,
+				accepted
+			});
+			if (accepted) {
+				acceptedCandidates += 1;
+				emit({
+					type: "candidateAccepted",
+					source: entry.source,
+					demos: countDemos(entry.block),
+					candidateId: acceptedCandidates,
+					candidate: entry.candidate,
+					aggregateScore: score,
+					instanceScores: require_reporting.instanceRow(evaluation),
+					...trackBestOutputs ? { outputs: evaluation.outputs } : {}
+				});
+				best = entry.candidate;
+				bestScore = score;
+				bestOutputs = evaluation.outputs;
+			}
+			if (stopAtScore !== void 0 && score >= stopAtScore) {
+				waveStop = "scoreReached";
+				draining = true;
+			}
 		}
+		if (waveError !== void 0) throw waveError;
 		await checkpoint();
-		if (stopAtScore !== void 0 && score >= stopAtScore) {
-			stopReason = "scoreReached";
-			drawn += 1;
+		if (waveStop !== void 0) {
+			stopReason = waveStop;
 			break;
 		}
 	}
 	if (signal?.aborted) stopReason = "aborted";
-	const testScore = testSet === void 0 ? void 0 : require_evaluation.measuredMean(await evaluator.evaluate({
+	const heldOut = testSet === void 0 ? void 0 : await evaluator.evaluate({
 		candidate: best,
 		batch: testSet,
 		ids: testIds,
@@ -224,13 +283,17 @@ async function run(args) {
 		candidateId: null,
 		iteration: evaluated.length,
 		charge: false
-	}));
-	onEvent?.({
+	});
+	const testScore = heldOut === void 0 ? void 0 : require_reporting.measuredMean(heldOut);
+	emit({
 		type: "finish",
 		reason: stopReason,
+		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: require_reporting.instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
 		bestCandidate: best,
@@ -245,7 +308,8 @@ async function run(args) {
 		...trackBestOutputs && bestOutputs !== void 0 ? { bestOutputs } : {},
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: testSet?.length ?? 0
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		stopReason
 	};
@@ -260,19 +324,21 @@ async function run(args) {
 		const requested = source === "unshuffled" ? maxDemos : minDemos + rng.nextInt(Math.max(1, maxDemos - minDemos + 1));
 		const affordable = Math.min(trainingSet.length, budget.remaining() - validationSet.length);
 		if (affordable < 1) return "";
-		const harvest = await require_demos.bootstrapDemos({
+		const harvest = await require_demos.harvestFewShotExamples({
 			adapter,
 			candidate: seedCandidate,
 			trainingSet,
 			...demoMinScore === void 0 ? {} : { minScore: demoMinScore },
 			maxDemos: requested,
 			maxMetricCalls: affordable,
+			...maxCostUsd === void 0 ? {} : { maxCostUsd: maxCostUsd - evaluator.usage().costUsd },
 			...source === "unshuffled" ? {} : { rng },
 			...renderDemo === void 0 ? {} : { renderDemo },
 			...signal === void 0 ? {} : { signal }
 		});
 		bootstrapMetricCalls += harvest.metricCalls;
 		budget.reserve(harvest.metricCalls);
+		evaluator.absorbUsage(harvest.usage);
 		return harvest.block;
 	}
 	function labeledBlock() {
@@ -298,11 +364,25 @@ function candidatePlan(args) {
 		...Array.from({ length: shuffledHarvests }, () => "bootstrapped")
 	];
 }
+/** Turns a sweep into a value, so a dispatched one never rejects unobserved. */
+function settled(sweeping) {
+	return sweeping.then((evaluation) => ({ evaluation }), (err) => ({
+		failed: true,
+		err
+	}));
+}
+/**
+* Range checks on the search knobs, run at construction so a configuration
+* that could never terminate is refused before a task is ever handed to it.
+*/
+function assertBootstrapSearchConfig(config) {
+	const { concurrency = 1 } = config;
+	if (!Number.isInteger(concurrency) || concurrency < 1) throw new Error(`concurrency must be a positive integer, received ${concurrency}`);
+}
 function countDemos(block) {
 	return block.split("<demo>").length - 1;
 }
-function defaultInstanceId(args) {
-	return String(args.index);
-}
 //#endregion
 exports.BootstrapSearchOptimizer = BootstrapSearchOptimizer;
+exports.isCandidateAccepted = require_reporting.isCandidateAccepted;
+exports.isRunFinished = require_reporting.isRunFinished;

package/dist/bootstrap-search/index.d.cts

@@ -1,7 +1,7 @@
-import { n as Candidate, t as Adapter } from "../types-CWv4IQFF.cjs";
+import { a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.cjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.cjs";
-import { r as DemoRenderer } from "../demos-BTuzFNsp.cjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-B7SpRwl7.cjs";
+import { r as DemoRenderer } from "../demos-CU9dy2oT.cjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Bh5DPRMH.cjs";
 //#region src/bootstrap-search/optimize.d.ts
 /**
  * Where a candidate's demo block came from. `zeroShot` holds no demos at all,
@@ -35,6 +35,18 @@ interface BootstrapSearchConfig {
    * every candidate, which is the reliable reading and the expensive one.
    */
   stopAtScore?: number;
+  /**
+   * How many candidates may be swept at once. Default 1.
+   *
+   * Harvesting stays in plan order however this is set — every harvest draws
+   * from the same random stream, and reordering them would make a seeded run
+   * unreproducible — so what overlaps is a sweep with the harvest of the
+   * candidates behind it. Two costs come with raising it: a checkpoint is
+   * taken per wave rather than per candidate, so a killed run loses up to this
+   * many candidates instead of one, and `stopAtScore` is honoured by sweeping
+   * one at a time, since a wave cannot know it has already passed the target.
+   */
+  concurrency?: number;
   seed?: number;
   trackBestOutputs?: boolean;
   /**
@@ -61,6 +73,13 @@ interface BootstrapSearchSnapshot {
   metricCalls: number;
   bootstrapMetricCalls: number;
   cacheHits: number;
+  /** Usage already spent, so a resumed run reports totals and honours ceilings. */
+  usage?: UsageTotals;
+  /**
+   * Candidates accepted so far. Reporters key rows by this id, so restarting it
+   * at zero makes a resumed run collide with the run it continues.
+   */
+  acceptedCandidates?: number;
   rngState: number;
   cache?: [string, CachedScore][];
 }
@@ -92,7 +111,8 @@ interface BootstrapSearchTask<Datum, Trajectory = unknown, Output = unknown, K e
   }) => string;
   /** Pass `false` to disable caching entirely. */
   cache?: EvaluationCache | false;
-  onEvent?: (event: BootstrapSearchEvent<NoInfer<K>>) => void;
+  /** Observers of the run. Every one sees every event; none can fail it. */
+  reporters?: readonly Reporter<BootstrapSearchEvent<NoInfer<K>>>[];
   /** Called with a resumable snapshot after every candidate is scored. */
   onCheckpoint?: (snapshot: BootstrapSearchSnapshot) => void | Promise<void>;
   /** Snapshot to continue from. */
@@ -112,13 +132,15 @@ type BootstrapSearchEvent<K extends string = string> = {
   demos: number;
   score: number;
   accepted: boolean;
-} | {
+} | ({
+  type: "candidateAccepted";
+  /** Which construction produced the demo block that won. */
+  source: DemoSource;
+  demos: number;
+} & CandidateAccepted<K>) | ({
   type: "finish";
   reason: BootstrapSearchStopReason;
-  bestScore: number;
-  metricCalls: number;
-  testScore?: number;
-};
+} & RunFinished);
 interface BootstrapSearchResult<K extends string = string, Output = unknown> extends OptimizerResult<K, BootstrapSearchStopReason, Output> {
   /** The seed's score, so the lift the demos bought is readable directly. */
   seedScore: number;
@@ -159,4 +181,4 @@ declare class BootstrapSearchOptimizer implements Optimizer<BootstrapSearchStopR
   optimize<Datum, Trajectory = unknown, Output = unknown, const K extends string = string>(task: BootstrapSearchTask<Datum, Trajectory, Output, K>): Promise<BootstrapSearchResult<K, Output>>;
 }
 //#endregion
-export { type BootstrapCandidate, type BootstrapSearchConfig, type BootstrapSearchEvent, BootstrapSearchOptimizer, type BootstrapSearchResult, type BootstrapSearchSnapshot, type BootstrapSearchStopReason, type BootstrapSearchTask, type DemoSource };
+export { type BootstrapCandidate, type BootstrapSearchConfig, type BootstrapSearchEvent, BootstrapSearchOptimizer, type BootstrapSearchResult, type BootstrapSearchSnapshot, type BootstrapSearchStopReason, type BootstrapSearchTask, type CandidateAccepted, type DemoSource, type OptimizerEvent, type ReportableEvent, type Reporter, type RunFinished, isCandidateAccepted, isRunFinished };