textopt 0.0.0 → 0.1.0

package/dist/bootstrap-search/index.d.mts

@@ -1,7 +1,7 @@
-import { n as Candidate, t as Adapter } from "../types-CWv4IQFF.mjs";
+import { a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.mjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.mjs";
-import { r as DemoRenderer } from "../demos-B0pVQjYC.mjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-DqCoth_w.mjs";
+import { r as DemoRenderer } from "../demos-1b7JiUue.mjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Ck6-e_8o.mjs";
 //#region src/bootstrap-search/optimize.d.ts
 /**
  * Where a candidate's demo block came from. `zeroShot` holds no demos at all,
@@ -35,6 +35,18 @@ interface BootstrapSearchConfig {
    * every candidate, which is the reliable reading and the expensive one.
    */
   stopAtScore?: number;
+  /**
+   * How many candidates may be swept at once. Default 1.
+   *
+   * Harvesting stays in plan order however this is set — every harvest draws
+   * from the same random stream, and reordering them would make a seeded run
+   * unreproducible — so what overlaps is a sweep with the harvest of the
+   * candidates behind it. Two costs come with raising it: a checkpoint is
+   * taken per wave rather than per candidate, so a killed run loses up to this
+   * many candidates instead of one, and `stopAtScore` is honoured by sweeping
+   * one at a time, since a wave cannot know it has already passed the target.
+   */
+  concurrency?: number;
   seed?: number;
   trackBestOutputs?: boolean;
   /**
@@ -61,6 +73,13 @@ interface BootstrapSearchSnapshot {
   metricCalls: number;
   bootstrapMetricCalls: number;
   cacheHits: number;
+  /** Usage already spent, so a resumed run reports totals and honours ceilings. */
+  usage?: UsageTotals;
+  /**
+   * Candidates accepted so far. Reporters key rows by this id, so restarting it
+   * at zero makes a resumed run collide with the run it continues.
+   */
+  acceptedCandidates?: number;
   rngState: number;
   cache?: [string, CachedScore][];
 }
@@ -92,7 +111,8 @@ interface BootstrapSearchTask<Datum, Trajectory = unknown, Output = unknown, K e
   }) => string;
   /** Pass `false` to disable caching entirely. */
   cache?: EvaluationCache | false;
-  onEvent?: (event: BootstrapSearchEvent<NoInfer<K>>) => void;
+  /** Observers of the run. Every one sees every event; none can fail it. */
+  reporters?: readonly Reporter<BootstrapSearchEvent<NoInfer<K>>>[];
   /** Called with a resumable snapshot after every candidate is scored. */
   onCheckpoint?: (snapshot: BootstrapSearchSnapshot) => void | Promise<void>;
   /** Snapshot to continue from. */
@@ -112,13 +132,15 @@ type BootstrapSearchEvent<K extends string = string> = {
   demos: number;
   score: number;
   accepted: boolean;
-} | {
+} | ({
+  type: "candidateAccepted";
+  /** Which construction produced the demo block that won. */
+  source: DemoSource;
+  demos: number;
+} & CandidateAccepted<K>) | ({
   type: "finish";
   reason: BootstrapSearchStopReason;
-  bestScore: number;
-  metricCalls: number;
-  testScore?: number;
-};
+} & RunFinished);
 interface BootstrapSearchResult<K extends string = string, Output = unknown> extends OptimizerResult<K, BootstrapSearchStopReason, Output> {
   /** The seed's score, so the lift the demos bought is readable directly. */
   seedScore: number;
@@ -159,4 +181,4 @@ declare class BootstrapSearchOptimizer implements Optimizer<BootstrapSearchStopR
   optimize<Datum, Trajectory = unknown, Output = unknown, const K extends string = string>(task: BootstrapSearchTask<Datum, Trajectory, Output, K>): Promise<BootstrapSearchResult<K, Output>>;
 }
 //#endregion
-export { type BootstrapCandidate, type BootstrapSearchConfig, type BootstrapSearchEvent, BootstrapSearchOptimizer, type BootstrapSearchResult, type BootstrapSearchSnapshot, type BootstrapSearchStopReason, type BootstrapSearchTask, type DemoSource };
+export { type BootstrapCandidate, type BootstrapSearchConfig, type BootstrapSearchEvent, BootstrapSearchOptimizer, type BootstrapSearchResult, type BootstrapSearchSnapshot, type BootstrapSearchStopReason, type BootstrapSearchTask, type CandidateAccepted, type DemoSource, type OptimizerEvent, type ReportableEvent, type Reporter, type RunFinished, isCandidateAccepted, isRunFinished };

package/dist/bootstrap-search/index.mjs

@@ -1,5 +1,5 @@
-import { a as requireMeasuredMean, d as createMemoryCache, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "../evaluation-BV0nSZVx.mjs";
-import { n as formatDemos, t as bootstrapDemos } from "../demos-Degx6UmP.mjs";
+import { _ as createMemoryCache, a as isRunFinished, b as createBudget, c as createEvaluator, d as createDeadline, f as assertResumable, g as candidateHash, h as componentNames, i as isCandidateAccepted, l as measuredMean, m as runFingerprint, n as flushReporters, o as BudgetExhausted, r as instanceRow, s as costExhausted, t as createEmitter, u as requireMeasuredMean, v as defaultInstanceId } from "../reporting-DY-DC4HG.mjs";
+import { n as harvestFewShotExamples, t as formatDemos } from "../demos-DE2oxNWX.mjs";
 import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
 //#region src/bootstrap-search/optimize.ts
 const DEFAULT_CANDIDATES = 16;
@@ -33,19 +33,25 @@ const DEFAULT_MAX_LABELED_DEMOS = 16;
 var BootstrapSearchOptimizer = class {
 	#config;
 	constructor(config = {}) {
+		assertBootstrapSearchConfig(config);
 		this.#config = config;
 	}
-	optimize(task) {
-		return run({
-			config: this.#config,
-			task
-		});
+	async optimize(task) {
+		try {
+			return await run({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await flushReporters(task.reporters ?? []);
+		}
 	}
 };
 async function run(args) {
 	const { config, task } = args;
-	const { candidates: shuffledHarvests = DEFAULT_CANDIDATES, maxDemos = DEFAULT_MAX_DEMOS, minDemos = DEFAULT_MIN_DEMOS, maxLabeledDemos = DEFAULT_MAX_LABELED_DEMOS, demoMinScore, stopAtScore, seed = 0, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, demoComponents, renderDemo, goldOutput, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
+	const { candidates: shuffledHarvests = DEFAULT_CANDIDATES, maxDemos = DEFAULT_MAX_DEMOS, minDemos = DEFAULT_MIN_DEMOS, maxLabeledDemos = DEFAULT_MAX_LABELED_DEMOS, demoMinScore, stopAtScore, concurrency = 1, seed = 0, trackBestOutputs = false, checkpointCache = true } = config;
+	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, demoComponents, renderDemo, goldOutput, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const emit = createEmitter(reporters);
 	const deadline = createDeadline({ maxWallClockMs });
 	const components = componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
@@ -88,8 +94,9 @@ async function run(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
-		onEvaluation: (event) => onEvent?.({
+		onEvaluation: (event) => emit({
 			type: "evaluation",
 			...event
 		})
@@ -99,7 +106,7 @@ async function run(args) {
 	let drawn = resumeFrom?.drawn ?? 0;
 	let bootstrapMetricCalls = resumeFrom?.bootstrapMetricCalls ?? 0;
 	let stopReason = "candidatesExhausted";
-	onEvent?.({
+	emit({
 		type: "start",
 		components,
 		validationSetSize: validationSet.length
@@ -115,13 +122,25 @@ async function run(args) {
 			iteration: evaluated.length
 		});
 	}
-	const seedScore = resumeFrom?.seedScore ?? requireMeasuredMean({
-		batch: await sweep(seedCandidate, "seed"),
+	const seedEvaluation = resumeFrom === void 0 ? await sweep(seedCandidate, "seed") : void 0;
+	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : requireMeasuredMean({
+		batch: seedEvaluation,
 		phase: "seed"
 	});
 	let best = resumeFrom?.best ?? seedCandidate;
 	let bestScore = resumeFrom?.bestScore ?? seedScore;
 	let bestOutputs;
+	let acceptedCandidates = resumeFrom?.acceptedCandidates ?? 0;
+	if (seedEvaluation !== void 0) emit({
+		type: "candidateAccepted",
+		source: "zeroShot",
+		demos: 0,
+		candidateId: 0,
+		candidate: seedCandidate,
+		aggregateScore: seedScore,
+		instanceScores: instanceRow(seedEvaluation),
+		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
+	});
 	function takeSnapshot() {
 		const cached = checkpointCache ? evaluationCache?.entries?.() : void 0;
 		return {
@@ -135,6 +154,8 @@ async function run(args) {
 			metricCalls: budget.spent(),
 			bootstrapMetricCalls,
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
+			acceptedCandidates,
 			rngState: rng.state(),
 			...cached === void 0 ? {} : { cache: cached }
 		};
@@ -147,7 +168,8 @@ async function run(args) {
 		shuffledHarvests,
 		labeled: goldOutput !== void 0
 	});
-	for (; drawn < plan.length; drawn += 1) {
+	const waveSize = stopAtScore === void 0 ? concurrency : 1;
+	while (drawn < plan.length) {
 		if (signal?.aborted) {
 			stopReason = "aborted";
 			break;
@@ -163,58 +185,95 @@ async function run(args) {
 			stopReason = "deadlineReached";
 			break;
 		}
-		if (!budget.canAfford(validationSet.length + 1)) {
-			stopReason = "budgetExhausted";
-			break;
-		}
-		const source = plan[drawn];
-		const block = await buildBlock(source);
-		const candidate = withDemos(block);
-		let evaluation;
-		try {
-			evaluation = await sweep(candidate, "validation");
-		} catch (err) {
-			if (err instanceof BudgetExhausted) {
-				stopReason = "budgetExhausted";
-				break;
-			}
-			if (signal?.aborted) {
-				stopReason = "aborted";
+		const wave = [];
+		const inFlight = /* @__PURE__ */ new Map();
+		let waveStop;
+		while (wave.length < waveSize && drawn + wave.length < plan.length) {
+			if (!budget.canAfford(validationSet.length + 1)) {
+				waveStop = "budgetExhausted";
 				break;
 			}
-			throw err;
+			const source = plan[drawn + wave.length];
+			const block = await buildBlock(source);
+			const candidate = withDemos(block);
+			const key = candidateHash(candidate);
+			const prior = inFlight.get(key);
+			const sweeping = prior === void 0 ? settled(sweep(candidate, "validation")) : prior.then(() => settled(sweep(candidate, "validation")));
+			inFlight.set(key, sweeping);
+			wave.push({
+				source,
+				candidate,
+				block,
+				sweep: sweeping
+			});
 		}
-		const score = measuredMean(evaluation);
-		if (score === void 0) continue;
-		const accepted = score > bestScore;
-		evaluated.push({
-			candidate,
-			source,
-			demos: countDemos(block),
-			score
-		});
-		onEvent?.({
-			type: "candidate",
-			index: evaluated.length - 1,
-			source,
-			demos: countDemos(block),
-			score,
-			accepted
-		});
-		if (accepted) {
-			best = candidate;
-			bestScore = score;
-			bestOutputs = evaluation.outputs;
+		let waveError;
+		let draining = false;
+		for (const entry of wave) {
+			const outcome = await entry.sweep;
+			if (draining) continue;
+			drawn += 1;
+			if (outcome.failed === true) {
+				draining = true;
+				if (outcome.err instanceof BudgetExhausted) {
+					waveStop = "budgetExhausted";
+					continue;
+				}
+				if (signal?.aborted) {
+					waveStop = "aborted";
+					continue;
+				}
+				waveError = outcome.err;
+				continue;
+			}
+			const evaluation = outcome.evaluation;
+			const score = measuredMean(evaluation);
+			if (score === void 0) continue;
+			const accepted = score > bestScore;
+			evaluated.push({
+				candidate: entry.candidate,
+				source: entry.source,
+				demos: countDemos(entry.block),
+				score
+			});
+			emit({
+				type: "candidate",
+				index: evaluated.length - 1,
+				source: entry.source,
+				demos: countDemos(entry.block),
+				score,
+				accepted
+			});
+			if (accepted) {
+				acceptedCandidates += 1;
+				emit({
+					type: "candidateAccepted",
+					source: entry.source,
+					demos: countDemos(entry.block),
+					candidateId: acceptedCandidates,
+					candidate: entry.candidate,
+					aggregateScore: score,
+					instanceScores: instanceRow(evaluation),
+					...trackBestOutputs ? { outputs: evaluation.outputs } : {}
+				});
+				best = entry.candidate;
+				bestScore = score;
+				bestOutputs = evaluation.outputs;
+			}
+			if (stopAtScore !== void 0 && score >= stopAtScore) {
+				waveStop = "scoreReached";
+				draining = true;
+			}
 		}
+		if (waveError !== void 0) throw waveError;
 		await checkpoint();
-		if (stopAtScore !== void 0 && score >= stopAtScore) {
-			stopReason = "scoreReached";
-			drawn += 1;
+		if (waveStop !== void 0) {
+			stopReason = waveStop;
 			break;
 		}
 	}
 	if (signal?.aborted) stopReason = "aborted";
-	const testScore = testSet === void 0 ? void 0 : measuredMean(await evaluator.evaluate({
+	const heldOut = testSet === void 0 ? void 0 : await evaluator.evaluate({
 		candidate: best,
 		batch: testSet,
 		ids: testIds,
@@ -223,13 +282,17 @@ async function run(args) {
 		candidateId: null,
 		iteration: evaluated.length,
 		charge: false
-	}));
-	onEvent?.({
+	});
+	const testScore = heldOut === void 0 ? void 0 : measuredMean(heldOut);
+	emit({
 		type: "finish",
 		reason: stopReason,
+		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
 		bestCandidate: best,
@@ -244,7 +307,8 @@ async function run(args) {
 		...trackBestOutputs && bestOutputs !== void 0 ? { bestOutputs } : {},
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: testSet?.length ?? 0
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		stopReason
 	};
@@ -259,19 +323,21 @@ async function run(args) {
 		const requested = source === "unshuffled" ? maxDemos : minDemos + rng.nextInt(Math.max(1, maxDemos - minDemos + 1));
 		const affordable = Math.min(trainingSet.length, budget.remaining() - validationSet.length);
 		if (affordable < 1) return "";
-		const harvest = await bootstrapDemos({
+		const harvest = await harvestFewShotExamples({
 			adapter,
 			candidate: seedCandidate,
 			trainingSet,
 			...demoMinScore === void 0 ? {} : { minScore: demoMinScore },
 			maxDemos: requested,
 			maxMetricCalls: affordable,
+			...maxCostUsd === void 0 ? {} : { maxCostUsd: maxCostUsd - evaluator.usage().costUsd },
 			...source === "unshuffled" ? {} : { rng },
 			...renderDemo === void 0 ? {} : { renderDemo },
 			...signal === void 0 ? {} : { signal }
 		});
 		bootstrapMetricCalls += harvest.metricCalls;
 		budget.reserve(harvest.metricCalls);
+		evaluator.absorbUsage(harvest.usage);
 		return harvest.block;
 	}
 	function labeledBlock() {
@@ -297,11 +363,23 @@ function candidatePlan(args) {
 		...Array.from({ length: shuffledHarvests }, () => "bootstrapped")
 	];
 }
+/** Turns a sweep into a value, so a dispatched one never rejects unobserved. */
+function settled(sweeping) {
+	return sweeping.then((evaluation) => ({ evaluation }), (err) => ({
+		failed: true,
+		err
+	}));
+}
+/**
+* Range checks on the search knobs, run at construction so a configuration
+* that could never terminate is refused before a task is ever handed to it.
+*/
+function assertBootstrapSearchConfig(config) {
+	const { concurrency = 1 } = config;
+	if (!Number.isInteger(concurrency) || concurrency < 1) throw new Error(`concurrency must be a positive integer, received ${concurrency}`);
+}
 function countDemos(block) {
 	return block.split("<demo>").length - 1;
 }
-function defaultInstanceId(args) {
-	return String(args.index);
-}
 //#endregion
-export { BootstrapSearchOptimizer };
+export { BootstrapSearchOptimizer, isCandidateAccepted, isRunFinished };

package/dist/{demos-B0pVQjYC.d.mts → demos-1b7JiUue.d.mts}

@@ -1,4 +1,4 @@
-import { n as Candidate, t as Adapter } from "./types-CWv4IQFF.mjs";
+import { c as Adapter, l as Candidate, v as UsageTotals } from "./reporting-DQbAohc9.mjs";
 import { t as Rng } from "./rng-BR5MOedA.mjs";
 //#region src/demos.d.ts
 /**
@@ -23,6 +23,8 @@ interface BootstrapResult<Datum, Output> {
   block: string;
   /** Rollouts this cost. Bootstrapping is cheap, not free. */
   metricCalls: number;
+  /** Tokens and dollars this cost, for a caller that bounds spend. */
+  usage: UsageTotals;
   attempted: number;
 }
 /**
@@ -36,7 +38,7 @@ interface BootstrapResult<Datum, Output> {
  * examples on what the output should look like — so a seed carrying both
  * starts somewhere neither reaches alone.
  */
-declare function bootstrapDemos<Datum, Trajectory, Output, K extends string = string>(args: {
+declare function harvestFewShotExamples<Datum, Trajectory, Output, K extends string = string>(args: {
   adapter: Adapter<Datum, Trajectory, Output, K>;
   /** The candidate to run. Usually the seed, sometimes a run's winner. */
   candidate: Candidate<K>;
@@ -61,6 +63,11 @@ declare function bootstrapDemos<Datum, Trajectory, Output, K extends string = st
   batchSize?: number;
   /** Ceiling on rollouts. Defaults to one pass over the trainingSet. */
   maxMetricCalls?: number;
+  /**
+   * Ceiling on dollars this pass may spend, checked between batches. Harvesting
+   * runs on its own evaluator, so a caller bounding spend has to say so here.
+   */
+  maxCostUsd?: number;
   /** Shuffles the trainingSet first, so demos are not all drawn from its head. */
   rng?: Rng;
   renderDemo?: DemoRenderer<Datum, Output>;
@@ -85,4 +92,4 @@ declare function formatDemos<Datum, Output>(demos: readonly Demo<Datum, Output>[
  */
 declare function parseDemos(text: string): Demo[];
 //#endregion
-export { formatDemos as a, bootstrapDemos as i, Demo as n, parseDemos as o, DemoRenderer as r, BootstrapResult as t };
+export { harvestFewShotExamples as a, formatDemos as i, Demo as n, parseDemos as o, DemoRenderer as r, BootstrapResult as t };

package/dist/{demos-BTuzFNsp.d.cts → demos-CU9dy2oT.d.cts}

@@ -1,4 +1,4 @@
-import { n as Candidate, t as Adapter } from "./types-CWv4IQFF.cjs";
+import { c as Adapter, l as Candidate, v as UsageTotals } from "./reporting-DQbAohc9.cjs";
 import { t as Rng } from "./rng-BR5MOedA.cjs";
 //#region src/demos.d.ts
 /**
@@ -23,6 +23,8 @@ interface BootstrapResult<Datum, Output> {
   block: string;
   /** Rollouts this cost. Bootstrapping is cheap, not free. */
   metricCalls: number;
+  /** Tokens and dollars this cost, for a caller that bounds spend. */
+  usage: UsageTotals;
   attempted: number;
 }
 /**
@@ -36,7 +38,7 @@ interface BootstrapResult<Datum, Output> {
  * examples on what the output should look like — so a seed carrying both
  * starts somewhere neither reaches alone.
  */
-declare function bootstrapDemos<Datum, Trajectory, Output, K extends string = string>(args: {
+declare function harvestFewShotExamples<Datum, Trajectory, Output, K extends string = string>(args: {
   adapter: Adapter<Datum, Trajectory, Output, K>;
   /** The candidate to run. Usually the seed, sometimes a run's winner. */
   candidate: Candidate<K>;
@@ -61,6 +63,11 @@ declare function bootstrapDemos<Datum, Trajectory, Output, K extends string = st
   batchSize?: number;
   /** Ceiling on rollouts. Defaults to one pass over the trainingSet. */
   maxMetricCalls?: number;
+  /**
+   * Ceiling on dollars this pass may spend, checked between batches. Harvesting
+   * runs on its own evaluator, so a caller bounding spend has to say so here.
+   */
+  maxCostUsd?: number;
   /** Shuffles the trainingSet first, so demos are not all drawn from its head. */
   rng?: Rng;
   renderDemo?: DemoRenderer<Datum, Output>;
@@ -85,4 +92,4 @@ declare function formatDemos<Datum, Output>(demos: readonly Demo<Datum, Output>[
  */
 declare function parseDemos(text: string): Demo[];
 //#endregion
-export { formatDemos as a, bootstrapDemos as i, Demo as n, parseDemos as o, DemoRenderer as r, BootstrapResult as t };
+export { harvestFewShotExamples as a, formatDemos as i, Demo as n, parseDemos as o, DemoRenderer as r, BootstrapResult as t };