textopt 0.0.0 → 0.1.0

package/dist/mipro/index.cjs

@@ -1,8 +1,8 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_evaluation = require("../evaluation-OZOp6TB7.cjs");
+const require_reporting = require("../reporting-CNHzbJC-.cjs");
 const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
-const require_math = require("../math-COOofUyv.cjs");
-const require_demos = require("../demos-B9BJiNKz.cjs");
+const require_math = require("../math-BhlziRPc.cjs");
+const require_demos = require("../demos-D2o0qSSo.cjs");
 const require_text = require("../text-CK_HB3su.cjs");
 const require_rng = require("../rng-DbA_rPIo.cjs");
 const require_sampling = require("../sampling-Dars7ctR.cjs");
@@ -212,10 +212,14 @@ var MiproOptimizer = class {
 		this.#config = config;
 	}
 	async optimize(task) {
-		return runMipro({
-			config: this.#config,
-			task
-		});
+		try {
+			return await runMipro({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await require_reporting.flushReporters(task.reporters ?? []);
+		}
 	}
 };
 function buildMiproPrompt(args) {
@@ -274,9 +278,10 @@ function buildDatasetSummaryPrompt(examples) {
 async function runMipro(args) {
 	const { config, task } = args;
 	const { instructionsPerComponent = DEFAULT_INSTRUCTIONS, minibatchSize = DEFAULT_MINIBATCH_SIZE, maxTrials = DEFAULT_MAX_TRIALS, startupTrials, gamma, surrogateSamples, multivariate, fullEvalInterval = DEFAULT_FULL_EVAL_INTERVAL, demoSets = DEFAULT_DEMO_SETS, maxDemos = DEFAULT_MAX_DEMOS, demoMinScore, exemplars = DEFAULT_EXEMPLARS, datasetSummary = true, summaryExamples = DEFAULT_SUMMARY_EXAMPLES, concurrency = 1, seed = 0, buildPrompt = buildMiproPrompt, tips = DEFAULT_TIPS, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, componentOptions, demoComponents, renderDemo, goldOutput, maxMetricCalls, renderDatum = renderDefault, batchSampler = require_sampling.createEpochShuffledSampler({ minibatchSize }), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
-	const deadline = require_evaluation.createDeadline({ maxWallClockMs });
-	const components = require_evaluation.componentNames(seedCandidate);
+	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, componentOptions, demoComponents, renderDemo, goldOutput, maxMetricCalls, renderDatum = renderDefault, batchSampler = require_sampling.createEpochShuffledSampler({ minibatchSize }), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_reporting.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const emit = require_reporting.createEmitter(reporters);
+	const deadline = require_reporting.createDeadline({ maxWallClockMs });
+	const components = require_reporting.componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
 	if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
 	if (components.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
@@ -294,24 +299,24 @@ async function runMipro(args) {
 		datum,
 		index
 	})) ?? [];
-	const fingerprint = require_evaluation.runFingerprint({
+	const fingerprint = require_reporting.runFingerprint({
 		seedCandidate,
 		trainingIds,
 		validationIds,
 		seed,
 		...cacheNamespace === void 0 ? {} : { cacheNamespace }
 	});
-	require_evaluation.assertResumable({
+	require_reporting.assertResumable({
 		fingerprint,
 		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
 	});
 	const rng = require_rng.createSeededRng(seed, resumeFrom?.rngState);
-	const budget = require_evaluation.createBudget({
+	const budget = require_reporting.createBudget({
 		maxMetricCalls,
 		spent: resumeFrom?.metricCalls ?? 0
 	});
-	const evaluationCache = cache === false ? void 0 : cache ?? require_evaluation.createMemoryCache();
-	const evaluator = require_evaluation.createEvaluator({
+	const evaluationCache = cache === false ? void 0 : cache ?? require_reporting.createMemoryCache();
+	const evaluator = require_reporting.createEvaluator({
 		adapter,
 		budget,
 		...retry === void 0 ? {} : { retry },
@@ -319,15 +324,16 @@ async function runMipro(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
-		onEvaluation: (event) => onEvent?.({
+		onEvaluation: (event) => emit({
 			type: "evaluation",
 			...event
 		})
 	});
 	evaluator.restore(resumeFrom?.cache ?? []);
 	if (resumeFrom?.sampler !== void 0) batchSampler.restore?.(resumeFrom.sampler);
-	onEvent?.({
+	emit({
 		type: "start",
 		components,
 		validationSetSize: validationSet.length
@@ -374,20 +380,26 @@ async function runMipro(args) {
 		for (let index = 0; index < demoSets; index += 1) {
 			const affordable = Math.min(trainingSet.length, budget.remaining() - validationSet.length);
 			if (affordable < 1) break;
+			if (require_reporting.costExhausted({
+				usage: evaluator.usage(),
+				maxCostUsd
+			})) break;
 			const requested = demoSets === 1 ? maxDemos : Math.round(1 + index * (maxDemos - 1) / (demoSets - 1));
-			const harvest = await require_demos.bootstrapDemos({
+			const harvest = await require_demos.harvestFewShotExamples({
 				adapter,
 				candidate: seedCandidate,
 				trainingSet,
 				...demoMinScore === void 0 ? {} : { minScore: demoMinScore },
 				maxDemos: requested,
 				maxMetricCalls: affordable,
+				...maxCostUsd === void 0 ? {} : { maxCostUsd: maxCostUsd - evaluator.usage().costUsd },
 				rng,
 				...renderDemo === void 0 ? {} : { renderDemo },
 				...signal === void 0 ? {} : { signal }
 			});
 			bootstrapMetricCalls += harvest.metricCalls;
 			budget.reserve(harvest.metricCalls);
+			evaluator.absorbUsage(harvest.usage);
 			if (harvest.demos.length > 0) blocks.push(require_demos.formatDemos(harvest.demos, renderDemo === void 0 ? {} : { render: renderDemo }));
 		}
 		return [.../* @__PURE__ */ new Set([
@@ -432,7 +444,7 @@ async function runMipro(args) {
 		});
 		menu[name] = [seedCandidate[name], ...drawn.filter((text) => text.length > 0)];
 	}
-	onEvent?.({
+	emit({
 		type: "menu",
 		menu,
 		reflectionCalls
@@ -462,11 +474,20 @@ async function runMipro(args) {
 		});
 	}
 	const seedEvaluation = resumeFrom === void 0 ? await fullSweep(seedCandidate, "seed") : void 0;
-	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_evaluation.requireMeasuredMean({
+	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_reporting.requireMeasuredMean({
 		batch: seedEvaluation,
 		phase: "seed"
 	});
 	if (seedEvaluation !== void 0) fullEvaluations += 1;
+	if (seedEvaluation !== void 0) emit({
+		type: "candidateAccepted",
+		trial: 0,
+		candidateId: 0,
+		candidate: seedCandidate,
+		aggregateScore: seedScore,
+		instanceScores: require_reporting.instanceRow(seedEvaluation),
+		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
+	});
 	if (resumeFrom === void 0) surrogateInput.push({
 		choices: menuSizes.map(() => 0),
 		score: seedScore
@@ -474,6 +495,7 @@ async function runMipro(args) {
 	let best = resumeFrom?.best ?? seedCandidate;
 	let bestScore = resumeFrom?.bestScore ?? seedScore;
 	/** Absent on a resumed run until a sweep wins: outputs are not checkpointed. */
+	let acceptedCandidates = resumeFrom?.acceptedCandidates ?? 0;
 	let bestOutputs = seedEvaluation?.outputs;
 	const readings = new Map(resumeFrom?.readings ?? []);
 	const swept = new Set(resumeFrom?.swept ?? []);
@@ -493,6 +515,8 @@ async function runMipro(args) {
 			bootstrapMetricCalls,
 			metricCalls: budget.spent(),
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
+			acceptedCandidates,
 			rngState: rng.state(),
 			observations: [...observations],
 			surrogateInput: surrogateInput.map((entry) => ({
@@ -535,12 +559,12 @@ async function runMipro(args) {
 		try {
 			evaluation = await fullSweep(candidate, "validation");
 		} catch (err) {
-			if (err instanceof require_evaluation.BudgetExhausted) return "budgetExhausted";
+			if (err instanceof require_reporting.BudgetExhausted) return "budgetExhausted";
 			if (signal?.aborted) return "aborted";
 			throw err;
 		}
 		fullEvaluations += 1;
-		const score = require_evaluation.measuredMean(evaluation);
+		const score = require_reporting.measuredMean(evaluation);
 		if (score === void 0) return "swept";
 		surrogateInput.push({
 			choices,
@@ -554,11 +578,15 @@ async function runMipro(args) {
 			best = candidate;
 			bestScore = score;
 			bestOutputs = evaluation.outputs;
-			onEvent?.({
-				type: "incumbent",
+			acceptedCandidates += 1;
+			emit({
+				type: "candidateAccepted",
 				trial,
-				choices,
-				score
+				candidateId: acceptedCandidates,
+				candidate,
+				aggregateScore: score,
+				instanceScores: require_reporting.instanceRow(evaluation),
+				...trackBestOutputs ? { outputs: evaluation.outputs } : {}
 			});
 		}
 		return "swept";
@@ -568,7 +596,7 @@ async function runMipro(args) {
 			stopReason = "aborted";
 			break;
 		}
-		if (require_evaluation.costExhausted({
+		if (require_reporting.costExhausted({
 			usage: evaluator.usage(),
 			maxCostUsd
 		})) {
@@ -613,9 +641,9 @@ async function runMipro(args) {
 				candidateId: null,
 				iteration: trial
 			});
-			minibatchScore = require_evaluation.measuredMean(evaluation);
+			minibatchScore = require_reporting.measuredMean(evaluation);
 		} catch (err) {
-			if (err instanceof require_evaluation.BudgetExhausted) {
+			if (err instanceof require_reporting.BudgetExhausted) {
 				stopReason = "budgetExhausted";
 				break;
 			}
@@ -625,47 +653,44 @@ async function runMipro(args) {
 			}
 			throw err;
 		}
-		if (minibatchScore === void 0) {
-			trial += 1;
-			continue;
+		if (minibatchScore !== void 0) {
+			surrogateInput.push({
+				choices,
+				score: minibatchScore
+			});
+			const key = choices.join(",");
+			readings.set(key, [...readings.get(key) ?? [], minibatchScore]);
+			const observation = {
+				trial,
+				choices,
+				minibatchScore,
+				promoted: false
+			};
+			observations.push(observation);
+			emit({
+				type: "trial",
+				trial,
+				choices,
+				minibatchScore,
+				promoted: false
+			});
 		}
-		surrogateInput.push({
-			choices,
-			score: minibatchScore
-		});
-		const key = choices.join(",");
-		readings.set(key, [...readings.get(key) ?? [], minibatchScore]);
-		const observation = {
-			trial,
-			choices,
-			minibatchScore,
-			promoted: false
-		};
-		observations.push(observation);
-		onEvent?.({
-			type: "trial",
-			trial,
-			choices,
-			minibatchScore,
-			promoted: false
-		});
 		trial += 1;
-		await checkpoint();
+		let cadenceStop;
 		if (trial % fullEvalInterval === 0) {
 			const outcome = await sweepBestUnswept();
-			if (outcome === "budgetExhausted" || outcome === "aborted") {
-				stopReason = outcome;
-				break;
-			}
-			if (outcome === "unaffordable") {
-				stopReason = "budgetExhausted";
-				break;
-			}
+			if (outcome === "budgetExhausted" || outcome === "aborted") cadenceStop = outcome;
+			if (outcome === "unaffordable") cadenceStop = "budgetExhausted";
+		}
+		await checkpoint();
+		if (cadenceStop !== void 0) {
+			stopReason = cadenceStop;
+			break;
 		}
 	}
 	if (stopReason === "maxTrials" && !signal?.aborted) await sweepBestUnswept();
 	if (signal?.aborted) stopReason = "aborted";
-	const testScore = testSet === void 0 ? void 0 : require_evaluation.measuredMean(await evaluator.evaluate({
+	const heldOut = testSet === void 0 ? void 0 : await evaluator.evaluate({
 		candidate: best,
 		batch: testSet,
 		ids: testIds,
@@ -674,13 +699,17 @@ async function runMipro(args) {
 		candidateId: null,
 		iteration: trial,
 		charge: false
-	}));
-	onEvent?.({
+	});
+	const testScore = heldOut === void 0 ? void 0 : require_reporting.measuredMean(heldOut);
+	emit({
 		type: "finish",
 		reason: stopReason,
+		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: require_reporting.instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
 		snapshot: takeSnapshot(),
@@ -691,7 +720,8 @@ async function runMipro(args) {
 		...trackBestOutputs ? { bestOutputs } : {},
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: evaluator.unchargedCalls()
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		trials: trial,
 		menu,
@@ -729,11 +759,9 @@ function assertConfig(config) {
 	if (config.gamma !== void 0 && (!Number.isFinite(config.gamma) || config.gamma <= 0 || config.gamma > 1)) throw new Error(`gamma must be within (0, 1], received ${config.gamma}`);
 	if (config.tips !== void 0 && config.tips.length === 0) throw new Error("tips must not be empty");
 }
-function defaultInstanceId(args) {
-	const hash = require_evaluation.stableHash(args.datum);
-	return hash === "" ? String(args.index) : hash;
-}
 //#endregion
 exports.MiproOptimizer = MiproOptimizer;
 exports.buildMiproPrompt = buildMiproPrompt;
+exports.isCandidateAccepted = require_reporting.isCandidateAccepted;
+exports.isRunFinished = require_reporting.isRunFinished;
 exports.proposeConfiguration = proposeConfiguration;

package/dist/mipro/index.d.cts

@@ -1,8 +1,8 @@
-import { n as Candidate, t as Adapter, u as TextModel } from "../types-CWv4IQFF.cjs";
+import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.cjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.cjs";
 import { t as Rng } from "../rng-BR5MOedA.cjs";
-import { r as DemoRenderer } from "../demos-BTuzFNsp.cjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-B7SpRwl7.cjs";
+import { r as DemoRenderer } from "../demos-CU9dy2oT.cjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Bh5DPRMH.cjs";
 import { t as BatchSampler } from "../sampling-axOwfZf5.cjs";
 //#region src/mipro/optimize.d.ts
 type MiproPromptBuilder = (args: {
@@ -131,6 +131,13 @@ interface MiproSnapshot {
   bootstrapMetricCalls: number;
   metricCalls: number;
   cacheHits: number;
+  /** Usage already spent, so a resumed run reports totals and honours ceilings. */
+  usage?: UsageTotals;
+  /**
+   * Candidates accepted so far. Reporters key rows by this id, so restarting it
+   * at zero makes a resumed run collide with the run it continues.
+   */
+  acceptedCandidates?: number;
   rngState: number;
   observations: MiproObservation[];
   /** What the surrogate was fitted on: one entry per measured trial. */
@@ -187,7 +194,8 @@ interface MiproTask<Datum, Trajectory = unknown, Output = unknown, K extends str
   }) => string;
   /** Pass `false` to disable caching entirely. */
   cache?: EvaluationCache | false;
-  onEvent?: (event: MiproEvent<NoInfer<K>>) => void;
+  /** Observers of the run. Every one sees every event; none can fail it. */
+  reporters?: readonly Reporter<MiproEvent<NoInfer<K>>>[];
   /**
    * Called with a resumable snapshot once the menus are built and after every
    * trial. Persist it and a killed run costs the last trial, not the menus.
@@ -214,18 +222,13 @@ type MiproEvent<K extends string = string> = {
   minibatchScore: number;
   /** True when the trial earned a full validation sweep. */
   promoted: boolean;
-} | {
-  type: "incumbent";
+} | ({
+  type: "candidateAccepted";
   trial: number;
-  choices: number[];
-  score: number;
-} | {
+} & CandidateAccepted<K>) | ({
   type: "finish";
   reason: MiproStopReason;
-  bestScore: number;
-  metricCalls: number;
-  testScore?: number;
-};
+} & RunFinished);
 interface MiproObservation {
   trial: number;
   choices: number[];
@@ -369,4 +372,4 @@ declare function proposeConfiguration(args: {
   rng: Rng;
 }): number[];
 //#endregion
-export { type MiproConfig, type MiproEvent, type MiproObservation, MiproOptimizer, type MiproPromptBuilder, type MiproResult, type MiproSnapshot, type MiproStopReason, type MiproTask, type Observation, buildMiproPrompt, proposeConfiguration };
+export { type CandidateAccepted, type MiproConfig, type MiproEvent, type MiproObservation, MiproOptimizer, type MiproPromptBuilder, type MiproResult, type MiproSnapshot, type MiproStopReason, type MiproTask, type Observation, type OptimizerEvent, type ReportableEvent, type Reporter, type RunFinished, buildMiproPrompt, isCandidateAccepted, isRunFinished, proposeConfiguration };

package/dist/mipro/index.d.mts

@@ -1,8 +1,8 @@
-import { n as Candidate, t as Adapter, u as TextModel } from "../types-CWv4IQFF.mjs";
+import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.mjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.mjs";
 import { t as Rng } from "../rng-BR5MOedA.mjs";
-import { r as DemoRenderer } from "../demos-B0pVQjYC.mjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-DqCoth_w.mjs";
+import { r as DemoRenderer } from "../demos-1b7JiUue.mjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Ck6-e_8o.mjs";
 import { t as BatchSampler } from "../sampling-DFo_7RNJ.mjs";
 //#region src/mipro/optimize.d.ts
 type MiproPromptBuilder = (args: {
@@ -131,6 +131,13 @@ interface MiproSnapshot {
   bootstrapMetricCalls: number;
   metricCalls: number;
   cacheHits: number;
+  /** Usage already spent, so a resumed run reports totals and honours ceilings. */
+  usage?: UsageTotals;
+  /**
+   * Candidates accepted so far. Reporters key rows by this id, so restarting it
+   * at zero makes a resumed run collide with the run it continues.
+   */
+  acceptedCandidates?: number;
   rngState: number;
   observations: MiproObservation[];
   /** What the surrogate was fitted on: one entry per measured trial. */
@@ -187,7 +194,8 @@ interface MiproTask<Datum, Trajectory = unknown, Output = unknown, K extends str
   }) => string;
   /** Pass `false` to disable caching entirely. */
   cache?: EvaluationCache | false;
-  onEvent?: (event: MiproEvent<NoInfer<K>>) => void;
+  /** Observers of the run. Every one sees every event; none can fail it. */
+  reporters?: readonly Reporter<MiproEvent<NoInfer<K>>>[];
   /**
    * Called with a resumable snapshot once the menus are built and after every
    * trial. Persist it and a killed run costs the last trial, not the menus.
@@ -214,18 +222,13 @@ type MiproEvent<K extends string = string> = {
   minibatchScore: number;
   /** True when the trial earned a full validation sweep. */
   promoted: boolean;
-} | {
-  type: "incumbent";
+} | ({
+  type: "candidateAccepted";
   trial: number;
-  choices: number[];
-  score: number;
-} | {
+} & CandidateAccepted<K>) | ({
   type: "finish";
   reason: MiproStopReason;
-  bestScore: number;
-  metricCalls: number;
-  testScore?: number;
-};
+} & RunFinished);
 interface MiproObservation {
   trial: number;
   choices: number[];
@@ -369,4 +372,4 @@ declare function proposeConfiguration(args: {
   rng: Rng;
 }): number[];
 //#endregion
-export { type MiproConfig, type MiproEvent, type MiproObservation, MiproOptimizer, type MiproPromptBuilder, type MiproResult, type MiproSnapshot, type MiproStopReason, type MiproTask, type Observation, buildMiproPrompt, proposeConfiguration };
+export { type CandidateAccepted, type MiproConfig, type MiproEvent, type MiproObservation, MiproOptimizer, type MiproPromptBuilder, type MiproResult, type MiproSnapshot, type MiproStopReason, type MiproTask, type Observation, type OptimizerEvent, type ReportableEvent, type Reporter, type RunFinished, buildMiproPrompt, isCandidateAccepted, isRunFinished, proposeConfiguration };