textopt 0.1.0 → 0.2.0

package/dist/opro/index.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_reporting = require("../reporting-CNHzbJC-.cjs");
+const require_warnings = require("../warnings-CWRJF-jA.cjs");
 const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
 const require_text = require("../text-CK_HB3su.cjs");
 const require_rng = require("../rng-DbA_rPIo.cjs");
@@ -49,7 +49,7 @@ var OproOptimizer = class {
 				task
 			});
 		} finally {
-			await require_reporting.flushReporters(task.reporters ?? []);
+			await require_warnings.flushReporters(task.reporters ?? []);
 		}
 	}
 };
@@ -95,10 +95,14 @@ function buildOproPrompt(args) {
 async function runOpro(args) {
 	const { config, task } = args;
 	const { proposalsPerRound = DEFAULT_PROPOSALS_PER_ROUND, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, maxReflectionCalls = Number.POSITIVE_INFINITY, seed = 0, historySize = DEFAULT_HISTORY_SIZE, exemplars = DEFAULT_EXEMPLARS, scoringSetSize, fullEvalInterval = DEFAULT_FULL_EVAL_INTERVAL, scoreScale = DEFAULT_SCORE_SCALE, buildPrompt = buildOproPrompt, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, renderDatum = renderDefault, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_reporting.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
-	const emit = require_reporting.createEmitter(reporters);
-	const deadline = require_reporting.createDeadline({ maxWallClockMs });
-	const components = require_reporting.componentNames(seedCandidate);
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, maxMetricCalls, renderDatum = renderDefault, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_warnings.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = require_warnings.resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
+	const emit = require_warnings.createEmitter(reporters);
+	const deadline = require_warnings.createDeadline({ maxWallClockMs });
+	const components = require_warnings.componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
 	if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
 	if (components.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
@@ -111,7 +115,7 @@ async function runOpro(args) {
 		datum,
 		index
 	})) ?? [];
-	const fingerprint = require_reporting.runFingerprint({
+	const fingerprint = require_warnings.runFingerprint({
 		seedCandidate,
 		trainingIds: trainingSet.map((datum, index) => instanceId({
 			datum,
@@ -121,17 +125,17 @@ async function runOpro(args) {
 		seed,
 		...cacheNamespace === void 0 ? {} : { cacheNamespace }
 	});
-	require_reporting.assertResumable({
+	require_warnings.assertResumable({
 		fingerprint,
 		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
 	});
 	const rng = require_rng.createSeededRng(seed, resumeFrom?.rngState);
-	const budget = require_reporting.createBudget({
+	const budget = require_warnings.createBudget({
 		maxMetricCalls,
 		spent: resumeFrom?.metricCalls ?? 0
 	});
-	const evaluationCache = cache === false ? void 0 : cache ?? require_reporting.createMemoryCache();
-	const evaluator = require_reporting.createEvaluator({
+	const evaluationCache = cache === false ? void 0 : cache ?? require_warnings.createMemoryCache();
+	const evaluator = require_warnings.createEvaluator({
 		adapter,
 		budget,
 		...retry === void 0 ? {} : { retry },
@@ -160,7 +164,7 @@ async function runOpro(args) {
 	function contextOf(candidate, component) {
 		const rest = {};
 		for (const name of components) if (name !== component) rest[name] = candidate[name];
-		return require_reporting.stableHash(rest);
+		return require_warnings.stableHash(rest);
 	}
 	const trajectory = [];
 	let round = resumeFrom?.round ?? 0;
@@ -212,7 +216,7 @@ async function runOpro(args) {
 		});
 	}
 	const seedEvaluation = resumeFrom === void 0 ? await sweep(seedCandidate, "seed") : void 0;
-	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_reporting.requireMeasuredMean({
+	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_warnings.requireMeasuredMean({
 		batch: seedEvaluation,
 		phase: "seed"
 	});
@@ -222,7 +226,7 @@ async function runOpro(args) {
 		candidateId: 0,
 		candidate: seedCandidate,
 		aggregateScore: seedScore,
-		instanceScores: require_reporting.instanceRow(seedEvaluation),
+		instanceScores: require_warnings.instanceRow(seedEvaluation),
 		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
 	});
 	let best = resumeFrom?.best ?? seedCandidate;
@@ -233,7 +237,7 @@ async function runOpro(args) {
 	let reported = resumeFrom?.reported ?? seedCandidate;
 	let lastSwept = resumeFrom === void 0 ? seedCandidate : resumeFrom.incumbentSwept ? best : {};
 	let bestSearchScore = resumeFrom?.bestSearchScore ?? seedScore;
-	if (resumeFrom === void 0 && scoringSet !== void 0) bestSearchScore = require_reporting.requireMeasuredMean({
+	if (resumeFrom === void 0 && scoringSet !== void 0) bestSearchScore = require_warnings.requireMeasuredMean({
 		batch: await screen(seedCandidate, "seed"),
 		phase: "seed"
 	});
@@ -257,7 +261,7 @@ async function runOpro(args) {
 			candidateId: acceptedCandidates,
 			candidate: args.candidate,
 			aggregateScore: args.score,
-			instanceScores: require_reporting.instanceRow(args.evaluation),
+			instanceScores: require_warnings.instanceRow(args.evaluation),
 			...trackBestOutputs ? { outputs: args.evaluation.outputs } : {}
 		});
 	}
@@ -265,7 +269,7 @@ async function runOpro(args) {
 		if (best === lastSwept || !budget.canAfford(validationSet.length)) return "ok";
 		try {
 			const evaluation = await sweep(best, "validation");
-			const full = require_reporting.measuredMean(evaluation);
+			const full = require_warnings.measuredMean(evaluation);
 			lastSwept = best;
 			if (full !== void 0 && full > bestScore) {
 				reported = best;
@@ -278,7 +282,7 @@ async function runOpro(args) {
 				});
 			}
 		} catch (err) {
-			if (err instanceof require_reporting.BudgetExhausted || signal?.aborted) return "stop";
+			if (err instanceof require_warnings.BudgetExhausted || signal?.aborted) return "stop";
 			throw err;
 		}
 		return "ok";
@@ -334,7 +338,7 @@ async function runOpro(args) {
 			stopReason = "proposalsExhausted";
 			break;
 		}
-		if (require_reporting.costExhausted({
+		if (require_warnings.costExhausted({
 			usage: evaluator.usage(),
 			maxCostUsd
 		})) {
@@ -409,7 +413,7 @@ async function runOpro(args) {
 				try {
 					return { evaluation: await screen(candidate, "validation") };
 				} catch (err) {
-					if (err instanceof require_reporting.BudgetExhausted) return { stop: "budgetExhausted" };
+					if (err instanceof require_warnings.BudgetExhausted) return { stop: "budgetExhausted" };
 					if (signal?.aborted) return { stop: "aborted" };
 					throw err;
 				}
@@ -423,7 +427,7 @@ async function runOpro(args) {
 			const candidate = scheduled[index];
 			const text = candidate[component];
 			const evaluation = outcome.evaluation;
-			const score = require_reporting.measuredMean(evaluation);
+			const score = require_warnings.measuredMean(evaluation);
 			if (score === void 0) continue;
 			const accepted = score > bestSearchScore;
 			history.push({
@@ -494,15 +498,16 @@ async function runOpro(args) {
 		iteration: round,
 		charge: false
 	});
-	const testScore = heldOut === void 0 ? void 0 : require_reporting.measuredMean(heldOut);
+	const testScore = heldOut === void 0 ? void 0 : require_warnings.measuredMean(heldOut);
 	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
 		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
 		...testScore === void 0 ? {} : { testScore },
-		...heldOut === void 0 ? {} : { testInstanceScores: require_reporting.instanceRow(heldOut) },
+		...heldOut === void 0 ? {} : { testInstanceScores: require_warnings.instanceRow(heldOut) },
 		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
@@ -522,6 +527,7 @@ async function runOpro(args) {
 		metricCalls: budget.spent(),
 		reflectionCalls,
 		cacheHits: evaluator.cacheHits(),
+		warnings,
 		stopReason
 	};
 }
@@ -562,5 +568,5 @@ function assertConfig(config) {
 //#endregion
 exports.OproOptimizer = OproOptimizer;
 exports.buildOproPrompt = buildOproPrompt;
-exports.isCandidateAccepted = require_reporting.isCandidateAccepted;
-exports.isRunFinished = require_reporting.isRunFinished;
+exports.isCandidateAccepted = require_warnings.isCandidateAccepted;
+exports.isRunFinished = require_warnings.isRunFinished;

package/dist/opro/index.d.cts

@@ -1,6 +1,6 @@
-import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.cjs";
+import { S as UsageTotals, a as RunFinished, f as Adapter, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, p as Candidate, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, x as TextModel } from "../reporting-bq007_2z.cjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.cjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Bh5DPRMH.cjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-4Zv-Zt2t.cjs";
 //#region src/opro/optimize.d.ts
 /** One instruction that was tried, and what it scored. */
 /** A history entry plus the system state its score was measured in. */

package/dist/opro/index.d.mts

@@ -1,6 +1,6 @@
-import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.mjs";
+import { S as UsageTotals, a as RunFinished, f as Adapter, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, p as Candidate, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, x as TextModel } from "../reporting-bq007_2z.mjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.mjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Ck6-e_8o.mjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Ds5mzYjz.mjs";
 //#region src/opro/optimize.d.ts
 /** One instruction that was tried, and what it scored. */
 /** A history entry plus the system state its score was measured in. */

package/dist/opro/index.mjs

@@ -1,4 +1,4 @@
-import { _ as createMemoryCache, a as isRunFinished, b as createBudget, c as createEvaluator, d as createDeadline, f as assertResumable, h as componentNames, i as isCandidateAccepted, l as measuredMean, m as runFingerprint, n as flushReporters, o as BudgetExhausted, r as instanceRow, s as costExhausted, t as createEmitter, u as requireMeasuredMean, v as defaultInstanceId, y as stableHash } from "../reporting-DY-DC4HG.mjs";
+import { S as createBudget, _ as componentNames, a as instanceRow, b as defaultInstanceId, c as BudgetExhausted, d as measuredMean, f as requireMeasuredMean, g as runFingerprint, i as flushReporters, l as costExhausted, m as assertResumable, o as isCandidateAccepted, p as createDeadline, r as createEmitter, s as isRunFinished, t as resolveValidationSet, u as createEvaluator, x as stableHash, y as createMemoryCache } from "../warnings-OxvDi9kN.mjs";
 import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
 import { t as parseProposedText } from "../text--v4Ffbus.mjs";
 import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
@@ -94,7 +94,11 @@ function buildOproPrompt(args) {
 async function runOpro(args) {
 	const { config, task } = args;
 	const { proposalsPerRound = DEFAULT_PROPOSALS_PER_ROUND, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, maxReflectionCalls = Number.POSITIVE_INFINITY, seed = 0, historySize = DEFAULT_HISTORY_SIZE, exemplars = DEFAULT_EXEMPLARS, scoringSetSize, fullEvalInterval = DEFAULT_FULL_EVAL_INTERVAL, scoreScale = DEFAULT_SCORE_SCALE, buildPrompt = buildOproPrompt, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, renderDatum = renderDefault, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, maxMetricCalls, renderDatum = renderDefault, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
 	const emit = createEmitter(reporters);
 	const deadline = createDeadline({ maxWallClockMs });
 	const components = componentNames(seedCandidate);
@@ -497,6 +501,7 @@ async function runOpro(args) {
 	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
 		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
@@ -521,6 +526,7 @@ async function runOpro(args) {
 		metricCalls: budget.spent(),
 		reflectionCalls,
 		cacheHits: evaluator.cacheHits(),
+		warnings,
 		stopReason
 	};
 }

package/dist/{optimizer-Bh5DPRMH.d.cts → optimizer-4Zv-Zt2t.d.cts}

@@ -1,4 +1,4 @@
-import { c as Adapter, d as EvaluationBatch, l as Candidate, m as EvaluationSplit, p as EvaluationPhase, v as UsageTotals } from "./reporting-DQbAohc9.cjs";
+import { S as UsageTotals, _ as EvaluationPhase, c as RunWarning, f as Adapter, h as EvaluationBatch, p as Candidate, v as EvaluationSplit } from "./reporting-bq007_2z.cjs";
 import { n as EvaluationCache, t as CachedScore } from "./cache-CuSo0NJ8.cjs";
 //#region src/budget.d.ts
 interface Budget {
@@ -214,8 +214,28 @@ declare function requireMeasuredMean(args: {
  */
 interface OptimizerTask<Datum, Trajectory = unknown, Output = unknown, K extends string = string> {
   seedCandidate: Candidate<K>;
+  /**
+   * Instances the search draws evidence from. Reflective optimizers mine these
+   * for what a candidate got wrong, so a training row earns its place by being
+   * diagnostic — a row every candidate already passes teaches the rewriter
+   * nothing.
+   *
+   * @see docs/data-prep.md
+   */
   trainingSet: readonly Datum[];
-  validationSet?: readonly Datum[];
+  /**
+   * Instances the search selects candidates against. Defaults to
+   * `trainingSet`, which is the right default for a first run and the wrong
+   * number to report from one — the result carries a warning saying so.
+   * `"reuseTraining"` is that same default with the caller's name on it, and
+   * carries no warning.
+   *
+   * Split by group rather than by row: near-duplicate instances that straddle
+   * the boundary leak, and the run reports a score nothing earned.
+   *
+   * @see docs/data-prep.md
+   */
+  validationSet?: readonly Datum[] | "reuseTraining";
   /**
    * `NoInfer` keeps the adapter out of `K`'s inference: an adapter built by a
    * factory knows nothing about component names, and one inference candidate of
@@ -229,6 +249,18 @@ interface OptimizerTask<Datum, Trajectory = unknown, Output = unknown, K extends
    * number in a result that no candidate was ever selected against.
    */
   testSet?: readonly Datum[];
+  /**
+   * Scored rollouts the search may spend. Cache hits do not count, and test
+   * rollouts are outside it entirely.
+   *
+   * Spending it is only worth anything if the metric separates candidates
+   * first: a metric that scores every candidate alike turns the whole budget
+   * into ranked ties, and the run reports a stop reason that looks like any
+   * other.
+   *
+   * @see docs/metric-preflight.md
+   * @see docs/tuning.md
+   */
   maxMetricCalls: number;
   /**
    * Dollars the run may spend, as reported by the adapter's usage. Checked
@@ -314,6 +346,13 @@ interface OptimizerResult<K extends string, Stop extends string, Output = unknow
    */
   testUsage?: UsageTotals;
   stopReason: Stop;
+  /**
+   * What this run cannot say about itself from its own numbers — selection
+   * that reused the training instances, a seed the metric could not separate.
+   * Empty when there is nothing to say. Never fatal, and repeated on the
+   * `finish` event so a reporter sees them next to the score.
+   */
+  warnings: RunWarning[];
 }
 /**
  * An optimizer: a task in, the best candidate it found out. Exactly one method.

package/dist/{optimizer-Ck6-e_8o.d.mts → optimizer-Ds5mzYjz.d.mts}

@@ -1,4 +1,4 @@
-import { c as Adapter, d as EvaluationBatch, l as Candidate, m as EvaluationSplit, p as EvaluationPhase, v as UsageTotals } from "./reporting-DQbAohc9.mjs";
+import { S as UsageTotals, _ as EvaluationPhase, c as RunWarning, f as Adapter, h as EvaluationBatch, p as Candidate, v as EvaluationSplit } from "./reporting-bq007_2z.mjs";
 import { n as EvaluationCache, t as CachedScore } from "./cache-CuSo0NJ8.mjs";
 //#region src/budget.d.ts
 interface Budget {
@@ -214,8 +214,28 @@ declare function requireMeasuredMean(args: {
  */
 interface OptimizerTask<Datum, Trajectory = unknown, Output = unknown, K extends string = string> {
   seedCandidate: Candidate<K>;
+  /**
+   * Instances the search draws evidence from. Reflective optimizers mine these
+   * for what a candidate got wrong, so a training row earns its place by being
+   * diagnostic — a row every candidate already passes teaches the rewriter
+   * nothing.
+   *
+   * @see docs/data-prep.md
+   */
   trainingSet: readonly Datum[];
-  validationSet?: readonly Datum[];
+  /**
+   * Instances the search selects candidates against. Defaults to
+   * `trainingSet`, which is the right default for a first run and the wrong
+   * number to report from one — the result carries a warning saying so.
+   * `"reuseTraining"` is that same default with the caller's name on it, and
+   * carries no warning.
+   *
+   * Split by group rather than by row: near-duplicate instances that straddle
+   * the boundary leak, and the run reports a score nothing earned.
+   *
+   * @see docs/data-prep.md
+   */
+  validationSet?: readonly Datum[] | "reuseTraining";
   /**
    * `NoInfer` keeps the adapter out of `K`'s inference: an adapter built by a
    * factory knows nothing about component names, and one inference candidate of
@@ -229,6 +249,18 @@ interface OptimizerTask<Datum, Trajectory = unknown, Output = unknown, K extends
    * number in a result that no candidate was ever selected against.
    */
   testSet?: readonly Datum[];
+  /**
+   * Scored rollouts the search may spend. Cache hits do not count, and test
+   * rollouts are outside it entirely.
+   *
+   * Spending it is only worth anything if the metric separates candidates
+   * first: a metric that scores every candidate alike turns the whole budget
+   * into ranked ties, and the run reports a stop reason that looks like any
+   * other.
+   *
+   * @see docs/metric-preflight.md
+   * @see docs/tuning.md
+   */
   maxMetricCalls: number;
   /**
    * Dollars the run may spend, as reported by the adapter's usage. Checked
@@ -314,6 +346,13 @@ interface OptimizerResult<K extends string, Stop extends string, Output = unknow
    */
   testUsage?: UsageTotals;
   stopReason: Stop;
+  /**
+   * What this run cannot say about itself from its own numbers — selection
+   * that reused the training instances, a seed the metric could not separate.
+   * Empty when there is nothing to say. Never fatal, and repeated on the
+   * `finish` event so a reporter sees them next to the score.
+   */
+  warnings: RunWarning[];
 }
 /**
  * An optimizer: a task in, the best candidate it found out. Exactly one method.

package/dist/random-search/index.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_reporting = require("../reporting-CNHzbJC-.cjs");
+const require_warnings = require("../warnings-CWRJF-jA.cjs");
 const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
 const require_text = require("../text-CK_HB3su.cjs");
 //#region src/random-search/optimize.ts
@@ -39,7 +39,7 @@ var RandomSearchOptimizer = class {
 				task
 			});
 		} finally {
-			await require_reporting.flushReporters(task.reporters ?? []);
+			await require_warnings.flushReporters(task.reporters ?? []);
 		}
 	}
 };
@@ -67,10 +67,14 @@ function buildParaphrasePrompt(args) {
 async function runRandomSearch(args) {
 	const { config, task } = args;
 	const { variants = DEFAULT_VARIANTS, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, buildPrompt = buildParaphrasePrompt, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_reporting.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
-	const emit = require_reporting.createEmitter(reporters);
-	const deadline = require_reporting.createDeadline({ maxWallClockMs });
-	const components = require_reporting.componentNames(seedCandidate);
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_warnings.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = require_warnings.resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
+	const emit = require_warnings.createEmitter(reporters);
+	const deadline = require_warnings.createDeadline({ maxWallClockMs });
+	const components = require_warnings.componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
 	if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
 	if (components.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
@@ -83,7 +87,7 @@ async function runRandomSearch(args) {
 		datum,
 		index
 	})) ?? [];
-	const fingerprint = require_reporting.runFingerprint({
+	const fingerprint = require_warnings.runFingerprint({
 		seedCandidate,
 		trainingIds: trainingSet.map((datum, index) => instanceId({
 			datum,
@@ -92,16 +96,16 @@ async function runRandomSearch(args) {
 		validationIds,
 		...cacheNamespace === void 0 ? {} : { cacheNamespace }
 	});
-	require_reporting.assertResumable({
+	require_warnings.assertResumable({
 		fingerprint,
 		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
 	});
-	const budget = require_reporting.createBudget({
+	const budget = require_warnings.createBudget({
 		maxMetricCalls,
 		spent: resumeFrom?.metricCalls ?? 0
 	});
-	const evaluationCache = cache === false ? void 0 : cache ?? require_reporting.createMemoryCache();
-	const evaluator = require_reporting.createEvaluator({
+	const evaluationCache = cache === false ? void 0 : cache ?? require_warnings.createMemoryCache();
+	const evaluator = require_warnings.createEvaluator({
 		adapter,
 		budget,
 		...retry === void 0 ? {} : { retry },
@@ -171,7 +175,7 @@ async function runRandomSearch(args) {
 		candidate: seedCandidate,
 		phase: "seed"
 	}) : void 0;
-	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_reporting.requireMeasuredMean({
+	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_warnings.requireMeasuredMean({
 		batch: seedEvaluation,
 		phase: "seed"
 	});
@@ -181,7 +185,7 @@ async function runRandomSearch(args) {
 		candidateId: 0,
 		candidate: seedCandidate,
 		aggregateScore: seedScore,
-		instanceScores: require_reporting.instanceRow(seedEvaluation),
+		instanceScores: require_warnings.instanceRow(seedEvaluation),
 		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
 	});
 	let best = resumeFrom?.best ?? seedCandidate;
@@ -195,7 +199,7 @@ async function runRandomSearch(args) {
 			stopReason = "aborted";
 			break;
 		}
-		if (require_reporting.costExhausted({
+		if (require_warnings.costExhausted({
 			usage: evaluator.usage(),
 			maxCostUsd
 		})) {
@@ -268,7 +272,7 @@ async function runRandomSearch(args) {
 						phase: "validation"
 					}) };
 				} catch (err) {
-					if (err instanceof require_reporting.BudgetExhausted) return { stop: "budgetExhausted" };
+					if (err instanceof require_warnings.BudgetExhausted) return { stop: "budgetExhausted" };
 					if (signal?.aborted) return { stop: "aborted" };
 					throw err;
 				}
@@ -280,7 +284,7 @@ async function runRandomSearch(args) {
 				break;
 			}
 			variantsEvaluated += 1;
-			const score = require_reporting.measuredMean(outcome.evaluation);
+			const score = require_warnings.measuredMean(outcome.evaluation);
 			if (score !== void 0 && score > bestScore) {
 				const candidate = scheduled[index];
 				acceptedCandidates += 1;
@@ -290,7 +294,7 @@ async function runRandomSearch(args) {
 					candidateId: acceptedCandidates,
 					candidate,
 					aggregateScore: score,
-					instanceScores: require_reporting.instanceRow(outcome.evaluation),
+					instanceScores: require_warnings.instanceRow(outcome.evaluation),
 					...trackBestOutputs ? { outputs: outcome.evaluation.outputs } : {}
 				});
 				best = candidate;
@@ -321,15 +325,16 @@ async function runRandomSearch(args) {
 		iteration: round,
 		charge: false
 	});
-	const testScore = heldOut === void 0 ? void 0 : require_reporting.measuredMean(heldOut);
+	const testScore = heldOut === void 0 ? void 0 : require_warnings.measuredMean(heldOut);
 	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
 		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
 		...testScore === void 0 ? {} : { testScore },
-		...heldOut === void 0 ? {} : { testInstanceScores: require_reporting.instanceRow(heldOut) },
+		...heldOut === void 0 ? {} : { testInstanceScores: require_warnings.instanceRow(heldOut) },
 		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
@@ -349,6 +354,7 @@ async function runRandomSearch(args) {
 		metricCalls: budget.spent(),
 		reflectionCalls,
 		cacheHits: evaluator.cacheHits(),
+		warnings,
 		stopReason
 	};
 }
@@ -361,5 +367,5 @@ function assertConfig(config) {
 //#endregion
 exports.RandomSearchOptimizer = RandomSearchOptimizer;
 exports.buildParaphrasePrompt = buildParaphrasePrompt;
-exports.isCandidateAccepted = require_reporting.isCandidateAccepted;
-exports.isRunFinished = require_reporting.isRunFinished;
+exports.isCandidateAccepted = require_warnings.isCandidateAccepted;
+exports.isRunFinished = require_warnings.isRunFinished;

package/dist/random-search/index.d.cts

@@ -1,6 +1,6 @@
-import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.cjs";
+import { S as UsageTotals, a as RunFinished, f as Adapter, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, p as Candidate, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, x as TextModel } from "../reporting-bq007_2z.cjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.cjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Bh5DPRMH.cjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-4Zv-Zt2t.cjs";
 //#region src/random-search/optimize.d.ts
 /** Builds the prompt one variant is drawn from. */
 type ParaphrasePromptBuilder = (args: {

package/dist/random-search/index.d.mts

@@ -1,6 +1,6 @@
-import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.mjs";
+import { S as UsageTotals, a as RunFinished, f as Adapter, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, p as Candidate, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, x as TextModel } from "../reporting-bq007_2z.mjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.mjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Ck6-e_8o.mjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Ds5mzYjz.mjs";
 //#region src/random-search/optimize.d.ts
 /** Builds the prompt one variant is drawn from. */
 type ParaphrasePromptBuilder = (args: {

package/dist/random-search/index.mjs

@@ -1,4 +1,4 @@
-import { _ as createMemoryCache, a as isRunFinished, b as createBudget, c as createEvaluator, d as createDeadline, f as assertResumable, h as componentNames, i as isCandidateAccepted, l as measuredMean, m as runFingerprint, n as flushReporters, o as BudgetExhausted, r as instanceRow, s as costExhausted, t as createEmitter, u as requireMeasuredMean, v as defaultInstanceId } from "../reporting-DY-DC4HG.mjs";
+import { S as createBudget, _ as componentNames, a as instanceRow, b as defaultInstanceId, c as BudgetExhausted, d as measuredMean, f as requireMeasuredMean, g as runFingerprint, i as flushReporters, l as costExhausted, m as assertResumable, o as isCandidateAccepted, p as createDeadline, r as createEmitter, s as isRunFinished, t as resolveValidationSet, u as createEvaluator, y as createMemoryCache } from "../warnings-OxvDi9kN.mjs";
 import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
 import { t as parseProposedText } from "../text--v4Ffbus.mjs";
 //#region src/random-search/optimize.ts
@@ -66,7 +66,11 @@ function buildParaphrasePrompt(args) {
 async function runRandomSearch(args) {
 	const { config, task } = args;
 	const { variants = DEFAULT_VARIANTS, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, buildPrompt = buildParaphrasePrompt, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
 	const emit = createEmitter(reporters);
 	const deadline = createDeadline({ maxWallClockMs });
 	const components = componentNames(seedCandidate);
@@ -324,6 +328,7 @@ async function runRandomSearch(args) {
 	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
 		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
@@ -348,6 +353,7 @@ async function runRandomSearch(args) {
 		metricCalls: budget.spent(),
 		reflectionCalls,
 		cacheHits: evaluator.cacheHits(),
+		warnings,
 		stopReason
 	};
 }