textopt 0.1.0 → 0.2.0

package/dist/{reflection-LRaAZP4e.d.mts → reflection-CMezGu6u.d.mts}

@@ -1,4 +1,4 @@
-import { _ as TextModel, a as RunFinished, c as Adapter, d as EvaluationBatch, l as Candidate, m as EvaluationSplit, p as EvaluationPhase, t as CandidateAccepted, v as UsageTotals } from "./reporting-DQbAohc9.mjs";
+import { S as UsageTotals, _ as EvaluationPhase, a as RunFinished, f as Adapter, h as EvaluationBatch, m as EvaluateArgs, p as Candidate, t as CandidateAccepted, v as EvaluationSplit, x as TextModel } from "./reporting-bq007_2z.mjs";
 import { t as CachedScore } from "./cache-CuSo0NJ8.mjs";
 import { t as Rng } from "./rng-BR5MOedA.mjs";
 //#region src/gepa/types.d.ts
@@ -56,12 +56,29 @@ interface ProposeArgs<K extends string = string> {
   reflect: TextModel;
   signal?: AbortSignal;
 }
+/**
+ * A scored batch that carries the per-instance diagnosis reflection is written
+ * from. `feedback` is optional on `EvaluationBatch` because the searches that
+ * never reflect have no use for it; here it is the input to the whole method,
+ * so it is required.
+ *
+ * An adapter that returns scores and no prose reduces every rollout to a
+ * number, and reflection then rewrites the instruction from a prompt whose
+ * feedback blocks are empty. That run spends its whole budget, reports a
+ * normal-looking `stopReason`, and has been doing blind search — which is
+ * exactly the failure no reading of the result can distinguish from a hard
+ * task. It is a type error instead.
+ */
+type ReflectiveBatch<Trajectory = unknown, Output = unknown> = EvaluationBatch<Trajectory, Output> & {
+  feedback: string[];
+};
 /**
  * An adapter GEPA can reflect against: evaluation, plus the traces reflection
  * reads. `makeReflectiveDataset` is what turns a scored batch into the
  * per-component evidence a reflection call is written from.
  */
 interface GepaAdapter<Datum, Trajectory = unknown, Output = unknown, K extends string = string> extends Adapter<Datum, Trajectory, Output, K> {
+  evaluate(args: EvaluateArgs<Datum, K>): Promise<ReflectiveBatch<Trajectory, Output>> | ReflectiveBatch<Trajectory, Output>;
   makeReflectiveDataset(args: MakeReflectiveDatasetArgs<Datum, Trajectory, Output, K>): Promise<ReflectiveDataset<K>> | ReflectiveDataset<K>;
   proposeNewTexts?(args: ProposeArgs<K>): Promise<ComponentPatch<K>> | ComponentPatch<K>;
 }
@@ -111,10 +128,21 @@ type ComponentSelector<K extends string = string> = (args: {
   iteration: number;
   rng: Rng;
 }) => K[];
-type AcceptancePolicy = (args: {
+type AcceptancePolicy = ((args: {
   parentScores: readonly number[];
   childScores: readonly number[];
-}) => boolean;
+}) => boolean) & {
+  /**
+   * Smallest minibatch this policy could ever accept on, when it has one. A
+   * policy that tests for significance has a floor its batch size must clear
+   * — a sign-flip test over three instances cannot report a p-value below
+   * 0.125 — and below it every proposal is rejected on arithmetic rather than
+   * on evidence. The run that results looks exactly like a search that found
+   * nothing, so the optimizer refuses the combination at construction instead
+   * of spending the budget discovering it.
+   */
+  minimumPairs?: number;
+};
 /**
  * Which validation instances a candidate is scored on, and how the best
  * candidate is read back out of possibly partial coverage.
@@ -276,4 +304,4 @@ declare function buildRewritePrompt(args: ReflectionPromptArgs): string;
  */
 declare function diverseReflectionStrategies(): ReflectionPromptBuilder[];
 //#endregion
-export { RejectedProposal as C, ReflectiveRecord as S, ValEvaluationPolicy as T, GepaStopReason as _, buildRewritePrompt as a, ProposeArgs as b, AcceptancePolicy as c, CandidateSource as d, ComponentPatch as f, GepaSnapshot as g, GepaEvent as h, buildReflectionPrompt as i, CandidateRecord as l, GepaAdapter as m, ReflectionPromptBuilder as n, buildSimplifyPrompt as o, ComponentSelector as p, buildGeneralizePrompt as r, diverseReflectionStrategies as s, ReflectionPromptArgs as t, CandidateSelector as u, MakeReflectiveDatasetArgs as v, SelectionState as w, ReflectiveDataset as x, ParetoFrontier as y };
+export { ReflectiveRecord as C, ValEvaluationPolicy as E, ReflectiveDataset as S, SelectionState as T, GepaStopReason as _, buildRewritePrompt as a, ProposeArgs as b, AcceptancePolicy as c, CandidateSource as d, ComponentPatch as f, GepaSnapshot as g, GepaEvent as h, buildReflectionPrompt as i, CandidateRecord as l, GepaAdapter as m, ReflectionPromptBuilder as n, buildSimplifyPrompt as o, ComponentSelector as p, buildGeneralizePrompt as r, diverseReflectionStrategies as s, ReflectionPromptArgs as t, CandidateSelector as u, MakeReflectiveDatasetArgs as v, RejectedProposal as w, ReflectiveBatch as x, ParetoFrontier as y };

package/dist/{reflection-Dt3QrXhM.d.cts → reflection-D0A7eahD.d.cts}

@@ -1,4 +1,4 @@
-import { _ as TextModel, a as RunFinished, c as Adapter, d as EvaluationBatch, l as Candidate, m as EvaluationSplit, p as EvaluationPhase, t as CandidateAccepted, v as UsageTotals } from "./reporting-DQbAohc9.cjs";
+import { S as UsageTotals, _ as EvaluationPhase, a as RunFinished, f as Adapter, h as EvaluationBatch, m as EvaluateArgs, p as Candidate, t as CandidateAccepted, v as EvaluationSplit, x as TextModel } from "./reporting-bq007_2z.cjs";
 import { t as CachedScore } from "./cache-CuSo0NJ8.cjs";
 import { t as Rng } from "./rng-BR5MOedA.cjs";
 //#region src/gepa/types.d.ts
@@ -56,12 +56,29 @@ interface ProposeArgs<K extends string = string> {
   reflect: TextModel;
   signal?: AbortSignal;
 }
+/**
+ * A scored batch that carries the per-instance diagnosis reflection is written
+ * from. `feedback` is optional on `EvaluationBatch` because the searches that
+ * never reflect have no use for it; here it is the input to the whole method,
+ * so it is required.
+ *
+ * An adapter that returns scores and no prose reduces every rollout to a
+ * number, and reflection then rewrites the instruction from a prompt whose
+ * feedback blocks are empty. That run spends its whole budget, reports a
+ * normal-looking `stopReason`, and has been doing blind search — which is
+ * exactly the failure no reading of the result can distinguish from a hard
+ * task. It is a type error instead.
+ */
+type ReflectiveBatch<Trajectory = unknown, Output = unknown> = EvaluationBatch<Trajectory, Output> & {
+  feedback: string[];
+};
 /**
  * An adapter GEPA can reflect against: evaluation, plus the traces reflection
  * reads. `makeReflectiveDataset` is what turns a scored batch into the
  * per-component evidence a reflection call is written from.
  */
 interface GepaAdapter<Datum, Trajectory = unknown, Output = unknown, K extends string = string> extends Adapter<Datum, Trajectory, Output, K> {
+  evaluate(args: EvaluateArgs<Datum, K>): Promise<ReflectiveBatch<Trajectory, Output>> | ReflectiveBatch<Trajectory, Output>;
   makeReflectiveDataset(args: MakeReflectiveDatasetArgs<Datum, Trajectory, Output, K>): Promise<ReflectiveDataset<K>> | ReflectiveDataset<K>;
   proposeNewTexts?(args: ProposeArgs<K>): Promise<ComponentPatch<K>> | ComponentPatch<K>;
 }
@@ -111,10 +128,21 @@ type ComponentSelector<K extends string = string> = (args: {
   iteration: number;
   rng: Rng;
 }) => K[];
-type AcceptancePolicy = (args: {
+type AcceptancePolicy = ((args: {
   parentScores: readonly number[];
   childScores: readonly number[];
-}) => boolean;
+}) => boolean) & {
+  /**
+   * Smallest minibatch this policy could ever accept on, when it has one. A
+   * policy that tests for significance has a floor its batch size must clear
+   * — a sign-flip test over three instances cannot report a p-value below
+   * 0.125 — and below it every proposal is rejected on arithmetic rather than
+   * on evidence. The run that results looks exactly like a search that found
+   * nothing, so the optimizer refuses the combination at construction instead
+   * of spending the budget discovering it.
+   */
+  minimumPairs?: number;
+};
 /**
  * Which validation instances a candidate is scored on, and how the best
  * candidate is read back out of possibly partial coverage.
@@ -276,4 +304,4 @@ declare function buildRewritePrompt(args: ReflectionPromptArgs): string;
  */
 declare function diverseReflectionStrategies(): ReflectionPromptBuilder[];
 //#endregion
-export { RejectedProposal as C, ReflectiveRecord as S, ValEvaluationPolicy as T, GepaStopReason as _, buildRewritePrompt as a, ProposeArgs as b, AcceptancePolicy as c, CandidateSource as d, ComponentPatch as f, GepaSnapshot as g, GepaEvent as h, buildReflectionPrompt as i, CandidateRecord as l, GepaAdapter as m, ReflectionPromptBuilder as n, buildSimplifyPrompt as o, ComponentSelector as p, buildGeneralizePrompt as r, diverseReflectionStrategies as s, ReflectionPromptArgs as t, CandidateSelector as u, MakeReflectiveDatasetArgs as v, SelectionState as w, ReflectiveDataset as x, ParetoFrontier as y };
+export { ReflectiveRecord as C, ValEvaluationPolicy as E, ReflectiveDataset as S, SelectionState as T, GepaStopReason as _, buildRewritePrompt as a, ProposeArgs as b, AcceptancePolicy as c, CandidateSource as d, ComponentPatch as f, GepaSnapshot as g, GepaEvent as h, buildReflectionPrompt as i, CandidateRecord as l, GepaAdapter as m, ReflectionPromptBuilder as n, buildSimplifyPrompt as o, ComponentSelector as p, buildGeneralizePrompt as r, diverseReflectionStrategies as s, ReflectionPromptArgs as t, CandidateSelector as u, MakeReflectiveDatasetArgs as v, RejectedProposal as w, ReflectiveBatch as x, ParetoFrontier as y };

package/dist/{reporting-DQbAohc9.d.cts → reporting-bq007_2z.d.cts}

@@ -126,6 +126,54 @@ type TextModel = (args: {
  */
 declare function componentNames<K extends string>(candidate: Partial<Candidate<K>>): K[];
 //#endregion
+//#region src/warnings.d.ts
+/**
+ * Something about a run that its own numbers cannot say.
+ *
+ * A search reports a score, a stop reason and what it spent, and every one of
+ * those reads the same whether the run measured what the caller thinks it did
+ * or not. These are the conditions under which a normal-looking result means
+ * less than it appears to: they never stop a run, and they are carried on the
+ * result and the `finish` event so a report can say so next to the number.
+ */
+interface RunWarning {
+  code: RunWarningCode;
+  message: string;
+}
+type RunWarningCode = "validationSetReusesTraining" | "seedScoreSaturated" | "seedScoreFloored";
+/**
+ * The validation set a run will actually select against, and whatever the
+ * choice costs it.
+ *
+ * Defaulting to the training set is the right default for a first run and the
+ * wrong number to report from one. It is worse than ordinary overfitting under
+ * reflective search: the reflection prompt asks the model to mine domain facts
+ * out of the traces it is shown, so those facts come out of the very instances
+ * that then select the candidate carrying them. `"reuseTraining"` is the same
+ * behaviour with the caller's name on it, and silences the warning.
+ */
+declare function resolveValidationSet<Datum>(args: {
+  validationSet: readonly Datum[] | "reuseTraining" | undefined;
+  trainingSet: readonly Datum[];
+}): {
+  validationSet: readonly Datum[];
+  warnings: RunWarning[];
+};
+/**
+ * What the seed's own validation row says about whether the run could have
+ * learned anything.
+ *
+ * A search ranks candidates by how they differ across instances, so a seed row
+ * with no spread leaves nothing to rank: at the ceiling every proposal is a tie
+ * the acceptance test resolves by noise, and at the floor no proposal has a
+ * partial improvement to build on. Both produce a run that spends its whole
+ * budget and reports a stop reason that looks like any other.
+ */
+declare function seedScoreWarnings(args: {
+  scores: readonly (number | undefined)[];
+  perfectScore: number;
+}): RunWarning[];
+//#endregion
 //#region src/reporting.d.ts
 /**
  * Where a run's progress goes, for any optimizer. Observability only:
@@ -212,6 +260,12 @@ interface RunFinished {
   testInstanceScores?: readonly (number | undefined)[];
   /** Aligned with `testInstanceScores`. Only under `trackBestOutputs`. */
   testOutputs?: readonly unknown[];
+  /**
+   * What the run cannot say about itself from its own numbers. A reporter that
+   * writes the score somewhere permanent writes these beside it, or the record
+   * outlives the only place the caveat was ever stated.
+   */
+  warnings: readonly RunWarning[];
 }
 /**
  * The two events every optimizer emits with a payload a reporter can read
@@ -237,4 +291,4 @@ declare function isRunFinished(event: OptimizerEvent): event is {
   type: "finish";
 } & RunFinished;
 //#endregion
-export { TextModel as _, RunFinished as a, Adapter as c, EvaluationBatch as d, EvaluationContext as f, ScoreResult as g, RolloutUsage as h, Reporter as i, Candidate as l, EvaluationSplit as m, OptimizerEvent as n, isCandidateAccepted as o, EvaluationPhase as p, ReportableEvent as r, isRunFinished as s, CandidateAccepted as t, EvaluateArgs as u, UsageTotals as v, componentNames as y };
+export { componentNames as C, UsageTotals as S, EvaluationPhase as _, RunFinished as a, ScoreResult as b, RunWarning as c, seedScoreWarnings as d, Adapter as f, EvaluationContext as g, EvaluationBatch as h, Reporter as i, RunWarningCode as l, EvaluateArgs as m, OptimizerEvent as n, isCandidateAccepted as o, Candidate as p, ReportableEvent as r, isRunFinished as s, CandidateAccepted as t, resolveValidationSet as u, EvaluationSplit as v, TextModel as x, RolloutUsage as y };

package/dist/{reporting-DQbAohc9.d.mts → reporting-bq007_2z.d.mts}

@@ -126,6 +126,54 @@ type TextModel = (args: {
  */
 declare function componentNames<K extends string>(candidate: Partial<Candidate<K>>): K[];
 //#endregion
+//#region src/warnings.d.ts
+/**
+ * Something about a run that its own numbers cannot say.
+ *
+ * A search reports a score, a stop reason and what it spent, and every one of
+ * those reads the same whether the run measured what the caller thinks it did
+ * or not. These are the conditions under which a normal-looking result means
+ * less than it appears to: they never stop a run, and they are carried on the
+ * result and the `finish` event so a report can say so next to the number.
+ */
+interface RunWarning {
+  code: RunWarningCode;
+  message: string;
+}
+type RunWarningCode = "validationSetReusesTraining" | "seedScoreSaturated" | "seedScoreFloored";
+/**
+ * The validation set a run will actually select against, and whatever the
+ * choice costs it.
+ *
+ * Defaulting to the training set is the right default for a first run and the
+ * wrong number to report from one. It is worse than ordinary overfitting under
+ * reflective search: the reflection prompt asks the model to mine domain facts
+ * out of the traces it is shown, so those facts come out of the very instances
+ * that then select the candidate carrying them. `"reuseTraining"` is the same
+ * behaviour with the caller's name on it, and silences the warning.
+ */
+declare function resolveValidationSet<Datum>(args: {
+  validationSet: readonly Datum[] | "reuseTraining" | undefined;
+  trainingSet: readonly Datum[];
+}): {
+  validationSet: readonly Datum[];
+  warnings: RunWarning[];
+};
+/**
+ * What the seed's own validation row says about whether the run could have
+ * learned anything.
+ *
+ * A search ranks candidates by how they differ across instances, so a seed row
+ * with no spread leaves nothing to rank: at the ceiling every proposal is a tie
+ * the acceptance test resolves by noise, and at the floor no proposal has a
+ * partial improvement to build on. Both produce a run that spends its whole
+ * budget and reports a stop reason that looks like any other.
+ */
+declare function seedScoreWarnings(args: {
+  scores: readonly (number | undefined)[];
+  perfectScore: number;
+}): RunWarning[];
+//#endregion
 //#region src/reporting.d.ts
 /**
  * Where a run's progress goes, for any optimizer. Observability only:
@@ -212,6 +260,12 @@ interface RunFinished {
   testInstanceScores?: readonly (number | undefined)[];
   /** Aligned with `testInstanceScores`. Only under `trackBestOutputs`. */
   testOutputs?: readonly unknown[];
+  /**
+   * What the run cannot say about itself from its own numbers. A reporter that
+   * writes the score somewhere permanent writes these beside it, or the record
+   * outlives the only place the caveat was ever stated.
+   */
+  warnings: readonly RunWarning[];
 }
 /**
  * The two events every optimizer emits with a payload a reporter can read
@@ -237,4 +291,4 @@ declare function isRunFinished(event: OptimizerEvent): event is {
   type: "finish";
 } & RunFinished;
 //#endregion
-export { TextModel as _, RunFinished as a, Adapter as c, EvaluationBatch as d, EvaluationContext as f, ScoreResult as g, RolloutUsage as h, Reporter as i, Candidate as l, EvaluationSplit as m, OptimizerEvent as n, isCandidateAccepted as o, EvaluationPhase as p, ReportableEvent as r, isRunFinished as s, CandidateAccepted as t, EvaluateArgs as u, UsageTotals as v, componentNames as y };
+export { componentNames as C, UsageTotals as S, EvaluationPhase as _, RunFinished as a, ScoreResult as b, RunWarning as c, seedScoreWarnings as d, Adapter as f, EvaluationContext as g, EvaluationBatch as h, Reporter as i, RunWarningCode as l, EvaluateArgs as m, OptimizerEvent as n, isCandidateAccepted as o, Candidate as p, ReportableEvent as r, isRunFinished as s, CandidateAccepted as t, resolveValidationSet as u, EvaluationSplit as v, TextModel as x, RolloutUsage as y };

package/dist/simba/index.cjs

@@ -1,7 +1,7 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_reporting = require("../reporting-CNHzbJC-.cjs");
+const require_warnings = require("../warnings-CWRJF-jA.cjs");
 const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
-const require_demos = require("../demos-D2o0qSSo.cjs");
+const require_demos = require("../demos-9v5ts7F3.cjs");
 const require_rng = require("../rng-DbA_rPIo.cjs");
 const require_sampling = require("../sampling-Dars7ctR.cjs");
 //#region src/simba/advice.ts
@@ -246,16 +246,20 @@ var SimbaOptimizer = class {
 				task
 			});
 		} finally {
-			await require_reporting.flushReporters(task.reporters ?? []);
+			await require_warnings.flushReporters(task.reporters ?? []);
 		}
 	}
 };
 async function run(args) {
 	const { config, task } = args;
 	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, candidates: candidateCount = DEFAULT_CANDIDATES, concurrency = 1, maxSteps = DEFAULT_MAX_STEPS, maxDemos = DEFAULT_MAX_DEMOS, samplingTemperature = DEFAULT_TEMPERATURE, candidateTemperature = DEFAULT_TEMPERATURE, strategies, maxReflectionCalls, seed = 0, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, demoComponents = [], instructionComponents, renderDemo, buildAdvicePrompt: buildPrompt = buildAdvicePrompt, sampler = require_sampling.createEpochShuffledSampler({ minibatchSize }), maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_reporting.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
-	const emit = require_reporting.createEmitter(reporters);
-	const components = require_reporting.componentNames(seedCandidate);
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, demoComponents = [], instructionComponents, renderDemo, buildAdvicePrompt: buildPrompt = buildAdvicePrompt, sampler = require_sampling.createEpochShuffledSampler({ minibatchSize }), maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_warnings.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = require_warnings.resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
+	const emit = require_warnings.createEmitter(reporters);
+	const components = require_warnings.componentNames(seedCandidate);
 	const nonDemo = components.filter((name) => !demoComponents.includes(name));
 	const ruleComponents = instructionComponents ?? (nonDemo.length > 0 ? nonDemo : components);
 	const enabled = strategies ?? (demoComponents.length > 0 ? ["appendDemo", "appendRule"] : ["appendRule"]);
@@ -278,25 +282,25 @@ async function run(args) {
 		datum,
 		index
 	})) ?? [];
-	const fingerprint = require_reporting.runFingerprint({
+	const fingerprint = require_warnings.runFingerprint({
 		seedCandidate,
 		trainingIds,
 		validationIds,
 		seed,
 		...cacheNamespace === void 0 ? {} : { cacheNamespace }
 	});
-	require_reporting.assertResumable({
+	require_warnings.assertResumable({
 		fingerprint,
 		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
 	});
-	const deadline = require_reporting.createDeadline({ maxWallClockMs });
+	const deadline = require_warnings.createDeadline({ maxWallClockMs });
 	const rng = require_rng.createSeededRng(seed, resumeFrom?.rngState);
-	const budget = require_reporting.createBudget({
+	const budget = require_warnings.createBudget({
 		maxMetricCalls,
 		spent: resumeFrom?.metricCalls ?? 0
 	});
-	const evaluationCache = cache === false ? void 0 : cache ?? require_reporting.createMemoryCache();
-	const evaluator = require_reporting.createEvaluator({
+	const evaluationCache = cache === false ? void 0 : cache ?? require_warnings.createMemoryCache();
+	const evaluator = require_warnings.createEvaluator({
 		adapter,
 		budget,
 		...retry === void 0 ? {} : { retry },
@@ -334,7 +338,7 @@ async function run(args) {
 			stopReason = "aborted";
 			break;
 		}
-		if (require_reporting.costExhausted({
+		if (require_warnings.costExhausted({
 			usage: evaluator.usage(),
 			maxCostUsd
 		})) {
@@ -389,7 +393,7 @@ async function run(args) {
 				outputs: evaluation.outputs,
 				...evaluation.feedback === void 0 ? {} : { feedback: evaluation.feedback }
 			});
-			programScores[programIndex].push(require_reporting.measuredMean(evaluation) ?? 0);
+			programScores[programIndex].push(require_warnings.measuredMean(evaluation) ?? 0);
 		}
 		if (samples.length === 0) {
 			stopReason = "budgetExhausted";
@@ -463,10 +467,10 @@ async function run(args) {
 		const scored = await mapDistinct({
 			items: scheduled,
 			limit: concurrency,
-			key: (entry) => require_reporting.candidateHash(entry.candidate),
+			key: (entry) => require_warnings.candidateHash(entry.candidate),
 			task: async (entry, index) => {
 				try {
-					return { score: require_reporting.measuredMean(await evaluator.evaluate({
+					return { score: require_warnings.measuredMean(await evaluator.evaluate({
 						candidate: entry.candidate,
 						batch,
 						ids: batchIds,
@@ -476,7 +480,7 @@ async function run(args) {
 						iteration: step
 					})) };
 				} catch (err) {
-					if (err instanceof require_reporting.BudgetExhausted) return { stop: true };
+					if (err instanceof require_warnings.BudgetExhausted) return { stop: true };
 					throw err;
 				}
 			}
@@ -537,7 +541,7 @@ async function run(args) {
 	const sweeps = await mapDistinct({
 		items: contenders,
 		limit: concurrency,
-		key: (contender) => require_reporting.candidateHash(contender.candidate),
+		key: (contender) => require_warnings.candidateHash(contender.candidate),
 		task: async (contender) => {
 			if (signal?.aborted) return { stop: true };
 			try {
@@ -551,11 +555,11 @@ async function run(args) {
 					iteration: step
 				});
 				return {
-					score: require_reporting.measuredMean(evaluation),
+					score: require_warnings.measuredMean(evaluation),
 					evaluation
 				};
 			} catch (err) {
-				if (err instanceof require_reporting.BudgetExhausted || signal?.aborted) return { stop: true };
+				if (err instanceof require_warnings.BudgetExhausted || signal?.aborted) return { stop: true };
 				throw err;
 			}
 		}
@@ -581,7 +585,7 @@ async function run(args) {
 				candidateId: acceptedCandidates,
 				candidate: contender.candidate,
 				aggregateScore: outcome.score,
-				instanceScores: require_reporting.instanceRow(outcome.evaluation),
+				instanceScores: require_warnings.instanceRow(outcome.evaluation),
 				...trackBestOutputs ? { outputs: outcome.evaluation.outputs } : {}
 			});
 		}
@@ -611,15 +615,16 @@ async function run(args) {
 		iteration: step,
 		charge: false
 	});
-	const testScore = heldOut === void 0 ? void 0 : require_reporting.measuredMean(heldOut);
+	const testScore = heldOut === void 0 ? void 0 : require_warnings.measuredMean(heldOut);
 	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
 		bestCandidateId: acceptedCandidates,
 		bestScore: best.score,
 		metricCalls: budget.spent(),
 		...testScore === void 0 ? {} : { testScore },
-		...heldOut === void 0 ? {} : { testInstanceScores: require_reporting.instanceRow(heldOut) },
+		...heldOut === void 0 ? {} : { testInstanceScores: require_warnings.instanceRow(heldOut) },
 		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
@@ -639,6 +644,7 @@ async function run(args) {
 			testMetricCalls: evaluator.unchargedCalls(),
 			testUsage: evaluator.unchargedUsage()
 		},
+		warnings,
 		stopReason
 	};
 	/**
@@ -827,8 +833,8 @@ exports.SimbaOptimizer = SimbaOptimizer;
 exports.buildAdvicePrompt = buildAdvicePrompt;
 exports.buildBuckets = buildBuckets;
 exports.evenlySpacedIndices = evenlySpacedIndices;
-exports.isCandidateAccepted = require_reporting.isCandidateAccepted;
-exports.isRunFinished = require_reporting.isRunFinished;
+exports.isCandidateAccepted = require_warnings.isCandidateAccepted;
+exports.isRunFinished = require_warnings.isRunFinished;
 exports.parseAdvice = parseAdvice;
 exports.percentile = percentile;
 exports.samplePoisson = samplePoisson;

package/dist/simba/index.d.cts

@@ -1,8 +1,8 @@
-import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.cjs";
+import { S as UsageTotals, a as RunFinished, f as Adapter, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, p as Candidate, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, x as TextModel } from "../reporting-bq007_2z.cjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.cjs";
 import { t as Rng } from "../rng-BR5MOedA.cjs";
-import { r as DemoRenderer } from "../demos-CU9dy2oT.cjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Bh5DPRMH.cjs";
+import { r as DemoRenderer } from "../demos-ByaLZy-Z.cjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-4Zv-Zt2t.cjs";
 import { t as BatchSampler } from "../sampling-axOwfZf5.cjs";
 //#region src/simba/advice.d.ts
 interface AdviceTrajectory<Output = unknown> {
@@ -58,7 +58,15 @@ interface SimbaFinalist<K extends string = string> {
   step: number;
 }
 interface SimbaConfig {
-  /** Instances per step. Default 32. */
+  /**
+   * Instances per step. Default 32.
+   *
+   * Minibatch defaults do not transfer between optimizers — GEPA's 3 and
+   * MIPRO's 35 mean different things, and SIMBA's ranks instances within a
+   * batch by how much its programs disagreed.
+   *
+   * @see docs/tuning.md
+   */
   minibatchSize?: number;
   /** Programs sampled per step, and candidates built from them. Default 6. */
   candidates?: number;
@@ -73,7 +81,18 @@ interface SimbaConfig {
    * which call returned first.
    */
   concurrency?: number;
-  /** Steps to run. Default 8. */
+  /**
+   * Steps to run. Default 8.
+   *
+   * A run has to be funded past the finalist reserve before any step happens:
+   * `min(candidates + 1, maxSteps + 1) * validationSet.length` rollouts are
+   * held back to sweep the finalists, and each step then costs
+   * `candidates * minibatchSize + minibatchSize`. Fund only the steps and the
+   * run stops after one, reporting `budgetExhausted` like any other exhausted
+   * budget.
+   *
+   * @see docs/tuning.md
+   */
   maxSteps?: number;
   /** Demos a candidate may hold before the loop starts dropping them. Default 4. */
   maxDemos?: number;

package/dist/simba/index.d.mts

@@ -1,8 +1,8 @@
-import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.mjs";
+import { S as UsageTotals, a as RunFinished, f as Adapter, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, p as Candidate, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, x as TextModel } from "../reporting-bq007_2z.mjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.mjs";
 import { t as Rng } from "../rng-BR5MOedA.mjs";
-import { r as DemoRenderer } from "../demos-1b7JiUue.mjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Ck6-e_8o.mjs";
+import { r as DemoRenderer } from "../demos-ASsSXYXA.mjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Ds5mzYjz.mjs";
 import { t as BatchSampler } from "../sampling-DFo_7RNJ.mjs";
 //#region src/simba/advice.d.ts
 interface AdviceTrajectory<Output = unknown> {
@@ -58,7 +58,15 @@ interface SimbaFinalist<K extends string = string> {
   step: number;
 }
 interface SimbaConfig {
-  /** Instances per step. Default 32. */
+  /**
+   * Instances per step. Default 32.
+   *
+   * Minibatch defaults do not transfer between optimizers — GEPA's 3 and
+   * MIPRO's 35 mean different things, and SIMBA's ranks instances within a
+   * batch by how much its programs disagreed.
+   *
+   * @see docs/tuning.md
+   */
   minibatchSize?: number;
   /** Programs sampled per step, and candidates built from them. Default 6. */
   candidates?: number;
@@ -73,7 +81,18 @@ interface SimbaConfig {
    * which call returned first.
    */
   concurrency?: number;
-  /** Steps to run. Default 8. */
+  /**
+   * Steps to run. Default 8.
+   *
+   * A run has to be funded past the finalist reserve before any step happens:
+   * `min(candidates + 1, maxSteps + 1) * validationSet.length` rollouts are
+   * held back to sweep the finalists, and each step then costs
+   * `candidates * minibatchSize + minibatchSize`. Fund only the steps and the
+   * run stops after one, reporting `budgetExhausted` like any other exhausted
+   * budget.
+   *
+   * @see docs/tuning.md
+   */
   maxSteps?: number;
   /** Demos a candidate may hold before the loop starts dropping them. Default 4. */
   maxDemos?: number;

package/dist/simba/index.mjs

@@ -1,6 +1,6 @@
-import { _ as createMemoryCache, a as isRunFinished, b as createBudget, c as createEvaluator, d as createDeadline, f as assertResumable, g as candidateHash, h as componentNames, i as isCandidateAccepted, l as measuredMean, m as runFingerprint, n as flushReporters, o as BudgetExhausted, r as instanceRow, s as costExhausted, t as createEmitter, v as defaultInstanceId } from "../reporting-DY-DC4HG.mjs";
+import { S as createBudget, _ as componentNames, a as instanceRow, b as defaultInstanceId, c as BudgetExhausted, d as measuredMean, g as runFingerprint, i as flushReporters, l as costExhausted, m as assertResumable, o as isCandidateAccepted, p as createDeadline, r as createEmitter, s as isRunFinished, t as resolveValidationSet, u as createEvaluator, v as candidateHash, y as createMemoryCache } from "../warnings-OxvDi9kN.mjs";
 import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
-import { i as replaceDemos, r as parseDemos } from "../demos-DE2oxNWX.mjs";
+import { i as replaceDemos, r as parseDemos } from "../demos-Brobjfuc.mjs";
 import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
 import { t as createEpochShuffledSampler } from "../sampling-CfHt7Gue.mjs";
 //#region src/simba/advice.ts
@@ -252,7 +252,11 @@ var SimbaOptimizer = class {
 async function run(args) {
 	const { config, task } = args;
 	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, candidates: candidateCount = DEFAULT_CANDIDATES, concurrency = 1, maxSteps = DEFAULT_MAX_STEPS, maxDemos = DEFAULT_MAX_DEMOS, samplingTemperature = DEFAULT_TEMPERATURE, candidateTemperature = DEFAULT_TEMPERATURE, strategies, maxReflectionCalls, seed = 0, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, demoComponents = [], instructionComponents, renderDemo, buildAdvicePrompt: buildPrompt = buildAdvicePrompt, sampler = createEpochShuffledSampler({ minibatchSize }), maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, demoComponents = [], instructionComponents, renderDemo, buildAdvicePrompt: buildPrompt = buildAdvicePrompt, sampler = createEpochShuffledSampler({ minibatchSize }), maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
 	const emit = createEmitter(reporters);
 	const components = componentNames(seedCandidate);
 	const nonDemo = components.filter((name) => !demoComponents.includes(name));
@@ -614,6 +618,7 @@ async function run(args) {
 	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
 		bestCandidateId: acceptedCandidates,
 		bestScore: best.score,
 		metricCalls: budget.spent(),
@@ -638,6 +643,7 @@ async function run(args) {
 			testMetricCalls: evaluator.unchargedCalls(),
 			testUsage: evaluator.unchargedUsage()
 		},
+		warnings,
 		stopReason
 	};
 	/**

package/dist/testing.d.cts

@@ -1,5 +1,5 @@
-import { _ as TextModel } from "./reporting-DQbAohc9.cjs";
-import { i as buildReflectionPrompt, m as GepaAdapter } from "./reflection-Dt3QrXhM.cjs";
+import { x as TextModel } from "./reporting-bq007_2z.cjs";
+import { i as buildReflectionPrompt, m as GepaAdapter } from "./reflection-D0A7eahD.cjs";
 //#region src/testing.d.ts
 /**
  * A deterministic, LLM-free system under optimization.

package/dist/testing.d.mts

@@ -1,5 +1,5 @@
-import { _ as TextModel } from "./reporting-DQbAohc9.mjs";
-import { i as buildReflectionPrompt, m as GepaAdapter } from "./reflection-LRaAZP4e.mjs";
+import { x as TextModel } from "./reporting-bq007_2z.mjs";
+import { i as buildReflectionPrompt, m as GepaAdapter } from "./reflection-CMezGu6u.mjs";
 //#region src/testing.d.ts
 /**
  * A deterministic, LLM-free system under optimization.