textopt 0.0.0 → 0.2.0

package/dist/random-search/index.mjs

@@ -1,4 +1,4 @@
-import { a as requireMeasuredMean, d as createMemoryCache, f as stableHash, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "../evaluation-BV0nSZVx.mjs";
+import { S as createBudget, _ as componentNames, a as instanceRow, b as defaultInstanceId, c as BudgetExhausted, d as measuredMean, f as requireMeasuredMean, g as runFingerprint, i as flushReporters, l as costExhausted, m as assertResumable, o as isCandidateAccepted, p as createDeadline, r as createEmitter, s as isRunFinished, t as resolveValidationSet, u as createEvaluator, y as createMemoryCache } from "../warnings-OxvDi9kN.mjs";
 import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
 import { t as parseProposedText } from "../text--v4Ffbus.mjs";
 //#region src/random-search/optimize.ts
@@ -32,10 +32,14 @@ var RandomSearchOptimizer = class {
 		this.#config = config;
 	}
 	async optimize(task) {
-		return runRandomSearch({
-			config: this.#config,
-			task
-		});
+		try {
+			return await runRandomSearch({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await flushReporters(task.reporters ?? []);
+		}
 	}
 };
 /**
@@ -62,7 +66,12 @@ function buildParaphrasePrompt(args) {
 async function runRandomSearch(args) {
 	const { config, task } = args;
 	const { variants = DEFAULT_VARIANTS, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, buildPrompt = buildParaphrasePrompt, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
+	const emit = createEmitter(reporters);
 	const deadline = createDeadline({ maxWallClockMs });
 	const components = componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
@@ -103,8 +112,9 @@ async function runRandomSearch(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
-		onEvaluation: (event) => onEvent?.({
+		onEvaluation: (event) => emit({
 			type: "evaluation",
 			...event
 		})
@@ -122,7 +132,7 @@ async function runRandomSearch(args) {
 	* no budget here bounds.
 	*/
 	let stalledRounds = 0;
-	onEvent?.({
+	emit({
 		type: "start",
 		components,
 		validationSetSize: validationSet.length
@@ -151,6 +161,8 @@ async function runRandomSearch(args) {
 			reflectionCalls,
 			metricCalls: budget.spent(),
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
+			acceptedCandidates,
 			...cached === void 0 ? {} : { cache: cached }
 		};
 	}
@@ -166,10 +178,20 @@ async function runRandomSearch(args) {
 		batch: seedEvaluation,
 		phase: "seed"
 	});
+	if (seedEvaluation !== void 0) emit({
+		type: "candidateAccepted",
+		round: 0,
+		candidateId: 0,
+		candidate: seedCandidate,
+		aggregateScore: seedScore,
+		instanceScores: instanceRow(seedEvaluation),
+		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
+	});
 	let best = resumeFrom?.best ?? seedCandidate;
 	let bestScore = resumeFrom?.bestScore ?? seedScore;
 	/** Absent on a resumed run until a variant wins: outputs are not checkpointed. */
 	let bestOutputs = seedEvaluation?.outputs;
+	let acceptedCandidates = resumeFrom?.acceptedCandidates ?? 0;
 	await checkpoint();
 	while (round < maxRounds) {
 		if (signal?.aborted) {
@@ -187,12 +209,12 @@ async function runRandomSearch(args) {
 			stopReason = "deadlineReached";
 			break;
 		}
-		if (!budget.canAfford(variants * validationSet.length)) {
+		if (!budget.canAfford(validationSet.length)) {
 			stopReason = "budgetExhausted";
 			break;
 		}
 		const component = components[round % components.length];
-		onEvent?.({
+		emit({
 			type: "roundStart",
 			round,
 			component
@@ -219,41 +241,64 @@ async function runRandomSearch(args) {
 		});
 		const unique = [...new Set(drawn)].filter((text) => text.length > 0 && text !== currentText);
 		let roundStop;
+		const scheduled = [];
+		let owed = 0;
 		for (const text of unique) {
 			const candidate = {
 				...best,
 				[component]: text
 			};
-			let evaluation;
-			try {
-				evaluation = await sweep({
-					candidate,
-					phase: "validation"
-				});
-			} catch (err) {
-				if (err instanceof BudgetExhausted) {
-					roundStop = "budgetExhausted";
-					break;
-				}
-				if (signal?.aborted) {
-					roundStop = "aborted";
-					break;
+			const uncached = evaluator.countUncached({
+				candidate,
+				ids: validationIds,
+				split: "val"
+			});
+			if (!budget.canAfford(owed + uncached)) {
+				roundStop = "budgetExhausted";
+				break;
+			}
+			owed += uncached;
+			scheduled.push(candidate);
+		}
+		const swept = await mapWithConcurrency({
+			items: scheduled,
+			limit: concurrency,
+			task: async (candidate) => {
+				if (signal?.aborted) return { stop: "aborted" };
+				try {
+					return { evaluation: await sweep({
+						candidate,
+						phase: "validation"
+					}) };
+				} catch (err) {
+					if (err instanceof BudgetExhausted) return { stop: "budgetExhausted" };
+					if (signal?.aborted) return { stop: "aborted" };
+					throw err;
 				}
-				throw err;
+			}
+		});
+		for (const [index, outcome] of swept.entries()) {
+			if (outcome.stop !== void 0) {
+				roundStop = outcome.stop;
+				break;
 			}
 			variantsEvaluated += 1;
-			const score = measuredMean(evaluation);
+			const score = measuredMean(outcome.evaluation);
 			if (score !== void 0 && score > bestScore) {
-				onEvent?.({
+				const candidate = scheduled[index];
+				acceptedCandidates += 1;
+				emit({
 					type: "candidateAccepted",
 					round,
-					component,
-					score,
-					previousScore: bestScore
+					candidateId: acceptedCandidates,
+					candidate,
+					aggregateScore: score,
+					instanceScores: instanceRow(outcome.evaluation),
+					...trackBestOutputs ? { outputs: outcome.evaluation.outputs } : {}
 				});
 				best = candidate;
 				bestScore = score;
-				bestOutputs = evaluation.outputs;
+				bestOutputs = outcome.evaluation.outputs;
 			}
 		}
 		round += 1;
@@ -269,7 +314,7 @@ async function runRandomSearch(args) {
 		}
 	}
 	if (signal?.aborted) stopReason = "aborted";
-	const testScore = testSet === void 0 ? void 0 : measuredMean(await evaluator.evaluate({
+	const heldOut = testSet === void 0 ? void 0 : await evaluator.evaluate({
 		candidate: best,
 		batch: testSet,
 		ids: testIds,
@@ -278,13 +323,18 @@ async function runRandomSearch(args) {
 		candidateId: null,
 		iteration: round,
 		charge: false
-	}));
-	onEvent?.({
+	});
+	const testScore = heldOut === void 0 ? void 0 : measuredMean(heldOut);
+	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
+		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
 		bestCandidate: best,
@@ -295,13 +345,15 @@ async function runRandomSearch(args) {
 		...trackBestOutputs ? { bestOutputs } : {},
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: evaluator.unchargedCalls()
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		rounds: round,
 		variantsEvaluated,
 		metricCalls: budget.spent(),
 		reflectionCalls,
 		cacheHits: evaluator.cacheHits(),
+		warnings,
 		stopReason
 	};
 }
@@ -311,9 +363,5 @@ function assertConfig(config) {
 	if (!Number.isInteger(concurrency) || concurrency < 1) throw new Error(`concurrency must be a positive integer, received ${concurrency}`);
 	if (maxRounds !== void 0 && (!Number.isInteger(maxRounds) || maxRounds < 1)) throw new Error(`maxRounds must be a positive integer, received ${maxRounds}`);
 }
-function defaultInstanceId(args) {
-	const hash = stableHash(args.datum);
-	return hash === "" ? String(args.index) : hash;
-}
 //#endregion
-export { RandomSearchOptimizer, buildParaphrasePrompt };
+export { RandomSearchOptimizer, buildParaphrasePrompt, isCandidateAccepted, isRunFinished };

package/dist/{reflection-Cr_upzU0.d.mts → reflection-CMezGu6u.d.mts}

@@ -1,4 +1,4 @@
-import { i as EvaluationBatch, n as Candidate, o as EvaluationPhase, s as EvaluationSplit, t as Adapter, u as TextModel } from "./types-CWv4IQFF.mjs";
+import { S as UsageTotals, _ as EvaluationPhase, a as RunFinished, f as Adapter, h as EvaluationBatch, m as EvaluateArgs, p as Candidate, t as CandidateAccepted, v as EvaluationSplit, x as TextModel } from "./reporting-bq007_2z.mjs";
 import { t as CachedScore } from "./cache-CuSo0NJ8.mjs";
 import { t as Rng } from "./rng-BR5MOedA.mjs";
 //#region src/gepa/types.d.ts
@@ -56,12 +56,29 @@ interface ProposeArgs<K extends string = string> {
   reflect: TextModel;
   signal?: AbortSignal;
 }
+/**
+ * A scored batch that carries the per-instance diagnosis reflection is written
+ * from. `feedback` is optional on `EvaluationBatch` because the searches that
+ * never reflect have no use for it; here it is the input to the whole method,
+ * so it is required.
+ *
+ * An adapter that returns scores and no prose reduces every rollout to a
+ * number, and reflection then rewrites the instruction from a prompt whose
+ * feedback blocks are empty. That run spends its whole budget, reports a
+ * normal-looking `stopReason`, and has been doing blind search — which is
+ * exactly the failure no reading of the result can distinguish from a hard
+ * task. It is a type error instead.
+ */
+type ReflectiveBatch<Trajectory = unknown, Output = unknown> = EvaluationBatch<Trajectory, Output> & {
+  feedback: string[];
+};
 /**
  * An adapter GEPA can reflect against: evaluation, plus the traces reflection
  * reads. `makeReflectiveDataset` is what turns a scored batch into the
  * per-component evidence a reflection call is written from.
  */
 interface GepaAdapter<Datum, Trajectory = unknown, Output = unknown, K extends string = string> extends Adapter<Datum, Trajectory, Output, K> {
+  evaluate(args: EvaluateArgs<Datum, K>): Promise<ReflectiveBatch<Trajectory, Output>> | ReflectiveBatch<Trajectory, Output>;
   makeReflectiveDataset(args: MakeReflectiveDatasetArgs<Datum, Trajectory, Output, K>): Promise<ReflectiveDataset<K>> | ReflectiveDataset<K>;
   proposeNewTexts?(args: ProposeArgs<K>): Promise<ComponentPatch<K>> | ComponentPatch<K>;
 }
@@ -111,10 +128,21 @@ type ComponentSelector<K extends string = string> = (args: {
   iteration: number;
   rng: Rng;
 }) => K[];
-type AcceptancePolicy = (args: {
+type AcceptancePolicy = ((args: {
   parentScores: readonly number[];
   childScores: readonly number[];
-}) => boolean;
+}) => boolean) & {
+  /**
+   * Smallest minibatch this policy could ever accept on, when it has one. A
+   * policy that tests for significance has a floor its batch size must clear
+   * — a sign-flip test over three instances cannot report a p-value below
+   * 0.125 — and below it every proposal is rejected on arithmetic rather than
+   * on evidence. The run that results looks exactly like a search that found
+   * nothing, so the optimizer refuses the combination at construction instead
+   * of spending the budget discovering it.
+   */
+  minimumPairs?: number;
+};
 /**
  * Which validation instances a candidate is scored on, and how the best
  * candidate is read back out of possibly partial coverage.
@@ -158,14 +186,12 @@ type GepaEvent<K extends string = string> = {
   parentId: number;
   componentsToUpdate: K[];
   changed: boolean;
-} | {
+} | ({
   type: "candidateAccepted";
   iteration: number;
-  candidateId: number;
   parentIds: number[];
-  aggregateScore: number;
   source: CandidateSource;
-} | {
+} & CandidateAccepted<K>) | {
   type: "candidateRejected";
   iteration: number;
   parentId: number;
@@ -182,14 +208,10 @@ type GepaEvent<K extends string = string> = {
   type: "error";
   iteration: number;
   err: unknown;
-} | {
+} | ({
   type: "finish";
   reason: GepaStopReason;
-  bestCandidateId: number;
-  metricCalls: number;
-  /** The winner's held-out score, when a testSet was given. */
-  testScore?: number;
-};
+} & RunFinished);
 /**
  * Everything needed to continue a run: the candidate pool with its scores, the
  * budget already spent, the position of the random stream, and the bookkeeping
@@ -219,6 +241,8 @@ interface GepaSnapshot {
   metricCalls: number;
   reflectionCalls: number;
   cacheHits: number;
+  /** Usage already spent, so a resumed run reports totals and honours ceilings. */
+  usage?: UsageTotals;
   rngState: number;
   /** Whatever the batch sampler reports from `state()`, when it has one. */
   sampler?: unknown;
@@ -280,4 +304,4 @@ declare function buildRewritePrompt(args: ReflectionPromptArgs): string;
  */
 declare function diverseReflectionStrategies(): ReflectionPromptBuilder[];
 //#endregion
-export { RejectedProposal as C, ReflectiveRecord as S, ValEvaluationPolicy as T, GepaStopReason as _, buildRewritePrompt as a, ProposeArgs as b, AcceptancePolicy as c, CandidateSource as d, ComponentPatch as f, GepaSnapshot as g, GepaEvent as h, buildReflectionPrompt as i, CandidateRecord as l, GepaAdapter as m, ReflectionPromptBuilder as n, buildSimplifyPrompt as o, ComponentSelector as p, buildGeneralizePrompt as r, diverseReflectionStrategies as s, ReflectionPromptArgs as t, CandidateSelector as u, MakeReflectiveDatasetArgs as v, SelectionState as w, ReflectiveDataset as x, ParetoFrontier as y };
+export { ReflectiveRecord as C, ValEvaluationPolicy as E, ReflectiveDataset as S, SelectionState as T, GepaStopReason as _, buildRewritePrompt as a, ProposeArgs as b, AcceptancePolicy as c, CandidateSource as d, ComponentPatch as f, GepaSnapshot as g, GepaEvent as h, buildReflectionPrompt as i, CandidateRecord as l, GepaAdapter as m, ReflectionPromptBuilder as n, buildSimplifyPrompt as o, ComponentSelector as p, buildGeneralizePrompt as r, diverseReflectionStrategies as s, ReflectionPromptArgs as t, CandidateSelector as u, MakeReflectiveDatasetArgs as v, RejectedProposal as w, ReflectiveBatch as x, ParetoFrontier as y };

package/dist/{reflection-CQToe-5B.d.cts → reflection-D0A7eahD.d.cts}

@@ -1,4 +1,4 @@
-import { i as EvaluationBatch, n as Candidate, o as EvaluationPhase, s as EvaluationSplit, t as Adapter, u as TextModel } from "./types-CWv4IQFF.cjs";
+import { S as UsageTotals, _ as EvaluationPhase, a as RunFinished, f as Adapter, h as EvaluationBatch, m as EvaluateArgs, p as Candidate, t as CandidateAccepted, v as EvaluationSplit, x as TextModel } from "./reporting-bq007_2z.cjs";
 import { t as CachedScore } from "./cache-CuSo0NJ8.cjs";
 import { t as Rng } from "./rng-BR5MOedA.cjs";
 //#region src/gepa/types.d.ts
@@ -56,12 +56,29 @@ interface ProposeArgs<K extends string = string> {
   reflect: TextModel;
   signal?: AbortSignal;
 }
+/**
+ * A scored batch that carries the per-instance diagnosis reflection is written
+ * from. `feedback` is optional on `EvaluationBatch` because the searches that
+ * never reflect have no use for it; here it is the input to the whole method,
+ * so it is required.
+ *
+ * An adapter that returns scores and no prose reduces every rollout to a
+ * number, and reflection then rewrites the instruction from a prompt whose
+ * feedback blocks are empty. That run spends its whole budget, reports a
+ * normal-looking `stopReason`, and has been doing blind search — which is
+ * exactly the failure no reading of the result can distinguish from a hard
+ * task. It is a type error instead.
+ */
+type ReflectiveBatch<Trajectory = unknown, Output = unknown> = EvaluationBatch<Trajectory, Output> & {
+  feedback: string[];
+};
 /**
  * An adapter GEPA can reflect against: evaluation, plus the traces reflection
  * reads. `makeReflectiveDataset` is what turns a scored batch into the
  * per-component evidence a reflection call is written from.
  */
 interface GepaAdapter<Datum, Trajectory = unknown, Output = unknown, K extends string = string> extends Adapter<Datum, Trajectory, Output, K> {
+  evaluate(args: EvaluateArgs<Datum, K>): Promise<ReflectiveBatch<Trajectory, Output>> | ReflectiveBatch<Trajectory, Output>;
   makeReflectiveDataset(args: MakeReflectiveDatasetArgs<Datum, Trajectory, Output, K>): Promise<ReflectiveDataset<K>> | ReflectiveDataset<K>;
   proposeNewTexts?(args: ProposeArgs<K>): Promise<ComponentPatch<K>> | ComponentPatch<K>;
 }
@@ -111,10 +128,21 @@ type ComponentSelector<K extends string = string> = (args: {
   iteration: number;
   rng: Rng;
 }) => K[];
-type AcceptancePolicy = (args: {
+type AcceptancePolicy = ((args: {
   parentScores: readonly number[];
   childScores: readonly number[];
-}) => boolean;
+}) => boolean) & {
+  /**
+   * Smallest minibatch this policy could ever accept on, when it has one. A
+   * policy that tests for significance has a floor its batch size must clear
+   * — a sign-flip test over three instances cannot report a p-value below
+   * 0.125 — and below it every proposal is rejected on arithmetic rather than
+   * on evidence. The run that results looks exactly like a search that found
+   * nothing, so the optimizer refuses the combination at construction instead
+   * of spending the budget discovering it.
+   */
+  minimumPairs?: number;
+};
 /**
  * Which validation instances a candidate is scored on, and how the best
  * candidate is read back out of possibly partial coverage.
@@ -158,14 +186,12 @@ type GepaEvent<K extends string = string> = {
   parentId: number;
   componentsToUpdate: K[];
   changed: boolean;
-} | {
+} | ({
   type: "candidateAccepted";
   iteration: number;
-  candidateId: number;
   parentIds: number[];
-  aggregateScore: number;
   source: CandidateSource;
-} | {
+} & CandidateAccepted<K>) | {
   type: "candidateRejected";
   iteration: number;
   parentId: number;
@@ -182,14 +208,10 @@ type GepaEvent<K extends string = string> = {
   type: "error";
   iteration: number;
   err: unknown;
-} | {
+} | ({
   type: "finish";
   reason: GepaStopReason;
-  bestCandidateId: number;
-  metricCalls: number;
-  /** The winner's held-out score, when a testSet was given. */
-  testScore?: number;
-};
+} & RunFinished);
 /**
  * Everything needed to continue a run: the candidate pool with its scores, the
  * budget already spent, the position of the random stream, and the bookkeeping
@@ -219,6 +241,8 @@ interface GepaSnapshot {
   metricCalls: number;
   reflectionCalls: number;
   cacheHits: number;
+  /** Usage already spent, so a resumed run reports totals and honours ceilings. */
+  usage?: UsageTotals;
   rngState: number;
   /** Whatever the batch sampler reports from `state()`, when it has one. */
   sampler?: unknown;
@@ -280,4 +304,4 @@ declare function buildRewritePrompt(args: ReflectionPromptArgs): string;
  */
 declare function diverseReflectionStrategies(): ReflectionPromptBuilder[];
 //#endregion
-export { RejectedProposal as C, ReflectiveRecord as S, ValEvaluationPolicy as T, GepaStopReason as _, buildRewritePrompt as a, ProposeArgs as b, AcceptancePolicy as c, CandidateSource as d, ComponentPatch as f, GepaSnapshot as g, GepaEvent as h, buildReflectionPrompt as i, CandidateRecord as l, GepaAdapter as m, ReflectionPromptBuilder as n, buildSimplifyPrompt as o, ComponentSelector as p, buildGeneralizePrompt as r, diverseReflectionStrategies as s, ReflectionPromptArgs as t, CandidateSelector as u, MakeReflectiveDatasetArgs as v, SelectionState as w, ReflectiveDataset as x, ParetoFrontier as y };
+export { ReflectiveRecord as C, ValEvaluationPolicy as E, ReflectiveDataset as S, SelectionState as T, GepaStopReason as _, buildRewritePrompt as a, ProposeArgs as b, AcceptancePolicy as c, CandidateSource as d, ComponentPatch as f, GepaSnapshot as g, GepaEvent as h, buildReflectionPrompt as i, CandidateRecord as l, GepaAdapter as m, ReflectionPromptBuilder as n, buildSimplifyPrompt as o, ComponentSelector as p, buildGeneralizePrompt as r, diverseReflectionStrategies as s, ReflectionPromptArgs as t, CandidateSelector as u, MakeReflectiveDatasetArgs as v, RejectedProposal as w, ReflectiveBatch as x, ParetoFrontier as y };