textopt 0.0.0

package/dist/random-search/index.mjs

@@ -0,0 +1,319 @@
+import { a as requireMeasuredMean, d as createMemoryCache, f as stableHash, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "../evaluation-BV0nSZVx.mjs";
+import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
+import { t as parseProposedText } from "../text--v4Ffbus.mjs";
+//#region src/random-search/optimize.ts
+const DEFAULT_VARIANTS = 4;
+/**
+* The ablation baseline: propose blind, evaluate in full, keep what wins.
+*
+* It exists to be beaten. Reflective search costs a frontier-model call per
+* proposal on top of its rollouts, and running the same budget through a
+* search that cannot read feedback at all puts a floor under what that call
+* has to buy. A GEPA run that does not clear this one is paying for machinery
+* it is not using.
+*
+* What the gap measures is the whole of that machinery, not reflection alone:
+* this drops feedback, Pareto parent selection, and minibatch screening
+* together, and GEPA's own ablations put candidate selection at several points
+* by itself. For reflection in isolation, keep GEPA and hand it a
+* `reflection.buildPrompt` that withholds the evidence — same frontier, same
+* screening, one variable.
+*
+* Not DSPy's `BootstrapFewShotWithRandomSearch`, which the literature usually
+* means by "random search" and which searches bootstrapped demo sets; nor
+* random search in the Bergstra–Bengio sense, since proposals are paraphrases
+* of the incumbent rather than independent draws. No reference implements this
+* baseline — it is original here, and the GEPA paper has no equivalent.
+*/
+var RandomSearchOptimizer = class {
+	#config;
+	constructor(config = {}) {
+		assertConfig(config);
+		this.#config = config;
+	}
+	async optimize(task) {
+		return runRandomSearch({
+			config: this.#config,
+			task
+		});
+	}
+};
+/**
+* Adapted from the GEPA reflection prompt with every source of evidence
+* removed. What is left is the ablation: the same model, the same component,
+* the same output format, and nothing to reason from.
+*/
+function buildParaphrasePrompt(args) {
+	const { componentName, currentText, attempt } = args;
+	return [
+		`Here is the current instruction for the "${componentName}" component of a larger system:`,
+		"",
+		"<current_instruction>",
+		currentText,
+		"</current_instruction>",
+		"",
+		"Write a different instruction for this component.",
+		"You have no information about how the current one has performed, so do not guess at its weaknesses — vary it instead.",
+		`Make variation ${attempt + 1} distinct from the others: change the wording, the level of detail, or the strategy it describes.`,
+		"",
+		"Return only the new instruction, inside a ``` block."
+	].join("\n");
+}
+async function runRandomSearch(args) {
+	const { config, task } = args;
+	const { variants = DEFAULT_VARIANTS, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, buildPrompt = buildParaphrasePrompt, trackBestOutputs = false, checkpointCache = true } = config;
+	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
+	const deadline = createDeadline({ maxWallClockMs });
+	const components = componentNames(seedCandidate);
+	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
+	if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
+	if (components.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
+	if (testSet !== void 0 && testSet.length === 0) throw new Error("optimize requires a non-empty testSet when one is given; omit it to skip held-out evaluation");
+	const validationIds = validationSet.map((datum, index) => instanceId({
+		datum,
+		index
+	}));
+	const testIds = testSet?.map((datum, index) => instanceId({
+		datum,
+		index
+	})) ?? [];
+	const fingerprint = runFingerprint({
+		seedCandidate,
+		trainingIds: trainingSet.map((datum, index) => instanceId({
+			datum,
+			index
+		})),
+		validationIds,
+		...cacheNamespace === void 0 ? {} : { cacheNamespace }
+	});
+	assertResumable({
+		fingerprint,
+		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
+	});
+	const budget = createBudget({
+		maxMetricCalls,
+		spent: resumeFrom?.metricCalls ?? 0
+	});
+	const evaluationCache = cache === false ? void 0 : cache ?? createMemoryCache();
+	const evaluator = createEvaluator({
+		adapter,
+		budget,
+		...retry === void 0 ? {} : { retry },
+		...cacheNamespace === void 0 ? {} : { cacheNamespace },
+		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
+		trackOutputs: trackBestOutputs,
+		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...signal === void 0 ? {} : { signal },
+		onEvaluation: (event) => onEvent?.({
+			type: "evaluation",
+			...event
+		})
+	});
+	evaluator.restore(resumeFrom?.cache ?? []);
+	let round = resumeFrom?.round ?? 0;
+	let variantsEvaluated = resumeFrom?.variantsEvaluated ?? 0;
+	let reflectionCalls = resumeFrom?.reflectionCalls ?? 0;
+	let stopReason = "maxRounds";
+	/**
+	* Consecutive rounds that neither spent a rollout nor improved on the
+	* incumbent. A proposer stuck on texts that are already cached costs
+	* nothing, so neither the metric budget nor the cost ceiling can end the
+	* run — without this the loop spins forever, burning reflection calls that
+	* no budget here bounds.
+	*/
+	let stalledRounds = 0;
+	onEvent?.({
+		type: "start",
+		components,
+		validationSetSize: validationSet.length
+	});
+	async function sweep(args) {
+		return evaluator.evaluate({
+			candidate: args.candidate,
+			batch: validationSet,
+			ids: validationIds,
+			split: "val",
+			phase: args.phase,
+			candidateId: null,
+			iteration: round
+		});
+	}
+	function takeSnapshot() {
+		const cached = checkpointCache ? evaluationCache?.entries?.() : void 0;
+		return {
+			version: 1,
+			fingerprint,
+			best,
+			bestScore,
+			seedScore,
+			round,
+			variantsEvaluated,
+			reflectionCalls,
+			metricCalls: budget.spent(),
+			cacheHits: evaluator.cacheHits(),
+			...cached === void 0 ? {} : { cache: cached }
+		};
+	}
+	async function checkpoint() {
+		if (onCheckpoint === void 0) return;
+		await onCheckpoint(takeSnapshot());
+	}
+	const seedEvaluation = resumeFrom === void 0 ? await sweep({
+		candidate: seedCandidate,
+		phase: "seed"
+	}) : void 0;
+	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : requireMeasuredMean({
+		batch: seedEvaluation,
+		phase: "seed"
+	});
+	let best = resumeFrom?.best ?? seedCandidate;
+	let bestScore = resumeFrom?.bestScore ?? seedScore;
+	/** Absent on a resumed run until a variant wins: outputs are not checkpointed. */
+	let bestOutputs = seedEvaluation?.outputs;
+	await checkpoint();
+	while (round < maxRounds) {
+		if (signal?.aborted) {
+			stopReason = "aborted";
+			break;
+		}
+		if (costExhausted({
+			usage: evaluator.usage(),
+			maxCostUsd
+		})) {
+			stopReason = "costExhausted";
+			break;
+		}
+		if (deadline.exceeded()) {
+			stopReason = "deadlineReached";
+			break;
+		}
+		if (!budget.canAfford(variants * validationSet.length)) {
+			stopReason = "budgetExhausted";
+			break;
+		}
+		const component = components[round % components.length];
+		onEvent?.({
+			type: "roundStart",
+			round,
+			component
+		});
+		const spentBefore = budget.spent();
+		const scoreBefore = bestScore;
+		const currentText = best[component];
+		const drawn = await mapWithConcurrency({
+			items: Array.from({ length: variants }, (_, attempt) => attempt),
+			limit: concurrency,
+			signal,
+			task: async (attempt) => {
+				reflectionCalls += 1;
+				const response = await reflect({
+					prompt: buildPrompt({
+						componentName: component,
+						currentText,
+						attempt
+					}),
+					...signal === void 0 ? {} : { signal }
+				});
+				return parseProposedText(response);
+			}
+		});
+		const unique = [...new Set(drawn)].filter((text) => text.length > 0 && text !== currentText);
+		let roundStop;
+		for (const text of unique) {
+			const candidate = {
+				...best,
+				[component]: text
+			};
+			let evaluation;
+			try {
+				evaluation = await sweep({
+					candidate,
+					phase: "validation"
+				});
+			} catch (err) {
+				if (err instanceof BudgetExhausted) {
+					roundStop = "budgetExhausted";
+					break;
+				}
+				if (signal?.aborted) {
+					roundStop = "aborted";
+					break;
+				}
+				throw err;
+			}
+			variantsEvaluated += 1;
+			const score = measuredMean(evaluation);
+			if (score !== void 0 && score > bestScore) {
+				onEvent?.({
+					type: "candidateAccepted",
+					round,
+					component,
+					score,
+					previousScore: bestScore
+				});
+				best = candidate;
+				bestScore = score;
+				bestOutputs = evaluation.outputs;
+			}
+		}
+		round += 1;
+		stalledRounds = budget.spent() === spentBefore && bestScore === scoreBefore ? stalledRounds + 1 : 0;
+		await checkpoint();
+		if (stalledRounds >= components.length) {
+			stopReason = "proposerStalled";
+			break;
+		}
+		if (roundStop !== void 0) {
+			stopReason = roundStop;
+			break;
+		}
+	}
+	if (signal?.aborted) stopReason = "aborted";
+	const testScore = testSet === void 0 ? void 0 : measuredMean(await evaluator.evaluate({
+		candidate: best,
+		batch: testSet,
+		ids: testIds,
+		split: "test",
+		phase: "test",
+		candidateId: null,
+		iteration: round,
+		charge: false
+	}));
+	onEvent?.({
+		type: "finish",
+		reason: stopReason,
+		bestScore,
+		metricCalls: budget.spent(),
+		...testScore === void 0 ? {} : { testScore }
+	});
+	return {
+		bestCandidate: best,
+		bestScore,
+		usage: evaluator.usage(),
+		seedScore,
+		snapshot: takeSnapshot(),
+		...trackBestOutputs ? { bestOutputs } : {},
+		...testScore === void 0 ? {} : {
+			testScore,
+			testMetricCalls: evaluator.unchargedCalls()
+		},
+		rounds: round,
+		variantsEvaluated,
+		metricCalls: budget.spent(),
+		reflectionCalls,
+		cacheHits: evaluator.cacheHits(),
+		stopReason
+	};
+}
+function assertConfig(config) {
+	const { variants = DEFAULT_VARIANTS, concurrency = 1, maxRounds } = config;
+	if (!Number.isInteger(variants) || variants < 1) throw new Error(`variants must be a positive integer, received ${variants}`);
+	if (!Number.isInteger(concurrency) || concurrency < 1) throw new Error(`concurrency must be a positive integer, received ${concurrency}`);
+	if (maxRounds !== void 0 && (!Number.isInteger(maxRounds) || maxRounds < 1)) throw new Error(`maxRounds must be a positive integer, received ${maxRounds}`);
+}
+function defaultInstanceId(args) {
+	const hash = stableHash(args.datum);
+	return hash === "" ? String(args.index) : hash;
+}
+//#endregion
+export { RandomSearchOptimizer, buildParaphrasePrompt };

package/dist/reflection-CQToe-5B.d.cts

@@ -0,0 +1,283 @@
+import { i as EvaluationBatch, n as Candidate, o as EvaluationPhase, s as EvaluationSplit, t as Adapter, u as TextModel } from "./types-CWv4IQFF.cjs";
+import { t as CachedScore } from "./cache-CuSo0NJ8.cjs";
+import { t as Rng } from "./rng-BR5MOedA.cjs";
+//#region src/gepa/types.d.ts
+/**
+ * One piece of evidence reflection reads: what went in, what came out, and how
+ * it scored. `Evidence` is the adapter's own slot — trace steps, retrieved
+ * documents, errors — typed rather than smuggled in through an index signature.
+ */
+interface ReflectiveRecord<Evidence = unknown> {
+  inputs: unknown;
+  generatedOutputs: unknown;
+  feedback: string;
+  score?: number;
+  evidence?: Evidence;
+}
+/**
+ * Component name -> records shown to the reflection model for that component.
+ *
+ * Partial, not total: an adapter only fills the components it was asked to
+ * update, so a component with nothing to diagnose is simply absent.
+ */
+type ReflectiveDataset<K extends string = string> = Partial<Record<K, ReflectiveRecord[]>>;
+interface MakeReflectiveDatasetArgs<Datum, Trajectory, Output, K extends string = string> {
+  candidate: Candidate<K>;
+  batch: readonly Datum[];
+  evaluation: EvaluationBatch<Trajectory, Output>;
+  componentsToUpdate: readonly K[];
+}
+/** New text for the subset of components a proposal actually changed. */
+type ComponentPatch<K extends string = string> = Partial<Record<K, string>>;
+/**
+ * A component text that was proposed and lost on the minibatch. Showing these
+ * back to the reflection model is what stops a run from re-deriving the same
+ * dead end: only exact duplicates are filtered structurally, so without this
+ * the model can spend the whole budget circling one bad idea.
+ */
+interface RejectedProposal {
+  text: string;
+  parentScore: number;
+  childScore: number;
+}
+interface ProposeArgs<K extends string = string> {
+  candidate: Candidate<K>;
+  reflectiveDataset: ReflectiveDataset<K>;
+  componentsToUpdate: readonly K[];
+  /** Component name -> texts already tried and rejected for it. */
+  rejectedProposals?: Partial<Record<K, RejectedProposal[]>>;
+  /**
+   * Which proposal this is in the run, counting from 0. Assigned when the
+   * iteration is planned rather than when it executes, so concurrent
+   * proposals get stable, distinct values instead of racing for a counter.
+   * The default proposer rotates its strategies on it.
+   */
+  attempt?: number;
+  reflect: TextModel;
+  signal?: AbortSignal;
+}
+/**
+ * An adapter GEPA can reflect against: evaluation, plus the traces reflection
+ * reads. `makeReflectiveDataset` is what turns a scored batch into the
+ * per-component evidence a reflection call is written from.
+ */
+interface GepaAdapter<Datum, Trajectory = unknown, Output = unknown, K extends string = string> extends Adapter<Datum, Trajectory, Output, K> {
+  makeReflectiveDataset(args: MakeReflectiveDatasetArgs<Datum, Trajectory, Output, K>): Promise<ReflectiveDataset<K>> | ReflectiveDataset<K>;
+  proposeNewTexts?(args: ProposeArgs<K>): Promise<ComponentPatch<K>> | ComponentPatch<K>;
+}
+type CandidateSource = "seed" | "mutation" | "merge";
+interface CandidateRecord<K extends string = string> {
+  id: number;
+  candidate: Candidate<K>;
+  parentIds: number[];
+  /**
+   * One entry per validation instance. `undefined` marks an instance the
+   * evaluation policy did not select for this candidate — unknown, not zero.
+   */
+  instanceScores: (number | undefined)[];
+  /** Mean over the instances that were scored. */
+  aggregateScore: number;
+  /** Mean of each objective over the evaluated validation instances. */
+  objectiveScores?: Record<string, number>;
+  source: CandidateSource;
+  updatedComponents: K[];
+  iteration: number;
+  /**
+   * Round-robin position this candidate resumes from the next time it is
+   * chosen as a parent. Per-candidate rather than global: a lineage that is
+   * only reselected every fifth iteration must still walk its own components
+   * one at a time instead of being pinned to whichever one the global counter
+   * happens to land on.
+   */
+  componentCursor: number;
+}
+/** What a Pareto frontier is taken over. */
+type ParetoFrontier = "instance" | "objective" | "hybrid";
+/** Read-only view handed to candidate selectors. */
+interface SelectionState {
+  scoreMatrix: readonly (readonly (number | undefined)[])[];
+  aggregateScores: readonly number[];
+  /** Per-candidate objective means, absent when the adapter reports none. */
+  objectiveScores?: readonly (Readonly<Record<string, number>> | undefined)[];
+}
+type CandidateSelector = (args: {
+  state: SelectionState;
+  rng: Rng;
+}) => number;
+type ComponentSelector<K extends string = string> = (args: {
+  candidate: Candidate<K>;
+  /** This candidate's own round-robin position, advanced after each selection. */
+  cursor: number;
+  iteration: number;
+  rng: Rng;
+}) => K[];
+type AcceptancePolicy = (args: {
+  parentScores: readonly number[];
+  childScores: readonly number[];
+}) => boolean;
+/**
+ * Which validation instances a candidate is scored on, and how the best
+ * candidate is read back out of possibly partial coverage.
+ *
+ * A full sweep per accepted candidate is the published behaviour and the
+ * default. Swapping in a partial policy trades frontier fidelity for rollouts:
+ * candidates scored on different instances are no longer strictly comparable,
+ * which is why picking the best is the policy's job too.
+ */
+interface ValEvaluationPolicy<Datum = unknown, K extends string = string> {
+  selectInstances(args: {
+    validationSet: readonly Datum[];
+    candidate: Candidate<K>;
+    records: readonly CandidateRecord<K>[];
+    iteration: number;
+    rng: Rng;
+  }): number[];
+  bestCandidate(records: readonly CandidateRecord<K>[]): number;
+}
+type GepaStopReason = "budgetExhausted" | "costExhausted" | "deadlineReached" | "reflectionBudgetExhausted" | "aborted" | "maxIterations";
+type GepaEvent<K extends string = string> = {
+  type: "start";
+  components: K[];
+  validationSetSize: number;
+} | {
+  type: "iterationStart";
+  iteration: number;
+  parentIds: number[];
+} | {
+  type: "evaluation";
+  iteration: number;
+  phase: EvaluationPhase;
+  split: EvaluationSplit;
+  candidateId: number | null;
+  metricCalls: number;
+  cacheHits: number;
+  meanScore: number;
+} | {
+  type: "proposal";
+  iteration: number;
+  parentId: number;
+  componentsToUpdate: K[];
+  changed: boolean;
+} | {
+  type: "candidateAccepted";
+  iteration: number;
+  candidateId: number;
+  parentIds: number[];
+  aggregateScore: number;
+  source: CandidateSource;
+} | {
+  type: "candidateRejected";
+  iteration: number;
+  parentId: number;
+  parentScore: number;
+  childScore: number;
+  source: CandidateSource;
+  /**
+   * "worse" lost to its parent on the minibatch. "notSelected" beat its
+   * parent but lost to a stronger proposal from the same iteration — it is
+   * not a dead end, and is never fed back to reflection as one.
+   */
+  reason: "worse" | "notSelected";
+} | {
+  type: "error";
+  iteration: number;
+  err: unknown;
+} | {
+  type: "finish";
+  reason: GepaStopReason;
+  bestCandidateId: number;
+  metricCalls: number;
+  /** The winner's held-out score, when a testSet was given. */
+  testScore?: number;
+};
+/**
+ * Everything needed to continue a run: the candidate pool with its scores, the
+ * budget already spent, the position of the random stream, and the bookkeeping
+ * that stops merges and proposals from being relitigated. Plain JSON — write it
+ * wherever you like and hand it back as `resumeFrom`.
+ *
+ * Deliberately not generic over component names. It leaves the process and
+ * comes back through `JSON.parse` with plain string keys, so the narrowing back
+ * to a run's own components happens once inside the engine, guarded by the
+ * fingerprint, rather than being a type the caller has to reconstruct.
+ *
+ * A resumed run follows the same trajectory an uninterrupted one would, as long
+ * as the batch sampler reports its state and the evaluation cache is either
+ * checkpointed or disabled — a cache that is neither leaves the resumed run
+ * paying again for rollouts the first one had already bought.
+ */
+interface GepaSnapshot {
+  version: 1;
+  /**
+   * Identifies the run this checkpoint came from — seed candidate, instance
+   * ids and seed. Resuming against a different setup is refused rather than
+   * silently scoring old candidates against new data.
+   */
+  fingerprint: string;
+  records: CandidateRecord[];
+  iteration: number;
+  metricCalls: number;
+  reflectionCalls: number;
+  cacheHits: number;
+  rngState: number;
+  /** Whatever the batch sampler reports from `state()`, when it has one. */
+  sampler?: unknown;
+  rejectedProposals: Record<string, RejectedProposal[]>;
+  merge: {
+    attempts: string[];
+    descriptions: string[];
+    due: number;
+    tested: number;
+    lastIterationAccepted: boolean;
+  };
+  /** Cached instance scores, when the cache can enumerate them. */
+  cache?: [string, CachedScore][];
+}
+//#endregion
+//#region src/gepa/reflection.d.ts
+interface ReflectionPromptArgs {
+  componentName: string;
+  currentText: string;
+  records: readonly ReflectiveRecord[];
+  /** Texts already tried for this component that lost to their parent. */
+  rejected?: readonly RejectedProposal[];
+}
+type ReflectionPromptBuilder = (args: ReflectionPromptArgs) => string;
+/**
+ * Adapted from the reflection prompt in the GEPA paper (Agrawal et al., 2025).
+ * The instruction to mine domain facts out of the traces matters as much as the
+ * instruction to fix failures — most of the lift comes from the model writing
+ * down knowledge the traces revealed.
+ */
+declare function buildReflectionPrompt(args: ReflectionPromptArgs): string;
+/**
+ * Cut rather than add. Reflective evolution only ever appends — every
+ * iteration diagnoses a failure and writes a rule preventing it — so an
+ * instruction grows monotonically until it is mostly edge cases that no longer
+ * fire. Nothing else in the loop ever removes one.
+ */
+declare function buildSimplifyPrompt(args: ReflectionPromptArgs): string;
+/**
+ * Replace a rule that fits the instances it was written from with the
+ * principle behind it. Feedback is drawn from minibatches, so a rule written
+ * to fix three examples routinely encodes those three examples.
+ */
+declare function buildGeneralizePrompt(args: ReflectionPromptArgs): string;
+/**
+ * Start from the evidence rather than from the incumbent. Every other strategy
+ * edits the current text, which anchors each proposal to whatever the search
+ * happened to reach first; this one is the only escape from a bad opening.
+ */
+declare function buildRewritePrompt(args: ReflectionPromptArgs): string;
+/**
+ * A rotation covering the four directions a proposal can move in: fix what is
+ * broken, cut what is dead, widen what is too narrow, and start over.
+ *
+ * Drawing a proposal k times from one template samples one direction k times.
+ * Rotating costs nothing extra — same call count, same rollouts — and is the
+ * cheapest diversity available. Opt in via `reflection.strategies`; the
+ * default stays the published single prompt.
+ */
+declare function diverseReflectionStrategies(): ReflectionPromptBuilder[];
+//#endregion
+export { RejectedProposal as C, ReflectiveRecord as S, ValEvaluationPolicy as T, GepaStopReason as _, buildRewritePrompt as a, ProposeArgs as b, AcceptancePolicy as c, CandidateSource as d, ComponentPatch as f, GepaSnapshot as g, GepaEvent as h, buildReflectionPrompt as i, CandidateRecord as l, GepaAdapter as m, ReflectionPromptBuilder as n, buildSimplifyPrompt as o, ComponentSelector as p, buildGeneralizePrompt as r, diverseReflectionStrategies as s, ReflectionPromptArgs as t, CandidateSelector as u, MakeReflectiveDatasetArgs as v, SelectionState as w, ReflectiveDataset as x, ParetoFrontier as y };