textopt 0.0.0

package/dist/opro/index.cjs

@@ -0,0 +1,487 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_evaluation = require("../evaluation-OZOp6TB7.cjs");
+const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
+const require_text = require("../text-CK_HB3su.cjs");
+const require_rng = require("../rng-DbA_rPIo.cjs");
+//#region src/opro/optimize.ts
+/**
+* Rounds that may pass without a proposal nobody has tried before, per
+* component, before the run gives up.
+*
+* Such a round spends no rollouts, so the budget guard — the only thing
+* bounding a run at the default round limit — never fires, and a proposal
+* model that has settled on one answer would loop forever. Waiting a few
+* rounds costs one prompt each and leaves room for a stochastic model to find
+* something new; waiting indefinitely is a hang.
+*/
+const BARREN_ROUNDS = 3;
+const DEFAULT_PROPOSALS_PER_ROUND = 8;
+const DEFAULT_HISTORY_SIZE = 20;
+const DEFAULT_EXEMPLARS = 3;
+const DEFAULT_SCORE_SCALE = 100;
+const DEFAULT_FULL_EVAL_INTERVAL = 3;
+/**
+* Optimization by prompting: show the model what has been tried and what each
+* attempt scored, and ask for something better.
+*
+* The regime this is for is the one GEPA cannot serve. GEPA's advantage over
+* blind search is the per-instance feedback string — a diagnosis of *why* a
+* rollout failed. Plenty of metrics cannot produce one: a reward model, a
+* preference score, a classifier's accuracy over a closed label set. Handed
+* those, reflection is asked to diagnose a failure it has no evidence about,
+* and pays a frontier-model call to guess.
+*
+* This search asks for less and needs less. One scalar per candidate is the
+* entire signal, and the model reasons over the *trajectory* of scores rather
+* than the anatomy of a single failure. Where feedback does exist, use GEPA:
+* a score history is a much thinner channel than a paragraph saying what broke.
+*/
+var OproOptimizer = class {
+	#config;
+	constructor(config = {}) {
+		assertConfig(config);
+		this.#config = config;
+	}
+	async optimize(task) {
+		return runOpro({
+			config: this.#config,
+			task
+		});
+	}
+};
+/**
+* Adapted from the meta-prompt in *Large Language Models as Optimizers* (Yang
+* et al., 2023). Ascending order is load-bearing: the paper finds the model
+* attends most to what is nearest the end of the prompt, so the best attempt
+* has to be last.
+*
+* Exemplars carry an `<INS>` marker for the same reason the paper's do: an
+* instruction written against the task in the abstract reads differently from
+* one written to sit in a particular slot. The marker is placed ahead of the
+* input, which is where a component's text usually goes but which this
+* optimizer cannot actually know — only the adapter composes the real prompt.
+* The paper also shows each exemplar's gold answer. Nothing here needs to know
+* what that is: `renderDatum` is handed the whole datum, so a caller holding
+* labels can render them alongside the input.
+*/
+function buildOproPrompt(args) {
+	const { componentName, history, exemplars } = args;
+	return [
+		`I am tuning the "${componentName}" component of a larger system. Below are the instructions I have tried, each with the score it achieved. Higher scores are better.`,
+		"",
+		"<attempts>",
+		history.map((attempt) => `score: ${attempt.score}\n<instruction>\n${attempt.text}\n</instruction>`).join("\n\n"),
+		"</attempts>",
+		...exemplars.length === 0 ? [] : [
+			"",
+			"Here are examples of the inputs this component receives. `<INS>` marks where your instruction is read:",
+			"",
+			"<inputs>",
+			exemplars.map((exemplar) => `input:\n<INS>\n${exemplar}`).join("\n\n"),
+			"</inputs>"
+		],
+		"",
+		"Write a new instruction that will score higher than every instruction above.",
+		"Work out what the higher-scoring instructions do that the lower-scoring ones do not, and push further in that direction.",
+		"Do not repeat an instruction that has already been tried.",
+		"",
+		"Return only the new instruction, inside a ``` block."
+	].join("\n");
+}
+async function runOpro(args) {
+	const { config, task } = args;
+	const { proposalsPerRound = DEFAULT_PROPOSALS_PER_ROUND, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, maxReflectionCalls = Number.POSITIVE_INFINITY, seed = 0, historySize = DEFAULT_HISTORY_SIZE, exemplars = DEFAULT_EXEMPLARS, scoringSetSize, fullEvalInterval = DEFAULT_FULL_EVAL_INTERVAL, scoreScale = DEFAULT_SCORE_SCALE, buildPrompt = buildOproPrompt, trackBestOutputs = false, checkpointCache = true } = config;
+	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, renderDatum = renderDefault, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
+	const deadline = require_evaluation.createDeadline({ maxWallClockMs });
+	const components = require_evaluation.componentNames(seedCandidate);
+	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
+	if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
+	if (components.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
+	if (testSet !== void 0 && testSet.length === 0) throw new Error("optimize requires a non-empty testSet when one is given; omit it to skip held-out evaluation");
+	const validationIds = validationSet.map((datum, index) => instanceId({
+		datum,
+		index
+	}));
+	const testIds = testSet?.map((datum, index) => instanceId({
+		datum,
+		index
+	})) ?? [];
+	const fingerprint = require_evaluation.runFingerprint({
+		seedCandidate,
+		trainingIds: trainingSet.map((datum, index) => instanceId({
+			datum,
+			index
+		})),
+		validationIds,
+		seed,
+		...cacheNamespace === void 0 ? {} : { cacheNamespace }
+	});
+	require_evaluation.assertResumable({
+		fingerprint,
+		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
+	});
+	const rng = require_rng.createSeededRng(seed, resumeFrom?.rngState);
+	const budget = require_evaluation.createBudget({
+		maxMetricCalls,
+		spent: resumeFrom?.metricCalls ?? 0
+	});
+	const evaluationCache = cache === false ? void 0 : cache ?? require_evaluation.createMemoryCache();
+	const evaluator = require_evaluation.createEvaluator({
+		adapter,
+		budget,
+		...retry === void 0 ? {} : { retry },
+		...cacheNamespace === void 0 ? {} : { cacheNamespace },
+		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
+		trackOutputs: trackBestOutputs,
+		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...signal === void 0 ? {} : { signal },
+		onEvaluation: (event) => onEvent?.({
+			type: "evaluation",
+			...event
+		})
+	});
+	evaluator.restore(resumeFrom?.cache ?? []);
+	function drawExemplars() {
+		return rng.sample(trainingSet, Math.min(exemplars, trainingSet.length)).map(renderDatum);
+	}
+	const scoringIndices = resumeFrom?.scoringIndices ?? (scoringSetSize === void 0 ? void 0 : rng.sample(trainingSet.map((_, index) => index), Math.min(scoringSetSize, trainingSet.length)));
+	const scoringSet = scoringIndices?.map((index) => trainingSet[index]);
+	const scoringIds = scoringIndices?.map((index) => instanceId({
+		datum: trainingSet[index],
+		index
+	}));
+	const histories = new Map(components.map((name) => [name, [...resumeFrom?.histories[name] ?? []]]));
+	function contextOf(candidate, component) {
+		const rest = {};
+		for (const name of components) if (name !== component) rest[name] = candidate[name];
+		return require_evaluation.stableHash(rest);
+	}
+	const trajectory = [];
+	let round = resumeFrom?.round ?? 0;
+	let reflectionCalls = resumeFrom?.reflectionCalls ?? 0;
+	let stopReason = "maxRounds";
+	onEvent?.({
+		type: "start",
+		components,
+		validationSetSize: validationSet.length
+	});
+	async function sweep(candidate, phase) {
+		return evaluator.evaluate({
+			candidate,
+			batch: validationSet,
+			ids: validationIds,
+			split: "val",
+			phase,
+			candidateId: null,
+			iteration: round
+		});
+	}
+	/**
+	* Screens a candidate. With a scoring set that is a slice of the training set;
+	* without one it is the full validation set, and screening and reporting are the
+	* same measurement.
+	*/
+	async function screen(candidate, phase) {
+		if (scoringSet === void 0) return sweep(candidate, phase);
+		return evaluator.evaluate({
+			candidate,
+			batch: scoringSet,
+			ids: scoringIds,
+			split: "train",
+			phase: "minibatch",
+			candidateId: null,
+			iteration: round
+		});
+	}
+	const seedEvaluation = resumeFrom === void 0 ? await sweep(seedCandidate, "seed") : void 0;
+	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_evaluation.requireMeasuredMean({
+		batch: seedEvaluation,
+		phase: "seed"
+	});
+	let best = resumeFrom?.best ?? seedCandidate;
+	let bestScore = resumeFrom?.bestScore ?? seedScore;
+	/** Absent on a resumed run until a sweep wins: outputs are not checkpointed. */
+	let bestOutputs = seedEvaluation?.outputs;
+	let reported = resumeFrom?.reported ?? seedCandidate;
+	let lastSwept = resumeFrom === void 0 ? seedCandidate : resumeFrom.incumbentSwept ? best : {};
+	let bestSearchScore = resumeFrom?.bestSearchScore ?? seedScore;
+	if (resumeFrom === void 0 && scoringSet !== void 0) bestSearchScore = require_evaluation.requireMeasuredMean({
+		batch: await screen(seedCandidate, "seed"),
+		phase: "seed"
+	});
+	/**
+	* Measures the incumbent on the whole validation set. The search chooses by subset
+	* score and can therefore chase something that only works on the subset, so
+	* what gets reported is the best candidate a full sweep has actually seen —
+	* never a subset number wearing a validation set label.
+	*/
+	async function refreshIncumbent() {
+		if (best === lastSwept || !budget.canAfford(validationSet.length)) return "ok";
+		try {
+			const evaluation = await sweep(best, "validation");
+			const full = require_evaluation.measuredMean(evaluation);
+			lastSwept = best;
+			if (full !== void 0 && full > bestScore) {
+				reported = best;
+				bestScore = full;
+				bestOutputs = evaluation.outputs;
+			}
+		} catch (err) {
+			if (err instanceof require_evaluation.BudgetExhausted || signal?.aborted) return "stop";
+			throw err;
+		}
+		return "ok";
+	}
+	if (resumeFrom === void 0) {
+		for (const name of components) histories.get(name)?.push({
+			text: seedCandidate[name],
+			score: scaleScore(bestSearchScore, scoreScale),
+			context: contextOf(seedCandidate, name)
+		});
+		trajectory.push({
+			round: 0,
+			component: components[0],
+			candidate: seedCandidate,
+			score: seedScore
+		});
+	}
+	function takeSnapshot() {
+		const cached = checkpointCache ? evaluationCache?.entries?.() : void 0;
+		return {
+			version: 1,
+			fingerprint,
+			best,
+			reported,
+			incumbentSwept: best === lastSwept,
+			bestScore,
+			bestSearchScore,
+			seedScore,
+			round,
+			reflectionCalls,
+			metricCalls: budget.spent(),
+			cacheHits: evaluator.cacheHits(),
+			rngState: rng.state(),
+			histories: Object.fromEntries([...histories].map(([name, attempts]) => [name, [...attempts]])),
+			...scoringIndices === void 0 ? {} : { scoringIndices },
+			...cached === void 0 ? {} : { cache: cached }
+		};
+	}
+	async function checkpoint() {
+		if (onCheckpoint === void 0) return;
+		await onCheckpoint(takeSnapshot());
+	}
+	await checkpoint();
+	let barrenRounds = 0;
+	while (round < maxRounds) {
+		if (signal?.aborted) {
+			stopReason = "aborted";
+			break;
+		}
+		if (barrenRounds >= BARREN_ROUNDS * components.length) {
+			stopReason = "proposalsExhausted";
+			break;
+		}
+		if (require_evaluation.costExhausted({
+			usage: evaluator.usage(),
+			maxCostUsd
+		})) {
+			stopReason = "costExhausted";
+			break;
+		}
+		if (deadline.exceeded()) {
+			stopReason = "deadlineReached";
+			break;
+		}
+		if (reflectionCalls >= maxReflectionCalls) {
+			stopReason = "reflectionBudgetExhausted";
+			break;
+		}
+		if (!budget.canAfford(validationSet.length)) {
+			stopReason = "budgetExhausted";
+			break;
+		}
+		const component = components[round % components.length];
+		const history = histories.get(component);
+		const context = contextOf(best, component);
+		const comparable = history.filter((attempt) => attempt.context === context);
+		onEvent?.({
+			type: "roundStart",
+			round,
+			component,
+			historySize: comparable.length
+		});
+		const affordableProposals = Math.min(proposalsPerRound, maxReflectionCalls - reflectionCalls);
+		const prompt = buildPrompt({
+			componentName: component,
+			history: topAttempts({
+				history: comparable,
+				keep: historySize
+			}),
+			exemplars: drawExemplars()
+		});
+		reflectionCalls += affordableProposals;
+		const drawn = await require_concurrency.mapWithConcurrency({
+			items: Array.from({ length: affordableProposals }, (_, index) => index),
+			limit: concurrency,
+			signal,
+			task: async () => require_text.parseProposedText(await reflect({
+				prompt,
+				...signal === void 0 ? {} : { signal }
+			}))
+		});
+		const tried = new Set(comparable.map((attempt) => attempt.text));
+		const unique = [...new Set(drawn)].filter((text) => text.length > 0 && !tried.has(text));
+		barrenRounds = unique.length === 0 ? barrenRounds + 1 : 0;
+		let roundStop;
+		for (const text of unique) {
+			const candidate = {
+				...best,
+				[component]: text
+			};
+			let evaluation;
+			try {
+				evaluation = await screen(candidate, "validation");
+			} catch (err) {
+				if (err instanceof require_evaluation.BudgetExhausted) {
+					roundStop = "budgetExhausted";
+					break;
+				}
+				if (signal?.aborted) {
+					roundStop = "aborted";
+					break;
+				}
+				throw err;
+			}
+			const score = require_evaluation.measuredMean(evaluation);
+			if (score === void 0) continue;
+			const accepted = score > bestSearchScore;
+			history.push({
+				text,
+				score: scaleScore(score, scoreScale),
+				context
+			});
+			trajectory.push({
+				round: round + 1,
+				component,
+				candidate,
+				score
+			});
+			onEvent?.({
+				type: "attempt",
+				round,
+				component,
+				score,
+				accepted
+			});
+			if (accepted) {
+				best = candidate;
+				bestSearchScore = score;
+				if (scoringSet === void 0) {
+					reported = candidate;
+					lastSwept = candidate;
+					bestScore = score;
+					bestOutputs = evaluation.outputs;
+				}
+				const scaled = scaleScore(score, scoreScale);
+				for (const name of components) {
+					if (name === component) continue;
+					histories.get(name)?.push({
+						text: candidate[name],
+						score: scaled,
+						context: contextOf(candidate, name)
+					});
+				}
+			}
+		}
+		round += 1;
+		await checkpoint();
+		if (roundStop !== void 0) {
+			stopReason = roundStop;
+			break;
+		}
+		if (scoringSet !== void 0 && round % fullEvalInterval === 0) {
+			if (await refreshIncumbent() === "stop") {
+				stopReason = signal?.aborted ? "aborted" : "budgetExhausted";
+				break;
+			}
+		}
+	}
+	if (scoringSet !== void 0 && !signal?.aborted) await refreshIncumbent();
+	if (signal?.aborted) stopReason = "aborted";
+	const testScore = testSet === void 0 ? void 0 : require_evaluation.measuredMean(await evaluator.evaluate({
+		candidate: best,
+		batch: testSet,
+		ids: testIds,
+		split: "test",
+		phase: "test",
+		candidateId: null,
+		iteration: round,
+		charge: false
+	}));
+	onEvent?.({
+		type: "finish",
+		reason: stopReason,
+		bestScore,
+		metricCalls: budget.spent(),
+		...testScore === void 0 ? {} : { testScore }
+	});
+	return {
+		snapshot: takeSnapshot(),
+		bestCandidate: reported,
+		bestScore,
+		usage: evaluator.usage(),
+		seedScore,
+		...trackBestOutputs ? { bestOutputs } : {},
+		...testScore === void 0 ? {} : {
+			testScore,
+			testMetricCalls: evaluator.unchargedCalls()
+		},
+		rounds: round,
+		trajectory,
+		metricCalls: budget.spent(),
+		reflectionCalls,
+		cacheHits: evaluator.cacheHits(),
+		stopReason
+	};
+}
+/**
+* The strongest `keep` attempts, ordered weakest first. Dropping the weak tail
+* rather than the old one is deliberate: the prompt's job is to show a
+* gradient, and an attempt that scored badly a hundred rounds ago still marks
+* the bottom of it.
+*/
+function topAttempts(args) {
+	const { history, keep } = args;
+	return [...history].sort((a, b) => b.score - a.score).slice(0, keep).sort((a, b) => a.score - b.score);
+}
+function scaleScore(score, scale) {
+	return Math.round(score * scale);
+}
+function renderDefault(datum) {
+	if (typeof datum === "string") return datum;
+	try {
+		return JSON.stringify(datum, null, 2) ?? String(datum);
+	} catch {
+		return String(datum);
+	}
+}
+function assertConfig(config) {
+	const positive = [
+		["proposalsPerRound", config.proposalsPerRound],
+		["concurrency", config.concurrency],
+		["maxRounds", config.maxRounds],
+		["maxReflectionCalls", config.maxReflectionCalls],
+		["historySize", config.historySize],
+		["scoringSetSize", config.scoringSetSize],
+		["fullEvalInterval", config.fullEvalInterval]
+	];
+	for (const [name, value] of positive) if (value !== void 0 && (!Number.isInteger(value) || value < 1)) throw new Error(`${name} must be a positive integer, received ${value}`);
+	if (config.exemplars !== void 0 && (!Number.isInteger(config.exemplars) || config.exemplars < 0)) throw new Error(`exemplars must be a non-negative integer, received ${config.exemplars}`);
+}
+function defaultInstanceId(args) {
+	const hash = require_evaluation.stableHash(args.datum);
+	return hash === "" ? String(args.index) : hash;
+}
+//#endregion
+exports.OproOptimizer = OproOptimizer;
+exports.buildOproPrompt = buildOproPrompt;