textopt 0.0.0 → 0.2.0

package/dist/gepa/index.mjs

@@ -1,7 +1,7 @@
-import { c as candidateFingerprint, d as createMemoryCache, f as stableHash, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "../evaluation-BV0nSZVx.mjs";
+import { S as createBudget, _ as componentNames, a as instanceRow, b as defaultInstanceId, c as BudgetExhausted, d as measuredMean, g as runFingerprint, h as candidateFingerprint, i as flushReporters, l as costExhausted, m as assertResumable, n as seedScoreWarnings, o as isCandidateAccepted, p as createDeadline, r as createEmitter, s as isRunFinished, t as resolveValidationSet, u as createEvaluator, y as createMemoryCache } from "../warnings-OxvDi9kN.mjs";
 import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
-import { i as sum, n as mean, r as signFlipPValue, t as argmax } from "../math-DhrDmpFS.mjs";
-import { n as formatDemos, r as parseDemos } from "../demos-Degx6UmP.mjs";
+import { a as sum, i as signFlipPValue, r as mean, t as argmax } from "../math-Dqme4rYz.mjs";
+import { r as parseDemos, t as formatDemos } from "../demos-Brobjfuc.mjs";
 import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
 import { a as createDefaultProposer, i as buildSimplifyPrompt, n as buildReflectionPrompt, o as diverseReflectionStrategies, r as buildRewritePrompt, t as buildGeneralizePrompt } from "../reflection-mwMhrjs_.mjs";
 import { t as createEpochShuffledSampler } from "../sampling-CfHt7Gue.mjs";
@@ -512,7 +512,9 @@ function hasSurvivor(args) {
 * `frontier` chooses what the fronts are taken over. "instance" is GEPA as
 * published. "objective" tracks candidates leading each named objective the
 * adapter reports, and "hybrid" pools both — a candidate then earns selection
-* weight for every instance it wins *and* every objective it leads.
+* weight for every instance it wins *and* every objective it leads. The
+* objectives are whatever the adapter put in `objectiveScores`, which for a
+* judge is every criterion it graded, including any at `weight: 0`.
 */
 function paretoSelector(args = {}) {
 	const { epsilon = 0, frontier = "instance" } = args;
@@ -659,7 +661,8 @@ function improvementAcceptance(args = {}) {
 */
 function pairedPermutationAcceptance(args = {}) {
 	const { alpha = .2, maxExact = 16 } = args;
-	return ({ parentScores, childScores }) => {
+	if (!Number.isFinite(alpha) || alpha <= 0 || alpha > 1) throw new Error(`alpha must be greater than 0 and at most 1, received ${alpha}`);
+	const policy = ({ parentScores, childScores }) => {
 		const differences = [];
 		for (let index = 0; index < parentScores.length; index += 1) differences.push(childScores[index] - parentScores[index]);
 		const observed = sum(differences);
@@ -670,6 +673,33 @@ function pairedPermutationAcceptance(args = {}) {
 			maxExact
 		}) <= alpha;
 	};
+	policy.minimumPairs = smallestAcceptableBatch({
+		alpha,
+		maxExact
+	});
+	return policy;
+}
+/**
+* Smallest paired batch on which a sign-flip test at `alpha` could return a
+* verdict of "accept" at all.
+*
+* Enumerating n non-zero differences gives 2^n equally likely sign
+* assignments, so the smallest attainable p-value is 2^-n and no batch below
+* `log2(1/alpha)` pairs can clear the bar. Past `maxExact` the p-value comes
+* from a normal approximation, which has no such floor — so when the exact
+* requirement is out of that regime's reach, the first batch that leaves the
+* regime is the honest answer rather than an exact size that never applies.
+*/
+function smallestAcceptableBatch(args) {
+	const { alpha, maxExact } = args;
+	let pairs = 1;
+	let smallestPValue = .5;
+	while (smallestPValue > alpha) {
+		pairs += 1;
+		smallestPValue /= 2;
+		if (pairs > maxExact) return maxExact + 1;
+	}
+	return pairs;
 }
 /**
 * Highest mean over the instances it was scored on, with wider coverage
@@ -738,16 +768,24 @@ var GepaOptimizer = class {
 		this.#config = config;
 	}
 	async optimize(task) {
-		return runGepa({
-			config: this.#config,
-			task
-		});
+		try {
+			return await runGepa({
+				config: this.#config,
+				task
+			});
+		} finally {
+			await flushReporters(task.reporters ?? []);
+		}
 	}
 };
 async function runGepa(args) {
 	const { config, task } = args;
 	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, seed = 0, candidateSelector = paretoSelector(), acceptance = improvementAcceptance(), merge, skipPerfectScore = true, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals, reflection, checkpointCache = true, trackBestOutputs = false, raiseOnError = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, componentSelector = roundRobinComponentSelector(), batchSampler = createEpochShuffledSampler({ minibatchSize }), valEvaluationPolicy = fullEvaluationPolicy(), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, maxMetricCalls, componentSelector = roundRobinComponentSelector(), batchSampler = createEpochShuffledSampler({ minibatchSize }), valEvaluationPolicy = fullEvaluationPolicy(), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
 	const deadline = createDeadline({ maxWallClockMs });
 	const seedComponents = componentNames(seedCandidate);
 	const mergeConfig = {
@@ -827,8 +865,26 @@ async function runGepa(args) {
 	let mergesDue = resumeFrom?.merge.due ?? 0;
 	let totalMergesTested = resumeFrom?.merge.tested ?? 0;
 	let lastIterationAccepted = resumeFrom?.merge.lastIterationAccepted ?? false;
-	function emit(event) {
-		onEvent?.(event);
+	const emit = createEmitter(reporters);
+	/**
+	* Everything an acceptance means, in one event: the text, the aggregate, and
+	* the row it put on the frontier. Emitted from one place because the merge
+	* path and the mutation path accept candidates separately, and a payload
+	* assembled twice is a payload that drifts.
+	*/
+	function emitAccepted(record) {
+		const outputs = outputsByCandidate.get(record.id);
+		emit({
+			type: "candidateAccepted",
+			iteration,
+			candidateId: record.id,
+			parentIds: record.parentIds,
+			aggregateScore: record.aggregateScore,
+			source: record.source,
+			candidate: record.candidate,
+			instanceScores: record.instanceScores,
+			...outputs === void 0 ? {} : { outputs }
+		});
 	}
 	/**
 	* Copies everything mutable: a snapshot handed to `onCheckpoint` is a record
@@ -845,6 +901,7 @@ async function runGepa(args) {
 			metricCalls: budget.spent(),
 			reflectionCalls,
 			cacheHits: evaluator.cacheHits(),
+			usage: evaluator.usage(),
 			...samplerState === void 0 ? {} : { sampler: samplerState },
 			rejectedProposals: snapshotRejections({
 				rejections: rejectedProposals,
@@ -873,6 +930,7 @@ async function runGepa(args) {
 		...evaluationCache === void 0 ? {} : { cache: evaluationCache },
 		trackOutputs: trackBestOutputs,
 		cacheHits: resumeFrom?.cacheHits ?? 0,
+		...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
 		...signal === void 0 ? {} : { signal },
 		onEvaluation: (event) => emit({
 			type: "evaluation",
@@ -992,18 +1050,23 @@ async function runGepa(args) {
 	if (records.length === 0) {
 		const seedInstances = selectValInstances(seedCandidate);
 		if (!budget.canAfford(seedInstances.length)) throw new Error(`maxMetricCalls (${maxMetricCalls}) is smaller than the ${seedInstances.length} validation instances selected for scoring; the seed candidate cannot be scored`);
-		addCandidate({
+		const seedEvaluation = await evaluateValidation({
+			candidate: seedCandidate,
+			instances: seedInstances,
+			phase: "seed",
+			candidateId: 0
+		});
+		emitAccepted(addCandidate({
 			candidate: seedCandidate,
 			parentIds: [],
-			evaluation: await evaluateValidation({
-				candidate: seedCandidate,
-				instances: seedInstances,
-				phase: "seed",
-				candidateId: 0
-			}),
+			evaluation: seedEvaluation,
 			source: "seed",
 			updatedComponents: []
-		});
+		}));
+		warnings.push(...seedScoreWarnings({
+			scores: seedEvaluation.scores,
+			perfectScore
+		}));
 		lastIterationAccepted = false;
 		mergesDue = 0;
 		await checkpoint();
@@ -1094,14 +1157,7 @@ async function runGepa(args) {
 		});
 		mergesDue -= 1;
 		totalMergesTested += 1;
-		emit({
-			type: "candidateAccepted",
-			iteration,
-			candidateId: record.id,
-			parentIds: record.parentIds,
-			aggregateScore: record.aggregateScore,
-			source: "merge"
-		});
+		emitAccepted(record);
 		return "attempted";
 	}
 	/**
@@ -1346,21 +1402,13 @@ async function runGepa(args) {
 				stop ??= "budgetExhausted";
 				break;
 			}
-			const record = addCandidate({
+			emitAccepted(addCandidate({
 				candidate: item.outcome.child,
 				parentIds: [item.outcome.plan.parent.id],
 				evaluation,
 				source: "mutation",
 				updatedComponents: componentNames(item.outcome.proposed)
-			});
-			emit({
-				type: "candidateAccepted",
-				iteration,
-				candidateId: record.id,
-				parentIds: record.parentIds,
-				aggregateScore: record.aggregateScore,
-				source: "mutation"
-			});
+			}));
 		}
 		return stop;
 	}
@@ -1444,7 +1492,7 @@ async function runGepa(args) {
 	}
 	const bestCandidateId = valEvaluationPolicy.bestCandidate(records);
 	const best = records[bestCandidateId];
-	const testScore = testSet === void 0 ? void 0 : measuredMean(await evaluateCached({
+	const heldOut = testSet === void 0 ? void 0 : await evaluateCached({
 		candidate: best.candidate,
 		batch: testSet,
 		ids: testIds,
@@ -1452,13 +1500,18 @@ async function runGepa(args) {
 		phase: "test",
 		candidateId: bestCandidateId,
 		charge: false
-	}));
+	});
+	const testScore = heldOut === void 0 ? void 0 : measuredMean(heldOut);
 	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
 		bestCandidateId,
+		bestScore: best.aggregateScore,
 		metricCalls: budget.spent(),
-		...testScore === void 0 ? {} : { testScore }
+		...testScore === void 0 ? {} : { testScore },
+		...heldOut === void 0 ? {} : { testInstanceScores: instanceRow(heldOut) },
+		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	const perObjectiveBest = collectPerObjectiveBest(records);
 	const bestOutputs = outputsByCandidate.get(bestCandidateId);
@@ -1469,7 +1522,8 @@ async function runGepa(args) {
 		bestCandidateId,
 		...testScore === void 0 ? {} : {
 			testScore,
-			testMetricCalls: evaluator.unchargedCalls()
+			testMetricCalls: evaluator.unchargedCalls(),
+			testUsage: evaluator.unchargedUsage()
 		},
 		...bestOutputs === void 0 ? {} : { bestOutputs },
 		candidates: records,
@@ -1480,6 +1534,7 @@ async function runGepa(args) {
 		reflectionCalls,
 		cacheHits: evaluator.cacheHits(),
 		iterations: iteration,
+		warnings,
 		stopReason,
 		snapshot: takeSnapshot()
 	};
@@ -1492,13 +1547,15 @@ async function runGepa(args) {
 function assertGepaConfig(config) {
 	if (config.reflection?.buildPrompt !== void 0 && config.reflection.strategies !== void 0) throw new Error("reflection takes buildPrompt or strategies, not both");
 	if (config.reflection?.strategies?.length === 0) throw new Error("reflection.strategies must not be empty");
-	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals } = config;
+	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, acceptance, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals } = config;
 	const proposalsPerIteration = proposals?.perIteration ?? 1;
 	const proposalConcurrency = proposals?.concurrency ?? 1;
 	if (!Number.isInteger(proposalsPerIteration) || proposalsPerIteration < 1) throw new Error(`proposals.perIteration must be a positive integer, received ${proposalsPerIteration}`);
 	if (!Number.isInteger(proposalConcurrency) || proposalConcurrency < 1) throw new Error(`proposals.concurrency must be a positive integer, received ${proposalConcurrency}`);
 	keepCount(proposals?.selection ?? "all");
 	if (!Number.isInteger(minibatchSize) || minibatchSize < 1) throw new Error(`minibatchSize must be a positive integer, received ${minibatchSize}`);
+	const minimumPairs = acceptance?.minimumPairs;
+	if (minimumPairs !== void 0 && minibatchSize < minimumPairs) throw new Error(`this acceptance policy cannot accept anything on fewer than ${minimumPairs} instances, but minibatchSize is ${minibatchSize}; raise minibatchSize or loosen the policy`);
 	if (!Number.isFinite(perfectScore)) throw new Error(`perfectScore must be a finite number, received ${perfectScore}`);
 	if (!Number.isInteger(rejectedProposalMemory) || rejectedProposalMemory < 0) throw new Error(`rejectedProposalMemory must be a non-negative integer, received ${rejectedProposalMemory}`);
 	if (maxIterations !== Number.POSITIVE_INFINITY && (!Number.isInteger(maxIterations) || maxIterations < 0)) throw new Error(`maxIterations must be a non-negative integer or Infinity, received ${maxIterations}`);
@@ -1617,17 +1674,6 @@ function collectDominatorIds(records) {
 	return [...ids].sort((a, b) => a - b);
 }
 /**
-* Names an instance by a hash of its content rather than by the content
-* itself: the id ends up inside every cache key and inside the checkpoint
-* fingerprint, and embedding whole examples there costs memory proportional to
-* the dataset for no benefit. Data that will not serialize falls back to its
-* position, which is stable for as long as the dataset order is.
-*/
-function defaultInstanceId(args) {
-	const hash = stableHash(args.datum);
-	return hash === "" ? String(args.index) : hash;
-}
-/**
 * The two rollout sets restricted to the instances both of them measured.
 *
 * Screening is a paired comparison over one minibatch: a transient row is a
@@ -1649,4 +1695,4 @@ function pairMeasured(args) {
 	};
 }
 //#endregion
-export { GepaOptimizer, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };
+export { GepaOptimizer, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, isCandidateAccepted, isRunFinished, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };

package/dist/index.cjs

@@ -1,11 +1,13 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_evaluation = require("./evaluation-OZOp6TB7.cjs");
+const require_warnings = require("./warnings-CWRJF-jA.cjs");
 const require_concurrency = require("./concurrency-C-cFzWW2.cjs");
-const require_math = require("./math-COOofUyv.cjs");
-const require_demos = require("./demos-B9BJiNKz.cjs");
+const require_math = require("./math-BhlziRPc.cjs");
+const require_demos = require("./demos-9v5ts7F3.cjs");
 const require_text = require("./text-CK_HB3su.cjs");
 //#region src/compare.ts
-const EXACT_LIMIT = 16;
+const EXACT_LIMIT = 20;
+/** Well above float subtraction noise (~1e-16), well below a real margin. */
+const DEGENERACY_TOLERANCE = 1e-9;
 /**
 * Run several optimizers over the same seeds and report which one actually won.
 *
@@ -20,7 +22,7 @@ const EXACT_LIMIT = 16;
 * seed is constructor config and every optimizer here is deterministic given
 * one — comparing two entrants at a single seed compares two anecdotes. It also
 * leaves the task where the caller builds it, which is the only place the
-* optimizer-specific parts of it (`reflect`, `cache`, `onEvent`) are known.
+* optimizer-specific parts of it (`reflect`, `cache`, `reporters`) are known.
 */
 async function compare(args) {
 	const { entrants, seeds, concurrency = 1 } = args;
@@ -43,6 +45,8 @@ async function compare(args) {
 				bestScore: result.bestScore,
 				...result.testScore === void 0 ? {} : { testScore: result.testScore },
 				metricCalls: result.metricCalls,
+				cacheHits: result.cacheHits,
+				reflectionCalls: result.reflectionCalls ?? 0,
 				costUsd: result.usage.costUsd,
 				stopReason: result.stopReason
 			};
@@ -53,17 +57,25 @@ async function compare(args) {
 		runs: runs.filter((run) => run.entrant === entrant)
 	}));
 	const winner = summaries.reduce((best, summary) => summary.meanScore > best.meanScore ? summary : best);
+	const rawPValues = summaries.map((summary) => summary.entrant === winner.entrant ? void 0 : margin({
+		winner: winner.entrant,
+		entrant: summary.entrant,
+		runs,
+		seeds
+	}));
+	const holmAdjusted = holmAdjustSparse({
+		pValues: rawPValues,
+		familySize: names.length - 1
+	});
 	return {
 		winner: winner.entrant,
-		summaries: summaries.map((summary) => summary.entrant === winner.entrant ? summary : {
+		summaries: summaries.map((summary, index) => ({
 			...summary,
-			pValueVsWinner: margin({
-				winner: winner.entrant,
-				entrant: summary.entrant,
-				runs,
-				seeds
-			})
-		}),
+			...rawPValues[index] === void 0 ? {} : {
+				pValueVsWinner: rawPValues[index],
+				pValueVsWinnerHolm: holmAdjusted[index]
+			}
+		})),
 		runs
 	};
 }
@@ -78,13 +90,28 @@ function summarize(args) {
 		minScore: Math.min(...scores),
 		maxScore: Math.max(...scores),
 		meanMetricCalls: mean(runs.map((run) => run.metricCalls)),
-		meanCostUsd: mean(runs.map((run) => run.costUsd))
+		meanCostUsd: mean(runs.map((run) => run.costUsd)),
+		meanCacheHits: mean(runs.map((run) => run.cacheHits)),
+		meanReflectionCalls: mean(runs.map((run) => run.reflectionCalls)),
+		distinctScores: new Set(scores).size
 	};
 }
 /**
 * Paired across seeds rather than pooled: the same seed puts both entrants on
 * the same sampling order, so the difference at a seed is a comparison and the
 * spread between seeds is not.
+*
+* Returns `undefined` when every seed produced the exact same nonzero margin.
+* That is not evidence of an n-seed-strong result — it is one realization the
+* seed never varied, and a sign-flip p-value would report a precision from n
+* independent trials that never happened. A margin of exactly zero every seed
+* is not this case: `signFlipPValue` already reports that honestly as 1, no
+* significance claimed either way, which is not a fabricated number.
+*
+* "Exact same" is judged within `DEGENERACY_TOLERANCE`, not `===`: subtracting
+* two scores that are equal in substance can still land a few ULPs apart
+* (0.95 - 0.55 and 0.9 - 0.5 differ at the 16th digit), and treating that as
+* n real trials would be the same fabrication this check exists to prevent.
 */
 function margin(args) {
 	const { winner, entrant, runs, seeds } = args;
@@ -99,6 +126,10 @@ function margin(args) {
 			seed
 		});
 	});
+	const [first] = differences;
+	if (first !== void 0) {
+		if (differences.reduce((widest, difference) => Math.max(widest, Math.abs(difference - first)), 0) < DEGENERACY_TOLERANCE && Math.abs(first) > DEGENERACY_TOLERANCE) return;
+	}
 	return require_math.signFlipPValue({
 		differences,
 		observed: differences.reduce((total, value) => total + value, 0),
@@ -109,6 +140,27 @@ function scoreOf(args) {
 	const { runs, entrant, seed } = args;
 	return runs.find((candidate) => candidate.entrant === entrant && candidate.seed === seed)?.score ?? 0;
 }
+/**
+* `holmAdjust` over the raw p-values that exist, skipping the slots a
+* withheld comparison left `undefined` — those still count toward
+* `familySize`, they just have nothing of their own to adjust.
+*/
+function holmAdjustSparse(args) {
+	const { pValues, familySize } = args;
+	const present = pValues.map((pValue, index) => ({
+		pValue,
+		index
+	})).filter((entry) => entry.pValue !== void 0);
+	const adjusted = require_math.holmAdjust({
+		pValues: present.map((entry) => entry.pValue),
+		familySize
+	});
+	const result = new Array(pValues.length).fill(void 0);
+	present.forEach((entry, rank) => {
+		result[entry.index] = adjusted[rank];
+	});
+	return result;
+}
 function mean(values) {
 	return values.reduce((total, value) => total + value, 0) / values.length;
 }
@@ -119,6 +171,51 @@ function standardDeviation(values) {
 	return Math.sqrt(variance);
 }
 //#endregion
+//#region src/distillation.ts
+const ROLES = /* @__PURE__ */ new Set([
+	"system",
+	"user",
+	"assistant"
+]);
+/**
+* Serialize harvested rollouts as JSONL, one training example per line.
+*
+* The chat-messages shape rather than any one vendor's: it is what Axolotl,
+* Together, Fireworks and the Hugging Face trainers all ingest, and the
+* providers that read it outlive the ones that do not.
+*
+* Returns the text rather than writing it. Only `file-cache` touches the
+* filesystem, and a caller uploading this straight to a provider should not
+* have to round-trip it through a file to do so.
+*/
+function toTrainingJsonl(args) {
+	const { rollouts, render } = args;
+	const lines = [];
+	for (const [index, rollout] of rollouts.entries()) {
+		const example = render({
+			rollout,
+			index
+		});
+		if (example === null) continue;
+		assertUsable(example, index);
+		lines.push(JSON.stringify(example));
+	}
+	return lines.join("\n");
+}
+/**
+* Checked here rather than left to the provider. A malformed line surfaces as
+* a rejected upload hours later, naming a line number in a file the caller
+* never wrote by hand; naming the rollout at the point it was rendered is the
+* same error while it is still fixable.
+*/
+function assertUsable(example, index) {
+	if (!Array.isArray(example.messages) || example.messages.length === 0) throw new Error(`rollout ${index} rendered no messages`);
+	for (const message of example.messages) {
+		if (!ROLES.has(message.role)) throw new Error(`rollout ${index} rendered the unknown role ${message.role}`);
+		if (typeof message.content !== "string") throw new Error(`rollout ${index} rendered non-text content`);
+	}
+}
+//#endregion
 //#region src/judge.ts
 const DEFAULT_SCALE = 5;
 const SCORE = /<score\s+name="([^"]+)"\s*>\s*([\d.]+)\s*<\/score>/g;
@@ -141,6 +238,11 @@ const FEEDBACK = /<feedback>([\s\S]*?)<\/feedback>/;
 function createJudge(args) {
 	const { model, criteria, scale = DEFAULT_SCALE, renderInput = stringify, renderOutput = stringify, buildPrompt = buildJudgePrompt } = args;
 	if (criteria.length === 0) throw new Error("createJudge requires at least one criterion");
+	if (!Number.isFinite(scale) || scale <= 0) throw new Error(`scale must be a positive number, received ${scale}`);
+	assertCriteria({
+		criteria,
+		scale
+	});
 	return async ({ input, output, expected, signal }) => {
 		return readVerdict({
 			response: await model({
@@ -184,6 +286,7 @@ function buildJudgePrompt(args) {
 		`Grade each criterion from 0 to ${scale}, where ${scale} is a perfect answer.`,
 		"",
 		"Then write feedback. It is read by a program that rewrites the system's instructions, not by a person reviewing this output, so say what the instructions should tell the system to do differently. Feedback about this particular answer is of no use to it.",
+		...expected === void 0 ? [] : ["", "Do not restate the expected answer, or any fact drawn from it, in the feedback. The instruction it is rewritten into is reused on inputs whose answers you have not seen: a fact copied out of the expected answer becomes an answer key memorised in the prompt, which raises the score on this input and teaches the system nothing. Name the kind of thing the answer was missing, not the thing itself."],
 		"",
 		"Reply in exactly this format and nothing else:",
 		...criteria.map((criterion) => `<score name="${criterion.name}">…</score>`),
@@ -202,6 +305,9 @@ function readVerdict(args) {
 	const graded = /* @__PURE__ */ new Map();
 	for (const match of response.matchAll(SCORE)) graded.set(match[1], Number(match[2]));
 	const objectiveScores = {};
+	let gated = false;
+	let weighted = 0;
+	let totalWeight = 0;
 	for (const criterion of criteria) {
 		const grade = graded.get(criterion.name);
 		if (grade === void 0 || Number.isNaN(grade)) return {
@@ -209,15 +315,30 @@ function readVerdict(args) {
 			feedback: `Judge did not grade "${criterion.name}".`,
 			transient: true
 		};
-		objectiveScores[criterion.name] = clamp(grade / scale);
+		const normalized = clamp(grade / scale);
+		objectiveScores[criterion.name] = normalized;
+		if (criterion.gate !== void 0 && grade < criterion.gate) gated = true;
+		const weight = criterion.weight ?? 1;
+		weighted += normalized * weight;
+		totalWeight += weight;
 	}
-	const grades = Object.values(objectiveScores);
 	return {
-		score: grades.reduce((total, grade) => total + grade, 0) / grades.length,
+		score: gated ? 0 : weighted / totalWeight,
 		feedback,
 		objectiveScores
 	};
 }
+function assertCriteria(args) {
+	const { criteria, scale } = args;
+	let totalWeight = 0;
+	for (const { name, weight = 1, gate } of criteria) {
+		if (!Number.isFinite(weight) || weight < 0) throw new Error(`weight on criterion "${name}" must be a non-negative number, received ${weight}`);
+		totalWeight += weight;
+		if (gate !== void 0 && (!Number.isFinite(gate) || gate <= 0)) throw new Error(`gate on criterion "${name}" must be greater than 0, received ${gate}; no grade can fall below 0`);
+		if (gate !== void 0 && gate > scale) throw new Error(`gate on criterion "${name}" is ${gate}, above the scale of ${scale}; no grade can reach it`);
+	}
+	if (totalWeight <= 0) throw new Error("criteria weights must sum to more than 0; at least one criterion has to count towards the score");
+}
 function stringify(value) {
 	return typeof value === "string" ? value : JSON.stringify(value);
 }
@@ -243,24 +364,30 @@ function priceUsage(args) {
 	};
 }
 //#endregion
-exports.BudgetExhausted = require_evaluation.BudgetExhausted;
-exports.assertResumable = require_evaluation.assertResumable;
-exports.bootstrapDemos = require_demos.bootstrapDemos;
+exports.BudgetExhausted = require_warnings.BudgetExhausted;
+exports.assertResumable = require_warnings.assertResumable;
 exports.buildJudgePrompt = buildJudgePrompt;
-exports.candidateFingerprint = require_evaluation.candidateFingerprint;
+exports.candidateFingerprint = require_warnings.candidateFingerprint;
 exports.compare = compare;
-exports.componentNames = require_evaluation.componentNames;
-exports.costExhausted = require_evaluation.costExhausted;
-exports.createBudget = require_evaluation.createBudget;
-exports.createDeadline = require_evaluation.createDeadline;
-exports.createEvaluator = require_evaluation.createEvaluator;
+exports.componentNames = require_warnings.componentNames;
+exports.costExhausted = require_warnings.costExhausted;
+exports.createBudget = require_warnings.createBudget;
+exports.createDeadline = require_warnings.createDeadline;
+exports.createEvaluator = require_warnings.createEvaluator;
 exports.createJudge = createJudge;
-exports.createMemoryCache = require_evaluation.createMemoryCache;
+exports.createMemoryCache = require_warnings.createMemoryCache;
 exports.formatDemos = require_demos.formatDemos;
+exports.harvestFewShotExamples = require_demos.harvestFewShotExamples;
+exports.harvestRollouts = require_demos.harvestRollouts;
+exports.isCandidateAccepted = require_warnings.isCandidateAccepted;
+exports.isRunFinished = require_warnings.isRunFinished;
 exports.mapWithConcurrency = require_concurrency.mapWithConcurrency;
-exports.measuredMean = require_evaluation.measuredMean;
+exports.measuredMean = require_warnings.measuredMean;
 exports.parseDemos = require_demos.parseDemos;
 exports.parseProposedText = require_text.parseProposedText;
 exports.priceUsage = priceUsage;
-exports.requireMeasuredMean = require_evaluation.requireMeasuredMean;
-exports.runFingerprint = require_evaluation.runFingerprint;
+exports.requireMeasuredMean = require_warnings.requireMeasuredMean;
+exports.resolveValidationSet = require_warnings.resolveValidationSet;
+exports.runFingerprint = require_warnings.runFingerprint;
+exports.seedScoreWarnings = require_warnings.seedScoreWarnings;
+exports.toTrainingJsonl = toTrainingJsonl;