textopt 0.1.0 → 0.2.0

package/dist/gepa/index.cjs

@@ -1,8 +1,8 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_reporting = require("../reporting-CNHzbJC-.cjs");
+const require_warnings = require("../warnings-CWRJF-jA.cjs");
 const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
 const require_math = require("../math-BhlziRPc.cjs");
-const require_demos = require("../demos-D2o0qSSo.cjs");
+const require_demos = require("../demos-9v5ts7F3.cjs");
 const require_rng = require("../rng-DbA_rPIo.cjs");
 const require_reflection = require("../reflection-DRfbk6hu.cjs");
 const require_sampling = require("../sampling-Dars7ctR.cjs");
@@ -334,7 +334,7 @@ function isEligibleAncestor(args) {
 */
 function hasComplementaryComponent(args) {
 	const { ancestor, left, right } = args;
-	return require_reporting.componentNames(ancestor).some((name) => {
+	return require_warnings.componentNames(ancestor).some((name) => {
 		const base = ancestor[name];
 		return (base === left[name] || base === right[name]) && left[name] !== right[name];
 	});
@@ -347,7 +347,7 @@ function mergeComponents(args) {
 	const rightRecord = records[rightId];
 	const candidate = { ...ancestor };
 	const sources = [];
-	for (const name of require_reporting.componentNames(ancestor)) {
+	for (const name of require_warnings.componentNames(ancestor)) {
 		const base = ancestor[name];
 		const leftText = leftRecord.candidate[name];
 		const rightText = rightRecord.candidate[name];
@@ -391,7 +391,7 @@ function buildAncestries(records) {
 	return ancestries;
 }
 function fingerprint(candidate) {
-	return JSON.stringify(require_reporting.componentNames(candidate).sort().map((name) => [name, candidate[name]]));
+	return JSON.stringify(require_warnings.componentNames(candidate).sort().map((name) => [name, candidate[name]]));
 }
 //#endregion
 //#region src/gepa/pareto.ts
@@ -513,7 +513,9 @@ function hasSurvivor(args) {
 * `frontier` chooses what the fronts are taken over. "instance" is GEPA as
 * published. "objective" tracks candidates leading each named objective the
 * adapter reports, and "hybrid" pools both — a candidate then earns selection
-* weight for every instance it wins *and* every objective it leads.
+* weight for every instance it wins *and* every objective it leads. The
+* objectives are whatever the adapter put in `objectiveScores`, which for a
+* judge is every criterion it graded, including any at `weight: 0`.
 */
 function paretoSelector(args = {}) {
 	const { epsilon = 0, frontier = "instance" } = args;
@@ -628,14 +630,14 @@ function subsampledEvaluationPolicy(args) {
 */
 function roundRobinComponentSelector() {
 	return ({ candidate, cursor }) => {
-		const names = require_reporting.componentNames(candidate);
+		const names = require_warnings.componentNames(candidate);
 		if (names.length === 0) throw new Error("Candidate has no components to update");
 		return [names[cursor % names.length]];
 	};
 }
 /** Update every component in a single reflection call. */
 function allComponentsSelector() {
-	return ({ candidate }) => require_reporting.componentNames(candidate);
+	return ({ candidate }) => require_warnings.componentNames(candidate);
 }
 /**
 * Accept a mutation only when it beats its parent on the same minibatch. Cheap
@@ -660,7 +662,8 @@ function improvementAcceptance(args = {}) {
 */
 function pairedPermutationAcceptance(args = {}) {
 	const { alpha = .2, maxExact = 16 } = args;
-	return ({ parentScores, childScores }) => {
+	if (!Number.isFinite(alpha) || alpha <= 0 || alpha > 1) throw new Error(`alpha must be greater than 0 and at most 1, received ${alpha}`);
+	const policy = ({ parentScores, childScores }) => {
 		const differences = [];
 		for (let index = 0; index < parentScores.length; index += 1) differences.push(childScores[index] - parentScores[index]);
 		const observed = require_math.sum(differences);
@@ -671,6 +674,33 @@ function pairedPermutationAcceptance(args = {}) {
 			maxExact
 		}) <= alpha;
 	};
+	policy.minimumPairs = smallestAcceptableBatch({
+		alpha,
+		maxExact
+	});
+	return policy;
+}
+/**
+* Smallest paired batch on which a sign-flip test at `alpha` could return a
+* verdict of "accept" at all.
+*
+* Enumerating n non-zero differences gives 2^n equally likely sign
+* assignments, so the smallest attainable p-value is 2^-n and no batch below
+* `log2(1/alpha)` pairs can clear the bar. Past `maxExact` the p-value comes
+* from a normal approximation, which has no such floor — so when the exact
+* requirement is out of that regime's reach, the first batch that leaves the
+* regime is the honest answer rather than an exact size that never applies.
+*/
+function smallestAcceptableBatch(args) {
+	const { alpha, maxExact } = args;
+	let pairs = 1;
+	let smallestPValue = .5;
+	while (smallestPValue > alpha) {
+		pairs += 1;
+		smallestPValue /= 2;
+		if (pairs > maxExact) return maxExact + 1;
+	}
+	return pairs;
 }
 /**
 * Highest mean over the instances it was scored on, with wider coverage
@@ -745,16 +775,20 @@ var GepaOptimizer = class {
 				task
 			});
 		} finally {
-			await require_reporting.flushReporters(task.reporters ?? []);
+			await require_warnings.flushReporters(task.reporters ?? []);
 		}
 	}
 };
 async function runGepa(args) {
 	const { config, task } = args;
 	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, seed = 0, candidateSelector = paretoSelector(), acceptance = improvementAcceptance(), merge, skipPerfectScore = true, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals, reflection, checkpointCache = true, trackBestOutputs = false, raiseOnError = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, componentSelector = roundRobinComponentSelector(), batchSampler = require_sampling.createEpochShuffledSampler({ minibatchSize }), valEvaluationPolicy = fullEvaluationPolicy(), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_reporting.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
-	const deadline = require_reporting.createDeadline({ maxWallClockMs });
-	const seedComponents = require_reporting.componentNames(seedCandidate);
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, maxMetricCalls, componentSelector = roundRobinComponentSelector(), batchSampler = require_sampling.createEpochShuffledSampler({ minibatchSize }), valEvaluationPolicy = fullEvaluationPolicy(), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_warnings.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = require_warnings.resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
+	const deadline = require_warnings.createDeadline({ maxWallClockMs });
+	const seedComponents = require_warnings.componentNames(seedCandidate);
 	const mergeConfig = {
 		enabled: merge?.enabled ?? seedComponents.length > 1,
 		maxInvocations: merge?.maxInvocations ?? DEFAULT_MAX_MERGES,
@@ -767,7 +801,7 @@ async function runGepa(args) {
 	if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet; the Pareto frontier is tracked over validation instances");
 	if (seedComponents.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
 	if (testSet !== void 0 && testSet.length === 0) throw new Error("optimize requires a non-empty testSet when one is given; omit it to skip held-out evaluation");
-	const evaluationCache = cache === false ? void 0 : cache ?? require_reporting.createMemoryCache();
+	const evaluationCache = cache === false ? void 0 : cache ?? require_warnings.createMemoryCache();
 	const propose = adapter.proposeNewTexts?.bind(adapter) ?? require_reflection.createDefaultProposer({
 		...reflection?.buildPrompt === void 0 ? {} : { buildPrompt: reflection.buildPrompt },
 		...reflection?.strategies === void 0 ? {} : { strategies: reflection.strategies },
@@ -788,19 +822,19 @@ async function runGepa(args) {
 		datum,
 		index
 	})) ?? [];
-	const fingerprint = require_reporting.runFingerprint({
+	const fingerprint = require_warnings.runFingerprint({
 		seedCandidate,
 		trainingIds,
 		validationIds,
 		seed,
 		...cacheNamespace === void 0 ? {} : { cacheNamespace }
 	});
-	require_reporting.assertResumable({
+	require_warnings.assertResumable({
 		fingerprint,
 		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
 	});
 	const rng = require_rng.createSeededRng(seed, resumeFrom?.rngState);
-	const budget = require_reporting.createBudget({
+	const budget = require_warnings.createBudget({
 		maxMetricCalls,
 		spent: resumeFrom?.metricCalls ?? 0
 	});
@@ -820,7 +854,7 @@ async function runGepa(args) {
 		records: resumeFrom?.records ?? [],
 		seedCandidate
 	});
-	const seenCandidates = new Set(records.map((record) => require_reporting.candidateFingerprint(record.candidate)));
+	const seenCandidates = new Set(records.map((record) => require_warnings.candidateFingerprint(record.candidate)));
 	const outputsByCandidate = /* @__PURE__ */ new Map();
 	const rejectedProposals = restoreRejections({
 		rejections: resumeFrom?.rejectedProposals ?? {},
@@ -832,7 +866,7 @@ async function runGepa(args) {
 	let mergesDue = resumeFrom?.merge.due ?? 0;
 	let totalMergesTested = resumeFrom?.merge.tested ?? 0;
 	let lastIterationAccepted = resumeFrom?.merge.lastIterationAccepted ?? false;
-	const emit = require_reporting.createEmitter(reporters);
+	const emit = require_warnings.createEmitter(reporters);
 	/**
 	* Everything an acceptance means, in one event: the text, the aggregate, and
 	* the row it put on the frontier. Emitted from one place because the merge
@@ -889,7 +923,7 @@ async function runGepa(args) {
 		if (onCheckpoint === void 0) return;
 		await onCheckpoint(takeSnapshot());
 	}
-	const evaluator = require_reporting.createEvaluator({
+	const evaluator = require_warnings.createEvaluator({
 		adapter,
 		budget,
 		...retry === void 0 ? {} : { retry },
@@ -977,7 +1011,7 @@ async function runGepa(args) {
 			componentCursor: inheritedCursor(args.parentIds)
 		};
 		records.push(record);
-		seenCandidates.add(require_reporting.candidateFingerprint(args.candidate));
+		seenCandidates.add(require_warnings.candidateFingerprint(args.candidate));
 		if (trackBestOutputs) outputsByCandidate.set(record.id, args.evaluation.outputs);
 		if (args.source !== "merge") {
 			lastIterationAccepted = true;
@@ -989,7 +1023,7 @@ async function runGepa(args) {
 	function rememberRejection(args) {
 		const { proposed, parentScore, childScore } = args;
 		if (rejectedProposalMemory <= 0) return;
-		for (const component of require_reporting.componentNames(proposed)) {
+		for (const component of require_warnings.componentNames(proposed)) {
 			const text = proposed[component];
 			if (text === void 0) continue;
 			const history = rejectedProposals[component] ?? [];
@@ -1017,18 +1051,23 @@ async function runGepa(args) {
 	if (records.length === 0) {
 		const seedInstances = selectValInstances(seedCandidate);
 		if (!budget.canAfford(seedInstances.length)) throw new Error(`maxMetricCalls (${maxMetricCalls}) is smaller than the ${seedInstances.length} validation instances selected for scoring; the seed candidate cannot be scored`);
+		const seedEvaluation = await evaluateValidation({
+			candidate: seedCandidate,
+			instances: seedInstances,
+			phase: "seed",
+			candidateId: 0
+		});
 		emitAccepted(addCandidate({
 			candidate: seedCandidate,
 			parentIds: [],
-			evaluation: await evaluateValidation({
-				candidate: seedCandidate,
-				instances: seedInstances,
-				phase: "seed",
-				candidateId: 0
-			}),
+			evaluation: seedEvaluation,
 			source: "seed",
 			updatedComponents: []
 		}));
+		warnings.push(...require_warnings.seedScoreWarnings({
+			scores: seedEvaluation.scores,
+			perfectScore
+		}));
 		lastIterationAccepted = false;
 		mergesDue = 0;
 		await checkpoint();
@@ -1115,7 +1154,7 @@ async function runGepa(args) {
 			parentIds: [...proposal.parentIds],
 			evaluation,
 			source: "merge",
-			updatedComponents: require_reporting.componentNames(proposal.candidate).filter((name) => proposal.candidate[name] !== ancestor.candidate[name])
+			updatedComponents: require_warnings.componentNames(proposal.candidate).filter((name) => proposal.candidate[name] !== ancestor.candidate[name])
 		});
 		mergesDue -= 1;
 		totalMergesTested += 1;
@@ -1158,7 +1197,7 @@ async function runGepa(args) {
 				candidate: parent.candidate,
 				source: "componentSelector"
 			});
-			parent.componentCursor = (parent.componentCursor + 1) % Math.max(1, require_reporting.componentNames(parent.candidate).length);
+			parent.componentCursor = (parent.componentCursor + 1) % Math.max(1, require_warnings.componentNames(parent.candidate).length);
 			plans.push({
 				parent,
 				batch: batchIndices.map((index) => trainingSet[index]),
@@ -1208,7 +1247,7 @@ async function runGepa(args) {
 			throw err;
 		}
 		assertComponents({
-			names: require_reporting.componentNames(proposed),
+			names: require_warnings.componentNames(proposed),
 			candidate: parent.candidate,
 			source: "proposeNewTexts"
 		});
@@ -1216,7 +1255,7 @@ async function runGepa(args) {
 			...parent.candidate,
 			...proposed
 		};
-		const changed = require_reporting.componentNames(proposed).length > 0 && !seenCandidates.has(require_reporting.candidateFingerprint(child));
+		const changed = require_warnings.componentNames(proposed).length > 0 && !seenCandidates.has(require_warnings.candidateFingerprint(child));
 		emit({
 			type: "proposal",
 			iteration,
@@ -1236,7 +1275,7 @@ async function runGepa(args) {
 				candidateId: null
 			});
 		} catch (err) {
-			if (err instanceof require_reporting.BudgetExhausted) return { status: "budgetExhausted" };
+			if (err instanceof require_warnings.BudgetExhausted) return { status: "budgetExhausted" };
 			throw err;
 		}
 		const screened = pairMeasured({
@@ -1279,7 +1318,7 @@ async function runGepa(args) {
 				stop ??= "reflectionBudgetExhausted";
 				continue;
 			}
-			const fingerprint = require_reporting.candidateFingerprint(outcome.child);
+			const fingerprint = require_warnings.candidateFingerprint(outcome.child);
 			if (claimed.has(fingerprint)) continue;
 			claimed.add(fingerprint);
 			if (!outcome.accepted) {
@@ -1351,7 +1390,7 @@ async function runGepa(args) {
 						})
 					};
 				} catch (err) {
-					if (err instanceof require_reporting.BudgetExhausted) return {
+					if (err instanceof require_warnings.BudgetExhausted) return {
 						item,
 						evaluation: void 0
 					};
@@ -1369,7 +1408,7 @@ async function runGepa(args) {
 				parentIds: [item.outcome.plan.parent.id],
 				evaluation,
 				source: "mutation",
-				updatedComponents: require_reporting.componentNames(item.outcome.proposed)
+				updatedComponents: require_warnings.componentNames(item.outcome.proposed)
 			}));
 		}
 		return stop;
@@ -1387,7 +1426,7 @@ async function runGepa(args) {
 			stopReason = "aborted";
 			break;
 		}
-		if (require_reporting.costExhausted({
+		if (require_warnings.costExhausted({
 			usage: evaluator.usage(),
 			maxCostUsd
 		})) {
@@ -1434,7 +1473,7 @@ async function runGepa(args) {
 				stopReason = "aborted";
 				break;
 			}
-			if (err instanceof require_reporting.BudgetExhausted) {
+			if (err instanceof require_warnings.BudgetExhausted) {
 				stopReason = "budgetExhausted";
 				break;
 			}
@@ -1463,15 +1502,16 @@ async function runGepa(args) {
 		candidateId: bestCandidateId,
 		charge: false
 	});
-	const testScore = heldOut === void 0 ? void 0 : require_reporting.measuredMean(heldOut);
+	const testScore = heldOut === void 0 ? void 0 : require_warnings.measuredMean(heldOut);
 	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
 		bestCandidateId,
 		bestScore: best.aggregateScore,
 		metricCalls: budget.spent(),
 		...testScore === void 0 ? {} : { testScore },
-		...heldOut === void 0 ? {} : { testInstanceScores: require_reporting.instanceRow(heldOut) },
+		...heldOut === void 0 ? {} : { testInstanceScores: require_warnings.instanceRow(heldOut) },
 		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	const perObjectiveBest = collectPerObjectiveBest(records);
@@ -1495,6 +1535,7 @@ async function runGepa(args) {
 		reflectionCalls,
 		cacheHits: evaluator.cacheHits(),
 		iterations: iteration,
+		warnings,
 		stopReason,
 		snapshot: takeSnapshot()
 	};
@@ -1507,13 +1548,15 @@ async function runGepa(args) {
 function assertGepaConfig(config) {
 	if (config.reflection?.buildPrompt !== void 0 && config.reflection.strategies !== void 0) throw new Error("reflection takes buildPrompt or strategies, not both");
 	if (config.reflection?.strategies?.length === 0) throw new Error("reflection.strategies must not be empty");
-	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals } = config;
+	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, acceptance, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals } = config;
 	const proposalsPerIteration = proposals?.perIteration ?? 1;
 	const proposalConcurrency = proposals?.concurrency ?? 1;
 	if (!Number.isInteger(proposalsPerIteration) || proposalsPerIteration < 1) throw new Error(`proposals.perIteration must be a positive integer, received ${proposalsPerIteration}`);
 	if (!Number.isInteger(proposalConcurrency) || proposalConcurrency < 1) throw new Error(`proposals.concurrency must be a positive integer, received ${proposalConcurrency}`);
 	keepCount(proposals?.selection ?? "all");
 	if (!Number.isInteger(minibatchSize) || minibatchSize < 1) throw new Error(`minibatchSize must be a positive integer, received ${minibatchSize}`);
+	const minimumPairs = acceptance?.minimumPairs;
+	if (minimumPairs !== void 0 && minibatchSize < minimumPairs) throw new Error(`this acceptance policy cannot accept anything on fewer than ${minimumPairs} instances, but minibatchSize is ${minibatchSize}; raise minibatchSize or loosen the policy`);
 	if (!Number.isFinite(perfectScore)) throw new Error(`perfectScore must be a finite number, received ${perfectScore}`);
 	if (!Number.isInteger(rejectedProposalMemory) || rejectedProposalMemory < 0) throw new Error(`rejectedProposalMemory must be a non-negative integer, received ${rejectedProposalMemory}`);
 	if (maxIterations !== Number.POSITIVE_INFINITY && (!Number.isInteger(maxIterations) || maxIterations < 0)) throw new Error(`maxIterations must be a non-negative integer or Infinity, received ${maxIterations}`);
@@ -1527,7 +1570,7 @@ function assertGepaConfig(config) {
 */
 function restoreRecords(args) {
 	const { records, seedCandidate } = args;
-	const known = new Set(require_reporting.componentNames(seedCandidate));
+	const known = new Set(require_warnings.componentNames(seedCandidate));
 	for (const record of records) {
 		const named = [...Object.keys(record.candidate), ...record.updatedComponents];
 		for (const name of named) if (!known.has(name)) throw new Error(`checkpoint names the component "${name}", which the seed candidate does not have (${[...known].join(", ")})`);
@@ -1666,8 +1709,8 @@ exports.diverseReflectionStrategies = require_reflection.diverseReflectionStrate
 exports.epsilonGreedySelector = epsilonGreedySelector;
 exports.fullEvaluationPolicy = fullEvaluationPolicy;
 exports.improvementAcceptance = improvementAcceptance;
-exports.isCandidateAccepted = require_reporting.isCandidateAccepted;
-exports.isRunFinished = require_reporting.isRunFinished;
+exports.isCandidateAccepted = require_warnings.isCandidateAccepted;
+exports.isRunFinished = require_warnings.isRunFinished;
 exports.lowerBoundEvaluationPolicy = lowerBoundEvaluationPolicy;
 exports.pairedPermutationAcceptance = pairedPermutationAcceptance;
 exports.paretoSelector = paretoSelector;

package/dist/gepa/index.d.cts

@@ -1,8 +1,8 @@
-import { _ as TextModel, a as RunFinished, g as ScoreResult, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted } from "../reporting-DQbAohc9.cjs";
+import { a as RunFinished, b as ScoreResult, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, x as TextModel } from "../reporting-bq007_2z.cjs";
 import { n as EvaluationCache } from "../cache-CuSo0NJ8.cjs";
-import { r as DemoRenderer } from "../demos-CU9dy2oT.cjs";
-import { n as OptimizerResult, r as OptimizerTask, t as Optimizer } from "../optimizer-Bh5DPRMH.cjs";
-import { C as RejectedProposal, S as ReflectiveRecord, T as ValEvaluationPolicy, _ as GepaStopReason, a as buildRewritePrompt, b as ProposeArgs, c as AcceptancePolicy, d as CandidateSource, f as ComponentPatch, g as GepaSnapshot, h as GepaEvent, i as buildReflectionPrompt, l as CandidateRecord, m as GepaAdapter, n as ReflectionPromptBuilder, o as buildSimplifyPrompt, p as ComponentSelector, r as buildGeneralizePrompt, s as diverseReflectionStrategies, t as ReflectionPromptArgs, u as CandidateSelector, v as MakeReflectiveDatasetArgs, w as SelectionState, x as ReflectiveDataset, y as ParetoFrontier } from "../reflection-Dt3QrXhM.cjs";
+import { r as DemoRenderer } from "../demos-ByaLZy-Z.cjs";
+import { n as OptimizerResult, r as OptimizerTask, t as Optimizer } from "../optimizer-4Zv-Zt2t.cjs";
+import { C as ReflectiveRecord, E as ValEvaluationPolicy, S as ReflectiveDataset, T as SelectionState, _ as GepaStopReason, a as buildRewritePrompt, b as ProposeArgs, c as AcceptancePolicy, d as CandidateSource, f as ComponentPatch, g as GepaSnapshot, h as GepaEvent, i as buildReflectionPrompt, l as CandidateRecord, m as GepaAdapter, n as ReflectionPromptBuilder, o as buildSimplifyPrompt, p as ComponentSelector, r as buildGeneralizePrompt, s as diverseReflectionStrategies, t as ReflectionPromptArgs, u as CandidateSelector, v as MakeReflectiveDatasetArgs, w as RejectedProposal, x as ReflectiveBatch, y as ParetoFrontier } from "../reflection-D0A7eahD.cjs";
 import { t as BatchSampler } from "../sampling-axOwfZf5.cjs";
 //#region src/gepa/demos.d.ts
 /**
@@ -305,7 +305,9 @@ declare class GepaOptimizer implements Optimizer<GepaStopReason> {
  * `frontier` chooses what the fronts are taken over. "instance" is GEPA as
  * published. "objective" tracks candidates leading each named objective the
  * adapter reports, and "hybrid" pools both — a candidate then earns selection
- * weight for every instance it wins *and* every objective it leads.
+ * weight for every instance it wins *and* every objective it leads. The
+ * objectives are whatever the adapter put in `objectiveScores`, which for a
+ * judge is every criterion it graded, including any at `weight: 0`.
  */
 declare function paretoSelector(args?: {
   epsilon?: number;
@@ -388,4 +390,4 @@ declare function pairedPermutationAcceptance(args?: {
   maxExact?: number;
 }): AcceptancePolicy;
 //#endregion
-export { type AcceptancePolicy, type CandidateAccepted, type CandidateRecord, type CandidateSelector, type CandidateSource, type ComponentPatch, type ComponentSelector, type GepaAdapter, type GepaConfig, type GepaEvent, GepaOptimizer, type GepaResult, type GepaSnapshot, type GepaStopReason, type GepaTask, type MakeReflectiveDatasetArgs, type OptimizerEvent, type PipelineModule, type PipelineStep, type PipelineTrace, type ProposeArgs, type ReflectionPromptArgs, type ReflectionPromptBuilder, type ReflectiveDataset, type ReflectiveRecord, type RejectedProposal, type ReportableEvent, type Reporter, type RunFinished, type SelectionState, type ValEvaluationPolicy, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, isCandidateAccepted, isRunFinished, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };
+export { type AcceptancePolicy, type CandidateAccepted, type CandidateRecord, type CandidateSelector, type CandidateSource, type ComponentPatch, type ComponentSelector, type GepaAdapter, type GepaConfig, type GepaEvent, GepaOptimizer, type GepaResult, type GepaSnapshot, type GepaStopReason, type GepaTask, type MakeReflectiveDatasetArgs, type OptimizerEvent, type PipelineModule, type PipelineStep, type PipelineTrace, type ProposeArgs, type ReflectionPromptArgs, type ReflectionPromptBuilder, type ReflectiveBatch, type ReflectiveDataset, type ReflectiveRecord, type RejectedProposal, type ReportableEvent, type Reporter, type RunFinished, type SelectionState, type ValEvaluationPolicy, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, isCandidateAccepted, isRunFinished, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };

package/dist/gepa/index.d.mts

@@ -1,8 +1,8 @@
-import { _ as TextModel, a as RunFinished, g as ScoreResult, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted } from "../reporting-DQbAohc9.mjs";
+import { a as RunFinished, b as ScoreResult, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, x as TextModel } from "../reporting-bq007_2z.mjs";
 import { n as EvaluationCache } from "../cache-CuSo0NJ8.mjs";
-import { r as DemoRenderer } from "../demos-1b7JiUue.mjs";
-import { n as OptimizerResult, r as OptimizerTask, t as Optimizer } from "../optimizer-Ck6-e_8o.mjs";
-import { C as RejectedProposal, S as ReflectiveRecord, T as ValEvaluationPolicy, _ as GepaStopReason, a as buildRewritePrompt, b as ProposeArgs, c as AcceptancePolicy, d as CandidateSource, f as ComponentPatch, g as GepaSnapshot, h as GepaEvent, i as buildReflectionPrompt, l as CandidateRecord, m as GepaAdapter, n as ReflectionPromptBuilder, o as buildSimplifyPrompt, p as ComponentSelector, r as buildGeneralizePrompt, s as diverseReflectionStrategies, t as ReflectionPromptArgs, u as CandidateSelector, v as MakeReflectiveDatasetArgs, w as SelectionState, x as ReflectiveDataset, y as ParetoFrontier } from "../reflection-LRaAZP4e.mjs";
+import { r as DemoRenderer } from "../demos-ASsSXYXA.mjs";
+import { n as OptimizerResult, r as OptimizerTask, t as Optimizer } from "../optimizer-Ds5mzYjz.mjs";
+import { C as ReflectiveRecord, E as ValEvaluationPolicy, S as ReflectiveDataset, T as SelectionState, _ as GepaStopReason, a as buildRewritePrompt, b as ProposeArgs, c as AcceptancePolicy, d as CandidateSource, f as ComponentPatch, g as GepaSnapshot, h as GepaEvent, i as buildReflectionPrompt, l as CandidateRecord, m as GepaAdapter, n as ReflectionPromptBuilder, o as buildSimplifyPrompt, p as ComponentSelector, r as buildGeneralizePrompt, s as diverseReflectionStrategies, t as ReflectionPromptArgs, u as CandidateSelector, v as MakeReflectiveDatasetArgs, w as RejectedProposal, x as ReflectiveBatch, y as ParetoFrontier } from "../reflection-CMezGu6u.mjs";
 import { t as BatchSampler } from "../sampling-DFo_7RNJ.mjs";
 //#region src/gepa/demos.d.ts
 /**
@@ -305,7 +305,9 @@ declare class GepaOptimizer implements Optimizer<GepaStopReason> {
  * `frontier` chooses what the fronts are taken over. "instance" is GEPA as
  * published. "objective" tracks candidates leading each named objective the
  * adapter reports, and "hybrid" pools both — a candidate then earns selection
- * weight for every instance it wins *and* every objective it leads.
+ * weight for every instance it wins *and* every objective it leads. The
+ * objectives are whatever the adapter put in `objectiveScores`, which for a
+ * judge is every criterion it graded, including any at `weight: 0`.
  */
 declare function paretoSelector(args?: {
   epsilon?: number;
@@ -388,4 +390,4 @@ declare function pairedPermutationAcceptance(args?: {
   maxExact?: number;
 }): AcceptancePolicy;
 //#endregion
-export { type AcceptancePolicy, type CandidateAccepted, type CandidateRecord, type CandidateSelector, type CandidateSource, type ComponentPatch, type ComponentSelector, type GepaAdapter, type GepaConfig, type GepaEvent, GepaOptimizer, type GepaResult, type GepaSnapshot, type GepaStopReason, type GepaTask, type MakeReflectiveDatasetArgs, type OptimizerEvent, type PipelineModule, type PipelineStep, type PipelineTrace, type ProposeArgs, type ReflectionPromptArgs, type ReflectionPromptBuilder, type ReflectiveDataset, type ReflectiveRecord, type RejectedProposal, type ReportableEvent, type Reporter, type RunFinished, type SelectionState, type ValEvaluationPolicy, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, isCandidateAccepted, isRunFinished, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };
+export { type AcceptancePolicy, type CandidateAccepted, type CandidateRecord, type CandidateSelector, type CandidateSource, type ComponentPatch, type ComponentSelector, type GepaAdapter, type GepaConfig, type GepaEvent, GepaOptimizer, type GepaResult, type GepaSnapshot, type GepaStopReason, type GepaTask, type MakeReflectiveDatasetArgs, type OptimizerEvent, type PipelineModule, type PipelineStep, type PipelineTrace, type ProposeArgs, type ReflectionPromptArgs, type ReflectionPromptBuilder, type ReflectiveBatch, type ReflectiveDataset, type ReflectiveRecord, type RejectedProposal, type ReportableEvent, type Reporter, type RunFinished, type SelectionState, type ValEvaluationPolicy, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, isCandidateAccepted, isRunFinished, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };

package/dist/gepa/index.mjs

@@ -1,7 +1,7 @@
-import { _ as createMemoryCache, a as isRunFinished, b as createBudget, c as createEvaluator, d as createDeadline, f as assertResumable, h as componentNames, i as isCandidateAccepted, l as measuredMean, m as runFingerprint, n as flushReporters, o as BudgetExhausted, p as candidateFingerprint, r as instanceRow, s as costExhausted, t as createEmitter, v as defaultInstanceId } from "../reporting-DY-DC4HG.mjs";
+import { S as createBudget, _ as componentNames, a as instanceRow, b as defaultInstanceId, c as BudgetExhausted, d as measuredMean, g as runFingerprint, h as candidateFingerprint, i as flushReporters, l as costExhausted, m as assertResumable, n as seedScoreWarnings, o as isCandidateAccepted, p as createDeadline, r as createEmitter, s as isRunFinished, t as resolveValidationSet, u as createEvaluator, y as createMemoryCache } from "../warnings-OxvDi9kN.mjs";
 import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
 import { a as sum, i as signFlipPValue, r as mean, t as argmax } from "../math-Dqme4rYz.mjs";
-import { r as parseDemos, t as formatDemos } from "../demos-DE2oxNWX.mjs";
+import { r as parseDemos, t as formatDemos } from "../demos-Brobjfuc.mjs";
 import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
 import { a as createDefaultProposer, i as buildSimplifyPrompt, n as buildReflectionPrompt, o as diverseReflectionStrategies, r as buildRewritePrompt, t as buildGeneralizePrompt } from "../reflection-mwMhrjs_.mjs";
 import { t as createEpochShuffledSampler } from "../sampling-CfHt7Gue.mjs";
@@ -512,7 +512,9 @@ function hasSurvivor(args) {
 * `frontier` chooses what the fronts are taken over. "instance" is GEPA as
 * published. "objective" tracks candidates leading each named objective the
 * adapter reports, and "hybrid" pools both — a candidate then earns selection
-* weight for every instance it wins *and* every objective it leads.
+* weight for every instance it wins *and* every objective it leads. The
+* objectives are whatever the adapter put in `objectiveScores`, which for a
+* judge is every criterion it graded, including any at `weight: 0`.
 */
 function paretoSelector(args = {}) {
 	const { epsilon = 0, frontier = "instance" } = args;
@@ -659,7 +661,8 @@ function improvementAcceptance(args = {}) {
 */
 function pairedPermutationAcceptance(args = {}) {
 	const { alpha = .2, maxExact = 16 } = args;
-	return ({ parentScores, childScores }) => {
+	if (!Number.isFinite(alpha) || alpha <= 0 || alpha > 1) throw new Error(`alpha must be greater than 0 and at most 1, received ${alpha}`);
+	const policy = ({ parentScores, childScores }) => {
 		const differences = [];
 		for (let index = 0; index < parentScores.length; index += 1) differences.push(childScores[index] - parentScores[index]);
 		const observed = sum(differences);
@@ -670,6 +673,33 @@ function pairedPermutationAcceptance(args = {}) {
 			maxExact
 		}) <= alpha;
 	};
+	policy.minimumPairs = smallestAcceptableBatch({
+		alpha,
+		maxExact
+	});
+	return policy;
+}
+/**
+* Smallest paired batch on which a sign-flip test at `alpha` could return a
+* verdict of "accept" at all.
+*
+* Enumerating n non-zero differences gives 2^n equally likely sign
+* assignments, so the smallest attainable p-value is 2^-n and no batch below
+* `log2(1/alpha)` pairs can clear the bar. Past `maxExact` the p-value comes
+* from a normal approximation, which has no such floor — so when the exact
+* requirement is out of that regime's reach, the first batch that leaves the
+* regime is the honest answer rather than an exact size that never applies.
+*/
+function smallestAcceptableBatch(args) {
+	const { alpha, maxExact } = args;
+	let pairs = 1;
+	let smallestPValue = .5;
+	while (smallestPValue > alpha) {
+		pairs += 1;
+		smallestPValue /= 2;
+		if (pairs > maxExact) return maxExact + 1;
+	}
+	return pairs;
 }
 /**
 * Highest mean over the instances it was scored on, with wider coverage
@@ -751,7 +781,11 @@ var GepaOptimizer = class {
 async function runGepa(args) {
 	const { config, task } = args;
 	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, seed = 0, candidateSelector = paretoSelector(), acceptance = improvementAcceptance(), merge, skipPerfectScore = true, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals, reflection, checkpointCache = true, trackBestOutputs = false, raiseOnError = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, componentSelector = roundRobinComponentSelector(), batchSampler = createEpochShuffledSampler({ minibatchSize }), valEvaluationPolicy = fullEvaluationPolicy(), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, maxMetricCalls, componentSelector = roundRobinComponentSelector(), batchSampler = createEpochShuffledSampler({ minibatchSize }), valEvaluationPolicy = fullEvaluationPolicy(), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
 	const deadline = createDeadline({ maxWallClockMs });
 	const seedComponents = componentNames(seedCandidate);
 	const mergeConfig = {
@@ -1016,18 +1050,23 @@ async function runGepa(args) {
 	if (records.length === 0) {
 		const seedInstances = selectValInstances(seedCandidate);
 		if (!budget.canAfford(seedInstances.length)) throw new Error(`maxMetricCalls (${maxMetricCalls}) is smaller than the ${seedInstances.length} validation instances selected for scoring; the seed candidate cannot be scored`);
+		const seedEvaluation = await evaluateValidation({
+			candidate: seedCandidate,
+			instances: seedInstances,
+			phase: "seed",
+			candidateId: 0
+		});
 		emitAccepted(addCandidate({
 			candidate: seedCandidate,
 			parentIds: [],
-			evaluation: await evaluateValidation({
-				candidate: seedCandidate,
-				instances: seedInstances,
-				phase: "seed",
-				candidateId: 0
-			}),
+			evaluation: seedEvaluation,
 			source: "seed",
 			updatedComponents: []
 		}));
+		warnings.push(...seedScoreWarnings({
+			scores: seedEvaluation.scores,
+			perfectScore
+		}));
 		lastIterationAccepted = false;
 		mergesDue = 0;
 		await checkpoint();
@@ -1466,6 +1505,7 @@ async function runGepa(args) {
 	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
 		bestCandidateId,
 		bestScore: best.aggregateScore,
 		metricCalls: budget.spent(),
@@ -1494,6 +1534,7 @@ async function runGepa(args) {
 		reflectionCalls,
 		cacheHits: evaluator.cacheHits(),
 		iterations: iteration,
+		warnings,
 		stopReason,
 		snapshot: takeSnapshot()
 	};
@@ -1506,13 +1547,15 @@ async function runGepa(args) {
 function assertGepaConfig(config) {
 	if (config.reflection?.buildPrompt !== void 0 && config.reflection.strategies !== void 0) throw new Error("reflection takes buildPrompt or strategies, not both");
 	if (config.reflection?.strategies?.length === 0) throw new Error("reflection.strategies must not be empty");
-	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals } = config;
+	const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, acceptance, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals } = config;
 	const proposalsPerIteration = proposals?.perIteration ?? 1;
 	const proposalConcurrency = proposals?.concurrency ?? 1;
 	if (!Number.isInteger(proposalsPerIteration) || proposalsPerIteration < 1) throw new Error(`proposals.perIteration must be a positive integer, received ${proposalsPerIteration}`);
 	if (!Number.isInteger(proposalConcurrency) || proposalConcurrency < 1) throw new Error(`proposals.concurrency must be a positive integer, received ${proposalConcurrency}`);
 	keepCount(proposals?.selection ?? "all");
 	if (!Number.isInteger(minibatchSize) || minibatchSize < 1) throw new Error(`minibatchSize must be a positive integer, received ${minibatchSize}`);
+	const minimumPairs = acceptance?.minimumPairs;
+	if (minimumPairs !== void 0 && minibatchSize < minimumPairs) throw new Error(`this acceptance policy cannot accept anything on fewer than ${minimumPairs} instances, but minibatchSize is ${minibatchSize}; raise minibatchSize or loosen the policy`);
 	if (!Number.isFinite(perfectScore)) throw new Error(`perfectScore must be a finite number, received ${perfectScore}`);
 	if (!Number.isInteger(rejectedProposalMemory) || rejectedProposalMemory < 0) throw new Error(`rejectedProposalMemory must be a non-negative integer, received ${rejectedProposalMemory}`);
 	if (maxIterations !== Number.POSITIVE_INFINITY && (!Number.isInteger(maxIterations) || maxIterations < 0)) throw new Error(`maxIterations must be a non-negative integer or Infinity, received ${maxIterations}`);