textopt 0.0.0 → 0.2.0

package/dist/index.mjs

@@ -1,10 +1,12 @@
-import { a as requireMeasuredMean, c as candidateFingerprint, d as createMemoryCache, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "./evaluation-BV0nSZVx.mjs";
+import { S as createBudget, _ as componentNames, c as BudgetExhausted, d as measuredMean, f as requireMeasuredMean, g as runFingerprint, h as candidateFingerprint, l as costExhausted, m as assertResumable, n as seedScoreWarnings, o as isCandidateAccepted, p as createDeadline, s as isRunFinished, t as resolveValidationSet, u as createEvaluator, y as createMemoryCache } from "./warnings-OxvDi9kN.mjs";
 import { t as mapWithConcurrency } from "./concurrency-D58PWeSk.mjs";
-import { r as signFlipPValue } from "./math-DhrDmpFS.mjs";
-import { n as formatDemos, r as parseDemos, t as bootstrapDemos } from "./demos-Degx6UmP.mjs";
+import { i as signFlipPValue, n as holmAdjust } from "./math-Dqme4rYz.mjs";
+import { a as harvestRollouts, n as harvestFewShotExamples, r as parseDemos, t as formatDemos } from "./demos-Brobjfuc.mjs";
 import { t as parseProposedText } from "./text--v4Ffbus.mjs";
 //#region src/compare.ts
-const EXACT_LIMIT = 16;
+const EXACT_LIMIT = 20;
+/** Well above float subtraction noise (~1e-16), well below a real margin. */
+const DEGENERACY_TOLERANCE = 1e-9;
 /**
 * Run several optimizers over the same seeds and report which one actually won.
 *
@@ -19,7 +21,7 @@ const EXACT_LIMIT = 16;
 * seed is constructor config and every optimizer here is deterministic given
 * one — comparing two entrants at a single seed compares two anecdotes. It also
 * leaves the task where the caller builds it, which is the only place the
-* optimizer-specific parts of it (`reflect`, `cache`, `onEvent`) are known.
+* optimizer-specific parts of it (`reflect`, `cache`, `reporters`) are known.
 */
 async function compare(args) {
 	const { entrants, seeds, concurrency = 1 } = args;
@@ -42,6 +44,8 @@ async function compare(args) {
 				bestScore: result.bestScore,
 				...result.testScore === void 0 ? {} : { testScore: result.testScore },
 				metricCalls: result.metricCalls,
+				cacheHits: result.cacheHits,
+				reflectionCalls: result.reflectionCalls ?? 0,
 				costUsd: result.usage.costUsd,
 				stopReason: result.stopReason
 			};
@@ -52,17 +56,25 @@ async function compare(args) {
 		runs: runs.filter((run) => run.entrant === entrant)
 	}));
 	const winner = summaries.reduce((best, summary) => summary.meanScore > best.meanScore ? summary : best);
+	const rawPValues = summaries.map((summary) => summary.entrant === winner.entrant ? void 0 : margin({
+		winner: winner.entrant,
+		entrant: summary.entrant,
+		runs,
+		seeds
+	}));
+	const holmAdjusted = holmAdjustSparse({
+		pValues: rawPValues,
+		familySize: names.length - 1
+	});
 	return {
 		winner: winner.entrant,
-		summaries: summaries.map((summary) => summary.entrant === winner.entrant ? summary : {
+		summaries: summaries.map((summary, index) => ({
 			...summary,
-			pValueVsWinner: margin({
-				winner: winner.entrant,
-				entrant: summary.entrant,
-				runs,
-				seeds
-			})
-		}),
+			...rawPValues[index] === void 0 ? {} : {
+				pValueVsWinner: rawPValues[index],
+				pValueVsWinnerHolm: holmAdjusted[index]
+			}
+		})),
 		runs
 	};
 }
@@ -77,13 +89,28 @@ function summarize(args) {
 		minScore: Math.min(...scores),
 		maxScore: Math.max(...scores),
 		meanMetricCalls: mean(runs.map((run) => run.metricCalls)),
-		meanCostUsd: mean(runs.map((run) => run.costUsd))
+		meanCostUsd: mean(runs.map((run) => run.costUsd)),
+		meanCacheHits: mean(runs.map((run) => run.cacheHits)),
+		meanReflectionCalls: mean(runs.map((run) => run.reflectionCalls)),
+		distinctScores: new Set(scores).size
 	};
 }
 /**
 * Paired across seeds rather than pooled: the same seed puts both entrants on
 * the same sampling order, so the difference at a seed is a comparison and the
 * spread between seeds is not.
+*
+* Returns `undefined` when every seed produced the exact same nonzero margin.
+* That is not evidence of an n-seed-strong result — it is one realization the
+* seed never varied, and a sign-flip p-value would report a precision from n
+* independent trials that never happened. A margin of exactly zero every seed
+* is not this case: `signFlipPValue` already reports that honestly as 1, no
+* significance claimed either way, which is not a fabricated number.
+*
+* "Exact same" is judged within `DEGENERACY_TOLERANCE`, not `===`: subtracting
+* two scores that are equal in substance can still land a few ULPs apart
+* (0.95 - 0.55 and 0.9 - 0.5 differ at the 16th digit), and treating that as
+* n real trials would be the same fabrication this check exists to prevent.
 */
 function margin(args) {
 	const { winner, entrant, runs, seeds } = args;
@@ -98,6 +125,10 @@ function margin(args) {
 			seed
 		});
 	});
+	const [first] = differences;
+	if (first !== void 0) {
+		if (differences.reduce((widest, difference) => Math.max(widest, Math.abs(difference - first)), 0) < DEGENERACY_TOLERANCE && Math.abs(first) > DEGENERACY_TOLERANCE) return;
+	}
 	return signFlipPValue({
 		differences,
 		observed: differences.reduce((total, value) => total + value, 0),
@@ -108,6 +139,27 @@ function scoreOf(args) {
 	const { runs, entrant, seed } = args;
 	return runs.find((candidate) => candidate.entrant === entrant && candidate.seed === seed)?.score ?? 0;
 }
+/**
+* `holmAdjust` over the raw p-values that exist, skipping the slots a
+* withheld comparison left `undefined` — those still count toward
+* `familySize`, they just have nothing of their own to adjust.
+*/
+function holmAdjustSparse(args) {
+	const { pValues, familySize } = args;
+	const present = pValues.map((pValue, index) => ({
+		pValue,
+		index
+	})).filter((entry) => entry.pValue !== void 0);
+	const adjusted = holmAdjust({
+		pValues: present.map((entry) => entry.pValue),
+		familySize
+	});
+	const result = new Array(pValues.length).fill(void 0);
+	present.forEach((entry, rank) => {
+		result[entry.index] = adjusted[rank];
+	});
+	return result;
+}
 function mean(values) {
 	return values.reduce((total, value) => total + value, 0) / values.length;
 }
@@ -118,6 +170,51 @@ function standardDeviation(values) {
 	return Math.sqrt(variance);
 }
 //#endregion
+//#region src/distillation.ts
+const ROLES = /* @__PURE__ */ new Set([
+	"system",
+	"user",
+	"assistant"
+]);
+/**
+* Serialize harvested rollouts as JSONL, one training example per line.
+*
+* The chat-messages shape rather than any one vendor's: it is what Axolotl,
+* Together, Fireworks and the Hugging Face trainers all ingest, and the
+* providers that read it outlive the ones that do not.
+*
+* Returns the text rather than writing it. Only `file-cache` touches the
+* filesystem, and a caller uploading this straight to a provider should not
+* have to round-trip it through a file to do so.
+*/
+function toTrainingJsonl(args) {
+	const { rollouts, render } = args;
+	const lines = [];
+	for (const [index, rollout] of rollouts.entries()) {
+		const example = render({
+			rollout,
+			index
+		});
+		if (example === null) continue;
+		assertUsable(example, index);
+		lines.push(JSON.stringify(example));
+	}
+	return lines.join("\n");
+}
+/**
+* Checked here rather than left to the provider. A malformed line surfaces as
+* a rejected upload hours later, naming a line number in a file the caller
+* never wrote by hand; naming the rollout at the point it was rendered is the
+* same error while it is still fixable.
+*/
+function assertUsable(example, index) {
+	if (!Array.isArray(example.messages) || example.messages.length === 0) throw new Error(`rollout ${index} rendered no messages`);
+	for (const message of example.messages) {
+		if (!ROLES.has(message.role)) throw new Error(`rollout ${index} rendered the unknown role ${message.role}`);
+		if (typeof message.content !== "string") throw new Error(`rollout ${index} rendered non-text content`);
+	}
+}
+//#endregion
 //#region src/judge.ts
 const DEFAULT_SCALE = 5;
 const SCORE = /<score\s+name="([^"]+)"\s*>\s*([\d.]+)\s*<\/score>/g;
@@ -140,6 +237,11 @@ const FEEDBACK = /<feedback>([\s\S]*?)<\/feedback>/;
 function createJudge(args) {
 	const { model, criteria, scale = DEFAULT_SCALE, renderInput = stringify, renderOutput = stringify, buildPrompt = buildJudgePrompt } = args;
 	if (criteria.length === 0) throw new Error("createJudge requires at least one criterion");
+	if (!Number.isFinite(scale) || scale <= 0) throw new Error(`scale must be a positive number, received ${scale}`);
+	assertCriteria({
+		criteria,
+		scale
+	});
 	return async ({ input, output, expected, signal }) => {
 		return readVerdict({
 			response: await model({
@@ -183,6 +285,7 @@ function buildJudgePrompt(args) {
 		`Grade each criterion from 0 to ${scale}, where ${scale} is a perfect answer.`,
 		"",
 		"Then write feedback. It is read by a program that rewrites the system's instructions, not by a person reviewing this output, so say what the instructions should tell the system to do differently. Feedback about this particular answer is of no use to it.",
+		...expected === void 0 ? [] : ["", "Do not restate the expected answer, or any fact drawn from it, in the feedback. The instruction it is rewritten into is reused on inputs whose answers you have not seen: a fact copied out of the expected answer becomes an answer key memorised in the prompt, which raises the score on this input and teaches the system nothing. Name the kind of thing the answer was missing, not the thing itself."],
 		"",
 		"Reply in exactly this format and nothing else:",
 		...criteria.map((criterion) => `<score name="${criterion.name}">…</score>`),
@@ -201,6 +304,9 @@ function readVerdict(args) {
 	const graded = /* @__PURE__ */ new Map();
 	for (const match of response.matchAll(SCORE)) graded.set(match[1], Number(match[2]));
 	const objectiveScores = {};
+	let gated = false;
+	let weighted = 0;
+	let totalWeight = 0;
 	for (const criterion of criteria) {
 		const grade = graded.get(criterion.name);
 		if (grade === void 0 || Number.isNaN(grade)) return {
@@ -208,15 +314,30 @@ function readVerdict(args) {
 			feedback: `Judge did not grade "${criterion.name}".`,
 			transient: true
 		};
-		objectiveScores[criterion.name] = clamp(grade / scale);
+		const normalized = clamp(grade / scale);
+		objectiveScores[criterion.name] = normalized;
+		if (criterion.gate !== void 0 && grade < criterion.gate) gated = true;
+		const weight = criterion.weight ?? 1;
+		weighted += normalized * weight;
+		totalWeight += weight;
 	}
-	const grades = Object.values(objectiveScores);
 	return {
-		score: grades.reduce((total, grade) => total + grade, 0) / grades.length,
+		score: gated ? 0 : weighted / totalWeight,
 		feedback,
 		objectiveScores
 	};
 }
+function assertCriteria(args) {
+	const { criteria, scale } = args;
+	let totalWeight = 0;
+	for (const { name, weight = 1, gate } of criteria) {
+		if (!Number.isFinite(weight) || weight < 0) throw new Error(`weight on criterion "${name}" must be a non-negative number, received ${weight}`);
+		totalWeight += weight;
+		if (gate !== void 0 && (!Number.isFinite(gate) || gate <= 0)) throw new Error(`gate on criterion "${name}" must be greater than 0, received ${gate}; no grade can fall below 0`);
+		if (gate !== void 0 && gate > scale) throw new Error(`gate on criterion "${name}" is ${gate}, above the scale of ${scale}; no grade can reach it`);
+	}
+	if (totalWeight <= 0) throw new Error("criteria weights must sum to more than 0; at least one criterion has to count towards the score");
+}
 function stringify(value) {
 	return typeof value === "string" ? value : JSON.stringify(value);
 }
@@ -242,4 +363,4 @@ function priceUsage(args) {
 	};
 }
 //#endregion
-export { BudgetExhausted, assertResumable, bootstrapDemos, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, runFingerprint };
+export { BudgetExhausted, assertResumable, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, harvestFewShotExamples, harvestRollouts, isCandidateAccepted, isRunFinished, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, resolveValidationSet, runFingerprint, seedScoreWarnings, toTrainingJsonl };

package/dist/{math-COOofUyv.cjs → math-BhlziRPc.cjs}

@@ -44,17 +44,62 @@ function signFlipPValue(args) {
 		differences: moved,
 		observed
 	});
-	const assignments = 2 ** moved.length;
-	let atLeastAsExtreme = 0;
-	for (let mask = 0; mask < assignments; mask += 1) {
-		let total = 0;
-		for (let index = 0; index < moved.length; index += 1) {
-			const sign = mask >> index & 1 ? -1 : 1;
-			total += sign * moved[index];
+	const sums = achievableSums(moved);
+	return sums.filter((total) => total >= observed).length / sums.length;
+}
+/**
+* Holm-Bonferroni step-down adjustment: the p-value each comparison would need
+* to survive on its own if the whole family were held to one error rate,
+* rather than letting the smallest of several tests read as significant by
+* volume alone. Sorted ascending, each rank is scaled by how many comparisons
+* are still in contention at that rank, and the running maximum keeps a later,
+* less-scaled rank from reporting looser than an earlier one already has.
+*
+* `familySize` may exceed `pValues.length`: a comparison `signFlipPValue`
+* could not test at all (every paired difference identical, so no p reflects
+* a real margin) is still a member of the family being controlled for, and
+* excluding it from the denominator would understate the correction owed to
+* the comparisons that could be tested.
+*/
+function holmAdjust(args) {
+	const { pValues, familySize } = args;
+	const ranked = pValues.map((pValue, index) => ({
+		pValue,
+		index
+	})).sort((a, b) => a.pValue - b.pValue);
+	const adjusted = new Array(pValues.length);
+	let runningMax = 0;
+	ranked.forEach(({ pValue, index }, rank) => {
+		runningMax = Math.max(runningMax, Math.min(1, pValue * (familySize - rank)));
+		adjusted[index] = runningMax;
+	});
+	return adjusted;
+}
+/**
+* Every total reachable by flipping some subset of `differences`' signs,
+* built by doubling rather than by scoring each of the 2^n sign masks
+* independently: after `k` differences there are 2^k sums, and folding in
+* difference `k+1` only ever adds or subtracts it from each of them, so the
+* whole enumeration costs O(2^n) instead of the O(2^n * n) a per-mask loop
+* pays for re-summing n terms every time. Measured at n=20, twenty seeds
+* being the bench's ceiling: about 63ms scoring masks one at a time against
+* about 16ms building sums this way — the difference between affording exact
+* enumeration through a full twenty-seed run and falling back to the normal
+* approximation, which is what raising `EXACT_LIMIT` in compare.ts to 20
+* relies on.
+*/
+function achievableSums(differences) {
+	let sums = [0];
+	for (const difference of differences) {
+		const next = new Array(sums.length * 2);
+		for (let index = 0; index < sums.length; index += 1) {
+			const total = sums[index];
+			next[index] = total + difference;
+			next[index + sums.length] = total - difference;
 		}
-		if (total >= observed) atLeastAsExtreme += 1;
+		sums = next;
 	}
-	return atLeastAsExtreme / assignments;
+	return sums;
 }
 /**
 * The same tail probability from a normal approximation, for batches too large
@@ -81,6 +126,12 @@ Object.defineProperty(exports, "argmax", {
 		return argmax;
 	}
 });
+Object.defineProperty(exports, "holmAdjust", {
+	enumerable: true,
+	get: function() {
+		return holmAdjust;
+	}
+});
 Object.defineProperty(exports, "mean", {
 	enumerable: true,
 	get: function() {

package/dist/math-Dqme4rYz.mjs

@@ -0,0 +1,123 @@
+//#region src/math.ts
+function argmax(values) {
+	let bestIndex = 0;
+	let bestValue = Number.NEGATIVE_INFINITY;
+	for (let index = 0; index < values.length; index += 1) {
+		const value = values[index];
+		if (value > bestValue) {
+			bestValue = value;
+			bestIndex = index;
+		}
+	}
+	return bestIndex;
+}
+/** Mean over the values that exist; unscored instances are not zeros. */
+function mean(values) {
+	let total = 0;
+	let count = 0;
+	for (const value of values) if (value !== void 0) {
+		total += value;
+		count += 1;
+	}
+	return count === 0 ? 0 : total / count;
+}
+function sum(values) {
+	let total = 0;
+	for (const value of values) total += value;
+	return total;
+}
+/**
+* One-sided p-value from a paired sign-flip (permutation) test: how often a
+* total this large arises when each difference is equally likely to have come
+* out the other way. Exact by enumeration for small samples, normal-approximated
+* past `maxExact` where 2^n stops being cheap.
+*
+* Paired and distribution-free, which is what a comparison of two runs over the
+* same instances needs — the scores are not independent draws and are not
+* normal, so a t-test on them is answering a different question.
+*/
+function signFlipPValue(args) {
+	const { differences, observed, maxExact } = args;
+	const moved = differences.filter((difference) => difference !== 0);
+	if (moved.length === 0) return 1;
+	if (moved.length > maxExact) return normalTailProbability({
+		differences: moved,
+		observed
+	});
+	const sums = achievableSums(moved);
+	return sums.filter((total) => total >= observed).length / sums.length;
+}
+/**
+* Holm-Bonferroni step-down adjustment: the p-value each comparison would need
+* to survive on its own if the whole family were held to one error rate,
+* rather than letting the smallest of several tests read as significant by
+* volume alone. Sorted ascending, each rank is scaled by how many comparisons
+* are still in contention at that rank, and the running maximum keeps a later,
+* less-scaled rank from reporting looser than an earlier one already has.
+*
+* `familySize` may exceed `pValues.length`: a comparison `signFlipPValue`
+* could not test at all (every paired difference identical, so no p reflects
+* a real margin) is still a member of the family being controlled for, and
+* excluding it from the denominator would understate the correction owed to
+* the comparisons that could be tested.
+*/
+function holmAdjust(args) {
+	const { pValues, familySize } = args;
+	const ranked = pValues.map((pValue, index) => ({
+		pValue,
+		index
+	})).sort((a, b) => a.pValue - b.pValue);
+	const adjusted = new Array(pValues.length);
+	let runningMax = 0;
+	ranked.forEach(({ pValue, index }, rank) => {
+		runningMax = Math.max(runningMax, Math.min(1, pValue * (familySize - rank)));
+		adjusted[index] = runningMax;
+	});
+	return adjusted;
+}
+/**
+* Every total reachable by flipping some subset of `differences`' signs,
+* built by doubling rather than by scoring each of the 2^n sign masks
+* independently: after `k` differences there are 2^k sums, and folding in
+* difference `k+1` only ever adds or subtracts it from each of them, so the
+* whole enumeration costs O(2^n) instead of the O(2^n * n) a per-mask loop
+* pays for re-summing n terms every time. Measured at n=20, twenty seeds
+* being the bench's ceiling: about 63ms scoring masks one at a time against
+* about 16ms building sums this way — the difference between affording exact
+* enumeration through a full twenty-seed run and falling back to the normal
+* approximation, which is what raising `EXACT_LIMIT` in compare.ts to 20
+* relies on.
+*/
+function achievableSums(differences) {
+	let sums = [0];
+	for (const difference of differences) {
+		const next = new Array(sums.length * 2);
+		for (let index = 0; index < sums.length; index += 1) {
+			const total = sums[index];
+			next[index] = total + difference;
+			next[index + sums.length] = total - difference;
+		}
+		sums = next;
+	}
+	return sums;
+}
+/**
+* The same tail probability from a normal approximation, for batches too large
+* to enumerate. Under the sign-flip null each difference has mean zero and
+* variance equal to its square, so the total's variance is their sum.
+*/
+function normalTailProbability(args) {
+	const { differences, observed } = args;
+	const variance = differences.reduce((total, difference) => total + difference * difference, 0);
+	if (variance === 0) return observed > 0 ? 0 : 1;
+	return 1 - standardNormalCdf(observed / Math.sqrt(variance));
+}
+/** Abramowitz and Stegun 7.1.26, which is accurate to about 1e-7. */
+function standardNormalCdf(value) {
+	const sign = value < 0 ? -1 : 1;
+	const scaled = Math.abs(value) / Math.SQRT2;
+	const t = 1 / (1 + .3275911 * scaled);
+	return .5 * (1 + sign * (1 - t * (.254829592 + t * (-.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429)))) * Math.exp(-scaled * scaled)));
+}
+//#endregion
+export { sum as a, signFlipPValue as i, holmAdjust as n, mean as r, argmax as t };