textopt 0.0.0 → 0.2.0

package/dist/{evaluation-OZOp6TB7.cjs → warnings-CWRJF-jA.cjs}

@@ -30,6 +30,8 @@ function createBudget(args) {
 }
 //#endregion
 //#region src/cache.ts
+/** What `stableHash` returns for any value that serializes to `{}`. */
+const EMPTY_OBJECT_HASH = stableHash({});
 /**
 * The loop re-evaluates unchanged candidates against the same validation
 * instances constantly (every accepted child inherits most of its parent's
@@ -65,6 +67,20 @@ function stableHash(value) {
 	}
 	return `${hash32(serialized, 2166136261)}${hash32(serialized, 16777619)}`;
 }
+/**
+* Names one data instance for the evaluation cache: a content hash, so the same
+* row is the same instance wherever it appears in a run.
+*
+* Falls back to the row's position when the datum carries nothing the hash can
+* read. A Map, a Set and a class instance holding its state privately all
+* serialize to `{}`, and an id two rows share serves each of them the score the
+* other measured. Position is a weaker id — it is only stable while the data is
+* — but it is one instance per row, which is what the cache needs to be sound.
+*/
+function defaultInstanceId(args) {
+	const hash = stableHash(args.datum);
+	return hash === "" || hash === EMPTY_OBJECT_HASH ? String(args.index) : hash;
+}
 function createMemoryCache(args = {}) {
 	const { maxEntries = 1e5, entries: initial = [] } = args;
 	const entries = new Map(initial);
@@ -172,25 +188,39 @@ const DEFAULT_RETRY = {
 */
 var BudgetExhausted = class extends Error {};
 function createEvaluator(args) {
-	const { adapter, budget, cache, trackOutputs = false, onEvaluation, signal, cacheHits: initialCacheHits = 0, retry, cacheNamespace } = args;
+	const { adapter, budget, cache, trackOutputs = false, onEvaluation, signal, cacheHits: initialCacheHits = 0, usage: initialUsage, retry, cacheNamespace } = args;
 	const { attempts: retryAttempts, delayMs: retryDelayMs } = {
 		...DEFAULT_RETRY,
 		...retry
 	};
+	if (initialUsage !== void 0) assertUsage({
+		reading: initialUsage,
+		source: "Checkpoint carries"
+	});
 	let cacheHits = initialCacheHits;
 	let unchargedCalls = 0;
 	const usage = {
+		inputTokens: 0,
+		outputTokens: 0,
+		totalTokens: 0,
+		costUsd: 0,
+		rollouts: 0,
+		...initialUsage
+	};
+	/** What `charge: false` bought, kept out of the totals a ceiling reads. */
+	const unchargedUsage = {
 		inputTokens: 0,
 		outputTokens: 0,
 		totalTokens: 0,
 		costUsd: 0,
 		rollouts: 0
 	};
-	/** Folds one adapter call's reported usage into the run's totals. */
+	/** Folds one adapter call's reported usage into the totals that bought it. */
 	function recordUsage(args) {
-		usage.rollouts += args.rollouts;
+		const totals = args.charge ? usage : unchargedUsage;
+		totals.rollouts += args.rollouts;
 		for (const rollout of args.evaluation.usage ?? []) addUsage({
-			totals: usage,
+			totals,
 			rollout
 		});
 	}
@@ -252,7 +282,8 @@ function createEvaluator(args) {
 			});
 			recordUsage({
 				evaluation,
-				rollouts: rows.length
+				rollouts: rows.length,
+				charge
 			});
 			return evaluation;
 		} catch (err) {
@@ -395,6 +426,18 @@ function createEvaluator(args) {
 		cacheHits: () => cacheHits,
 		unchargedCalls: () => unchargedCalls,
 		usage: () => ({ ...usage }),
+		unchargedUsage: () => ({ ...unchargedUsage }),
+		absorbUsage: (spent) => {
+			assertUsage({
+				reading: spent,
+				source: "Absorbed usage carries"
+			});
+			usage.inputTokens += spent.inputTokens;
+			usage.outputTokens += spent.outputTokens;
+			usage.totalTokens += spent.totalTokens;
+			usage.costUsd += spent.costUsd;
+			usage.rollouts += spent.rollouts;
+		},
 		entries: () => cache?.entries?.(),
 		restore: (entries) => {
 			for (const [key, cached] of entries) cache?.set(key, cached);
@@ -471,11 +514,29 @@ function requireMeasuredMean(args) {
 function addUsage(args) {
 	const { totals, rollout } = args;
 	const { inputTokens = 0, outputTokens = 0, costUsd = 0 } = rollout;
+	assertUsage({
+		reading: rollout,
+		source: "Adapter reported"
+	});
 	totals.inputTokens += inputTokens;
 	totals.outputTokens += outputTokens;
 	totals.totalTokens += rollout.totalTokens ?? inputTokens + outputTokens;
 	totals.costUsd += costUsd;
 }
+/**
+* Refuses a reading the totals cannot hold, where it enters rather than once it
+* has been absorbed. A NaN folded in makes every later `maxCostUsd` comparison
+* false, so the ceiling stops holding without saying so, and a value that is
+* not a number at all concatenates onto the totals instead of adding to them.
+* `RolloutUsage` binds TypeScript callers and nothing else.
+*/
+function assertUsage(args) {
+	const { reading, source } = args;
+	for (const [field, value] of Object.entries(reading)) {
+		if (value === void 0) continue;
+		if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new Error(`${source} ${field} as ${value}; usage must be a non-negative finite number`);
+	}
+}
 function transientIndices(evaluation) {
 	const { transient } = evaluation;
 	if (transient === void 0) return [];
@@ -518,6 +579,114 @@ function delay(milliseconds) {
 	});
 }
 //#endregion
+//#region src/reporting.ts
+/**
+* Narrows an event off any optimizer's union to an acceptance. The tag is
+* enough: every optimizer's `candidateAccepted` intersects `CandidateAccepted`,
+* so carrying the payload is a compile-time obligation rather than a hope.
+*/
+function isCandidateAccepted(event) {
+	return event.type === "candidateAccepted";
+}
+/** Narrows an event off any optimizer's union to the end of the run. */
+function isRunFinished(event) {
+	return event.type === "finish";
+}
+/**
+* Fans one event out to every reporter, absorbing whatever they throw. A
+* reporter is an observer of the search, never a participant in it: a logging
+* endpoint that is down must not decide a run's outcome.
+*/
+function createEmitter(reporters) {
+	return function emit(event) {
+		for (const reporter of reporters) try {
+			reporter.onEvent?.(event);
+		} catch (err) {
+			console.warn("[textopt] reporter threw while handling an event", {
+				type: event.type,
+				err
+			});
+		}
+	};
+}
+/**
+* Gives every reporter its one chance to upload what it buffered. Called from
+* a `finally` rather than after the run: a reporter that buffers has the most
+* to say about a run that aborted or threw, and that is exactly the run that
+* never reaches its last line.
+*/
+async function flushReporters(reporters) {
+	await Promise.all(reporters.map(async (reporter) => {
+		try {
+			await reporter.flush?.();
+		} catch (err) {
+			console.warn("[textopt] reporter threw while flushing", { err });
+		}
+	}));
+}
+/**
+* A scored batch as the per-instance row a reporter should read: an instance
+* an infrastructure failure left unmeasured becomes `undefined` rather than
+* the zero the adapter reported for it.
+*
+* The same distinction `measuredMean` makes when it averages — a row of zeros
+* and a row of unknowns describe very different runs.
+*/
+function instanceRow(batch) {
+	return batch.scores.map((score, index) => batch.transient[index] === true ? void 0 : score);
+}
+//#endregion
+//#region src/warnings.ts
+/**
+* The validation set a run will actually select against, and whatever the
+* choice costs it.
+*
+* Defaulting to the training set is the right default for a first run and the
+* wrong number to report from one. It is worse than ordinary overfitting under
+* reflective search: the reflection prompt asks the model to mine domain facts
+* out of the traces it is shown, so those facts come out of the very instances
+* that then select the candidate carrying them. `"reuseTraining"` is the same
+* behaviour with the caller's name on it, and silences the warning.
+*/
+function resolveValidationSet(args) {
+	const { validationSet, trainingSet } = args;
+	if (validationSet === void 0) return {
+		validationSet: trainingSet,
+		warnings: [{
+			code: "validationSetReusesTraining",
+			message: "No validationSet was given, so the search selected candidates on the training instances reflection read. bestScore is fitted to them; pass a validationSet, or a testSet to measure the gap, or validationSet: \"reuseTraining\" to accept it."
+		}]
+	};
+	return {
+		validationSet: validationSet === "reuseTraining" ? trainingSet : validationSet,
+		warnings: []
+	};
+}
+/**
+* What the seed's own validation row says about whether the run could have
+* learned anything.
+*
+* A search ranks candidates by how they differ across instances, so a seed row
+* with no spread leaves nothing to rank: at the ceiling every proposal is a tie
+* the acceptance test resolves by noise, and at the floor no proposal has a
+* partial improvement to build on. Both produce a run that spends its whole
+* budget and reports a stop reason that looks like any other.
+*/
+function seedScoreWarnings(args) {
+	const { scores, perfectScore } = args;
+	const measured = scores.filter((score) => score !== void 0);
+	if (measured.length === 0) return [];
+	if (measured.every((score) => score >= perfectScore)) return [{
+		code: "seedScoreSaturated",
+		message: `The seed candidate already scores ${perfectScore} on every validation instance, so no proposal has anything to improve on and the search ranks ties. Use a harder validation set or a metric that separates these instances.`
+	}];
+	if (measured.every((score) => score <= 0)) return [{
+		code: "seedScoreFloored",
+		message: "The seed candidate scores 0 on every validation instance. That is a seed with everything to gain when the feedback says what is missing, and a metric that scores nothing when it does not — the score alone cannot tell the two apart. Check the metric against a candidate you know is good before reading much into this run."
+	}];
+	return [];
+}
+//#endregion
 Object.defineProperty(exports, "BudgetExhausted", {
 	enumerable: true,
 	get: function() {
@@ -536,6 +705,12 @@ Object.defineProperty(exports, "candidateFingerprint", {
 		return candidateFingerprint;
 	}
 });
+Object.defineProperty(exports, "candidateHash", {
+	enumerable: true,
+	get: function() {
+		return candidateHash;
+	}
+});
 Object.defineProperty(exports, "componentNames", {
 	enumerable: true,
 	get: function() {
@@ -560,6 +735,12 @@ Object.defineProperty(exports, "createDeadline", {
 		return createDeadline;
 	}
 });
+Object.defineProperty(exports, "createEmitter", {
+	enumerable: true,
+	get: function() {
+		return createEmitter;
+	}
+});
 Object.defineProperty(exports, "createEvaluator", {
 	enumerable: true,
 	get: function() {
@@ -572,6 +753,36 @@ Object.defineProperty(exports, "createMemoryCache", {
 		return createMemoryCache;
 	}
 });
+Object.defineProperty(exports, "defaultInstanceId", {
+	enumerable: true,
+	get: function() {
+		return defaultInstanceId;
+	}
+});
+Object.defineProperty(exports, "flushReporters", {
+	enumerable: true,
+	get: function() {
+		return flushReporters;
+	}
+});
+Object.defineProperty(exports, "instanceRow", {
+	enumerable: true,
+	get: function() {
+		return instanceRow;
+	}
+});
+Object.defineProperty(exports, "isCandidateAccepted", {
+	enumerable: true,
+	get: function() {
+		return isCandidateAccepted;
+	}
+});
+Object.defineProperty(exports, "isRunFinished", {
+	enumerable: true,
+	get: function() {
+		return isRunFinished;
+	}
+});
 Object.defineProperty(exports, "measuredMean", {
 	enumerable: true,
 	get: function() {
@@ -584,12 +795,24 @@ Object.defineProperty(exports, "requireMeasuredMean", {
 		return requireMeasuredMean;
 	}
 });
+Object.defineProperty(exports, "resolveValidationSet", {
+	enumerable: true,
+	get: function() {
+		return resolveValidationSet;
+	}
+});
 Object.defineProperty(exports, "runFingerprint", {
 	enumerable: true,
 	get: function() {
 		return runFingerprint;
 	}
 });
+Object.defineProperty(exports, "seedScoreWarnings", {
+	enumerable: true,
+	get: function() {
+		return seedScoreWarnings;
+	}
+});
 Object.defineProperty(exports, "stableHash", {
 	enumerable: true,
 	get: function() {

package/dist/{evaluation-BV0nSZVx.mjs → warnings-OxvDi9kN.mjs}

@@ -30,6 +30,8 @@ function createBudget(args) {
 }
 //#endregion
 //#region src/cache.ts
+/** What `stableHash` returns for any value that serializes to `{}`. */
+const EMPTY_OBJECT_HASH = stableHash({});
 /**
 * The loop re-evaluates unchanged candidates against the same validation
 * instances constantly (every accepted child inherits most of its parent's
@@ -65,6 +67,20 @@ function stableHash(value) {
 	}
 	return `${hash32(serialized, 2166136261)}${hash32(serialized, 16777619)}`;
 }
+/**
+* Names one data instance for the evaluation cache: a content hash, so the same
+* row is the same instance wherever it appears in a run.
+*
+* Falls back to the row's position when the datum carries nothing the hash can
+* read. A Map, a Set and a class instance holding its state privately all
+* serialize to `{}`, and an id two rows share serves each of them the score the
+* other measured. Position is a weaker id — it is only stable while the data is
+* — but it is one instance per row, which is what the cache needs to be sound.
+*/
+function defaultInstanceId(args) {
+	const hash = stableHash(args.datum);
+	return hash === "" || hash === EMPTY_OBJECT_HASH ? String(args.index) : hash;
+}
 function createMemoryCache(args = {}) {
 	const { maxEntries = 1e5, entries: initial = [] } = args;
 	const entries = new Map(initial);
@@ -172,25 +188,39 @@ const DEFAULT_RETRY = {
 */
 var BudgetExhausted = class extends Error {};
 function createEvaluator(args) {
-	const { adapter, budget, cache, trackOutputs = false, onEvaluation, signal, cacheHits: initialCacheHits = 0, retry, cacheNamespace } = args;
+	const { adapter, budget, cache, trackOutputs = false, onEvaluation, signal, cacheHits: initialCacheHits = 0, usage: initialUsage, retry, cacheNamespace } = args;
 	const { attempts: retryAttempts, delayMs: retryDelayMs } = {
 		...DEFAULT_RETRY,
 		...retry
 	};
+	if (initialUsage !== void 0) assertUsage({
+		reading: initialUsage,
+		source: "Checkpoint carries"
+	});
 	let cacheHits = initialCacheHits;
 	let unchargedCalls = 0;
 	const usage = {
+		inputTokens: 0,
+		outputTokens: 0,
+		totalTokens: 0,
+		costUsd: 0,
+		rollouts: 0,
+		...initialUsage
+	};
+	/** What `charge: false` bought, kept out of the totals a ceiling reads. */
+	const unchargedUsage = {
 		inputTokens: 0,
 		outputTokens: 0,
 		totalTokens: 0,
 		costUsd: 0,
 		rollouts: 0
 	};
-	/** Folds one adapter call's reported usage into the run's totals. */
+	/** Folds one adapter call's reported usage into the totals that bought it. */
 	function recordUsage(args) {
-		usage.rollouts += args.rollouts;
+		const totals = args.charge ? usage : unchargedUsage;
+		totals.rollouts += args.rollouts;
 		for (const rollout of args.evaluation.usage ?? []) addUsage({
-			totals: usage,
+			totals,
 			rollout
 		});
 	}
@@ -252,7 +282,8 @@ function createEvaluator(args) {
 			});
 			recordUsage({
 				evaluation,
-				rollouts: rows.length
+				rollouts: rows.length,
+				charge
 			});
 			return evaluation;
 		} catch (err) {
@@ -395,6 +426,18 @@ function createEvaluator(args) {
 		cacheHits: () => cacheHits,
 		unchargedCalls: () => unchargedCalls,
 		usage: () => ({ ...usage }),
+		unchargedUsage: () => ({ ...unchargedUsage }),
+		absorbUsage: (spent) => {
+			assertUsage({
+				reading: spent,
+				source: "Absorbed usage carries"
+			});
+			usage.inputTokens += spent.inputTokens;
+			usage.outputTokens += spent.outputTokens;
+			usage.totalTokens += spent.totalTokens;
+			usage.costUsd += spent.costUsd;
+			usage.rollouts += spent.rollouts;
+		},
 		entries: () => cache?.entries?.(),
 		restore: (entries) => {
 			for (const [key, cached] of entries) cache?.set(key, cached);
@@ -471,11 +514,29 @@ function requireMeasuredMean(args) {
 function addUsage(args) {
 	const { totals, rollout } = args;
 	const { inputTokens = 0, outputTokens = 0, costUsd = 0 } = rollout;
+	assertUsage({
+		reading: rollout,
+		source: "Adapter reported"
+	});
 	totals.inputTokens += inputTokens;
 	totals.outputTokens += outputTokens;
 	totals.totalTokens += rollout.totalTokens ?? inputTokens + outputTokens;
 	totals.costUsd += costUsd;
 }
+/**
+* Refuses a reading the totals cannot hold, where it enters rather than once it
+* has been absorbed. A NaN folded in makes every later `maxCostUsd` comparison
+* false, so the ceiling stops holding without saying so, and a value that is
+* not a number at all concatenates onto the totals instead of adding to them.
+* `RolloutUsage` binds TypeScript callers and nothing else.
+*/
+function assertUsage(args) {
+	const { reading, source } = args;
+	for (const [field, value] of Object.entries(reading)) {
+		if (value === void 0) continue;
+		if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new Error(`${source} ${field} as ${value}; usage must be a non-negative finite number`);
+	}
+}
 function transientIndices(evaluation) {
 	const { transient } = evaluation;
 	if (transient === void 0) return [];
@@ -518,4 +579,112 @@ function delay(milliseconds) {
 	});
 }
 //#endregion
-export { requireMeasuredMean as a, candidateFingerprint as c, createMemoryCache as d, stableHash as f, measuredMean as i, runFingerprint as l, costExhausted as n, createDeadline as o, createBudget as p, createEvaluator as r, assertResumable as s, BudgetExhausted as t, componentNames as u };
+//#region src/reporting.ts
+/**
+* Narrows an event off any optimizer's union to an acceptance. The tag is
+* enough: every optimizer's `candidateAccepted` intersects `CandidateAccepted`,
+* so carrying the payload is a compile-time obligation rather than a hope.
+*/
+function isCandidateAccepted(event) {
+	return event.type === "candidateAccepted";
+}
+/** Narrows an event off any optimizer's union to the end of the run. */
+function isRunFinished(event) {
+	return event.type === "finish";
+}
+/**
+* Fans one event out to every reporter, absorbing whatever they throw. A
+* reporter is an observer of the search, never a participant in it: a logging
+* endpoint that is down must not decide a run's outcome.
+*/
+function createEmitter(reporters) {
+	return function emit(event) {
+		for (const reporter of reporters) try {
+			reporter.onEvent?.(event);
+		} catch (err) {
+			console.warn("[textopt] reporter threw while handling an event", {
+				type: event.type,
+				err
+			});
+		}
+	};
+}
+/**
+* Gives every reporter its one chance to upload what it buffered. Called from
+* a `finally` rather than after the run: a reporter that buffers has the most
+* to say about a run that aborted or threw, and that is exactly the run that
+* never reaches its last line.
+*/
+async function flushReporters(reporters) {
+	await Promise.all(reporters.map(async (reporter) => {
+		try {
+			await reporter.flush?.();
+		} catch (err) {
+			console.warn("[textopt] reporter threw while flushing", { err });
+		}
+	}));
+}
+/**
+* A scored batch as the per-instance row a reporter should read: an instance
+* an infrastructure failure left unmeasured becomes `undefined` rather than
+* the zero the adapter reported for it.
+*
+* The same distinction `measuredMean` makes when it averages — a row of zeros
+* and a row of unknowns describe very different runs.
+*/
+function instanceRow(batch) {
+	return batch.scores.map((score, index) => batch.transient[index] === true ? void 0 : score);
+}
+//#endregion
+//#region src/warnings.ts
+/**
+* The validation set a run will actually select against, and whatever the
+* choice costs it.
+*
+* Defaulting to the training set is the right default for a first run and the
+* wrong number to report from one. It is worse than ordinary overfitting under
+* reflective search: the reflection prompt asks the model to mine domain facts
+* out of the traces it is shown, so those facts come out of the very instances
+* that then select the candidate carrying them. `"reuseTraining"` is the same
+* behaviour with the caller's name on it, and silences the warning.
+*/
+function resolveValidationSet(args) {
+	const { validationSet, trainingSet } = args;
+	if (validationSet === void 0) return {
+		validationSet: trainingSet,
+		warnings: [{
+			code: "validationSetReusesTraining",
+			message: "No validationSet was given, so the search selected candidates on the training instances reflection read. bestScore is fitted to them; pass a validationSet, or a testSet to measure the gap, or validationSet: \"reuseTraining\" to accept it."
+		}]
+	};
+	return {
+		validationSet: validationSet === "reuseTraining" ? trainingSet : validationSet,
+		warnings: []
+	};
+}
+/**
+* What the seed's own validation row says about whether the run could have
+* learned anything.
+*
+* A search ranks candidates by how they differ across instances, so a seed row
+* with no spread leaves nothing to rank: at the ceiling every proposal is a tie
+* the acceptance test resolves by noise, and at the floor no proposal has a
+* partial improvement to build on. Both produce a run that spends its whole
+* budget and reports a stop reason that looks like any other.
+*/
+function seedScoreWarnings(args) {
+	const { scores, perfectScore } = args;
+	const measured = scores.filter((score) => score !== void 0);
+	if (measured.length === 0) return [];
+	if (measured.every((score) => score >= perfectScore)) return [{
+		code: "seedScoreSaturated",
+		message: `The seed candidate already scores ${perfectScore} on every validation instance, so no proposal has anything to improve on and the search ranks ties. Use a harder validation set or a metric that separates these instances.`
+	}];
+	if (measured.every((score) => score <= 0)) return [{
+		code: "seedScoreFloored",
+		message: "The seed candidate scores 0 on every validation instance. That is a seed with everything to gain when the feedback says what is missing, and a metric that scores nothing when it does not — the score alone cannot tell the two apart. Check the metric against a candidate you know is good before reading much into this run."
+	}];
+	return [];
+}
+//#endregion
+export { createBudget as S, componentNames as _, instanceRow as a, defaultInstanceId as b, BudgetExhausted as c, measuredMean as d, requireMeasuredMean as f, runFingerprint as g, candidateFingerprint as h, flushReporters as i, costExhausted as l, assertResumable as m, seedScoreWarnings as n, isCandidateAccepted as o, createDeadline as p, createEmitter as r, isRunFinished as s, resolveValidationSet as t, createEvaluator as u, candidateHash as v, stableHash as x, createMemoryCache as y };