textopt 0.0.0 → 0.1.0

package/dist/demos-D2o0qSSo.cjs

@@ -0,0 +1,244 @@
+const require_reporting = require("./reporting-CNHzbJC-.cjs");
+//#region src/harvest.ts
+/**
+* Run a candidate over data and keep the rollouts the metric rewarded.
+*
+* The library's one paid collection primitive, with two consumers: a few-shot
+* block wants four of these, and a distillation set wants thousands. Both are
+* the same pass — run the candidate, score it, keep what cleared the bar — so
+* both share the budget, retry and transient-failure handling that pass needs.
+*
+* Which data to sweep is the caller's decision and the consequential one. A
+* validation set is the wrong choice: it is the set that selected the candidate,
+* so the rollouts it yields are enriched for the candidate's fit to those
+* instances rather than to the task. Prefer the training set, or a pool held
+* out of the run entirely.
+*/
+async function harvestRollouts(args) {
+	const { adapter, candidate, data, minScore, maxRollouts = Number.POSITIVE_INFINITY, batchSize = Math.min(maxRollouts, data.length), maxMetricCalls = data.length, maxCostUsd, rng, signal } = args;
+	if (data.length === 0) throw new Error("harvestRollouts requires non-empty data");
+	const budget = require_reporting.createBudget({ maxMetricCalls });
+	const evaluator = require_reporting.createEvaluator({
+		adapter,
+		budget,
+		...signal === void 0 ? {} : { signal }
+	});
+	const order = rng === void 0 ? [...data] : rng.shuffle(data);
+	const rollouts = [];
+	let attempted = 0;
+	for (let start = 0; start < order.length; start += batchSize) {
+		if (rollouts.length >= maxRollouts || signal?.aborted) break;
+		if (require_reporting.costExhausted({
+			usage: evaluator.usage(),
+			maxCostUsd
+		})) break;
+		const batch = order.slice(start, start + Math.min(batchSize, budget.remaining()));
+		if (batch.length === 0) break;
+		const evaluation = await evaluator.evaluateTraced({
+			candidate,
+			batch,
+			split: "train",
+			phase: "seed",
+			candidateId: null,
+			iteration: 0
+		});
+		if (evaluation === null) break;
+		attempted += batch.length;
+		for (let index = 0; index < batch.length; index += 1) {
+			const score = evaluation.scores[index];
+			if (!(minScore === void 0 ? score > 0 : score >= minScore) || rollouts.length >= maxRollouts) continue;
+			rollouts.push({
+				input: batch[index],
+				output: evaluation.outputs[index],
+				score
+			});
+		}
+	}
+	return {
+		rollouts,
+		metricCalls: budget.spent(),
+		usage: evaluator.usage(),
+		attempted
+	};
+}
+//#endregion
+//#region src/demos.ts
+const DEMO_OPEN = "<demo>";
+const DEMO_CLOSE = "</demo>";
+const DEMO_BLOCK = /<demo>\s*([\s\S]*?)\s*<\/demo>/g;
+const DEMO_PARTS = /<input>\s*([\s\S]*?)\s*<\/input>\s*<output>\s*([\s\S]*?)\s*<\/output>/;
+const DELIMITER_TAG = /<(\/?)(demo|input|output)>/g;
+const ESCAPED_DELIMITER_TAG = /&lt;(\/?)(demo|input|output)>/g;
+const DEFAULT_MAX_DEMOS = 4;
+/**
+* Harvest demonstrations by running a candidate over the training set and keeping
+* the rollouts the metric rewarded.
+*
+* The cheapest signal in the whole library: a rollout that scored well is
+* already paid for, and turning it into a few-shot block costs one pass over
+* the data rather than a search. Instruction search and demonstrations pull on
+* different parts of a model's behaviour — instructions on what to do,
+* examples on what the output should look like — so a seed carrying both
+* starts somewhere neither reaches alone.
+*/
+async function harvestFewShotExamples(args) {
+	const { adapter, candidate, trainingSet, minScore, maxDemos = DEFAULT_MAX_DEMOS, batchSize = maxDemos, maxMetricCalls = trainingSet.length, maxCostUsd, rng, renderDemo, signal } = args;
+	if (trainingSet.length === 0) throw new Error("harvestFewShotExamples requires a non-empty trainingSet");
+	const harvest = await harvestRollouts({
+		adapter,
+		candidate,
+		data: trainingSet,
+		maxRollouts: maxDemos,
+		batchSize,
+		maxMetricCalls,
+		...maxCostUsd === void 0 ? {} : { maxCostUsd },
+		...minScore === void 0 ? {} : { minScore },
+		...rng === void 0 ? {} : { rng },
+		...signal === void 0 ? {} : { signal }
+	});
+	return {
+		demos: harvest.rollouts,
+		block: formatDemos(harvest.rollouts, renderDemo === void 0 ? {} : { render: renderDemo }),
+		metricCalls: harvest.metricCalls,
+		usage: harvest.usage,
+		attempted: harvest.attempted
+	};
+}
+/**
+* Render demos as the text a candidate component holds.
+*
+* Delimited rather than free-form so `parseDemos` can read them back: a demo
+* component is edited over the course of a run, and a block that cannot be
+* parsed can only be replaced wholesale, throwing away every example found
+* before it.
+*/
+function formatDemos(demos, options = {}) {
+	const { render = renderDefault } = options;
+	if (demos.length === 0) return "";
+	return demos.map((demo, index) => `${DEMO_OPEN}\n${render({
+		demo,
+		index
+	})}\n${DEMO_CLOSE}`).join("\n");
+}
+/**
+* Rewrite the demos a component holds, leaving everything else it says intact.
+*
+* A component is not always only examples. SIMBA appends advice to the same
+* text it appends demonstrations to, so replacing the component wholesale with
+* a fresh block would delete the instructions the other mutation wrote. The
+* replacement lands where the first demo was, so a block a caller placed after
+* its preamble stays after it.
+*/
+function replaceDemos(args) {
+	const { text, demos, render } = args;
+	const block = formatDemos(demos, render === void 0 ? {} : { render });
+	const [before, ...rest] = text.replace(DEMO_BLOCK, "\0").split("\0");
+	if (rest.length === 0) return join([text, block]);
+	return join([
+		before ?? "",
+		block,
+		rest.join("")
+	]);
+}
+/**
+* Recover the demos from a formatted block, ignoring anything written around
+* them. Text a model rewrote and mangled yields the demos it left intact
+* rather than throwing: a malformed example is worth less than the rest of the
+* block, not more than it.
+*/
+function parseDemos(text) {
+	const demos = [];
+	for (const match of text.matchAll(DEMO_BLOCK)) {
+		const parts = (match[1] ?? "").match(DEMO_PARTS);
+		if (parts === null) continue;
+		const input = parseValue(parts[1] ?? "");
+		const output = parseValue(parts[2] ?? "");
+		demos.push({
+			input,
+			output
+		});
+	}
+	return demos;
+}
+/** Joins what survived a replacement, without leaving blank runs behind. */
+function join(parts) {
+	return parts.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
+}
+function renderDefault(args) {
+	const { demo } = args;
+	return [
+		"<input>",
+		serialize(demo.input),
+		"</input>",
+		"<output>",
+		serialize(demo.output),
+		"</output>"
+	].join("\n");
+}
+/** Strings stay as they are; anything else is shown as JSON. */
+function serialize(value) {
+	if (typeof value === "string") return escapeDelimiters(value);
+	try {
+		return escapeDelimiters(JSON.stringify(value, null, 2) ?? String(value));
+	} catch {
+		return escapeDelimiters(String(value));
+	}
+}
+function parseValue(text) {
+	const unescaped = unescapeDelimiters(text);
+	try {
+		return JSON.parse(unescaped);
+	} catch {
+		return unescaped;
+	}
+}
+/**
+* Neutralize the tags a demo block is delimited by, so a value carrying one
+* cannot end the block it sits in.
+*
+* A system that quotes its own prompt back produces exactly that: the output
+* worth keeping is a demonstration containing `</demo>`, and appending it raw
+* closes the outer block early. What comes back is not the demo that went in,
+* and SIMBA reparses and rewrites the block at every step, so the damage
+* compounds over a run rather than showing up once.
+*
+* The escape is escaped first, so a value that already reads `&lt;demo>`
+* survives the round trip as itself.
+*/
+function escapeDelimiters(text) {
+	return text.replaceAll("&lt;", "&amp;lt;").replace(DELIMITER_TAG, "&lt;$1$2>");
+}
+function unescapeDelimiters(text) {
+	return text.replace(ESCAPED_DELIMITER_TAG, "<$1$2>").replaceAll("&amp;lt;", "&lt;");
+}
+//#endregion
+Object.defineProperty(exports, "formatDemos", {
+	enumerable: true,
+	get: function() {
+		return formatDemos;
+	}
+});
+Object.defineProperty(exports, "harvestFewShotExamples", {
+	enumerable: true,
+	get: function() {
+		return harvestFewShotExamples;
+	}
+});
+Object.defineProperty(exports, "harvestRollouts", {
+	enumerable: true,
+	get: function() {
+		return harvestRollouts;
+	}
+});
+Object.defineProperty(exports, "parseDemos", {
+	enumerable: true,
+	get: function() {
+		return parseDemos;
+	}
+});
+Object.defineProperty(exports, "replaceDemos", {
+	enumerable: true,
+	get: function() {
+		return replaceDemos;
+	}
+});

package/dist/demos-DE2oxNWX.mjs

@@ -0,0 +1,215 @@
+import { b as createBudget, c as createEvaluator, s as costExhausted } from "./reporting-DY-DC4HG.mjs";
+//#region src/harvest.ts
+/**
+* Run a candidate over data and keep the rollouts the metric rewarded.
+*
+* The library's one paid collection primitive, with two consumers: a few-shot
+* block wants four of these, and a distillation set wants thousands. Both are
+* the same pass — run the candidate, score it, keep what cleared the bar — so
+* both share the budget, retry and transient-failure handling that pass needs.
+*
+* Which data to sweep is the caller's decision and the consequential one. A
+* validation set is the wrong choice: it is the set that selected the candidate,
+* so the rollouts it yields are enriched for the candidate's fit to those
+* instances rather than to the task. Prefer the training set, or a pool held
+* out of the run entirely.
+*/
+async function harvestRollouts(args) {
+	const { adapter, candidate, data, minScore, maxRollouts = Number.POSITIVE_INFINITY, batchSize = Math.min(maxRollouts, data.length), maxMetricCalls = data.length, maxCostUsd, rng, signal } = args;
+	if (data.length === 0) throw new Error("harvestRollouts requires non-empty data");
+	const budget = createBudget({ maxMetricCalls });
+	const evaluator = createEvaluator({
+		adapter,
+		budget,
+		...signal === void 0 ? {} : { signal }
+	});
+	const order = rng === void 0 ? [...data] : rng.shuffle(data);
+	const rollouts = [];
+	let attempted = 0;
+	for (let start = 0; start < order.length; start += batchSize) {
+		if (rollouts.length >= maxRollouts || signal?.aborted) break;
+		if (costExhausted({
+			usage: evaluator.usage(),
+			maxCostUsd
+		})) break;
+		const batch = order.slice(start, start + Math.min(batchSize, budget.remaining()));
+		if (batch.length === 0) break;
+		const evaluation = await evaluator.evaluateTraced({
+			candidate,
+			batch,
+			split: "train",
+			phase: "seed",
+			candidateId: null,
+			iteration: 0
+		});
+		if (evaluation === null) break;
+		attempted += batch.length;
+		for (let index = 0; index < batch.length; index += 1) {
+			const score = evaluation.scores[index];
+			if (!(minScore === void 0 ? score > 0 : score >= minScore) || rollouts.length >= maxRollouts) continue;
+			rollouts.push({
+				input: batch[index],
+				output: evaluation.outputs[index],
+				score
+			});
+		}
+	}
+	return {
+		rollouts,
+		metricCalls: budget.spent(),
+		usage: evaluator.usage(),
+		attempted
+	};
+}
+//#endregion
+//#region src/demos.ts
+const DEMO_OPEN = "<demo>";
+const DEMO_CLOSE = "</demo>";
+const DEMO_BLOCK = /<demo>\s*([\s\S]*?)\s*<\/demo>/g;
+const DEMO_PARTS = /<input>\s*([\s\S]*?)\s*<\/input>\s*<output>\s*([\s\S]*?)\s*<\/output>/;
+const DELIMITER_TAG = /<(\/?)(demo|input|output)>/g;
+const ESCAPED_DELIMITER_TAG = /&lt;(\/?)(demo|input|output)>/g;
+const DEFAULT_MAX_DEMOS = 4;
+/**
+* Harvest demonstrations by running a candidate over the training set and keeping
+* the rollouts the metric rewarded.
+*
+* The cheapest signal in the whole library: a rollout that scored well is
+* already paid for, and turning it into a few-shot block costs one pass over
+* the data rather than a search. Instruction search and demonstrations pull on
+* different parts of a model's behaviour — instructions on what to do,
+* examples on what the output should look like — so a seed carrying both
+* starts somewhere neither reaches alone.
+*/
+async function harvestFewShotExamples(args) {
+	const { adapter, candidate, trainingSet, minScore, maxDemos = DEFAULT_MAX_DEMOS, batchSize = maxDemos, maxMetricCalls = trainingSet.length, maxCostUsd, rng, renderDemo, signal } = args;
+	if (trainingSet.length === 0) throw new Error("harvestFewShotExamples requires a non-empty trainingSet");
+	const harvest = await harvestRollouts({
+		adapter,
+		candidate,
+		data: trainingSet,
+		maxRollouts: maxDemos,
+		batchSize,
+		maxMetricCalls,
+		...maxCostUsd === void 0 ? {} : { maxCostUsd },
+		...minScore === void 0 ? {} : { minScore },
+		...rng === void 0 ? {} : { rng },
+		...signal === void 0 ? {} : { signal }
+	});
+	return {
+		demos: harvest.rollouts,
+		block: formatDemos(harvest.rollouts, renderDemo === void 0 ? {} : { render: renderDemo }),
+		metricCalls: harvest.metricCalls,
+		usage: harvest.usage,
+		attempted: harvest.attempted
+	};
+}
+/**
+* Render demos as the text a candidate component holds.
+*
+* Delimited rather than free-form so `parseDemos` can read them back: a demo
+* component is edited over the course of a run, and a block that cannot be
+* parsed can only be replaced wholesale, throwing away every example found
+* before it.
+*/
+function formatDemos(demos, options = {}) {
+	const { render = renderDefault } = options;
+	if (demos.length === 0) return "";
+	return demos.map((demo, index) => `${DEMO_OPEN}\n${render({
+		demo,
+		index
+	})}\n${DEMO_CLOSE}`).join("\n");
+}
+/**
+* Rewrite the demos a component holds, leaving everything else it says intact.
+*
+* A component is not always only examples. SIMBA appends advice to the same
+* text it appends demonstrations to, so replacing the component wholesale with
+* a fresh block would delete the instructions the other mutation wrote. The
+* replacement lands where the first demo was, so a block a caller placed after
+* its preamble stays after it.
+*/
+function replaceDemos(args) {
+	const { text, demos, render } = args;
+	const block = formatDemos(demos, render === void 0 ? {} : { render });
+	const [before, ...rest] = text.replace(DEMO_BLOCK, "\0").split("\0");
+	if (rest.length === 0) return join([text, block]);
+	return join([
+		before ?? "",
+		block,
+		rest.join("")
+	]);
+}
+/**
+* Recover the demos from a formatted block, ignoring anything written around
+* them. Text a model rewrote and mangled yields the demos it left intact
+* rather than throwing: a malformed example is worth less than the rest of the
+* block, not more than it.
+*/
+function parseDemos(text) {
+	const demos = [];
+	for (const match of text.matchAll(DEMO_BLOCK)) {
+		const parts = (match[1] ?? "").match(DEMO_PARTS);
+		if (parts === null) continue;
+		const input = parseValue(parts[1] ?? "");
+		const output = parseValue(parts[2] ?? "");
+		demos.push({
+			input,
+			output
+		});
+	}
+	return demos;
+}
+/** Joins what survived a replacement, without leaving blank runs behind. */
+function join(parts) {
+	return parts.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
+}
+function renderDefault(args) {
+	const { demo } = args;
+	return [
+		"<input>",
+		serialize(demo.input),
+		"</input>",
+		"<output>",
+		serialize(demo.output),
+		"</output>"
+	].join("\n");
+}
+/** Strings stay as they are; anything else is shown as JSON. */
+function serialize(value) {
+	if (typeof value === "string") return escapeDelimiters(value);
+	try {
+		return escapeDelimiters(JSON.stringify(value, null, 2) ?? String(value));
+	} catch {
+		return escapeDelimiters(String(value));
+	}
+}
+function parseValue(text) {
+	const unescaped = unescapeDelimiters(text);
+	try {
+		return JSON.parse(unescaped);
+	} catch {
+		return unescaped;
+	}
+}
+/**
+* Neutralize the tags a demo block is delimited by, so a value carrying one
+* cannot end the block it sits in.
+*
+* A system that quotes its own prompt back produces exactly that: the output
+* worth keeping is a demonstration containing `</demo>`, and appending it raw
+* closes the outer block early. What comes back is not the demo that went in,
+* and SIMBA reparses and rewrites the block at every step, so the damage
+* compounds over a run rather than showing up once.
+*
+* The escape is escaped first, so a value that already reads `&lt;demo>`
+* survives the round trip as itself.
+*/
+function escapeDelimiters(text) {
+	return text.replaceAll("&lt;", "&amp;lt;").replace(DELIMITER_TAG, "&lt;$1$2>");
+}
+function unescapeDelimiters(text) {
+	return text.replace(ESCAPED_DELIMITER_TAG, "<$1$2>").replaceAll("&amp;lt;", "&lt;");
+}
+//#endregion
+export { harvestRollouts as a, replaceDemos as i, harvestFewShotExamples as n, parseDemos as r, formatDemos as t };

package/dist/file-cache.cjs

@@ -17,7 +17,9 @@ let node_path = require("node:path");
 function createFileCache(args) {
 	const { path, maxEntries = 1e6 } = args;
 	(0, node_fs.mkdirSync)((0, node_path.dirname)(path), { recursive: true });
-	const entries = readLog(path);
+	const log = readLog(path);
+	const entries = log.entries;
+	if (log.unterminated) (0, node_fs.appendFileSync)(path, "\n");
 	return {
 		get: (key) => entries.get(key),
 		set: (key, cached) => {
@@ -42,14 +44,20 @@ function readLog(path) {
 	try {
 		contents = (0, node_fs.readFileSync)(path, "utf8");
 	} catch {
-		return entries;
+		return {
+			entries,
+			unterminated: false
+		};
 	}
 	for (const line of contents.split("\n")) {
 		if (line.length === 0) continue;
 		const entry = parseEntry(line);
 		if (entry !== void 0) entries.set(entry[0], entry[1]);
 	}
-	return entries;
+	return {
+		entries,
+		unterminated: contents.length > 0 && !contents.endsWith("\n")
+	};
 }
 function parseEntry(line) {
 	let parsed;

package/dist/file-cache.mjs

@@ -16,7 +16,9 @@ import { dirname } from "node:path";
 function createFileCache(args) {
 	const { path, maxEntries = 1e6 } = args;
 	mkdirSync(dirname(path), { recursive: true });
-	const entries = readLog(path);
+	const log = readLog(path);
+	const entries = log.entries;
+	if (log.unterminated) appendFileSync(path, "\n");
 	return {
 		get: (key) => entries.get(key),
 		set: (key, cached) => {
@@ -41,14 +43,20 @@ function readLog(path) {
 	try {
 		contents = readFileSync(path, "utf8");
 	} catch {
-		return entries;
+		return {
+			entries,
+			unterminated: false
+		};
 	}
 	for (const line of contents.split("\n")) {
 		if (line.length === 0) continue;
 		const entry = parseEntry(line);
 		if (entry !== void 0) entries.set(entry[0], entry[1]);
 	}
-	return entries;
+	return {
+		entries,
+		unterminated: contents.length > 0 && !contents.endsWith("\n")
+	};
 }
 function parseEntry(line) {
 	let parsed;