textopt 0.0.0

package/dist/demos-BTuzFNsp.d.cts

@@ -0,0 +1,88 @@
+import { n as Candidate, t as Adapter } from "./types-CWv4IQFF.cjs";
+import { t as Rng } from "./rng-BR5MOedA.cjs";
+//#region src/demos.d.ts
+/**
+ * One worked example: what went in, what a run of the system produced, and
+ * how that output scored. A demo is harvested, never written — its value is
+ * that the output is one the system actually produced and the metric actually
+ * rewarded.
+ */
+interface Demo<Datum = unknown, Output = unknown> {
+  input: Datum;
+  output: Output;
+  /** Present on harvested demos, absent on ones recovered from a block. */
+  score?: number;
+}
+type DemoRenderer<Datum = unknown, Output = unknown> = (args: {
+  demo: Demo<Datum, Output>;
+  index: number;
+}) => string;
+interface BootstrapResult<Datum, Output> {
+  demos: Demo<Datum, Output>[];
+  /** The demos as a candidate component, ready to seed a run with. */
+  block: string;
+  /** Rollouts this cost. Bootstrapping is cheap, not free. */
+  metricCalls: number;
+  attempted: number;
+}
+/**
+ * Harvest demonstrations by running a candidate over the training set and keeping
+ * the rollouts the metric rewarded.
+ *
+ * The cheapest signal in the whole library: a rollout that scored well is
+ * already paid for, and turning it into a few-shot block costs one pass over
+ * the data rather than a search. Instruction search and demonstrations pull on
+ * different parts of a model's behaviour — instructions on what to do,
+ * examples on what the output should look like — so a seed carrying both
+ * starts somewhere neither reaches alone.
+ */
+declare function bootstrapDemos<Datum, Trajectory, Output, K extends string = string>(args: {
+  adapter: Adapter<Datum, Trajectory, Output, K>;
+  /** The candidate to run. Usually the seed, sometimes a run's winner. */
+  candidate: Candidate<K>;
+  trainingSet: readonly Datum[];
+  /**
+   * Score a rollout must reach to be kept. Unset keeps every rollout the
+   * metric rewarded at all, which is what MIPROv2's bootstrapper does without
+   * a `metric_threshold`: it keeps a trace on any truthy score and only
+   * compares against a number once one is configured.
+   *
+   * Demanding a perfect score instead is the right call for a boolean metric
+   * and the wrong one for a graded metric, where it throws away every rollout
+   * that was most of the way there — which on a hard task is all of them.
+   */
+  minScore?: number;
+  /** Demos to collect before stopping. Default 4. */
+  maxDemos?: number;
+  /**
+   * Instances per rollout batch. Smaller batches stop closer to the moment
+   * enough demos exist, at the cost of less concurrency inside the adapter.
+   */
+  batchSize?: number;
+  /** Ceiling on rollouts. Defaults to one pass over the trainingSet. */
+  maxMetricCalls?: number;
+  /** Shuffles the trainingSet first, so demos are not all drawn from its head. */
+  rng?: Rng;
+  renderDemo?: DemoRenderer<Datum, Output>;
+  signal?: AbortSignal;
+}): Promise<BootstrapResult<Datum, Output>>;
+/**
+ * Render demos as the text a candidate component holds.
+ *
+ * Delimited rather than free-form so `parseDemos` can read them back: a demo
+ * component is edited over the course of a run, and a block that cannot be
+ * parsed can only be replaced wholesale, throwing away every example found
+ * before it.
+ */
+declare function formatDemos<Datum, Output>(demos: readonly Demo<Datum, Output>[], options?: {
+  render?: DemoRenderer<Datum, Output>;
+}): string;
+/**
+ * Recover the demos from a formatted block, ignoring anything written around
+ * them. Text a model rewrote and mangled yields the demos it left intact
+ * rather than throwing: a malformed example is worth less than the rest of the
+ * block, not more than it.
+ */
+declare function parseDemos(text: string): Demo[];
+//#endregion
+export { formatDemos as a, bootstrapDemos as i, Demo as n, parseDemos as o, DemoRenderer as r, BootstrapResult as t };

package/dist/demos-Degx6UmP.mjs

@@ -0,0 +1,126 @@
+import { p as createBudget, r as createEvaluator } from "./evaluation-BV0nSZVx.mjs";
+//#region src/demos.ts
+const DEMO_OPEN = "<demo>";
+const DEMO_CLOSE = "</demo>";
+const DEMO_BLOCK = /<demo>\s*([\s\S]*?)\s*<\/demo>/g;
+const DEMO_PARTS = /<input>\s*([\s\S]*?)\s*<\/input>\s*<output>\s*([\s\S]*?)\s*<\/output>/;
+const DEFAULT_MAX_DEMOS = 4;
+/**
+* Harvest demonstrations by running a candidate over the training set and keeping
+* the rollouts the metric rewarded.
+*
+* The cheapest signal in the whole library: a rollout that scored well is
+* already paid for, and turning it into a few-shot block costs one pass over
+* the data rather than a search. Instruction search and demonstrations pull on
+* different parts of a model's behaviour — instructions on what to do,
+* examples on what the output should look like — so a seed carrying both
+* starts somewhere neither reaches alone.
+*/
+async function bootstrapDemos(args) {
+	const { adapter, candidate, trainingSet, minScore, maxDemos = DEFAULT_MAX_DEMOS, batchSize = maxDemos, maxMetricCalls = trainingSet.length, rng, renderDemo, signal } = args;
+	if (trainingSet.length === 0) throw new Error("bootstrapDemos requires a non-empty trainingSet");
+	const budget = createBudget({ maxMetricCalls });
+	const evaluator = createEvaluator({
+		adapter,
+		budget,
+		...signal === void 0 ? {} : { signal }
+	});
+	const order = rng === void 0 ? [...trainingSet] : rng.shuffle(trainingSet);
+	const demos = [];
+	let attempted = 0;
+	for (let start = 0; start < order.length; start += batchSize) {
+		if (demos.length >= maxDemos || signal?.aborted) break;
+		const batch = order.slice(start, start + Math.min(batchSize, budget.remaining()));
+		if (batch.length === 0) break;
+		const evaluation = await evaluator.evaluateTraced({
+			candidate,
+			batch,
+			split: "train",
+			phase: "seed",
+			candidateId: null,
+			iteration: 0
+		});
+		if (evaluation === null) break;
+		attempted += batch.length;
+		for (let index = 0; index < batch.length; index += 1) {
+			const score = evaluation.scores[index];
+			if (!(minScore === void 0 ? score > 0 : score >= minScore) || demos.length >= maxDemos) continue;
+			demos.push({
+				input: batch[index],
+				output: evaluation.outputs[index],
+				score
+			});
+		}
+	}
+	return {
+		demos,
+		block: formatDemos(demos, renderDemo === void 0 ? {} : { render: renderDemo }),
+		metricCalls: budget.spent(),
+		attempted
+	};
+}
+/**
+* Render demos as the text a candidate component holds.
+*
+* Delimited rather than free-form so `parseDemos` can read them back: a demo
+* component is edited over the course of a run, and a block that cannot be
+* parsed can only be replaced wholesale, throwing away every example found
+* before it.
+*/
+function formatDemos(demos, options = {}) {
+	const { render = renderDefault } = options;
+	if (demos.length === 0) return "";
+	return demos.map((demo, index) => `${DEMO_OPEN}\n${render({
+		demo,
+		index
+	})}\n${DEMO_CLOSE}`).join("\n");
+}
+/**
+* Recover the demos from a formatted block, ignoring anything written around
+* them. Text a model rewrote and mangled yields the demos it left intact
+* rather than throwing: a malformed example is worth less than the rest of the
+* block, not more than it.
+*/
+function parseDemos(text) {
+	const demos = [];
+	for (const match of text.matchAll(DEMO_BLOCK)) {
+		const parts = (match[1] ?? "").match(DEMO_PARTS);
+		if (parts === null) continue;
+		const input = parseValue(parts[1] ?? "");
+		const output = parseValue(parts[2] ?? "");
+		demos.push({
+			input,
+			output
+		});
+	}
+	return demos;
+}
+function renderDefault(args) {
+	const { demo } = args;
+	return [
+		"<input>",
+		serialize(demo.input),
+		"</input>",
+		"<output>",
+		serialize(demo.output),
+		"</output>"
+	].join("\n");
+}
+/** Strings stay as they are; anything else is shown as JSON. */
+function serialize(value) {
+	if (typeof value === "string") return value;
+	try {
+		return JSON.stringify(value, null, 2) ?? String(value);
+	} catch {
+		return String(value);
+	}
+}
+function parseValue(text) {
+	try {
+		return JSON.parse(text);
+	} catch {
+		return text;
+	}
+}
+//#endregion
+export { formatDemos as n, parseDemos as r, bootstrapDemos as t };