textopt 0.0.0 → 0.2.0

package/dist/demos-B9BJiNKz.cjs

@@ -1,143 +0,0 @@
-const require_evaluation = require("./evaluation-OZOp6TB7.cjs");
-//#region src/demos.ts
-const DEMO_OPEN = "<demo>";
-const DEMO_CLOSE = "</demo>";
-const DEMO_BLOCK = /<demo>\s*([\s\S]*?)\s*<\/demo>/g;
-const DEMO_PARTS = /<input>\s*([\s\S]*?)\s*<\/input>\s*<output>\s*([\s\S]*?)\s*<\/output>/;
-const DEFAULT_MAX_DEMOS = 4;
-/**
-* Harvest demonstrations by running a candidate over the training set and keeping
-* the rollouts the metric rewarded.
-*
-* The cheapest signal in the whole library: a rollout that scored well is
-* already paid for, and turning it into a few-shot block costs one pass over
-* the data rather than a search. Instruction search and demonstrations pull on
-* different parts of a model's behaviour — instructions on what to do,
-* examples on what the output should look like — so a seed carrying both
-* starts somewhere neither reaches alone.
-*/
-async function bootstrapDemos(args) {
-	const { adapter, candidate, trainingSet, minScore, maxDemos = DEFAULT_MAX_DEMOS, batchSize = maxDemos, maxMetricCalls = trainingSet.length, rng, renderDemo, signal } = args;
-	if (trainingSet.length === 0) throw new Error("bootstrapDemos requires a non-empty trainingSet");
-	const budget = require_evaluation.createBudget({ maxMetricCalls });
-	const evaluator = require_evaluation.createEvaluator({
-		adapter,
-		budget,
-		...signal === void 0 ? {} : { signal }
-	});
-	const order = rng === void 0 ? [...trainingSet] : rng.shuffle(trainingSet);
-	const demos = [];
-	let attempted = 0;
-	for (let start = 0; start < order.length; start += batchSize) {
-		if (demos.length >= maxDemos || signal?.aborted) break;
-		const batch = order.slice(start, start + Math.min(batchSize, budget.remaining()));
-		if (batch.length === 0) break;
-		const evaluation = await evaluator.evaluateTraced({
-			candidate,
-			batch,
-			split: "train",
-			phase: "seed",
-			candidateId: null,
-			iteration: 0
-		});
-		if (evaluation === null) break;
-		attempted += batch.length;
-		for (let index = 0; index < batch.length; index += 1) {
-			const score = evaluation.scores[index];
-			if (!(minScore === void 0 ? score > 0 : score >= minScore) || demos.length >= maxDemos) continue;
-			demos.push({
-				input: batch[index],
-				output: evaluation.outputs[index],
-				score
-			});
-		}
-	}
-	return {
-		demos,
-		block: formatDemos(demos, renderDemo === void 0 ? {} : { render: renderDemo }),
-		metricCalls: budget.spent(),
-		attempted
-	};
-}
-/**
-* Render demos as the text a candidate component holds.
-*
-* Delimited rather than free-form so `parseDemos` can read them back: a demo
-* component is edited over the course of a run, and a block that cannot be
-* parsed can only be replaced wholesale, throwing away every example found
-* before it.
-*/
-function formatDemos(demos, options = {}) {
-	const { render = renderDefault } = options;
-	if (demos.length === 0) return "";
-	return demos.map((demo, index) => `${DEMO_OPEN}\n${render({
-		demo,
-		index
-	})}\n${DEMO_CLOSE}`).join("\n");
-}
-/**
-* Recover the demos from a formatted block, ignoring anything written around
-* them. Text a model rewrote and mangled yields the demos it left intact
-* rather than throwing: a malformed example is worth less than the rest of the
-* block, not more than it.
-*/
-function parseDemos(text) {
-	const demos = [];
-	for (const match of text.matchAll(DEMO_BLOCK)) {
-		const parts = (match[1] ?? "").match(DEMO_PARTS);
-		if (parts === null) continue;
-		const input = parseValue(parts[1] ?? "");
-		const output = parseValue(parts[2] ?? "");
-		demos.push({
-			input,
-			output
-		});
-	}
-	return demos;
-}
-function renderDefault(args) {
-	const { demo } = args;
-	return [
-		"<input>",
-		serialize(demo.input),
-		"</input>",
-		"<output>",
-		serialize(demo.output),
-		"</output>"
-	].join("\n");
-}
-/** Strings stay as they are; anything else is shown as JSON. */
-function serialize(value) {
-	if (typeof value === "string") return value;
-	try {
-		return JSON.stringify(value, null, 2) ?? String(value);
-	} catch {
-		return String(value);
-	}
-}
-function parseValue(text) {
-	try {
-		return JSON.parse(text);
-	} catch {
-		return text;
-	}
-}
-//#endregion
-Object.defineProperty(exports, "bootstrapDemos", {
-	enumerable: true,
-	get: function() {
-		return bootstrapDemos;
-	}
-});
-Object.defineProperty(exports, "formatDemos", {
-	enumerable: true,
-	get: function() {
-		return formatDemos;
-	}
-});
-Object.defineProperty(exports, "parseDemos", {
-	enumerable: true,
-	get: function() {
-		return parseDemos;
-	}
-});

package/dist/demos-Degx6UmP.mjs

@@ -1,126 +0,0 @@
-import { p as createBudget, r as createEvaluator } from "./evaluation-BV0nSZVx.mjs";
-//#region src/demos.ts
-const DEMO_OPEN = "<demo>";
-const DEMO_CLOSE = "</demo>";
-const DEMO_BLOCK = /<demo>\s*([\s\S]*?)\s*<\/demo>/g;
-const DEMO_PARTS = /<input>\s*([\s\S]*?)\s*<\/input>\s*<output>\s*([\s\S]*?)\s*<\/output>/;
-const DEFAULT_MAX_DEMOS = 4;
-/**
-* Harvest demonstrations by running a candidate over the training set and keeping
-* the rollouts the metric rewarded.
-*
-* The cheapest signal in the whole library: a rollout that scored well is
-* already paid for, and turning it into a few-shot block costs one pass over
-* the data rather than a search. Instruction search and demonstrations pull on
-* different parts of a model's behaviour — instructions on what to do,
-* examples on what the output should look like — so a seed carrying both
-* starts somewhere neither reaches alone.
-*/
-async function bootstrapDemos(args) {
-	const { adapter, candidate, trainingSet, minScore, maxDemos = DEFAULT_MAX_DEMOS, batchSize = maxDemos, maxMetricCalls = trainingSet.length, rng, renderDemo, signal } = args;
-	if (trainingSet.length === 0) throw new Error("bootstrapDemos requires a non-empty trainingSet");
-	const budget = createBudget({ maxMetricCalls });
-	const evaluator = createEvaluator({
-		adapter,
-		budget,
-		...signal === void 0 ? {} : { signal }
-	});
-	const order = rng === void 0 ? [...trainingSet] : rng.shuffle(trainingSet);
-	const demos = [];
-	let attempted = 0;
-	for (let start = 0; start < order.length; start += batchSize) {
-		if (demos.length >= maxDemos || signal?.aborted) break;
-		const batch = order.slice(start, start + Math.min(batchSize, budget.remaining()));
-		if (batch.length === 0) break;
-		const evaluation = await evaluator.evaluateTraced({
-			candidate,
-			batch,
-			split: "train",
-			phase: "seed",
-			candidateId: null,
-			iteration: 0
-		});
-		if (evaluation === null) break;
-		attempted += batch.length;
-		for (let index = 0; index < batch.length; index += 1) {
-			const score = evaluation.scores[index];
-			if (!(minScore === void 0 ? score > 0 : score >= minScore) || demos.length >= maxDemos) continue;
-			demos.push({
-				input: batch[index],
-				output: evaluation.outputs[index],
-				score
-			});
-		}
-	}
-	return {
-		demos,
-		block: formatDemos(demos, renderDemo === void 0 ? {} : { render: renderDemo }),
-		metricCalls: budget.spent(),
-		attempted
-	};
-}
-/**
-* Render demos as the text a candidate component holds.
-*
-* Delimited rather than free-form so `parseDemos` can read them back: a demo
-* component is edited over the course of a run, and a block that cannot be
-* parsed can only be replaced wholesale, throwing away every example found
-* before it.
-*/
-function formatDemos(demos, options = {}) {
-	const { render = renderDefault } = options;
-	if (demos.length === 0) return "";
-	return demos.map((demo, index) => `${DEMO_OPEN}\n${render({
-		demo,
-		index
-	})}\n${DEMO_CLOSE}`).join("\n");
-}
-/**
-* Recover the demos from a formatted block, ignoring anything written around
-* them. Text a model rewrote and mangled yields the demos it left intact
-* rather than throwing: a malformed example is worth less than the rest of the
-* block, not more than it.
-*/
-function parseDemos(text) {
-	const demos = [];
-	for (const match of text.matchAll(DEMO_BLOCK)) {
-		const parts = (match[1] ?? "").match(DEMO_PARTS);
-		if (parts === null) continue;
-		const input = parseValue(parts[1] ?? "");
-		const output = parseValue(parts[2] ?? "");
-		demos.push({
-			input,
-			output
-		});
-	}
-	return demos;
-}
-function renderDefault(args) {
-	const { demo } = args;
-	return [
-		"<input>",
-		serialize(demo.input),
-		"</input>",
-		"<output>",
-		serialize(demo.output),
-		"</output>"
-	].join("\n");
-}
-/** Strings stay as they are; anything else is shown as JSON. */
-function serialize(value) {
-	if (typeof value === "string") return value;
-	try {
-		return JSON.stringify(value, null, 2) ?? String(value);
-	} catch {
-		return String(value);
-	}
-}
-function parseValue(text) {
-	try {
-		return JSON.parse(text);
-	} catch {
-		return text;
-	}
-}
-//#endregion
-export { formatDemos as n, parseDemos as r, bootstrapDemos as t };

package/dist/math-DhrDmpFS.mjs

@@ -1,78 +0,0 @@
-//#region src/math.ts
-function argmax(values) {
-	let bestIndex = 0;
-	let bestValue = Number.NEGATIVE_INFINITY;
-	for (let index = 0; index < values.length; index += 1) {
-		const value = values[index];
-		if (value > bestValue) {
-			bestValue = value;
-			bestIndex = index;
-		}
-	}
-	return bestIndex;
-}
-/** Mean over the values that exist; unscored instances are not zeros. */
-function mean(values) {
-	let total = 0;
-	let count = 0;
-	for (const value of values) if (value !== void 0) {
-		total += value;
-		count += 1;
-	}
-	return count === 0 ? 0 : total / count;
-}
-function sum(values) {
-	let total = 0;
-	for (const value of values) total += value;
-	return total;
-}
-/**
-* One-sided p-value from a paired sign-flip (permutation) test: how often a
-* total this large arises when each difference is equally likely to have come
-* out the other way. Exact by enumeration for small samples, normal-approximated
-* past `maxExact` where 2^n stops being cheap.
-*
-* Paired and distribution-free, which is what a comparison of two runs over the
-* same instances needs — the scores are not independent draws and are not
-* normal, so a t-test on them is answering a different question.
-*/
-function signFlipPValue(args) {
-	const { differences, observed, maxExact } = args;
-	const moved = differences.filter((difference) => difference !== 0);
-	if (moved.length === 0) return 1;
-	if (moved.length > maxExact) return normalTailProbability({
-		differences: moved,
-		observed
-	});
-	const assignments = 2 ** moved.length;
-	let atLeastAsExtreme = 0;
-	for (let mask = 0; mask < assignments; mask += 1) {
-		let total = 0;
-		for (let index = 0; index < moved.length; index += 1) {
-			const sign = mask >> index & 1 ? -1 : 1;
-			total += sign * moved[index];
-		}
-		if (total >= observed) atLeastAsExtreme += 1;
-	}
-	return atLeastAsExtreme / assignments;
-}
-/**
-* The same tail probability from a normal approximation, for batches too large
-* to enumerate. Under the sign-flip null each difference has mean zero and
-* variance equal to its square, so the total's variance is their sum.
-*/
-function normalTailProbability(args) {
-	const { differences, observed } = args;
-	const variance = differences.reduce((total, difference) => total + difference * difference, 0);
-	if (variance === 0) return observed > 0 ? 0 : 1;
-	return 1 - standardNormalCdf(observed / Math.sqrt(variance));
-}
-/** Abramowitz and Stegun 7.1.26, which is accurate to about 1e-7. */
-function standardNormalCdf(value) {
-	const sign = value < 0 ? -1 : 1;
-	const scaled = Math.abs(value) / Math.SQRT2;
-	const t = 1 / (1 + .3275911 * scaled);
-	return .5 * (1 + sign * (1 - t * (.254829592 + t * (-.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429)))) * Math.exp(-scaled * scaled)));
-}
-//#endregion
-export { sum as i, mean as n, signFlipPValue as r, argmax as t };

package/dist/types-CWv4IQFF.d.cts

@@ -1,129 +0,0 @@
-//#region src/types.d.ts
-/**
- * A candidate is a map of named text components to their current text. This is
- * the unit of optimization — prompts, instructions, code, tool descriptions,
- * anything expressible as a named string.
- *
- * `K` is the union of component names, inferred from the seed candidate, so a
- * misspelled component is a compile error rather than a silent no-op.
- */
-type Candidate<K extends string = string> = Record<K, string>;
-/**
- * What one rollout consumed. Every field is optional because providers report
- * different subsets, and a partial reading is still worth more than none.
- */
-interface RolloutUsage {
-  inputTokens?: number;
-  outputTokens?: number;
-  /** Defaults to the sum of the two token counts when they are reported. */
-  totalTokens?: number;
-  costUsd?: number;
-}
-/** Usage summed over a run, alongside the rollouts that produced it. */
-interface UsageTotals {
-  inputTokens: number;
-  outputTokens: number;
-  totalTokens: number;
-  costUsd: number;
-  /** Fresh rollouts counted here. Cached instances buy nothing. */
-  rollouts: number;
-}
-/**
- * Result of running a candidate over a batch of data instances.
- *
- * `scores` is the load-bearing field: one number per instance, higher is
- * better. `feedback` is a per-instance textual diagnosis of what went wrong,
- * which a reflective optimizer reads to write a better candidate.
- */
-interface EvaluationBatch<Trajectory = unknown, Output = unknown> {
-  outputs: Output[];
-  scores: number[];
-  /**
-   * What each rollout consumed. Rollout counts are the budget, but they are a
-   * poor proxy for spend: reflective search grows the text it optimizes, so
-   * the same rollout costs more late in a run than early in it.
-   */
-  usage?: RolloutUsage[];
-  feedback?: string[];
-  trajectories?: Trajectory[];
-  objectiveScores?: Record<string, number>[];
-  /**
-   * Per-instance: true when the score reflects an infrastructure failure
-   * rather than the candidate's behaviour. Transient scores are never written
-   * to the evaluation cache.
-   */
-  transient?: boolean[];
-}
-/**
- * What a per-instance scorer returns. Shared by every adapter so scorers are
- * portable between them — a Braintrust scorer works in a LangChain run.
- */
-interface ScoreResult {
-  score: number;
-  feedback?: string;
-  objectiveScores?: Record<string, number>;
-  /** What this rollout consumed, when the caller can see it. */
-  usage?: RolloutUsage;
-  /**
-   * Marks a score produced by an infrastructure failure — a rate limit, a
-   * network blip, a provider 5xx — rather than by the candidate. Without this
-   * the engine cannot tell such a zero from a genuine one, and would cache it
-   * permanently against the candidate.
-   */
-  transient?: boolean;
-}
-interface EvaluateArgs<Datum, K extends string = string> {
-  batch: readonly Datum[];
-  candidate: Candidate<K>;
-  captureTraces: boolean;
-  /**
-   * Where this batch sits in the run. Forward it to whatever tracing the
-   * system under optimization already has — without it a run is thousands of
-   * indistinguishable rollouts, and no trace can be tied back to the iteration
-   * whose score moved.
-   */
-  run: EvaluationContext;
-  signal?: AbortSignal;
-}
-/**
- * Identifies one evaluation within a run. `candidateId` is null while the
- * candidate is still a proposal being screened on a minibatch: it has no
- * record, and inventing an id for it would collide with the one it gets if it
- * is accepted.
- */
-interface EvaluationContext {
-  iteration: number;
-  phase: EvaluationPhase;
-  split: EvaluationSplit;
-  candidateId: number | null;
-}
-type EvaluationPhase = "seed" | "minibatch" | "validation" | "test";
-/**
- * Which dataset an instance id was drawn from. Each split numbers its ids
- * independently, so the same id can name three different instances; the cache
- * key has to keep them apart.
- */
-type EvaluationSplit = "train" | "val" | "test";
-/**
- * The single integration seam between an optimizer and a system under
- * optimization. Everything framework-specific — LangChain, the AI SDK,
- * Braintrust — lives in an implementation of this interface.
- */
-interface Adapter<Datum, Trajectory = unknown, Output = unknown, K extends string = string> {
-  evaluate(args: EvaluateArgs<Datum, K>): Promise<EvaluationBatch<Trajectory, Output>> | EvaluationBatch<Trajectory, Output>;
-}
-/** Provider-agnostic text model: text in, text out. */
-type TextModel = (args: {
-  prompt: string;
-  signal?: AbortSignal;
-}) => Promise<string>;
-/**
- * The component names of a candidate, as the union they were inferred from.
- *
- * `Object.keys` widens a closed key union back to `string`. This is the one
- * place that narrowing happens, so every other caller stays assertion-free.
- * Accepts a partial so it also names the components of a component patch.
- */
-declare function componentNames<K extends string>(candidate: Partial<Candidate<K>>): K[];
-//#endregion
-export { EvaluationContext as a, RolloutUsage as c, UsageTotals as d, componentNames as f, EvaluationBatch as i, ScoreResult as l, Candidate as n, EvaluationPhase as o, EvaluateArgs as r, EvaluationSplit as s, Adapter as t, TextModel as u };

package/dist/types-CWv4IQFF.d.mts

@@ -1,129 +0,0 @@
-//#region src/types.d.ts
-/**
- * A candidate is a map of named text components to their current text. This is
- * the unit of optimization — prompts, instructions, code, tool descriptions,
- * anything expressible as a named string.
- *
- * `K` is the union of component names, inferred from the seed candidate, so a
- * misspelled component is a compile error rather than a silent no-op.
- */
-type Candidate<K extends string = string> = Record<K, string>;
-/**
- * What one rollout consumed. Every field is optional because providers report
- * different subsets, and a partial reading is still worth more than none.
- */
-interface RolloutUsage {
-  inputTokens?: number;
-  outputTokens?: number;
-  /** Defaults to the sum of the two token counts when they are reported. */
-  totalTokens?: number;
-  costUsd?: number;
-}
-/** Usage summed over a run, alongside the rollouts that produced it. */
-interface UsageTotals {
-  inputTokens: number;
-  outputTokens: number;
-  totalTokens: number;
-  costUsd: number;
-  /** Fresh rollouts counted here. Cached instances buy nothing. */
-  rollouts: number;
-}
-/**
- * Result of running a candidate over a batch of data instances.
- *
- * `scores` is the load-bearing field: one number per instance, higher is
- * better. `feedback` is a per-instance textual diagnosis of what went wrong,
- * which a reflective optimizer reads to write a better candidate.
- */
-interface EvaluationBatch<Trajectory = unknown, Output = unknown> {
-  outputs: Output[];
-  scores: number[];
-  /**
-   * What each rollout consumed. Rollout counts are the budget, but they are a
-   * poor proxy for spend: reflective search grows the text it optimizes, so
-   * the same rollout costs more late in a run than early in it.
-   */
-  usage?: RolloutUsage[];
-  feedback?: string[];
-  trajectories?: Trajectory[];
-  objectiveScores?: Record<string, number>[];
-  /**
-   * Per-instance: true when the score reflects an infrastructure failure
-   * rather than the candidate's behaviour. Transient scores are never written
-   * to the evaluation cache.
-   */
-  transient?: boolean[];
-}
-/**
- * What a per-instance scorer returns. Shared by every adapter so scorers are
- * portable between them — a Braintrust scorer works in a LangChain run.
- */
-interface ScoreResult {
-  score: number;
-  feedback?: string;
-  objectiveScores?: Record<string, number>;
-  /** What this rollout consumed, when the caller can see it. */
-  usage?: RolloutUsage;
-  /**
-   * Marks a score produced by an infrastructure failure — a rate limit, a
-   * network blip, a provider 5xx — rather than by the candidate. Without this
-   * the engine cannot tell such a zero from a genuine one, and would cache it
-   * permanently against the candidate.
-   */
-  transient?: boolean;
-}
-interface EvaluateArgs<Datum, K extends string = string> {
-  batch: readonly Datum[];
-  candidate: Candidate<K>;
-  captureTraces: boolean;
-  /**
-   * Where this batch sits in the run. Forward it to whatever tracing the
-   * system under optimization already has — without it a run is thousands of
-   * indistinguishable rollouts, and no trace can be tied back to the iteration
-   * whose score moved.
-   */
-  run: EvaluationContext;
-  signal?: AbortSignal;
-}
-/**
- * Identifies one evaluation within a run. `candidateId` is null while the
- * candidate is still a proposal being screened on a minibatch: it has no
- * record, and inventing an id for it would collide with the one it gets if it
- * is accepted.
- */
-interface EvaluationContext {
-  iteration: number;
-  phase: EvaluationPhase;
-  split: EvaluationSplit;
-  candidateId: number | null;
-}
-type EvaluationPhase = "seed" | "minibatch" | "validation" | "test";
-/**
- * Which dataset an instance id was drawn from. Each split numbers its ids
- * independently, so the same id can name three different instances; the cache
- * key has to keep them apart.
- */
-type EvaluationSplit = "train" | "val" | "test";
-/**
- * The single integration seam between an optimizer and a system under
- * optimization. Everything framework-specific — LangChain, the AI SDK,
- * Braintrust — lives in an implementation of this interface.
- */
-interface Adapter<Datum, Trajectory = unknown, Output = unknown, K extends string = string> {
-  evaluate(args: EvaluateArgs<Datum, K>): Promise<EvaluationBatch<Trajectory, Output>> | EvaluationBatch<Trajectory, Output>;
-}
-/** Provider-agnostic text model: text in, text out. */
-type TextModel = (args: {
-  prompt: string;
-  signal?: AbortSignal;
-}) => Promise<string>;
-/**
- * The component names of a candidate, as the union they were inferred from.
- *
- * `Object.keys` widens a closed key union back to `string`. This is the one
- * place that narrowing happens, so every other caller stays assertion-free.
- * Accepts a partial so it also names the components of a component patch.
- */
-declare function componentNames<K extends string>(candidate: Partial<Candidate<K>>): K[];
-//#endregion
-export { EvaluationContext as a, RolloutUsage as c, UsageTotals as d, componentNames as f, EvaluationBatch as i, ScoreResult as l, Candidate as n, EvaluationPhase as o, EvaluateArgs as r, EvaluationSplit as s, Adapter as t, TextModel as u };