textopt 0.0.0

package/dist/testing.cjs

@@ -0,0 +1,155 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_reflection = require("./reflection-DRfbk6hu.cjs");
+//#region src/testing.ts
+const KEYWORD_EXAMPLES = [
+	{
+		question: "How do I reset a device?",
+		required: ["hold", "ten seconds"]
+	},
+	{
+		question: "How do I contact support?",
+		required: ["ticket", "portal"]
+	},
+	{
+		question: "What is the refund window?",
+		required: ["thirty days"]
+	},
+	{
+		question: "How do I upgrade a plan?",
+		required: ["billing", "prorated"]
+	}
+];
+/** Terms the keyword metric rewards, interleaved with terms it ignores. */
+const SAMPLING_POOL = [
+	"hold",
+	"sprocket",
+	"ten seconds",
+	"lorem",
+	"ticket",
+	"widget",
+	"portal",
+	"colour",
+	"thirty days",
+	"ipsum",
+	"billing",
+	"gizmo",
+	"prorated",
+	"flange"
+];
+function createKeywordAdapter() {
+	return {
+		evaluate: ({ batch, candidate }) => {
+			const answer = Object.values(candidate).join(" ");
+			const trajectories = [];
+			const scores = [];
+			const feedback = [];
+			for (const example of batch) {
+				const missing = example.required.filter((term) => !answer.toLowerCase().includes(term.toLowerCase()));
+				const score = (example.required.length - missing.length) / example.required.length;
+				scores.push(score);
+				feedback.push(missing.length === 0 ? "All required terms present." : `Missing required terms: ${missing.join(", ")}`);
+				trajectories.push({
+					question: example.question,
+					instruction: answer,
+					answer,
+					missing
+				});
+			}
+			return {
+				outputs: trajectories.map((trajectory) => trajectory.answer),
+				scores,
+				feedback,
+				trajectories
+			};
+		},
+		makeReflectiveDataset: ({ batch, evaluation, componentsToUpdate }) => {
+			const records = batch.map((example, index) => ({
+				inputs: { question: example.question },
+				generatedOutputs: evaluation.outputs[index] ?? "",
+				feedback: evaluation.feedback?.[index] ?? "",
+				score: evaluation.scores[index]
+			}));
+			return Object.fromEntries(componentsToUpdate.map((component) => [component, records]));
+		}
+	};
+}
+/**
+* A deterministic stand-in for a reflection model: it reads the feedback in the
+* prompt and folds the missing terms into the current instruction.
+*/
+function createKeywordReflector() {
+	return async ({ prompt }) => {
+		const current = extractCurrentInstruction(prompt);
+		return `\`\`\`\n${[current, ...extractMissingTerms(prompt).filter((term) => !current.toLowerCase().includes(term.toLowerCase()))].filter(Boolean).join(" ")}\n\`\`\``;
+	};
+}
+/**
+* A stand-in for a model asked to rewrite text it has been told nothing about.
+* It appends one term per call, cycling a pool in which only every other entry
+* is useful — which is what blind proposal actually is: a draw from a space
+* where some samples happen to help.
+*
+* Feedback-driven reflectors read the prompt; this one deliberately does not,
+* so a search using it cannot benefit from evidence even if it is offered.
+*/
+function createSamplingReflector(args = {}) {
+	const { pool = SAMPLING_POOL } = args;
+	let cursor = 0;
+	return async ({ prompt }) => {
+		const term = pool[cursor % pool.length];
+		cursor += 1;
+		return `\`\`\`\n${[extractCurrentInstruction(prompt), term].filter(Boolean).join(" ")}\n\`\`\``;
+	};
+}
+/**
+* A stand-in for a model that reads a score history and climbs it: it takes
+* the highest-scoring attempt it is shown, keeps it, and extends it by one
+* term. Unlike `createSamplingReflector` it depends on the prompt carrying
+* scores, so a search that shows it none makes no progress with it.
+*/
+function createHillClimbingReflector(args = {}) {
+	const { pool = SAMPLING_POOL } = args;
+	let cursor = 0;
+	return async ({ prompt }) => {
+		const best = extractBestAttempt(prompt);
+		const term = pool[cursor % pool.length];
+		cursor += 1;
+		return `\`\`\`\n${[best, term].filter(Boolean).join(" ")}\n\`\`\``;
+	};
+}
+/** A reflection model that always proposes something strictly worse. */
+function createDegradingReflector() {
+	return async () => "```\nno useful information\n```";
+}
+function extractCurrentInstruction(prompt) {
+	return prompt.match(/<current_instruction>\n([\s\S]*?)\n<\/current_instruction>/)?.[1]?.trim() ?? "";
+}
+/** The instruction beside the highest score in a scored-attempt prompt. */
+function extractBestAttempt(prompt) {
+	let bestScore = Number.NEGATIVE_INFINITY;
+	let best = "";
+	for (const match of prompt.matchAll(/score:\s*([\d.]+)[\s\S]*?<instruction>\n?([\s\S]*?)\n?<\/instruction>/g)) {
+		const score = Number(match[1]);
+		if (score > bestScore) {
+			bestScore = score;
+			best = (match[2] ?? "").trim();
+		}
+	}
+	return best;
+}
+function extractMissingTerms(prompt) {
+	const terms = /* @__PURE__ */ new Set();
+	for (const match of prompt.matchAll(/Missing required terms: ([^"\\\n]+)/g)) for (const term of (match[1] ?? "").split(",")) {
+		const trimmed = term.trim();
+		if (trimmed.length > 0) terms.add(trimmed);
+	}
+	return [...terms];
+}
+//#endregion
+exports.KEYWORD_EXAMPLES = KEYWORD_EXAMPLES;
+exports.buildReflectionPrompt = require_reflection.buildReflectionPrompt;
+exports.createDegradingReflector = createDegradingReflector;
+exports.createHillClimbingReflector = createHillClimbingReflector;
+exports.createKeywordAdapter = createKeywordAdapter;
+exports.createKeywordReflector = createKeywordReflector;
+exports.createSamplingReflector = createSamplingReflector;

package/dist/testing.d.cts

@@ -0,0 +1,53 @@
+import { u as TextModel } from "./types-CWv4IQFF.cjs";
+import { i as buildReflectionPrompt, m as GepaAdapter } from "./reflection-CQToe-5B.cjs";
+//#region src/testing.d.ts
+/**
+ * A deterministic, LLM-free system under optimization.
+ *
+ * It exists so the engine, and any adapter you write, can be exercised end to
+ * end in milliseconds: the "model" answers with the candidate text itself and
+ * the metric rewards covering the required terms. Optimization therefore has a
+ * real gradient to climb without a single network call.
+ */
+interface KeywordExample {
+  question: string;
+  required: string[];
+}
+interface KeywordTrajectory {
+  question: string;
+  instruction: string;
+  answer: string;
+  missing: string[];
+}
+declare const KEYWORD_EXAMPLES: KeywordExample[];
+declare function createKeywordAdapter(): GepaAdapter<KeywordExample, KeywordTrajectory, string>;
+/**
+ * A deterministic stand-in for a reflection model: it reads the feedback in the
+ * prompt and folds the missing terms into the current instruction.
+ */
+declare function createKeywordReflector(): TextModel;
+/**
+ * A stand-in for a model asked to rewrite text it has been told nothing about.
+ * It appends one term per call, cycling a pool in which only every other entry
+ * is useful — which is what blind proposal actually is: a draw from a space
+ * where some samples happen to help.
+ *
+ * Feedback-driven reflectors read the prompt; this one deliberately does not,
+ * so a search using it cannot benefit from evidence even if it is offered.
+ */
+declare function createSamplingReflector(args?: {
+  pool?: readonly string[];
+}): TextModel;
+/**
+ * A stand-in for a model that reads a score history and climbs it: it takes
+ * the highest-scoring attempt it is shown, keeps it, and extends it by one
+ * term. Unlike `createSamplingReflector` it depends on the prompt carrying
+ * scores, so a search that shows it none makes no progress with it.
+ */
+declare function createHillClimbingReflector(args?: {
+  pool?: readonly string[];
+}): TextModel;
+/** A reflection model that always proposes something strictly worse. */
+declare function createDegradingReflector(): TextModel;
+//#endregion
+export { KEYWORD_EXAMPLES, KeywordExample, KeywordTrajectory, buildReflectionPrompt, createDegradingReflector, createHillClimbingReflector, createKeywordAdapter, createKeywordReflector, createSamplingReflector };

package/dist/testing.d.mts

@@ -0,0 +1,53 @@
+import { u as TextModel } from "./types-CWv4IQFF.mjs";
+import { i as buildReflectionPrompt, m as GepaAdapter } from "./reflection-Cr_upzU0.mjs";
+//#region src/testing.d.ts
+/**
+ * A deterministic, LLM-free system under optimization.
+ *
+ * It exists so the engine, and any adapter you write, can be exercised end to
+ * end in milliseconds: the "model" answers with the candidate text itself and
+ * the metric rewards covering the required terms. Optimization therefore has a
+ * real gradient to climb without a single network call.
+ */
+interface KeywordExample {
+  question: string;
+  required: string[];
+}
+interface KeywordTrajectory {
+  question: string;
+  instruction: string;
+  answer: string;
+  missing: string[];
+}
+declare const KEYWORD_EXAMPLES: KeywordExample[];
+declare function createKeywordAdapter(): GepaAdapter<KeywordExample, KeywordTrajectory, string>;
+/**
+ * A deterministic stand-in for a reflection model: it reads the feedback in the
+ * prompt and folds the missing terms into the current instruction.
+ */
+declare function createKeywordReflector(): TextModel;
+/**
+ * A stand-in for a model asked to rewrite text it has been told nothing about.
+ * It appends one term per call, cycling a pool in which only every other entry
+ * is useful — which is what blind proposal actually is: a draw from a space
+ * where some samples happen to help.
+ *
+ * Feedback-driven reflectors read the prompt; this one deliberately does not,
+ * so a search using it cannot benefit from evidence even if it is offered.
+ */
+declare function createSamplingReflector(args?: {
+  pool?: readonly string[];
+}): TextModel;
+/**
+ * A stand-in for a model that reads a score history and climbs it: it takes
+ * the highest-scoring attempt it is shown, keeps it, and extends it by one
+ * term. Unlike `createSamplingReflector` it depends on the prompt carrying
+ * scores, so a search that shows it none makes no progress with it.
+ */
+declare function createHillClimbingReflector(args?: {
+  pool?: readonly string[];
+}): TextModel;
+/** A reflection model that always proposes something strictly worse. */
+declare function createDegradingReflector(): TextModel;
+//#endregion
+export { KEYWORD_EXAMPLES, KeywordExample, KeywordTrajectory, buildReflectionPrompt, createDegradingReflector, createHillClimbingReflector, createKeywordAdapter, createKeywordReflector, createSamplingReflector };

package/dist/testing.mjs

@@ -0,0 +1,148 @@
+import { n as buildReflectionPrompt } from "./reflection-mwMhrjs_.mjs";
+//#region src/testing.ts
+const KEYWORD_EXAMPLES = [
+	{
+		question: "How do I reset a device?",
+		required: ["hold", "ten seconds"]
+	},
+	{
+		question: "How do I contact support?",
+		required: ["ticket", "portal"]
+	},
+	{
+		question: "What is the refund window?",
+		required: ["thirty days"]
+	},
+	{
+		question: "How do I upgrade a plan?",
+		required: ["billing", "prorated"]
+	}
+];
+/** Terms the keyword metric rewards, interleaved with terms it ignores. */
+const SAMPLING_POOL = [
+	"hold",
+	"sprocket",
+	"ten seconds",
+	"lorem",
+	"ticket",
+	"widget",
+	"portal",
+	"colour",
+	"thirty days",
+	"ipsum",
+	"billing",
+	"gizmo",
+	"prorated",
+	"flange"
+];
+function createKeywordAdapter() {
+	return {
+		evaluate: ({ batch, candidate }) => {
+			const answer = Object.values(candidate).join(" ");
+			const trajectories = [];
+			const scores = [];
+			const feedback = [];
+			for (const example of batch) {
+				const missing = example.required.filter((term) => !answer.toLowerCase().includes(term.toLowerCase()));
+				const score = (example.required.length - missing.length) / example.required.length;
+				scores.push(score);
+				feedback.push(missing.length === 0 ? "All required terms present." : `Missing required terms: ${missing.join(", ")}`);
+				trajectories.push({
+					question: example.question,
+					instruction: answer,
+					answer,
+					missing
+				});
+			}
+			return {
+				outputs: trajectories.map((trajectory) => trajectory.answer),
+				scores,
+				feedback,
+				trajectories
+			};
+		},
+		makeReflectiveDataset: ({ batch, evaluation, componentsToUpdate }) => {
+			const records = batch.map((example, index) => ({
+				inputs: { question: example.question },
+				generatedOutputs: evaluation.outputs[index] ?? "",
+				feedback: evaluation.feedback?.[index] ?? "",
+				score: evaluation.scores[index]
+			}));
+			return Object.fromEntries(componentsToUpdate.map((component) => [component, records]));
+		}
+	};
+}
+/**
+* A deterministic stand-in for a reflection model: it reads the feedback in the
+* prompt and folds the missing terms into the current instruction.
+*/
+function createKeywordReflector() {
+	return async ({ prompt }) => {
+		const current = extractCurrentInstruction(prompt);
+		return `\`\`\`\n${[current, ...extractMissingTerms(prompt).filter((term) => !current.toLowerCase().includes(term.toLowerCase()))].filter(Boolean).join(" ")}\n\`\`\``;
+	};
+}
+/**
+* A stand-in for a model asked to rewrite text it has been told nothing about.
+* It appends one term per call, cycling a pool in which only every other entry
+* is useful — which is what blind proposal actually is: a draw from a space
+* where some samples happen to help.
+*
+* Feedback-driven reflectors read the prompt; this one deliberately does not,
+* so a search using it cannot benefit from evidence even if it is offered.
+*/
+function createSamplingReflector(args = {}) {
+	const { pool = SAMPLING_POOL } = args;
+	let cursor = 0;
+	return async ({ prompt }) => {
+		const term = pool[cursor % pool.length];
+		cursor += 1;
+		return `\`\`\`\n${[extractCurrentInstruction(prompt), term].filter(Boolean).join(" ")}\n\`\`\``;
+	};
+}
+/**
+* A stand-in for a model that reads a score history and climbs it: it takes
+* the highest-scoring attempt it is shown, keeps it, and extends it by one
+* term. Unlike `createSamplingReflector` it depends on the prompt carrying
+* scores, so a search that shows it none makes no progress with it.
+*/
+function createHillClimbingReflector(args = {}) {
+	const { pool = SAMPLING_POOL } = args;
+	let cursor = 0;
+	return async ({ prompt }) => {
+		const best = extractBestAttempt(prompt);
+		const term = pool[cursor % pool.length];
+		cursor += 1;
+		return `\`\`\`\n${[best, term].filter(Boolean).join(" ")}\n\`\`\``;
+	};
+}
+/** A reflection model that always proposes something strictly worse. */
+function createDegradingReflector() {
+	return async () => "```\nno useful information\n```";
+}
+function extractCurrentInstruction(prompt) {
+	return prompt.match(/<current_instruction>\n([\s\S]*?)\n<\/current_instruction>/)?.[1]?.trim() ?? "";
+}
+/** The instruction beside the highest score in a scored-attempt prompt. */
+function extractBestAttempt(prompt) {
+	let bestScore = Number.NEGATIVE_INFINITY;
+	let best = "";
+	for (const match of prompt.matchAll(/score:\s*([\d.]+)[\s\S]*?<instruction>\n?([\s\S]*?)\n?<\/instruction>/g)) {
+		const score = Number(match[1]);
+		if (score > bestScore) {
+			bestScore = score;
+			best = (match[2] ?? "").trim();
+		}
+	}
+	return best;
+}
+function extractMissingTerms(prompt) {
+	const terms = /* @__PURE__ */ new Set();
+	for (const match of prompt.matchAll(/Missing required terms: ([^"\\\n]+)/g)) for (const term of (match[1] ?? "").split(",")) {
+		const trimmed = term.trim();
+		if (trimmed.length > 0) terms.add(trimmed);
+	}
+	return [...terms];
+}
+//#endregion
+export { KEYWORD_EXAMPLES, buildReflectionPrompt, createDegradingReflector, createHillClimbingReflector, createKeywordAdapter, createKeywordReflector, createSamplingReflector };

package/dist/text--v4Ffbus.mjs

@@ -0,0 +1,21 @@
+//#region src/text.ts
+const LANGUAGE_TAG = /^[a-zA-Z0-9_+.-]*\n/;
+const DANGLING_OPEN_FENCE = /^\s*```\S*\n?/;
+const DANGLING_CLOSE_FENCE = /\n?```\s*$/;
+/**
+* Pull the proposed text out of the reflection model's response.
+*
+* Spans the *first* fence to the *last* one rather than matching blocks
+* individually: proposed instructions routinely contain their own fenced
+* examples, and per-block matching would silently return only the trailing
+* fragment. A response with a single fence was truncated mid-generation, so the
+* stray fence is stripped and the partial text kept.
+*/
+function parseProposedText(response) {
+	const start = response.indexOf("```");
+	const end = response.lastIndexOf("```");
+	if (start !== -1 && start !== end) return response.slice(start + 3, end).replace(LANGUAGE_TAG, "").trim();
+	return response.replace(DANGLING_OPEN_FENCE, "").replace(DANGLING_CLOSE_FENCE, "").trim();
+}
+//#endregion
+export { parseProposedText as t };

package/dist/text-CK_HB3su.cjs

@@ -0,0 +1,26 @@
+//#region src/text.ts
+const LANGUAGE_TAG = /^[a-zA-Z0-9_+.-]*\n/;
+const DANGLING_OPEN_FENCE = /^\s*```\S*\n?/;
+const DANGLING_CLOSE_FENCE = /\n?```\s*$/;
+/**
+* Pull the proposed text out of the reflection model's response.
+*
+* Spans the *first* fence to the *last* one rather than matching blocks
+* individually: proposed instructions routinely contain their own fenced
+* examples, and per-block matching would silently return only the trailing
+* fragment. A response with a single fence was truncated mid-generation, so the
+* stray fence is stripped and the partial text kept.
+*/
+function parseProposedText(response) {
+	const start = response.indexOf("```");
+	const end = response.lastIndexOf("```");
+	if (start !== -1 && start !== end) return response.slice(start + 3, end).replace(LANGUAGE_TAG, "").trim();
+	return response.replace(DANGLING_OPEN_FENCE, "").replace(DANGLING_CLOSE_FENCE, "").trim();
+}
+//#endregion
+Object.defineProperty(exports, "parseProposedText", {
+	enumerable: true,
+	get: function() {
+		return parseProposedText;
+	}
+});

package/dist/types-CWv4IQFF.d.cts

@@ -0,0 +1,129 @@
+//#region src/types.d.ts
+/**
+ * A candidate is a map of named text components to their current text. This is
+ * the unit of optimization — prompts, instructions, code, tool descriptions,
+ * anything expressible as a named string.
+ *
+ * `K` is the union of component names, inferred from the seed candidate, so a
+ * misspelled component is a compile error rather than a silent no-op.
+ */
+type Candidate<K extends string = string> = Record<K, string>;
+/**
+ * What one rollout consumed. Every field is optional because providers report
+ * different subsets, and a partial reading is still worth more than none.
+ */
+interface RolloutUsage {
+  inputTokens?: number;
+  outputTokens?: number;
+  /** Defaults to the sum of the two token counts when they are reported. */
+  totalTokens?: number;
+  costUsd?: number;
+}
+/** Usage summed over a run, alongside the rollouts that produced it. */
+interface UsageTotals {
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
+  costUsd: number;
+  /** Fresh rollouts counted here. Cached instances buy nothing. */
+  rollouts: number;
+}
+/**
+ * Result of running a candidate over a batch of data instances.
+ *
+ * `scores` is the load-bearing field: one number per instance, higher is
+ * better. `feedback` is a per-instance textual diagnosis of what went wrong,
+ * which a reflective optimizer reads to write a better candidate.
+ */
+interface EvaluationBatch<Trajectory = unknown, Output = unknown> {
+  outputs: Output[];
+  scores: number[];
+  /**
+   * What each rollout consumed. Rollout counts are the budget, but they are a
+   * poor proxy for spend: reflective search grows the text it optimizes, so
+   * the same rollout costs more late in a run than early in it.
+   */
+  usage?: RolloutUsage[];
+  feedback?: string[];
+  trajectories?: Trajectory[];
+  objectiveScores?: Record<string, number>[];
+  /**
+   * Per-instance: true when the score reflects an infrastructure failure
+   * rather than the candidate's behaviour. Transient scores are never written
+   * to the evaluation cache.
+   */
+  transient?: boolean[];
+}
+/**
+ * What a per-instance scorer returns. Shared by every adapter so scorers are
+ * portable between them — a Braintrust scorer works in a LangChain run.
+ */
+interface ScoreResult {
+  score: number;
+  feedback?: string;
+  objectiveScores?: Record<string, number>;
+  /** What this rollout consumed, when the caller can see it. */
+  usage?: RolloutUsage;
+  /**
+   * Marks a score produced by an infrastructure failure — a rate limit, a
+   * network blip, a provider 5xx — rather than by the candidate. Without this
+   * the engine cannot tell such a zero from a genuine one, and would cache it
+   * permanently against the candidate.
+   */
+  transient?: boolean;
+}
+interface EvaluateArgs<Datum, K extends string = string> {
+  batch: readonly Datum[];
+  candidate: Candidate<K>;
+  captureTraces: boolean;
+  /**
+   * Where this batch sits in the run. Forward it to whatever tracing the
+   * system under optimization already has — without it a run is thousands of
+   * indistinguishable rollouts, and no trace can be tied back to the iteration
+   * whose score moved.
+   */
+  run: EvaluationContext;
+  signal?: AbortSignal;
+}
+/**
+ * Identifies one evaluation within a run. `candidateId` is null while the
+ * candidate is still a proposal being screened on a minibatch: it has no
+ * record, and inventing an id for it would collide with the one it gets if it
+ * is accepted.
+ */
+interface EvaluationContext {
+  iteration: number;
+  phase: EvaluationPhase;
+  split: EvaluationSplit;
+  candidateId: number | null;
+}
+type EvaluationPhase = "seed" | "minibatch" | "validation" | "test";
+/**
+ * Which dataset an instance id was drawn from. Each split numbers its ids
+ * independently, so the same id can name three different instances; the cache
+ * key has to keep them apart.
+ */
+type EvaluationSplit = "train" | "val" | "test";
+/**
+ * The single integration seam between an optimizer and a system under
+ * optimization. Everything framework-specific — LangChain, the AI SDK,
+ * Braintrust — lives in an implementation of this interface.
+ */
+interface Adapter<Datum, Trajectory = unknown, Output = unknown, K extends string = string> {
+  evaluate(args: EvaluateArgs<Datum, K>): Promise<EvaluationBatch<Trajectory, Output>> | EvaluationBatch<Trajectory, Output>;
+}
+/** Provider-agnostic text model: text in, text out. */
+type TextModel = (args: {
+  prompt: string;
+  signal?: AbortSignal;
+}) => Promise<string>;
+/**
+ * The component names of a candidate, as the union they were inferred from.
+ *
+ * `Object.keys` widens a closed key union back to `string`. This is the one
+ * place that narrowing happens, so every other caller stays assertion-free.
+ * Accepts a partial so it also names the components of a component patch.
+ */
+declare function componentNames<K extends string>(candidate: Partial<Candidate<K>>): K[];
+//#endregion
+export { EvaluationContext as a, RolloutUsage as c, UsageTotals as d, componentNames as f, EvaluationBatch as i, ScoreResult as l, Candidate as n, EvaluationPhase as o, EvaluateArgs as r, EvaluationSplit as s, Adapter as t, TextModel as u };