textopt 0.0.0

package/dist/index.cjs

@@ -0,0 +1,266 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_evaluation = require("./evaluation-OZOp6TB7.cjs");
+const require_concurrency = require("./concurrency-C-cFzWW2.cjs");
+const require_math = require("./math-COOofUyv.cjs");
+const require_demos = require("./demos-B9BJiNKz.cjs");
+const require_text = require("./text-CK_HB3su.cjs");
+//#region src/compare.ts
+const EXACT_LIMIT = 16;
+/**
+* Run several optimizers over the same seeds and report which one actually won.
+*
+* Two things make this worth a helper rather than a for-loop. The first is that
+* it ranks on `testScore` where a run reports one: the validation score is the
+* number the search selected against for its whole run, so an entrant that
+* overfits looks strongest on exactly the number it fitted. The second is that
+* a difference in means over a handful of seeds is usually noise, and the
+* paired sign-flip p-value against the winner is what says whether it is.
+*
+* Entrants are functions of a seed rather than optimizer instances, because the
+* seed is constructor config and every optimizer here is deterministic given
+* one — comparing two entrants at a single seed compares two anecdotes. It also
+* leaves the task where the caller builds it, which is the only place the
+* optimizer-specific parts of it (`reflect`, `cache`, `onEvent`) are known.
+*/
+async function compare(args) {
+	const { entrants, seeds, concurrency = 1 } = args;
+	const names = Object.keys(entrants);
+	if (names.length === 0) throw new Error("compare requires at least one entrant");
+	if (seeds.length === 0) throw new Error("compare requires at least one seed");
+	const grid = names.flatMap((entrant) => seeds.map((seed) => ({
+		entrant,
+		seed
+	})));
+	const runs = await require_concurrency.mapWithConcurrency({
+		items: grid,
+		limit: concurrency,
+		task: async ({ entrant, seed }) => {
+			const result = await entrants[entrant]({ seed });
+			return {
+				entrant,
+				seed,
+				score: result.testScore ?? result.bestScore,
+				bestScore: result.bestScore,
+				...result.testScore === void 0 ? {} : { testScore: result.testScore },
+				metricCalls: result.metricCalls,
+				costUsd: result.usage.costUsd,
+				stopReason: result.stopReason
+			};
+		}
+	});
+	const summaries = names.map((entrant) => summarize({
+		entrant,
+		runs: runs.filter((run) => run.entrant === entrant)
+	}));
+	const winner = summaries.reduce((best, summary) => summary.meanScore > best.meanScore ? summary : best);
+	return {
+		winner: winner.entrant,
+		summaries: summaries.map((summary) => summary.entrant === winner.entrant ? summary : {
+			...summary,
+			pValueVsWinner: margin({
+				winner: winner.entrant,
+				entrant: summary.entrant,
+				runs,
+				seeds
+			})
+		}),
+		runs
+	};
+}
+function summarize(args) {
+	const { entrant, runs } = args;
+	const scores = runs.map((run) => run.score);
+	return {
+		entrant,
+		runs: runs.length,
+		meanScore: mean(scores),
+		sdScore: standardDeviation(scores),
+		minScore: Math.min(...scores),
+		maxScore: Math.max(...scores),
+		meanMetricCalls: mean(runs.map((run) => run.metricCalls)),
+		meanCostUsd: mean(runs.map((run) => run.costUsd))
+	};
+}
+/**
+* Paired across seeds rather than pooled: the same seed puts both entrants on
+* the same sampling order, so the difference at a seed is a comparison and the
+* spread between seeds is not.
+*/
+function margin(args) {
+	const { winner, entrant, runs, seeds } = args;
+	const differences = seeds.map((seed) => {
+		return scoreOf({
+			runs,
+			entrant: winner,
+			seed
+		}) - scoreOf({
+			runs,
+			entrant,
+			seed
+		});
+	});
+	return require_math.signFlipPValue({
+		differences,
+		observed: differences.reduce((total, value) => total + value, 0),
+		maxExact: EXACT_LIMIT
+	});
+}
+function scoreOf(args) {
+	const { runs, entrant, seed } = args;
+	return runs.find((candidate) => candidate.entrant === entrant && candidate.seed === seed)?.score ?? 0;
+}
+function mean(values) {
+	return values.reduce((total, value) => total + value, 0) / values.length;
+}
+function standardDeviation(values) {
+	if (values.length < 2) return 0;
+	const average = mean(values);
+	const variance = values.reduce((total, value) => total + (value - average) ** 2, 0) / (values.length - 1);
+	return Math.sqrt(variance);
+}
+//#endregion
+//#region src/judge.ts
+const DEFAULT_SCALE = 5;
+const SCORE = /<score\s+name="([^"]+)"\s*>\s*([\d.]+)\s*<\/score>/g;
+const FEEDBACK = /<feedback>([\s\S]*?)<\/feedback>/;
+/**
+* A model-graded metric that returns written feedback alongside the score.
+*
+* The feedback is the point. A judge that returns only a number reduces a whole
+* paragraph of diagnosis to one scalar, and reflective search — the thing most
+* likely to be pointed at a task too open-ended to score by string match — runs
+* on exactly that diagnosis. So the prompt demands feedback, and demands it be
+* addressed to the instructions rather than to the graded output: "this answer
+* should have mentioned the refund window" tells a rewriting model nothing that
+* "the instruction never says to state the refund window" does not say better.
+*
+* Grades on a small integer scale and normalizes afterwards. Models discriminate
+* between 2 and 4 far more reliably than between 0.4 and 0.8, and the scale is
+* the caller's to widen once they have seen the judge bunch its answers.
+*/
+function createJudge(args) {
+	const { model, criteria, scale = DEFAULT_SCALE, renderInput = stringify, renderOutput = stringify, buildPrompt = buildJudgePrompt } = args;
+	if (criteria.length === 0) throw new Error("createJudge requires at least one criterion");
+	return async ({ input, output, expected, signal }) => {
+		return readVerdict({
+			response: await model({
+				prompt: buildPrompt({
+					input: renderInput(input),
+					output: renderOutput(output),
+					expected: expected === void 0 ? void 0 : renderOutput(expected),
+					criteria,
+					scale
+				}),
+				signal
+			}),
+			criteria,
+			scale
+		});
+	};
+}
+function buildJudgePrompt(args) {
+	const { input, output, expected, criteria, scale } = args;
+	return [
+		"You are grading one output of an automated system against the criteria below.",
+		"",
+		"<input>",
+		input,
+		"</input>",
+		"",
+		"<output>",
+		output,
+		"</output>",
+		...expected === void 0 ? [] : [
+			"",
+			"<expected_answer>",
+			expected,
+			"</expected_answer>"
+		],
+		"",
+		"<criteria>",
+		criteria.map((criterion) => `<${criterion.name}>${criterion.description}</${criterion.name}>`).join("\n"),
+		"</criteria>",
+		"",
+		`Grade each criterion from 0 to ${scale}, where ${scale} is a perfect answer.`,
+		"",
+		"Then write feedback. It is read by a program that rewrites the system's instructions, not by a person reviewing this output, so say what the instructions should tell the system to do differently. Feedback about this particular answer is of no use to it.",
+		"",
+		"Reply in exactly this format and nothing else:",
+		...criteria.map((criterion) => `<score name="${criterion.name}">…</score>`),
+		"<feedback>…</feedback>"
+	].join("\n");
+}
+/**
+* A judge that answered off-format graded nothing, so the result is unknown
+* rather than bad. Reporting it as transient is what keeps a formatting failure
+* out of the cache and off the candidate's record — the same treatment a rate
+* limit gets, for the same reason.
+*/
+function readVerdict(args) {
+	const { response, criteria, scale } = args;
+	const feedback = response.match(FEEDBACK)?.[1]?.trim() ?? "";
+	const graded = /* @__PURE__ */ new Map();
+	for (const match of response.matchAll(SCORE)) graded.set(match[1], Number(match[2]));
+	const objectiveScores = {};
+	for (const criterion of criteria) {
+		const grade = graded.get(criterion.name);
+		if (grade === void 0 || Number.isNaN(grade)) return {
+			score: 0,
+			feedback: `Judge did not grade "${criterion.name}".`,
+			transient: true
+		};
+		objectiveScores[criterion.name] = clamp(grade / scale);
+	}
+	const grades = Object.values(objectiveScores);
+	return {
+		score: grades.reduce((total, grade) => total + grade, 0) / grades.length,
+		feedback,
+		objectiveScores
+	};
+}
+function stringify(value) {
+	return typeof value === "string" ? value : JSON.stringify(value);
+}
+function clamp(value) {
+	return Math.min(1, Math.max(0, value));
+}
+//#endregion
+//#region src/usage.ts
+const TOKENS_PER_PRICED_UNIT = 1e6;
+/**
+* Costs a token reading, leaving it untouched when no price list is given.
+*
+* Prices belong to the caller rather than to this library: they change without
+* warning, differ per account, and a stale table baked in here would report
+* confident numbers that are quietly wrong.
+*/
+function priceUsage(args) {
+	const { usage, pricing } = args;
+	if (pricing === void 0) return usage;
+	return {
+		...usage,
+		costUsd: ((usage.inputTokens ?? 0) * pricing.inputPerMillionUsd + (usage.outputTokens ?? 0) * pricing.outputPerMillionUsd) / TOKENS_PER_PRICED_UNIT
+	};
+}
+//#endregion
+exports.BudgetExhausted = require_evaluation.BudgetExhausted;
+exports.assertResumable = require_evaluation.assertResumable;
+exports.bootstrapDemos = require_demos.bootstrapDemos;
+exports.buildJudgePrompt = buildJudgePrompt;
+exports.candidateFingerprint = require_evaluation.candidateFingerprint;
+exports.compare = compare;
+exports.componentNames = require_evaluation.componentNames;
+exports.costExhausted = require_evaluation.costExhausted;
+exports.createBudget = require_evaluation.createBudget;
+exports.createDeadline = require_evaluation.createDeadline;
+exports.createEvaluator = require_evaluation.createEvaluator;
+exports.createJudge = createJudge;
+exports.createMemoryCache = require_evaluation.createMemoryCache;
+exports.formatDemos = require_demos.formatDemos;
+exports.mapWithConcurrency = require_concurrency.mapWithConcurrency;
+exports.measuredMean = require_evaluation.measuredMean;
+exports.parseDemos = require_demos.parseDemos;
+exports.parseProposedText = require_text.parseProposedText;
+exports.priceUsage = priceUsage;
+exports.requireMeasuredMean = require_evaluation.requireMeasuredMean;
+exports.runFingerprint = require_evaluation.runFingerprint;

package/dist/index.d.cts

@@ -0,0 +1,221 @@
+import { a as EvaluationContext, c as RolloutUsage, d as UsageTotals, f as componentNames, i as EvaluationBatch, l as ScoreResult, n as Candidate, o as EvaluationPhase, r as EvaluateArgs, s as EvaluationSplit, t as Adapter, u as TextModel } from "./types-CWv4IQFF.cjs";
+import { n as EvaluationCache, r as createMemoryCache, t as CachedScore } from "./cache-CuSo0NJ8.cjs";
+import { t as Rng } from "./rng-BR5MOedA.cjs";
+import { a as formatDemos, i as bootstrapDemos, n as Demo, o as parseDemos, r as DemoRenderer, t as BootstrapResult } from "./demos-BTuzFNsp.cjs";
+import { a as EvaluateBatchArgs, c as Evaluator, d as costExhausted, f as createEvaluator, g as createBudget, h as Budget, i as BudgetExhausted, l as RetryPolicy, m as requireMeasuredMean, n as OptimizerResult, o as EvaluateTracedArgs, p as measuredMean, r as OptimizerTask, s as EvaluationEvent, t as Optimizer, u as ScoredBatch } from "./optimizer-B7SpRwl7.cjs";
+import { t as BatchSampler } from "./sampling-axOwfZf5.cjs";
+//#region src/checkpoint.d.ts
+/**
+ * Identifies the run a checkpoint came from: seed candidate, instance ids,
+ * seed, and cache namespace.
+ *
+ * Hashed rather than embedded, because it goes into every snapshot and is only
+ * ever compared for equality. The namespace is part of it because a snapshot
+ * carries cached scores, and resuming under a different one would replay
+ * measurements of a system the run is no longer running. The test set is
+ * deliberately absent: it never touches selection, so adding one to a resumed
+ * run changes nothing about what that run would have done.
+ */
+declare function runFingerprint(args: {
+  seedCandidate: Candidate;
+  trainingIds: readonly string[];
+  validationIds: readonly string[];
+  seed?: number;
+  cacheNamespace?: string;
+}): string;
+/**
+ * Refuses a checkpoint from a different run rather than silently scoring old
+ * candidates against new data — the failure mode that produces a plausible
+ * result nobody can reproduce.
+ */
+declare function assertResumable(args: {
+  fingerprint: string;
+  snapshot?: {
+    fingerprint: string;
+  };
+}): void;
+/** A candidate's identity: component names and their text, order-independent. */
+declare function candidateFingerprint<K extends string>(candidate: Candidate<K>): string;
+//#endregion
+//#region src/compare.d.ts
+interface ComparisonRun {
+  entrant: string;
+  seed: number;
+  /** What the entrants are ranked on: the held-out score when there is one. */
+  score: number;
+  bestScore: number;
+  testScore?: number;
+  metricCalls: number;
+  costUsd: number;
+  stopReason: string;
+}
+interface ComparisonSummary {
+  entrant: string;
+  runs: number;
+  meanScore: number;
+  sdScore: number;
+  minScore: number;
+  maxScore: number;
+  meanMetricCalls: number;
+  meanCostUsd: number;
+  /**
+   * How often the winner's margin over this entrant would arise if the two
+   * were equally good and each seed's outcome were a coin flip. Absent for the
+   * winner itself.
+   */
+  pValueVsWinner?: number;
+}
+interface Comparison {
+  /** Highest mean score. Read `pValueVsWinner` before believing it. */
+  winner: string;
+  summaries: ComparisonSummary[];
+  runs: ComparisonRun[];
+}
+/**
+ * Run several optimizers over the same seeds and report which one actually won.
+ *
+ * Two things make this worth a helper rather than a for-loop. The first is that
+ * it ranks on `testScore` where a run reports one: the validation score is the
+ * number the search selected against for its whole run, so an entrant that
+ * overfits looks strongest on exactly the number it fitted. The second is that
+ * a difference in means over a handful of seeds is usually noise, and the
+ * paired sign-flip p-value against the winner is what says whether it is.
+ *
+ * Entrants are functions of a seed rather than optimizer instances, because the
+ * seed is constructor config and every optimizer here is deterministic given
+ * one — comparing two entrants at a single seed compares two anecdotes. It also
+ * leaves the task where the caller builds it, which is the only place the
+ * optimizer-specific parts of it (`reflect`, `cache`, `onEvent`) are known.
+ */
+declare function compare<K extends string, Output = unknown>(args: {
+  entrants: Record<string, (args: {
+    seed: number;
+  }) => Promise<OptimizerResult<K, string, Output>>>;
+  seeds: readonly number[];
+  /** Runs in flight at once. Default 1. */
+  concurrency?: number;
+}): Promise<Comparison>;
+//#endregion
+//#region src/concurrency.d.ts
+/**
+ * Order-preserving concurrency-limited map. An optimization run's workload is
+ * entirely IO bound, so adapters use this to fan out a batch across a bounded
+ * number of in-flight model calls.
+ */
+declare function mapWithConcurrency<Item, Result>(args: {
+  items: readonly Item[];
+  limit: number;
+  task: (item: Item, index: number) => Promise<Result>;
+  /**
+   * Checked before each dispatch. Aborting stops the fan-out rather than
+   * letting the remaining batch items spend rollouts on a cancelled run.
+   */
+  signal?: AbortSignal;
+}): Promise<Result[]>;
+//#endregion
+//#region src/deadline.d.ts
+interface Deadline {
+  exceeded(): boolean;
+  /** Milliseconds left, `Infinity` when no limit was set. Never negative. */
+  remainingMs(): number;
+}
+/**
+ * A wall-clock limit on a run.
+ *
+ * Rollout and cost ceilings both bound what a run *spends*, and neither bounds
+ * how long it takes: a run waiting on a rate-limited provider can sit for an
+ * hour without spending a dollar. A deadline is what makes an optimizer safe to
+ * put behind a request timeout or a nightly job.
+ *
+ * Checked between evaluations, so a run overruns by at most the length of one.
+ * The clock is injectable because a deadline that can only be tested by waiting
+ * is a deadline nobody tests.
+ */
+declare function createDeadline(args: {
+  maxWallClockMs?: number;
+  now?: () => number;
+}): Deadline;
+//#endregion
+//#region src/judge.d.ts
+/** One thing the judge grades, and what a perfect answer looks like for it. */
+interface JudgeCriterion {
+  name: string;
+  description: string;
+}
+type JudgePromptBuilder = (args: {
+  input: string;
+  output: string;
+  expected?: string;
+  criteria: readonly JudgeCriterion[];
+  scale: number;
+}) => string;
+type Judge<Datum, Output> = (args: {
+  input: Datum;
+  output: Output;
+  /** The gold answer, when the caller has one. */
+  expected?: Output;
+  signal?: AbortSignal;
+}) => Promise<ScoreResult>;
+/**
+ * A model-graded metric that returns written feedback alongside the score.
+ *
+ * The feedback is the point. A judge that returns only a number reduces a whole
+ * paragraph of diagnosis to one scalar, and reflective search — the thing most
+ * likely to be pointed at a task too open-ended to score by string match — runs
+ * on exactly that diagnosis. So the prompt demands feedback, and demands it be
+ * addressed to the instructions rather than to the graded output: "this answer
+ * should have mentioned the refund window" tells a rewriting model nothing that
+ * "the instruction never says to state the refund window" does not say better.
+ *
+ * Grades on a small integer scale and normalizes afterwards. Models discriminate
+ * between 2 and 4 far more reliably than between 0.4 and 0.8, and the scale is
+ * the caller's to widen once they have seen the judge bunch its answers.
+ */
+declare function createJudge<Datum = string, Output = string>(args: {
+  model: TextModel;
+  criteria: readonly JudgeCriterion[];
+  /** Highest grade the judge may award per criterion. Default 5. */
+  scale?: number;
+  renderInput?: (input: Datum) => string;
+  renderOutput?: (output: Output) => string;
+  buildPrompt?: JudgePromptBuilder;
+}): Judge<Datum, Output>;
+declare function buildJudgePrompt(args: {
+  input: string;
+  output: string;
+  expected?: string;
+  criteria: readonly JudgeCriterion[];
+  scale: number;
+}): string;
+//#endregion
+//#region src/text.d.ts
+/**
+ * Pull the proposed text out of the reflection model's response.
+ *
+ * Spans the *first* fence to the *last* one rather than matching blocks
+ * individually: proposed instructions routinely contain their own fenced
+ * examples, and per-block matching would silently return only the trailing
+ * fragment. A response with a single fence was truncated mid-generation, so the
+ * stray fence is stripped and the partial text kept.
+ */
+declare function parseProposedText(response: string): string;
+//#endregion
+//#region src/usage.d.ts
+/** What a model charges, in the per-million-tokens units every vendor quotes. */
+interface TokenPricing {
+  inputPerMillionUsd: number;
+  outputPerMillionUsd: number;
+}
+/**
+ * Costs a token reading, leaving it untouched when no price list is given.
+ *
+ * Prices belong to the caller rather than to this library: they change without
+ * warning, differ per account, and a stale table baked in here would report
+ * confident numbers that are quietly wrong.
+ */
+declare function priceUsage(args: {
+  usage: RolloutUsage;
+  pricing?: TokenPricing;
+}): RolloutUsage;
+//#endregion
+export { type Adapter, type BatchSampler, type BootstrapResult, type Budget, BudgetExhausted, type CachedScore, type Candidate, type Comparison, type ComparisonRun, type ComparisonSummary, type Deadline, type Demo, type DemoRenderer, type EvaluateArgs, type EvaluateBatchArgs, type EvaluateTracedArgs, type EvaluationBatch, type EvaluationCache, type EvaluationContext, type EvaluationEvent, type EvaluationPhase, type EvaluationSplit, type Evaluator, type Judge, type JudgeCriterion, type JudgePromptBuilder, type Optimizer, type OptimizerResult, type OptimizerTask, type RetryPolicy, type Rng, type RolloutUsage, type ScoreResult, type ScoredBatch, type TextModel, type TokenPricing, type UsageTotals, assertResumable, bootstrapDemos, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, runFingerprint };