textopt 0.0.0 → 0.1.0

package/dist/reporting-DQbAohc9.d.mts

@@ -0,0 +1,240 @@
+//#region src/types.d.ts
+/**
+ * A candidate is a map of named text components to their current text. This is
+ * the unit of optimization — prompts, instructions, code, tool descriptions,
+ * anything expressible as a named string.
+ *
+ * `K` is the union of component names, inferred from the seed candidate, so a
+ * misspelled component is a compile error rather than a silent no-op.
+ */
+type Candidate<K extends string = string> = Record<K, string>;
+/**
+ * What one rollout consumed. Every field is optional because providers report
+ * different subsets, and a partial reading is still worth more than none.
+ */
+interface RolloutUsage {
+  inputTokens?: number;
+  outputTokens?: number;
+  /** Defaults to the sum of the two token counts when they are reported. */
+  totalTokens?: number;
+  costUsd?: number;
+}
+/** Usage summed over a run, alongside the rollouts that produced it. */
+interface UsageTotals {
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
+  costUsd: number;
+  /** Fresh rollouts counted here. Cached instances buy nothing. */
+  rollouts: number;
+}
+/**
+ * Result of running a candidate over a batch of data instances.
+ *
+ * `scores` is the load-bearing field: one number per instance, higher is
+ * better. `feedback` is a per-instance textual diagnosis of what went wrong,
+ * which a reflective optimizer reads to write a better candidate.
+ */
+interface EvaluationBatch<Trajectory = unknown, Output = unknown> {
+  outputs: Output[];
+  scores: number[];
+  /**
+   * What each rollout consumed. Rollout counts are the budget, but they are a
+   * poor proxy for spend: reflective search grows the text it optimizes, so
+   * the same rollout costs more late in a run than early in it.
+   */
+  usage?: RolloutUsage[];
+  feedback?: string[];
+  trajectories?: Trajectory[];
+  objectiveScores?: Record<string, number>[];
+  /**
+   * Per-instance: true when the score reflects an infrastructure failure
+   * rather than the candidate's behaviour. Transient scores are never written
+   * to the evaluation cache.
+   */
+  transient?: boolean[];
+}
+/**
+ * What a per-instance scorer returns. Shared by every adapter so scorers are
+ * portable between them — a Braintrust scorer works in a LangChain run.
+ */
+interface ScoreResult {
+  score: number;
+  feedback?: string;
+  objectiveScores?: Record<string, number>;
+  /** What this rollout consumed, when the caller can see it. */
+  usage?: RolloutUsage;
+  /**
+   * Marks a score produced by an infrastructure failure — a rate limit, a
+   * network blip, a provider 5xx — rather than by the candidate. Without this
+   * the engine cannot tell such a zero from a genuine one, and would cache it
+   * permanently against the candidate.
+   */
+  transient?: boolean;
+}
+interface EvaluateArgs<Datum, K extends string = string> {
+  batch: readonly Datum[];
+  candidate: Candidate<K>;
+  captureTraces: boolean;
+  /**
+   * Where this batch sits in the run. Forward it to whatever tracing the
+   * system under optimization already has — without it a run is thousands of
+   * indistinguishable rollouts, and no trace can be tied back to the iteration
+   * whose score moved.
+   */
+  run: EvaluationContext;
+  signal?: AbortSignal;
+}
+/**
+ * Identifies one evaluation within a run. `candidateId` is null while the
+ * candidate is still a proposal being screened on a minibatch: it has no
+ * record, and inventing an id for it would collide with the one it gets if it
+ * is accepted.
+ */
+interface EvaluationContext {
+  iteration: number;
+  phase: EvaluationPhase;
+  split: EvaluationSplit;
+  candidateId: number | null;
+}
+type EvaluationPhase = "seed" | "minibatch" | "validation" | "test";
+/**
+ * Which dataset an instance id was drawn from. Each split numbers its ids
+ * independently, so the same id can name three different instances; the cache
+ * key has to keep them apart.
+ */
+type EvaluationSplit = "train" | "val" | "test";
+/**
+ * The single integration seam between an optimizer and a system under
+ * optimization. Everything framework-specific — LangChain, the AI SDK,
+ * Braintrust — lives in an implementation of this interface.
+ */
+interface Adapter<Datum, Trajectory = unknown, Output = unknown, K extends string = string> {
+  evaluate(args: EvaluateArgs<Datum, K>): Promise<EvaluationBatch<Trajectory, Output>> | EvaluationBatch<Trajectory, Output>;
+}
+/** Provider-agnostic text model: text in, text out. */
+type TextModel = (args: {
+  prompt: string;
+  signal?: AbortSignal;
+}) => Promise<string>;
+/**
+ * The component names of a candidate, as the union they were inferred from.
+ *
+ * `Object.keys` widens a closed key union back to `string`. This is the one
+ * place that narrowing happens, so every other caller stays assertion-free.
+ * Accepts a partial so it also names the components of a component patch.
+ */
+declare function componentNames<K extends string>(candidate: Partial<Candidate<K>>): K[];
+//#endregion
+//#region src/reporting.d.ts
+/**
+ * Where a run's progress goes, for any optimizer. Observability only:
+ * persisting a run so it can be resumed is `onCheckpoint`, which is durability
+ * and a separate concern.
+ *
+ * Generic over the event union rather than one union covering every optimizer:
+ * a search emits what it actually has, and a reporter written against one
+ * optimizer still type-checks against the events it reads.
+ */
+/**
+ * The least an optimizer's event satisfies. A reporter typed against this
+ * accepts every optimizer's union, because a literal tag is assignable to
+ * `string` and the parameter position is contravariant.
+ */
+interface OptimizerEvent {
+  type: string;
+}
+interface Reporter<Event> {
+  /**
+   * Called on the search's hot path, synchronously. A reporter that ships
+   * anywhere over a network buffers here and uploads in `flush`, or it charges
+   * every iteration for its latency.
+   *
+   * A reporter that throws is warned about and skipped: observability never
+   * fails a run.
+   */
+  onEvent?: (event: Event) => void;
+  /** Awaited once as the run ends, including when it ends by throwing. */
+  flush?: () => Promise<void>;
+}
+/**
+ * The payload every optimizer's `candidateAccepted` carries, so one reporter
+ * can read an acceptance without knowing which search produced it. Each
+ * optimizer intersects its own fields onto this — GEPA its lineage, MIPRO its
+ * menu choices — the way the event unions already intersect `EvaluationEvent`.
+ *
+ * Emitted only when the incumbent moves and a full validation sweep measured
+ * it. An optimizer that accepts on a minibatch reports the acceptance in its
+ * own event and emits this one once the sweep that confirms it lands, so
+ * `instanceScores` never means "a subset, and you work out which".
+ */
+interface CandidateAccepted<K extends string = string> {
+  /** Identifies the candidate within the run. */
+  candidateId: number;
+  /** The text that scored, so a move is readable next to the edit. */
+  candidate: Candidate<K>;
+  /** Mean over the validation set, which is what selection is decided on. */
+  aggregateScore: number;
+  /**
+   * Per-instance scores, aligned with the validation set. `undefined` marks an
+   * instance an infrastructure failure left unmeasured — unknown, not zero.
+   *
+   * Handed out by reference rather than copied: a run emits this once per
+   * accepted candidate, and copying a validation-set-sized array that often to
+   * guard against a listener that writes to it costs every run to protect a
+   * listener that should not exist.
+   */
+  instanceScores: readonly (number | undefined)[];
+  /** Aligned with `instanceScores`. Present only under `trackBestOutputs`. */
+  outputs?: readonly unknown[];
+}
+/**
+ * The payload every optimizer's `finish` carries. `reason` stays per-optimizer
+ * because the stop reasons genuinely differ — only GEPA can exhaust a
+ * reflection budget.
+ */
+interface RunFinished {
+  /** The winner, named the way `CandidateAccepted.candidateId` names it. */
+  bestCandidateId: number;
+  bestScore: number;
+  metricCalls: number;
+  /** The winner's held-out score, when a testSet was given. */
+  testScore?: number;
+  /**
+   * The winner's per-instance held-out scores, aligned with the testSet and
+   * present whenever `testScore` is. `undefined` marks an instance an
+   * infrastructure failure left unmeasured, which is what `testScore` averages
+   * over too.
+   *
+   * The mean is the number selection never saw; this is where the gap below
+   * `bestScore` came from.
+   */
+  testInstanceScores?: readonly (number | undefined)[];
+  /** Aligned with `testInstanceScores`. Only under `trackBestOutputs`. */
+  testOutputs?: readonly unknown[];
+}
+/**
+ * The two events every optimizer emits with a payload a reporter can read
+ * without knowing which search produced it. A cross-optimizer reporter takes
+ * this as its event type: it is a supertype of every optimizer's own union, so
+ * one reporter drops into any optimizer's `reporters` array.
+ */
+type ReportableEvent<K extends string = string> = ({
+  type: "candidateAccepted";
+} & CandidateAccepted<K>) | ({
+  type: "finish";
+} & RunFinished);
+/**
+ * Narrows an event off any optimizer's union to an acceptance. The tag is
+ * enough: every optimizer's `candidateAccepted` intersects `CandidateAccepted`,
+ * so carrying the payload is a compile-time obligation rather than a hope.
+ */
+declare function isCandidateAccepted<K extends string = string>(event: OptimizerEvent): event is {
+  type: "candidateAccepted";
+} & CandidateAccepted<K>;
+/** Narrows an event off any optimizer's union to the end of the run. */
+declare function isRunFinished(event: OptimizerEvent): event is {
+  type: "finish";
+} & RunFinished;
+//#endregion
+export { TextModel as _, RunFinished as a, Adapter as c, EvaluationBatch as d, EvaluationContext as f, ScoreResult as g, RolloutUsage as h, Reporter as i, Candidate as l, EvaluationSplit as m, OptimizerEvent as n, isCandidateAccepted as o, EvaluationPhase as p, ReportableEvent as r, isRunFinished as s, CandidateAccepted as t, EvaluateArgs as u, UsageTotals as v, componentNames as y };

package/dist/{evaluation-BV0nSZVx.mjs → reporting-DY-DC4HG.mjs}

@@ -30,6 +30,8 @@ function createBudget(args) {
 }
 //#endregion
 //#region src/cache.ts
+/** What `stableHash` returns for any value that serializes to `{}`. */
+const EMPTY_OBJECT_HASH = stableHash({});
 /**
 * The loop re-evaluates unchanged candidates against the same validation
 * instances constantly (every accepted child inherits most of its parent's
@@ -65,6 +67,20 @@ function stableHash(value) {
 	}
 	return `${hash32(serialized, 2166136261)}${hash32(serialized, 16777619)}`;
 }
+/**
+* Names one data instance for the evaluation cache: a content hash, so the same
+* row is the same instance wherever it appears in a run.
+*
+* Falls back to the row's position when the datum carries nothing the hash can
+* read. A Map, a Set and a class instance holding its state privately all
+* serialize to `{}`, and an id two rows share serves each of them the score the
+* other measured. Position is a weaker id — it is only stable while the data is
+* — but it is one instance per row, which is what the cache needs to be sound.
+*/
+function defaultInstanceId(args) {
+	const hash = stableHash(args.datum);
+	return hash === "" || hash === EMPTY_OBJECT_HASH ? String(args.index) : hash;
+}
 function createMemoryCache(args = {}) {
 	const { maxEntries = 1e5, entries: initial = [] } = args;
 	const entries = new Map(initial);
@@ -172,25 +188,39 @@ const DEFAULT_RETRY = {
 */
 var BudgetExhausted = class extends Error {};
 function createEvaluator(args) {
-	const { adapter, budget, cache, trackOutputs = false, onEvaluation, signal, cacheHits: initialCacheHits = 0, retry, cacheNamespace } = args;
+	const { adapter, budget, cache, trackOutputs = false, onEvaluation, signal, cacheHits: initialCacheHits = 0, usage: initialUsage, retry, cacheNamespace } = args;
 	const { attempts: retryAttempts, delayMs: retryDelayMs } = {
 		...DEFAULT_RETRY,
 		...retry
 	};
+	if (initialUsage !== void 0) assertUsage({
+		reading: initialUsage,
+		source: "Checkpoint carries"
+	});
 	let cacheHits = initialCacheHits;
 	let unchargedCalls = 0;
 	const usage = {
+		inputTokens: 0,
+		outputTokens: 0,
+		totalTokens: 0,
+		costUsd: 0,
+		rollouts: 0,
+		...initialUsage
+	};
+	/** What `charge: false` bought, kept out of the totals a ceiling reads. */
+	const unchargedUsage = {
 		inputTokens: 0,
 		outputTokens: 0,
 		totalTokens: 0,
 		costUsd: 0,
 		rollouts: 0
 	};
-	/** Folds one adapter call's reported usage into the run's totals. */
+	/** Folds one adapter call's reported usage into the totals that bought it. */
 	function recordUsage(args) {
-		usage.rollouts += args.rollouts;
+		const totals = args.charge ? usage : unchargedUsage;
+		totals.rollouts += args.rollouts;
 		for (const rollout of args.evaluation.usage ?? []) addUsage({
-			totals: usage,
+			totals,
 			rollout
 		});
 	}
@@ -252,7 +282,8 @@ function createEvaluator(args) {
 			});
 			recordUsage({
 				evaluation,
-				rollouts: rows.length
+				rollouts: rows.length,
+				charge
 			});
 			return evaluation;
 		} catch (err) {
@@ -395,6 +426,18 @@ function createEvaluator(args) {
 		cacheHits: () => cacheHits,
 		unchargedCalls: () => unchargedCalls,
 		usage: () => ({ ...usage }),
+		unchargedUsage: () => ({ ...unchargedUsage }),
+		absorbUsage: (spent) => {
+			assertUsage({
+				reading: spent,
+				source: "Absorbed usage carries"
+			});
+			usage.inputTokens += spent.inputTokens;
+			usage.outputTokens += spent.outputTokens;
+			usage.totalTokens += spent.totalTokens;
+			usage.costUsd += spent.costUsd;
+			usage.rollouts += spent.rollouts;
+		},
 		entries: () => cache?.entries?.(),
 		restore: (entries) => {
 			for (const [key, cached] of entries) cache?.set(key, cached);
@@ -471,11 +514,29 @@ function requireMeasuredMean(args) {
 function addUsage(args) {
 	const { totals, rollout } = args;
 	const { inputTokens = 0, outputTokens = 0, costUsd = 0 } = rollout;
+	assertUsage({
+		reading: rollout,
+		source: "Adapter reported"
+	});
 	totals.inputTokens += inputTokens;
 	totals.outputTokens += outputTokens;
 	totals.totalTokens += rollout.totalTokens ?? inputTokens + outputTokens;
 	totals.costUsd += costUsd;
 }
+/**
+* Refuses a reading the totals cannot hold, where it enters rather than once it
+* has been absorbed. A NaN folded in makes every later `maxCostUsd` comparison
+* false, so the ceiling stops holding without saying so, and a value that is
+* not a number at all concatenates onto the totals instead of adding to them.
+* `RolloutUsage` binds TypeScript callers and nothing else.
+*/
+function assertUsage(args) {
+	const { reading, source } = args;
+	for (const [field, value] of Object.entries(reading)) {
+		if (value === void 0) continue;
+		if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new Error(`${source} ${field} as ${value}; usage must be a non-negative finite number`);
+	}
+}
 function transientIndices(evaluation) {
 	const { transient } = evaluation;
 	if (transient === void 0) return [];
@@ -518,4 +579,61 @@ function delay(milliseconds) {
 	});
 }
 //#endregion
-export { requireMeasuredMean as a, candidateFingerprint as c, createMemoryCache as d, stableHash as f, measuredMean as i, runFingerprint as l, costExhausted as n, createDeadline as o, createBudget as p, createEvaluator as r, assertResumable as s, BudgetExhausted as t, componentNames as u };
+//#region src/reporting.ts
+/**
+* Narrows an event off any optimizer's union to an acceptance. The tag is
+* enough: every optimizer's `candidateAccepted` intersects `CandidateAccepted`,
+* so carrying the payload is a compile-time obligation rather than a hope.
+*/
+function isCandidateAccepted(event) {
+	return event.type === "candidateAccepted";
+}
+/** Narrows an event off any optimizer's union to the end of the run. */
+function isRunFinished(event) {
+	return event.type === "finish";
+}
+/**
+* Fans one event out to every reporter, absorbing whatever they throw. A
+* reporter is an observer of the search, never a participant in it: a logging
+* endpoint that is down must not decide a run's outcome.
+*/
+function createEmitter(reporters) {
+	return function emit(event) {
+		for (const reporter of reporters) try {
+			reporter.onEvent?.(event);
+		} catch (err) {
+			console.warn("[textopt] reporter threw while handling an event", {
+				type: event.type,
+				err
+			});
+		}
+	};
+}
+/**
+* Gives every reporter its one chance to upload what it buffered. Called from
+* a `finally` rather than after the run: a reporter that buffers has the most
+* to say about a run that aborted or threw, and that is exactly the run that
+* never reaches its last line.
+*/
+async function flushReporters(reporters) {
+	await Promise.all(reporters.map(async (reporter) => {
+		try {
+			await reporter.flush?.();
+		} catch (err) {
+			console.warn("[textopt] reporter threw while flushing", { err });
+		}
+	}));
+}
+/**
+* A scored batch as the per-instance row a reporter should read: an instance
+* an infrastructure failure left unmeasured becomes `undefined` rather than
+* the zero the adapter reported for it.
+*
+* The same distinction `measuredMean` makes when it averages — a row of zeros
+* and a row of unknowns describe very different runs.
+*/
+function instanceRow(batch) {
+	return batch.scores.map((score, index) => batch.transient[index] === true ? void 0 : score);
+}
+//#endregion
+export { createMemoryCache as _, isRunFinished as a, createBudget as b, createEvaluator as c, createDeadline as d, assertResumable as f, candidateHash as g, componentNames as h, isCandidateAccepted as i, measuredMean as l, runFingerprint as m, flushReporters as n, BudgetExhausted as o, candidateFingerprint as p, instanceRow as r, costExhausted as s, createEmitter as t, requireMeasuredMean as u, defaultInstanceId as v, stableHash as y };