textopt 0.0.0 → 0.2.0

package/dist/reporting-bq007_2z.d.cts

@@ -0,0 +1,294 @@
+//#region src/types.d.ts
+/**
+ * A candidate is a map of named text components to their current text. This is
+ * the unit of optimization — prompts, instructions, code, tool descriptions,
+ * anything expressible as a named string.
+ *
+ * `K` is the union of component names, inferred from the seed candidate, so a
+ * misspelled component is a compile error rather than a silent no-op.
+ */
+type Candidate<K extends string = string> = Record<K, string>;
+/**
+ * What one rollout consumed. Every field is optional because providers report
+ * different subsets, and a partial reading is still worth more than none.
+ */
+interface RolloutUsage {
+  inputTokens?: number;
+  outputTokens?: number;
+  /** Defaults to the sum of the two token counts when they are reported. */
+  totalTokens?: number;
+  costUsd?: number;
+}
+/** Usage summed over a run, alongside the rollouts that produced it. */
+interface UsageTotals {
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
+  costUsd: number;
+  /** Fresh rollouts counted here. Cached instances buy nothing. */
+  rollouts: number;
+}
+/**
+ * Result of running a candidate over a batch of data instances.
+ *
+ * `scores` is the load-bearing field: one number per instance, higher is
+ * better. `feedback` is a per-instance textual diagnosis of what went wrong,
+ * which a reflective optimizer reads to write a better candidate.
+ */
+interface EvaluationBatch<Trajectory = unknown, Output = unknown> {
+  outputs: Output[];
+  scores: number[];
+  /**
+   * What each rollout consumed. Rollout counts are the budget, but they are a
+   * poor proxy for spend: reflective search grows the text it optimizes, so
+   * the same rollout costs more late in a run than early in it.
+   */
+  usage?: RolloutUsage[];
+  feedback?: string[];
+  trajectories?: Trajectory[];
+  objectiveScores?: Record<string, number>[];
+  /**
+   * Per-instance: true when the score reflects an infrastructure failure
+   * rather than the candidate's behaviour. Transient scores are never written
+   * to the evaluation cache.
+   */
+  transient?: boolean[];
+}
+/**
+ * What a per-instance scorer returns. Shared by every adapter so scorers are
+ * portable between them — a Braintrust scorer works in a LangChain run.
+ */
+interface ScoreResult {
+  score: number;
+  feedback?: string;
+  objectiveScores?: Record<string, number>;
+  /** What this rollout consumed, when the caller can see it. */
+  usage?: RolloutUsage;
+  /**
+   * Marks a score produced by an infrastructure failure — a rate limit, a
+   * network blip, a provider 5xx — rather than by the candidate. Without this
+   * the engine cannot tell such a zero from a genuine one, and would cache it
+   * permanently against the candidate.
+   */
+  transient?: boolean;
+}
+interface EvaluateArgs<Datum, K extends string = string> {
+  batch: readonly Datum[];
+  candidate: Candidate<K>;
+  captureTraces: boolean;
+  /**
+   * Where this batch sits in the run. Forward it to whatever tracing the
+   * system under optimization already has — without it a run is thousands of
+   * indistinguishable rollouts, and no trace can be tied back to the iteration
+   * whose score moved.
+   */
+  run: EvaluationContext;
+  signal?: AbortSignal;
+}
+/**
+ * Identifies one evaluation within a run. `candidateId` is null while the
+ * candidate is still a proposal being screened on a minibatch: it has no
+ * record, and inventing an id for it would collide with the one it gets if it
+ * is accepted.
+ */
+interface EvaluationContext {
+  iteration: number;
+  phase: EvaluationPhase;
+  split: EvaluationSplit;
+  candidateId: number | null;
+}
+type EvaluationPhase = "seed" | "minibatch" | "validation" | "test";
+/**
+ * Which dataset an instance id was drawn from. Each split numbers its ids
+ * independently, so the same id can name three different instances; the cache
+ * key has to keep them apart.
+ */
+type EvaluationSplit = "train" | "val" | "test";
+/**
+ * The single integration seam between an optimizer and a system under
+ * optimization. Everything framework-specific — LangChain, the AI SDK,
+ * Braintrust — lives in an implementation of this interface.
+ */
+interface Adapter<Datum, Trajectory = unknown, Output = unknown, K extends string = string> {
+  evaluate(args: EvaluateArgs<Datum, K>): Promise<EvaluationBatch<Trajectory, Output>> | EvaluationBatch<Trajectory, Output>;
+}
+/** Provider-agnostic text model: text in, text out. */
+type TextModel = (args: {
+  prompt: string;
+  signal?: AbortSignal;
+}) => Promise<string>;
+/**
+ * The component names of a candidate, as the union they were inferred from.
+ *
+ * `Object.keys` widens a closed key union back to `string`. This is the one
+ * place that narrowing happens, so every other caller stays assertion-free.
+ * Accepts a partial so it also names the components of a component patch.
+ */
+declare function componentNames<K extends string>(candidate: Partial<Candidate<K>>): K[];
+//#endregion
+//#region src/warnings.d.ts
+/**
+ * Something about a run that its own numbers cannot say.
+ *
+ * A search reports a score, a stop reason and what it spent, and every one of
+ * those reads the same whether the run measured what the caller thinks it did
+ * or not. These are the conditions under which a normal-looking result means
+ * less than it appears to: they never stop a run, and they are carried on the
+ * result and the `finish` event so a report can say so next to the number.
+ */
+interface RunWarning {
+  code: RunWarningCode;
+  message: string;
+}
+type RunWarningCode = "validationSetReusesTraining" | "seedScoreSaturated" | "seedScoreFloored";
+/**
+ * The validation set a run will actually select against, and whatever the
+ * choice costs it.
+ *
+ * Defaulting to the training set is the right default for a first run and the
+ * wrong number to report from one. It is worse than ordinary overfitting under
+ * reflective search: the reflection prompt asks the model to mine domain facts
+ * out of the traces it is shown, so those facts come out of the very instances
+ * that then select the candidate carrying them. `"reuseTraining"` is the same
+ * behaviour with the caller's name on it, and silences the warning.
+ */
+declare function resolveValidationSet<Datum>(args: {
+  validationSet: readonly Datum[] | "reuseTraining" | undefined;
+  trainingSet: readonly Datum[];
+}): {
+  validationSet: readonly Datum[];
+  warnings: RunWarning[];
+};
+/**
+ * What the seed's own validation row says about whether the run could have
+ * learned anything.
+ *
+ * A search ranks candidates by how they differ across instances, so a seed row
+ * with no spread leaves nothing to rank: at the ceiling every proposal is a tie
+ * the acceptance test resolves by noise, and at the floor no proposal has a
+ * partial improvement to build on. Both produce a run that spends its whole
+ * budget and reports a stop reason that looks like any other.
+ */
+declare function seedScoreWarnings(args: {
+  scores: readonly (number | undefined)[];
+  perfectScore: number;
+}): RunWarning[];
+//#endregion
+//#region src/reporting.d.ts
+/**
+ * Where a run's progress goes, for any optimizer. Observability only:
+ * persisting a run so it can be resumed is `onCheckpoint`, which is durability
+ * and a separate concern.
+ *
+ * Generic over the event union rather than one union covering every optimizer:
+ * a search emits what it actually has, and a reporter written against one
+ * optimizer still type-checks against the events it reads.
+ */
+/**
+ * The least an optimizer's event satisfies. A reporter typed against this
+ * accepts every optimizer's union, because a literal tag is assignable to
+ * `string` and the parameter position is contravariant.
+ */
+interface OptimizerEvent {
+  type: string;
+}
+interface Reporter<Event> {
+  /**
+   * Called on the search's hot path, synchronously. A reporter that ships
+   * anywhere over a network buffers here and uploads in `flush`, or it charges
+   * every iteration for its latency.
+   *
+   * A reporter that throws is warned about and skipped: observability never
+   * fails a run.
+   */
+  onEvent?: (event: Event) => void;
+  /** Awaited once as the run ends, including when it ends by throwing. */
+  flush?: () => Promise<void>;
+}
+/**
+ * The payload every optimizer's `candidateAccepted` carries, so one reporter
+ * can read an acceptance without knowing which search produced it. Each
+ * optimizer intersects its own fields onto this — GEPA its lineage, MIPRO its
+ * menu choices — the way the event unions already intersect `EvaluationEvent`.
+ *
+ * Emitted only when the incumbent moves and a full validation sweep measured
+ * it. An optimizer that accepts on a minibatch reports the acceptance in its
+ * own event and emits this one once the sweep that confirms it lands, so
+ * `instanceScores` never means "a subset, and you work out which".
+ */
+interface CandidateAccepted<K extends string = string> {
+  /** Identifies the candidate within the run. */
+  candidateId: number;
+  /** The text that scored, so a move is readable next to the edit. */
+  candidate: Candidate<K>;
+  /** Mean over the validation set, which is what selection is decided on. */
+  aggregateScore: number;
+  /**
+   * Per-instance scores, aligned with the validation set. `undefined` marks an
+   * instance an infrastructure failure left unmeasured — unknown, not zero.
+   *
+   * Handed out by reference rather than copied: a run emits this once per
+   * accepted candidate, and copying a validation-set-sized array that often to
+   * guard against a listener that writes to it costs every run to protect a
+   * listener that should not exist.
+   */
+  instanceScores: readonly (number | undefined)[];
+  /** Aligned with `instanceScores`. Present only under `trackBestOutputs`. */
+  outputs?: readonly unknown[];
+}
+/**
+ * The payload every optimizer's `finish` carries. `reason` stays per-optimizer
+ * because the stop reasons genuinely differ — only GEPA can exhaust a
+ * reflection budget.
+ */
+interface RunFinished {
+  /** The winner, named the way `CandidateAccepted.candidateId` names it. */
+  bestCandidateId: number;
+  bestScore: number;
+  metricCalls: number;
+  /** The winner's held-out score, when a testSet was given. */
+  testScore?: number;
+  /**
+   * The winner's per-instance held-out scores, aligned with the testSet and
+   * present whenever `testScore` is. `undefined` marks an instance an
+   * infrastructure failure left unmeasured, which is what `testScore` averages
+   * over too.
+   *
+   * The mean is the number selection never saw; this is where the gap below
+   * `bestScore` came from.
+   */
+  testInstanceScores?: readonly (number | undefined)[];
+  /** Aligned with `testInstanceScores`. Only under `trackBestOutputs`. */
+  testOutputs?: readonly unknown[];
+  /**
+   * What the run cannot say about itself from its own numbers. A reporter that
+   * writes the score somewhere permanent writes these beside it, or the record
+   * outlives the only place the caveat was ever stated.
+   */
+  warnings: readonly RunWarning[];
+}
+/**
+ * The two events every optimizer emits with a payload a reporter can read
+ * without knowing which search produced it. A cross-optimizer reporter takes
+ * this as its event type: it is a supertype of every optimizer's own union, so
+ * one reporter drops into any optimizer's `reporters` array.
+ */
+type ReportableEvent<K extends string = string> = ({
+  type: "candidateAccepted";
+} & CandidateAccepted<K>) | ({
+  type: "finish";
+} & RunFinished);
+/**
+ * Narrows an event off any optimizer's union to an acceptance. The tag is
+ * enough: every optimizer's `candidateAccepted` intersects `CandidateAccepted`,
+ * so carrying the payload is a compile-time obligation rather than a hope.
+ */
+declare function isCandidateAccepted<K extends string = string>(event: OptimizerEvent): event is {
+  type: "candidateAccepted";
+} & CandidateAccepted<K>;
+/** Narrows an event off any optimizer's union to the end of the run. */
+declare function isRunFinished(event: OptimizerEvent): event is {
+  type: "finish";
+} & RunFinished;
+//#endregion
+export { componentNames as C, UsageTotals as S, EvaluationPhase as _, RunFinished as a, ScoreResult as b, RunWarning as c, seedScoreWarnings as d, Adapter as f, EvaluationContext as g, EvaluationBatch as h, Reporter as i, RunWarningCode as l, EvaluateArgs as m, OptimizerEvent as n, isCandidateAccepted as o, Candidate as p, ReportableEvent as r, isRunFinished as s, CandidateAccepted as t, resolveValidationSet as u, EvaluationSplit as v, TextModel as x, RolloutUsage as y };

package/dist/reporting-bq007_2z.d.mts

@@ -0,0 +1,294 @@
+//#region src/types.d.ts
+/**
+ * A candidate is a map of named text components to their current text. This is
+ * the unit of optimization — prompts, instructions, code, tool descriptions,
+ * anything expressible as a named string.
+ *
+ * `K` is the union of component names, inferred from the seed candidate, so a
+ * misspelled component is a compile error rather than a silent no-op.
+ */
+type Candidate<K extends string = string> = Record<K, string>;
+/**
+ * What one rollout consumed. Every field is optional because providers report
+ * different subsets, and a partial reading is still worth more than none.
+ */
+interface RolloutUsage {
+  inputTokens?: number;
+  outputTokens?: number;
+  /** Defaults to the sum of the two token counts when they are reported. */
+  totalTokens?: number;
+  costUsd?: number;
+}
+/** Usage summed over a run, alongside the rollouts that produced it. */
+interface UsageTotals {
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
+  costUsd: number;
+  /** Fresh rollouts counted here. Cached instances buy nothing. */
+  rollouts: number;
+}
+/**
+ * Result of running a candidate over a batch of data instances.
+ *
+ * `scores` is the load-bearing field: one number per instance, higher is
+ * better. `feedback` is a per-instance textual diagnosis of what went wrong,
+ * which a reflective optimizer reads to write a better candidate.
+ */
+interface EvaluationBatch<Trajectory = unknown, Output = unknown> {
+  outputs: Output[];
+  scores: number[];
+  /**
+   * What each rollout consumed. Rollout counts are the budget, but they are a
+   * poor proxy for spend: reflective search grows the text it optimizes, so
+   * the same rollout costs more late in a run than early in it.
+   */
+  usage?: RolloutUsage[];
+  feedback?: string[];
+  trajectories?: Trajectory[];
+  objectiveScores?: Record<string, number>[];
+  /**
+   * Per-instance: true when the score reflects an infrastructure failure
+   * rather than the candidate's behaviour. Transient scores are never written
+   * to the evaluation cache.
+   */
+  transient?: boolean[];
+}
+/**
+ * What a per-instance scorer returns. Shared by every adapter so scorers are
+ * portable between them — a Braintrust scorer works in a LangChain run.
+ */
+interface ScoreResult {
+  score: number;
+  feedback?: string;
+  objectiveScores?: Record<string, number>;
+  /** What this rollout consumed, when the caller can see it. */
+  usage?: RolloutUsage;
+  /**
+   * Marks a score produced by an infrastructure failure — a rate limit, a
+   * network blip, a provider 5xx — rather than by the candidate. Without this
+   * the engine cannot tell such a zero from a genuine one, and would cache it
+   * permanently against the candidate.
+   */
+  transient?: boolean;
+}
+interface EvaluateArgs<Datum, K extends string = string> {
+  batch: readonly Datum[];
+  candidate: Candidate<K>;
+  captureTraces: boolean;
+  /**
+   * Where this batch sits in the run. Forward it to whatever tracing the
+   * system under optimization already has — without it a run is thousands of
+   * indistinguishable rollouts, and no trace can be tied back to the iteration
+   * whose score moved.
+   */
+  run: EvaluationContext;
+  signal?: AbortSignal;
+}
+/**
+ * Identifies one evaluation within a run. `candidateId` is null while the
+ * candidate is still a proposal being screened on a minibatch: it has no
+ * record, and inventing an id for it would collide with the one it gets if it
+ * is accepted.
+ */
+interface EvaluationContext {
+  iteration: number;
+  phase: EvaluationPhase;
+  split: EvaluationSplit;
+  candidateId: number | null;
+}
+type EvaluationPhase = "seed" | "minibatch" | "validation" | "test";
+/**
+ * Which dataset an instance id was drawn from. Each split numbers its ids
+ * independently, so the same id can name three different instances; the cache
+ * key has to keep them apart.
+ */
+type EvaluationSplit = "train" | "val" | "test";
+/**
+ * The single integration seam between an optimizer and a system under
+ * optimization. Everything framework-specific — LangChain, the AI SDK,
+ * Braintrust — lives in an implementation of this interface.
+ */
+interface Adapter<Datum, Trajectory = unknown, Output = unknown, K extends string = string> {
+  evaluate(args: EvaluateArgs<Datum, K>): Promise<EvaluationBatch<Trajectory, Output>> | EvaluationBatch<Trajectory, Output>;
+}
+/** Provider-agnostic text model: text in, text out. */
+type TextModel = (args: {
+  prompt: string;
+  signal?: AbortSignal;
+}) => Promise<string>;
+/**
+ * The component names of a candidate, as the union they were inferred from.
+ *
+ * `Object.keys` widens a closed key union back to `string`. This is the one
+ * place that narrowing happens, so every other caller stays assertion-free.
+ * Accepts a partial so it also names the components of a component patch.
+ */
+declare function componentNames<K extends string>(candidate: Partial<Candidate<K>>): K[];
+//#endregion
+//#region src/warnings.d.ts
+/**
+ * Something about a run that its own numbers cannot say.
+ *
+ * A search reports a score, a stop reason and what it spent, and every one of
+ * those reads the same whether the run measured what the caller thinks it did
+ * or not. These are the conditions under which a normal-looking result means
+ * less than it appears to: they never stop a run, and they are carried on the
+ * result and the `finish` event so a report can say so next to the number.
+ */
+interface RunWarning {
+  code: RunWarningCode;
+  message: string;
+}
+type RunWarningCode = "validationSetReusesTraining" | "seedScoreSaturated" | "seedScoreFloored";
+/**
+ * The validation set a run will actually select against, and whatever the
+ * choice costs it.
+ *
+ * Defaulting to the training set is the right default for a first run and the
+ * wrong number to report from one. It is worse than ordinary overfitting under
+ * reflective search: the reflection prompt asks the model to mine domain facts
+ * out of the traces it is shown, so those facts come out of the very instances
+ * that then select the candidate carrying them. `"reuseTraining"` is the same
+ * behaviour with the caller's name on it, and silences the warning.
+ */
+declare function resolveValidationSet<Datum>(args: {
+  validationSet: readonly Datum[] | "reuseTraining" | undefined;
+  trainingSet: readonly Datum[];
+}): {
+  validationSet: readonly Datum[];
+  warnings: RunWarning[];
+};
+/**
+ * What the seed's own validation row says about whether the run could have
+ * learned anything.
+ *
+ * A search ranks candidates by how they differ across instances, so a seed row
+ * with no spread leaves nothing to rank: at the ceiling every proposal is a tie
+ * the acceptance test resolves by noise, and at the floor no proposal has a
+ * partial improvement to build on. Both produce a run that spends its whole
+ * budget and reports a stop reason that looks like any other.
+ */
+declare function seedScoreWarnings(args: {
+  scores: readonly (number | undefined)[];
+  perfectScore: number;
+}): RunWarning[];
+//#endregion
+//#region src/reporting.d.ts
+/**
+ * Where a run's progress goes, for any optimizer. Observability only:
+ * persisting a run so it can be resumed is `onCheckpoint`, which is durability
+ * and a separate concern.
+ *
+ * Generic over the event union rather than one union covering every optimizer:
+ * a search emits what it actually has, and a reporter written against one
+ * optimizer still type-checks against the events it reads.
+ */
+/**
+ * The least an optimizer's event satisfies. A reporter typed against this
+ * accepts every optimizer's union, because a literal tag is assignable to
+ * `string` and the parameter position is contravariant.
+ */
+interface OptimizerEvent {
+  type: string;
+}
+interface Reporter<Event> {
+  /**
+   * Called on the search's hot path, synchronously. A reporter that ships
+   * anywhere over a network buffers here and uploads in `flush`, or it charges
+   * every iteration for its latency.
+   *
+   * A reporter that throws is warned about and skipped: observability never
+   * fails a run.
+   */
+  onEvent?: (event: Event) => void;
+  /** Awaited once as the run ends, including when it ends by throwing. */
+  flush?: () => Promise<void>;
+}
+/**
+ * The payload every optimizer's `candidateAccepted` carries, so one reporter
+ * can read an acceptance without knowing which search produced it. Each
+ * optimizer intersects its own fields onto this — GEPA its lineage, MIPRO its
+ * menu choices — the way the event unions already intersect `EvaluationEvent`.
+ *
+ * Emitted only when the incumbent moves and a full validation sweep measured
+ * it. An optimizer that accepts on a minibatch reports the acceptance in its
+ * own event and emits this one once the sweep that confirms it lands, so
+ * `instanceScores` never means "a subset, and you work out which".
+ */
+interface CandidateAccepted<K extends string = string> {
+  /** Identifies the candidate within the run. */
+  candidateId: number;
+  /** The text that scored, so a move is readable next to the edit. */
+  candidate: Candidate<K>;
+  /** Mean over the validation set, which is what selection is decided on. */
+  aggregateScore: number;
+  /**
+   * Per-instance scores, aligned with the validation set. `undefined` marks an
+   * instance an infrastructure failure left unmeasured — unknown, not zero.
+   *
+   * Handed out by reference rather than copied: a run emits this once per
+   * accepted candidate, and copying a validation-set-sized array that often to
+   * guard against a listener that writes to it costs every run to protect a
+   * listener that should not exist.
+   */
+  instanceScores: readonly (number | undefined)[];
+  /** Aligned with `instanceScores`. Present only under `trackBestOutputs`. */
+  outputs?: readonly unknown[];
+}
+/**
+ * The payload every optimizer's `finish` carries. `reason` stays per-optimizer
+ * because the stop reasons genuinely differ — only GEPA can exhaust a
+ * reflection budget.
+ */
+interface RunFinished {
+  /** The winner, named the way `CandidateAccepted.candidateId` names it. */
+  bestCandidateId: number;
+  bestScore: number;
+  metricCalls: number;
+  /** The winner's held-out score, when a testSet was given. */
+  testScore?: number;
+  /**
+   * The winner's per-instance held-out scores, aligned with the testSet and
+   * present whenever `testScore` is. `undefined` marks an instance an
+   * infrastructure failure left unmeasured, which is what `testScore` averages
+   * over too.
+   *
+   * The mean is the number selection never saw; this is where the gap below
+   * `bestScore` came from.
+   */
+  testInstanceScores?: readonly (number | undefined)[];
+  /** Aligned with `testInstanceScores`. Only under `trackBestOutputs`. */
+  testOutputs?: readonly unknown[];
+  /**
+   * What the run cannot say about itself from its own numbers. A reporter that
+   * writes the score somewhere permanent writes these beside it, or the record
+   * outlives the only place the caveat was ever stated.
+   */
+  warnings: readonly RunWarning[];
+}
+/**
+ * The two events every optimizer emits with a payload a reporter can read
+ * without knowing which search produced it. A cross-optimizer reporter takes
+ * this as its event type: it is a supertype of every optimizer's own union, so
+ * one reporter drops into any optimizer's `reporters` array.
+ */
+type ReportableEvent<K extends string = string> = ({
+  type: "candidateAccepted";
+} & CandidateAccepted<K>) | ({
+  type: "finish";
+} & RunFinished);
+/**
+ * Narrows an event off any optimizer's union to an acceptance. The tag is
+ * enough: every optimizer's `candidateAccepted` intersects `CandidateAccepted`,
+ * so carrying the payload is a compile-time obligation rather than a hope.
+ */
+declare function isCandidateAccepted<K extends string = string>(event: OptimizerEvent): event is {
+  type: "candidateAccepted";
+} & CandidateAccepted<K>;
+/** Narrows an event off any optimizer's union to the end of the run. */
+declare function isRunFinished(event: OptimizerEvent): event is {
+  type: "finish";
+} & RunFinished;
+//#endregion
+export { componentNames as C, UsageTotals as S, EvaluationPhase as _, RunFinished as a, ScoreResult as b, RunWarning as c, seedScoreWarnings as d, Adapter as f, EvaluationContext as g, EvaluationBatch as h, Reporter as i, RunWarningCode as l, EvaluateArgs as m, OptimizerEvent as n, isCandidateAccepted as o, Candidate as p, ReportableEvent as r, isRunFinished as s, CandidateAccepted as t, resolveValidationSet as u, EvaluationSplit as v, TextModel as x, RolloutUsage as y };