textopt 0.0.0

package/dist/opro/index.d.cts

@@ -0,0 +1,230 @@
+import { n as Candidate, t as Adapter, u as TextModel } from "../types-CWv4IQFF.cjs";
+import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.cjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-B7SpRwl7.cjs";
+//#region src/opro/optimize.d.ts
+/** One instruction that was tried, and what it scored. */
+/** A history entry plus the system state its score was measured in. */
+interface RecordedAttempt extends ScoredAttempt {
+  context: string;
+}
+interface ScoredAttempt {
+  text: string;
+  score: number;
+}
+type OproPromptBuilder = (args: {
+  componentName: string;
+  /** Attempts so far, ascending by score — the weakest first. */
+  history: readonly ScoredAttempt[];
+  /** Rendered task inputs, for grounding. Empty when none were requested. */
+  exemplars: readonly string[];
+}) => string;
+interface OproConfig {
+  /**
+   * Instructions drawn per round. Default 8, as in the paper.
+   *
+   * All eight come from one prompt, so they differ only by the sampling
+   * temperature of `reflect`. The paper runs its optimizer at 1.0 and finds
+   * below 0.5 explores too little to escape a plateau. At temperature 0 the
+   * eight drafts are identical, dedup collapses them to one, and the round
+   * costs eight reflection calls to try a single instruction — set a
+   * temperature on the model before raising this.
+   */
+  proposalsPerRound?: number;
+  /** How many of them may be in flight at once. Default 1. */
+  concurrency?: number;
+  maxRounds?: number;
+  /**
+   * Reflection is the expensive half of this search and no metric budget
+   * covers it, so it is bounded separately.
+   */
+  maxReflectionCalls?: number;
+  seed?: number;
+  /**
+   * Scored attempts the prompt carries, strongest kept. Default 20, matching
+   * the paper's `max_num_instructions`.
+   *
+   * The reference also drops attempts below an absolute score threshold
+   * (`old_instruction_score_threshold`, 0.3 for its GPT scorers) before this
+   * cut. Nothing here does: the weak tail is what marks the bottom of the
+   * range the model is reading a gradient across, and once twenty decent
+   * attempts exist the strongest-kept rule has retired it anyway.
+   */
+  historySize?: number;
+  /**
+   * Task inputs shown for grounding, redrawn each round. Default 3, the
+   * reference's `num_few_shot_questions_for_instruction_refinement`, which
+   * resamples them per step under its default `random` selection.
+   */
+  exemplars?: number;
+  /**
+   * Instances drawn once from the training set to screen proposals on. Unset means
+   * every proposal is measured on the whole validation set, which is the reliable
+   * reading and the expensive one: a round of eight proposals against a
+   * 500-instance validation set costs 4000 rollouts before anything is learned.
+   *
+   * The paper screens on a small fixed slice of the training set and checks the
+   * full set periodically, which is what lets the validation set be large enough to
+   * trust. The slice is drawn once and never resampled — the meta-prompt ranks
+   * attempts against each other, so they have to be measured on the same
+   * instances or the ranking is noise.
+   */
+  scoringSetSize?: number;
+  /**
+   * Rounds between full validation set sweeps of the incumbent. Default 3, the paper's
+   * `eval_interval`. Only used when `scoringSetSize` is set; without it every
+   * proposal is already a full sweep.
+   */
+  fullEvalInterval?: number;
+  /**
+   * What scores are multiplied by before being shown. Models discriminate
+   * between 41 and 68 far more reliably than between 0.41 and 0.68, which is
+   * the whole mechanism this search runs on. Default 100.
+   */
+  scoreScale?: number;
+  buildPrompt?: OproPromptBuilder;
+  trackBestOutputs?: boolean;
+  /**
+   * Include cached instance scores in every checkpoint. Leaving them out keeps
+   * snapshots small at the cost of a resumed run re-paying for rollouts it
+   * cannot look up. Default true.
+   */
+  checkpointCache?: boolean;
+}
+/**
+ * Everything needed to continue a run: the per-component score histories the
+ * meta-prompt is written from, the incumbent, the budget already spent, the
+ * position of the random stream, and the screening slice — which is drawn once
+ * and must survive a resume, since attempts screened on different instances
+ * are not the gradient this search reads.
+ */
+interface OproSnapshot {
+  version: 1;
+  fingerprint: string;
+  best: Candidate;
+  reported: Candidate;
+  /** Whether the incumbent has already been confirmed by a full sweep. */
+  incumbentSwept: boolean;
+  bestScore: number;
+  bestSearchScore: number;
+  seedScore: number;
+  round: number;
+  reflectionCalls: number;
+  metricCalls: number;
+  cacheHits: number;
+  rngState: number;
+  /** Component name -> every text tried for it, with what it scored. */
+  histories: Record<string, RecordedAttempt[]>;
+  /** Training set positions the screening slice was drawn from. */
+  scoringIndices?: number[];
+  cache?: [string, CachedScore][];
+}
+interface OproTask<Datum, Trajectory = unknown, Output = unknown, K extends string = string> extends OptimizerTask<Datum, Trajectory, Output, K> {
+  /**
+   * The base adapter, not `GepaAdapter`: this search reads scores only, so it
+   * never asks for traces or a reflective dataset.
+   */
+  adapter: Adapter<Datum, Trajectory, Output, NoInfer<K>>;
+  reflect: TextModel;
+  /** Renders a task input for the prompt. Defaults to JSON. */
+  renderDatum?: (datum: NoInfer<Datum>) => string;
+  instanceId?: (args: {
+    datum: NoInfer<Datum>;
+    index: number;
+  }) => string;
+  /** Pass `false` to disable caching entirely. */
+  cache?: EvaluationCache | false;
+  onEvent?: (event: OproEvent<NoInfer<K>>) => void;
+  /**
+   * Called with a resumable snapshot after the seed is scored and after every
+   * round. Persist it and a killed run costs the last round, not all of them.
+   */
+  onCheckpoint?: (snapshot: OproSnapshot) => void | Promise<void>;
+  /** Snapshot to continue from, instead of starting at the seed candidate. */
+  resumeFrom?: OproSnapshot;
+}
+type OproStopReason = "budgetExhausted" | "costExhausted" | "deadlineReached" | "reflectionBudgetExhausted" | "proposalsExhausted" | "maxRounds" | "aborted";
+type OproEvent<K extends string = string> = {
+  type: "start";
+  components: K[];
+  validationSetSize: number;
+} | {
+  type: "roundStart";
+  round: number;
+  component: K;
+  historySize: number;
+} | ({
+  type: "evaluation";
+} & EvaluationEvent) | {
+  type: "attempt";
+  round: number;
+  component: K;
+  score: number;
+  /** True when this attempt became the new incumbent. */
+  accepted: boolean;
+} | {
+  type: "finish";
+  reason: OproStopReason;
+  bestScore: number;
+  metricCalls: number;
+  testScore?: number;
+};
+interface OproAttempt<K extends string = string> {
+  round: number;
+  component: K;
+  candidate: Candidate<K>;
+  score: number;
+}
+interface OproResult<K extends string = string, Output = unknown> extends OptimizerResult<K, OproStopReason, Output> {
+  /** The seed's score, so the lift the search bought is readable directly. */
+  seedScore: number;
+  rounds: number;
+  /** Every candidate scored, in the order it was tried. */
+  trajectory: OproAttempt<K>[];
+  reflectionCalls: number;
+  cacheHits: number;
+  /** State as of the last round, ready to hand back as `resumeFrom`. */
+  snapshot: OproSnapshot;
+}
+/**
+ * Optimization by prompting: show the model what has been tried and what each
+ * attempt scored, and ask for something better.
+ *
+ * The regime this is for is the one GEPA cannot serve. GEPA's advantage over
+ * blind search is the per-instance feedback string — a diagnosis of *why* a
+ * rollout failed. Plenty of metrics cannot produce one: a reward model, a
+ * preference score, a classifier's accuracy over a closed label set. Handed
+ * those, reflection is asked to diagnose a failure it has no evidence about,
+ * and pays a frontier-model call to guess.
+ *
+ * This search asks for less and needs less. One scalar per candidate is the
+ * entire signal, and the model reasons over the *trajectory* of scores rather
+ * than the anatomy of a single failure. Where feedback does exist, use GEPA:
+ * a score history is a much thinner channel than a paragraph saying what broke.
+ */
+declare class OproOptimizer implements Optimizer<OproStopReason> {
+  #private;
+  constructor(config?: OproConfig);
+  optimize<Datum, Trajectory = unknown, Output = unknown, const K extends string = string>(task: OproTask<Datum, Trajectory, Output, K>): Promise<OproResult<K, Output>>;
+}
+/**
+ * Adapted from the meta-prompt in *Large Language Models as Optimizers* (Yang
+ * et al., 2023). Ascending order is load-bearing: the paper finds the model
+ * attends most to what is nearest the end of the prompt, so the best attempt
+ * has to be last.
+ *
+ * Exemplars carry an `<INS>` marker for the same reason the paper's do: an
+ * instruction written against the task in the abstract reads differently from
+ * one written to sit in a particular slot. The marker is placed ahead of the
+ * input, which is where a component's text usually goes but which this
+ * optimizer cannot actually know — only the adapter composes the real prompt.
+ * The paper also shows each exemplar's gold answer. Nothing here needs to know
+ * what that is: `renderDatum` is handed the whole datum, so a caller holding
+ * labels can render them alongside the input.
+ */
+declare function buildOproPrompt(args: {
+  componentName: string;
+  history: readonly ScoredAttempt[];
+  exemplars: readonly string[];
+}): string;
+//#endregion
+export { type OproAttempt, type OproConfig, type OproEvent, OproOptimizer, type OproPromptBuilder, type OproResult, type OproSnapshot, type OproStopReason, type OproTask, type ScoredAttempt, buildOproPrompt };

package/dist/opro/index.d.mts

@@ -0,0 +1,230 @@
+import { n as Candidate, t as Adapter, u as TextModel } from "../types-CWv4IQFF.mjs";
+import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.mjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-DqCoth_w.mjs";
+//#region src/opro/optimize.d.ts
+/** One instruction that was tried, and what it scored. */
+/** A history entry plus the system state its score was measured in. */
+interface RecordedAttempt extends ScoredAttempt {
+  context: string;
+}
+interface ScoredAttempt {
+  text: string;
+  score: number;
+}
+type OproPromptBuilder = (args: {
+  componentName: string;
+  /** Attempts so far, ascending by score — the weakest first. */
+  history: readonly ScoredAttempt[];
+  /** Rendered task inputs, for grounding. Empty when none were requested. */
+  exemplars: readonly string[];
+}) => string;
+interface OproConfig {
+  /**
+   * Instructions drawn per round. Default 8, as in the paper.
+   *
+   * All eight come from one prompt, so they differ only by the sampling
+   * temperature of `reflect`. The paper runs its optimizer at 1.0 and finds
+   * below 0.5 explores too little to escape a plateau. At temperature 0 the
+   * eight drafts are identical, dedup collapses them to one, and the round
+   * costs eight reflection calls to try a single instruction — set a
+   * temperature on the model before raising this.
+   */
+  proposalsPerRound?: number;
+  /** How many of them may be in flight at once. Default 1. */
+  concurrency?: number;
+  maxRounds?: number;
+  /**
+   * Reflection is the expensive half of this search and no metric budget
+   * covers it, so it is bounded separately.
+   */
+  maxReflectionCalls?: number;
+  seed?: number;
+  /**
+   * Scored attempts the prompt carries, strongest kept. Default 20, matching
+   * the paper's `max_num_instructions`.
+   *
+   * The reference also drops attempts below an absolute score threshold
+   * (`old_instruction_score_threshold`, 0.3 for its GPT scorers) before this
+   * cut. Nothing here does: the weak tail is what marks the bottom of the
+   * range the model is reading a gradient across, and once twenty decent
+   * attempts exist the strongest-kept rule has retired it anyway.
+   */
+  historySize?: number;
+  /**
+   * Task inputs shown for grounding, redrawn each round. Default 3, the
+   * reference's `num_few_shot_questions_for_instruction_refinement`, which
+   * resamples them per step under its default `random` selection.
+   */
+  exemplars?: number;
+  /**
+   * Instances drawn once from the training set to screen proposals on. Unset means
+   * every proposal is measured on the whole validation set, which is the reliable
+   * reading and the expensive one: a round of eight proposals against a
+   * 500-instance validation set costs 4000 rollouts before anything is learned.
+   *
+   * The paper screens on a small fixed slice of the training set and checks the
+   * full set periodically, which is what lets the validation set be large enough to
+   * trust. The slice is drawn once and never resampled — the meta-prompt ranks
+   * attempts against each other, so they have to be measured on the same
+   * instances or the ranking is noise.
+   */
+  scoringSetSize?: number;
+  /**
+   * Rounds between full validation set sweeps of the incumbent. Default 3, the paper's
+   * `eval_interval`. Only used when `scoringSetSize` is set; without it every
+   * proposal is already a full sweep.
+   */
+  fullEvalInterval?: number;
+  /**
+   * What scores are multiplied by before being shown. Models discriminate
+   * between 41 and 68 far more reliably than between 0.41 and 0.68, which is
+   * the whole mechanism this search runs on. Default 100.
+   */
+  scoreScale?: number;
+  buildPrompt?: OproPromptBuilder;
+  trackBestOutputs?: boolean;
+  /**
+   * Include cached instance scores in every checkpoint. Leaving them out keeps
+   * snapshots small at the cost of a resumed run re-paying for rollouts it
+   * cannot look up. Default true.
+   */
+  checkpointCache?: boolean;
+}
+/**
+ * Everything needed to continue a run: the per-component score histories the
+ * meta-prompt is written from, the incumbent, the budget already spent, the
+ * position of the random stream, and the screening slice — which is drawn once
+ * and must survive a resume, since attempts screened on different instances
+ * are not the gradient this search reads.
+ */
+interface OproSnapshot {
+  version: 1;
+  fingerprint: string;
+  best: Candidate;
+  reported: Candidate;
+  /** Whether the incumbent has already been confirmed by a full sweep. */
+  incumbentSwept: boolean;
+  bestScore: number;
+  bestSearchScore: number;
+  seedScore: number;
+  round: number;
+  reflectionCalls: number;
+  metricCalls: number;
+  cacheHits: number;
+  rngState: number;
+  /** Component name -> every text tried for it, with what it scored. */
+  histories: Record<string, RecordedAttempt[]>;
+  /** Training set positions the screening slice was drawn from. */
+  scoringIndices?: number[];
+  cache?: [string, CachedScore][];
+}
+interface OproTask<Datum, Trajectory = unknown, Output = unknown, K extends string = string> extends OptimizerTask<Datum, Trajectory, Output, K> {
+  /**
+   * The base adapter, not `GepaAdapter`: this search reads scores only, so it
+   * never asks for traces or a reflective dataset.
+   */
+  adapter: Adapter<Datum, Trajectory, Output, NoInfer<K>>;
+  reflect: TextModel;
+  /** Renders a task input for the prompt. Defaults to JSON. */
+  renderDatum?: (datum: NoInfer<Datum>) => string;
+  instanceId?: (args: {
+    datum: NoInfer<Datum>;
+    index: number;
+  }) => string;
+  /** Pass `false` to disable caching entirely. */
+  cache?: EvaluationCache | false;
+  onEvent?: (event: OproEvent<NoInfer<K>>) => void;
+  /**
+   * Called with a resumable snapshot after the seed is scored and after every
+   * round. Persist it and a killed run costs the last round, not all of them.
+   */
+  onCheckpoint?: (snapshot: OproSnapshot) => void | Promise<void>;
+  /** Snapshot to continue from, instead of starting at the seed candidate. */
+  resumeFrom?: OproSnapshot;
+}
+type OproStopReason = "budgetExhausted" | "costExhausted" | "deadlineReached" | "reflectionBudgetExhausted" | "proposalsExhausted" | "maxRounds" | "aborted";
+type OproEvent<K extends string = string> = {
+  type: "start";
+  components: K[];
+  validationSetSize: number;
+} | {
+  type: "roundStart";
+  round: number;
+  component: K;
+  historySize: number;
+} | ({
+  type: "evaluation";
+} & EvaluationEvent) | {
+  type: "attempt";
+  round: number;
+  component: K;
+  score: number;
+  /** True when this attempt became the new incumbent. */
+  accepted: boolean;
+} | {
+  type: "finish";
+  reason: OproStopReason;
+  bestScore: number;
+  metricCalls: number;
+  testScore?: number;
+};
+interface OproAttempt<K extends string = string> {
+  round: number;
+  component: K;
+  candidate: Candidate<K>;
+  score: number;
+}
+interface OproResult<K extends string = string, Output = unknown> extends OptimizerResult<K, OproStopReason, Output> {
+  /** The seed's score, so the lift the search bought is readable directly. */
+  seedScore: number;
+  rounds: number;
+  /** Every candidate scored, in the order it was tried. */
+  trajectory: OproAttempt<K>[];
+  reflectionCalls: number;
+  cacheHits: number;
+  /** State as of the last round, ready to hand back as `resumeFrom`. */
+  snapshot: OproSnapshot;
+}
+/**
+ * Optimization by prompting: show the model what has been tried and what each
+ * attempt scored, and ask for something better.
+ *
+ * The regime this is for is the one GEPA cannot serve. GEPA's advantage over
+ * blind search is the per-instance feedback string — a diagnosis of *why* a
+ * rollout failed. Plenty of metrics cannot produce one: a reward model, a
+ * preference score, a classifier's accuracy over a closed label set. Handed
+ * those, reflection is asked to diagnose a failure it has no evidence about,
+ * and pays a frontier-model call to guess.
+ *
+ * This search asks for less and needs less. One scalar per candidate is the
+ * entire signal, and the model reasons over the *trajectory* of scores rather
+ * than the anatomy of a single failure. Where feedback does exist, use GEPA:
+ * a score history is a much thinner channel than a paragraph saying what broke.
+ */
+declare class OproOptimizer implements Optimizer<OproStopReason> {
+  #private;
+  constructor(config?: OproConfig);
+  optimize<Datum, Trajectory = unknown, Output = unknown, const K extends string = string>(task: OproTask<Datum, Trajectory, Output, K>): Promise<OproResult<K, Output>>;
+}
+/**
+ * Adapted from the meta-prompt in *Large Language Models as Optimizers* (Yang
+ * et al., 2023). Ascending order is load-bearing: the paper finds the model
+ * attends most to what is nearest the end of the prompt, so the best attempt
+ * has to be last.
+ *
+ * Exemplars carry an `<INS>` marker for the same reason the paper's do: an
+ * instruction written against the task in the abstract reads differently from
+ * one written to sit in a particular slot. The marker is placed ahead of the
+ * input, which is where a component's text usually goes but which this
+ * optimizer cannot actually know — only the adapter composes the real prompt.
+ * The paper also shows each exemplar's gold answer. Nothing here needs to know
+ * what that is: `renderDatum` is handed the whole datum, so a caller holding
+ * labels can render them alongside the input.
+ */
+declare function buildOproPrompt(args: {
+  componentName: string;
+  history: readonly ScoredAttempt[];
+  exemplars: readonly string[];
+}): string;
+//#endregion
+export { type OproAttempt, type OproConfig, type OproEvent, OproOptimizer, type OproPromptBuilder, type OproResult, type OproSnapshot, type OproStopReason, type OproTask, type ScoredAttempt, buildOproPrompt };