textopt 0.0.0

package/dist/mipro/index.d.mts

@@ -0,0 +1,372 @@
+import { n as Candidate, t as Adapter, u as TextModel } from "../types-CWv4IQFF.mjs";
+import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.mjs";
+import { t as Rng } from "../rng-BR5MOedA.mjs";
+import { r as DemoRenderer } from "../demos-B0pVQjYC.mjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-DqCoth_w.mjs";
+import { t as BatchSampler } from "../sampling-DFo_7RNJ.mjs";
+//#region src/mipro/optimize.d.ts
+type MiproPromptBuilder = (args: {
+  componentName: string;
+  seedText: string;
+  /** Rendered task inputs, for grounding. Empty when none were requested. */
+  exemplars: readonly string[];
+  /** A style hint, varied per draw so the menu is not four of one idea. */
+  tip: string;
+}) => string;
+interface MiproConfig {
+  /**
+   * Instructions generated per component, beyond the seed. Ignored for a
+   * component the caller supplied a menu for. Default 3.
+   */
+  instructionsPerComponent?: number;
+  /**
+   * Instances a trial is scored on. Default 35, MIPROv2's `minibatch_size`.
+   *
+   * The surrogate reads these means as evidence, so the size sets how much of
+   * what it learns is signal. Shrinking it is the cheapest way to buy trials
+   * and the fastest way to make them worthless: the good/bad split at thirty
+   * trials is three or four observations, and a lucky small minibatch is
+   * enough to put the wrong configuration among them.
+   *
+   * MIPROv2 also abandons minibatching altogether when the validation set is
+   * 50 instances or fewer (`MIN_MINIBATCH_SIZE`), evaluating every trial in
+   * full. textopt has no such mode; on a small validation set, set this to the
+   * set's size to get the same behaviour.
+   */
+  minibatchSize?: number;
+  /** Configurations evaluated. Default 30. */
+  maxTrials?: number;
+  /** Trials drawn uniformly before the surrogate takes over. Default 10. */
+  startupTrials?: number;
+  /**
+   * Fraction of observations the surrogate treats as good. Defaults to
+   * Optuna's rule — a tenth of them, capped at 25 — which narrows the good set
+   * as observations accumulate rather than holding a fixed share.
+   */
+  gamma?: number;
+  /** Configurations the surrogate draws per trial. Default 24. */
+  surrogateSamples?: number;
+  /**
+   * Whether the surrogate models components jointly. Default true, matching
+   * MIPROv2. Set false when the components do independent jobs — see
+   * `proposeConfiguration` for the tradeoff.
+   */
+  multivariate?: boolean;
+  /**
+   * Trials between full validation sweeps. Every interval, the configuration
+   * with the best *average* minibatch reading that has not been swept yet is
+   * evaluated in full. Averaging is the point: a single minibatch is a noisy
+   * reading, and promoting on one alone lets a lucky draw decide the run.
+   * Default 5, MIPROv2's `minibatch_full_eval_steps`.
+   *
+   * MIPROv2 sweeps when `trial_num % (minibatch_full_eval_steps + 1) == 0`,
+   * which reads like every sixth trial but is not: Optuna numbers the seed
+   * baseline and each full evaluation as trials of their own, so the six slots
+   * hold five minibatch trials and one sweep. Its own budgeting says the same
+   * thing directly — `num_trials // minibatch_full_eval_steps + 1` sweeps for
+   * `num_trials` trials. This counts only minibatch trials, so 5 here and 5
+   * there describe one schedule.
+   */
+  fullEvalInterval?: number;
+  /**
+   * Bootstrapped demo sets generated per demo component, beyond the seed text
+   * and the zero-shot option. Default 3.
+   */
+  demoSets?: number;
+  /** Demos in the largest generated set. Default 4. */
+  maxDemos?: number;
+  /**
+   * Score a rollout must reach to be kept as a demo. Unset keeps every rollout
+   * the metric rewarded at all, as MIPROv2 does without a `metric_threshold`.
+   * Set 1 to demand a perfect score, which suits a boolean metric and discards
+   * most of a graded one.
+   */
+  demoMinScore?: number;
+  /** Task inputs shown when generating instructions. Default 3. */
+  exemplars?: number;
+  /**
+   * Summarize the training set with one reflection call and show that summary to
+   * the proposer, as MIPROv2's grounded proposer does. Default true. The
+   * summary reads more data than the exemplars can fit, so it describes the
+   * task rather than a few instances of it. Costs one reflection call, and is
+   * skipped entirely when every menu was supplied and nothing is proposed.
+   */
+  datasetSummary?: boolean;
+  /** Trainset entries the summary is written from. Default 10. */
+  summaryExamples?: number;
+  /** How many instruction proposals may be in flight at once. Default 1. */
+  concurrency?: number;
+  seed?: number;
+  buildPrompt?: MiproPromptBuilder;
+  /** Replaces the built-in style hints. */
+  tips?: readonly string[];
+  trackBestOutputs?: boolean;
+  /**
+   * Include cached instance scores in every checkpoint. Leaving them out keeps
+   * snapshots small at the cost of a resumed run re-paying for rollouts it
+   * cannot look up. Default true.
+   */
+  checkpointCache?: boolean;
+}
+/**
+ * Everything needed to continue a run.
+ *
+ * The menus matter most. Building them is the expensive half of a MIPRO run —
+ * a reflection call per instruction and a harvesting pass per demo set — and
+ * they are also what every trial's choice vector indexes into, so a resumed
+ * run that rebuilt them would both pay twice and reinterpret every
+ * observation it had already made.
+ */
+interface MiproSnapshot {
+  version: 1;
+  fingerprint: string;
+  /** Component name -> its option menu, in the order choices index it. */
+  menu: Record<string, string[]>;
+  best: Candidate;
+  bestScore: number;
+  seedScore: number;
+  trial: number;
+  fullEvaluations: number;
+  reflectionCalls: number;
+  bootstrapMetricCalls: number;
+  metricCalls: number;
+  cacheHits: number;
+  rngState: number;
+  observations: MiproObservation[];
+  /** What the surrogate was fitted on: one entry per measured trial. */
+  surrogateInput: {
+    choices: number[];
+    score: number;
+  }[];
+  /** Configuration key -> its minibatch readings so far. */
+  readings: [string, number[]][];
+  /** Configuration keys a full sweep has already bought. */
+  swept: string[];
+  /** Whatever the batch sampler reports from `state()`, when it has one. */
+  sampler?: unknown;
+  cache?: [string, CachedScore][];
+}
+interface MiproTask<Datum, Trajectory = unknown, Output = unknown, K extends string = string> extends OptimizerTask<Datum, Trajectory, Output, K> {
+  /**
+   * The base adapter, not `GepaAdapter`: the surrogate reads scores only, so
+   * this search never asks for traces or a reflective dataset.
+   */
+  adapter: Adapter<Datum, Trajectory, Output, NoInfer<K>>;
+  /** Generates the menu for components no menu was supplied for. */
+  reflect: TextModel;
+  /**
+   * Menu entries for a component, used verbatim and never rewritten. This is
+   * where a bootstrapped demo block belongs: demos are harvested, not
+   * authored, and handing them to a rewriting model destroys them.
+   */
+  componentOptions?: Partial<Record<NoInfer<K>, readonly string[]>>;
+  /**
+   * Components holding few-shot demo blocks. Their menus are bootstrapped from
+   * the trainingSet rather than written by `reflect` — MIPROv2 searches
+   * instructions and demonstrations together, and a demo is evidence a rollout
+   * actually produced, which asking a model to author would destroy.
+   */
+  demoComponents?: readonly NoInfer<K>[];
+  /** Renders a harvested rollout as demo text. Defaults to JSON. */
+  renderDemo?: DemoRenderer<NoInfer<Datum>, NoInfer<Output>>;
+  /**
+   * The gold output for a training datum, where the caller has labels. Supply
+   * it and every demo component keeps a labels-only set on its menu, as
+   * MIPROv2 does — the one demo set that costs no rollouts at all, since the
+   * output is known rather than produced. Return `undefined` for an unlabelled
+   * datum. Nothing generic can infer this: only the caller knows which part of
+   * a datum is the answer.
+   */
+  goldOutput?: (datum: NoInfer<Datum>) => NoInfer<Output> | undefined;
+  /** Renders a task input for the proposal prompt. Defaults to JSON. */
+  renderDatum?: (datum: NoInfer<Datum>) => string;
+  batchSampler?: BatchSampler<NoInfer<Datum>>;
+  instanceId?: (args: {
+    datum: NoInfer<Datum>;
+    index: number;
+  }) => string;
+  /** Pass `false` to disable caching entirely. */
+  cache?: EvaluationCache | false;
+  onEvent?: (event: MiproEvent<NoInfer<K>>) => void;
+  /**
+   * Called with a resumable snapshot once the menus are built and after every
+   * trial. Persist it and a killed run costs the last trial, not the menus.
+   */
+  onCheckpoint?: (snapshot: MiproSnapshot) => void | Promise<void>;
+  /** Snapshot to continue from, instead of starting at the seed candidate. */
+  resumeFrom?: MiproSnapshot;
+}
+type MiproStopReason = "budgetExhausted" | "costExhausted" | "deadlineReached" | "maxTrials" | "aborted";
+type MiproEvent<K extends string = string> = {
+  type: "start";
+  components: K[];
+  validationSetSize: number;
+} | {
+  type: "menu";
+  menu: Record<K, string[]>;
+  reflectionCalls: number;
+} | ({
+  type: "evaluation";
+} & EvaluationEvent) | {
+  type: "trial";
+  trial: number;
+  choices: number[];
+  minibatchScore: number;
+  /** True when the trial earned a full validation sweep. */
+  promoted: boolean;
+} | {
+  type: "incumbent";
+  trial: number;
+  choices: number[];
+  score: number;
+} | {
+  type: "finish";
+  reason: MiproStopReason;
+  bestScore: number;
+  metricCalls: number;
+  testScore?: number;
+};
+interface MiproObservation {
+  trial: number;
+  choices: number[];
+  minibatchScore: number;
+  promoted: boolean;
+  /** The full validation score, present only on promoted trials. */
+  score?: number;
+}
+interface MiproResult<K extends string = string, Output = unknown> extends OptimizerResult<K, MiproStopReason, Output> {
+  /** The seed's full validation score, so the lift is readable directly. */
+  seedScore: number;
+  trials: number;
+  /** The search space that was actually explored, per component. */
+  menu: Record<K, string[]>;
+  observations: MiproObservation[];
+  /** Trials that earned a full sweep. The rest were minibatch readings only. */
+  fullEvaluations: number;
+  /** Rollouts spent harvesting demos, included in `metricCalls`. */
+  bootstrapMetricCalls: number;
+  reflectionCalls: number;
+  cacheHits: number;
+  /** State as of the last trial, ready to hand back as `resumeFrom`. */
+  snapshot: MiproSnapshot;
+}
+/**
+ * Joint search over a fixed menu, guided by a surrogate.
+ *
+ * The gap this fills is interaction between components. Reflective search
+ * updates one component per iteration and screens it in isolation, so a pair
+ * of components that only pay off together is invisible to it — a routing rule
+ * and the prompt it routes to, an output format and the instruction that
+ * assumes it. Merge recombines lineages after the fact but never proposes a
+ * joint move.
+ *
+ * This search proposes a menu of options per component up front, then treats
+ * the choice of one option per component as a single categorical
+ * configuration and lets a TPE decide which to spend a trial on. Trials run on
+ * minibatches; a configuration that beats the best minibatch reading earns a
+ * full sweep before it can become the incumbent, so the number reported is
+ * never a lucky minibatch.
+ *
+ * What it gives up is the ability to write text it did not think of at the
+ * start. The menu is fixed at trial one — reflective search keeps writing new
+ * text for the whole run. Neither dominates; they fail differently.
+ */
+declare class MiproOptimizer implements Optimizer<MiproStopReason> {
+  #private;
+  constructor(config?: MiproConfig);
+  optimize<Datum, Trajectory = unknown, Output = unknown, const K extends string = string>(task: MiproTask<Datum, Trajectory, Output, K>): Promise<MiproResult<K, Output>>;
+}
+declare function buildMiproPrompt(args: {
+  componentName: string;
+  seedText: string;
+  exemplars: readonly string[];
+  tip: string;
+  /** The other components' current text, so the proposal fits the system. */
+  siblings?: Readonly<Record<string, string>>;
+  /** What the trainingSet looks like as a whole, beyond the exemplars. */
+  datasetSummary?: string;
+}): string;
+//#endregion
+//#region src/mipro/tpe.d.ts
+/** One configuration that was scored: an index into each component's menu. */
+interface Observation {
+  choices: readonly number[];
+  score: number;
+}
+/**
+ * Tree-structured Parzen Estimator over a categorical space.
+ *
+ * The search space here is a menu index per component. A TPE splits the
+ * observations into the good ones and the rest, models the density of each,
+ * and proposes the configuration that maximizes their ratio: sample where
+ * good configurations live, prefer where bad ones do not.
+ *
+ * The densities are mixtures with one kernel centred on each observation,
+ * rather than one histogram per component. That distinction is the whole
+ * reason this optimizer exists. Per-component histograms only record how often
+ * an option appears among good trials, so a space where every option is
+ * equally common and only the *pairing* matters looks completely flat to them.
+ * A kernel centred on an observed configuration keeps its components together,
+ * so "B works, but only alongside A" survives into the proposal. It is what
+ * Optuna calls a multivariate sampler, and what MIPROv2 turns on.
+ *
+ * What this buys over sampling every combination: a candidate with five
+ * components and four options each has 1024 configurations and a budget for
+ * perhaps thirty evaluations. Per-component hill climbing cannot see
+ * interactions at all; enumerating cannot afford to. This can do both, at the
+ * cost of being an estimate.
+ */
+declare function proposeConfiguration(args: {
+  observations: readonly Observation[];
+  /** Menu length per component, positionally. */
+  menuSizes: readonly number[];
+  /**
+   * Fraction of observations treated as good. Defaults to Optuna's rule — a
+   * tenth of them, never more than 25 — which narrows the good set as the run
+   * goes on. A fixed fraction keeps admitting weaker observations as evidence
+   * accumulates, which is backwards: the more you have measured, the less a
+   * middling result should count as something to aim at.
+   */
+  gamma?: number;
+  /** Configurations drawn before the best is chosen. Default 24. */
+  samples?: number;
+  /** Observations required before the model is trusted at all. Default 10. */
+  startupTrials?: number;
+  /**
+   * Total weight of the uniform prior, shared out across the kernels. Optuna's
+   * `prior_weight`. Each kernel carries `priorWeight / kernelCount` of it, so
+   * an observed option stands at `1 + kernelCount / priorWeight` against one
+   * beside it — twice as likely under a single kernel, and progressively
+   * sharper as observations accumulate.
+   */
+  priorWeight?: number;
+  /**
+   * Model components jointly rather than one at a time. Default true, as
+   * MIPROv2 sets on Optuna.
+   *
+   * Joint modelling is the only way to represent "this option works, but only
+   * beside that one", and it costs something where no such dependency exists.
+   * Kernels sit on observed *combinations*, so evidence does not carry across
+   * to combinations nobody has drawn; independent histograms generalize from
+   * far fewer trials but cannot express a dependency at all.
+   *
+   * Measured on a chained objective, where a component pays off only when the
+   * one before it is also right — the dependency independent histograms cannot
+   * represent. On five components of five options, 3125 configurations against
+   * 60 trials over 15 seeds, joint reaches a mean best of 0.91 and solves 10
+   * runs where independent reaches 0.80 and solves 6. On a 16-configuration
+   * space that 30 trials nearly enumerate, joint solves 20 runs of 20 against
+   * independent's 16.
+   *
+   * Joint led on the small space only once the kernels were given Optuna's
+   * smoothing. While each kernel stayed as wide as it was on the first trial,
+   * joint lost that case 15 runs to 16, and the default was argued for on the
+   * grounds that the case it lost was the one that needed a surrogate least.
+   * It no longer loses it, so that argument is retired: re-measure before
+   * repeating either number.
+   */
+  multivariate?: boolean;
+  rng: Rng;
+}): number[];
+//#endregion
+export { type MiproConfig, type MiproEvent, type MiproObservation, MiproOptimizer, type MiproPromptBuilder, type MiproResult, type MiproSnapshot, type MiproStopReason, type MiproTask, type Observation, buildMiproPrompt, proposeConfiguration };