textopt 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +509 -0
  3. package/dist/bootstrap-search/index.cjs +308 -0
  4. package/dist/bootstrap-search/index.d.cts +162 -0
  5. package/dist/bootstrap-search/index.d.mts +162 -0
  6. package/dist/bootstrap-search/index.mjs +307 -0
  7. package/dist/cache-CuSo0NJ8.d.cts +24 -0
  8. package/dist/cache-CuSo0NJ8.d.mts +24 -0
  9. package/dist/concurrency-C-cFzWW2.cjs +44 -0
  10. package/dist/concurrency-D58PWeSk.mjs +39 -0
  11. package/dist/demos-B0pVQjYC.d.mts +88 -0
  12. package/dist/demos-B9BJiNKz.cjs +143 -0
  13. package/dist/demos-BTuzFNsp.d.cts +88 -0
  14. package/dist/demos-Degx6UmP.mjs +126 -0
  15. package/dist/evaluation-BV0nSZVx.mjs +521 -0
  16. package/dist/evaluation-OZOp6TB7.cjs +598 -0
  17. package/dist/file-cache.cjs +70 -0
  18. package/dist/file-cache.d.cts +21 -0
  19. package/dist/file-cache.d.mts +21 -0
  20. package/dist/file-cache.mjs +69 -0
  21. package/dist/gepa/index.cjs +1671 -0
  22. package/dist/gepa/index.d.cts +385 -0
  23. package/dist/gepa/index.d.mts +385 -0
  24. package/dist/gepa/index.mjs +1652 -0
  25. package/dist/index.cjs +266 -0
  26. package/dist/index.d.cts +221 -0
  27. package/dist/index.d.mts +221 -0
  28. package/dist/index.mjs +245 -0
  29. package/dist/math-COOofUyv.cjs +101 -0
  30. package/dist/math-DhrDmpFS.mjs +78 -0
  31. package/dist/mipro/index.cjs +739 -0
  32. package/dist/mipro/index.d.cts +372 -0
  33. package/dist/mipro/index.d.mts +372 -0
  34. package/dist/mipro/index.mjs +736 -0
  35. package/dist/opro/index.cjs +487 -0
  36. package/dist/opro/index.d.cts +230 -0
  37. package/dist/opro/index.d.mts +230 -0
  38. package/dist/opro/index.mjs +485 -0
  39. package/dist/optimizer-B7SpRwl7.d.cts +288 -0
  40. package/dist/optimizer-DqCoth_w.d.mts +288 -0
  41. package/dist/random-search/index.cjs +321 -0
  42. package/dist/random-search/index.d.cts +156 -0
  43. package/dist/random-search/index.d.mts +156 -0
  44. package/dist/random-search/index.mjs +319 -0
  45. package/dist/reflection-CQToe-5B.d.cts +283 -0
  46. package/dist/reflection-Cr_upzU0.d.mts +283 -0
  47. package/dist/reflection-DRfbk6hu.cjs +249 -0
  48. package/dist/reflection-mwMhrjs_.mjs +214 -0
  49. package/dist/rng-BR5MOedA.d.cts +22 -0
  50. package/dist/rng-BR5MOedA.d.mts +22 -0
  51. package/dist/rng-DbA_rPIo.cjs +67 -0
  52. package/dist/rng-Dtc5eZ_W.mjs +62 -0
  53. package/dist/sampling-CfHt7Gue.mjs +59 -0
  54. package/dist/sampling-DFo_7RNJ.d.mts +23 -0
  55. package/dist/sampling-Dars7ctR.cjs +64 -0
  56. package/dist/sampling-axOwfZf5.d.cts +23 -0
  57. package/dist/simba/index.cjs +709 -0
  58. package/dist/simba/index.d.cts +289 -0
  59. package/dist/simba/index.d.mts +289 -0
  60. package/dist/simba/index.mjs +700 -0
  61. package/dist/testing.cjs +155 -0
  62. package/dist/testing.d.cts +53 -0
  63. package/dist/testing.d.mts +53 -0
  64. package/dist/testing.mjs +148 -0
  65. package/dist/text--v4Ffbus.mjs +21 -0
  66. package/dist/text-CK_HB3su.cjs +26 -0
  67. package/dist/types-CWv4IQFF.d.cts +129 -0
  68. package/dist/types-CWv4IQFF.d.mts +129 -0
  69. package/package.json +135 -0
@@ -0,0 +1,307 @@
1
+ import { a as requireMeasuredMean, d as createMemoryCache, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "../evaluation-BV0nSZVx.mjs";
2
+ import { n as formatDemos, t as bootstrapDemos } from "../demos-Degx6UmP.mjs";
3
+ import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
4
+ //#region src/bootstrap-search/optimize.ts
5
+ const DEFAULT_CANDIDATES = 16;
6
+ const DEFAULT_MAX_DEMOS = 4;
7
+ const DEFAULT_MIN_DEMOS = 1;
8
+ const DEFAULT_MAX_LABELED_DEMOS = 16;
9
+ /**
10
+ * Bootstrapped few-shot search: harvest demonstrations from rollouts the metric
11
+ * already rewarded, and pick the set that scores best.
12
+ *
13
+ * DSPy's `BootstrapFewShotWithRandomSearch`, which is what the literature
14
+ * usually means by "random search" over prompts. It is the only optimizer here
15
+ * that calls no model to write text: every candidate is assembled from outputs
16
+ * the system itself produced, so the search costs rollouts and nothing else.
17
+ * That makes it the right first thing to try — it is cheap, it needs no
18
+ * frontier model, and on tasks where the instruction is already adequate and
19
+ * the format is not, it is often the whole win.
20
+ *
21
+ * The fixed candidates come first and in DSPy's order: zero-shot (seed -3),
22
+ * labels-only when gold outputs exist (seed -2), and one unshuffled harvest at
23
+ * full size (seed -1). Shuffled harvests of random size follow. Keeping
24
+ * zero-shot in the running is not a formality — demonstrations can hurt, and a
25
+ * search that cannot return "no demos" has no baseline to report against.
26
+ *
27
+ * One deviation, deliberate: DSPy bootstraps each predictor separately from the
28
+ * traces of one pass. This library's adapter runs the whole system, so a
29
+ * harvest is a set of end-to-end rollouts and every demo component is given the
30
+ * same block. For per-module demos, use `createPipelineAdapter` with GEPA,
31
+ * which sees each module's own inputs and outputs.
32
+ */
33
+ var BootstrapSearchOptimizer = class {
34
+ #config;
35
+ constructor(config = {}) {
36
+ this.#config = config;
37
+ }
38
+ optimize(task) {
39
+ return run({
40
+ config: this.#config,
41
+ task
42
+ });
43
+ }
44
+ };
45
+ async function run(args) {
46
+ const { config, task } = args;
47
+ const { candidates: shuffledHarvests = DEFAULT_CANDIDATES, maxDemos = DEFAULT_MAX_DEMOS, minDemos = DEFAULT_MIN_DEMOS, maxLabeledDemos = DEFAULT_MAX_LABELED_DEMOS, demoMinScore, stopAtScore, seed = 0, trackBestOutputs = false, checkpointCache = true } = config;
48
+ const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, demoComponents, renderDemo, goldOutput, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
49
+ const deadline = createDeadline({ maxWallClockMs });
50
+ const components = componentNames(seedCandidate);
51
+ if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
52
+ if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
53
+ if (demoComponents.length === 0) throw new Error("optimize requires at least one demoComponent: this search has nothing to put demonstrations in otherwise");
54
+ if (testSet !== void 0 && testSet.length === 0) throw new Error("optimize requires a non-empty testSet when one is given; omit it to skip held-out evaluation");
55
+ const validationIds = validationSet.map((datum, index) => instanceId({
56
+ datum,
57
+ index
58
+ }));
59
+ const testIds = testSet?.map((datum, index) => instanceId({
60
+ datum,
61
+ index
62
+ })) ?? [];
63
+ const fingerprint = runFingerprint({
64
+ seedCandidate,
65
+ trainingIds: trainingSet.map((datum, index) => instanceId({
66
+ datum,
67
+ index
68
+ })),
69
+ validationIds,
70
+ seed,
71
+ ...cacheNamespace === void 0 ? {} : { cacheNamespace }
72
+ });
73
+ assertResumable({
74
+ fingerprint,
75
+ ...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
76
+ });
77
+ const rng = createSeededRng(seed, resumeFrom?.rngState);
78
+ const budget = createBudget({
79
+ maxMetricCalls,
80
+ spent: resumeFrom?.metricCalls ?? 0
81
+ });
82
+ const evaluationCache = cache === false ? void 0 : cache ?? createMemoryCache();
83
+ const evaluator = createEvaluator({
84
+ adapter,
85
+ budget,
86
+ ...retry === void 0 ? {} : { retry },
87
+ ...cacheNamespace === void 0 ? {} : { cacheNamespace },
88
+ ...evaluationCache === void 0 ? {} : { cache: evaluationCache },
89
+ trackOutputs: trackBestOutputs,
90
+ cacheHits: resumeFrom?.cacheHits ?? 0,
91
+ ...signal === void 0 ? {} : { signal },
92
+ onEvaluation: (event) => onEvent?.({
93
+ type: "evaluation",
94
+ ...event
95
+ })
96
+ });
97
+ evaluator.restore(resumeFrom?.cache ?? []);
98
+ const evaluated = [...resumeFrom?.candidates ?? []];
99
+ let drawn = resumeFrom?.drawn ?? 0;
100
+ let bootstrapMetricCalls = resumeFrom?.bootstrapMetricCalls ?? 0;
101
+ let stopReason = "candidatesExhausted";
102
+ onEvent?.({
103
+ type: "start",
104
+ components,
105
+ validationSetSize: validationSet.length
106
+ });
107
+ async function sweep(candidate, phase) {
108
+ return evaluator.evaluate({
109
+ candidate,
110
+ batch: validationSet,
111
+ ids: validationIds,
112
+ split: "val",
113
+ phase,
114
+ candidateId: null,
115
+ iteration: evaluated.length
116
+ });
117
+ }
118
+ const seedScore = resumeFrom?.seedScore ?? requireMeasuredMean({
119
+ batch: await sweep(seedCandidate, "seed"),
120
+ phase: "seed"
121
+ });
122
+ let best = resumeFrom?.best ?? seedCandidate;
123
+ let bestScore = resumeFrom?.bestScore ?? seedScore;
124
+ let bestOutputs;
125
+ function takeSnapshot() {
126
+ const cached = checkpointCache ? evaluationCache?.entries?.() : void 0;
127
+ return {
128
+ version: 1,
129
+ fingerprint,
130
+ candidates: [...evaluated],
131
+ best,
132
+ bestScore,
133
+ seedScore,
134
+ drawn,
135
+ metricCalls: budget.spent(),
136
+ bootstrapMetricCalls,
137
+ cacheHits: evaluator.cacheHits(),
138
+ rngState: rng.state(),
139
+ ...cached === void 0 ? {} : { cache: cached }
140
+ };
141
+ }
142
+ async function checkpoint() {
143
+ if (onCheckpoint === void 0) return;
144
+ await onCheckpoint(takeSnapshot());
145
+ }
146
+ const plan = candidatePlan({
147
+ shuffledHarvests,
148
+ labeled: goldOutput !== void 0
149
+ });
150
+ for (; drawn < plan.length; drawn += 1) {
151
+ if (signal?.aborted) {
152
+ stopReason = "aborted";
153
+ break;
154
+ }
155
+ if (costExhausted({
156
+ usage: evaluator.usage(),
157
+ maxCostUsd
158
+ })) {
159
+ stopReason = "costExhausted";
160
+ break;
161
+ }
162
+ if (deadline.exceeded()) {
163
+ stopReason = "deadlineReached";
164
+ break;
165
+ }
166
+ if (!budget.canAfford(validationSet.length + 1)) {
167
+ stopReason = "budgetExhausted";
168
+ break;
169
+ }
170
+ const source = plan[drawn];
171
+ const block = await buildBlock(source);
172
+ const candidate = withDemos(block);
173
+ let evaluation;
174
+ try {
175
+ evaluation = await sweep(candidate, "validation");
176
+ } catch (err) {
177
+ if (err instanceof BudgetExhausted) {
178
+ stopReason = "budgetExhausted";
179
+ break;
180
+ }
181
+ if (signal?.aborted) {
182
+ stopReason = "aborted";
183
+ break;
184
+ }
185
+ throw err;
186
+ }
187
+ const score = measuredMean(evaluation);
188
+ if (score === void 0) continue;
189
+ const accepted = score > bestScore;
190
+ evaluated.push({
191
+ candidate,
192
+ source,
193
+ demos: countDemos(block),
194
+ score
195
+ });
196
+ onEvent?.({
197
+ type: "candidate",
198
+ index: evaluated.length - 1,
199
+ source,
200
+ demos: countDemos(block),
201
+ score,
202
+ accepted
203
+ });
204
+ if (accepted) {
205
+ best = candidate;
206
+ bestScore = score;
207
+ bestOutputs = evaluation.outputs;
208
+ }
209
+ await checkpoint();
210
+ if (stopAtScore !== void 0 && score >= stopAtScore) {
211
+ stopReason = "scoreReached";
212
+ drawn += 1;
213
+ break;
214
+ }
215
+ }
216
+ if (signal?.aborted) stopReason = "aborted";
217
+ const testScore = testSet === void 0 ? void 0 : measuredMean(await evaluator.evaluate({
218
+ candidate: best,
219
+ batch: testSet,
220
+ ids: testIds,
221
+ split: "test",
222
+ phase: "test",
223
+ candidateId: null,
224
+ iteration: evaluated.length,
225
+ charge: false
226
+ }));
227
+ onEvent?.({
228
+ type: "finish",
229
+ reason: stopReason,
230
+ bestScore,
231
+ metricCalls: budget.spent(),
232
+ ...testScore === void 0 ? {} : { testScore }
233
+ });
234
+ return {
235
+ bestCandidate: best,
236
+ bestScore,
237
+ usage: evaluator.usage(),
238
+ seedScore,
239
+ candidates: evaluated,
240
+ bootstrapMetricCalls,
241
+ cacheHits: evaluator.cacheHits(),
242
+ metricCalls: budget.spent(),
243
+ snapshot: takeSnapshot(),
244
+ ...trackBestOutputs && bestOutputs !== void 0 ? { bestOutputs } : {},
245
+ ...testScore === void 0 ? {} : {
246
+ testScore,
247
+ testMetricCalls: testSet?.length ?? 0
248
+ },
249
+ stopReason
250
+ };
251
+ function withDemos(block) {
252
+ const candidate = { ...seedCandidate };
253
+ for (const name of demoComponents) candidate[name] = block;
254
+ return candidate;
255
+ }
256
+ async function buildBlock(source) {
257
+ if (source === "zeroShot") return "";
258
+ if (source === "labeled") return labeledBlock();
259
+ const requested = source === "unshuffled" ? maxDemos : minDemos + rng.nextInt(Math.max(1, maxDemos - minDemos + 1));
260
+ const affordable = Math.min(trainingSet.length, budget.remaining() - validationSet.length);
261
+ if (affordable < 1) return "";
262
+ const harvest = await bootstrapDemos({
263
+ adapter,
264
+ candidate: seedCandidate,
265
+ trainingSet,
266
+ ...demoMinScore === void 0 ? {} : { minScore: demoMinScore },
267
+ maxDemos: requested,
268
+ maxMetricCalls: affordable,
269
+ ...source === "unshuffled" ? {} : { rng },
270
+ ...renderDemo === void 0 ? {} : { renderDemo },
271
+ ...signal === void 0 ? {} : { signal }
272
+ });
273
+ bootstrapMetricCalls += harvest.metricCalls;
274
+ budget.reserve(harvest.metricCalls);
275
+ return harvest.block;
276
+ }
277
+ function labeledBlock() {
278
+ const labelled = trainingSet.map((datum) => ({
279
+ input: datum,
280
+ output: goldOutput?.(datum)
281
+ })).filter((demo) => demo.output !== void 0).slice(0, maxLabeledDemos);
282
+ return formatDemos(labelled, renderDemo === void 0 ? {} : { render: renderDemo });
283
+ }
284
+ }
285
+ /**
286
+ * The order candidates are tried in, following DSPy's special seeds: zero-shot
287
+ * first, then labels-only, then one unshuffled full-size harvest, then the
288
+ * shuffled ones. Cheapest and most reliable first, so a run cut short by its
289
+ * budget still has the baseline it needs to report against.
290
+ */
291
+ function candidatePlan(args) {
292
+ const { shuffledHarvests, labeled } = args;
293
+ return [
294
+ "zeroShot",
295
+ ...labeled ? ["labeled"] : [],
296
+ "unshuffled",
297
+ ...Array.from({ length: shuffledHarvests }, () => "bootstrapped")
298
+ ];
299
+ }
300
+ function countDemos(block) {
301
+ return block.split("<demo>").length - 1;
302
+ }
303
+ function defaultInstanceId(args) {
304
+ return String(args.index);
305
+ }
306
+ //#endregion
307
+ export { BootstrapSearchOptimizer };
@@ -0,0 +1,24 @@
1
+ //#region src/cache.d.ts
2
+ /**
3
+ * What the cache stores per (candidate, instance): the metric the frontier is
4
+ * built from, plus the per-objective breakdown when the adapter reports one.
5
+ * Both come from the same rollout, so caching the score without the objectives
6
+ * would force a re-run to recover them.
7
+ */
8
+ interface CachedScore {
9
+ score: number;
10
+ objectiveScores?: Record<string, number>;
11
+ }
12
+ interface EvaluationCache {
13
+ get(key: string): CachedScore | undefined;
14
+ set(key: string, cached: CachedScore): void;
15
+ /** Entries for checkpointing. Omit on caches that are already durable. */
16
+ entries?(): [string, CachedScore][];
17
+ }
18
+ declare function createMemoryCache(args?: {
19
+ maxEntries?: number;
20
+ /** Entries from a previous run's checkpoint. */
21
+ entries?: readonly [string, CachedScore][];
22
+ }): EvaluationCache;
23
+ //#endregion
24
+ export { EvaluationCache as n, createMemoryCache as r, CachedScore as t };
@@ -0,0 +1,24 @@
1
+ //#region src/cache.d.ts
2
+ /**
3
+ * What the cache stores per (candidate, instance): the metric the frontier is
4
+ * built from, plus the per-objective breakdown when the adapter reports one.
5
+ * Both come from the same rollout, so caching the score without the objectives
6
+ * would force a re-run to recover them.
7
+ */
8
+ interface CachedScore {
9
+ score: number;
10
+ objectiveScores?: Record<string, number>;
11
+ }
12
+ interface EvaluationCache {
13
+ get(key: string): CachedScore | undefined;
14
+ set(key: string, cached: CachedScore): void;
15
+ /** Entries for checkpointing. Omit on caches that are already durable. */
16
+ entries?(): [string, CachedScore][];
17
+ }
18
+ declare function createMemoryCache(args?: {
19
+ maxEntries?: number;
20
+ /** Entries from a previous run's checkpoint. */
21
+ entries?: readonly [string, CachedScore][];
22
+ }): EvaluationCache;
23
+ //#endregion
24
+ export { EvaluationCache as n, createMemoryCache as r, CachedScore as t };
@@ -0,0 +1,44 @@
1
+ //#region src/concurrency.ts
2
+ /**
3
+ * Order-preserving concurrency-limited map. An optimization run's workload is
4
+ * entirely IO bound, so adapters use this to fan out a batch across a bounded
5
+ * number of in-flight model calls.
6
+ */
7
+ async function mapWithConcurrency(args) {
8
+ const { items, limit, task, signal } = args;
9
+ if (items.length === 0) return [];
10
+ signal?.throwIfAborted();
11
+ const workerCount = Math.max(1, Math.min(Math.floor(limit), items.length));
12
+ const results = new Array(items.length);
13
+ let cursor = 0;
14
+ /**
15
+ * The first failure, kept rather than thrown so the workers that are already
16
+ * running settle before the caller gets control back. A rejection that left
17
+ * tasks running in the background would go on spending an optimizer's budget
18
+ * and writing its caches after the run it belonged to had ended.
19
+ */
20
+ let failure;
21
+ async function worker() {
22
+ while (cursor < items.length && failure === void 0) {
23
+ const index = cursor;
24
+ cursor += 1;
25
+ try {
26
+ signal?.throwIfAborted();
27
+ results[index] = await task(items[index], index);
28
+ } catch (err) {
29
+ failure ??= { err };
30
+ return;
31
+ }
32
+ }
33
+ }
34
+ await Promise.all(Array.from({ length: workerCount }, () => worker()));
35
+ if (failure !== void 0) throw failure.err;
36
+ return results;
37
+ }
38
+ //#endregion
39
+ Object.defineProperty(exports, "mapWithConcurrency", {
40
+ enumerable: true,
41
+ get: function() {
42
+ return mapWithConcurrency;
43
+ }
44
+ });
@@ -0,0 +1,39 @@
1
+ //#region src/concurrency.ts
2
+ /**
3
+ * Order-preserving concurrency-limited map. An optimization run's workload is
4
+ * entirely IO bound, so adapters use this to fan out a batch across a bounded
5
+ * number of in-flight model calls.
6
+ */
7
+ async function mapWithConcurrency(args) {
8
+ const { items, limit, task, signal } = args;
9
+ if (items.length === 0) return [];
10
+ signal?.throwIfAborted();
11
+ const workerCount = Math.max(1, Math.min(Math.floor(limit), items.length));
12
+ const results = new Array(items.length);
13
+ let cursor = 0;
14
+ /**
15
+ * The first failure, kept rather than thrown so the workers that are already
16
+ * running settle before the caller gets control back. A rejection that left
17
+ * tasks running in the background would go on spending an optimizer's budget
18
+ * and writing its caches after the run it belonged to had ended.
19
+ */
20
+ let failure;
21
+ async function worker() {
22
+ while (cursor < items.length && failure === void 0) {
23
+ const index = cursor;
24
+ cursor += 1;
25
+ try {
26
+ signal?.throwIfAborted();
27
+ results[index] = await task(items[index], index);
28
+ } catch (err) {
29
+ failure ??= { err };
30
+ return;
31
+ }
32
+ }
33
+ }
34
+ await Promise.all(Array.from({ length: workerCount }, () => worker()));
35
+ if (failure !== void 0) throw failure.err;
36
+ return results;
37
+ }
38
+ //#endregion
39
+ export { mapWithConcurrency as t };
@@ -0,0 +1,88 @@
1
+ import { n as Candidate, t as Adapter } from "./types-CWv4IQFF.mjs";
2
+ import { t as Rng } from "./rng-BR5MOedA.mjs";
3
+ //#region src/demos.d.ts
4
+ /**
5
+ * One worked example: what went in, what a run of the system produced, and
6
+ * how that output scored. A demo is harvested, never written — its value is
7
+ * that the output is one the system actually produced and the metric actually
8
+ * rewarded.
9
+ */
10
+ interface Demo<Datum = unknown, Output = unknown> {
11
+ input: Datum;
12
+ output: Output;
13
+ /** Present on harvested demos, absent on ones recovered from a block. */
14
+ score?: number;
15
+ }
16
+ type DemoRenderer<Datum = unknown, Output = unknown> = (args: {
17
+ demo: Demo<Datum, Output>;
18
+ index: number;
19
+ }) => string;
20
+ interface BootstrapResult<Datum, Output> {
21
+ demos: Demo<Datum, Output>[];
22
+ /** The demos as a candidate component, ready to seed a run with. */
23
+ block: string;
24
+ /** Rollouts this cost. Bootstrapping is cheap, not free. */
25
+ metricCalls: number;
26
+ attempted: number;
27
+ }
28
+ /**
29
+ * Harvest demonstrations by running a candidate over the training set and keeping
30
+ * the rollouts the metric rewarded.
31
+ *
32
+ * The cheapest signal in the whole library: a rollout that scored well is
33
+ * already paid for, and turning it into a few-shot block costs one pass over
34
+ * the data rather than a search. Instruction search and demonstrations pull on
35
+ * different parts of a model's behaviour — instructions on what to do,
36
+ * examples on what the output should look like — so a seed carrying both
37
+ * starts somewhere neither reaches alone.
38
+ */
39
+ declare function bootstrapDemos<Datum, Trajectory, Output, K extends string = string>(args: {
40
+ adapter: Adapter<Datum, Trajectory, Output, K>;
41
+ /** The candidate to run. Usually the seed, sometimes a run's winner. */
42
+ candidate: Candidate<K>;
43
+ trainingSet: readonly Datum[];
44
+ /**
45
+ * Score a rollout must reach to be kept. Unset keeps every rollout the
46
+ * metric rewarded at all, which is what MIPROv2's bootstrapper does without
47
+ * a `metric_threshold`: it keeps a trace on any truthy score and only
48
+ * compares against a number once one is configured.
49
+ *
50
+ * Demanding a perfect score instead is the right call for a boolean metric
51
+ * and the wrong one for a graded metric, where it throws away every rollout
52
+ * that was most of the way there — which on a hard task is all of them.
53
+ */
54
+ minScore?: number;
55
+ /** Demos to collect before stopping. Default 4. */
56
+ maxDemos?: number;
57
+ /**
58
+ * Instances per rollout batch. Smaller batches stop closer to the moment
59
+ * enough demos exist, at the cost of less concurrency inside the adapter.
60
+ */
61
+ batchSize?: number;
62
+ /** Ceiling on rollouts. Defaults to one pass over the trainingSet. */
63
+ maxMetricCalls?: number;
64
+ /** Shuffles the trainingSet first, so demos are not all drawn from its head. */
65
+ rng?: Rng;
66
+ renderDemo?: DemoRenderer<Datum, Output>;
67
+ signal?: AbortSignal;
68
+ }): Promise<BootstrapResult<Datum, Output>>;
69
+ /**
70
+ * Render demos as the text a candidate component holds.
71
+ *
72
+ * Delimited rather than free-form so `parseDemos` can read them back: a demo
73
+ * component is edited over the course of a run, and a block that cannot be
74
+ * parsed can only be replaced wholesale, throwing away every example found
75
+ * before it.
76
+ */
77
+ declare function formatDemos<Datum, Output>(demos: readonly Demo<Datum, Output>[], options?: {
78
+ render?: DemoRenderer<Datum, Output>;
79
+ }): string;
80
+ /**
81
+ * Recover the demos from a formatted block, ignoring anything written around
82
+ * them. Text a model rewrote and mangled yields the demos it left intact
83
+ * rather than throwing: a malformed example is worth less than the rest of the
84
+ * block, not more than it.
85
+ */
86
+ declare function parseDemos(text: string): Demo[];
87
+ //#endregion
88
+ export { formatDemos as a, bootstrapDemos as i, Demo as n, parseDemos as o, DemoRenderer as r, BootstrapResult as t };
@@ -0,0 +1,143 @@
1
+ const require_evaluation = require("./evaluation-OZOp6TB7.cjs");
2
+ //#region src/demos.ts
3
+ const DEMO_OPEN = "<demo>";
4
+ const DEMO_CLOSE = "</demo>";
5
+ const DEMO_BLOCK = /<demo>\s*([\s\S]*?)\s*<\/demo>/g;
6
+ const DEMO_PARTS = /<input>\s*([\s\S]*?)\s*<\/input>\s*<output>\s*([\s\S]*?)\s*<\/output>/;
7
+ const DEFAULT_MAX_DEMOS = 4;
8
+ /**
9
+ * Harvest demonstrations by running a candidate over the training set and keeping
10
+ * the rollouts the metric rewarded.
11
+ *
12
+ * The cheapest signal in the whole library: a rollout that scored well is
13
+ * already paid for, and turning it into a few-shot block costs one pass over
14
+ * the data rather than a search. Instruction search and demonstrations pull on
15
+ * different parts of a model's behaviour — instructions on what to do,
16
+ * examples on what the output should look like — so a seed carrying both
17
+ * starts somewhere neither reaches alone.
18
+ */
19
+ async function bootstrapDemos(args) {
20
+ const { adapter, candidate, trainingSet, minScore, maxDemos = DEFAULT_MAX_DEMOS, batchSize = maxDemos, maxMetricCalls = trainingSet.length, rng, renderDemo, signal } = args;
21
+ if (trainingSet.length === 0) throw new Error("bootstrapDemos requires a non-empty trainingSet");
22
+ const budget = require_evaluation.createBudget({ maxMetricCalls });
23
+ const evaluator = require_evaluation.createEvaluator({
24
+ adapter,
25
+ budget,
26
+ ...signal === void 0 ? {} : { signal }
27
+ });
28
+ const order = rng === void 0 ? [...trainingSet] : rng.shuffle(trainingSet);
29
+ const demos = [];
30
+ let attempted = 0;
31
+ for (let start = 0; start < order.length; start += batchSize) {
32
+ if (demos.length >= maxDemos || signal?.aborted) break;
33
+ const batch = order.slice(start, start + Math.min(batchSize, budget.remaining()));
34
+ if (batch.length === 0) break;
35
+ const evaluation = await evaluator.evaluateTraced({
36
+ candidate,
37
+ batch,
38
+ split: "train",
39
+ phase: "seed",
40
+ candidateId: null,
41
+ iteration: 0
42
+ });
43
+ if (evaluation === null) break;
44
+ attempted += batch.length;
45
+ for (let index = 0; index < batch.length; index += 1) {
46
+ const score = evaluation.scores[index];
47
+ if (!(minScore === void 0 ? score > 0 : score >= minScore) || demos.length >= maxDemos) continue;
48
+ demos.push({
49
+ input: batch[index],
50
+ output: evaluation.outputs[index],
51
+ score
52
+ });
53
+ }
54
+ }
55
+ return {
56
+ demos,
57
+ block: formatDemos(demos, renderDemo === void 0 ? {} : { render: renderDemo }),
58
+ metricCalls: budget.spent(),
59
+ attempted
60
+ };
61
+ }
62
+ /**
63
+ * Render demos as the text a candidate component holds.
64
+ *
65
+ * Delimited rather than free-form so `parseDemos` can read them back: a demo
66
+ * component is edited over the course of a run, and a block that cannot be
67
+ * parsed can only be replaced wholesale, throwing away every example found
68
+ * before it.
69
+ */
70
+ function formatDemos(demos, options = {}) {
71
+ const { render = renderDefault } = options;
72
+ if (demos.length === 0) return "";
73
+ return demos.map((demo, index) => `${DEMO_OPEN}\n${render({
74
+ demo,
75
+ index
76
+ })}\n${DEMO_CLOSE}`).join("\n");
77
+ }
78
+ /**
79
+ * Recover the demos from a formatted block, ignoring anything written around
80
+ * them. Text a model rewrote and mangled yields the demos it left intact
81
+ * rather than throwing: a malformed example is worth less than the rest of the
82
+ * block, not more than it.
83
+ */
84
+ function parseDemos(text) {
85
+ const demos = [];
86
+ for (const match of text.matchAll(DEMO_BLOCK)) {
87
+ const parts = (match[1] ?? "").match(DEMO_PARTS);
88
+ if (parts === null) continue;
89
+ const input = parseValue(parts[1] ?? "");
90
+ const output = parseValue(parts[2] ?? "");
91
+ demos.push({
92
+ input,
93
+ output
94
+ });
95
+ }
96
+ return demos;
97
+ }
98
+ function renderDefault(args) {
99
+ const { demo } = args;
100
+ return [
101
+ "<input>",
102
+ serialize(demo.input),
103
+ "</input>",
104
+ "<output>",
105
+ serialize(demo.output),
106
+ "</output>"
107
+ ].join("\n");
108
+ }
109
+ /** Strings stay as they are; anything else is shown as JSON. */
110
+ function serialize(value) {
111
+ if (typeof value === "string") return value;
112
+ try {
113
+ return JSON.stringify(value, null, 2) ?? String(value);
114
+ } catch {
115
+ return String(value);
116
+ }
117
+ }
118
+ function parseValue(text) {
119
+ try {
120
+ return JSON.parse(text);
121
+ } catch {
122
+ return text;
123
+ }
124
+ }
125
+ //#endregion
126
+ Object.defineProperty(exports, "bootstrapDemos", {
127
+ enumerable: true,
128
+ get: function() {
129
+ return bootstrapDemos;
130
+ }
131
+ });
132
+ Object.defineProperty(exports, "formatDemos", {
133
+ enumerable: true,
134
+ get: function() {
135
+ return formatDemos;
136
+ }
137
+ });
138
+ Object.defineProperty(exports, "parseDemos", {
139
+ enumerable: true,
140
+ get: function() {
141
+ return parseDemos;
142
+ }
143
+ });