textopt 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +509 -0
- package/dist/bootstrap-search/index.cjs +308 -0
- package/dist/bootstrap-search/index.d.cts +162 -0
- package/dist/bootstrap-search/index.d.mts +162 -0
- package/dist/bootstrap-search/index.mjs +307 -0
- package/dist/cache-CuSo0NJ8.d.cts +24 -0
- package/dist/cache-CuSo0NJ8.d.mts +24 -0
- package/dist/concurrency-C-cFzWW2.cjs +44 -0
- package/dist/concurrency-D58PWeSk.mjs +39 -0
- package/dist/demos-B0pVQjYC.d.mts +88 -0
- package/dist/demos-B9BJiNKz.cjs +143 -0
- package/dist/demos-BTuzFNsp.d.cts +88 -0
- package/dist/demos-Degx6UmP.mjs +126 -0
- package/dist/evaluation-BV0nSZVx.mjs +521 -0
- package/dist/evaluation-OZOp6TB7.cjs +598 -0
- package/dist/file-cache.cjs +70 -0
- package/dist/file-cache.d.cts +21 -0
- package/dist/file-cache.d.mts +21 -0
- package/dist/file-cache.mjs +69 -0
- package/dist/gepa/index.cjs +1671 -0
- package/dist/gepa/index.d.cts +385 -0
- package/dist/gepa/index.d.mts +385 -0
- package/dist/gepa/index.mjs +1652 -0
- package/dist/index.cjs +266 -0
- package/dist/index.d.cts +221 -0
- package/dist/index.d.mts +221 -0
- package/dist/index.mjs +245 -0
- package/dist/math-COOofUyv.cjs +101 -0
- package/dist/math-DhrDmpFS.mjs +78 -0
- package/dist/mipro/index.cjs +739 -0
- package/dist/mipro/index.d.cts +372 -0
- package/dist/mipro/index.d.mts +372 -0
- package/dist/mipro/index.mjs +736 -0
- package/dist/opro/index.cjs +487 -0
- package/dist/opro/index.d.cts +230 -0
- package/dist/opro/index.d.mts +230 -0
- package/dist/opro/index.mjs +485 -0
- package/dist/optimizer-B7SpRwl7.d.cts +288 -0
- package/dist/optimizer-DqCoth_w.d.mts +288 -0
- package/dist/random-search/index.cjs +321 -0
- package/dist/random-search/index.d.cts +156 -0
- package/dist/random-search/index.d.mts +156 -0
- package/dist/random-search/index.mjs +319 -0
- package/dist/reflection-CQToe-5B.d.cts +283 -0
- package/dist/reflection-Cr_upzU0.d.mts +283 -0
- package/dist/reflection-DRfbk6hu.cjs +249 -0
- package/dist/reflection-mwMhrjs_.mjs +214 -0
- package/dist/rng-BR5MOedA.d.cts +22 -0
- package/dist/rng-BR5MOedA.d.mts +22 -0
- package/dist/rng-DbA_rPIo.cjs +67 -0
- package/dist/rng-Dtc5eZ_W.mjs +62 -0
- package/dist/sampling-CfHt7Gue.mjs +59 -0
- package/dist/sampling-DFo_7RNJ.d.mts +23 -0
- package/dist/sampling-Dars7ctR.cjs +64 -0
- package/dist/sampling-axOwfZf5.d.cts +23 -0
- package/dist/simba/index.cjs +709 -0
- package/dist/simba/index.d.cts +289 -0
- package/dist/simba/index.d.mts +289 -0
- package/dist/simba/index.mjs +700 -0
- package/dist/testing.cjs +155 -0
- package/dist/testing.d.cts +53 -0
- package/dist/testing.d.mts +53 -0
- package/dist/testing.mjs +148 -0
- package/dist/text--v4Ffbus.mjs +21 -0
- package/dist/text-CK_HB3su.cjs +26 -0
- package/dist/types-CWv4IQFF.d.cts +129 -0
- package/dist/types-CWv4IQFF.d.mts +129 -0
- package/package.json +135 -0
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
import { a as requireMeasuredMean, d as createMemoryCache, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "../evaluation-BV0nSZVx.mjs";
|
|
2
|
+
import { n as formatDemos, t as bootstrapDemos } from "../demos-Degx6UmP.mjs";
|
|
3
|
+
import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
|
|
4
|
+
//#region src/bootstrap-search/optimize.ts
|
|
5
|
+
const DEFAULT_CANDIDATES = 16;
|
|
6
|
+
const DEFAULT_MAX_DEMOS = 4;
|
|
7
|
+
const DEFAULT_MIN_DEMOS = 1;
|
|
8
|
+
const DEFAULT_MAX_LABELED_DEMOS = 16;
|
|
9
|
+
/**
|
|
10
|
+
* Bootstrapped few-shot search: harvest demonstrations from rollouts the metric
|
|
11
|
+
* already rewarded, and pick the set that scores best.
|
|
12
|
+
*
|
|
13
|
+
* DSPy's `BootstrapFewShotWithRandomSearch`, which is what the literature
|
|
14
|
+
* usually means by "random search" over prompts. It is the only optimizer here
|
|
15
|
+
* that calls no model to write text: every candidate is assembled from outputs
|
|
16
|
+
* the system itself produced, so the search costs rollouts and nothing else.
|
|
17
|
+
* That makes it the right first thing to try — it is cheap, it needs no
|
|
18
|
+
* frontier model, and on tasks where the instruction is already adequate and
|
|
19
|
+
* the format is not, it is often the whole win.
|
|
20
|
+
*
|
|
21
|
+
* The fixed candidates come first and in DSPy's order: zero-shot (seed -3),
|
|
22
|
+
* labels-only when gold outputs exist (seed -2), and one unshuffled harvest at
|
|
23
|
+
* full size (seed -1). Shuffled harvests of random size follow. Keeping
|
|
24
|
+
* zero-shot in the running is not a formality — demonstrations can hurt, and a
|
|
25
|
+
* search that cannot return "no demos" has no baseline to report against.
|
|
26
|
+
*
|
|
27
|
+
* One deviation, deliberate: DSPy bootstraps each predictor separately from the
|
|
28
|
+
* traces of one pass. This library's adapter runs the whole system, so a
|
|
29
|
+
* harvest is a set of end-to-end rollouts and every demo component is given the
|
|
30
|
+
* same block. For per-module demos, use `createPipelineAdapter` with GEPA,
|
|
31
|
+
* which sees each module's own inputs and outputs.
|
|
32
|
+
*/
|
|
33
|
+
var BootstrapSearchOptimizer = class {
|
|
34
|
+
#config;
|
|
35
|
+
constructor(config = {}) {
|
|
36
|
+
this.#config = config;
|
|
37
|
+
}
|
|
38
|
+
optimize(task) {
|
|
39
|
+
return run({
|
|
40
|
+
config: this.#config,
|
|
41
|
+
task
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
async function run(args) {
|
|
46
|
+
const { config, task } = args;
|
|
47
|
+
const { candidates: shuffledHarvests = DEFAULT_CANDIDATES, maxDemos = DEFAULT_MAX_DEMOS, minDemos = DEFAULT_MIN_DEMOS, maxLabeledDemos = DEFAULT_MAX_LABELED_DEMOS, demoMinScore, stopAtScore, seed = 0, trackBestOutputs = false, checkpointCache = true } = config;
|
|
48
|
+
const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, demoComponents, renderDemo, goldOutput, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
|
|
49
|
+
const deadline = createDeadline({ maxWallClockMs });
|
|
50
|
+
const components = componentNames(seedCandidate);
|
|
51
|
+
if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
|
|
52
|
+
if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
|
|
53
|
+
if (demoComponents.length === 0) throw new Error("optimize requires at least one demoComponent: this search has nothing to put demonstrations in otherwise");
|
|
54
|
+
if (testSet !== void 0 && testSet.length === 0) throw new Error("optimize requires a non-empty testSet when one is given; omit it to skip held-out evaluation");
|
|
55
|
+
const validationIds = validationSet.map((datum, index) => instanceId({
|
|
56
|
+
datum,
|
|
57
|
+
index
|
|
58
|
+
}));
|
|
59
|
+
const testIds = testSet?.map((datum, index) => instanceId({
|
|
60
|
+
datum,
|
|
61
|
+
index
|
|
62
|
+
})) ?? [];
|
|
63
|
+
const fingerprint = runFingerprint({
|
|
64
|
+
seedCandidate,
|
|
65
|
+
trainingIds: trainingSet.map((datum, index) => instanceId({
|
|
66
|
+
datum,
|
|
67
|
+
index
|
|
68
|
+
})),
|
|
69
|
+
validationIds,
|
|
70
|
+
seed,
|
|
71
|
+
...cacheNamespace === void 0 ? {} : { cacheNamespace }
|
|
72
|
+
});
|
|
73
|
+
assertResumable({
|
|
74
|
+
fingerprint,
|
|
75
|
+
...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
|
|
76
|
+
});
|
|
77
|
+
const rng = createSeededRng(seed, resumeFrom?.rngState);
|
|
78
|
+
const budget = createBudget({
|
|
79
|
+
maxMetricCalls,
|
|
80
|
+
spent: resumeFrom?.metricCalls ?? 0
|
|
81
|
+
});
|
|
82
|
+
const evaluationCache = cache === false ? void 0 : cache ?? createMemoryCache();
|
|
83
|
+
const evaluator = createEvaluator({
|
|
84
|
+
adapter,
|
|
85
|
+
budget,
|
|
86
|
+
...retry === void 0 ? {} : { retry },
|
|
87
|
+
...cacheNamespace === void 0 ? {} : { cacheNamespace },
|
|
88
|
+
...evaluationCache === void 0 ? {} : { cache: evaluationCache },
|
|
89
|
+
trackOutputs: trackBestOutputs,
|
|
90
|
+
cacheHits: resumeFrom?.cacheHits ?? 0,
|
|
91
|
+
...signal === void 0 ? {} : { signal },
|
|
92
|
+
onEvaluation: (event) => onEvent?.({
|
|
93
|
+
type: "evaluation",
|
|
94
|
+
...event
|
|
95
|
+
})
|
|
96
|
+
});
|
|
97
|
+
evaluator.restore(resumeFrom?.cache ?? []);
|
|
98
|
+
const evaluated = [...resumeFrom?.candidates ?? []];
|
|
99
|
+
let drawn = resumeFrom?.drawn ?? 0;
|
|
100
|
+
let bootstrapMetricCalls = resumeFrom?.bootstrapMetricCalls ?? 0;
|
|
101
|
+
let stopReason = "candidatesExhausted";
|
|
102
|
+
onEvent?.({
|
|
103
|
+
type: "start",
|
|
104
|
+
components,
|
|
105
|
+
validationSetSize: validationSet.length
|
|
106
|
+
});
|
|
107
|
+
async function sweep(candidate, phase) {
|
|
108
|
+
return evaluator.evaluate({
|
|
109
|
+
candidate,
|
|
110
|
+
batch: validationSet,
|
|
111
|
+
ids: validationIds,
|
|
112
|
+
split: "val",
|
|
113
|
+
phase,
|
|
114
|
+
candidateId: null,
|
|
115
|
+
iteration: evaluated.length
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
const seedScore = resumeFrom?.seedScore ?? requireMeasuredMean({
|
|
119
|
+
batch: await sweep(seedCandidate, "seed"),
|
|
120
|
+
phase: "seed"
|
|
121
|
+
});
|
|
122
|
+
let best = resumeFrom?.best ?? seedCandidate;
|
|
123
|
+
let bestScore = resumeFrom?.bestScore ?? seedScore;
|
|
124
|
+
let bestOutputs;
|
|
125
|
+
function takeSnapshot() {
|
|
126
|
+
const cached = checkpointCache ? evaluationCache?.entries?.() : void 0;
|
|
127
|
+
return {
|
|
128
|
+
version: 1,
|
|
129
|
+
fingerprint,
|
|
130
|
+
candidates: [...evaluated],
|
|
131
|
+
best,
|
|
132
|
+
bestScore,
|
|
133
|
+
seedScore,
|
|
134
|
+
drawn,
|
|
135
|
+
metricCalls: budget.spent(),
|
|
136
|
+
bootstrapMetricCalls,
|
|
137
|
+
cacheHits: evaluator.cacheHits(),
|
|
138
|
+
rngState: rng.state(),
|
|
139
|
+
...cached === void 0 ? {} : { cache: cached }
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
async function checkpoint() {
|
|
143
|
+
if (onCheckpoint === void 0) return;
|
|
144
|
+
await onCheckpoint(takeSnapshot());
|
|
145
|
+
}
|
|
146
|
+
const plan = candidatePlan({
|
|
147
|
+
shuffledHarvests,
|
|
148
|
+
labeled: goldOutput !== void 0
|
|
149
|
+
});
|
|
150
|
+
for (; drawn < plan.length; drawn += 1) {
|
|
151
|
+
if (signal?.aborted) {
|
|
152
|
+
stopReason = "aborted";
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
155
|
+
if (costExhausted({
|
|
156
|
+
usage: evaluator.usage(),
|
|
157
|
+
maxCostUsd
|
|
158
|
+
})) {
|
|
159
|
+
stopReason = "costExhausted";
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
if (deadline.exceeded()) {
|
|
163
|
+
stopReason = "deadlineReached";
|
|
164
|
+
break;
|
|
165
|
+
}
|
|
166
|
+
if (!budget.canAfford(validationSet.length + 1)) {
|
|
167
|
+
stopReason = "budgetExhausted";
|
|
168
|
+
break;
|
|
169
|
+
}
|
|
170
|
+
const source = plan[drawn];
|
|
171
|
+
const block = await buildBlock(source);
|
|
172
|
+
const candidate = withDemos(block);
|
|
173
|
+
let evaluation;
|
|
174
|
+
try {
|
|
175
|
+
evaluation = await sweep(candidate, "validation");
|
|
176
|
+
} catch (err) {
|
|
177
|
+
if (err instanceof BudgetExhausted) {
|
|
178
|
+
stopReason = "budgetExhausted";
|
|
179
|
+
break;
|
|
180
|
+
}
|
|
181
|
+
if (signal?.aborted) {
|
|
182
|
+
stopReason = "aborted";
|
|
183
|
+
break;
|
|
184
|
+
}
|
|
185
|
+
throw err;
|
|
186
|
+
}
|
|
187
|
+
const score = measuredMean(evaluation);
|
|
188
|
+
if (score === void 0) continue;
|
|
189
|
+
const accepted = score > bestScore;
|
|
190
|
+
evaluated.push({
|
|
191
|
+
candidate,
|
|
192
|
+
source,
|
|
193
|
+
demos: countDemos(block),
|
|
194
|
+
score
|
|
195
|
+
});
|
|
196
|
+
onEvent?.({
|
|
197
|
+
type: "candidate",
|
|
198
|
+
index: evaluated.length - 1,
|
|
199
|
+
source,
|
|
200
|
+
demos: countDemos(block),
|
|
201
|
+
score,
|
|
202
|
+
accepted
|
|
203
|
+
});
|
|
204
|
+
if (accepted) {
|
|
205
|
+
best = candidate;
|
|
206
|
+
bestScore = score;
|
|
207
|
+
bestOutputs = evaluation.outputs;
|
|
208
|
+
}
|
|
209
|
+
await checkpoint();
|
|
210
|
+
if (stopAtScore !== void 0 && score >= stopAtScore) {
|
|
211
|
+
stopReason = "scoreReached";
|
|
212
|
+
drawn += 1;
|
|
213
|
+
break;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
if (signal?.aborted) stopReason = "aborted";
|
|
217
|
+
const testScore = testSet === void 0 ? void 0 : measuredMean(await evaluator.evaluate({
|
|
218
|
+
candidate: best,
|
|
219
|
+
batch: testSet,
|
|
220
|
+
ids: testIds,
|
|
221
|
+
split: "test",
|
|
222
|
+
phase: "test",
|
|
223
|
+
candidateId: null,
|
|
224
|
+
iteration: evaluated.length,
|
|
225
|
+
charge: false
|
|
226
|
+
}));
|
|
227
|
+
onEvent?.({
|
|
228
|
+
type: "finish",
|
|
229
|
+
reason: stopReason,
|
|
230
|
+
bestScore,
|
|
231
|
+
metricCalls: budget.spent(),
|
|
232
|
+
...testScore === void 0 ? {} : { testScore }
|
|
233
|
+
});
|
|
234
|
+
return {
|
|
235
|
+
bestCandidate: best,
|
|
236
|
+
bestScore,
|
|
237
|
+
usage: evaluator.usage(),
|
|
238
|
+
seedScore,
|
|
239
|
+
candidates: evaluated,
|
|
240
|
+
bootstrapMetricCalls,
|
|
241
|
+
cacheHits: evaluator.cacheHits(),
|
|
242
|
+
metricCalls: budget.spent(),
|
|
243
|
+
snapshot: takeSnapshot(),
|
|
244
|
+
...trackBestOutputs && bestOutputs !== void 0 ? { bestOutputs } : {},
|
|
245
|
+
...testScore === void 0 ? {} : {
|
|
246
|
+
testScore,
|
|
247
|
+
testMetricCalls: testSet?.length ?? 0
|
|
248
|
+
},
|
|
249
|
+
stopReason
|
|
250
|
+
};
|
|
251
|
+
function withDemos(block) {
|
|
252
|
+
const candidate = { ...seedCandidate };
|
|
253
|
+
for (const name of demoComponents) candidate[name] = block;
|
|
254
|
+
return candidate;
|
|
255
|
+
}
|
|
256
|
+
async function buildBlock(source) {
|
|
257
|
+
if (source === "zeroShot") return "";
|
|
258
|
+
if (source === "labeled") return labeledBlock();
|
|
259
|
+
const requested = source === "unshuffled" ? maxDemos : minDemos + rng.nextInt(Math.max(1, maxDemos - minDemos + 1));
|
|
260
|
+
const affordable = Math.min(trainingSet.length, budget.remaining() - validationSet.length);
|
|
261
|
+
if (affordable < 1) return "";
|
|
262
|
+
const harvest = await bootstrapDemos({
|
|
263
|
+
adapter,
|
|
264
|
+
candidate: seedCandidate,
|
|
265
|
+
trainingSet,
|
|
266
|
+
...demoMinScore === void 0 ? {} : { minScore: demoMinScore },
|
|
267
|
+
maxDemos: requested,
|
|
268
|
+
maxMetricCalls: affordable,
|
|
269
|
+
...source === "unshuffled" ? {} : { rng },
|
|
270
|
+
...renderDemo === void 0 ? {} : { renderDemo },
|
|
271
|
+
...signal === void 0 ? {} : { signal }
|
|
272
|
+
});
|
|
273
|
+
bootstrapMetricCalls += harvest.metricCalls;
|
|
274
|
+
budget.reserve(harvest.metricCalls);
|
|
275
|
+
return harvest.block;
|
|
276
|
+
}
|
|
277
|
+
function labeledBlock() {
|
|
278
|
+
const labelled = trainingSet.map((datum) => ({
|
|
279
|
+
input: datum,
|
|
280
|
+
output: goldOutput?.(datum)
|
|
281
|
+
})).filter((demo) => demo.output !== void 0).slice(0, maxLabeledDemos);
|
|
282
|
+
return formatDemos(labelled, renderDemo === void 0 ? {} : { render: renderDemo });
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* The order candidates are tried in, following DSPy's special seeds: zero-shot
|
|
287
|
+
* first, then labels-only, then one unshuffled full-size harvest, then the
|
|
288
|
+
* shuffled ones. Cheapest and most reliable first, so a run cut short by its
|
|
289
|
+
* budget still has the baseline it needs to report against.
|
|
290
|
+
*/
|
|
291
|
+
function candidatePlan(args) {
|
|
292
|
+
const { shuffledHarvests, labeled } = args;
|
|
293
|
+
return [
|
|
294
|
+
"zeroShot",
|
|
295
|
+
...labeled ? ["labeled"] : [],
|
|
296
|
+
"unshuffled",
|
|
297
|
+
...Array.from({ length: shuffledHarvests }, () => "bootstrapped")
|
|
298
|
+
];
|
|
299
|
+
}
|
|
300
|
+
function countDemos(block) {
|
|
301
|
+
return block.split("<demo>").length - 1;
|
|
302
|
+
}
|
|
303
|
+
function defaultInstanceId(args) {
|
|
304
|
+
return String(args.index);
|
|
305
|
+
}
|
|
306
|
+
//#endregion
|
|
307
|
+
export { BootstrapSearchOptimizer };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
//#region src/cache.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* What the cache stores per (candidate, instance): the metric the frontier is
|
|
4
|
+
* built from, plus the per-objective breakdown when the adapter reports one.
|
|
5
|
+
* Both come from the same rollout, so caching the score without the objectives
|
|
6
|
+
* would force a re-run to recover them.
|
|
7
|
+
*/
|
|
8
|
+
interface CachedScore {
|
|
9
|
+
score: number;
|
|
10
|
+
objectiveScores?: Record<string, number>;
|
|
11
|
+
}
|
|
12
|
+
interface EvaluationCache {
|
|
13
|
+
get(key: string): CachedScore | undefined;
|
|
14
|
+
set(key: string, cached: CachedScore): void;
|
|
15
|
+
/** Entries for checkpointing. Omit on caches that are already durable. */
|
|
16
|
+
entries?(): [string, CachedScore][];
|
|
17
|
+
}
|
|
18
|
+
declare function createMemoryCache(args?: {
|
|
19
|
+
maxEntries?: number;
|
|
20
|
+
/** Entries from a previous run's checkpoint. */
|
|
21
|
+
entries?: readonly [string, CachedScore][];
|
|
22
|
+
}): EvaluationCache;
|
|
23
|
+
//#endregion
|
|
24
|
+
export { EvaluationCache as n, createMemoryCache as r, CachedScore as t };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
//#region src/cache.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* What the cache stores per (candidate, instance): the metric the frontier is
|
|
4
|
+
* built from, plus the per-objective breakdown when the adapter reports one.
|
|
5
|
+
* Both come from the same rollout, so caching the score without the objectives
|
|
6
|
+
* would force a re-run to recover them.
|
|
7
|
+
*/
|
|
8
|
+
interface CachedScore {
|
|
9
|
+
score: number;
|
|
10
|
+
objectiveScores?: Record<string, number>;
|
|
11
|
+
}
|
|
12
|
+
interface EvaluationCache {
|
|
13
|
+
get(key: string): CachedScore | undefined;
|
|
14
|
+
set(key: string, cached: CachedScore): void;
|
|
15
|
+
/** Entries for checkpointing. Omit on caches that are already durable. */
|
|
16
|
+
entries?(): [string, CachedScore][];
|
|
17
|
+
}
|
|
18
|
+
declare function createMemoryCache(args?: {
|
|
19
|
+
maxEntries?: number;
|
|
20
|
+
/** Entries from a previous run's checkpoint. */
|
|
21
|
+
entries?: readonly [string, CachedScore][];
|
|
22
|
+
}): EvaluationCache;
|
|
23
|
+
//#endregion
|
|
24
|
+
export { EvaluationCache as n, createMemoryCache as r, CachedScore as t };
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
//#region src/concurrency.ts
|
|
2
|
+
/**
|
|
3
|
+
* Order-preserving concurrency-limited map. An optimization run's workload is
|
|
4
|
+
* entirely IO bound, so adapters use this to fan out a batch across a bounded
|
|
5
|
+
* number of in-flight model calls.
|
|
6
|
+
*/
|
|
7
|
+
async function mapWithConcurrency(args) {
|
|
8
|
+
const { items, limit, task, signal } = args;
|
|
9
|
+
if (items.length === 0) return [];
|
|
10
|
+
signal?.throwIfAborted();
|
|
11
|
+
const workerCount = Math.max(1, Math.min(Math.floor(limit), items.length));
|
|
12
|
+
const results = new Array(items.length);
|
|
13
|
+
let cursor = 0;
|
|
14
|
+
/**
|
|
15
|
+
* The first failure, kept rather than thrown so the workers that are already
|
|
16
|
+
* running settle before the caller gets control back. A rejection that left
|
|
17
|
+
* tasks running in the background would go on spending an optimizer's budget
|
|
18
|
+
* and writing its caches after the run it belonged to had ended.
|
|
19
|
+
*/
|
|
20
|
+
let failure;
|
|
21
|
+
async function worker() {
|
|
22
|
+
while (cursor < items.length && failure === void 0) {
|
|
23
|
+
const index = cursor;
|
|
24
|
+
cursor += 1;
|
|
25
|
+
try {
|
|
26
|
+
signal?.throwIfAborted();
|
|
27
|
+
results[index] = await task(items[index], index);
|
|
28
|
+
} catch (err) {
|
|
29
|
+
failure ??= { err };
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
|
35
|
+
if (failure !== void 0) throw failure.err;
|
|
36
|
+
return results;
|
|
37
|
+
}
|
|
38
|
+
//#endregion
|
|
39
|
+
Object.defineProperty(exports, "mapWithConcurrency", {
|
|
40
|
+
enumerable: true,
|
|
41
|
+
get: function() {
|
|
42
|
+
return mapWithConcurrency;
|
|
43
|
+
}
|
|
44
|
+
});
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
//#region src/concurrency.ts
|
|
2
|
+
/**
|
|
3
|
+
* Order-preserving concurrency-limited map. An optimization run's workload is
|
|
4
|
+
* entirely IO bound, so adapters use this to fan out a batch across a bounded
|
|
5
|
+
* number of in-flight model calls.
|
|
6
|
+
*/
|
|
7
|
+
async function mapWithConcurrency(args) {
|
|
8
|
+
const { items, limit, task, signal } = args;
|
|
9
|
+
if (items.length === 0) return [];
|
|
10
|
+
signal?.throwIfAborted();
|
|
11
|
+
const workerCount = Math.max(1, Math.min(Math.floor(limit), items.length));
|
|
12
|
+
const results = new Array(items.length);
|
|
13
|
+
let cursor = 0;
|
|
14
|
+
/**
|
|
15
|
+
* The first failure, kept rather than thrown so the workers that are already
|
|
16
|
+
* running settle before the caller gets control back. A rejection that left
|
|
17
|
+
* tasks running in the background would go on spending an optimizer's budget
|
|
18
|
+
* and writing its caches after the run it belonged to had ended.
|
|
19
|
+
*/
|
|
20
|
+
let failure;
|
|
21
|
+
async function worker() {
|
|
22
|
+
while (cursor < items.length && failure === void 0) {
|
|
23
|
+
const index = cursor;
|
|
24
|
+
cursor += 1;
|
|
25
|
+
try {
|
|
26
|
+
signal?.throwIfAborted();
|
|
27
|
+
results[index] = await task(items[index], index);
|
|
28
|
+
} catch (err) {
|
|
29
|
+
failure ??= { err };
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
|
35
|
+
if (failure !== void 0) throw failure.err;
|
|
36
|
+
return results;
|
|
37
|
+
}
|
|
38
|
+
//#endregion
|
|
39
|
+
export { mapWithConcurrency as t };
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { n as Candidate, t as Adapter } from "./types-CWv4IQFF.mjs";
|
|
2
|
+
import { t as Rng } from "./rng-BR5MOedA.mjs";
|
|
3
|
+
//#region src/demos.d.ts
|
|
4
|
+
/**
|
|
5
|
+
* One worked example: what went in, what a run of the system produced, and
|
|
6
|
+
* how that output scored. A demo is harvested, never written — its value is
|
|
7
|
+
* that the output is one the system actually produced and the metric actually
|
|
8
|
+
* rewarded.
|
|
9
|
+
*/
|
|
10
|
+
interface Demo<Datum = unknown, Output = unknown> {
|
|
11
|
+
input: Datum;
|
|
12
|
+
output: Output;
|
|
13
|
+
/** Present on harvested demos, absent on ones recovered from a block. */
|
|
14
|
+
score?: number;
|
|
15
|
+
}
|
|
16
|
+
type DemoRenderer<Datum = unknown, Output = unknown> = (args: {
|
|
17
|
+
demo: Demo<Datum, Output>;
|
|
18
|
+
index: number;
|
|
19
|
+
}) => string;
|
|
20
|
+
interface BootstrapResult<Datum, Output> {
|
|
21
|
+
demos: Demo<Datum, Output>[];
|
|
22
|
+
/** The demos as a candidate component, ready to seed a run with. */
|
|
23
|
+
block: string;
|
|
24
|
+
/** Rollouts this cost. Bootstrapping is cheap, not free. */
|
|
25
|
+
metricCalls: number;
|
|
26
|
+
attempted: number;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Harvest demonstrations by running a candidate over the training set and keeping
|
|
30
|
+
* the rollouts the metric rewarded.
|
|
31
|
+
*
|
|
32
|
+
* The cheapest signal in the whole library: a rollout that scored well is
|
|
33
|
+
* already paid for, and turning it into a few-shot block costs one pass over
|
|
34
|
+
* the data rather than a search. Instruction search and demonstrations pull on
|
|
35
|
+
* different parts of a model's behaviour — instructions on what to do,
|
|
36
|
+
* examples on what the output should look like — so a seed carrying both
|
|
37
|
+
* starts somewhere neither reaches alone.
|
|
38
|
+
*/
|
|
39
|
+
declare function bootstrapDemos<Datum, Trajectory, Output, K extends string = string>(args: {
|
|
40
|
+
adapter: Adapter<Datum, Trajectory, Output, K>;
|
|
41
|
+
/** The candidate to run. Usually the seed, sometimes a run's winner. */
|
|
42
|
+
candidate: Candidate<K>;
|
|
43
|
+
trainingSet: readonly Datum[];
|
|
44
|
+
/**
|
|
45
|
+
* Score a rollout must reach to be kept. Unset keeps every rollout the
|
|
46
|
+
* metric rewarded at all, which is what MIPROv2's bootstrapper does without
|
|
47
|
+
* a `metric_threshold`: it keeps a trace on any truthy score and only
|
|
48
|
+
* compares against a number once one is configured.
|
|
49
|
+
*
|
|
50
|
+
* Demanding a perfect score instead is the right call for a boolean metric
|
|
51
|
+
* and the wrong one for a graded metric, where it throws away every rollout
|
|
52
|
+
* that was most of the way there — which on a hard task is all of them.
|
|
53
|
+
*/
|
|
54
|
+
minScore?: number;
|
|
55
|
+
/** Demos to collect before stopping. Default 4. */
|
|
56
|
+
maxDemos?: number;
|
|
57
|
+
/**
|
|
58
|
+
* Instances per rollout batch. Smaller batches stop closer to the moment
|
|
59
|
+
* enough demos exist, at the cost of less concurrency inside the adapter.
|
|
60
|
+
*/
|
|
61
|
+
batchSize?: number;
|
|
62
|
+
/** Ceiling on rollouts. Defaults to one pass over the trainingSet. */
|
|
63
|
+
maxMetricCalls?: number;
|
|
64
|
+
/** Shuffles the trainingSet first, so demos are not all drawn from its head. */
|
|
65
|
+
rng?: Rng;
|
|
66
|
+
renderDemo?: DemoRenderer<Datum, Output>;
|
|
67
|
+
signal?: AbortSignal;
|
|
68
|
+
}): Promise<BootstrapResult<Datum, Output>>;
|
|
69
|
+
/**
|
|
70
|
+
* Render demos as the text a candidate component holds.
|
|
71
|
+
*
|
|
72
|
+
* Delimited rather than free-form so `parseDemos` can read them back: a demo
|
|
73
|
+
* component is edited over the course of a run, and a block that cannot be
|
|
74
|
+
* parsed can only be replaced wholesale, throwing away every example found
|
|
75
|
+
* before it.
|
|
76
|
+
*/
|
|
77
|
+
declare function formatDemos<Datum, Output>(demos: readonly Demo<Datum, Output>[], options?: {
|
|
78
|
+
render?: DemoRenderer<Datum, Output>;
|
|
79
|
+
}): string;
|
|
80
|
+
/**
|
|
81
|
+
* Recover the demos from a formatted block, ignoring anything written around
|
|
82
|
+
* them. Text a model rewrote and mangled yields the demos it left intact
|
|
83
|
+
* rather than throwing: a malformed example is worth less than the rest of the
|
|
84
|
+
* block, not more than it.
|
|
85
|
+
*/
|
|
86
|
+
declare function parseDemos(text: string): Demo[];
|
|
87
|
+
//#endregion
|
|
88
|
+
export { formatDemos as a, bootstrapDemos as i, Demo as n, parseDemos as o, DemoRenderer as r, BootstrapResult as t };
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
const require_evaluation = require("./evaluation-OZOp6TB7.cjs");
|
|
2
|
+
//#region src/demos.ts
|
|
3
|
+
const DEMO_OPEN = "<demo>";
|
|
4
|
+
const DEMO_CLOSE = "</demo>";
|
|
5
|
+
const DEMO_BLOCK = /<demo>\s*([\s\S]*?)\s*<\/demo>/g;
|
|
6
|
+
const DEMO_PARTS = /<input>\s*([\s\S]*?)\s*<\/input>\s*<output>\s*([\s\S]*?)\s*<\/output>/;
|
|
7
|
+
const DEFAULT_MAX_DEMOS = 4;
|
|
8
|
+
/**
|
|
9
|
+
* Harvest demonstrations by running a candidate over the training set and keeping
|
|
10
|
+
* the rollouts the metric rewarded.
|
|
11
|
+
*
|
|
12
|
+
* The cheapest signal in the whole library: a rollout that scored well is
|
|
13
|
+
* already paid for, and turning it into a few-shot block costs one pass over
|
|
14
|
+
* the data rather than a search. Instruction search and demonstrations pull on
|
|
15
|
+
* different parts of a model's behaviour — instructions on what to do,
|
|
16
|
+
* examples on what the output should look like — so a seed carrying both
|
|
17
|
+
* starts somewhere neither reaches alone.
|
|
18
|
+
*/
|
|
19
|
+
async function bootstrapDemos(args) {
|
|
20
|
+
const { adapter, candidate, trainingSet, minScore, maxDemos = DEFAULT_MAX_DEMOS, batchSize = maxDemos, maxMetricCalls = trainingSet.length, rng, renderDemo, signal } = args;
|
|
21
|
+
if (trainingSet.length === 0) throw new Error("bootstrapDemos requires a non-empty trainingSet");
|
|
22
|
+
const budget = require_evaluation.createBudget({ maxMetricCalls });
|
|
23
|
+
const evaluator = require_evaluation.createEvaluator({
|
|
24
|
+
adapter,
|
|
25
|
+
budget,
|
|
26
|
+
...signal === void 0 ? {} : { signal }
|
|
27
|
+
});
|
|
28
|
+
const order = rng === void 0 ? [...trainingSet] : rng.shuffle(trainingSet);
|
|
29
|
+
const demos = [];
|
|
30
|
+
let attempted = 0;
|
|
31
|
+
for (let start = 0; start < order.length; start += batchSize) {
|
|
32
|
+
if (demos.length >= maxDemos || signal?.aborted) break;
|
|
33
|
+
const batch = order.slice(start, start + Math.min(batchSize, budget.remaining()));
|
|
34
|
+
if (batch.length === 0) break;
|
|
35
|
+
const evaluation = await evaluator.evaluateTraced({
|
|
36
|
+
candidate,
|
|
37
|
+
batch,
|
|
38
|
+
split: "train",
|
|
39
|
+
phase: "seed",
|
|
40
|
+
candidateId: null,
|
|
41
|
+
iteration: 0
|
|
42
|
+
});
|
|
43
|
+
if (evaluation === null) break;
|
|
44
|
+
attempted += batch.length;
|
|
45
|
+
for (let index = 0; index < batch.length; index += 1) {
|
|
46
|
+
const score = evaluation.scores[index];
|
|
47
|
+
if (!(minScore === void 0 ? score > 0 : score >= minScore) || demos.length >= maxDemos) continue;
|
|
48
|
+
demos.push({
|
|
49
|
+
input: batch[index],
|
|
50
|
+
output: evaluation.outputs[index],
|
|
51
|
+
score
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
demos,
|
|
57
|
+
block: formatDemos(demos, renderDemo === void 0 ? {} : { render: renderDemo }),
|
|
58
|
+
metricCalls: budget.spent(),
|
|
59
|
+
attempted
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Render demos as the text a candidate component holds.
|
|
64
|
+
*
|
|
65
|
+
* Delimited rather than free-form so `parseDemos` can read them back: a demo
|
|
66
|
+
* component is edited over the course of a run, and a block that cannot be
|
|
67
|
+
* parsed can only be replaced wholesale, throwing away every example found
|
|
68
|
+
* before it.
|
|
69
|
+
*/
|
|
70
|
+
function formatDemos(demos, options = {}) {
|
|
71
|
+
const { render = renderDefault } = options;
|
|
72
|
+
if (demos.length === 0) return "";
|
|
73
|
+
return demos.map((demo, index) => `${DEMO_OPEN}\n${render({
|
|
74
|
+
demo,
|
|
75
|
+
index
|
|
76
|
+
})}\n${DEMO_CLOSE}`).join("\n");
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Recover the demos from a formatted block, ignoring anything written around
|
|
80
|
+
* them. Text a model rewrote and mangled yields the demos it left intact
|
|
81
|
+
* rather than throwing: a malformed example is worth less than the rest of the
|
|
82
|
+
* block, not more than it.
|
|
83
|
+
*/
|
|
84
|
+
function parseDemos(text) {
|
|
85
|
+
const demos = [];
|
|
86
|
+
for (const match of text.matchAll(DEMO_BLOCK)) {
|
|
87
|
+
const parts = (match[1] ?? "").match(DEMO_PARTS);
|
|
88
|
+
if (parts === null) continue;
|
|
89
|
+
const input = parseValue(parts[1] ?? "");
|
|
90
|
+
const output = parseValue(parts[2] ?? "");
|
|
91
|
+
demos.push({
|
|
92
|
+
input,
|
|
93
|
+
output
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
return demos;
|
|
97
|
+
}
|
|
98
|
+
function renderDefault(args) {
|
|
99
|
+
const { demo } = args;
|
|
100
|
+
return [
|
|
101
|
+
"<input>",
|
|
102
|
+
serialize(demo.input),
|
|
103
|
+
"</input>",
|
|
104
|
+
"<output>",
|
|
105
|
+
serialize(demo.output),
|
|
106
|
+
"</output>"
|
|
107
|
+
].join("\n");
|
|
108
|
+
}
|
|
109
|
+
/** Strings stay as they are; anything else is shown as JSON. */
|
|
110
|
+
function serialize(value) {
|
|
111
|
+
if (typeof value === "string") return value;
|
|
112
|
+
try {
|
|
113
|
+
return JSON.stringify(value, null, 2) ?? String(value);
|
|
114
|
+
} catch {
|
|
115
|
+
return String(value);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
function parseValue(text) {
|
|
119
|
+
try {
|
|
120
|
+
return JSON.parse(text);
|
|
121
|
+
} catch {
|
|
122
|
+
return text;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
//#endregion
|
|
126
|
+
Object.defineProperty(exports, "bootstrapDemos", {
|
|
127
|
+
enumerable: true,
|
|
128
|
+
get: function() {
|
|
129
|
+
return bootstrapDemos;
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
Object.defineProperty(exports, "formatDemos", {
|
|
133
|
+
enumerable: true,
|
|
134
|
+
get: function() {
|
|
135
|
+
return formatDemos;
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
Object.defineProperty(exports, "parseDemos", {
|
|
139
|
+
enumerable: true,
|
|
140
|
+
get: function() {
|
|
141
|
+
return parseDemos;
|
|
142
|
+
}
|
|
143
|
+
});
|