textopt 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +509 -0
- package/dist/bootstrap-search/index.cjs +308 -0
- package/dist/bootstrap-search/index.d.cts +162 -0
- package/dist/bootstrap-search/index.d.mts +162 -0
- package/dist/bootstrap-search/index.mjs +307 -0
- package/dist/cache-CuSo0NJ8.d.cts +24 -0
- package/dist/cache-CuSo0NJ8.d.mts +24 -0
- package/dist/concurrency-C-cFzWW2.cjs +44 -0
- package/dist/concurrency-D58PWeSk.mjs +39 -0
- package/dist/demos-B0pVQjYC.d.mts +88 -0
- package/dist/demos-B9BJiNKz.cjs +143 -0
- package/dist/demos-BTuzFNsp.d.cts +88 -0
- package/dist/demos-Degx6UmP.mjs +126 -0
- package/dist/evaluation-BV0nSZVx.mjs +521 -0
- package/dist/evaluation-OZOp6TB7.cjs +598 -0
- package/dist/file-cache.cjs +70 -0
- package/dist/file-cache.d.cts +21 -0
- package/dist/file-cache.d.mts +21 -0
- package/dist/file-cache.mjs +69 -0
- package/dist/gepa/index.cjs +1671 -0
- package/dist/gepa/index.d.cts +385 -0
- package/dist/gepa/index.d.mts +385 -0
- package/dist/gepa/index.mjs +1652 -0
- package/dist/index.cjs +266 -0
- package/dist/index.d.cts +221 -0
- package/dist/index.d.mts +221 -0
- package/dist/index.mjs +245 -0
- package/dist/math-COOofUyv.cjs +101 -0
- package/dist/math-DhrDmpFS.mjs +78 -0
- package/dist/mipro/index.cjs +739 -0
- package/dist/mipro/index.d.cts +372 -0
- package/dist/mipro/index.d.mts +372 -0
- package/dist/mipro/index.mjs +736 -0
- package/dist/opro/index.cjs +487 -0
- package/dist/opro/index.d.cts +230 -0
- package/dist/opro/index.d.mts +230 -0
- package/dist/opro/index.mjs +485 -0
- package/dist/optimizer-B7SpRwl7.d.cts +288 -0
- package/dist/optimizer-DqCoth_w.d.mts +288 -0
- package/dist/random-search/index.cjs +321 -0
- package/dist/random-search/index.d.cts +156 -0
- package/dist/random-search/index.d.mts +156 -0
- package/dist/random-search/index.mjs +319 -0
- package/dist/reflection-CQToe-5B.d.cts +283 -0
- package/dist/reflection-Cr_upzU0.d.mts +283 -0
- package/dist/reflection-DRfbk6hu.cjs +249 -0
- package/dist/reflection-mwMhrjs_.mjs +214 -0
- package/dist/rng-BR5MOedA.d.cts +22 -0
- package/dist/rng-BR5MOedA.d.mts +22 -0
- package/dist/rng-DbA_rPIo.cjs +67 -0
- package/dist/rng-Dtc5eZ_W.mjs +62 -0
- package/dist/sampling-CfHt7Gue.mjs +59 -0
- package/dist/sampling-DFo_7RNJ.d.mts +23 -0
- package/dist/sampling-Dars7ctR.cjs +64 -0
- package/dist/sampling-axOwfZf5.d.cts +23 -0
- package/dist/simba/index.cjs +709 -0
- package/dist/simba/index.d.cts +289 -0
- package/dist/simba/index.d.mts +289 -0
- package/dist/simba/index.mjs +700 -0
- package/dist/testing.cjs +155 -0
- package/dist/testing.d.cts +53 -0
- package/dist/testing.d.mts +53 -0
- package/dist/testing.mjs +148 -0
- package/dist/text--v4Ffbus.mjs +21 -0
- package/dist/text-CK_HB3su.cjs +26 -0
- package/dist/types-CWv4IQFF.d.cts +129 -0
- package/dist/types-CWv4IQFF.d.mts +129 -0
- package/package.json +135 -0
|
@@ -0,0 +1,1652 @@
|
|
|
1
|
+
import { c as candidateFingerprint, d as createMemoryCache, f as stableHash, i as measuredMean, l as runFingerprint, n as costExhausted, o as createDeadline, p as createBudget, r as createEvaluator, s as assertResumable, t as BudgetExhausted, u as componentNames } from "../evaluation-BV0nSZVx.mjs";
|
|
2
|
+
import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
|
|
3
|
+
import { i as sum, n as mean, r as signFlipPValue, t as argmax } from "../math-DhrDmpFS.mjs";
|
|
4
|
+
import { n as formatDemos, r as parseDemos } from "../demos-Degx6UmP.mjs";
|
|
5
|
+
import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
|
|
6
|
+
import { a as createDefaultProposer, i as buildSimplifyPrompt, n as buildReflectionPrompt, o as diverseReflectionStrategies, r as buildRewritePrompt, t as buildGeneralizePrompt } from "../reflection-mwMhrjs_.mjs";
|
|
7
|
+
import { t as createEpochShuffledSampler } from "../sampling-CfHt7Gue.mjs";
|
|
8
|
+
//#region src/gepa/demos.ts
|
|
9
|
+
const DEFAULT_MAX_DEMOS = 4;
|
|
10
|
+
const DEFAULT_MIN_SCORE = 1;
|
|
11
|
+
/**
|
|
12
|
+
* A proposer that fills demonstration components from rollouts the run has
|
|
13
|
+
* already paid for.
|
|
14
|
+
*
|
|
15
|
+
* The reflective dataset carries every minibatch rollout's input, output and
|
|
16
|
+
* score, so the successful ones are a few-shot block sitting in memory. Taking
|
|
17
|
+
* them costs no rollout and no reflection call — the search buys demos as a
|
|
18
|
+
* side effect of the evaluations it was making anyway.
|
|
19
|
+
*
|
|
20
|
+
* A proposal appends to the block its parent already holds rather than
|
|
21
|
+
* replacing it: a block built from one minibatch alone would hold three or
|
|
22
|
+
* four examples and forget every earlier one. Accumulation therefore follows
|
|
23
|
+
* the accepted lineage — a demo only persists if the candidate carrying it
|
|
24
|
+
* beat its parent, which is the same bar every other component is held to.
|
|
25
|
+
*/
|
|
26
|
+
function createDemoProposer(args) {
|
|
27
|
+
const { components, minScore = DEFAULT_MIN_SCORE, maxDemos = DEFAULT_MAX_DEMOS, render, fallback = createDefaultProposer() } = args;
|
|
28
|
+
if (components.length === 0) throw new Error("createDemoProposer requires at least one component");
|
|
29
|
+
const demoComponents = new Set(components);
|
|
30
|
+
return async (proposeArgs) => {
|
|
31
|
+
const { candidate, reflectiveDataset, componentsToUpdate } = proposeArgs;
|
|
32
|
+
const demoTargets = componentsToUpdate.filter((name) => demoComponents.has(name));
|
|
33
|
+
const others = componentsToUpdate.filter((name) => !demoComponents.has(name));
|
|
34
|
+
const patch = others.length === 0 ? {} : await fallback({
|
|
35
|
+
...proposeArgs,
|
|
36
|
+
componentsToUpdate: others
|
|
37
|
+
});
|
|
38
|
+
for (const name of demoTargets) {
|
|
39
|
+
const harvested = harvestDemos({
|
|
40
|
+
records: reflectiveDataset[name] ?? [],
|
|
41
|
+
minScore
|
|
42
|
+
});
|
|
43
|
+
if (harvested.length === 0) continue;
|
|
44
|
+
const kept = mergeDemos({
|
|
45
|
+
existing: parseDemos(candidate[name] ?? ""),
|
|
46
|
+
harvested,
|
|
47
|
+
maxDemos
|
|
48
|
+
});
|
|
49
|
+
const block = formatDemos(kept, render === void 0 ? {} : { render });
|
|
50
|
+
if (block !== candidate[name]) patch[name] = block;
|
|
51
|
+
}
|
|
52
|
+
return patch;
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
function harvestDemos(args) {
|
|
56
|
+
const { records, minScore } = args;
|
|
57
|
+
return records.filter((record) => (record.score ?? Number.NEGATIVE_INFINITY) >= minScore).map((record) => ({
|
|
58
|
+
input: record.inputs,
|
|
59
|
+
output: record.generatedOutputs,
|
|
60
|
+
score: record.score
|
|
61
|
+
}));
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Newest wins on overflow. A demo harvested later came from a stronger
|
|
65
|
+
* candidate, since a weaker one would not have scored highly enough to be
|
|
66
|
+
* harvested at all — so the tail of the block is the better end of it.
|
|
67
|
+
*/
|
|
68
|
+
function mergeDemos(args) {
|
|
69
|
+
const { existing, harvested, maxDemos } = args;
|
|
70
|
+
const merged = [...existing];
|
|
71
|
+
const seen = new Set(existing.map((demo) => keyOf(demo.input)));
|
|
72
|
+
for (const demo of harvested) {
|
|
73
|
+
const key = keyOf(demo.input);
|
|
74
|
+
if (seen.has(key)) continue;
|
|
75
|
+
seen.add(key);
|
|
76
|
+
merged.push(demo);
|
|
77
|
+
}
|
|
78
|
+
return merged.slice(Math.max(0, merged.length - maxDemos));
|
|
79
|
+
}
|
|
80
|
+
function keyOf(input) {
|
|
81
|
+
try {
|
|
82
|
+
return JSON.stringify(input) ?? String(input);
|
|
83
|
+
} catch {
|
|
84
|
+
return String(input);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
//#endregion
|
|
88
|
+
//#region src/gepa/pipeline.ts
|
|
89
|
+
const DEFAULT_CONCURRENCY = 1;
|
|
90
|
+
/**
|
|
91
|
+
* A GEPA adapter for a system built from several modules in sequence, where
|
|
92
|
+
* each module's instruction is its own candidate component.
|
|
93
|
+
*
|
|
94
|
+
* The work this saves is attribution. Reflection on a multi-module system is
|
|
95
|
+
* only as good as the evidence it sees, and the evidence a module needs is what
|
|
96
|
+
* *it* received and produced — not the pipeline's input and final answer, which
|
|
97
|
+
* is what an adapter written in a hurry ends up showing every component.
|
|
98
|
+
*
|
|
99
|
+
* The feedback is end-to-end, and every module sees the same string. That is
|
|
100
|
+
* the honest default rather than a shortcut: a metric scores the final output,
|
|
101
|
+
* so nothing in a score alone says which module lost the point. A caller who
|
|
102
|
+
* can attribute better should score the steps themselves — `score` is handed
|
|
103
|
+
* the whole trace for exactly that.
|
|
104
|
+
*
|
|
105
|
+
* Errors from a module are not caught. A helper cannot tell a rate limit from a
|
|
106
|
+
* bug in a module, and guessing wrong either buries the bug or fails the run
|
|
107
|
+
* over a blip. Classify inside `run` and return a transient `ScoreResult` from
|
|
108
|
+
* `score`, or let the optimizer's `raiseOnError` decide.
|
|
109
|
+
*/
|
|
110
|
+
function createPipelineAdapter(args) {
|
|
111
|
+
const { modules, input = (datum) => datum, score, concurrency = DEFAULT_CONCURRENCY } = args;
|
|
112
|
+
if (modules.length === 0) throw new Error("createPipelineAdapter requires at least one module");
|
|
113
|
+
return {
|
|
114
|
+
evaluate: async ({ batch, candidate, captureTraces, signal }) => {
|
|
115
|
+
const rollouts = await mapWithConcurrency({
|
|
116
|
+
items: batch,
|
|
117
|
+
limit: concurrency,
|
|
118
|
+
signal,
|
|
119
|
+
task: async (datum) => {
|
|
120
|
+
const trace = await runPipeline({
|
|
121
|
+
modules,
|
|
122
|
+
candidate,
|
|
123
|
+
datum,
|
|
124
|
+
input,
|
|
125
|
+
signal
|
|
126
|
+
});
|
|
127
|
+
const output = trace.steps.at(-1).output;
|
|
128
|
+
return {
|
|
129
|
+
trace,
|
|
130
|
+
output,
|
|
131
|
+
scored: await score({
|
|
132
|
+
datum,
|
|
133
|
+
output,
|
|
134
|
+
steps: trace.steps
|
|
135
|
+
})
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
const batchResult = {
|
|
140
|
+
outputs: rollouts.map((rollout) => rollout.output),
|
|
141
|
+
scores: rollouts.map((rollout) => rollout.scored.score),
|
|
142
|
+
feedback: rollouts.map((rollout) => rollout.scored.feedback ?? ""),
|
|
143
|
+
usage: rollouts.map((rollout) => rollout.scored.usage ?? {}),
|
|
144
|
+
transient: rollouts.map((rollout) => rollout.scored.transient ?? false),
|
|
145
|
+
objectiveScores: rollouts.map((rollout) => rollout.scored.objectiveScores ?? {})
|
|
146
|
+
};
|
|
147
|
+
return captureTraces ? {
|
|
148
|
+
...batchResult,
|
|
149
|
+
trajectories: rollouts.map((r) => r.trace)
|
|
150
|
+
} : batchResult;
|
|
151
|
+
},
|
|
152
|
+
makeReflectiveDataset: ({ evaluation, componentsToUpdate }) => {
|
|
153
|
+
const dataset = {};
|
|
154
|
+
for (const component of componentsToUpdate) dataset[component] = recordsFor({
|
|
155
|
+
component,
|
|
156
|
+
evaluation
|
|
157
|
+
});
|
|
158
|
+
return dataset;
|
|
159
|
+
}
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
async function runPipeline(args) {
|
|
163
|
+
const { modules, candidate, datum, input, signal } = args;
|
|
164
|
+
const steps = [];
|
|
165
|
+
let carried = input(datum);
|
|
166
|
+
for (const module of modules) {
|
|
167
|
+
const output = await module.run({
|
|
168
|
+
instruction: candidate[module.component],
|
|
169
|
+
input: carried,
|
|
170
|
+
datum,
|
|
171
|
+
signal
|
|
172
|
+
});
|
|
173
|
+
steps.push({
|
|
174
|
+
component: module.component,
|
|
175
|
+
input: carried,
|
|
176
|
+
output
|
|
177
|
+
});
|
|
178
|
+
carried = output;
|
|
179
|
+
}
|
|
180
|
+
return { steps };
|
|
181
|
+
}
|
|
182
|
+
function recordsFor(args) {
|
|
183
|
+
const { component, evaluation } = args;
|
|
184
|
+
return (evaluation.trajectories ?? []).map((trace, index) => {
|
|
185
|
+
const step = trace.steps.find((entry) => entry.component === component);
|
|
186
|
+
return {
|
|
187
|
+
inputs: step?.input,
|
|
188
|
+
generatedOutputs: step?.output,
|
|
189
|
+
feedback: evaluation.feedback?.[index] ?? "",
|
|
190
|
+
score: evaluation.scores[index],
|
|
191
|
+
evidence: { component }
|
|
192
|
+
};
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
//#endregion
|
|
196
|
+
//#region src/gepa/merge.ts
|
|
197
|
+
const DEFAULT_MAX_ATTEMPTS = 10;
|
|
198
|
+
const DEFAULT_VAL_OVERLAP_FLOOR = 5;
|
|
199
|
+
/**
|
|
200
|
+
* System-aware merge (GEPA's crossover). Two dominator lineages that descend
|
|
201
|
+
* from a common ancestor neither of them regressed against can be recombined
|
|
202
|
+
* for free: for every component, take the version from whichever descendant
|
|
203
|
+
* actually moved it. Sampling is random rather than greedy — always merging the
|
|
204
|
+
* two strongest lineages collapses the frontier's diversity, which is the thing
|
|
205
|
+
* merge exists to exploit.
|
|
206
|
+
*
|
|
207
|
+
* Returns a proposal to be *tested*, not an accepted candidate: the caller
|
|
208
|
+
* still has to score it.
|
|
209
|
+
*/
|
|
210
|
+
function proposeMerge(args) {
|
|
211
|
+
const { records, pool, rng, attempted, attemptedDescriptions, maxAttempts = DEFAULT_MAX_ATTEMPTS, valOverlapFloor = DEFAULT_VAL_OVERLAP_FLOOR } = args;
|
|
212
|
+
if (pool.length < 2 || records.length < 3) return null;
|
|
213
|
+
const ancestries = buildAncestries(records);
|
|
214
|
+
const existing = new Set(records.map((record) => fingerprint(record.candidate)));
|
|
215
|
+
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
216
|
+
const triplet = sampleTriplet({
|
|
217
|
+
records,
|
|
218
|
+
pool,
|
|
219
|
+
rng,
|
|
220
|
+
attempted,
|
|
221
|
+
ancestries,
|
|
222
|
+
maxAttempts,
|
|
223
|
+
valOverlapFloor
|
|
224
|
+
});
|
|
225
|
+
if (triplet === null) continue;
|
|
226
|
+
const { parentIds, ancestorId } = triplet;
|
|
227
|
+
const merged = mergeComponents({
|
|
228
|
+
records,
|
|
229
|
+
parentIds,
|
|
230
|
+
ancestorId,
|
|
231
|
+
rng
|
|
232
|
+
});
|
|
233
|
+
const descriptionKey = `${parentIds[0]}:${parentIds[1]}:${merged.sources.join(",")}`;
|
|
234
|
+
if (attemptedDescriptions.has(descriptionKey) || existing.has(fingerprint(merged.candidate))) continue;
|
|
235
|
+
return {
|
|
236
|
+
candidate: merged.candidate,
|
|
237
|
+
parentIds,
|
|
238
|
+
ancestorId,
|
|
239
|
+
attemptKey: `${parentIds[0]}:${parentIds[1]}:${ancestorId}`,
|
|
240
|
+
descriptionKey
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
return null;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Picks the validation instances a merge is judged on: up to `size` ids, drawn
|
|
247
|
+
* evenly from the instances each parent uniquely wins plus the ones they tie
|
|
248
|
+
* on. A uniform sample would usually miss the handful of instances that
|
|
249
|
+
* distinguish the parents at all, which is exactly where a merge either pays
|
|
250
|
+
* off or breaks.
|
|
251
|
+
*/
|
|
252
|
+
function selectMergeSubsample(args) {
|
|
253
|
+
const { scores1, scores2, rng, size = 5 } = args;
|
|
254
|
+
const ids = scores1.map((_, index) => index).filter((index) => scores1[index] !== void 0 && scores2[index] !== void 0);
|
|
255
|
+
if (ids.length === 0) return [];
|
|
256
|
+
const firstWins = ids.filter((id) => scores1[id] > scores2[id]);
|
|
257
|
+
const secondWins = ids.filter((id) => scores2[id] > scores1[id]);
|
|
258
|
+
const ties = ids.filter((id) => !firstWins.includes(id) && !secondWins.includes(id));
|
|
259
|
+
const perBucket = Math.max(1, Math.ceil(size / 3));
|
|
260
|
+
const selected = [];
|
|
261
|
+
for (const bucket of [
|
|
262
|
+
firstWins,
|
|
263
|
+
secondWins,
|
|
264
|
+
ties
|
|
265
|
+
]) {
|
|
266
|
+
if (selected.length >= size) break;
|
|
267
|
+
const available = bucket.filter((id) => !selected.includes(id));
|
|
268
|
+
const take = Math.min(available.length, perBucket, size - selected.length);
|
|
269
|
+
if (take > 0) selected.push(...rng.sample(available, take));
|
|
270
|
+
}
|
|
271
|
+
const remaining = size - selected.length;
|
|
272
|
+
if (remaining > 0) {
|
|
273
|
+
const unused = ids.filter((id) => !selected.includes(id));
|
|
274
|
+
if (unused.length >= remaining) selected.push(...rng.sample(unused, remaining));
|
|
275
|
+
else for (let index = 0; index < remaining; index += 1) selected.push(rng.pick(ids));
|
|
276
|
+
}
|
|
277
|
+
return selected.slice(0, size);
|
|
278
|
+
}
|
|
279
|
+
function sampleTriplet(args) {
|
|
280
|
+
const { records, pool, rng, attempted, ancestries, maxAttempts, valOverlapFloor } = args;
|
|
281
|
+
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
282
|
+
const [first, second] = rng.sample(pool, 2);
|
|
283
|
+
if (first === second) continue;
|
|
284
|
+
const left = Math.min(first, second);
|
|
285
|
+
const right = Math.max(first, second);
|
|
286
|
+
const leftAncestry = ancestries[left];
|
|
287
|
+
const rightAncestry = ancestries[right];
|
|
288
|
+
if (leftAncestry.has(right) || rightAncestry.has(left)) continue;
|
|
289
|
+
if (overlap({
|
|
290
|
+
records,
|
|
291
|
+
left,
|
|
292
|
+
right
|
|
293
|
+
}) < valOverlapFloor) continue;
|
|
294
|
+
const eligible = [...leftAncestry].filter((id) => rightAncestry.has(id)).filter((ancestorId) => isEligibleAncestor({
|
|
295
|
+
records,
|
|
296
|
+
left,
|
|
297
|
+
right,
|
|
298
|
+
ancestorId,
|
|
299
|
+
attempted
|
|
300
|
+
}));
|
|
301
|
+
if (eligible.length === 0) continue;
|
|
302
|
+
return {
|
|
303
|
+
parentIds: [left, right],
|
|
304
|
+
ancestorId: rng.weighted(eligible, eligible.map((id) => records[id].aggregateScore))
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
return null;
|
|
308
|
+
}
|
|
309
|
+
function overlap(args) {
|
|
310
|
+
const leftScores = args.records[args.left].instanceScores;
|
|
311
|
+
const rightScores = args.records[args.right].instanceScores;
|
|
312
|
+
let shared = 0;
|
|
313
|
+
for (let index = 0; index < leftScores.length; index += 1) if (leftScores[index] !== void 0 && rightScores[index] !== void 0) shared += 1;
|
|
314
|
+
return shared;
|
|
315
|
+
}
|
|
316
|
+
function isEligibleAncestor(args) {
|
|
317
|
+
const { records, left, right, ancestorId, attempted } = args;
|
|
318
|
+
if (attempted.has(`${left}:${right}:${ancestorId}`)) return false;
|
|
319
|
+
const ancestor = records[ancestorId];
|
|
320
|
+
const leftRecord = records[left];
|
|
321
|
+
const rightRecord = records[right];
|
|
322
|
+
if (ancestor.aggregateScore > leftRecord.aggregateScore || ancestor.aggregateScore > rightRecord.aggregateScore) return false;
|
|
323
|
+
return hasComplementaryComponent({
|
|
324
|
+
ancestor: ancestor.candidate,
|
|
325
|
+
left: leftRecord.candidate,
|
|
326
|
+
right: rightRecord.candidate
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* True when at least one component was changed by exactly one descendant. Two
|
|
331
|
+
* lineages that rewrote every shared component give the merge nothing to
|
|
332
|
+
* attribute an improvement to.
|
|
333
|
+
*/
|
|
334
|
+
function hasComplementaryComponent(args) {
|
|
335
|
+
const { ancestor, left, right } = args;
|
|
336
|
+
return componentNames(ancestor).some((name) => {
|
|
337
|
+
const base = ancestor[name];
|
|
338
|
+
return (base === left[name] || base === right[name]) && left[name] !== right[name];
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
function mergeComponents(args) {
|
|
342
|
+
const { records, parentIds, ancestorId, rng } = args;
|
|
343
|
+
const [leftId, rightId] = parentIds;
|
|
344
|
+
const ancestor = records[ancestorId].candidate;
|
|
345
|
+
const leftRecord = records[leftId];
|
|
346
|
+
const rightRecord = records[rightId];
|
|
347
|
+
const candidate = { ...ancestor };
|
|
348
|
+
const sources = [];
|
|
349
|
+
for (const name of componentNames(ancestor)) {
|
|
350
|
+
const base = ancestor[name];
|
|
351
|
+
const leftText = leftRecord.candidate[name];
|
|
352
|
+
const rightText = rightRecord.candidate[name];
|
|
353
|
+
const sourceId = resolveComponentSource({
|
|
354
|
+
base,
|
|
355
|
+
leftText,
|
|
356
|
+
rightText,
|
|
357
|
+
leftRecord,
|
|
358
|
+
rightRecord,
|
|
359
|
+
rng
|
|
360
|
+
});
|
|
361
|
+
candidate[name] = records[sourceId].candidate[name];
|
|
362
|
+
sources.push(sourceId);
|
|
363
|
+
}
|
|
364
|
+
return {
|
|
365
|
+
candidate,
|
|
366
|
+
sources
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
function resolveComponentSource(args) {
|
|
370
|
+
const { base, leftText, rightText, leftRecord, rightRecord, rng } = args;
|
|
371
|
+
if ((base === leftText || base === rightText) && leftText !== rightText) return base === leftText ? rightRecord.id : leftRecord.id;
|
|
372
|
+
if (base !== leftText && base !== rightText) {
|
|
373
|
+
if (leftRecord.aggregateScore > rightRecord.aggregateScore) return leftRecord.id;
|
|
374
|
+
if (rightRecord.aggregateScore > leftRecord.aggregateScore) return rightRecord.id;
|
|
375
|
+
return rng.pick([leftRecord.id, rightRecord.id]);
|
|
376
|
+
}
|
|
377
|
+
return leftRecord.id;
|
|
378
|
+
}
|
|
379
|
+
/** Strict ancestors of each record, keyed by record id. */
|
|
380
|
+
function buildAncestries(records) {
|
|
381
|
+
const ancestries = [];
|
|
382
|
+
for (const record of records) {
|
|
383
|
+
const ancestry = /* @__PURE__ */ new Set();
|
|
384
|
+
for (const parentId of record.parentIds) {
|
|
385
|
+
ancestry.add(parentId);
|
|
386
|
+
for (const id of ancestries[parentId] ?? []) ancestry.add(id);
|
|
387
|
+
}
|
|
388
|
+
ancestries[record.id] = ancestry;
|
|
389
|
+
}
|
|
390
|
+
return ancestries;
|
|
391
|
+
}
|
|
392
|
+
function fingerprint(candidate) {
|
|
393
|
+
return JSON.stringify(componentNames(candidate).sort().map((name) => [name, candidate[name]]));
|
|
394
|
+
}
|
|
395
|
+
//#endregion
|
|
396
|
+
//#region src/gepa/pareto.ts
|
|
397
|
+
function computeInstanceBests(scoreMatrix) {
|
|
398
|
+
const first = scoreMatrix[0];
|
|
399
|
+
if (first === void 0) return [];
|
|
400
|
+
const bests = first.map((score) => score ?? Number.NEGATIVE_INFINITY);
|
|
401
|
+
for (let candidate = 1; candidate < scoreMatrix.length; candidate += 1) {
|
|
402
|
+
const row = scoreMatrix[candidate];
|
|
403
|
+
for (let instance = 0; instance < bests.length; instance += 1) {
|
|
404
|
+
const score = row[instance] ?? Number.NEGATIVE_INFINITY;
|
|
405
|
+
if (score > bests[instance]) bests[instance] = score;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
return bests;
|
|
409
|
+
}
|
|
410
|
+
/**
|
|
411
|
+
* For each instance, the set of candidates achieving the best score on it.
|
|
412
|
+
* Candidates within `epsilon` of the best count as tied, which keeps noisy
|
|
413
|
+
* metrics from collapsing the frontier to a single lineage.
|
|
414
|
+
*/
|
|
415
|
+
function buildInstanceFronts(args) {
|
|
416
|
+
const { scoreMatrix, epsilon = 0 } = args;
|
|
417
|
+
return computeInstanceBests(scoreMatrix).map((best, instance) => {
|
|
418
|
+
const front = /* @__PURE__ */ new Set();
|
|
419
|
+
for (let candidate = 0; candidate < scoreMatrix.length; candidate += 1) {
|
|
420
|
+
const score = scoreMatrix[candidate][instance];
|
|
421
|
+
if (score !== void 0 && score >= best - epsilon) front.add(candidate);
|
|
422
|
+
}
|
|
423
|
+
return front;
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
/** The best value any candidate reached on each objective. */
|
|
427
|
+
function objectiveBests(objectiveScores) {
|
|
428
|
+
const bests = {};
|
|
429
|
+
for (const scores of objectiveScores) for (const [objective, value] of Object.entries(scores ?? {})) {
|
|
430
|
+
const best = bests[objective];
|
|
431
|
+
if (best === void 0 || value > best) bests[objective] = value;
|
|
432
|
+
}
|
|
433
|
+
return bests;
|
|
434
|
+
}
|
|
435
|
+
/**
|
|
436
|
+
* The objective-wise counterpart of `buildInstanceFronts`: one front per named
|
|
437
|
+
* objective, holding the candidates that lead it. Optimizing a system against
|
|
438
|
+
* several metrics at once — accuracy against cost, quality against latency —
|
|
439
|
+
* means the interesting candidates are the ones that lead *an* objective, which
|
|
440
|
+
* an average over instances hides.
|
|
441
|
+
*/
|
|
442
|
+
function buildObjectiveFronts(args) {
|
|
443
|
+
const { objectiveScores, epsilon = 0 } = args;
|
|
444
|
+
const bests = objectiveBests(objectiveScores);
|
|
445
|
+
return Object.entries(bests).map(([objective, best]) => {
|
|
446
|
+
const front = /* @__PURE__ */ new Set();
|
|
447
|
+
for (let candidate = 0; candidate < objectiveScores.length; candidate += 1) {
|
|
448
|
+
const value = objectiveScores[candidate]?.[objective];
|
|
449
|
+
if (value !== void 0 && value >= best - epsilon) front.add(candidate);
|
|
450
|
+
}
|
|
451
|
+
return front;
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
/**
|
|
455
|
+
* Drop candidates that contribute nothing unique: a candidate is dominated when
|
|
456
|
+
* every instance it wins is also won by some surviving candidate. Candidates are
|
|
457
|
+
* considered in ascending aggregate score, so weaker duplicates are removed
|
|
458
|
+
* first. Mirrors `remove_dominated_programs` in the reference implementation.
|
|
459
|
+
*/
|
|
460
|
+
function pruneDominatedFronts(args) {
|
|
461
|
+
const { fronts, aggregateScores } = args;
|
|
462
|
+
const frontsByCandidate = indexFrontsByCandidate(fronts);
|
|
463
|
+
const ordered = [...frontsByCandidate.keys()].sort((a, b) => (aggregateScores[a] ?? 0) - (aggregateScores[b] ?? 0));
|
|
464
|
+
const survivors = new Set(ordered);
|
|
465
|
+
for (const candidate of ordered) {
|
|
466
|
+
survivors.delete(candidate);
|
|
467
|
+
if (!frontsByCandidate.get(candidate).every((index) => hasSurvivor({
|
|
468
|
+
front: fronts[index],
|
|
469
|
+
survivors
|
|
470
|
+
}))) survivors.add(candidate);
|
|
471
|
+
}
|
|
472
|
+
return fronts.map((front) => new Set([...front].filter((candidate) => survivors.has(candidate))));
|
|
473
|
+
}
|
|
474
|
+
/**
|
|
475
|
+
* Sample a parent candidate with probability proportional to the number of
|
|
476
|
+
* instances it is best on, after dominated candidates are pruned.
|
|
477
|
+
*/
|
|
478
|
+
function selectParetoCandidate(args) {
|
|
479
|
+
const { fronts, aggregateScores, rng } = args;
|
|
480
|
+
const pruned = pruneDominatedFronts({
|
|
481
|
+
fronts,
|
|
482
|
+
aggregateScores
|
|
483
|
+
});
|
|
484
|
+
const samplingPool = [];
|
|
485
|
+
for (const front of pruned) for (const candidate of front) samplingPool.push(candidate);
|
|
486
|
+
if (samplingPool.length === 0) return argmax(aggregateScores);
|
|
487
|
+
return rng.pick(samplingPool);
|
|
488
|
+
}
|
|
489
|
+
function indexFrontsByCandidate(fronts) {
|
|
490
|
+
const index = /* @__PURE__ */ new Map();
|
|
491
|
+
fronts.forEach((front, position) => {
|
|
492
|
+
for (const candidate of front) {
|
|
493
|
+
const owned = index.get(candidate);
|
|
494
|
+
if (owned === void 0) index.set(candidate, [position]);
|
|
495
|
+
else owned.push(position);
|
|
496
|
+
}
|
|
497
|
+
});
|
|
498
|
+
return index;
|
|
499
|
+
}
|
|
500
|
+
function hasSurvivor(args) {
|
|
501
|
+
const { front, survivors } = args;
|
|
502
|
+
for (const candidate of front) if (survivors.has(candidate)) return true;
|
|
503
|
+
return false;
|
|
504
|
+
}
|
|
505
|
+
//#endregion
|
|
506
|
+
//#region src/gepa/strategies.ts
|
|
507
|
+
/**
|
|
508
|
+
* Default parent selection: sample from the instance-wise Pareto frontier with
|
|
509
|
+
* probability proportional to how many validation instances a candidate is best
|
|
510
|
+
* on. This is what keeps GEPA from tunnelling into one lineage.
|
|
511
|
+
*
|
|
512
|
+
* `frontier` chooses what the fronts are taken over. "instance" is GEPA as
|
|
513
|
+
* published. "objective" tracks candidates leading each named objective the
|
|
514
|
+
* adapter reports, and "hybrid" pools both — a candidate then earns selection
|
|
515
|
+
* weight for every instance it wins *and* every objective it leads.
|
|
516
|
+
*/
|
|
517
|
+
function paretoSelector(args = {}) {
|
|
518
|
+
const { epsilon = 0, frontier = "instance" } = args;
|
|
519
|
+
return ({ state, rng }) => {
|
|
520
|
+
const fronts = [];
|
|
521
|
+
if (frontier !== "objective") fronts.push(...buildInstanceFronts({
|
|
522
|
+
scoreMatrix: state.scoreMatrix,
|
|
523
|
+
epsilon
|
|
524
|
+
}));
|
|
525
|
+
if (frontier !== "instance") {
|
|
526
|
+
const objectiveFronts = buildObjectiveFronts({
|
|
527
|
+
objectiveScores: state.objectiveScores ?? [],
|
|
528
|
+
epsilon
|
|
529
|
+
});
|
|
530
|
+
if (objectiveFronts.length === 0) throw new Error(`paretoSelector frontier "${frontier}" needs objective scores, but no candidate has any; have the adapter return objectiveScores or use frontier "instance"`);
|
|
531
|
+
fronts.push(...objectiveFronts);
|
|
532
|
+
}
|
|
533
|
+
return selectParetoCandidate({
|
|
534
|
+
fronts,
|
|
535
|
+
aggregateScores: state.aggregateScores,
|
|
536
|
+
rng
|
|
537
|
+
});
|
|
538
|
+
};
|
|
539
|
+
}
|
|
540
|
+
/** Greedy hill climbing. Useful as an ablation baseline. */
|
|
541
|
+
function currentBestSelector() {
|
|
542
|
+
return ({ state }) => argmax(state.aggregateScores);
|
|
543
|
+
}
|
|
544
|
+
function epsilonGreedySelector(args) {
|
|
545
|
+
const { epsilon } = args;
|
|
546
|
+
return ({ state, rng }) => {
|
|
547
|
+
if (rng.next() < epsilon) return rng.nextInt(state.aggregateScores.length);
|
|
548
|
+
return argmax(state.aggregateScores);
|
|
549
|
+
};
|
|
550
|
+
}
|
|
551
|
+
/** Pareto selection restricted to the top k candidates by aggregate score. */
|
|
552
|
+
function topKParetoSelector(args) {
|
|
553
|
+
const { k, epsilon = 0 } = args;
|
|
554
|
+
return ({ state, rng }) => {
|
|
555
|
+
const ranked = state.aggregateScores.map((score, index) => ({
|
|
556
|
+
score,
|
|
557
|
+
index
|
|
558
|
+
})).sort((a, b) => b.score - a.score).slice(0, k).map((entry) => entry.index);
|
|
559
|
+
const allowed = new Set(ranked);
|
|
560
|
+
const nonEmpty = buildInstanceFronts({
|
|
561
|
+
scoreMatrix: state.scoreMatrix,
|
|
562
|
+
epsilon
|
|
563
|
+
}).map((front) => new Set([...front].filter((id) => allowed.has(id)))).filter((front) => front.size > 0);
|
|
564
|
+
if (nonEmpty.length === 0) return argmax(state.aggregateScores);
|
|
565
|
+
return selectParetoCandidate({
|
|
566
|
+
fronts: nonEmpty,
|
|
567
|
+
aggregateScores: state.aggregateScores,
|
|
568
|
+
rng
|
|
569
|
+
});
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
/**
|
|
573
|
+
* Score every accepted candidate on the whole validation set. This is GEPA as
|
|
574
|
+
* published: the frontier is exact, and the cost is one full sweep per
|
|
575
|
+
* acceptance.
|
|
576
|
+
*/
|
|
577
|
+
function fullEvaluationPolicy() {
|
|
578
|
+
return {
|
|
579
|
+
selectInstances: ({ validationSet }) => validationSet.map((_, index) => index),
|
|
580
|
+
bestCandidate: bestByMeanThenCoverage
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
/**
|
|
584
|
+
* A full sweep, with the winner chosen by a lower confidence bound on its mean
|
|
585
|
+
* rather than by the mean itself: `mean - z * standardError`, over the
|
|
586
|
+
* instances the candidate was scored on.
|
|
587
|
+
*
|
|
588
|
+
* The reference picks the highest mean. That is an argmax over every candidate
|
|
589
|
+
* a run produced, all measured on the same instances, so the winner is
|
|
590
|
+
* systematically the one whose instance-level noise happened to land in its
|
|
591
|
+
* favour — the gap this library reports between `bestScore` and `testScore`.
|
|
592
|
+
* Penalising spread prefers a candidate that was even across the set to one
|
|
593
|
+
* carried by a few instances, at the cost of sometimes returning a genuinely
|
|
594
|
+
* better but less consistent candidate. It changes only which candidate is
|
|
595
|
+
* reported, never which ones the search explores.
|
|
596
|
+
*/
|
|
597
|
+
function lowerBoundEvaluationPolicy(args = {}) {
|
|
598
|
+
const { z = 1 } = args;
|
|
599
|
+
return {
|
|
600
|
+
selectInstances: ({ validationSet }) => validationSet.map((_, index) => index),
|
|
601
|
+
bestCandidate: (records) => bestByLowerBound({
|
|
602
|
+
records,
|
|
603
|
+
z
|
|
604
|
+
})
|
|
605
|
+
};
|
|
606
|
+
}
|
|
607
|
+
/**
|
|
608
|
+
* Score each candidate on a random subset of the validation set. Cheaper per
|
|
609
|
+
* acceptance, at the cost of comparing candidates measured on different
|
|
610
|
+
* instances — coverage breaks ties, so a candidate cannot win by having been
|
|
611
|
+
* asked fewer questions.
|
|
612
|
+
*/
|
|
613
|
+
function subsampledEvaluationPolicy(args) {
|
|
614
|
+
const { size } = args;
|
|
615
|
+
if (!Number.isFinite(size) || size <= 0) throw new Error(`subsampledEvaluationPolicy requires a positive size, received ${size}`);
|
|
616
|
+
return {
|
|
617
|
+
selectInstances: ({ validationSet, rng }) => rng.sample(validationSet.map((_, index) => index), size),
|
|
618
|
+
bestCandidate: bestByMeanThenCoverage
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* One component per selection, cycling in declaration order from the parent's
|
|
623
|
+
* own cursor. Updating a single component at a time is what makes the minibatch
|
|
624
|
+
* acceptance test attributable; keying off the parent's cursor rather than the
|
|
625
|
+
* global iteration is what guarantees every component of a rarely-selected
|
|
626
|
+
* lineage eventually gets a turn.
|
|
627
|
+
*/
|
|
628
|
+
function roundRobinComponentSelector() {
|
|
629
|
+
return ({ candidate, cursor }) => {
|
|
630
|
+
const names = componentNames(candidate);
|
|
631
|
+
if (names.length === 0) throw new Error("Candidate has no components to update");
|
|
632
|
+
return [names[cursor % names.length]];
|
|
633
|
+
};
|
|
634
|
+
}
|
|
635
|
+
/** Update every component in a single reflection call. */
|
|
636
|
+
function allComponentsSelector() {
|
|
637
|
+
return ({ candidate }) => componentNames(candidate);
|
|
638
|
+
}
|
|
639
|
+
/**
|
|
640
|
+
* Accept a mutation only when it beats its parent on the same minibatch. Cheap
|
|
641
|
+
* gate that keeps full validation sweeps for candidates that showed a signal.
|
|
642
|
+
*/
|
|
643
|
+
function improvementAcceptance(args = {}) {
|
|
644
|
+
const { minImprovement = 0 } = args;
|
|
645
|
+
return ({ parentScores, childScores }) => sum(childScores) > sum(parentScores) + minImprovement;
|
|
646
|
+
}
|
|
647
|
+
/**
|
|
648
|
+
* Accepts a child only when a paired permutation test over the minibatch says
|
|
649
|
+
* its gain is unlikely to be noise. Pairs are the same instances run by both
|
|
650
|
+
* candidates, so the test is over the per-instance differences.
|
|
651
|
+
*
|
|
652
|
+
* The reference accepts on any sum improvement. On a minibatch of three that
|
|
653
|
+
* promotes a candidate one lucky rollout ahead, which is what fills a pool
|
|
654
|
+
* with children the validation sweep then discards. The cost is that small
|
|
655
|
+
* batches cannot produce small p-values at all — with three instances the
|
|
656
|
+
* smallest attainable is 0.125 — so this accepts only a clean sweep there.
|
|
657
|
+
* That is the honest reading of three rollouts, not a limitation to tune
|
|
658
|
+
* around: raise `minibatchSize` to buy the power to detect smaller gains.
|
|
659
|
+
*/
|
|
660
|
+
function pairedPermutationAcceptance(args = {}) {
|
|
661
|
+
const { alpha = .2, maxExact = 16 } = args;
|
|
662
|
+
return ({ parentScores, childScores }) => {
|
|
663
|
+
const differences = [];
|
|
664
|
+
for (let index = 0; index < parentScores.length; index += 1) differences.push(childScores[index] - parentScores[index]);
|
|
665
|
+
const observed = sum(differences);
|
|
666
|
+
if (observed <= 0) return false;
|
|
667
|
+
return signFlipPValue({
|
|
668
|
+
differences,
|
|
669
|
+
observed,
|
|
670
|
+
maxExact
|
|
671
|
+
}) <= alpha;
|
|
672
|
+
};
|
|
673
|
+
}
|
|
674
|
+
/**
|
|
675
|
+
* Highest mean over the instances it was scored on, with wider coverage
|
|
676
|
+
* winning a tie: a candidate measured on more instances has earned the same
|
|
677
|
+
* mean against more evidence.
|
|
678
|
+
*/
|
|
679
|
+
function bestByMeanThenCoverage(records) {
|
|
680
|
+
let bestId = 0;
|
|
681
|
+
let bestScore = Number.NEGATIVE_INFINITY;
|
|
682
|
+
let bestCoverage = -1;
|
|
683
|
+
for (const record of records) {
|
|
684
|
+
const coverage = record.instanceScores.filter((score) => score !== void 0).length;
|
|
685
|
+
if (record.aggregateScore > bestScore || record.aggregateScore === bestScore && coverage > bestCoverage) {
|
|
686
|
+
bestId = record.id;
|
|
687
|
+
bestScore = record.aggregateScore;
|
|
688
|
+
bestCoverage = coverage;
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
return bestId;
|
|
692
|
+
}
|
|
693
|
+
/**
|
|
694
|
+
* Highest lower bound on the mean, over the instances each candidate was
|
|
695
|
+
* scored on. The bound is `mean - z * standardError`; a candidate scored on
|
|
696
|
+
* one instance has no spread to measure, so it is ranked on its mean alone
|
|
697
|
+
* rather than being flattered by an undefined variance.
|
|
698
|
+
*/
|
|
699
|
+
function bestByLowerBound(args) {
|
|
700
|
+
const { records, z } = args;
|
|
701
|
+
let bestId = 0;
|
|
702
|
+
let bestBound = Number.NEGATIVE_INFINITY;
|
|
703
|
+
for (const record of records) {
|
|
704
|
+
const scored = record.instanceScores.filter((score) => score !== void 0);
|
|
705
|
+
if (scored.length === 0) continue;
|
|
706
|
+
const bound = scored.length < 2 ? record.aggregateScore : record.aggregateScore - z * standardError(scored);
|
|
707
|
+
if (bound > bestBound) {
|
|
708
|
+
bestBound = bound;
|
|
709
|
+
bestId = record.id;
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
return bestId;
|
|
713
|
+
}
|
|
714
|
+
function standardError(scores) {
|
|
715
|
+
const mean = scores.reduce((total, score) => total + score, 0) / scores.length;
|
|
716
|
+
const variance = scores.reduce((total, score) => total + (score - mean) ** 2, 0) / (scores.length - 1);
|
|
717
|
+
return Math.sqrt(variance / scores.length);
|
|
718
|
+
}
|
|
719
|
+
//#endregion
|
|
720
|
+
//#region src/gepa/optimize.ts
|
|
721
|
+
const DEFAULT_MINIBATCH_SIZE = 3;
|
|
722
|
+
const DEFAULT_REJECTED_PROPOSAL_MEMORY = 3;
|
|
723
|
+
const DEFAULT_MAX_MERGES = 5;
|
|
724
|
+
const MERGE_SUBSAMPLE_SIZE = 5;
|
|
725
|
+
var ReflectionBudgetExhausted = class extends Error {};
|
|
726
|
+
/**
|
|
727
|
+
* Reflective prompt evolution: propose, screen on a minibatch, promote what
|
|
728
|
+
* survives, and track the Pareto frontier of everything promoted.
|
|
729
|
+
*
|
|
730
|
+
* One instance is a configured search that can be run against any number of
|
|
731
|
+
* tasks. It holds no run state, so two runs never share a shuffle position, a
|
|
732
|
+
* budget or a candidate pool.
|
|
733
|
+
*/
|
|
734
|
+
var GepaOptimizer = class {
|
|
735
|
+
#config;
|
|
736
|
+
constructor(config = {}) {
|
|
737
|
+
assertGepaConfig(config);
|
|
738
|
+
this.#config = config;
|
|
739
|
+
}
|
|
740
|
+
async optimize(task) {
|
|
741
|
+
return runGepa({
|
|
742
|
+
config: this.#config,
|
|
743
|
+
task
|
|
744
|
+
});
|
|
745
|
+
}
|
|
746
|
+
};
|
|
747
|
+
async function runGepa(args) {
|
|
748
|
+
const { config, task } = args;
|
|
749
|
+
const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, seed = 0, candidateSelector = paretoSelector(), acceptance = improvementAcceptance(), merge, skipPerfectScore = true, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals, reflection, checkpointCache = true, trackBestOutputs = false, raiseOnError = true } = config;
|
|
750
|
+
const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, componentSelector = roundRobinComponentSelector(), batchSampler = createEpochShuffledSampler({ minibatchSize }), valEvaluationPolicy = fullEvaluationPolicy(), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
|
|
751
|
+
const deadline = createDeadline({ maxWallClockMs });
|
|
752
|
+
const seedComponents = componentNames(seedCandidate);
|
|
753
|
+
const mergeConfig = {
|
|
754
|
+
enabled: merge?.enabled ?? seedComponents.length > 1,
|
|
755
|
+
maxInvocations: merge?.maxInvocations ?? DEFAULT_MAX_MERGES,
|
|
756
|
+
...merge?.valOverlapFloor === void 0 ? {} : { valOverlapFloor: merge.valOverlapFloor }
|
|
757
|
+
};
|
|
758
|
+
const proposalsPerIteration = proposals?.perIteration ?? 1;
|
|
759
|
+
const proposalConcurrency = proposals?.concurrency ?? 1;
|
|
760
|
+
const survivorsPerIteration = keepCount(proposals?.selection ?? "all");
|
|
761
|
+
if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
|
|
762
|
+
if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet; the Pareto frontier is tracked over validation instances");
|
|
763
|
+
if (seedComponents.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
|
|
764
|
+
if (testSet !== void 0 && testSet.length === 0) throw new Error("optimize requires a non-empty testSet when one is given; omit it to skip held-out evaluation");
|
|
765
|
+
const evaluationCache = cache === false ? void 0 : cache ?? createMemoryCache();
|
|
766
|
+
const propose = adapter.proposeNewTexts?.bind(adapter) ?? createDefaultProposer({
|
|
767
|
+
...reflection?.buildPrompt === void 0 ? {} : { buildPrompt: reflection.buildPrompt },
|
|
768
|
+
...reflection?.strategies === void 0 ? {} : { strategies: reflection.strategies },
|
|
769
|
+
limits: {
|
|
770
|
+
...reflection?.maxRecords === void 0 ? {} : { maxRecords: reflection.maxRecords },
|
|
771
|
+
...reflection?.maxCharacters === void 0 ? {} : { maxCharacters: reflection.maxCharacters }
|
|
772
|
+
}
|
|
773
|
+
});
|
|
774
|
+
const trainingIds = trainingSet.map((datum, index) => instanceId({
|
|
775
|
+
datum,
|
|
776
|
+
index
|
|
777
|
+
}));
|
|
778
|
+
const validationIds = validationSet.map((datum, index) => instanceId({
|
|
779
|
+
datum,
|
|
780
|
+
index
|
|
781
|
+
}));
|
|
782
|
+
const testIds = testSet?.map((datum, index) => instanceId({
|
|
783
|
+
datum,
|
|
784
|
+
index
|
|
785
|
+
})) ?? [];
|
|
786
|
+
const fingerprint = runFingerprint({
|
|
787
|
+
seedCandidate,
|
|
788
|
+
trainingIds,
|
|
789
|
+
validationIds,
|
|
790
|
+
seed,
|
|
791
|
+
...cacheNamespace === void 0 ? {} : { cacheNamespace }
|
|
792
|
+
});
|
|
793
|
+
assertResumable({
|
|
794
|
+
fingerprint,
|
|
795
|
+
...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
|
|
796
|
+
});
|
|
797
|
+
const rng = createSeededRng(seed, resumeFrom?.rngState);
|
|
798
|
+
const budget = createBudget({
|
|
799
|
+
maxMetricCalls,
|
|
800
|
+
spent: resumeFrom?.metricCalls ?? 0
|
|
801
|
+
});
|
|
802
|
+
if (resumeFrom?.sampler !== void 0) batchSampler.restore?.(resumeFrom.sampler);
|
|
803
|
+
let reflectionCalls = resumeFrom?.reflectionCalls ?? 0;
|
|
804
|
+
/**
|
|
805
|
+
* The reflection budget is enforced at the call, not at the proposal: an
|
|
806
|
+
* adapter's own proposer may make any number of calls, and a cap that only
|
|
807
|
+
* counted proposals would not bound it.
|
|
808
|
+
*/
|
|
809
|
+
const countedReflect = async (args) => {
|
|
810
|
+
if (reflection?.maxCalls !== void 0 && reflectionCalls >= reflection.maxCalls) throw new ReflectionBudgetExhausted();
|
|
811
|
+
reflectionCalls += 1;
|
|
812
|
+
return reflect(args);
|
|
813
|
+
};
|
|
814
|
+
const records = restoreRecords({
|
|
815
|
+
records: resumeFrom?.records ?? [],
|
|
816
|
+
seedCandidate
|
|
817
|
+
});
|
|
818
|
+
const seenCandidates = new Set(records.map((record) => candidateFingerprint(record.candidate)));
|
|
819
|
+
const outputsByCandidate = /* @__PURE__ */ new Map();
|
|
820
|
+
const rejectedProposals = restoreRejections({
|
|
821
|
+
rejections: resumeFrom?.rejectedProposals ?? {},
|
|
822
|
+
components: seedComponents
|
|
823
|
+
});
|
|
824
|
+
let iteration = resumeFrom?.iteration ?? 0;
|
|
825
|
+
const mergeAttempts = new Set(resumeFrom?.merge.attempts);
|
|
826
|
+
const mergeDescriptions = new Set(resumeFrom?.merge.descriptions);
|
|
827
|
+
let mergesDue = resumeFrom?.merge.due ?? 0;
|
|
828
|
+
let totalMergesTested = resumeFrom?.merge.tested ?? 0;
|
|
829
|
+
let lastIterationAccepted = resumeFrom?.merge.lastIterationAccepted ?? false;
|
|
830
|
+
function emit(event) {
|
|
831
|
+
onEvent?.(event);
|
|
832
|
+
}
|
|
833
|
+
/**
|
|
834
|
+
* Copies everything mutable: a snapshot handed to `onCheckpoint` is a record
|
|
835
|
+
* of that moment, and would otherwise keep growing as the run continues.
|
|
836
|
+
*/
|
|
837
|
+
function takeSnapshot() {
|
|
838
|
+
const cached = checkpointCache ? evaluationCache?.entries?.() : void 0;
|
|
839
|
+
const samplerState = batchSampler.state?.();
|
|
840
|
+
return {
|
|
841
|
+
version: 1,
|
|
842
|
+
fingerprint,
|
|
843
|
+
records: copyRecords(records),
|
|
844
|
+
iteration,
|
|
845
|
+
metricCalls: budget.spent(),
|
|
846
|
+
reflectionCalls,
|
|
847
|
+
cacheHits: evaluator.cacheHits(),
|
|
848
|
+
...samplerState === void 0 ? {} : { sampler: samplerState },
|
|
849
|
+
rejectedProposals: snapshotRejections({
|
|
850
|
+
rejections: rejectedProposals,
|
|
851
|
+
components: seedComponents
|
|
852
|
+
}),
|
|
853
|
+
rngState: rng.state(),
|
|
854
|
+
merge: {
|
|
855
|
+
attempts: [...mergeAttempts],
|
|
856
|
+
descriptions: [...mergeDescriptions],
|
|
857
|
+
due: mergesDue,
|
|
858
|
+
tested: totalMergesTested,
|
|
859
|
+
lastIterationAccepted
|
|
860
|
+
},
|
|
861
|
+
...cached === void 0 ? {} : { cache: cached }
|
|
862
|
+
};
|
|
863
|
+
}
|
|
864
|
+
async function checkpoint() {
|
|
865
|
+
if (onCheckpoint === void 0) return;
|
|
866
|
+
await onCheckpoint(takeSnapshot());
|
|
867
|
+
}
|
|
868
|
+
const evaluator = createEvaluator({
|
|
869
|
+
adapter,
|
|
870
|
+
budget,
|
|
871
|
+
...retry === void 0 ? {} : { retry },
|
|
872
|
+
...cacheNamespace === void 0 ? {} : { cacheNamespace },
|
|
873
|
+
...evaluationCache === void 0 ? {} : { cache: evaluationCache },
|
|
874
|
+
trackOutputs: trackBestOutputs,
|
|
875
|
+
cacheHits: resumeFrom?.cacheHits ?? 0,
|
|
876
|
+
...signal === void 0 ? {} : { signal },
|
|
877
|
+
onEvaluation: (event) => emit({
|
|
878
|
+
type: "evaluation",
|
|
879
|
+
...event
|
|
880
|
+
})
|
|
881
|
+
});
|
|
882
|
+
evaluator.restore(resumeFrom?.cache ?? []);
|
|
883
|
+
/**
|
|
884
|
+
* Evaluates a candidate on a batch at the current iteration. The cache, the
|
|
885
|
+
* budget and the transient-failure rules live in the shared evaluator; what
|
|
886
|
+
* belongs to GEPA is only which batch, and when.
|
|
887
|
+
*/
|
|
888
|
+
async function evaluateCached(args) {
|
|
889
|
+
return evaluator.evaluate({
|
|
890
|
+
...args,
|
|
891
|
+
iteration
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
/**
|
|
895
|
+
* Scores a candidate on the validation instances the policy selects, and
|
|
896
|
+
* spreads the result back over the full validation set — instances the
|
|
897
|
+
* policy skipped stay `undefined`, which every consumer reads as unknown
|
|
898
|
+
* rather than as a zero.
|
|
899
|
+
*/
|
|
900
|
+
async function evaluateValidation(args) {
|
|
901
|
+
const { candidate, instances, phase, candidateId } = args;
|
|
902
|
+
const dense = await evaluateCached({
|
|
903
|
+
candidate,
|
|
904
|
+
batch: instances.map((index) => validationSet[index]),
|
|
905
|
+
ids: instances.map((index) => validationIds[index]),
|
|
906
|
+
split: "val",
|
|
907
|
+
phase,
|
|
908
|
+
candidateId
|
|
909
|
+
});
|
|
910
|
+
const scores = new Array(validationSet.length).fill(void 0);
|
|
911
|
+
const objectiveScores = new Array(validationSet.length).fill(void 0);
|
|
912
|
+
const outputs = new Array(validationSet.length).fill(void 0);
|
|
913
|
+
instances.forEach((instance, position) => {
|
|
914
|
+
if (dense.transient[position] === true) return;
|
|
915
|
+
scores[instance] = dense.scores[position];
|
|
916
|
+
objectiveScores[instance] = dense.objectiveScores[position];
|
|
917
|
+
outputs[instance] = dense.outputs[position];
|
|
918
|
+
});
|
|
919
|
+
return {
|
|
920
|
+
scores,
|
|
921
|
+
objectiveScores,
|
|
922
|
+
outputs
|
|
923
|
+
};
|
|
924
|
+
}
|
|
925
|
+
/** The validation instances this candidate should be scored on. */
|
|
926
|
+
function selectValInstances(candidate) {
|
|
927
|
+
const selected = valEvaluationPolicy.selectInstances({
|
|
928
|
+
validationSet,
|
|
929
|
+
candidate,
|
|
930
|
+
records,
|
|
931
|
+
iteration,
|
|
932
|
+
rng
|
|
933
|
+
});
|
|
934
|
+
if (selected.length === 0) throw new Error("valEvaluationPolicy selected no validation instances; a candidate cannot be scored");
|
|
935
|
+
return selected;
|
|
936
|
+
}
|
|
937
|
+
function addCandidate(args) {
|
|
938
|
+
const objectiveScores = meanObjectives({
|
|
939
|
+
rows: args.evaluation.objectiveScores,
|
|
940
|
+
scores: args.evaluation.scores
|
|
941
|
+
});
|
|
942
|
+
const record = {
|
|
943
|
+
id: records.length,
|
|
944
|
+
candidate: args.candidate,
|
|
945
|
+
parentIds: args.parentIds,
|
|
946
|
+
instanceScores: args.evaluation.scores,
|
|
947
|
+
aggregateScore: mean(args.evaluation.scores),
|
|
948
|
+
...objectiveScores === void 0 ? {} : { objectiveScores },
|
|
949
|
+
source: args.source,
|
|
950
|
+
updatedComponents: args.updatedComponents,
|
|
951
|
+
iteration,
|
|
952
|
+
componentCursor: inheritedCursor(args.parentIds)
|
|
953
|
+
};
|
|
954
|
+
records.push(record);
|
|
955
|
+
seenCandidates.add(candidateFingerprint(args.candidate));
|
|
956
|
+
if (trackBestOutputs) outputsByCandidate.set(record.id, args.evaluation.outputs);
|
|
957
|
+
if (args.source !== "merge") {
|
|
958
|
+
lastIterationAccepted = true;
|
|
959
|
+
if (mergeConfig.enabled && totalMergesTested < mergeConfig.maxInvocations) mergesDue += 1;
|
|
960
|
+
}
|
|
961
|
+
return record;
|
|
962
|
+
}
|
|
963
|
+
/** Keeps the most recent rejections per component, oldest dropped first. */
|
|
964
|
+
function rememberRejection(args) {
|
|
965
|
+
const { proposed, parentScore, childScore } = args;
|
|
966
|
+
if (rejectedProposalMemory <= 0) return;
|
|
967
|
+
for (const component of componentNames(proposed)) {
|
|
968
|
+
const text = proposed[component];
|
|
969
|
+
if (text === void 0) continue;
|
|
970
|
+
const history = rejectedProposals[component] ?? [];
|
|
971
|
+
history.unshift({
|
|
972
|
+
text,
|
|
973
|
+
parentScore,
|
|
974
|
+
childScore
|
|
975
|
+
});
|
|
976
|
+
rejectedProposals[component] = history.slice(0, rejectedProposalMemory);
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
function inheritedCursor(parentIds) {
|
|
980
|
+
let cursor = 0;
|
|
981
|
+
for (const parentId of parentIds) {
|
|
982
|
+
const parent = records[parentId];
|
|
983
|
+
if (parent !== void 0 && parent.componentCursor > cursor) cursor = parent.componentCursor;
|
|
984
|
+
}
|
|
985
|
+
return cursor;
|
|
986
|
+
}
|
|
987
|
+
emit({
|
|
988
|
+
type: "start",
|
|
989
|
+
components: seedComponents,
|
|
990
|
+
validationSetSize: validationSet.length
|
|
991
|
+
});
|
|
992
|
+
if (records.length === 0) {
|
|
993
|
+
const seedInstances = selectValInstances(seedCandidate);
|
|
994
|
+
if (!budget.canAfford(seedInstances.length)) throw new Error(`maxMetricCalls (${maxMetricCalls}) is smaller than the ${seedInstances.length} validation instances selected for scoring; the seed candidate cannot be scored`);
|
|
995
|
+
addCandidate({
|
|
996
|
+
candidate: seedCandidate,
|
|
997
|
+
parentIds: [],
|
|
998
|
+
evaluation: await evaluateValidation({
|
|
999
|
+
candidate: seedCandidate,
|
|
1000
|
+
instances: seedInstances,
|
|
1001
|
+
phase: "seed",
|
|
1002
|
+
candidateId: 0
|
|
1003
|
+
}),
|
|
1004
|
+
source: "seed",
|
|
1005
|
+
updatedComponents: []
|
|
1006
|
+
});
|
|
1007
|
+
lastIterationAccepted = false;
|
|
1008
|
+
mergesDue = 0;
|
|
1009
|
+
await checkpoint();
|
|
1010
|
+
}
|
|
1011
|
+
let stopReason = "budgetExhausted";
|
|
1012
|
+
/**
|
|
1013
|
+
* Proposes and gates one merge. Returns "none" when nothing was tested — the
|
|
1014
|
+
* iteration then falls through to reflective mutation, exactly as it would
|
|
1015
|
+
* have without merging enabled. A merge that cannot be afforded is skipped,
|
|
1016
|
+
* never treated as the end of the run.
|
|
1017
|
+
*/
|
|
1018
|
+
async function tryMerge() {
|
|
1019
|
+
const proposal = proposeMerge({
|
|
1020
|
+
records,
|
|
1021
|
+
pool: collectDominatorIds(records),
|
|
1022
|
+
rng,
|
|
1023
|
+
attempted: mergeAttempts,
|
|
1024
|
+
attemptedDescriptions: mergeDescriptions,
|
|
1025
|
+
...mergeConfig.valOverlapFloor === void 0 ? {} : { valOverlapFloor: mergeConfig.valOverlapFloor }
|
|
1026
|
+
});
|
|
1027
|
+
if (proposal === null) return "none";
|
|
1028
|
+
const [leftId, rightId] = proposal.parentIds;
|
|
1029
|
+
const left = records[leftId];
|
|
1030
|
+
const right = records[rightId];
|
|
1031
|
+
const subsample = selectMergeSubsample({
|
|
1032
|
+
scores1: left.instanceScores,
|
|
1033
|
+
scores2: right.instanceScores,
|
|
1034
|
+
rng,
|
|
1035
|
+
size: MERGE_SUBSAMPLE_SIZE
|
|
1036
|
+
});
|
|
1037
|
+
if (subsample.length === 0) return "none";
|
|
1038
|
+
const unique = [...new Set(subsample)];
|
|
1039
|
+
const uniqueIds = unique.map((index) => validationIds[index]);
|
|
1040
|
+
if (!budget.canAfford(evaluator.countUncached({
|
|
1041
|
+
candidate: proposal.candidate,
|
|
1042
|
+
ids: uniqueIds,
|
|
1043
|
+
split: "val"
|
|
1044
|
+
}))) return "none";
|
|
1045
|
+
emit({
|
|
1046
|
+
type: "iterationStart",
|
|
1047
|
+
iteration,
|
|
1048
|
+
parentIds: [...proposal.parentIds]
|
|
1049
|
+
});
|
|
1050
|
+
mergeAttempts.add(proposal.attemptKey);
|
|
1051
|
+
mergeDescriptions.add(proposal.descriptionKey);
|
|
1052
|
+
const uniqueEvaluation = await evaluateCached({
|
|
1053
|
+
candidate: proposal.candidate,
|
|
1054
|
+
batch: unique.map((index) => validationSet[index]),
|
|
1055
|
+
ids: uniqueIds,
|
|
1056
|
+
split: "val",
|
|
1057
|
+
phase: "minibatch",
|
|
1058
|
+
candidateId: null
|
|
1059
|
+
});
|
|
1060
|
+
const scoreByIndex = new Map(unique.map((index, position) => [index, uniqueEvaluation.scores[position]]));
|
|
1061
|
+
const mergedSum = sum(subsample.map((index) => scoreByIndex.get(index)));
|
|
1062
|
+
const parentBest = Math.max(sum(subsample.map((index) => left.instanceScores[index])), sum(subsample.map((index) => right.instanceScores[index])));
|
|
1063
|
+
if (mergedSum < parentBest) {
|
|
1064
|
+
emit({
|
|
1065
|
+
type: "candidateRejected",
|
|
1066
|
+
iteration,
|
|
1067
|
+
parentId: leftId,
|
|
1068
|
+
parentScore: parentBest,
|
|
1069
|
+
childScore: mergedSum,
|
|
1070
|
+
source: "merge",
|
|
1071
|
+
reason: "worse"
|
|
1072
|
+
});
|
|
1073
|
+
return "attempted";
|
|
1074
|
+
}
|
|
1075
|
+
const mergeInstances = selectValInstances(proposal.candidate);
|
|
1076
|
+
if (!budget.canAfford(evaluator.countUncached({
|
|
1077
|
+
candidate: proposal.candidate,
|
|
1078
|
+
ids: mergeInstances.map((index) => validationIds[index]),
|
|
1079
|
+
split: "val"
|
|
1080
|
+
}))) return "attempted";
|
|
1081
|
+
const evaluation = await evaluateValidation({
|
|
1082
|
+
candidate: proposal.candidate,
|
|
1083
|
+
instances: mergeInstances,
|
|
1084
|
+
phase: "validation",
|
|
1085
|
+
candidateId: records.length
|
|
1086
|
+
});
|
|
1087
|
+
const ancestor = records[proposal.ancestorId];
|
|
1088
|
+
const record = addCandidate({
|
|
1089
|
+
candidate: proposal.candidate,
|
|
1090
|
+
parentIds: [...proposal.parentIds],
|
|
1091
|
+
evaluation,
|
|
1092
|
+
source: "merge",
|
|
1093
|
+
updatedComponents: componentNames(proposal.candidate).filter((name) => proposal.candidate[name] !== ancestor.candidate[name])
|
|
1094
|
+
});
|
|
1095
|
+
mergesDue -= 1;
|
|
1096
|
+
totalMergesTested += 1;
|
|
1097
|
+
emit({
|
|
1098
|
+
type: "candidateAccepted",
|
|
1099
|
+
iteration,
|
|
1100
|
+
candidateId: record.id,
|
|
1101
|
+
parentIds: record.parentIds,
|
|
1102
|
+
aggregateScore: record.aggregateScore,
|
|
1103
|
+
source: "merge"
|
|
1104
|
+
});
|
|
1105
|
+
return "attempted";
|
|
1106
|
+
}
|
|
1107
|
+
/**
|
|
1108
|
+
* Draws every proposal an iteration will make, before any of them runs.
|
|
1109
|
+
*
|
|
1110
|
+
* Each draw consumes the random stream — parent, minibatch, component — so
|
|
1111
|
+
* they all happen here, in order, on the same frontier snapshot. Doing it
|
|
1112
|
+
* inside the concurrent phase instead would make the whole run's trajectory
|
|
1113
|
+
* depend on which network call returned first.
|
|
1114
|
+
*/
|
|
1115
|
+
function planProposals() {
|
|
1116
|
+
const state = {
|
|
1117
|
+
scoreMatrix: records.map((record) => record.instanceScores),
|
|
1118
|
+
aggregateScores: records.map((record) => record.aggregateScore),
|
|
1119
|
+
objectiveScores: records.map((record) => record.objectiveScores)
|
|
1120
|
+
};
|
|
1121
|
+
const plans = [];
|
|
1122
|
+
for (let slot = 0; slot < proposalsPerIteration; slot += 1) {
|
|
1123
|
+
const parent = records[candidateSelector({
|
|
1124
|
+
state,
|
|
1125
|
+
rng
|
|
1126
|
+
})];
|
|
1127
|
+
const batchIndices = batchSampler({
|
|
1128
|
+
trainingSet,
|
|
1129
|
+
iteration: iteration * proposalsPerIteration + slot,
|
|
1130
|
+
rng
|
|
1131
|
+
});
|
|
1132
|
+
const componentsToUpdate = componentSelector({
|
|
1133
|
+
candidate: parent.candidate,
|
|
1134
|
+
cursor: parent.componentCursor,
|
|
1135
|
+
iteration,
|
|
1136
|
+
rng
|
|
1137
|
+
});
|
|
1138
|
+
assertComponents({
|
|
1139
|
+
names: componentsToUpdate,
|
|
1140
|
+
candidate: parent.candidate,
|
|
1141
|
+
source: "componentSelector"
|
|
1142
|
+
});
|
|
1143
|
+
parent.componentCursor = (parent.componentCursor + 1) % Math.max(1, componentNames(parent.candidate).length);
|
|
1144
|
+
plans.push({
|
|
1145
|
+
parent,
|
|
1146
|
+
batch: batchIndices.map((index) => trainingSet[index]),
|
|
1147
|
+
batchIds: batchIndices.map((index) => trainingIds[index]),
|
|
1148
|
+
componentsToUpdate,
|
|
1149
|
+
attempt: iteration * proposalsPerIteration + slot
|
|
1150
|
+
});
|
|
1151
|
+
}
|
|
1152
|
+
return plans;
|
|
1153
|
+
}
|
|
1154
|
+
/**
|
|
1155
|
+
* Reflects on one parent and screens the result on that parent's own
|
|
1156
|
+
* minibatch. Everything here is IO the run can overlap; nothing here mutates
|
|
1157
|
+
* the candidate pool, which is what makes overlapping it safe.
|
|
1158
|
+
*/
|
|
1159
|
+
async function runProposal(plan) {
|
|
1160
|
+
const { parent, batch, batchIds, componentsToUpdate, attempt } = plan;
|
|
1161
|
+
const parentEvaluation = await evaluator.evaluateTraced({
|
|
1162
|
+
batch,
|
|
1163
|
+
candidate: parent.candidate,
|
|
1164
|
+
split: "train",
|
|
1165
|
+
phase: "minibatch",
|
|
1166
|
+
candidateId: parent.id,
|
|
1167
|
+
iteration
|
|
1168
|
+
});
|
|
1169
|
+
if (parentEvaluation === null) return { status: "budgetExhausted" };
|
|
1170
|
+
if (skipPerfectScore && parentEvaluation.scores.every((score) => score >= perfectScore)) return { status: "skipped" };
|
|
1171
|
+
const reflectiveDataset = await adapter.makeReflectiveDataset({
|
|
1172
|
+
candidate: parent.candidate,
|
|
1173
|
+
batch,
|
|
1174
|
+
evaluation: parentEvaluation,
|
|
1175
|
+
componentsToUpdate
|
|
1176
|
+
});
|
|
1177
|
+
let proposed;
|
|
1178
|
+
try {
|
|
1179
|
+
proposed = await propose({
|
|
1180
|
+
candidate: parent.candidate,
|
|
1181
|
+
reflectiveDataset,
|
|
1182
|
+
componentsToUpdate,
|
|
1183
|
+
rejectedProposals,
|
|
1184
|
+
attempt,
|
|
1185
|
+
reflect: countedReflect,
|
|
1186
|
+
signal
|
|
1187
|
+
});
|
|
1188
|
+
} catch (err) {
|
|
1189
|
+
if (err instanceof ReflectionBudgetExhausted) return { status: "reflectionExhausted" };
|
|
1190
|
+
throw err;
|
|
1191
|
+
}
|
|
1192
|
+
assertComponents({
|
|
1193
|
+
names: componentNames(proposed),
|
|
1194
|
+
candidate: parent.candidate,
|
|
1195
|
+
source: "proposeNewTexts"
|
|
1196
|
+
});
|
|
1197
|
+
const child = {
|
|
1198
|
+
...parent.candidate,
|
|
1199
|
+
...proposed
|
|
1200
|
+
};
|
|
1201
|
+
const changed = componentNames(proposed).length > 0 && !seenCandidates.has(candidateFingerprint(child));
|
|
1202
|
+
emit({
|
|
1203
|
+
type: "proposal",
|
|
1204
|
+
iteration,
|
|
1205
|
+
parentId: parent.id,
|
|
1206
|
+
componentsToUpdate: [...componentsToUpdate],
|
|
1207
|
+
changed
|
|
1208
|
+
});
|
|
1209
|
+
if (!changed) return { status: "skipped" };
|
|
1210
|
+
let childEvaluation;
|
|
1211
|
+
try {
|
|
1212
|
+
childEvaluation = await evaluateCached({
|
|
1213
|
+
candidate: child,
|
|
1214
|
+
batch,
|
|
1215
|
+
ids: batchIds,
|
|
1216
|
+
split: "train",
|
|
1217
|
+
phase: "minibatch",
|
|
1218
|
+
candidateId: null
|
|
1219
|
+
});
|
|
1220
|
+
} catch (err) {
|
|
1221
|
+
if (err instanceof BudgetExhausted) return { status: "budgetExhausted" };
|
|
1222
|
+
throw err;
|
|
1223
|
+
}
|
|
1224
|
+
const screened = pairMeasured({
|
|
1225
|
+
parent: parentEvaluation,
|
|
1226
|
+
child: childEvaluation
|
|
1227
|
+
});
|
|
1228
|
+
if (screened.parentScores.length === 0) return { status: "skipped" };
|
|
1229
|
+
return {
|
|
1230
|
+
status: "screened",
|
|
1231
|
+
plan,
|
|
1232
|
+
child,
|
|
1233
|
+
proposed,
|
|
1234
|
+
parentScore: mean(screened.parentScores),
|
|
1235
|
+
childScore: mean(screened.childScores),
|
|
1236
|
+
improvement: sum(screened.childScores) - sum(screened.parentScores),
|
|
1237
|
+
accepted: acceptance(screened)
|
|
1238
|
+
};
|
|
1239
|
+
}
|
|
1240
|
+
/**
|
|
1241
|
+
* Turns screened proposals into candidates: rejections first, then the
|
|
1242
|
+
* survivors the selection policy keeps, each paying for its own validation
|
|
1243
|
+
* sweep. Returns a stop reason when the iteration ran the run out of budget.
|
|
1244
|
+
*/
|
|
1245
|
+
async function commitProposals(outcomes) {
|
|
1246
|
+
let stop;
|
|
1247
|
+
const improved = [];
|
|
1248
|
+
/**
|
|
1249
|
+
* Children two siblings converged on. Resolved here rather than while the
|
|
1250
|
+
* proposals were in flight: the first one in plan order keeps the child,
|
|
1251
|
+
* whichever of them finished first.
|
|
1252
|
+
*/
|
|
1253
|
+
const claimed = /* @__PURE__ */ new Set();
|
|
1254
|
+
for (const outcome of outcomes) {
|
|
1255
|
+
if (outcome.status === "skipped") continue;
|
|
1256
|
+
if (outcome.status === "budgetExhausted") {
|
|
1257
|
+
stop ??= "budgetExhausted";
|
|
1258
|
+
continue;
|
|
1259
|
+
}
|
|
1260
|
+
if (outcome.status === "reflectionExhausted") {
|
|
1261
|
+
stop ??= "reflectionBudgetExhausted";
|
|
1262
|
+
continue;
|
|
1263
|
+
}
|
|
1264
|
+
const fingerprint = candidateFingerprint(outcome.child);
|
|
1265
|
+
if (claimed.has(fingerprint)) continue;
|
|
1266
|
+
claimed.add(fingerprint);
|
|
1267
|
+
if (!outcome.accepted) {
|
|
1268
|
+
rememberRejection({
|
|
1269
|
+
proposed: outcome.proposed,
|
|
1270
|
+
parentScore: outcome.parentScore,
|
|
1271
|
+
childScore: outcome.childScore
|
|
1272
|
+
});
|
|
1273
|
+
emit({
|
|
1274
|
+
type: "candidateRejected",
|
|
1275
|
+
iteration,
|
|
1276
|
+
parentId: outcome.plan.parent.id,
|
|
1277
|
+
parentScore: outcome.parentScore,
|
|
1278
|
+
childScore: outcome.childScore,
|
|
1279
|
+
source: "mutation",
|
|
1280
|
+
reason: "worse"
|
|
1281
|
+
});
|
|
1282
|
+
continue;
|
|
1283
|
+
}
|
|
1284
|
+
improved.push(outcome);
|
|
1285
|
+
}
|
|
1286
|
+
const survivors = selectSurvivors(improved);
|
|
1287
|
+
for (const outcome of improved) {
|
|
1288
|
+
if (survivors.includes(outcome)) continue;
|
|
1289
|
+
emit({
|
|
1290
|
+
type: "candidateRejected",
|
|
1291
|
+
iteration,
|
|
1292
|
+
parentId: outcome.plan.parent.id,
|
|
1293
|
+
parentScore: outcome.parentScore,
|
|
1294
|
+
childScore: outcome.childScore,
|
|
1295
|
+
source: "mutation",
|
|
1296
|
+
reason: "notSelected"
|
|
1297
|
+
});
|
|
1298
|
+
}
|
|
1299
|
+
const baseId = records.length;
|
|
1300
|
+
const scheduled = [];
|
|
1301
|
+
let owed = 0;
|
|
1302
|
+
for (const outcome of survivors) {
|
|
1303
|
+
const instances = selectValInstances(outcome.child);
|
|
1304
|
+
const uncached = evaluator.countUncached({
|
|
1305
|
+
candidate: outcome.child,
|
|
1306
|
+
ids: instances.map((index) => validationIds[index]),
|
|
1307
|
+
split: "val"
|
|
1308
|
+
});
|
|
1309
|
+
if (!budget.canAfford(owed + uncached)) {
|
|
1310
|
+
stop ??= "budgetExhausted";
|
|
1311
|
+
break;
|
|
1312
|
+
}
|
|
1313
|
+
owed += uncached;
|
|
1314
|
+
scheduled.push({
|
|
1315
|
+
outcome,
|
|
1316
|
+
candidateId: baseId + scheduled.length,
|
|
1317
|
+
instances
|
|
1318
|
+
});
|
|
1319
|
+
}
|
|
1320
|
+
const scored = await mapWithConcurrency({
|
|
1321
|
+
items: scheduled,
|
|
1322
|
+
limit: proposalConcurrency,
|
|
1323
|
+
signal,
|
|
1324
|
+
task: async (item) => {
|
|
1325
|
+
try {
|
|
1326
|
+
return {
|
|
1327
|
+
item,
|
|
1328
|
+
evaluation: await evaluateValidation({
|
|
1329
|
+
candidate: item.outcome.child,
|
|
1330
|
+
instances: item.instances,
|
|
1331
|
+
phase: "validation",
|
|
1332
|
+
candidateId: item.candidateId
|
|
1333
|
+
})
|
|
1334
|
+
};
|
|
1335
|
+
} catch (err) {
|
|
1336
|
+
if (err instanceof BudgetExhausted) return {
|
|
1337
|
+
item,
|
|
1338
|
+
evaluation: void 0
|
|
1339
|
+
};
|
|
1340
|
+
throw err;
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
});
|
|
1344
|
+
for (const { item, evaluation } of scored) {
|
|
1345
|
+
if (evaluation === void 0) {
|
|
1346
|
+
stop ??= "budgetExhausted";
|
|
1347
|
+
break;
|
|
1348
|
+
}
|
|
1349
|
+
const record = addCandidate({
|
|
1350
|
+
candidate: item.outcome.child,
|
|
1351
|
+
parentIds: [item.outcome.plan.parent.id],
|
|
1352
|
+
evaluation,
|
|
1353
|
+
source: "mutation",
|
|
1354
|
+
updatedComponents: componentNames(item.outcome.proposed)
|
|
1355
|
+
});
|
|
1356
|
+
emit({
|
|
1357
|
+
type: "candidateAccepted",
|
|
1358
|
+
iteration,
|
|
1359
|
+
candidateId: record.id,
|
|
1360
|
+
parentIds: record.parentIds,
|
|
1361
|
+
aggregateScore: record.aggregateScore,
|
|
1362
|
+
source: "mutation"
|
|
1363
|
+
});
|
|
1364
|
+
}
|
|
1365
|
+
return stop;
|
|
1366
|
+
}
|
|
1367
|
+
/** The improving proposals an iteration keeps, in the order they were made. */
|
|
1368
|
+
function selectSurvivors(improved) {
|
|
1369
|
+
if (improved.length <= survivorsPerIteration) return [...improved];
|
|
1370
|
+
return improved.map((outcome, index) => ({
|
|
1371
|
+
outcome,
|
|
1372
|
+
index
|
|
1373
|
+
})).sort((a, b) => b.outcome.improvement - a.outcome.improvement || a.index - b.index).slice(0, survivorsPerIteration).sort((a, b) => a.index - b.index).map((entry) => entry.outcome);
|
|
1374
|
+
}
|
|
1375
|
+
while (true) {
|
|
1376
|
+
if (signal?.aborted) {
|
|
1377
|
+
stopReason = "aborted";
|
|
1378
|
+
break;
|
|
1379
|
+
}
|
|
1380
|
+
if (costExhausted({
|
|
1381
|
+
usage: evaluator.usage(),
|
|
1382
|
+
maxCostUsd
|
|
1383
|
+
})) {
|
|
1384
|
+
stopReason = "costExhausted";
|
|
1385
|
+
break;
|
|
1386
|
+
}
|
|
1387
|
+
if (deadline.exceeded()) {
|
|
1388
|
+
stopReason = "deadlineReached";
|
|
1389
|
+
break;
|
|
1390
|
+
}
|
|
1391
|
+
if (iteration >= maxIterations) {
|
|
1392
|
+
stopReason = "maxIterations";
|
|
1393
|
+
break;
|
|
1394
|
+
}
|
|
1395
|
+
if (!budget.canAfford(proposalsPerIteration * minibatchSize * 2 + validationSet.length)) {
|
|
1396
|
+
stopReason = "budgetExhausted";
|
|
1397
|
+
break;
|
|
1398
|
+
}
|
|
1399
|
+
if (reflection?.maxCalls !== void 0 && reflectionCalls >= reflection.maxCalls) {
|
|
1400
|
+
stopReason = "reflectionBudgetExhausted";
|
|
1401
|
+
break;
|
|
1402
|
+
}
|
|
1403
|
+
const spentBeforeIteration = budget.spent();
|
|
1404
|
+
let pendingStop;
|
|
1405
|
+
try {
|
|
1406
|
+
const mergeScheduled = mergeConfig.enabled && mergesDue > 0 && lastIterationAccepted && totalMergesTested < mergeConfig.maxInvocations;
|
|
1407
|
+
lastIterationAccepted = false;
|
|
1408
|
+
if (!(mergeScheduled && await tryMerge() === "attempted")) {
|
|
1409
|
+
const plans = planProposals();
|
|
1410
|
+
emit({
|
|
1411
|
+
type: "iterationStart",
|
|
1412
|
+
iteration,
|
|
1413
|
+
parentIds: plans.map((plan) => plan.parent.id)
|
|
1414
|
+
});
|
|
1415
|
+
pendingStop = await commitProposals(await mapWithConcurrency({
|
|
1416
|
+
items: plans,
|
|
1417
|
+
limit: proposalConcurrency,
|
|
1418
|
+
task: runProposal,
|
|
1419
|
+
signal
|
|
1420
|
+
}));
|
|
1421
|
+
}
|
|
1422
|
+
} catch (err) {
|
|
1423
|
+
if (signal?.aborted) {
|
|
1424
|
+
stopReason = "aborted";
|
|
1425
|
+
break;
|
|
1426
|
+
}
|
|
1427
|
+
if (err instanceof BudgetExhausted) {
|
|
1428
|
+
stopReason = "budgetExhausted";
|
|
1429
|
+
break;
|
|
1430
|
+
}
|
|
1431
|
+
if (raiseOnError || budget.spent() === spentBeforeIteration) throw err;
|
|
1432
|
+
emit({
|
|
1433
|
+
type: "error",
|
|
1434
|
+
iteration,
|
|
1435
|
+
err
|
|
1436
|
+
});
|
|
1437
|
+
}
|
|
1438
|
+
iteration += 1;
|
|
1439
|
+
await checkpoint();
|
|
1440
|
+
if (pendingStop !== void 0) {
|
|
1441
|
+
stopReason = pendingStop;
|
|
1442
|
+
break;
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
const bestCandidateId = valEvaluationPolicy.bestCandidate(records);
|
|
1446
|
+
const best = records[bestCandidateId];
|
|
1447
|
+
const testScore = testSet === void 0 ? void 0 : measuredMean(await evaluateCached({
|
|
1448
|
+
candidate: best.candidate,
|
|
1449
|
+
batch: testSet,
|
|
1450
|
+
ids: testIds,
|
|
1451
|
+
split: "test",
|
|
1452
|
+
phase: "test",
|
|
1453
|
+
candidateId: bestCandidateId,
|
|
1454
|
+
charge: false
|
|
1455
|
+
}));
|
|
1456
|
+
emit({
|
|
1457
|
+
type: "finish",
|
|
1458
|
+
reason: stopReason,
|
|
1459
|
+
bestCandidateId,
|
|
1460
|
+
metricCalls: budget.spent(),
|
|
1461
|
+
...testScore === void 0 ? {} : { testScore }
|
|
1462
|
+
});
|
|
1463
|
+
const perObjectiveBest = collectPerObjectiveBest(records);
|
|
1464
|
+
const bestOutputs = outputsByCandidate.get(bestCandidateId);
|
|
1465
|
+
return {
|
|
1466
|
+
bestCandidate: best.candidate,
|
|
1467
|
+
bestScore: best.aggregateScore,
|
|
1468
|
+
usage: evaluator.usage(),
|
|
1469
|
+
bestCandidateId,
|
|
1470
|
+
...testScore === void 0 ? {} : {
|
|
1471
|
+
testScore,
|
|
1472
|
+
testMetricCalls: evaluator.unchargedCalls()
|
|
1473
|
+
},
|
|
1474
|
+
...bestOutputs === void 0 ? {} : { bestOutputs },
|
|
1475
|
+
candidates: records,
|
|
1476
|
+
paretoFrontier: collectDominatorIds(records).map((id) => records[id]),
|
|
1477
|
+
...perObjectiveBest === void 0 ? {} : { perObjectiveBest },
|
|
1478
|
+
scoreMatrix: records.map((record) => [...record.instanceScores]),
|
|
1479
|
+
metricCalls: budget.spent(),
|
|
1480
|
+
reflectionCalls,
|
|
1481
|
+
cacheHits: evaluator.cacheHits(),
|
|
1482
|
+
iterations: iteration,
|
|
1483
|
+
stopReason,
|
|
1484
|
+
snapshot: takeSnapshot()
|
|
1485
|
+
};
|
|
1486
|
+
}
|
|
1487
|
+
/**
|
|
1488
|
+
* Range checks on the search knobs, run at construction so a configuration
|
|
1489
|
+
* that could never terminate is refused before a task is ever handed to it.
|
|
1490
|
+
* Task-shaped checks stay in `runGepa`, where the data is.
|
|
1491
|
+
*/
|
|
1492
|
+
function assertGepaConfig(config) {
|
|
1493
|
+
if (config.reflection?.buildPrompt !== void 0 && config.reflection.strategies !== void 0) throw new Error("reflection takes buildPrompt or strategies, not both");
|
|
1494
|
+
if (config.reflection?.strategies?.length === 0) throw new Error("reflection.strategies must not be empty");
|
|
1495
|
+
const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals } = config;
|
|
1496
|
+
const proposalsPerIteration = proposals?.perIteration ?? 1;
|
|
1497
|
+
const proposalConcurrency = proposals?.concurrency ?? 1;
|
|
1498
|
+
if (!Number.isInteger(proposalsPerIteration) || proposalsPerIteration < 1) throw new Error(`proposals.perIteration must be a positive integer, received ${proposalsPerIteration}`);
|
|
1499
|
+
if (!Number.isInteger(proposalConcurrency) || proposalConcurrency < 1) throw new Error(`proposals.concurrency must be a positive integer, received ${proposalConcurrency}`);
|
|
1500
|
+
keepCount(proposals?.selection ?? "all");
|
|
1501
|
+
if (!Number.isInteger(minibatchSize) || minibatchSize < 1) throw new Error(`minibatchSize must be a positive integer, received ${minibatchSize}`);
|
|
1502
|
+
if (!Number.isFinite(perfectScore)) throw new Error(`perfectScore must be a finite number, received ${perfectScore}`);
|
|
1503
|
+
if (!Number.isInteger(rejectedProposalMemory) || rejectedProposalMemory < 0) throw new Error(`rejectedProposalMemory must be a non-negative integer, received ${rejectedProposalMemory}`);
|
|
1504
|
+
if (maxIterations !== Number.POSITIVE_INFINITY && (!Number.isInteger(maxIterations) || maxIterations < 0)) throw new Error(`maxIterations must be a non-negative integer or Infinity, received ${maxIterations}`);
|
|
1505
|
+
}
|
|
1506
|
+
/**
|
|
1507
|
+
* The single narrowing point for a snapshot's candidate pool. A snapshot is
|
|
1508
|
+
* JSON that left the process and came back with plain string keys, so every
|
|
1509
|
+
* record is checked against the seed's components before it is read as one of
|
|
1510
|
+
* them — behind the fingerprint check, which has already established that the
|
|
1511
|
+
* snapshot belongs to this run.
|
|
1512
|
+
*/
|
|
1513
|
+
function restoreRecords(args) {
|
|
1514
|
+
const { records, seedCandidate } = args;
|
|
1515
|
+
const known = new Set(componentNames(seedCandidate));
|
|
1516
|
+
for (const record of records) {
|
|
1517
|
+
const named = [...Object.keys(record.candidate), ...record.updatedComponents];
|
|
1518
|
+
for (const name of named) if (!known.has(name)) throw new Error(`checkpoint names the component "${name}", which the seed candidate does not have (${[...known].join(", ")})`);
|
|
1519
|
+
}
|
|
1520
|
+
return copyRecords(records);
|
|
1521
|
+
}
|
|
1522
|
+
/**
|
|
1523
|
+
* Rejections arrive from a snapshot keyed by plain strings. Reading them
|
|
1524
|
+
* through the seed's own component names narrows them without an assertion,
|
|
1525
|
+
* and drops anything the seed no longer has.
|
|
1526
|
+
*/
|
|
1527
|
+
function restoreRejections(args) {
|
|
1528
|
+
const { rejections, components } = args;
|
|
1529
|
+
const restored = {};
|
|
1530
|
+
for (const component of components) {
|
|
1531
|
+
const history = rejections[component];
|
|
1532
|
+
if (history !== void 0) restored[component] = history.map((entry) => ({ ...entry }));
|
|
1533
|
+
}
|
|
1534
|
+
return restored;
|
|
1535
|
+
}
|
|
1536
|
+
/** The inverse: back to the plain string keys a snapshot is written with. */
|
|
1537
|
+
function snapshotRejections(args) {
|
|
1538
|
+
const { rejections, components } = args;
|
|
1539
|
+
const copy = {};
|
|
1540
|
+
for (const component of components) {
|
|
1541
|
+
const history = rejections[component];
|
|
1542
|
+
if (history !== void 0) copy[component] = history.map((entry) => ({ ...entry }));
|
|
1543
|
+
}
|
|
1544
|
+
return copy;
|
|
1545
|
+
}
|
|
1546
|
+
/**
|
|
1547
|
+
* Patches are merged over the parent, so a name the candidate does not have is
|
|
1548
|
+
* added rather than refused: the run would go on optimizing text the system
|
|
1549
|
+
* under optimization never reads, and every descendant would carry it.
|
|
1550
|
+
*/
|
|
1551
|
+
function assertComponents(args) {
|
|
1552
|
+
const { names, candidate, source } = args;
|
|
1553
|
+
for (const name of names) if (!Object.hasOwn(candidate, name)) throw new Error(`${source} named "${name}", which is not a component of the candidate (${Object.keys(candidate).join(", ")})`);
|
|
1554
|
+
}
|
|
1555
|
+
/** Copies everything a run mutates in place, so a snapshot never aliases one. */
|
|
1556
|
+
function copyRecords(records) {
|
|
1557
|
+
return records.map((record) => ({
|
|
1558
|
+
...record,
|
|
1559
|
+
parentIds: [...record.parentIds],
|
|
1560
|
+
instanceScores: [...record.instanceScores],
|
|
1561
|
+
updatedComponents: [...record.updatedComponents],
|
|
1562
|
+
...record.objectiveScores === void 0 ? {} : { objectiveScores: { ...record.objectiveScores } }
|
|
1563
|
+
}));
|
|
1564
|
+
}
|
|
1565
|
+
/** How many improving proposals an iteration is allowed to keep. */
|
|
1566
|
+
function keepCount(selection) {
|
|
1567
|
+
if (selection === "all") return Number.POSITIVE_INFINITY;
|
|
1568
|
+
if (selection === "best") return 1;
|
|
1569
|
+
if (!Number.isInteger(selection.keep) || selection.keep < 1) throw new Error(`proposals.selection.keep must be a positive integer, received ${selection.keep}`);
|
|
1570
|
+
return selection.keep;
|
|
1571
|
+
}
|
|
1572
|
+
/**
|
|
1573
|
+
* Mean of each objective over the instances this candidate was scored on.
|
|
1574
|
+
*
|
|
1575
|
+
* An objective only some of those instances reported is left out rather than
|
|
1576
|
+
* averaged over the ones that did: candidates are compared objective by
|
|
1577
|
+
* objective on the frontier, and a mean over one instance is not the same
|
|
1578
|
+
* measurement as a mean over forty.
|
|
1579
|
+
*/
|
|
1580
|
+
function meanObjectives(args) {
|
|
1581
|
+
const { rows, scores } = args;
|
|
1582
|
+
const measured = rows.filter((_, index) => scores[index] !== void 0);
|
|
1583
|
+
const totals = /* @__PURE__ */ new Map();
|
|
1584
|
+
for (const row of measured) for (const [objective, value] of Object.entries(row ?? {})) {
|
|
1585
|
+
const running = totals.get(objective) ?? {
|
|
1586
|
+
total: 0,
|
|
1587
|
+
count: 0
|
|
1588
|
+
};
|
|
1589
|
+
running.total += value;
|
|
1590
|
+
running.count += 1;
|
|
1591
|
+
totals.set(objective, running);
|
|
1592
|
+
}
|
|
1593
|
+
const complete = [...totals].filter(([, { count }]) => count === measured.length);
|
|
1594
|
+
if (complete.length === 0) return;
|
|
1595
|
+
return Object.fromEntries(complete.map(([objective, { total, count }]) => [objective, total / count]));
|
|
1596
|
+
}
|
|
1597
|
+
function collectPerObjectiveBest(records) {
|
|
1598
|
+
const bests = objectiveBests(records.map((record) => record.objectiveScores));
|
|
1599
|
+
if (Object.keys(bests).length === 0) return;
|
|
1600
|
+
return Object.fromEntries(Object.entries(bests).map(([objective, score]) => [objective, {
|
|
1601
|
+
score,
|
|
1602
|
+
candidateIds: records.filter((record) => record.objectiveScores?.[objective] === score).map((record) => record.id)
|
|
1603
|
+
}]));
|
|
1604
|
+
}
|
|
1605
|
+
/**
|
|
1606
|
+
* Candidates that uniquely win at least one validation instance once dominated
|
|
1607
|
+
* lineages are pruned. This is both the reported frontier and the pool merge
|
|
1608
|
+
* draws its parents from.
|
|
1609
|
+
*/
|
|
1610
|
+
function collectDominatorIds(records) {
|
|
1611
|
+
const fronts = pruneDominatedFronts({
|
|
1612
|
+
fronts: buildInstanceFronts({ scoreMatrix: records.map((record) => record.instanceScores) }),
|
|
1613
|
+
aggregateScores: records.map((record) => record.aggregateScore)
|
|
1614
|
+
});
|
|
1615
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1616
|
+
for (const front of fronts) for (const id of front) ids.add(id);
|
|
1617
|
+
return [...ids].sort((a, b) => a - b);
|
|
1618
|
+
}
|
|
1619
|
+
/**
|
|
1620
|
+
* Names an instance by a hash of its content rather than by the content
|
|
1621
|
+
* itself: the id ends up inside every cache key and inside the checkpoint
|
|
1622
|
+
* fingerprint, and embedding whole examples there costs memory proportional to
|
|
1623
|
+
* the dataset for no benefit. Data that will not serialize falls back to its
|
|
1624
|
+
* position, which is stable for as long as the dataset order is.
|
|
1625
|
+
*/
|
|
1626
|
+
function defaultInstanceId(args) {
|
|
1627
|
+
const hash = stableHash(args.datum);
|
|
1628
|
+
return hash === "" ? String(args.index) : hash;
|
|
1629
|
+
}
|
|
1630
|
+
/**
|
|
1631
|
+
* The two rollout sets restricted to the instances both of them measured.
|
|
1632
|
+
*
|
|
1633
|
+
* Screening is a paired comparison over one minibatch: a transient row is a
|
|
1634
|
+
* rollout that never happened, and leaving it in scores the candidate that ran
|
|
1635
|
+
* against the infrastructure failure of the one that did not.
|
|
1636
|
+
*/
|
|
1637
|
+
function pairMeasured(args) {
|
|
1638
|
+
const { parent, child } = args;
|
|
1639
|
+
const parentScores = [];
|
|
1640
|
+
const childScores = [];
|
|
1641
|
+
for (let index = 0; index < parent.scores.length; index += 1) {
|
|
1642
|
+
if (parent.transient?.[index] === true || child.transient?.[index] === true) continue;
|
|
1643
|
+
parentScores.push(parent.scores[index]);
|
|
1644
|
+
childScores.push(child.scores[index]);
|
|
1645
|
+
}
|
|
1646
|
+
return {
|
|
1647
|
+
parentScores,
|
|
1648
|
+
childScores
|
|
1649
|
+
};
|
|
1650
|
+
}
|
|
1651
|
+
//#endregion
|
|
1652
|
+
export { GepaOptimizer, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };
|