textopt 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +509 -0
- package/dist/bootstrap-search/index.cjs +308 -0
- package/dist/bootstrap-search/index.d.cts +162 -0
- package/dist/bootstrap-search/index.d.mts +162 -0
- package/dist/bootstrap-search/index.mjs +307 -0
- package/dist/cache-CuSo0NJ8.d.cts +24 -0
- package/dist/cache-CuSo0NJ8.d.mts +24 -0
- package/dist/concurrency-C-cFzWW2.cjs +44 -0
- package/dist/concurrency-D58PWeSk.mjs +39 -0
- package/dist/demos-B0pVQjYC.d.mts +88 -0
- package/dist/demos-B9BJiNKz.cjs +143 -0
- package/dist/demos-BTuzFNsp.d.cts +88 -0
- package/dist/demos-Degx6UmP.mjs +126 -0
- package/dist/evaluation-BV0nSZVx.mjs +521 -0
- package/dist/evaluation-OZOp6TB7.cjs +598 -0
- package/dist/file-cache.cjs +70 -0
- package/dist/file-cache.d.cts +21 -0
- package/dist/file-cache.d.mts +21 -0
- package/dist/file-cache.mjs +69 -0
- package/dist/gepa/index.cjs +1671 -0
- package/dist/gepa/index.d.cts +385 -0
- package/dist/gepa/index.d.mts +385 -0
- package/dist/gepa/index.mjs +1652 -0
- package/dist/index.cjs +266 -0
- package/dist/index.d.cts +221 -0
- package/dist/index.d.mts +221 -0
- package/dist/index.mjs +245 -0
- package/dist/math-COOofUyv.cjs +101 -0
- package/dist/math-DhrDmpFS.mjs +78 -0
- package/dist/mipro/index.cjs +739 -0
- package/dist/mipro/index.d.cts +372 -0
- package/dist/mipro/index.d.mts +372 -0
- package/dist/mipro/index.mjs +736 -0
- package/dist/opro/index.cjs +487 -0
- package/dist/opro/index.d.cts +230 -0
- package/dist/opro/index.d.mts +230 -0
- package/dist/opro/index.mjs +485 -0
- package/dist/optimizer-B7SpRwl7.d.cts +288 -0
- package/dist/optimizer-DqCoth_w.d.mts +288 -0
- package/dist/random-search/index.cjs +321 -0
- package/dist/random-search/index.d.cts +156 -0
- package/dist/random-search/index.d.mts +156 -0
- package/dist/random-search/index.mjs +319 -0
- package/dist/reflection-CQToe-5B.d.cts +283 -0
- package/dist/reflection-Cr_upzU0.d.mts +283 -0
- package/dist/reflection-DRfbk6hu.cjs +249 -0
- package/dist/reflection-mwMhrjs_.mjs +214 -0
- package/dist/rng-BR5MOedA.d.cts +22 -0
- package/dist/rng-BR5MOedA.d.mts +22 -0
- package/dist/rng-DbA_rPIo.cjs +67 -0
- package/dist/rng-Dtc5eZ_W.mjs +62 -0
- package/dist/sampling-CfHt7Gue.mjs +59 -0
- package/dist/sampling-DFo_7RNJ.d.mts +23 -0
- package/dist/sampling-Dars7ctR.cjs +64 -0
- package/dist/sampling-axOwfZf5.d.cts +23 -0
- package/dist/simba/index.cjs +709 -0
- package/dist/simba/index.d.cts +289 -0
- package/dist/simba/index.d.mts +289 -0
- package/dist/simba/index.mjs +700 -0
- package/dist/testing.cjs +155 -0
- package/dist/testing.d.cts +53 -0
- package/dist/testing.d.mts +53 -0
- package/dist/testing.mjs +148 -0
- package/dist/text--v4Ffbus.mjs +21 -0
- package/dist/text-CK_HB3su.cjs +26 -0
- package/dist/types-CWv4IQFF.d.cts +129 -0
- package/dist/types-CWv4IQFF.d.mts +129 -0
- package/package.json +135 -0
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_evaluation = require("../evaluation-OZOp6TB7.cjs");
|
|
3
|
+
const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
|
|
4
|
+
const require_text = require("../text-CK_HB3su.cjs");
|
|
5
|
+
//#region src/random-search/optimize.ts
|
|
6
|
+
const DEFAULT_VARIANTS = 4;
|
|
7
|
+
/**
|
|
8
|
+
* The ablation baseline: propose blind, evaluate in full, keep what wins.
|
|
9
|
+
*
|
|
10
|
+
* It exists to be beaten. Reflective search costs a frontier-model call per
|
|
11
|
+
* proposal on top of its rollouts, and running the same budget through a
|
|
12
|
+
* search that cannot read feedback at all puts a floor under what that call
|
|
13
|
+
* has to buy. A GEPA run that does not clear this one is paying for machinery
|
|
14
|
+
* it is not using.
|
|
15
|
+
*
|
|
16
|
+
* What the gap measures is the whole of that machinery, not reflection alone:
|
|
17
|
+
* this drops feedback, Pareto parent selection, and minibatch screening
|
|
18
|
+
* together, and GEPA's own ablations put candidate selection at several points
|
|
19
|
+
* by itself. For reflection in isolation, keep GEPA and hand it a
|
|
20
|
+
* `reflection.buildPrompt` that withholds the evidence — same frontier, same
|
|
21
|
+
* screening, one variable.
|
|
22
|
+
*
|
|
23
|
+
* Not DSPy's `BootstrapFewShotWithRandomSearch`, which the literature usually
|
|
24
|
+
* means by "random search" and which searches bootstrapped demo sets; nor
|
|
25
|
+
* random search in the Bergstra–Bengio sense, since proposals are paraphrases
|
|
26
|
+
* of the incumbent rather than independent draws. No reference implements this
|
|
27
|
+
* baseline — it is original here, and the GEPA paper has no equivalent.
|
|
28
|
+
*/
|
|
29
|
+
var RandomSearchOptimizer = class {
|
|
30
|
+
#config;
|
|
31
|
+
constructor(config = {}) {
|
|
32
|
+
assertConfig(config);
|
|
33
|
+
this.#config = config;
|
|
34
|
+
}
|
|
35
|
+
async optimize(task) {
|
|
36
|
+
return runRandomSearch({
|
|
37
|
+
config: this.#config,
|
|
38
|
+
task
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* Adapted from the GEPA reflection prompt with every source of evidence
|
|
44
|
+
* removed. What is left is the ablation: the same model, the same component,
|
|
45
|
+
* the same output format, and nothing to reason from.
|
|
46
|
+
*/
|
|
47
|
+
function buildParaphrasePrompt(args) {
|
|
48
|
+
const { componentName, currentText, attempt } = args;
|
|
49
|
+
return [
|
|
50
|
+
`Here is the current instruction for the "${componentName}" component of a larger system:`,
|
|
51
|
+
"",
|
|
52
|
+
"<current_instruction>",
|
|
53
|
+
currentText,
|
|
54
|
+
"</current_instruction>",
|
|
55
|
+
"",
|
|
56
|
+
"Write a different instruction for this component.",
|
|
57
|
+
"You have no information about how the current one has performed, so do not guess at its weaknesses — vary it instead.",
|
|
58
|
+
`Make variation ${attempt + 1} distinct from the others: change the wording, the level of detail, or the strategy it describes.`,
|
|
59
|
+
"",
|
|
60
|
+
"Return only the new instruction, inside a ``` block."
|
|
61
|
+
].join("\n");
|
|
62
|
+
}
|
|
63
|
+
async function runRandomSearch(args) {
|
|
64
|
+
const { config, task } = args;
|
|
65
|
+
const { variants = DEFAULT_VARIANTS, concurrency = 1, maxRounds = Number.POSITIVE_INFINITY, buildPrompt = buildParaphrasePrompt, trackBestOutputs = false, checkpointCache = true } = config;
|
|
66
|
+
const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, onEvent, onCheckpoint, resumeFrom, signal } = task;
|
|
67
|
+
const deadline = require_evaluation.createDeadline({ maxWallClockMs });
|
|
68
|
+
const components = require_evaluation.componentNames(seedCandidate);
|
|
69
|
+
if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
|
|
70
|
+
if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
|
|
71
|
+
if (components.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
|
|
72
|
+
if (testSet !== void 0 && testSet.length === 0) throw new Error("optimize requires a non-empty testSet when one is given; omit it to skip held-out evaluation");
|
|
73
|
+
const validationIds = validationSet.map((datum, index) => instanceId({
|
|
74
|
+
datum,
|
|
75
|
+
index
|
|
76
|
+
}));
|
|
77
|
+
const testIds = testSet?.map((datum, index) => instanceId({
|
|
78
|
+
datum,
|
|
79
|
+
index
|
|
80
|
+
})) ?? [];
|
|
81
|
+
const fingerprint = require_evaluation.runFingerprint({
|
|
82
|
+
seedCandidate,
|
|
83
|
+
trainingIds: trainingSet.map((datum, index) => instanceId({
|
|
84
|
+
datum,
|
|
85
|
+
index
|
|
86
|
+
})),
|
|
87
|
+
validationIds,
|
|
88
|
+
...cacheNamespace === void 0 ? {} : { cacheNamespace }
|
|
89
|
+
});
|
|
90
|
+
require_evaluation.assertResumable({
|
|
91
|
+
fingerprint,
|
|
92
|
+
...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
|
|
93
|
+
});
|
|
94
|
+
const budget = require_evaluation.createBudget({
|
|
95
|
+
maxMetricCalls,
|
|
96
|
+
spent: resumeFrom?.metricCalls ?? 0
|
|
97
|
+
});
|
|
98
|
+
const evaluationCache = cache === false ? void 0 : cache ?? require_evaluation.createMemoryCache();
|
|
99
|
+
const evaluator = require_evaluation.createEvaluator({
|
|
100
|
+
adapter,
|
|
101
|
+
budget,
|
|
102
|
+
...retry === void 0 ? {} : { retry },
|
|
103
|
+
...cacheNamespace === void 0 ? {} : { cacheNamespace },
|
|
104
|
+
...evaluationCache === void 0 ? {} : { cache: evaluationCache },
|
|
105
|
+
trackOutputs: trackBestOutputs,
|
|
106
|
+
cacheHits: resumeFrom?.cacheHits ?? 0,
|
|
107
|
+
...signal === void 0 ? {} : { signal },
|
|
108
|
+
onEvaluation: (event) => onEvent?.({
|
|
109
|
+
type: "evaluation",
|
|
110
|
+
...event
|
|
111
|
+
})
|
|
112
|
+
});
|
|
113
|
+
evaluator.restore(resumeFrom?.cache ?? []);
|
|
114
|
+
let round = resumeFrom?.round ?? 0;
|
|
115
|
+
let variantsEvaluated = resumeFrom?.variantsEvaluated ?? 0;
|
|
116
|
+
let reflectionCalls = resumeFrom?.reflectionCalls ?? 0;
|
|
117
|
+
let stopReason = "maxRounds";
|
|
118
|
+
/**
|
|
119
|
+
* Consecutive rounds that neither spent a rollout nor improved on the
|
|
120
|
+
* incumbent. A proposer stuck on texts that are already cached costs
|
|
121
|
+
* nothing, so neither the metric budget nor the cost ceiling can end the
|
|
122
|
+
* run — without this the loop spins forever, burning reflection calls that
|
|
123
|
+
* no budget here bounds.
|
|
124
|
+
*/
|
|
125
|
+
let stalledRounds = 0;
|
|
126
|
+
onEvent?.({
|
|
127
|
+
type: "start",
|
|
128
|
+
components,
|
|
129
|
+
validationSetSize: validationSet.length
|
|
130
|
+
});
|
|
131
|
+
async function sweep(args) {
|
|
132
|
+
return evaluator.evaluate({
|
|
133
|
+
candidate: args.candidate,
|
|
134
|
+
batch: validationSet,
|
|
135
|
+
ids: validationIds,
|
|
136
|
+
split: "val",
|
|
137
|
+
phase: args.phase,
|
|
138
|
+
candidateId: null,
|
|
139
|
+
iteration: round
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
function takeSnapshot() {
|
|
143
|
+
const cached = checkpointCache ? evaluationCache?.entries?.() : void 0;
|
|
144
|
+
return {
|
|
145
|
+
version: 1,
|
|
146
|
+
fingerprint,
|
|
147
|
+
best,
|
|
148
|
+
bestScore,
|
|
149
|
+
seedScore,
|
|
150
|
+
round,
|
|
151
|
+
variantsEvaluated,
|
|
152
|
+
reflectionCalls,
|
|
153
|
+
metricCalls: budget.spent(),
|
|
154
|
+
cacheHits: evaluator.cacheHits(),
|
|
155
|
+
...cached === void 0 ? {} : { cache: cached }
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
async function checkpoint() {
|
|
159
|
+
if (onCheckpoint === void 0) return;
|
|
160
|
+
await onCheckpoint(takeSnapshot());
|
|
161
|
+
}
|
|
162
|
+
const seedEvaluation = resumeFrom === void 0 ? await sweep({
|
|
163
|
+
candidate: seedCandidate,
|
|
164
|
+
phase: "seed"
|
|
165
|
+
}) : void 0;
|
|
166
|
+
const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_evaluation.requireMeasuredMean({
|
|
167
|
+
batch: seedEvaluation,
|
|
168
|
+
phase: "seed"
|
|
169
|
+
});
|
|
170
|
+
let best = resumeFrom?.best ?? seedCandidate;
|
|
171
|
+
let bestScore = resumeFrom?.bestScore ?? seedScore;
|
|
172
|
+
/** Absent on a resumed run until a variant wins: outputs are not checkpointed. */
|
|
173
|
+
let bestOutputs = seedEvaluation?.outputs;
|
|
174
|
+
await checkpoint();
|
|
175
|
+
while (round < maxRounds) {
|
|
176
|
+
if (signal?.aborted) {
|
|
177
|
+
stopReason = "aborted";
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
180
|
+
if (require_evaluation.costExhausted({
|
|
181
|
+
usage: evaluator.usage(),
|
|
182
|
+
maxCostUsd
|
|
183
|
+
})) {
|
|
184
|
+
stopReason = "costExhausted";
|
|
185
|
+
break;
|
|
186
|
+
}
|
|
187
|
+
if (deadline.exceeded()) {
|
|
188
|
+
stopReason = "deadlineReached";
|
|
189
|
+
break;
|
|
190
|
+
}
|
|
191
|
+
if (!budget.canAfford(variants * validationSet.length)) {
|
|
192
|
+
stopReason = "budgetExhausted";
|
|
193
|
+
break;
|
|
194
|
+
}
|
|
195
|
+
const component = components[round % components.length];
|
|
196
|
+
onEvent?.({
|
|
197
|
+
type: "roundStart",
|
|
198
|
+
round,
|
|
199
|
+
component
|
|
200
|
+
});
|
|
201
|
+
const spentBefore = budget.spent();
|
|
202
|
+
const scoreBefore = bestScore;
|
|
203
|
+
const currentText = best[component];
|
|
204
|
+
const drawn = await require_concurrency.mapWithConcurrency({
|
|
205
|
+
items: Array.from({ length: variants }, (_, attempt) => attempt),
|
|
206
|
+
limit: concurrency,
|
|
207
|
+
signal,
|
|
208
|
+
task: async (attempt) => {
|
|
209
|
+
reflectionCalls += 1;
|
|
210
|
+
const response = await reflect({
|
|
211
|
+
prompt: buildPrompt({
|
|
212
|
+
componentName: component,
|
|
213
|
+
currentText,
|
|
214
|
+
attempt
|
|
215
|
+
}),
|
|
216
|
+
...signal === void 0 ? {} : { signal }
|
|
217
|
+
});
|
|
218
|
+
return require_text.parseProposedText(response);
|
|
219
|
+
}
|
|
220
|
+
});
|
|
221
|
+
const unique = [...new Set(drawn)].filter((text) => text.length > 0 && text !== currentText);
|
|
222
|
+
let roundStop;
|
|
223
|
+
for (const text of unique) {
|
|
224
|
+
const candidate = {
|
|
225
|
+
...best,
|
|
226
|
+
[component]: text
|
|
227
|
+
};
|
|
228
|
+
let evaluation;
|
|
229
|
+
try {
|
|
230
|
+
evaluation = await sweep({
|
|
231
|
+
candidate,
|
|
232
|
+
phase: "validation"
|
|
233
|
+
});
|
|
234
|
+
} catch (err) {
|
|
235
|
+
if (err instanceof require_evaluation.BudgetExhausted) {
|
|
236
|
+
roundStop = "budgetExhausted";
|
|
237
|
+
break;
|
|
238
|
+
}
|
|
239
|
+
if (signal?.aborted) {
|
|
240
|
+
roundStop = "aborted";
|
|
241
|
+
break;
|
|
242
|
+
}
|
|
243
|
+
throw err;
|
|
244
|
+
}
|
|
245
|
+
variantsEvaluated += 1;
|
|
246
|
+
const score = require_evaluation.measuredMean(evaluation);
|
|
247
|
+
if (score !== void 0 && score > bestScore) {
|
|
248
|
+
onEvent?.({
|
|
249
|
+
type: "candidateAccepted",
|
|
250
|
+
round,
|
|
251
|
+
component,
|
|
252
|
+
score,
|
|
253
|
+
previousScore: bestScore
|
|
254
|
+
});
|
|
255
|
+
best = candidate;
|
|
256
|
+
bestScore = score;
|
|
257
|
+
bestOutputs = evaluation.outputs;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
round += 1;
|
|
261
|
+
stalledRounds = budget.spent() === spentBefore && bestScore === scoreBefore ? stalledRounds + 1 : 0;
|
|
262
|
+
await checkpoint();
|
|
263
|
+
if (stalledRounds >= components.length) {
|
|
264
|
+
stopReason = "proposerStalled";
|
|
265
|
+
break;
|
|
266
|
+
}
|
|
267
|
+
if (roundStop !== void 0) {
|
|
268
|
+
stopReason = roundStop;
|
|
269
|
+
break;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
if (signal?.aborted) stopReason = "aborted";
|
|
273
|
+
const testScore = testSet === void 0 ? void 0 : require_evaluation.measuredMean(await evaluator.evaluate({
|
|
274
|
+
candidate: best,
|
|
275
|
+
batch: testSet,
|
|
276
|
+
ids: testIds,
|
|
277
|
+
split: "test",
|
|
278
|
+
phase: "test",
|
|
279
|
+
candidateId: null,
|
|
280
|
+
iteration: round,
|
|
281
|
+
charge: false
|
|
282
|
+
}));
|
|
283
|
+
onEvent?.({
|
|
284
|
+
type: "finish",
|
|
285
|
+
reason: stopReason,
|
|
286
|
+
bestScore,
|
|
287
|
+
metricCalls: budget.spent(),
|
|
288
|
+
...testScore === void 0 ? {} : { testScore }
|
|
289
|
+
});
|
|
290
|
+
return {
|
|
291
|
+
bestCandidate: best,
|
|
292
|
+
bestScore,
|
|
293
|
+
usage: evaluator.usage(),
|
|
294
|
+
seedScore,
|
|
295
|
+
snapshot: takeSnapshot(),
|
|
296
|
+
...trackBestOutputs ? { bestOutputs } : {},
|
|
297
|
+
...testScore === void 0 ? {} : {
|
|
298
|
+
testScore,
|
|
299
|
+
testMetricCalls: evaluator.unchargedCalls()
|
|
300
|
+
},
|
|
301
|
+
rounds: round,
|
|
302
|
+
variantsEvaluated,
|
|
303
|
+
metricCalls: budget.spent(),
|
|
304
|
+
reflectionCalls,
|
|
305
|
+
cacheHits: evaluator.cacheHits(),
|
|
306
|
+
stopReason
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
function assertConfig(config) {
|
|
310
|
+
const { variants = DEFAULT_VARIANTS, concurrency = 1, maxRounds } = config;
|
|
311
|
+
if (!Number.isInteger(variants) || variants < 1) throw new Error(`variants must be a positive integer, received ${variants}`);
|
|
312
|
+
if (!Number.isInteger(concurrency) || concurrency < 1) throw new Error(`concurrency must be a positive integer, received ${concurrency}`);
|
|
313
|
+
if (maxRounds !== void 0 && (!Number.isInteger(maxRounds) || maxRounds < 1)) throw new Error(`maxRounds must be a positive integer, received ${maxRounds}`);
|
|
314
|
+
}
|
|
315
|
+
function defaultInstanceId(args) {
|
|
316
|
+
const hash = require_evaluation.stableHash(args.datum);
|
|
317
|
+
return hash === "" ? String(args.index) : hash;
|
|
318
|
+
}
|
|
319
|
+
//#endregion
|
|
320
|
+
exports.RandomSearchOptimizer = RandomSearchOptimizer;
|
|
321
|
+
exports.buildParaphrasePrompt = buildParaphrasePrompt;
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { n as Candidate, t as Adapter, u as TextModel } from "../types-CWv4IQFF.cjs";
|
|
2
|
+
import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.cjs";
|
|
3
|
+
import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-B7SpRwl7.cjs";
|
|
4
|
+
//#region src/random-search/optimize.d.ts
|
|
5
|
+
/** Builds the prompt one variant is drawn from. */
|
|
6
|
+
type ParaphrasePromptBuilder = (args: {
|
|
7
|
+
componentName: string;
|
|
8
|
+
currentText: string;
|
|
9
|
+
/** Which of this round's variants is being drawn, from 0. */
|
|
10
|
+
attempt: number;
|
|
11
|
+
}) => string;
|
|
12
|
+
/**
|
|
13
|
+
* How the baseline searches. Deliberately small: every knob GEPA has that this
|
|
14
|
+
* one lacks is a knob whose value GEPA has to earn.
|
|
15
|
+
*/
|
|
16
|
+
interface RandomSearchConfig {
|
|
17
|
+
/** Variants drawn per round, each evaluated in full. Default 4. */
|
|
18
|
+
variants?: number;
|
|
19
|
+
/** How many of them may be in flight at once. Default 1. */
|
|
20
|
+
concurrency?: number;
|
|
21
|
+
maxRounds?: number;
|
|
22
|
+
/** Replaces the default paraphrase template. */
|
|
23
|
+
buildPrompt?: ParaphrasePromptBuilder;
|
|
24
|
+
/** Keep what the winner produced on each validation instance. */
|
|
25
|
+
trackBestOutputs?: boolean;
|
|
26
|
+
/**
|
|
27
|
+
* Include cached instance scores in every checkpoint. Leaving them out keeps
|
|
28
|
+
* snapshots small at the cost of a resumed run re-paying for rollouts it
|
|
29
|
+
* cannot look up. Default true.
|
|
30
|
+
*/
|
|
31
|
+
checkpointCache?: boolean;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Everything needed to continue a run: the incumbent and its score, the budget
|
|
35
|
+
* already spent, and the bookkeeping that stops the seed from being re-scored.
|
|
36
|
+
* Plain JSON — persist it and hand it back as `resumeFrom`.
|
|
37
|
+
*
|
|
38
|
+
* There is no random stream here because this search has none: its variants
|
|
39
|
+
* come from the proposer alone.
|
|
40
|
+
*/
|
|
41
|
+
interface RandomSearchSnapshot {
|
|
42
|
+
version: 1;
|
|
43
|
+
/**
|
|
44
|
+
* Identifies the run this came from — seed candidate, instance ids, cache
|
|
45
|
+
* namespace. Resuming against a different setup is refused rather than
|
|
46
|
+
* silently scoring an old incumbent against new data.
|
|
47
|
+
*/
|
|
48
|
+
fingerprint: string;
|
|
49
|
+
best: Candidate;
|
|
50
|
+
bestScore: number;
|
|
51
|
+
seedScore: number;
|
|
52
|
+
round: number;
|
|
53
|
+
variantsEvaluated: number;
|
|
54
|
+
reflectionCalls: number;
|
|
55
|
+
metricCalls: number;
|
|
56
|
+
cacheHits: number;
|
|
57
|
+
/** Cached instance scores, when the cache can enumerate them. */
|
|
58
|
+
cache?: [string, CachedScore][];
|
|
59
|
+
}
|
|
60
|
+
interface RandomSearchTask<Datum, Trajectory = unknown, Output = unknown, K extends string = string> extends OptimizerTask<Datum, Trajectory, Output, K> {
|
|
61
|
+
/**
|
|
62
|
+
* The base adapter, not `GepaAdapter`: this search never reflects, so it has
|
|
63
|
+
* no use for a reflective dataset and does not ask for one.
|
|
64
|
+
*/
|
|
65
|
+
adapter: Adapter<Datum, Trajectory, Output, NoInfer<K>>;
|
|
66
|
+
/** Rewrites a component's text. Sees the text and nothing else. */
|
|
67
|
+
reflect: TextModel;
|
|
68
|
+
instanceId?: (args: {
|
|
69
|
+
datum: NoInfer<Datum>;
|
|
70
|
+
index: number;
|
|
71
|
+
}) => string;
|
|
72
|
+
/** Pass `false` to disable caching entirely. */
|
|
73
|
+
cache?: EvaluationCache | false;
|
|
74
|
+
onEvent?: (event: RandomSearchEvent<NoInfer<K>>) => void;
|
|
75
|
+
/**
|
|
76
|
+
* Called with a resumable snapshot after the seed is scored and after every
|
|
77
|
+
* round. Persist it and a killed run costs the last round, not all of them.
|
|
78
|
+
*/
|
|
79
|
+
onCheckpoint?: (snapshot: RandomSearchSnapshot) => void | Promise<void>;
|
|
80
|
+
/** Snapshot to continue from, instead of starting at the seed candidate. */
|
|
81
|
+
resumeFrom?: RandomSearchSnapshot;
|
|
82
|
+
}
|
|
83
|
+
type RandomSearchStopReason = "budgetExhausted" | "costExhausted" | "deadlineReached" | "maxRounds" | "proposerStalled" | "aborted";
|
|
84
|
+
type RandomSearchEvent<K extends string = string> = {
|
|
85
|
+
type: "start";
|
|
86
|
+
components: K[];
|
|
87
|
+
validationSetSize: number;
|
|
88
|
+
} | {
|
|
89
|
+
type: "roundStart";
|
|
90
|
+
round: number;
|
|
91
|
+
component: K;
|
|
92
|
+
} | ({
|
|
93
|
+
type: "evaluation";
|
|
94
|
+
} & EvaluationEvent) | {
|
|
95
|
+
type: "candidateAccepted";
|
|
96
|
+
round: number;
|
|
97
|
+
component: K;
|
|
98
|
+
score: number;
|
|
99
|
+
previousScore: number;
|
|
100
|
+
} | {
|
|
101
|
+
type: "finish";
|
|
102
|
+
reason: RandomSearchStopReason;
|
|
103
|
+
bestScore: number;
|
|
104
|
+
metricCalls: number;
|
|
105
|
+
testScore?: number;
|
|
106
|
+
};
|
|
107
|
+
interface RandomSearchResult<K extends string = string, Output = unknown> extends OptimizerResult<K, RandomSearchStopReason, Output> {
|
|
108
|
+
/** The seed's score, so the lift the search bought is readable directly. */
|
|
109
|
+
seedScore: number;
|
|
110
|
+
rounds: number;
|
|
111
|
+
/** Variants drawn and evaluated, including the ones that lost. */
|
|
112
|
+
variantsEvaluated: number;
|
|
113
|
+
reflectionCalls: number;
|
|
114
|
+
cacheHits: number;
|
|
115
|
+
/** State as of the last round, ready to hand back as `resumeFrom`. */
|
|
116
|
+
snapshot: RandomSearchSnapshot;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* The ablation baseline: propose blind, evaluate in full, keep what wins.
|
|
120
|
+
*
|
|
121
|
+
* It exists to be beaten. Reflective search costs a frontier-model call per
|
|
122
|
+
* proposal on top of its rollouts, and running the same budget through a
|
|
123
|
+
* search that cannot read feedback at all puts a floor under what that call
|
|
124
|
+
* has to buy. A GEPA run that does not clear this one is paying for machinery
|
|
125
|
+
* it is not using.
|
|
126
|
+
*
|
|
127
|
+
* What the gap measures is the whole of that machinery, not reflection alone:
|
|
128
|
+
* this drops feedback, Pareto parent selection, and minibatch screening
|
|
129
|
+
* together, and GEPA's own ablations put candidate selection at several points
|
|
130
|
+
* by itself. For reflection in isolation, keep GEPA and hand it a
|
|
131
|
+
* `reflection.buildPrompt` that withholds the evidence — same frontier, same
|
|
132
|
+
* screening, one variable.
|
|
133
|
+
*
|
|
134
|
+
* Not DSPy's `BootstrapFewShotWithRandomSearch`, which the literature usually
|
|
135
|
+
* means by "random search" and which searches bootstrapped demo sets; nor
|
|
136
|
+
* random search in the Bergstra–Bengio sense, since proposals are paraphrases
|
|
137
|
+
* of the incumbent rather than independent draws. No reference implements this
|
|
138
|
+
* baseline — it is original here, and the GEPA paper has no equivalent.
|
|
139
|
+
*/
|
|
140
|
+
declare class RandomSearchOptimizer implements Optimizer<RandomSearchStopReason> {
|
|
141
|
+
#private;
|
|
142
|
+
constructor(config?: RandomSearchConfig);
|
|
143
|
+
optimize<Datum, Trajectory = unknown, Output = unknown, const K extends string = string>(task: RandomSearchTask<Datum, Trajectory, Output, K>): Promise<RandomSearchResult<K, Output>>;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Adapted from the GEPA reflection prompt with every source of evidence
|
|
147
|
+
* removed. What is left is the ablation: the same model, the same component,
|
|
148
|
+
* the same output format, and nothing to reason from.
|
|
149
|
+
*/
|
|
150
|
+
declare function buildParaphrasePrompt(args: {
|
|
151
|
+
componentName: string;
|
|
152
|
+
currentText: string;
|
|
153
|
+
attempt: number;
|
|
154
|
+
}): string;
|
|
155
|
+
//#endregion
|
|
156
|
+
export { type ParaphrasePromptBuilder, type RandomSearchConfig, type RandomSearchEvent, RandomSearchOptimizer, type RandomSearchResult, type RandomSearchSnapshot, type RandomSearchStopReason, type RandomSearchTask, buildParaphrasePrompt };
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { n as Candidate, t as Adapter, u as TextModel } from "../types-CWv4IQFF.mjs";
|
|
2
|
+
import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.mjs";
|
|
3
|
+
import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-DqCoth_w.mjs";
|
|
4
|
+
//#region src/random-search/optimize.d.ts
|
|
5
|
+
/** Builds the prompt one variant is drawn from. */
|
|
6
|
+
type ParaphrasePromptBuilder = (args: {
|
|
7
|
+
componentName: string;
|
|
8
|
+
currentText: string;
|
|
9
|
+
/** Which of this round's variants is being drawn, from 0. */
|
|
10
|
+
attempt: number;
|
|
11
|
+
}) => string;
|
|
12
|
+
/**
|
|
13
|
+
* How the baseline searches. Deliberately small: every knob GEPA has that this
|
|
14
|
+
* one lacks is a knob whose value GEPA has to earn.
|
|
15
|
+
*/
|
|
16
|
+
interface RandomSearchConfig {
|
|
17
|
+
/** Variants drawn per round, each evaluated in full. Default 4. */
|
|
18
|
+
variants?: number;
|
|
19
|
+
/** How many of them may be in flight at once. Default 1. */
|
|
20
|
+
concurrency?: number;
|
|
21
|
+
maxRounds?: number;
|
|
22
|
+
/** Replaces the default paraphrase template. */
|
|
23
|
+
buildPrompt?: ParaphrasePromptBuilder;
|
|
24
|
+
/** Keep what the winner produced on each validation instance. */
|
|
25
|
+
trackBestOutputs?: boolean;
|
|
26
|
+
/**
|
|
27
|
+
* Include cached instance scores in every checkpoint. Leaving them out keeps
|
|
28
|
+
* snapshots small at the cost of a resumed run re-paying for rollouts it
|
|
29
|
+
* cannot look up. Default true.
|
|
30
|
+
*/
|
|
31
|
+
checkpointCache?: boolean;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Everything needed to continue a run: the incumbent and its score, the budget
|
|
35
|
+
* already spent, and the bookkeeping that stops the seed from being re-scored.
|
|
36
|
+
* Plain JSON — persist it and hand it back as `resumeFrom`.
|
|
37
|
+
*
|
|
38
|
+
* There is no random stream here because this search has none: its variants
|
|
39
|
+
* come from the proposer alone.
|
|
40
|
+
*/
|
|
41
|
+
interface RandomSearchSnapshot {
|
|
42
|
+
version: 1;
|
|
43
|
+
/**
|
|
44
|
+
* Identifies the run this came from — seed candidate, instance ids, cache
|
|
45
|
+
* namespace. Resuming against a different setup is refused rather than
|
|
46
|
+
* silently scoring an old incumbent against new data.
|
|
47
|
+
*/
|
|
48
|
+
fingerprint: string;
|
|
49
|
+
best: Candidate;
|
|
50
|
+
bestScore: number;
|
|
51
|
+
seedScore: number;
|
|
52
|
+
round: number;
|
|
53
|
+
variantsEvaluated: number;
|
|
54
|
+
reflectionCalls: number;
|
|
55
|
+
metricCalls: number;
|
|
56
|
+
cacheHits: number;
|
|
57
|
+
/** Cached instance scores, when the cache can enumerate them. */
|
|
58
|
+
cache?: [string, CachedScore][];
|
|
59
|
+
}
|
|
60
|
+
interface RandomSearchTask<Datum, Trajectory = unknown, Output = unknown, K extends string = string> extends OptimizerTask<Datum, Trajectory, Output, K> {
|
|
61
|
+
/**
|
|
62
|
+
* The base adapter, not `GepaAdapter`: this search never reflects, so it has
|
|
63
|
+
* no use for a reflective dataset and does not ask for one.
|
|
64
|
+
*/
|
|
65
|
+
adapter: Adapter<Datum, Trajectory, Output, NoInfer<K>>;
|
|
66
|
+
/** Rewrites a component's text. Sees the text and nothing else. */
|
|
67
|
+
reflect: TextModel;
|
|
68
|
+
instanceId?: (args: {
|
|
69
|
+
datum: NoInfer<Datum>;
|
|
70
|
+
index: number;
|
|
71
|
+
}) => string;
|
|
72
|
+
/** Pass `false` to disable caching entirely. */
|
|
73
|
+
cache?: EvaluationCache | false;
|
|
74
|
+
onEvent?: (event: RandomSearchEvent<NoInfer<K>>) => void;
|
|
75
|
+
/**
|
|
76
|
+
* Called with a resumable snapshot after the seed is scored and after every
|
|
77
|
+
* round. Persist it and a killed run costs the last round, not all of them.
|
|
78
|
+
*/
|
|
79
|
+
onCheckpoint?: (snapshot: RandomSearchSnapshot) => void | Promise<void>;
|
|
80
|
+
/** Snapshot to continue from, instead of starting at the seed candidate. */
|
|
81
|
+
resumeFrom?: RandomSearchSnapshot;
|
|
82
|
+
}
|
|
83
|
+
type RandomSearchStopReason = "budgetExhausted" | "costExhausted" | "deadlineReached" | "maxRounds" | "proposerStalled" | "aborted";
|
|
84
|
+
type RandomSearchEvent<K extends string = string> = {
|
|
85
|
+
type: "start";
|
|
86
|
+
components: K[];
|
|
87
|
+
validationSetSize: number;
|
|
88
|
+
} | {
|
|
89
|
+
type: "roundStart";
|
|
90
|
+
round: number;
|
|
91
|
+
component: K;
|
|
92
|
+
} | ({
|
|
93
|
+
type: "evaluation";
|
|
94
|
+
} & EvaluationEvent) | {
|
|
95
|
+
type: "candidateAccepted";
|
|
96
|
+
round: number;
|
|
97
|
+
component: K;
|
|
98
|
+
score: number;
|
|
99
|
+
previousScore: number;
|
|
100
|
+
} | {
|
|
101
|
+
type: "finish";
|
|
102
|
+
reason: RandomSearchStopReason;
|
|
103
|
+
bestScore: number;
|
|
104
|
+
metricCalls: number;
|
|
105
|
+
testScore?: number;
|
|
106
|
+
};
|
|
107
|
+
interface RandomSearchResult<K extends string = string, Output = unknown> extends OptimizerResult<K, RandomSearchStopReason, Output> {
|
|
108
|
+
/** The seed's score, so the lift the search bought is readable directly. */
|
|
109
|
+
seedScore: number;
|
|
110
|
+
rounds: number;
|
|
111
|
+
/** Variants drawn and evaluated, including the ones that lost. */
|
|
112
|
+
variantsEvaluated: number;
|
|
113
|
+
reflectionCalls: number;
|
|
114
|
+
cacheHits: number;
|
|
115
|
+
/** State as of the last round, ready to hand back as `resumeFrom`. */
|
|
116
|
+
snapshot: RandomSearchSnapshot;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* The ablation baseline: propose blind, evaluate in full, keep what wins.
|
|
120
|
+
*
|
|
121
|
+
* It exists to be beaten. Reflective search costs a frontier-model call per
|
|
122
|
+
* proposal on top of its rollouts, and running the same budget through a
|
|
123
|
+
* search that cannot read feedback at all puts a floor under what that call
|
|
124
|
+
* has to buy. A GEPA run that does not clear this one is paying for machinery
|
|
125
|
+
* it is not using.
|
|
126
|
+
*
|
|
127
|
+
* What the gap measures is the whole of that machinery, not reflection alone:
|
|
128
|
+
* this drops feedback, Pareto parent selection, and minibatch screening
|
|
129
|
+
* together, and GEPA's own ablations put candidate selection at several points
|
|
130
|
+
* by itself. For reflection in isolation, keep GEPA and hand it a
|
|
131
|
+
* `reflection.buildPrompt` that withholds the evidence — same frontier, same
|
|
132
|
+
* screening, one variable.
|
|
133
|
+
*
|
|
134
|
+
* Not DSPy's `BootstrapFewShotWithRandomSearch`, which the literature usually
|
|
135
|
+
* means by "random search" and which searches bootstrapped demo sets; nor
|
|
136
|
+
* random search in the Bergstra–Bengio sense, since proposals are paraphrases
|
|
137
|
+
* of the incumbent rather than independent draws. No reference implements this
|
|
138
|
+
* baseline — it is original here, and the GEPA paper has no equivalent.
|
|
139
|
+
*/
|
|
140
|
+
declare class RandomSearchOptimizer implements Optimizer<RandomSearchStopReason> {
|
|
141
|
+
#private;
|
|
142
|
+
constructor(config?: RandomSearchConfig);
|
|
143
|
+
optimize<Datum, Trajectory = unknown, Output = unknown, const K extends string = string>(task: RandomSearchTask<Datum, Trajectory, Output, K>): Promise<RandomSearchResult<K, Output>>;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Adapted from the GEPA reflection prompt with every source of evidence
|
|
147
|
+
* removed. What is left is the ablation: the same model, the same component,
|
|
148
|
+
* the same output format, and nothing to reason from.
|
|
149
|
+
*/
|
|
150
|
+
declare function buildParaphrasePrompt(args: {
|
|
151
|
+
componentName: string;
|
|
152
|
+
currentText: string;
|
|
153
|
+
attempt: number;
|
|
154
|
+
}): string;
|
|
155
|
+
//#endregion
|
|
156
|
+
export { type ParaphrasePromptBuilder, type RandomSearchConfig, type RandomSearchEvent, RandomSearchOptimizer, type RandomSearchResult, type RandomSearchSnapshot, type RandomSearchStopReason, type RandomSearchTask, buildParaphrasePrompt };
|