textopt 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +509 -0
- package/dist/bootstrap-search/index.cjs +308 -0
- package/dist/bootstrap-search/index.d.cts +162 -0
- package/dist/bootstrap-search/index.d.mts +162 -0
- package/dist/bootstrap-search/index.mjs +307 -0
- package/dist/cache-CuSo0NJ8.d.cts +24 -0
- package/dist/cache-CuSo0NJ8.d.mts +24 -0
- package/dist/concurrency-C-cFzWW2.cjs +44 -0
- package/dist/concurrency-D58PWeSk.mjs +39 -0
- package/dist/demos-B0pVQjYC.d.mts +88 -0
- package/dist/demos-B9BJiNKz.cjs +143 -0
- package/dist/demos-BTuzFNsp.d.cts +88 -0
- package/dist/demos-Degx6UmP.mjs +126 -0
- package/dist/evaluation-BV0nSZVx.mjs +521 -0
- package/dist/evaluation-OZOp6TB7.cjs +598 -0
- package/dist/file-cache.cjs +70 -0
- package/dist/file-cache.d.cts +21 -0
- package/dist/file-cache.d.mts +21 -0
- package/dist/file-cache.mjs +69 -0
- package/dist/gepa/index.cjs +1671 -0
- package/dist/gepa/index.d.cts +385 -0
- package/dist/gepa/index.d.mts +385 -0
- package/dist/gepa/index.mjs +1652 -0
- package/dist/index.cjs +266 -0
- package/dist/index.d.cts +221 -0
- package/dist/index.d.mts +221 -0
- package/dist/index.mjs +245 -0
- package/dist/math-COOofUyv.cjs +101 -0
- package/dist/math-DhrDmpFS.mjs +78 -0
- package/dist/mipro/index.cjs +739 -0
- package/dist/mipro/index.d.cts +372 -0
- package/dist/mipro/index.d.mts +372 -0
- package/dist/mipro/index.mjs +736 -0
- package/dist/opro/index.cjs +487 -0
- package/dist/opro/index.d.cts +230 -0
- package/dist/opro/index.d.mts +230 -0
- package/dist/opro/index.mjs +485 -0
- package/dist/optimizer-B7SpRwl7.d.cts +288 -0
- package/dist/optimizer-DqCoth_w.d.mts +288 -0
- package/dist/random-search/index.cjs +321 -0
- package/dist/random-search/index.d.cts +156 -0
- package/dist/random-search/index.d.mts +156 -0
- package/dist/random-search/index.mjs +319 -0
- package/dist/reflection-CQToe-5B.d.cts +283 -0
- package/dist/reflection-Cr_upzU0.d.mts +283 -0
- package/dist/reflection-DRfbk6hu.cjs +249 -0
- package/dist/reflection-mwMhrjs_.mjs +214 -0
- package/dist/rng-BR5MOedA.d.cts +22 -0
- package/dist/rng-BR5MOedA.d.mts +22 -0
- package/dist/rng-DbA_rPIo.cjs +67 -0
- package/dist/rng-Dtc5eZ_W.mjs +62 -0
- package/dist/sampling-CfHt7Gue.mjs +59 -0
- package/dist/sampling-DFo_7RNJ.d.mts +23 -0
- package/dist/sampling-Dars7ctR.cjs +64 -0
- package/dist/sampling-axOwfZf5.d.cts +23 -0
- package/dist/simba/index.cjs +709 -0
- package/dist/simba/index.d.cts +289 -0
- package/dist/simba/index.d.mts +289 -0
- package/dist/simba/index.mjs +700 -0
- package/dist/testing.cjs +155 -0
- package/dist/testing.d.cts +53 -0
- package/dist/testing.d.mts +53 -0
- package/dist/testing.mjs +148 -0
- package/dist/text--v4Ffbus.mjs +21 -0
- package/dist/text-CK_HB3su.cjs +26 -0
- package/dist/types-CWv4IQFF.d.cts +129 -0
- package/dist/types-CWv4IQFF.d.mts +129 -0
- package/package.json +135 -0
|
@@ -0,0 +1,598 @@
|
|
|
1
|
+
//#region src/budget.ts
|
|
2
|
+
/**
|
|
3
|
+
* Rollouts are the currency of prompt optimization: an optimizer's cost is
|
|
4
|
+
* measured in metric calls, not iterations. The engine debits this budget
|
|
5
|
+
* before every evaluation and stops when it can no longer afford the next one.
|
|
6
|
+
*
|
|
7
|
+
* Debiting happens up front, as an atomic reserve-then-refund rather than a
|
|
8
|
+
* check followed by a charge: proposals evaluated concurrently would otherwise
|
|
9
|
+
* each see the same remaining allowance and all spend it.
|
|
10
|
+
*/
|
|
11
|
+
function createBudget(args) {
|
|
12
|
+
const { maxMetricCalls, spent = 0 } = args;
|
|
13
|
+
if (!Number.isFinite(maxMetricCalls) || maxMetricCalls <= 0) throw new Error(`maxMetricCalls must be a positive number, received ${maxMetricCalls}`);
|
|
14
|
+
let used = spent;
|
|
15
|
+
return {
|
|
16
|
+
maxMetricCalls,
|
|
17
|
+
spent: () => used,
|
|
18
|
+
remaining: () => maxMetricCalls - used,
|
|
19
|
+
canAfford: (calls) => used + calls <= maxMetricCalls,
|
|
20
|
+
reserve: (calls) => {
|
|
21
|
+
if (used + calls > maxMetricCalls) return false;
|
|
22
|
+
used += calls;
|
|
23
|
+
return true;
|
|
24
|
+
},
|
|
25
|
+
refund: (calls) => {
|
|
26
|
+
if (calls > used) throw new Error(`Cannot refund ${calls} metric calls; only ${used} are reserved`);
|
|
27
|
+
used -= calls;
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
//#endregion
|
|
32
|
+
//#region src/cache.ts
|
|
33
|
+
/**
|
|
34
|
+
* The loop re-evaluates unchanged candidates against the same validation
|
|
35
|
+
* instances constantly (every accepted child inherits most of its parent's
|
|
36
|
+
* text). Caching per (split, candidate text, instance) is the single largest
|
|
37
|
+
* cost saver available, and cached hits are not charged to the metric budget.
|
|
38
|
+
*/
|
|
39
|
+
function evaluationCacheKey(args) {
|
|
40
|
+
const { hash, instanceId, split, namespace } = args;
|
|
41
|
+
return `${namespace === void 0 ? split : `${namespace}:${split}`}:${hash}:${instanceId}`;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Identifies a candidate by its text. Hoisted out of the key so a sweep over a
|
|
45
|
+
* thousand validation instances hashes the candidate once instead of a
|
|
46
|
+
* thousand times — the candidate is the long part of the key, the instance id
|
|
47
|
+
* is not.
|
|
48
|
+
*/
|
|
49
|
+
function candidateHash(candidate) {
|
|
50
|
+
const serialized = Object.keys(candidate).sort().map((name) => `${name}\u0000${candidate[name]}`).join("");
|
|
51
|
+
return `${hash32(serialized, 2166136261)}${hash32(serialized, 16777619)}`;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* A short, collision-resistant id for arbitrary data, used to name validation
|
|
55
|
+
* instances. Two independent 32-bit passes rather than one: a single pass
|
|
56
|
+
* collides at roughly one pair per thousand instances, and a collision here
|
|
57
|
+
* would silently share cached scores between two different examples.
|
|
58
|
+
*/
|
|
59
|
+
function stableHash(value) {
|
|
60
|
+
let serialized;
|
|
61
|
+
try {
|
|
62
|
+
serialized = JSON.stringify(value) ?? "undefined";
|
|
63
|
+
} catch {
|
|
64
|
+
return "";
|
|
65
|
+
}
|
|
66
|
+
return `${hash32(serialized, 2166136261)}${hash32(serialized, 16777619)}`;
|
|
67
|
+
}
|
|
68
|
+
function createMemoryCache(args = {}) {
|
|
69
|
+
const { maxEntries = 1e5, entries: initial = [] } = args;
|
|
70
|
+
const entries = new Map(initial);
|
|
71
|
+
return {
|
|
72
|
+
get: (key) => entries.get(key),
|
|
73
|
+
set: (key, cached) => {
|
|
74
|
+
if (entries.size >= maxEntries && !entries.has(key)) {
|
|
75
|
+
const oldest = entries.keys().next();
|
|
76
|
+
if (!oldest.done) entries.delete(oldest.value);
|
|
77
|
+
}
|
|
78
|
+
entries.set(key, cached);
|
|
79
|
+
},
|
|
80
|
+
entries: () => [...entries]
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
function hash32(value, seed) {
|
|
84
|
+
let hash = seed;
|
|
85
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
86
|
+
hash ^= value.charCodeAt(index);
|
|
87
|
+
hash = Math.imul(hash, 16777619);
|
|
88
|
+
}
|
|
89
|
+
return (hash >>> 0).toString(36).padStart(7, "0");
|
|
90
|
+
}
|
|
91
|
+
//#endregion
|
|
92
|
+
//#region src/types.ts
|
|
93
|
+
/**
|
|
94
|
+
* The component names of a candidate, as the union they were inferred from.
|
|
95
|
+
*
|
|
96
|
+
* `Object.keys` widens a closed key union back to `string`. This is the one
|
|
97
|
+
* place that narrowing happens, so every other caller stays assertion-free.
|
|
98
|
+
* Accepts a partial so it also names the components of a component patch.
|
|
99
|
+
*/
|
|
100
|
+
function componentNames(candidate) {
|
|
101
|
+
return Object.keys(candidate);
|
|
102
|
+
}
|
|
103
|
+
//#endregion
|
|
104
|
+
//#region src/checkpoint.ts
|
|
105
|
+
/**
|
|
106
|
+
* Identifies the run a checkpoint came from: seed candidate, instance ids,
|
|
107
|
+
* seed, and cache namespace.
|
|
108
|
+
*
|
|
109
|
+
* Hashed rather than embedded, because it goes into every snapshot and is only
|
|
110
|
+
* ever compared for equality. The namespace is part of it because a snapshot
|
|
111
|
+
* carries cached scores, and resuming under a different one would replay
|
|
112
|
+
* measurements of a system the run is no longer running. The test set is
|
|
113
|
+
* deliberately absent: it never touches selection, so adding one to a resumed
|
|
114
|
+
* run changes nothing about what that run would have done.
|
|
115
|
+
*/
|
|
116
|
+
function runFingerprint(args) {
|
|
117
|
+
const { seedCandidate, trainingIds, validationIds, seed, cacheNamespace } = args;
|
|
118
|
+
return stableHash({
|
|
119
|
+
seed,
|
|
120
|
+
seedCandidate: candidateFingerprint(seedCandidate),
|
|
121
|
+
trainingIds,
|
|
122
|
+
validationIds,
|
|
123
|
+
cacheNamespace
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Refuses a checkpoint from a different run rather than silently scoring old
|
|
128
|
+
* candidates against new data — the failure mode that produces a plausible
|
|
129
|
+
* result nobody can reproduce.
|
|
130
|
+
*/
|
|
131
|
+
function assertResumable(args) {
|
|
132
|
+
const { fingerprint, snapshot } = args;
|
|
133
|
+
if (snapshot !== void 0 && snapshot.fingerprint !== fingerprint) throw new Error("checkpoint does not belong to this run: the seed candidate, instance ids, seed or cache namespace differ from the ones it was taken with");
|
|
134
|
+
}
|
|
135
|
+
/** A candidate's identity: component names and their text, order-independent. */
|
|
136
|
+
function candidateFingerprint(candidate) {
|
|
137
|
+
return JSON.stringify(componentNames(candidate).sort().map((name) => [name, candidate[name]]));
|
|
138
|
+
}
|
|
139
|
+
//#endregion
|
|
140
|
+
//#region src/deadline.ts
|
|
141
|
+
/**
|
|
142
|
+
* A wall-clock limit on a run.
|
|
143
|
+
*
|
|
144
|
+
* Rollout and cost ceilings both bound what a run *spends*, and neither bounds
|
|
145
|
+
* how long it takes: a run waiting on a rate-limited provider can sit for an
|
|
146
|
+
* hour without spending a dollar. A deadline is what makes an optimizer safe to
|
|
147
|
+
* put behind a request timeout or a nightly job.
|
|
148
|
+
*
|
|
149
|
+
* Checked between evaluations, so a run overruns by at most the length of one.
|
|
150
|
+
* The clock is injectable because a deadline that can only be tested by waiting
|
|
151
|
+
* is a deadline nobody tests.
|
|
152
|
+
*/
|
|
153
|
+
function createDeadline(args) {
|
|
154
|
+
const { maxWallClockMs, now = Date.now } = args;
|
|
155
|
+
const startedAt = now();
|
|
156
|
+
return {
|
|
157
|
+
exceeded: () => maxWallClockMs !== void 0 && now() - startedAt >= maxWallClockMs,
|
|
158
|
+
remainingMs: () => maxWallClockMs === void 0 ? Number.POSITIVE_INFINITY : Math.max(0, maxWallClockMs - (now() - startedAt))
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
//#endregion
|
|
162
|
+
//#region src/evaluation.ts
|
|
163
|
+
const DEFAULT_RETRY = {
|
|
164
|
+
attempts: 2,
|
|
165
|
+
delayMs: 500
|
|
166
|
+
};
|
|
167
|
+
/**
|
|
168
|
+
* Raised when a reservation cannot be met mid-flight. A concurrent evaluation
|
|
169
|
+
* cannot check the budget and then spend it — another may take the remainder
|
|
170
|
+
* in between — so running out is reported where it happens and turned into a
|
|
171
|
+
* stop reason by whichever loop is driving.
|
|
172
|
+
*/
|
|
173
|
+
var BudgetExhausted = class extends Error {};
|
|
174
|
+
function createEvaluator(args) {
|
|
175
|
+
const { adapter, budget, cache, trackOutputs = false, onEvaluation, signal, cacheHits: initialCacheHits = 0, retry, cacheNamespace } = args;
|
|
176
|
+
const { attempts: retryAttempts, delayMs: retryDelayMs } = {
|
|
177
|
+
...DEFAULT_RETRY,
|
|
178
|
+
...retry
|
|
179
|
+
};
|
|
180
|
+
let cacheHits = initialCacheHits;
|
|
181
|
+
let unchargedCalls = 0;
|
|
182
|
+
const usage = {
|
|
183
|
+
inputTokens: 0,
|
|
184
|
+
outputTokens: 0,
|
|
185
|
+
totalTokens: 0,
|
|
186
|
+
costUsd: 0,
|
|
187
|
+
rollouts: 0
|
|
188
|
+
};
|
|
189
|
+
/** Folds one adapter call's reported usage into the run's totals. */
|
|
190
|
+
function recordUsage(args) {
|
|
191
|
+
usage.rollouts += args.rollouts;
|
|
192
|
+
for (const rollout of args.evaluation.usage ?? []) addUsage({
|
|
193
|
+
totals: usage,
|
|
194
|
+
rollout
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Runs `rows` of `batch`, retrying the instances the adapter reports as
|
|
199
|
+
* infrastructure failures. Retries are charged like any other rollout —
|
|
200
|
+
* they cost the same money — but they are never allowed to overdraw: a run
|
|
201
|
+
* that cannot afford another attempt keeps the transient row instead.
|
|
202
|
+
*/
|
|
203
|
+
async function runWithRetries(call) {
|
|
204
|
+
const { rows } = call;
|
|
205
|
+
let merged = await runRows(call);
|
|
206
|
+
let calls = rows.length;
|
|
207
|
+
for (let attempt = 0; attempt < retryAttempts; attempt += 1) {
|
|
208
|
+
const failed = transientIndices(merged);
|
|
209
|
+
if (failed.length === 0 || !affordable({
|
|
210
|
+
calls: failed.length,
|
|
211
|
+
charge: call.charge
|
|
212
|
+
})) return {
|
|
213
|
+
evaluation: merged,
|
|
214
|
+
calls
|
|
215
|
+
};
|
|
216
|
+
await delay(retryDelayMs * 2 ** attempt);
|
|
217
|
+
signal?.throwIfAborted();
|
|
218
|
+
const retried = await runRows({
|
|
219
|
+
...call,
|
|
220
|
+
rows: failed.map((index) => rows[index])
|
|
221
|
+
});
|
|
222
|
+
calls += failed.length;
|
|
223
|
+
merged = mergeRows({
|
|
224
|
+
base: merged,
|
|
225
|
+
positions: failed,
|
|
226
|
+
rows: retried
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
return {
|
|
230
|
+
evaluation: merged,
|
|
231
|
+
calls
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
/** One adapter call, charged up front and refunded if it never happened. */
|
|
235
|
+
async function runRows(call) {
|
|
236
|
+
const { candidate, rows, captureTraces, charge, ...context } = call;
|
|
237
|
+
reserve({
|
|
238
|
+
calls: rows.length,
|
|
239
|
+
charge
|
|
240
|
+
});
|
|
241
|
+
try {
|
|
242
|
+
const evaluation = await adapter.evaluate({
|
|
243
|
+
batch: rows,
|
|
244
|
+
candidate,
|
|
245
|
+
captureTraces,
|
|
246
|
+
run: context,
|
|
247
|
+
signal
|
|
248
|
+
});
|
|
249
|
+
assertEvaluation({
|
|
250
|
+
evaluation,
|
|
251
|
+
expected: rows.length
|
|
252
|
+
});
|
|
253
|
+
recordUsage({
|
|
254
|
+
evaluation,
|
|
255
|
+
rollouts: rows.length
|
|
256
|
+
});
|
|
257
|
+
return evaluation;
|
|
258
|
+
} catch (err) {
|
|
259
|
+
release({
|
|
260
|
+
calls: rows.length,
|
|
261
|
+
charge
|
|
262
|
+
});
|
|
263
|
+
throw err;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
function affordable(args) {
|
|
267
|
+
return !args.charge || budget.canAfford(args.calls);
|
|
268
|
+
}
|
|
269
|
+
function reserve(args) {
|
|
270
|
+
if (!args.charge) {
|
|
271
|
+
unchargedCalls += args.calls;
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
if (!budget.reserve(args.calls)) throw new BudgetExhausted();
|
|
275
|
+
}
|
|
276
|
+
function release(args) {
|
|
277
|
+
if (args.charge) budget.refund(args.calls);
|
|
278
|
+
else unchargedCalls -= args.calls;
|
|
279
|
+
}
|
|
280
|
+
return {
|
|
281
|
+
countUncached: ({ candidate, ids, split }) => {
|
|
282
|
+
if (cache === void 0) return ids.length;
|
|
283
|
+
const hash = candidateHash(candidate);
|
|
284
|
+
return ids.filter((id) => cache.get(evaluationCacheKey({
|
|
285
|
+
hash,
|
|
286
|
+
instanceId: id,
|
|
287
|
+
split,
|
|
288
|
+
...cacheNamespace === void 0 ? {} : { namespace: cacheNamespace }
|
|
289
|
+
})) === void 0).length;
|
|
290
|
+
},
|
|
291
|
+
evaluateTraced: async ({ candidate, batch, split, phase, candidateId, iteration }) => {
|
|
292
|
+
let run;
|
|
293
|
+
try {
|
|
294
|
+
run = await runWithRetries({
|
|
295
|
+
candidate,
|
|
296
|
+
rows: batch,
|
|
297
|
+
captureTraces: true,
|
|
298
|
+
split,
|
|
299
|
+
phase,
|
|
300
|
+
candidateId,
|
|
301
|
+
iteration,
|
|
302
|
+
charge: true
|
|
303
|
+
});
|
|
304
|
+
} catch (err) {
|
|
305
|
+
if (err instanceof BudgetExhausted) return null;
|
|
306
|
+
throw err;
|
|
307
|
+
}
|
|
308
|
+
onEvaluation?.({
|
|
309
|
+
iteration,
|
|
310
|
+
phase,
|
|
311
|
+
split,
|
|
312
|
+
candidateId,
|
|
313
|
+
metricCalls: run.calls,
|
|
314
|
+
cacheHits: 0,
|
|
315
|
+
meanScore: measuredMean(run.evaluation) ?? 0
|
|
316
|
+
});
|
|
317
|
+
return run.evaluation;
|
|
318
|
+
},
|
|
319
|
+
evaluate: async (call) => {
|
|
320
|
+
const { candidate, batch, ids, split, phase, candidateId, iteration, charge = true } = call;
|
|
321
|
+
const hash = candidateHash(candidate);
|
|
322
|
+
const scores = new Array(batch.length);
|
|
323
|
+
const objectiveScores = new Array(batch.length);
|
|
324
|
+
const outputs = new Array(batch.length).fill(void 0);
|
|
325
|
+
const transient = new Array(batch.length).fill(false);
|
|
326
|
+
const pendingIndices = [];
|
|
327
|
+
for (let index = 0; index < batch.length; index += 1) {
|
|
328
|
+
const cached = cache?.get(evaluationCacheKey({
|
|
329
|
+
hash,
|
|
330
|
+
instanceId: ids[index],
|
|
331
|
+
split,
|
|
332
|
+
...cacheNamespace === void 0 ? {} : { namespace: cacheNamespace }
|
|
333
|
+
}));
|
|
334
|
+
if (cached === void 0) pendingIndices.push(index);
|
|
335
|
+
else {
|
|
336
|
+
scores[index] = cached.score;
|
|
337
|
+
objectiveScores[index] = cached.objectiveScores;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
cacheHits += batch.length - pendingIndices.length;
|
|
341
|
+
let metricCalls = 0;
|
|
342
|
+
if (pendingIndices.length > 0) {
|
|
343
|
+
const run = await runWithRetries({
|
|
344
|
+
candidate,
|
|
345
|
+
rows: pendingIndices.map((index) => batch[index]),
|
|
346
|
+
captureTraces: false,
|
|
347
|
+
split,
|
|
348
|
+
phase,
|
|
349
|
+
candidateId,
|
|
350
|
+
iteration,
|
|
351
|
+
charge
|
|
352
|
+
});
|
|
353
|
+
const evaluation = run.evaluation;
|
|
354
|
+
metricCalls = run.calls;
|
|
355
|
+
pendingIndices.forEach((batchIndex, resultIndex) => {
|
|
356
|
+
const score = evaluation.scores[resultIndex];
|
|
357
|
+
const objectives = evaluation.objectiveScores?.[resultIndex];
|
|
358
|
+
scores[batchIndex] = score;
|
|
359
|
+
objectiveScores[batchIndex] = objectives;
|
|
360
|
+
if (trackOutputs) outputs[batchIndex] = evaluation.outputs[resultIndex];
|
|
361
|
+
if (evaluation.transient?.[resultIndex] === true) {
|
|
362
|
+
transient[batchIndex] = true;
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
cache?.set(evaluationCacheKey({
|
|
366
|
+
hash,
|
|
367
|
+
instanceId: ids[batchIndex],
|
|
368
|
+
split,
|
|
369
|
+
...cacheNamespace === void 0 ? {} : { namespace: cacheNamespace }
|
|
370
|
+
}), objectives === void 0 ? { score } : {
|
|
371
|
+
score,
|
|
372
|
+
objectiveScores: objectives
|
|
373
|
+
});
|
|
374
|
+
});
|
|
375
|
+
}
|
|
376
|
+
onEvaluation?.({
|
|
377
|
+
iteration,
|
|
378
|
+
phase,
|
|
379
|
+
split,
|
|
380
|
+
candidateId,
|
|
381
|
+
metricCalls,
|
|
382
|
+
cacheHits: batch.length - pendingIndices.length,
|
|
383
|
+
meanScore: measuredMean({
|
|
384
|
+
scores,
|
|
385
|
+
transient
|
|
386
|
+
}) ?? 0
|
|
387
|
+
});
|
|
388
|
+
return {
|
|
389
|
+
scores,
|
|
390
|
+
objectiveScores,
|
|
391
|
+
outputs,
|
|
392
|
+
transient
|
|
393
|
+
};
|
|
394
|
+
},
|
|
395
|
+
cacheHits: () => cacheHits,
|
|
396
|
+
unchargedCalls: () => unchargedCalls,
|
|
397
|
+
usage: () => ({ ...usage }),
|
|
398
|
+
entries: () => cache?.entries?.(),
|
|
399
|
+
restore: (entries) => {
|
|
400
|
+
for (const [key, cached] of entries) cache?.set(key, cached);
|
|
401
|
+
}
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
/**
|
|
405
|
+
* A NaN score never raises on its own: every comparison that decides fronts or
|
|
406
|
+
* the best candidate is false for NaN, so the candidate silently becomes an
|
|
407
|
+
* unselectable phantom that still consumed budget. Every other array is read
|
|
408
|
+
* positionally against the batch, so a short one misattributes a diagnosis, an
|
|
409
|
+
* objective or an infrastructure failure to the wrong instance. Catch both at
|
|
410
|
+
* the boundary.
|
|
411
|
+
*/
|
|
412
|
+
function assertEvaluation(args) {
|
|
413
|
+
const { evaluation, expected } = args;
|
|
414
|
+
const { scores } = evaluation;
|
|
415
|
+
const aligned = [
|
|
416
|
+
["scores", scores],
|
|
417
|
+
["outputs", evaluation.outputs],
|
|
418
|
+
["feedback", evaluation.feedback],
|
|
419
|
+
["objectiveScores", evaluation.objectiveScores],
|
|
420
|
+
["usage", evaluation.usage],
|
|
421
|
+
["transient", evaluation.transient]
|
|
422
|
+
];
|
|
423
|
+
for (const [name, values] of aligned) if (values !== void 0 && values.length !== expected) throw new Error(`Adapter returned ${values.length} ${name} for a batch of ${expected}; ${name} must align one-to-one with the batch`);
|
|
424
|
+
for (let index = 0; index < scores.length; index += 1) {
|
|
425
|
+
const score = scores[index];
|
|
426
|
+
if (typeof score !== "number" || !Number.isFinite(score)) throw new Error(`Adapter returned a non-finite score at index ${index}: ${String(score)}`);
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
/**
|
|
430
|
+
* Mean over the rows that measured the candidate. Transient rows measured the
|
|
431
|
+
* infrastructure instead, so averaging their zeros in would reject a candidate
|
|
432
|
+
* for an outage; undefined when no row measured anything at all, which is a
|
|
433
|
+
* batch that says nothing rather than a batch that says zero.
|
|
434
|
+
*/
|
|
435
|
+
function measuredMean(batch) {
|
|
436
|
+
const { scores, transient } = batch;
|
|
437
|
+
let total = 0;
|
|
438
|
+
let count = 0;
|
|
439
|
+
for (let index = 0; index < scores.length; index += 1) {
|
|
440
|
+
if (transient?.[index] === true) continue;
|
|
441
|
+
total += scores[index];
|
|
442
|
+
count += 1;
|
|
443
|
+
}
|
|
444
|
+
return count === 0 ? void 0 : total / count;
|
|
445
|
+
}
|
|
446
|
+
/**
|
|
447
|
+
* Whether a run has spent what it was allowed to. Checked between evaluations,
|
|
448
|
+
* because usage is only known once a rollout has been paid for.
|
|
449
|
+
*/
|
|
450
|
+
function costExhausted(args) {
|
|
451
|
+
return args.maxCostUsd !== void 0 && args.usage.costUsd >= args.maxCostUsd;
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* The measured mean of an evaluation a run cannot continue without — its seed
|
|
455
|
+
* baseline, and the sweeps it compares everything against.
|
|
456
|
+
*
|
|
457
|
+
* Reporting zero for a batch in which nothing ran would set the search a
|
|
458
|
+
* baseline no rollout produced, and every later comparison would be made
|
|
459
|
+
* against it. Failing here names the provider outage instead.
|
|
460
|
+
*/
|
|
461
|
+
function requireMeasuredMean(args) {
|
|
462
|
+
const value = measuredMean(args.batch);
|
|
463
|
+
if (value === void 0) throw new Error(`Every rollout in the ${args.phase} evaluation failed transiently; it measured the infrastructure rather than the candidate`);
|
|
464
|
+
return value;
|
|
465
|
+
}
|
|
466
|
+
/**
|
|
467
|
+
* Adds one rollout's reading to a running total. `totalTokens` is derived when
|
|
468
|
+
* the provider reports only the two halves, because a caller comparing runs
|
|
469
|
+
* should not have to know which providers report which fields.
|
|
470
|
+
*/
|
|
471
|
+
function addUsage(args) {
|
|
472
|
+
const { totals, rollout } = args;
|
|
473
|
+
const { inputTokens = 0, outputTokens = 0, costUsd = 0 } = rollout;
|
|
474
|
+
totals.inputTokens += inputTokens;
|
|
475
|
+
totals.outputTokens += outputTokens;
|
|
476
|
+
totals.totalTokens += rollout.totalTokens ?? inputTokens + outputTokens;
|
|
477
|
+
totals.costUsd += costUsd;
|
|
478
|
+
}
|
|
479
|
+
function transientIndices(evaluation) {
|
|
480
|
+
const { transient } = evaluation;
|
|
481
|
+
if (transient === void 0) return [];
|
|
482
|
+
const indices = [];
|
|
483
|
+
for (let index = 0; index < transient.length; index += 1) if (transient[index] === true) indices.push(index);
|
|
484
|
+
return indices;
|
|
485
|
+
}
|
|
486
|
+
/**
|
|
487
|
+
* Writes a re-run of some rows back over the batch they came from, field by
|
|
488
|
+
* field. Aligned arrays are read positionally everywhere downstream, so a
|
|
489
|
+
* retried row has to land in the position its instance occupies.
|
|
490
|
+
*/
|
|
491
|
+
function mergeRows(args) {
|
|
492
|
+
const { base, positions, rows } = args;
|
|
493
|
+
const merged = {
|
|
494
|
+
outputs: [...base.outputs],
|
|
495
|
+
scores: [...base.scores],
|
|
496
|
+
...base.feedback === void 0 ? {} : { feedback: [...base.feedback] },
|
|
497
|
+
...base.trajectories === void 0 ? {} : { trajectories: [...base.trajectories] },
|
|
498
|
+
...base.objectiveScores === void 0 ? {} : { objectiveScores: [...base.objectiveScores] },
|
|
499
|
+
transient: base.transient === void 0 ? [] : [...base.transient]
|
|
500
|
+
};
|
|
501
|
+
positions.forEach((position, row) => {
|
|
502
|
+
merged.outputs[position] = rows.outputs[row];
|
|
503
|
+
merged.scores[position] = rows.scores[row];
|
|
504
|
+
if (merged.feedback !== void 0) merged.feedback[position] = rows.feedback?.[row] ?? "";
|
|
505
|
+
if (merged.trajectories !== void 0 && rows.trajectories !== void 0) merged.trajectories[position] = rows.trajectories[row];
|
|
506
|
+
if (merged.objectiveScores !== void 0) {
|
|
507
|
+
const objectives = rows.objectiveScores?.[row];
|
|
508
|
+
if (objectives !== void 0) merged.objectiveScores[position] = objectives;
|
|
509
|
+
}
|
|
510
|
+
merged.transient[position] = rows.transient?.[row] === true;
|
|
511
|
+
});
|
|
512
|
+
return merged;
|
|
513
|
+
}
|
|
514
|
+
function delay(milliseconds) {
|
|
515
|
+
if (milliseconds <= 0) return Promise.resolve();
|
|
516
|
+
return new Promise((resolve) => {
|
|
517
|
+
setTimeout(resolve, milliseconds);
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
//#endregion
|
|
521
|
+
Object.defineProperty(exports, "BudgetExhausted", {
|
|
522
|
+
enumerable: true,
|
|
523
|
+
get: function() {
|
|
524
|
+
return BudgetExhausted;
|
|
525
|
+
}
|
|
526
|
+
});
|
|
527
|
+
Object.defineProperty(exports, "assertResumable", {
|
|
528
|
+
enumerable: true,
|
|
529
|
+
get: function() {
|
|
530
|
+
return assertResumable;
|
|
531
|
+
}
|
|
532
|
+
});
|
|
533
|
+
Object.defineProperty(exports, "candidateFingerprint", {
|
|
534
|
+
enumerable: true,
|
|
535
|
+
get: function() {
|
|
536
|
+
return candidateFingerprint;
|
|
537
|
+
}
|
|
538
|
+
});
|
|
539
|
+
Object.defineProperty(exports, "componentNames", {
|
|
540
|
+
enumerable: true,
|
|
541
|
+
get: function() {
|
|
542
|
+
return componentNames;
|
|
543
|
+
}
|
|
544
|
+
});
|
|
545
|
+
Object.defineProperty(exports, "costExhausted", {
|
|
546
|
+
enumerable: true,
|
|
547
|
+
get: function() {
|
|
548
|
+
return costExhausted;
|
|
549
|
+
}
|
|
550
|
+
});
|
|
551
|
+
Object.defineProperty(exports, "createBudget", {
|
|
552
|
+
enumerable: true,
|
|
553
|
+
get: function() {
|
|
554
|
+
return createBudget;
|
|
555
|
+
}
|
|
556
|
+
});
|
|
557
|
+
Object.defineProperty(exports, "createDeadline", {
|
|
558
|
+
enumerable: true,
|
|
559
|
+
get: function() {
|
|
560
|
+
return createDeadline;
|
|
561
|
+
}
|
|
562
|
+
});
|
|
563
|
+
Object.defineProperty(exports, "createEvaluator", {
|
|
564
|
+
enumerable: true,
|
|
565
|
+
get: function() {
|
|
566
|
+
return createEvaluator;
|
|
567
|
+
}
|
|
568
|
+
});
|
|
569
|
+
Object.defineProperty(exports, "createMemoryCache", {
|
|
570
|
+
enumerable: true,
|
|
571
|
+
get: function() {
|
|
572
|
+
return createMemoryCache;
|
|
573
|
+
}
|
|
574
|
+
});
|
|
575
|
+
Object.defineProperty(exports, "measuredMean", {
|
|
576
|
+
enumerable: true,
|
|
577
|
+
get: function() {
|
|
578
|
+
return measuredMean;
|
|
579
|
+
}
|
|
580
|
+
});
|
|
581
|
+
Object.defineProperty(exports, "requireMeasuredMean", {
|
|
582
|
+
enumerable: true,
|
|
583
|
+
get: function() {
|
|
584
|
+
return requireMeasuredMean;
|
|
585
|
+
}
|
|
586
|
+
});
|
|
587
|
+
Object.defineProperty(exports, "runFingerprint", {
|
|
588
|
+
enumerable: true,
|
|
589
|
+
get: function() {
|
|
590
|
+
return runFingerprint;
|
|
591
|
+
}
|
|
592
|
+
});
|
|
593
|
+
Object.defineProperty(exports, "stableHash", {
|
|
594
|
+
enumerable: true,
|
|
595
|
+
get: function() {
|
|
596
|
+
return stableHash;
|
|
597
|
+
}
|
|
598
|
+
});
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
let node_fs = require("node:fs");
|
|
3
|
+
let node_path = require("node:path");
|
|
4
|
+
//#region src/file-cache.ts
|
|
5
|
+
/**
|
|
6
|
+
* An evaluation cache that outlives the process, as an append-only log.
|
|
7
|
+
*
|
|
8
|
+
* A long run against a real provider is measured in hours and dollars, and an
|
|
9
|
+
* in-memory cache throws all of it away when the run ends — a crashed run, a
|
|
10
|
+
* re-run with a changed budget, or a second experiment over the same
|
|
11
|
+
* validation set all pay for identical rollouts again.
|
|
12
|
+
*
|
|
13
|
+
* Append-only rather than rewritten: a score is never invalidated (the key
|
|
14
|
+
* names the candidate, the instance, and the environment), and a log survives
|
|
15
|
+
* a process killed mid-write, which a file rewritten in place does not.
|
|
16
|
+
*/
|
|
17
|
+
function createFileCache(args) {
|
|
18
|
+
const { path, maxEntries = 1e6 } = args;
|
|
19
|
+
(0, node_fs.mkdirSync)((0, node_path.dirname)(path), { recursive: true });
|
|
20
|
+
const entries = readLog(path);
|
|
21
|
+
return {
|
|
22
|
+
get: (key) => entries.get(key),
|
|
23
|
+
set: (key, cached) => {
|
|
24
|
+
if (entries.size >= maxEntries && !entries.has(key)) {
|
|
25
|
+
const oldest = entries.keys().next();
|
|
26
|
+
if (!oldest.done) entries.delete(oldest.value);
|
|
27
|
+
}
|
|
28
|
+
entries.set(key, cached);
|
|
29
|
+
(0, node_fs.appendFileSync)(path, `${JSON.stringify([key, cached])}\n`);
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Later records win, so a re-measured instance replaces its earlier reading.
|
|
35
|
+
* A record that does not parse is dropped rather than fatal: the last line of
|
|
36
|
+
* a log whose process was killed mid-write is routinely half-written, and
|
|
37
|
+
* losing one cached score is not worth failing a run over.
|
|
38
|
+
*/
|
|
39
|
+
function readLog(path) {
|
|
40
|
+
const entries = /* @__PURE__ */ new Map();
|
|
41
|
+
let contents;
|
|
42
|
+
try {
|
|
43
|
+
contents = (0, node_fs.readFileSync)(path, "utf8");
|
|
44
|
+
} catch {
|
|
45
|
+
return entries;
|
|
46
|
+
}
|
|
47
|
+
for (const line of contents.split("\n")) {
|
|
48
|
+
if (line.length === 0) continue;
|
|
49
|
+
const entry = parseEntry(line);
|
|
50
|
+
if (entry !== void 0) entries.set(entry[0], entry[1]);
|
|
51
|
+
}
|
|
52
|
+
return entries;
|
|
53
|
+
}
|
|
54
|
+
function parseEntry(line) {
|
|
55
|
+
let parsed;
|
|
56
|
+
try {
|
|
57
|
+
parsed = JSON.parse(line);
|
|
58
|
+
} catch {
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
if (!Array.isArray(parsed) || parsed.length !== 2) return;
|
|
62
|
+
const [key, cached] = parsed;
|
|
63
|
+
if (typeof key !== "string" || !isCachedScore(cached)) return;
|
|
64
|
+
return [key, cached];
|
|
65
|
+
}
|
|
66
|
+
function isCachedScore(value) {
|
|
67
|
+
return typeof value === "object" && value !== null && typeof value.score === "number";
|
|
68
|
+
}
|
|
69
|
+
//#endregion
|
|
70
|
+
exports.createFileCache = createFileCache;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { n as EvaluationCache } from "./cache-CuSo0NJ8.cjs";
|
|
2
|
+
//#region src/file-cache.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* An evaluation cache that outlives the process, as an append-only log.
|
|
5
|
+
*
|
|
6
|
+
* A long run against a real provider is measured in hours and dollars, and an
|
|
7
|
+
* in-memory cache throws all of it away when the run ends — a crashed run, a
|
|
8
|
+
* re-run with a changed budget, or a second experiment over the same
|
|
9
|
+
* validation set all pay for identical rollouts again.
|
|
10
|
+
*
|
|
11
|
+
* Append-only rather than rewritten: a score is never invalidated (the key
|
|
12
|
+
* names the candidate, the instance, and the environment), and a log survives
|
|
13
|
+
* a process killed mid-write, which a file rewritten in place does not.
|
|
14
|
+
*/
|
|
15
|
+
declare function createFileCache(args: {
|
|
16
|
+
path: string;
|
|
17
|
+
/** Entries kept in memory. The file itself is never trimmed. */
|
|
18
|
+
maxEntries?: number;
|
|
19
|
+
}): EvaluationCache;
|
|
20
|
+
//#endregion
|
|
21
|
+
export { createFileCache };
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { n as EvaluationCache } from "./cache-CuSo0NJ8.mjs";
|
|
2
|
+
//#region src/file-cache.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* An evaluation cache that outlives the process, as an append-only log.
|
|
5
|
+
*
|
|
6
|
+
* A long run against a real provider is measured in hours and dollars, and an
|
|
7
|
+
* in-memory cache throws all of it away when the run ends — a crashed run, a
|
|
8
|
+
* re-run with a changed budget, or a second experiment over the same
|
|
9
|
+
* validation set all pay for identical rollouts again.
|
|
10
|
+
*
|
|
11
|
+
* Append-only rather than rewritten: a score is never invalidated (the key
|
|
12
|
+
* names the candidate, the instance, and the environment), and a log survives
|
|
13
|
+
* a process killed mid-write, which a file rewritten in place does not.
|
|
14
|
+
*/
|
|
15
|
+
declare function createFileCache(args: {
|
|
16
|
+
path: string;
|
|
17
|
+
/** Entries kept in memory. The file itself is never trimmed. */
|
|
18
|
+
maxEntries?: number;
|
|
19
|
+
}): EvaluationCache;
|
|
20
|
+
//#endregion
|
|
21
|
+
export { createFileCache };
|