textopt 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -22
- package/dist/bootstrap-search/index.cjs +153 -73
- package/dist/bootstrap-search/index.d.cts +32 -10
- package/dist/bootstrap-search/index.d.mts +32 -10
- package/dist/bootstrap-search/index.mjs +144 -66
- package/dist/{demos-B0pVQjYC.d.mts → demos-1b7JiUue.d.mts} +10 -3
- package/dist/{demos-BTuzFNsp.d.cts → demos-CU9dy2oT.d.cts} +10 -3
- package/dist/demos-D2o0qSSo.cjs +244 -0
- package/dist/demos-DE2oxNWX.mjs +215 -0
- package/dist/file-cache.cjs +11 -3
- package/dist/file-cache.mjs +11 -3
- package/dist/gepa/index.cjs +76 -71
- package/dist/gepa/index.d.cts +12 -6
- package/dist/gepa/index.d.mts +12 -6
- package/dist/gepa/index.mjs +49 -46
- package/dist/index.cjs +129 -27
- package/dist/index.d.cts +145 -7
- package/dist/index.d.mts +145 -7
- package/dist/index.mjs +113 -15
- package/dist/{math-COOofUyv.cjs → math-BhlziRPc.cjs} +60 -9
- package/dist/math-Dqme4rYz.mjs +123 -0
- package/dist/mipro/index.cjs +98 -70
- package/dist/mipro/index.d.cts +17 -14
- package/dist/mipro/index.d.mts +17 -14
- package/dist/mipro/index.mjs +84 -58
- package/dist/opro/index.cjs +130 -51
- package/dist/opro/index.d.cts +17 -9
- package/dist/opro/index.d.mts +17 -9
- package/dist/opro/index.mjs +115 -38
- package/dist/{optimizer-B7SpRwl7.d.cts → optimizer-Bh5DPRMH.d.cts} +50 -4
- package/dist/{optimizer-DqCoth_w.d.mts → optimizer-Ck6-e_8o.d.mts} +50 -4
- package/dist/random-search/index.cjs +93 -49
- package/dist/random-search/index.d.cts +15 -13
- package/dist/random-search/index.d.mts +15 -13
- package/dist/random-search/index.mjs +83 -41
- package/dist/{reflection-CQToe-5B.d.cts → reflection-Dt3QrXhM.d.cts} +7 -11
- package/dist/{reflection-Cr_upzU0.d.mts → reflection-LRaAZP4e.d.mts} +7 -11
- package/dist/{evaluation-OZOp6TB7.cjs → reporting-CNHzbJC-.cjs} +165 -5
- package/dist/reporting-DQbAohc9.d.cts +240 -0
- package/dist/reporting-DQbAohc9.d.mts +240 -0
- package/dist/{evaluation-BV0nSZVx.mjs → reporting-DY-DC4HG.mjs} +124 -6
- package/dist/simba/index.cjs +210 -83
- package/dist/simba/index.d.cts +32 -11
- package/dist/simba/index.d.mts +32 -11
- package/dist/simba/index.mjs +200 -75
- package/dist/testing.cjs +1 -0
- package/dist/testing.d.cts +5 -3
- package/dist/testing.d.mts +5 -3
- package/dist/testing.mjs +1 -1
- package/package.json +4 -3
- package/dist/demos-B9BJiNKz.cjs +0 -143
- package/dist/demos-Degx6UmP.mjs +0 -126
- package/dist/math-DhrDmpFS.mjs +0 -78
- package/dist/types-CWv4IQFF.d.cts +0 -129
- package/dist/types-CWv4IQFF.d.mts +0 -129
package/dist/gepa/index.cjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const
|
|
2
|
+
const require_reporting = require("../reporting-CNHzbJC-.cjs");
|
|
3
3
|
const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
|
|
4
|
-
const require_math = require("../math-
|
|
5
|
-
const require_demos = require("../demos-
|
|
4
|
+
const require_math = require("../math-BhlziRPc.cjs");
|
|
5
|
+
const require_demos = require("../demos-D2o0qSSo.cjs");
|
|
6
6
|
const require_rng = require("../rng-DbA_rPIo.cjs");
|
|
7
7
|
const require_reflection = require("../reflection-DRfbk6hu.cjs");
|
|
8
8
|
const require_sampling = require("../sampling-Dars7ctR.cjs");
|
|
@@ -334,7 +334,7 @@ function isEligibleAncestor(args) {
|
|
|
334
334
|
*/
|
|
335
335
|
function hasComplementaryComponent(args) {
|
|
336
336
|
const { ancestor, left, right } = args;
|
|
337
|
-
return
|
|
337
|
+
return require_reporting.componentNames(ancestor).some((name) => {
|
|
338
338
|
const base = ancestor[name];
|
|
339
339
|
return (base === left[name] || base === right[name]) && left[name] !== right[name];
|
|
340
340
|
});
|
|
@@ -347,7 +347,7 @@ function mergeComponents(args) {
|
|
|
347
347
|
const rightRecord = records[rightId];
|
|
348
348
|
const candidate = { ...ancestor };
|
|
349
349
|
const sources = [];
|
|
350
|
-
for (const name of
|
|
350
|
+
for (const name of require_reporting.componentNames(ancestor)) {
|
|
351
351
|
const base = ancestor[name];
|
|
352
352
|
const leftText = leftRecord.candidate[name];
|
|
353
353
|
const rightText = rightRecord.candidate[name];
|
|
@@ -391,7 +391,7 @@ function buildAncestries(records) {
|
|
|
391
391
|
return ancestries;
|
|
392
392
|
}
|
|
393
393
|
function fingerprint(candidate) {
|
|
394
|
-
return JSON.stringify(
|
|
394
|
+
return JSON.stringify(require_reporting.componentNames(candidate).sort().map((name) => [name, candidate[name]]));
|
|
395
395
|
}
|
|
396
396
|
//#endregion
|
|
397
397
|
//#region src/gepa/pareto.ts
|
|
@@ -628,14 +628,14 @@ function subsampledEvaluationPolicy(args) {
|
|
|
628
628
|
*/
|
|
629
629
|
function roundRobinComponentSelector() {
|
|
630
630
|
return ({ candidate, cursor }) => {
|
|
631
|
-
const names =
|
|
631
|
+
const names = require_reporting.componentNames(candidate);
|
|
632
632
|
if (names.length === 0) throw new Error("Candidate has no components to update");
|
|
633
633
|
return [names[cursor % names.length]];
|
|
634
634
|
};
|
|
635
635
|
}
|
|
636
636
|
/** Update every component in a single reflection call. */
|
|
637
637
|
function allComponentsSelector() {
|
|
638
|
-
return ({ candidate }) =>
|
|
638
|
+
return ({ candidate }) => require_reporting.componentNames(candidate);
|
|
639
639
|
}
|
|
640
640
|
/**
|
|
641
641
|
* Accept a mutation only when it beats its parent on the same minibatch. Cheap
|
|
@@ -739,18 +739,22 @@ var GepaOptimizer = class {
|
|
|
739
739
|
this.#config = config;
|
|
740
740
|
}
|
|
741
741
|
async optimize(task) {
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
742
|
+
try {
|
|
743
|
+
return await runGepa({
|
|
744
|
+
config: this.#config,
|
|
745
|
+
task
|
|
746
|
+
});
|
|
747
|
+
} finally {
|
|
748
|
+
await require_reporting.flushReporters(task.reporters ?? []);
|
|
749
|
+
}
|
|
746
750
|
}
|
|
747
751
|
};
|
|
748
752
|
async function runGepa(args) {
|
|
749
753
|
const { config, task } = args;
|
|
750
754
|
const { minibatchSize = DEFAULT_MINIBATCH_SIZE, maxIterations = Number.POSITIVE_INFINITY, seed = 0, candidateSelector = paretoSelector(), acceptance = improvementAcceptance(), merge, skipPerfectScore = true, perfectScore = 1, rejectedProposalMemory = DEFAULT_REJECTED_PROPOSAL_MEMORY, proposals, reflection, checkpointCache = true, trackBestOutputs = false, raiseOnError = true } = config;
|
|
751
|
-
const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, componentSelector = roundRobinComponentSelector(), batchSampler = require_sampling.createEpochShuffledSampler({ minibatchSize }), valEvaluationPolicy = fullEvaluationPolicy(), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId,
|
|
752
|
-
const deadline =
|
|
753
|
-
const seedComponents =
|
|
755
|
+
const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, maxMetricCalls, componentSelector = roundRobinComponentSelector(), batchSampler = require_sampling.createEpochShuffledSampler({ minibatchSize }), valEvaluationPolicy = fullEvaluationPolicy(), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_reporting.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
|
|
756
|
+
const deadline = require_reporting.createDeadline({ maxWallClockMs });
|
|
757
|
+
const seedComponents = require_reporting.componentNames(seedCandidate);
|
|
754
758
|
const mergeConfig = {
|
|
755
759
|
enabled: merge?.enabled ?? seedComponents.length > 1,
|
|
756
760
|
maxInvocations: merge?.maxInvocations ?? DEFAULT_MAX_MERGES,
|
|
@@ -763,7 +767,7 @@ async function runGepa(args) {
|
|
|
763
767
|
if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet; the Pareto frontier is tracked over validation instances");
|
|
764
768
|
if (seedComponents.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
|
|
765
769
|
if (testSet !== void 0 && testSet.length === 0) throw new Error("optimize requires a non-empty testSet when one is given; omit it to skip held-out evaluation");
|
|
766
|
-
const evaluationCache = cache === false ? void 0 : cache ??
|
|
770
|
+
const evaluationCache = cache === false ? void 0 : cache ?? require_reporting.createMemoryCache();
|
|
767
771
|
const propose = adapter.proposeNewTexts?.bind(adapter) ?? require_reflection.createDefaultProposer({
|
|
768
772
|
...reflection?.buildPrompt === void 0 ? {} : { buildPrompt: reflection.buildPrompt },
|
|
769
773
|
...reflection?.strategies === void 0 ? {} : { strategies: reflection.strategies },
|
|
@@ -784,19 +788,19 @@ async function runGepa(args) {
|
|
|
784
788
|
datum,
|
|
785
789
|
index
|
|
786
790
|
})) ?? [];
|
|
787
|
-
const fingerprint =
|
|
791
|
+
const fingerprint = require_reporting.runFingerprint({
|
|
788
792
|
seedCandidate,
|
|
789
793
|
trainingIds,
|
|
790
794
|
validationIds,
|
|
791
795
|
seed,
|
|
792
796
|
...cacheNamespace === void 0 ? {} : { cacheNamespace }
|
|
793
797
|
});
|
|
794
|
-
|
|
798
|
+
require_reporting.assertResumable({
|
|
795
799
|
fingerprint,
|
|
796
800
|
...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
|
|
797
801
|
});
|
|
798
802
|
const rng = require_rng.createSeededRng(seed, resumeFrom?.rngState);
|
|
799
|
-
const budget =
|
|
803
|
+
const budget = require_reporting.createBudget({
|
|
800
804
|
maxMetricCalls,
|
|
801
805
|
spent: resumeFrom?.metricCalls ?? 0
|
|
802
806
|
});
|
|
@@ -816,7 +820,7 @@ async function runGepa(args) {
|
|
|
816
820
|
records: resumeFrom?.records ?? [],
|
|
817
821
|
seedCandidate
|
|
818
822
|
});
|
|
819
|
-
const seenCandidates = new Set(records.map((record) =>
|
|
823
|
+
const seenCandidates = new Set(records.map((record) => require_reporting.candidateFingerprint(record.candidate)));
|
|
820
824
|
const outputsByCandidate = /* @__PURE__ */ new Map();
|
|
821
825
|
const rejectedProposals = restoreRejections({
|
|
822
826
|
rejections: resumeFrom?.rejectedProposals ?? {},
|
|
@@ -828,8 +832,26 @@ async function runGepa(args) {
|
|
|
828
832
|
let mergesDue = resumeFrom?.merge.due ?? 0;
|
|
829
833
|
let totalMergesTested = resumeFrom?.merge.tested ?? 0;
|
|
830
834
|
let lastIterationAccepted = resumeFrom?.merge.lastIterationAccepted ?? false;
|
|
831
|
-
|
|
832
|
-
|
|
835
|
+
const emit = require_reporting.createEmitter(reporters);
|
|
836
|
+
/**
|
|
837
|
+
* Everything an acceptance means, in one event: the text, the aggregate, and
|
|
838
|
+
* the row it put on the frontier. Emitted from one place because the merge
|
|
839
|
+
* path and the mutation path accept candidates separately, and a payload
|
|
840
|
+
* assembled twice is a payload that drifts.
|
|
841
|
+
*/
|
|
842
|
+
function emitAccepted(record) {
|
|
843
|
+
const outputs = outputsByCandidate.get(record.id);
|
|
844
|
+
emit({
|
|
845
|
+
type: "candidateAccepted",
|
|
846
|
+
iteration,
|
|
847
|
+
candidateId: record.id,
|
|
848
|
+
parentIds: record.parentIds,
|
|
849
|
+
aggregateScore: record.aggregateScore,
|
|
850
|
+
source: record.source,
|
|
851
|
+
candidate: record.candidate,
|
|
852
|
+
instanceScores: record.instanceScores,
|
|
853
|
+
...outputs === void 0 ? {} : { outputs }
|
|
854
|
+
});
|
|
833
855
|
}
|
|
834
856
|
/**
|
|
835
857
|
* Copies everything mutable: a snapshot handed to `onCheckpoint` is a record
|
|
@@ -846,6 +868,7 @@ async function runGepa(args) {
|
|
|
846
868
|
metricCalls: budget.spent(),
|
|
847
869
|
reflectionCalls,
|
|
848
870
|
cacheHits: evaluator.cacheHits(),
|
|
871
|
+
usage: evaluator.usage(),
|
|
849
872
|
...samplerState === void 0 ? {} : { sampler: samplerState },
|
|
850
873
|
rejectedProposals: snapshotRejections({
|
|
851
874
|
rejections: rejectedProposals,
|
|
@@ -866,7 +889,7 @@ async function runGepa(args) {
|
|
|
866
889
|
if (onCheckpoint === void 0) return;
|
|
867
890
|
await onCheckpoint(takeSnapshot());
|
|
868
891
|
}
|
|
869
|
-
const evaluator =
|
|
892
|
+
const evaluator = require_reporting.createEvaluator({
|
|
870
893
|
adapter,
|
|
871
894
|
budget,
|
|
872
895
|
...retry === void 0 ? {} : { retry },
|
|
@@ -874,6 +897,7 @@ async function runGepa(args) {
|
|
|
874
897
|
...evaluationCache === void 0 ? {} : { cache: evaluationCache },
|
|
875
898
|
trackOutputs: trackBestOutputs,
|
|
876
899
|
cacheHits: resumeFrom?.cacheHits ?? 0,
|
|
900
|
+
...resumeFrom?.usage === void 0 ? {} : { usage: resumeFrom.usage },
|
|
877
901
|
...signal === void 0 ? {} : { signal },
|
|
878
902
|
onEvaluation: (event) => emit({
|
|
879
903
|
type: "evaluation",
|
|
@@ -953,7 +977,7 @@ async function runGepa(args) {
|
|
|
953
977
|
componentCursor: inheritedCursor(args.parentIds)
|
|
954
978
|
};
|
|
955
979
|
records.push(record);
|
|
956
|
-
seenCandidates.add(
|
|
980
|
+
seenCandidates.add(require_reporting.candidateFingerprint(args.candidate));
|
|
957
981
|
if (trackBestOutputs) outputsByCandidate.set(record.id, args.evaluation.outputs);
|
|
958
982
|
if (args.source !== "merge") {
|
|
959
983
|
lastIterationAccepted = true;
|
|
@@ -965,7 +989,7 @@ async function runGepa(args) {
|
|
|
965
989
|
function rememberRejection(args) {
|
|
966
990
|
const { proposed, parentScore, childScore } = args;
|
|
967
991
|
if (rejectedProposalMemory <= 0) return;
|
|
968
|
-
for (const component of
|
|
992
|
+
for (const component of require_reporting.componentNames(proposed)) {
|
|
969
993
|
const text = proposed[component];
|
|
970
994
|
if (text === void 0) continue;
|
|
971
995
|
const history = rejectedProposals[component] ?? [];
|
|
@@ -993,7 +1017,7 @@ async function runGepa(args) {
|
|
|
993
1017
|
if (records.length === 0) {
|
|
994
1018
|
const seedInstances = selectValInstances(seedCandidate);
|
|
995
1019
|
if (!budget.canAfford(seedInstances.length)) throw new Error(`maxMetricCalls (${maxMetricCalls}) is smaller than the ${seedInstances.length} validation instances selected for scoring; the seed candidate cannot be scored`);
|
|
996
|
-
addCandidate({
|
|
1020
|
+
emitAccepted(addCandidate({
|
|
997
1021
|
candidate: seedCandidate,
|
|
998
1022
|
parentIds: [],
|
|
999
1023
|
evaluation: await evaluateValidation({
|
|
@@ -1004,7 +1028,7 @@ async function runGepa(args) {
|
|
|
1004
1028
|
}),
|
|
1005
1029
|
source: "seed",
|
|
1006
1030
|
updatedComponents: []
|
|
1007
|
-
});
|
|
1031
|
+
}));
|
|
1008
1032
|
lastIterationAccepted = false;
|
|
1009
1033
|
mergesDue = 0;
|
|
1010
1034
|
await checkpoint();
|
|
@@ -1091,18 +1115,11 @@ async function runGepa(args) {
|
|
|
1091
1115
|
parentIds: [...proposal.parentIds],
|
|
1092
1116
|
evaluation,
|
|
1093
1117
|
source: "merge",
|
|
1094
|
-
updatedComponents:
|
|
1118
|
+
updatedComponents: require_reporting.componentNames(proposal.candidate).filter((name) => proposal.candidate[name] !== ancestor.candidate[name])
|
|
1095
1119
|
});
|
|
1096
1120
|
mergesDue -= 1;
|
|
1097
1121
|
totalMergesTested += 1;
|
|
1098
|
-
|
|
1099
|
-
type: "candidateAccepted",
|
|
1100
|
-
iteration,
|
|
1101
|
-
candidateId: record.id,
|
|
1102
|
-
parentIds: record.parentIds,
|
|
1103
|
-
aggregateScore: record.aggregateScore,
|
|
1104
|
-
source: "merge"
|
|
1105
|
-
});
|
|
1122
|
+
emitAccepted(record);
|
|
1106
1123
|
return "attempted";
|
|
1107
1124
|
}
|
|
1108
1125
|
/**
|
|
@@ -1141,7 +1158,7 @@ async function runGepa(args) {
|
|
|
1141
1158
|
candidate: parent.candidate,
|
|
1142
1159
|
source: "componentSelector"
|
|
1143
1160
|
});
|
|
1144
|
-
parent.componentCursor = (parent.componentCursor + 1) % Math.max(1,
|
|
1161
|
+
parent.componentCursor = (parent.componentCursor + 1) % Math.max(1, require_reporting.componentNames(parent.candidate).length);
|
|
1145
1162
|
plans.push({
|
|
1146
1163
|
parent,
|
|
1147
1164
|
batch: batchIndices.map((index) => trainingSet[index]),
|
|
@@ -1191,7 +1208,7 @@ async function runGepa(args) {
|
|
|
1191
1208
|
throw err;
|
|
1192
1209
|
}
|
|
1193
1210
|
assertComponents({
|
|
1194
|
-
names:
|
|
1211
|
+
names: require_reporting.componentNames(proposed),
|
|
1195
1212
|
candidate: parent.candidate,
|
|
1196
1213
|
source: "proposeNewTexts"
|
|
1197
1214
|
});
|
|
@@ -1199,7 +1216,7 @@ async function runGepa(args) {
|
|
|
1199
1216
|
...parent.candidate,
|
|
1200
1217
|
...proposed
|
|
1201
1218
|
};
|
|
1202
|
-
const changed =
|
|
1219
|
+
const changed = require_reporting.componentNames(proposed).length > 0 && !seenCandidates.has(require_reporting.candidateFingerprint(child));
|
|
1203
1220
|
emit({
|
|
1204
1221
|
type: "proposal",
|
|
1205
1222
|
iteration,
|
|
@@ -1219,7 +1236,7 @@ async function runGepa(args) {
|
|
|
1219
1236
|
candidateId: null
|
|
1220
1237
|
});
|
|
1221
1238
|
} catch (err) {
|
|
1222
|
-
if (err instanceof
|
|
1239
|
+
if (err instanceof require_reporting.BudgetExhausted) return { status: "budgetExhausted" };
|
|
1223
1240
|
throw err;
|
|
1224
1241
|
}
|
|
1225
1242
|
const screened = pairMeasured({
|
|
@@ -1262,7 +1279,7 @@ async function runGepa(args) {
|
|
|
1262
1279
|
stop ??= "reflectionBudgetExhausted";
|
|
1263
1280
|
continue;
|
|
1264
1281
|
}
|
|
1265
|
-
const fingerprint =
|
|
1282
|
+
const fingerprint = require_reporting.candidateFingerprint(outcome.child);
|
|
1266
1283
|
if (claimed.has(fingerprint)) continue;
|
|
1267
1284
|
claimed.add(fingerprint);
|
|
1268
1285
|
if (!outcome.accepted) {
|
|
@@ -1334,7 +1351,7 @@ async function runGepa(args) {
|
|
|
1334
1351
|
})
|
|
1335
1352
|
};
|
|
1336
1353
|
} catch (err) {
|
|
1337
|
-
if (err instanceof
|
|
1354
|
+
if (err instanceof require_reporting.BudgetExhausted) return {
|
|
1338
1355
|
item,
|
|
1339
1356
|
evaluation: void 0
|
|
1340
1357
|
};
|
|
@@ -1347,21 +1364,13 @@ async function runGepa(args) {
|
|
|
1347
1364
|
stop ??= "budgetExhausted";
|
|
1348
1365
|
break;
|
|
1349
1366
|
}
|
|
1350
|
-
|
|
1367
|
+
emitAccepted(addCandidate({
|
|
1351
1368
|
candidate: item.outcome.child,
|
|
1352
1369
|
parentIds: [item.outcome.plan.parent.id],
|
|
1353
1370
|
evaluation,
|
|
1354
1371
|
source: "mutation",
|
|
1355
|
-
updatedComponents:
|
|
1356
|
-
});
|
|
1357
|
-
emit({
|
|
1358
|
-
type: "candidateAccepted",
|
|
1359
|
-
iteration,
|
|
1360
|
-
candidateId: record.id,
|
|
1361
|
-
parentIds: record.parentIds,
|
|
1362
|
-
aggregateScore: record.aggregateScore,
|
|
1363
|
-
source: "mutation"
|
|
1364
|
-
});
|
|
1372
|
+
updatedComponents: require_reporting.componentNames(item.outcome.proposed)
|
|
1373
|
+
}));
|
|
1365
1374
|
}
|
|
1366
1375
|
return stop;
|
|
1367
1376
|
}
|
|
@@ -1378,7 +1387,7 @@ async function runGepa(args) {
|
|
|
1378
1387
|
stopReason = "aborted";
|
|
1379
1388
|
break;
|
|
1380
1389
|
}
|
|
1381
|
-
if (
|
|
1390
|
+
if (require_reporting.costExhausted({
|
|
1382
1391
|
usage: evaluator.usage(),
|
|
1383
1392
|
maxCostUsd
|
|
1384
1393
|
})) {
|
|
@@ -1425,7 +1434,7 @@ async function runGepa(args) {
|
|
|
1425
1434
|
stopReason = "aborted";
|
|
1426
1435
|
break;
|
|
1427
1436
|
}
|
|
1428
|
-
if (err instanceof
|
|
1437
|
+
if (err instanceof require_reporting.BudgetExhausted) {
|
|
1429
1438
|
stopReason = "budgetExhausted";
|
|
1430
1439
|
break;
|
|
1431
1440
|
}
|
|
@@ -1445,7 +1454,7 @@ async function runGepa(args) {
|
|
|
1445
1454
|
}
|
|
1446
1455
|
const bestCandidateId = valEvaluationPolicy.bestCandidate(records);
|
|
1447
1456
|
const best = records[bestCandidateId];
|
|
1448
|
-
const
|
|
1457
|
+
const heldOut = testSet === void 0 ? void 0 : await evaluateCached({
|
|
1449
1458
|
candidate: best.candidate,
|
|
1450
1459
|
batch: testSet,
|
|
1451
1460
|
ids: testIds,
|
|
@@ -1453,13 +1462,17 @@ async function runGepa(args) {
|
|
|
1453
1462
|
phase: "test",
|
|
1454
1463
|
candidateId: bestCandidateId,
|
|
1455
1464
|
charge: false
|
|
1456
|
-
})
|
|
1465
|
+
});
|
|
1466
|
+
const testScore = heldOut === void 0 ? void 0 : require_reporting.measuredMean(heldOut);
|
|
1457
1467
|
emit({
|
|
1458
1468
|
type: "finish",
|
|
1459
1469
|
reason: stopReason,
|
|
1460
1470
|
bestCandidateId,
|
|
1471
|
+
bestScore: best.aggregateScore,
|
|
1461
1472
|
metricCalls: budget.spent(),
|
|
1462
|
-
...testScore === void 0 ? {} : { testScore }
|
|
1473
|
+
...testScore === void 0 ? {} : { testScore },
|
|
1474
|
+
...heldOut === void 0 ? {} : { testInstanceScores: require_reporting.instanceRow(heldOut) },
|
|
1475
|
+
...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
|
|
1463
1476
|
});
|
|
1464
1477
|
const perObjectiveBest = collectPerObjectiveBest(records);
|
|
1465
1478
|
const bestOutputs = outputsByCandidate.get(bestCandidateId);
|
|
@@ -1470,7 +1483,8 @@ async function runGepa(args) {
|
|
|
1470
1483
|
bestCandidateId,
|
|
1471
1484
|
...testScore === void 0 ? {} : {
|
|
1472
1485
|
testScore,
|
|
1473
|
-
testMetricCalls: evaluator.unchargedCalls()
|
|
1486
|
+
testMetricCalls: evaluator.unchargedCalls(),
|
|
1487
|
+
testUsage: evaluator.unchargedUsage()
|
|
1474
1488
|
},
|
|
1475
1489
|
...bestOutputs === void 0 ? {} : { bestOutputs },
|
|
1476
1490
|
candidates: records,
|
|
@@ -1513,7 +1527,7 @@ function assertGepaConfig(config) {
|
|
|
1513
1527
|
*/
|
|
1514
1528
|
function restoreRecords(args) {
|
|
1515
1529
|
const { records, seedCandidate } = args;
|
|
1516
|
-
const known = new Set(
|
|
1530
|
+
const known = new Set(require_reporting.componentNames(seedCandidate));
|
|
1517
1531
|
for (const record of records) {
|
|
1518
1532
|
const named = [...Object.keys(record.candidate), ...record.updatedComponents];
|
|
1519
1533
|
for (const name of named) if (!known.has(name)) throw new Error(`checkpoint names the component "${name}", which the seed candidate does not have (${[...known].join(", ")})`);
|
|
@@ -1618,17 +1632,6 @@ function collectDominatorIds(records) {
|
|
|
1618
1632
|
return [...ids].sort((a, b) => a - b);
|
|
1619
1633
|
}
|
|
1620
1634
|
/**
|
|
1621
|
-
* Names an instance by a hash of its content rather than by the content
|
|
1622
|
-
* itself: the id ends up inside every cache key and inside the checkpoint
|
|
1623
|
-
* fingerprint, and embedding whole examples there costs memory proportional to
|
|
1624
|
-
* the dataset for no benefit. Data that will not serialize falls back to its
|
|
1625
|
-
* position, which is stable for as long as the dataset order is.
|
|
1626
|
-
*/
|
|
1627
|
-
function defaultInstanceId(args) {
|
|
1628
|
-
const hash = require_evaluation.stableHash(args.datum);
|
|
1629
|
-
return hash === "" ? String(args.index) : hash;
|
|
1630
|
-
}
|
|
1631
|
-
/**
|
|
1632
1635
|
* The two rollout sets restricted to the instances both of them measured.
|
|
1633
1636
|
*
|
|
1634
1637
|
* Screening is a paired comparison over one minibatch: a transient row is a
|
|
@@ -1663,6 +1666,8 @@ exports.diverseReflectionStrategies = require_reflection.diverseReflectionStrate
|
|
|
1663
1666
|
exports.epsilonGreedySelector = epsilonGreedySelector;
|
|
1664
1667
|
exports.fullEvaluationPolicy = fullEvaluationPolicy;
|
|
1665
1668
|
exports.improvementAcceptance = improvementAcceptance;
|
|
1669
|
+
exports.isCandidateAccepted = require_reporting.isCandidateAccepted;
|
|
1670
|
+
exports.isRunFinished = require_reporting.isRunFinished;
|
|
1666
1671
|
exports.lowerBoundEvaluationPolicy = lowerBoundEvaluationPolicy;
|
|
1667
1672
|
exports.pairedPermutationAcceptance = pairedPermutationAcceptance;
|
|
1668
1673
|
exports.paretoSelector = paretoSelector;
|
package/dist/gepa/index.d.cts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { _ as TextModel, a as RunFinished, g as ScoreResult, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted } from "../reporting-DQbAohc9.cjs";
|
|
2
2
|
import { n as EvaluationCache } from "../cache-CuSo0NJ8.cjs";
|
|
3
|
-
import { r as DemoRenderer } from "../demos-
|
|
4
|
-
import { n as OptimizerResult, r as OptimizerTask, t as Optimizer } from "../optimizer-
|
|
5
|
-
import { C as RejectedProposal, S as ReflectiveRecord, T as ValEvaluationPolicy, _ as GepaStopReason, a as buildRewritePrompt, b as ProposeArgs, c as AcceptancePolicy, d as CandidateSource, f as ComponentPatch, g as GepaSnapshot, h as GepaEvent, i as buildReflectionPrompt, l as CandidateRecord, m as GepaAdapter, n as ReflectionPromptBuilder, o as buildSimplifyPrompt, p as ComponentSelector, r as buildGeneralizePrompt, s as diverseReflectionStrategies, t as ReflectionPromptArgs, u as CandidateSelector, v as MakeReflectiveDatasetArgs, w as SelectionState, x as ReflectiveDataset, y as ParetoFrontier } from "../reflection-
|
|
3
|
+
import { r as DemoRenderer } from "../demos-CU9dy2oT.cjs";
|
|
4
|
+
import { n as OptimizerResult, r as OptimizerTask, t as Optimizer } from "../optimizer-Bh5DPRMH.cjs";
|
|
5
|
+
import { C as RejectedProposal, S as ReflectiveRecord, T as ValEvaluationPolicy, _ as GepaStopReason, a as buildRewritePrompt, b as ProposeArgs, c as AcceptancePolicy, d as CandidateSource, f as ComponentPatch, g as GepaSnapshot, h as GepaEvent, i as buildReflectionPrompt, l as CandidateRecord, m as GepaAdapter, n as ReflectionPromptBuilder, o as buildSimplifyPrompt, p as ComponentSelector, r as buildGeneralizePrompt, s as diverseReflectionStrategies, t as ReflectionPromptArgs, u as CandidateSelector, v as MakeReflectiveDatasetArgs, w as SelectionState, x as ReflectiveDataset, y as ParetoFrontier } from "../reflection-Dt3QrXhM.cjs";
|
|
6
6
|
import { t as BatchSampler } from "../sampling-axOwfZf5.cjs";
|
|
7
7
|
//#region src/gepa/demos.d.ts
|
|
8
8
|
/**
|
|
@@ -246,7 +246,13 @@ interface GepaTask<Datum, Trajectory = unknown, Output = unknown, K extends stri
|
|
|
246
246
|
* counts against a reference run directly.
|
|
247
247
|
*/
|
|
248
248
|
cache?: EvaluationCache | false;
|
|
249
|
-
|
|
249
|
+
/**
|
|
250
|
+
* Where the run's events go. An array because a run usually has more than
|
|
251
|
+
* one audience — a progress line on the terminal and a permanent record
|
|
252
|
+
* somewhere else — and teeing one callback by hand is how one of them ends
|
|
253
|
+
* up silently dropped.
|
|
254
|
+
*/
|
|
255
|
+
reporters?: readonly Reporter<GepaEvent<NoInfer<K>>>[];
|
|
250
256
|
/**
|
|
251
257
|
* Called with a resumable snapshot after the seed is scored and after every
|
|
252
258
|
* iteration. Persist it and a killed run costs the last iteration, not all
|
|
@@ -382,4 +388,4 @@ declare function pairedPermutationAcceptance(args?: {
|
|
|
382
388
|
maxExact?: number;
|
|
383
389
|
}): AcceptancePolicy;
|
|
384
390
|
//#endregion
|
|
385
|
-
export { type AcceptancePolicy, type CandidateRecord, type CandidateSelector, type CandidateSource, type ComponentPatch, type ComponentSelector, type GepaAdapter, type GepaConfig, type GepaEvent, GepaOptimizer, type GepaResult, type GepaSnapshot, type GepaStopReason, type GepaTask, type MakeReflectiveDatasetArgs, type PipelineModule, type PipelineStep, type PipelineTrace, type ProposeArgs, type ReflectionPromptArgs, type ReflectionPromptBuilder, type ReflectiveDataset, type ReflectiveRecord, type RejectedProposal, type SelectionState, type ValEvaluationPolicy, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };
|
|
391
|
+
export { type AcceptancePolicy, type CandidateAccepted, type CandidateRecord, type CandidateSelector, type CandidateSource, type ComponentPatch, type ComponentSelector, type GepaAdapter, type GepaConfig, type GepaEvent, GepaOptimizer, type GepaResult, type GepaSnapshot, type GepaStopReason, type GepaTask, type MakeReflectiveDatasetArgs, type OptimizerEvent, type PipelineModule, type PipelineStep, type PipelineTrace, type ProposeArgs, type ReflectionPromptArgs, type ReflectionPromptBuilder, type ReflectiveDataset, type ReflectiveRecord, type RejectedProposal, type ReportableEvent, type Reporter, type RunFinished, type SelectionState, type ValEvaluationPolicy, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, isCandidateAccepted, isRunFinished, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };
|
package/dist/gepa/index.d.mts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { _ as TextModel, a as RunFinished, g as ScoreResult, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted } from "../reporting-DQbAohc9.mjs";
|
|
2
2
|
import { n as EvaluationCache } from "../cache-CuSo0NJ8.mjs";
|
|
3
|
-
import { r as DemoRenderer } from "../demos-
|
|
4
|
-
import { n as OptimizerResult, r as OptimizerTask, t as Optimizer } from "../optimizer-
|
|
5
|
-
import { C as RejectedProposal, S as ReflectiveRecord, T as ValEvaluationPolicy, _ as GepaStopReason, a as buildRewritePrompt, b as ProposeArgs, c as AcceptancePolicy, d as CandidateSource, f as ComponentPatch, g as GepaSnapshot, h as GepaEvent, i as buildReflectionPrompt, l as CandidateRecord, m as GepaAdapter, n as ReflectionPromptBuilder, o as buildSimplifyPrompt, p as ComponentSelector, r as buildGeneralizePrompt, s as diverseReflectionStrategies, t as ReflectionPromptArgs, u as CandidateSelector, v as MakeReflectiveDatasetArgs, w as SelectionState, x as ReflectiveDataset, y as ParetoFrontier } from "../reflection-
|
|
3
|
+
import { r as DemoRenderer } from "../demos-1b7JiUue.mjs";
|
|
4
|
+
import { n as OptimizerResult, r as OptimizerTask, t as Optimizer } from "../optimizer-Ck6-e_8o.mjs";
|
|
5
|
+
import { C as RejectedProposal, S as ReflectiveRecord, T as ValEvaluationPolicy, _ as GepaStopReason, a as buildRewritePrompt, b as ProposeArgs, c as AcceptancePolicy, d as CandidateSource, f as ComponentPatch, g as GepaSnapshot, h as GepaEvent, i as buildReflectionPrompt, l as CandidateRecord, m as GepaAdapter, n as ReflectionPromptBuilder, o as buildSimplifyPrompt, p as ComponentSelector, r as buildGeneralizePrompt, s as diverseReflectionStrategies, t as ReflectionPromptArgs, u as CandidateSelector, v as MakeReflectiveDatasetArgs, w as SelectionState, x as ReflectiveDataset, y as ParetoFrontier } from "../reflection-LRaAZP4e.mjs";
|
|
6
6
|
import { t as BatchSampler } from "../sampling-DFo_7RNJ.mjs";
|
|
7
7
|
//#region src/gepa/demos.d.ts
|
|
8
8
|
/**
|
|
@@ -246,7 +246,13 @@ interface GepaTask<Datum, Trajectory = unknown, Output = unknown, K extends stri
|
|
|
246
246
|
* counts against a reference run directly.
|
|
247
247
|
*/
|
|
248
248
|
cache?: EvaluationCache | false;
|
|
249
|
-
|
|
249
|
+
/**
|
|
250
|
+
* Where the run's events go. An array because a run usually has more than
|
|
251
|
+
* one audience — a progress line on the terminal and a permanent record
|
|
252
|
+
* somewhere else — and teeing one callback by hand is how one of them ends
|
|
253
|
+
* up silently dropped.
|
|
254
|
+
*/
|
|
255
|
+
reporters?: readonly Reporter<GepaEvent<NoInfer<K>>>[];
|
|
250
256
|
/**
|
|
251
257
|
* Called with a resumable snapshot after the seed is scored and after every
|
|
252
258
|
* iteration. Persist it and a killed run costs the last iteration, not all
|
|
@@ -382,4 +388,4 @@ declare function pairedPermutationAcceptance(args?: {
|
|
|
382
388
|
maxExact?: number;
|
|
383
389
|
}): AcceptancePolicy;
|
|
384
390
|
//#endregion
|
|
385
|
-
export { type AcceptancePolicy, type CandidateRecord, type CandidateSelector, type CandidateSource, type ComponentPatch, type ComponentSelector, type GepaAdapter, type GepaConfig, type GepaEvent, GepaOptimizer, type GepaResult, type GepaSnapshot, type GepaStopReason, type GepaTask, type MakeReflectiveDatasetArgs, type PipelineModule, type PipelineStep, type PipelineTrace, type ProposeArgs, type ReflectionPromptArgs, type ReflectionPromptBuilder, type ReflectiveDataset, type ReflectiveRecord, type RejectedProposal, type SelectionState, type ValEvaluationPolicy, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };
|
|
391
|
+
export { type AcceptancePolicy, type CandidateAccepted, type CandidateRecord, type CandidateSelector, type CandidateSource, type ComponentPatch, type ComponentSelector, type GepaAdapter, type GepaConfig, type GepaEvent, GepaOptimizer, type GepaResult, type GepaSnapshot, type GepaStopReason, type GepaTask, type MakeReflectiveDatasetArgs, type OptimizerEvent, type PipelineModule, type PipelineStep, type PipelineTrace, type ProposeArgs, type ReflectionPromptArgs, type ReflectionPromptBuilder, type ReflectiveDataset, type ReflectiveRecord, type RejectedProposal, type ReportableEvent, type Reporter, type RunFinished, type SelectionState, type ValEvaluationPolicy, allComponentsSelector, buildGeneralizePrompt, buildReflectionPrompt, buildRewritePrompt, buildSimplifyPrompt, createDemoProposer, createPipelineAdapter, currentBestSelector, diverseReflectionStrategies, epsilonGreedySelector, fullEvaluationPolicy, improvementAcceptance, isCandidateAccepted, isRunFinished, lowerBoundEvaluationPolicy, pairedPermutationAcceptance, paretoSelector, roundRobinComponentSelector, subsampledEvaluationPolicy, topKParetoSelector };
|