poe-code 3.0.205 → 3.0.206
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/braintrust.js +2 -18
- package/dist/cli/commands/braintrust.js.map +1 -1
- package/dist/cli/commands/experiment.js +2 -1
- package/dist/cli/commands/experiment.js.map +1 -1
- package/dist/cli/commands/pipeline.js +2 -1
- package/dist/cli/commands/pipeline.js.map +1 -1
- package/dist/cli/commands/spawn.js +1 -1
- package/dist/cli/commands/spawn.js.map +1 -1
- package/dist/index.js +42430 -42425
- package/dist/index.js.map +4 -4
- package/dist/providers/poe-agent.js +905 -15
- package/dist/providers/poe-agent.js.map +4 -4
- package/package.json +1 -1
- package/packages/superintendent/dist/commands/run.d.ts +1 -1
- package/packages/superintendent/dist/commands/run.js +2 -1
|
@@ -3793,19 +3793,6 @@ var init_merge_callbacks = __esm({
|
|
|
3793
3793
|
}
|
|
3794
3794
|
});
|
|
3795
3795
|
|
|
3796
|
-
// packages/poe-code-config/src/integrations-loader.ts
|
|
3797
|
-
async function loadIntegrations(config) {
|
|
3798
|
-
if (!config.integrations?.braintrust?.enabled) return null;
|
|
3799
|
-
const packageName = "@poe-code/braintrust";
|
|
3800
|
-
const mod = await import(packageName);
|
|
3801
|
-
return mod.bootstrap(config);
|
|
3802
|
-
}
|
|
3803
|
-
var init_integrations_loader = __esm({
|
|
3804
|
-
"packages/poe-code-config/src/integrations-loader.ts"() {
|
|
3805
|
-
"use strict";
|
|
3806
|
-
}
|
|
3807
|
-
});
|
|
3808
|
-
|
|
3809
3796
|
// packages/poe-code-config/src/models.ts
|
|
3810
3797
|
var init_models = __esm({
|
|
3811
3798
|
"packages/poe-code-config/src/models.ts"() {
|
|
@@ -4351,7 +4338,6 @@ var init_src5 = __esm({
|
|
|
4351
4338
|
init_config();
|
|
4352
4339
|
init_merge2();
|
|
4353
4340
|
init_merge_callbacks();
|
|
4354
|
-
init_integrations_loader();
|
|
4355
4341
|
init_models();
|
|
4356
4342
|
init_memory();
|
|
4357
4343
|
init_resolve2();
|
|
@@ -22826,6 +22812,25 @@ var init_adapters = __esm({
|
|
|
22826
22812
|
}
|
|
22827
22813
|
});
|
|
22828
22814
|
|
|
22815
|
+
// packages/agent-spawn/src/acp/meta.ts
|
|
22816
|
+
function stampReceiveTime(event, ts) {
|
|
22817
|
+
if (event === null || typeof event !== "object") {
|
|
22818
|
+
return event;
|
|
22819
|
+
}
|
|
22820
|
+
const target = event;
|
|
22821
|
+
const existing = target._meta;
|
|
22822
|
+
if (existing && typeof existing.ts === "number") {
|
|
22823
|
+
return event;
|
|
22824
|
+
}
|
|
22825
|
+
target._meta = existing ? { ...existing, ts } : { ts };
|
|
22826
|
+
return event;
|
|
22827
|
+
}
|
|
22828
|
+
var init_meta = __esm({
|
|
22829
|
+
"packages/agent-spawn/src/acp/meta.ts"() {
|
|
22830
|
+
"use strict";
|
|
22831
|
+
}
|
|
22832
|
+
});
|
|
22833
|
+
|
|
22829
22834
|
// packages/agent-spawn/src/acp/spawn.ts
|
|
22830
22835
|
function createAbortError4() {
|
|
22831
22836
|
const error2 = new Error("Agent spawn aborted");
|
|
@@ -22999,7 +23004,7 @@ function spawnStreaming(options) {
|
|
|
22999
23004
|
const events = (async function* () {
|
|
23000
23005
|
for await (const output of adapter(queue.lines())) {
|
|
23001
23006
|
if (!isAcpEvent(output)) continue;
|
|
23002
|
-
yield output;
|
|
23007
|
+
yield stampReceiveTime(output, Date.now());
|
|
23003
23008
|
}
|
|
23004
23009
|
})();
|
|
23005
23010
|
const done = (async () => {
|
|
@@ -23034,6 +23039,7 @@ var init_spawn2 = __esm({
|
|
|
23034
23039
|
init_register_factories();
|
|
23035
23040
|
init_src9();
|
|
23036
23041
|
init_adapters();
|
|
23042
|
+
init_meta();
|
|
23037
23043
|
init_resolve_config();
|
|
23038
23044
|
init_mcp_args();
|
|
23039
23045
|
init_model_utils();
|
|
@@ -23049,6 +23055,7 @@ var init_spawn_acp = __esm({
|
|
|
23049
23055
|
init_src8();
|
|
23050
23056
|
init_src();
|
|
23051
23057
|
init_configs();
|
|
23058
|
+
init_meta();
|
|
23052
23059
|
init_session_update_converter();
|
|
23053
23060
|
}
|
|
23054
23061
|
});
|
|
@@ -26699,6 +26706,889 @@ async function executePoeAgent(agentSpec, input, createAgent = agent) {
|
|
|
26699
26706
|
init_src12();
|
|
26700
26707
|
init_src5();
|
|
26701
26708
|
|
|
26709
|
+
// packages/braintrust/src/client.ts
|
|
26710
|
+
function createClient(opts) {
|
|
26711
|
+
let sdkPromise;
|
|
26712
|
+
let rootLoggerPromise;
|
|
26713
|
+
const experiments = /* @__PURE__ */ new Map();
|
|
26714
|
+
let lastError = null;
|
|
26715
|
+
let errorCount = 0;
|
|
26716
|
+
const recordError = (err, ctx) => {
|
|
26717
|
+
try {
|
|
26718
|
+
errorCount += 1;
|
|
26719
|
+
lastError = `${ctx}: ${formatError(err)}`;
|
|
26720
|
+
} catch {
|
|
26721
|
+
errorCount += 1;
|
|
26722
|
+
lastError = `${ctx}: unknown error`;
|
|
26723
|
+
}
|
|
26724
|
+
};
|
|
26725
|
+
const loadSdk = async () => {
|
|
26726
|
+
sdkPromise ??= import("braintrust").catch((err) => {
|
|
26727
|
+
recordError(err, "load sdk");
|
|
26728
|
+
return void 0;
|
|
26729
|
+
});
|
|
26730
|
+
return sdkPromise;
|
|
26731
|
+
};
|
|
26732
|
+
const sdkOptions = () => ({
|
|
26733
|
+
projectName: opts.project,
|
|
26734
|
+
apiKey: opts.apiKey,
|
|
26735
|
+
apiUrl: opts.apiUrl
|
|
26736
|
+
});
|
|
26737
|
+
const client = {
|
|
26738
|
+
getSdk: loadSdk,
|
|
26739
|
+
async getRootLogger() {
|
|
26740
|
+
rootLoggerPromise ??= (async () => {
|
|
26741
|
+
try {
|
|
26742
|
+
const sdk = await loadSdk();
|
|
26743
|
+
return sdk?.initLogger(sdkOptions());
|
|
26744
|
+
} catch (err) {
|
|
26745
|
+
recordError(err, "init logger");
|
|
26746
|
+
return void 0;
|
|
26747
|
+
}
|
|
26748
|
+
})();
|
|
26749
|
+
return rootLoggerPromise;
|
|
26750
|
+
},
|
|
26751
|
+
async getExperiment(name) {
|
|
26752
|
+
if (!experiments.has(name)) {
|
|
26753
|
+
experiments.set(name, (async () => {
|
|
26754
|
+
try {
|
|
26755
|
+
const sdk = await loadSdk();
|
|
26756
|
+
return sdk?.initExperiment({
|
|
26757
|
+
...sdkOptions(),
|
|
26758
|
+
experimentName: name
|
|
26759
|
+
});
|
|
26760
|
+
} catch (err) {
|
|
26761
|
+
recordError(err, `init experiment ${name}`);
|
|
26762
|
+
return void 0;
|
|
26763
|
+
}
|
|
26764
|
+
})());
|
|
26765
|
+
}
|
|
26766
|
+
return experiments.get(name);
|
|
26767
|
+
},
|
|
26768
|
+
async flush(timeoutMs) {
|
|
26769
|
+
const flushAll = (async () => {
|
|
26770
|
+
const sdk = await sdkPromise;
|
|
26771
|
+
if (sdk === void 0) {
|
|
26772
|
+
return;
|
|
26773
|
+
}
|
|
26774
|
+
const targets = [
|
|
26775
|
+
await rootLoggerPromise,
|
|
26776
|
+
...await Promise.all(experiments.values())
|
|
26777
|
+
].filter((target) => target !== void 0);
|
|
26778
|
+
const flushTargets = Promise.all(
|
|
26779
|
+
targets.map((target) => sdk.flush(target))
|
|
26780
|
+
).then(() => void 0);
|
|
26781
|
+
await flushTargets;
|
|
26782
|
+
})().catch((err) => {
|
|
26783
|
+
recordError(err, "flush");
|
|
26784
|
+
});
|
|
26785
|
+
try {
|
|
26786
|
+
await Promise.race([
|
|
26787
|
+
flushAll,
|
|
26788
|
+
new Promise((resolve2) => {
|
|
26789
|
+
setTimeout(resolve2, timeoutMs);
|
|
26790
|
+
})
|
|
26791
|
+
]);
|
|
26792
|
+
} catch (err) {
|
|
26793
|
+
recordError(err, "flush");
|
|
26794
|
+
}
|
|
26795
|
+
},
|
|
26796
|
+
recordError,
|
|
26797
|
+
status() {
|
|
26798
|
+
return {
|
|
26799
|
+
lastError,
|
|
26800
|
+
errorCount,
|
|
26801
|
+
project: opts.project
|
|
26802
|
+
};
|
|
26803
|
+
}
|
|
26804
|
+
};
|
|
26805
|
+
return client;
|
|
26806
|
+
}
|
|
26807
|
+
function formatError(err) {
|
|
26808
|
+
if (err instanceof Error) {
|
|
26809
|
+
return err.message;
|
|
26810
|
+
}
|
|
26811
|
+
if (typeof err === "string") {
|
|
26812
|
+
return err;
|
|
26813
|
+
}
|
|
26814
|
+
try {
|
|
26815
|
+
return JSON.stringify(err);
|
|
26816
|
+
} catch {
|
|
26817
|
+
return String(err);
|
|
26818
|
+
}
|
|
26819
|
+
}
|
|
26820
|
+
|
|
26821
|
+
// packages/braintrust/src/redact.ts
|
|
26822
|
+
var MAX_STRING_BYTES = 65536;
|
|
26823
|
+
var MAX_JSON_BYTES = 262144;
|
|
26824
|
+
var BINARY_SCAN_BYTES = 1024;
|
|
26825
|
+
function redact(value) {
|
|
26826
|
+
const serialized = JSON.stringify(value);
|
|
26827
|
+
if (serialized !== void 0) {
|
|
26828
|
+
const originalBytes = Buffer.byteLength(serialized, "utf8");
|
|
26829
|
+
if (originalBytes > MAX_JSON_BYTES) {
|
|
26830
|
+
return `[truncated:${originalBytes}]`;
|
|
26831
|
+
}
|
|
26832
|
+
}
|
|
26833
|
+
return redactLeaf(value);
|
|
26834
|
+
}
|
|
26835
|
+
function redactLeaf(value) {
|
|
26836
|
+
if (typeof value === "string") {
|
|
26837
|
+
const bytes = Buffer.byteLength(value, "utf8");
|
|
26838
|
+
return bytes > MAX_STRING_BYTES ? `[truncated:${bytes}]` : value;
|
|
26839
|
+
}
|
|
26840
|
+
if (value instanceof Uint8Array) {
|
|
26841
|
+
const scanLength = Math.min(value.byteLength, BINARY_SCAN_BYTES);
|
|
26842
|
+
for (let index = 0; index < scanLength; index += 1) {
|
|
26843
|
+
if (value[index] === 0) {
|
|
26844
|
+
return `[binary:${value.byteLength}]`;
|
|
26845
|
+
}
|
|
26846
|
+
}
|
|
26847
|
+
return value;
|
|
26848
|
+
}
|
|
26849
|
+
if (Array.isArray(value)) {
|
|
26850
|
+
return value.map((item) => redactLeaf(item));
|
|
26851
|
+
}
|
|
26852
|
+
if (value !== null && typeof value === "object") {
|
|
26853
|
+
return Object.fromEntries(
|
|
26854
|
+
Object.entries(value).map(([key2, item]) => [key2, redactLeaf(item)])
|
|
26855
|
+
);
|
|
26856
|
+
}
|
|
26857
|
+
return value;
|
|
26858
|
+
}
|
|
26859
|
+
|
|
26860
|
+
// packages/braintrust/src/row-builder.ts
|
|
26861
|
+
function makePipelineRowState(client) {
|
|
26862
|
+
const rows = /* @__PURE__ */ new Map();
|
|
26863
|
+
return {
|
|
26864
|
+
start(progress) {
|
|
26865
|
+
try {
|
|
26866
|
+
const key2 = pipelineKey(progress);
|
|
26867
|
+
const row = {
|
|
26868
|
+
progress,
|
|
26869
|
+
span: openCurrentChildSpan(client, {
|
|
26870
|
+
name: `step:${readPipelineStep(progress)}:${readPipelineIndex(progress)}`,
|
|
26871
|
+
type: "task"
|
|
26872
|
+
}, "pipeline step start")
|
|
26873
|
+
};
|
|
26874
|
+
rows.set(key2, row);
|
|
26875
|
+
} catch (err) {
|
|
26876
|
+
client.recordError(err, "pipeline step start");
|
|
26877
|
+
}
|
|
26878
|
+
},
|
|
26879
|
+
complete(progress) {
|
|
26880
|
+
try {
|
|
26881
|
+
const key2 = pipelineKey(progress);
|
|
26882
|
+
const row = rows.get(key2);
|
|
26883
|
+
rows.delete(key2);
|
|
26884
|
+
void (async () => {
|
|
26885
|
+
try {
|
|
26886
|
+
const span = row?.span === void 0 ? await openCurrentChildSpan(client, {
|
|
26887
|
+
name: `step:${readPipelineStep(progress)}:${readPipelineIndex(progress)}`,
|
|
26888
|
+
type: "task"
|
|
26889
|
+
}, "pipeline step complete") : await row.span;
|
|
26890
|
+
if (span === void 0) {
|
|
26891
|
+
return;
|
|
26892
|
+
}
|
|
26893
|
+
try {
|
|
26894
|
+
span.log(buildPipelineCompletionLog(row?.progress ?? progress, progress));
|
|
26895
|
+
} finally {
|
|
26896
|
+
span.end();
|
|
26897
|
+
}
|
|
26898
|
+
} catch (err) {
|
|
26899
|
+
client.recordError(err, "pipeline step complete");
|
|
26900
|
+
}
|
|
26901
|
+
})();
|
|
26902
|
+
} catch (err) {
|
|
26903
|
+
client.recordError(err, "pipeline step complete");
|
|
26904
|
+
}
|
|
26905
|
+
}
|
|
26906
|
+
};
|
|
26907
|
+
}
|
|
26908
|
+
async function logSuperintendentRole(client, role, result) {
|
|
26909
|
+
try {
|
|
26910
|
+
const span = await openCurrentChildSpan(client, {
|
|
26911
|
+
name: `role:${role}`,
|
|
26912
|
+
type: "task"
|
|
26913
|
+
}, `superintendent ${role}`);
|
|
26914
|
+
if (span === void 0) {
|
|
26915
|
+
return;
|
|
26916
|
+
}
|
|
26917
|
+
try {
|
|
26918
|
+
span.log(buildSuperintendentLog(role, result));
|
|
26919
|
+
} finally {
|
|
26920
|
+
span.end();
|
|
26921
|
+
}
|
|
26922
|
+
} catch (err) {
|
|
26923
|
+
client.recordError(err, `superintendent ${role}`);
|
|
26924
|
+
}
|
|
26925
|
+
}
|
|
26926
|
+
function makeExperimentIterationState(client, experimentName) {
|
|
26927
|
+
const rows = /* @__PURE__ */ new Map();
|
|
26928
|
+
let latestBaseline;
|
|
26929
|
+
return {
|
|
26930
|
+
async start(index, agent2) {
|
|
26931
|
+
try {
|
|
26932
|
+
const experiment = asSpanParent(await client.getExperiment(experimentName));
|
|
26933
|
+
rows.set(index, {
|
|
26934
|
+
span: experiment.startSpan({
|
|
26935
|
+
name: `iteration:${index}`,
|
|
26936
|
+
type: "task"
|
|
26937
|
+
}),
|
|
26938
|
+
agent: agent2,
|
|
26939
|
+
iteration: index,
|
|
26940
|
+
baseline: latestBaseline,
|
|
26941
|
+
metrics: {}
|
|
26942
|
+
});
|
|
26943
|
+
} catch (err) {
|
|
26944
|
+
client.recordError(err, `experiment ${experimentName} iteration start`);
|
|
26945
|
+
}
|
|
26946
|
+
},
|
|
26947
|
+
baseline(b) {
|
|
26948
|
+
try {
|
|
26949
|
+
latestBaseline = { ...b };
|
|
26950
|
+
for (const row of rows.values()) {
|
|
26951
|
+
row.baseline = latestBaseline;
|
|
26952
|
+
}
|
|
26953
|
+
} catch (err) {
|
|
26954
|
+
client.recordError(err, `experiment ${experimentName} baseline`);
|
|
26955
|
+
}
|
|
26956
|
+
},
|
|
26957
|
+
metric(name, value) {
|
|
26958
|
+
try {
|
|
26959
|
+
const row = readLatestRow(rows);
|
|
26960
|
+
if (row !== void 0 && Number.isFinite(value)) {
|
|
26961
|
+
row.metrics[name] = value;
|
|
26962
|
+
}
|
|
26963
|
+
} catch (err) {
|
|
26964
|
+
client.recordError(err, `experiment ${experimentName} metric`);
|
|
26965
|
+
}
|
|
26966
|
+
},
|
|
26967
|
+
commit(hash) {
|
|
26968
|
+
try {
|
|
26969
|
+
const row = readLatestRow(rows);
|
|
26970
|
+
if (row !== void 0) {
|
|
26971
|
+
row.commit = hash;
|
|
26972
|
+
}
|
|
26973
|
+
} catch (err) {
|
|
26974
|
+
client.recordError(err, `experiment ${experimentName} commit`);
|
|
26975
|
+
}
|
|
26976
|
+
},
|
|
26977
|
+
reset(hash) {
|
|
26978
|
+
try {
|
|
26979
|
+
const row = readLatestRow(rows);
|
|
26980
|
+
if (row !== void 0) {
|
|
26981
|
+
row.reset = hash;
|
|
26982
|
+
}
|
|
26983
|
+
} catch (err) {
|
|
26984
|
+
client.recordError(err, `experiment ${experimentName} reset`);
|
|
26985
|
+
}
|
|
26986
|
+
},
|
|
26987
|
+
async complete(index, entry) {
|
|
26988
|
+
const row = rows.get(index);
|
|
26989
|
+
rows.delete(index);
|
|
26990
|
+
try {
|
|
26991
|
+
if (row?.span === void 0) {
|
|
26992
|
+
return;
|
|
26993
|
+
}
|
|
26994
|
+
try {
|
|
26995
|
+
row.span.log(buildExperimentLog(row, entry));
|
|
26996
|
+
} finally {
|
|
26997
|
+
row.span.end();
|
|
26998
|
+
}
|
|
26999
|
+
} catch (err) {
|
|
27000
|
+
client.recordError(err, `experiment ${experimentName} iteration complete`);
|
|
27001
|
+
}
|
|
27002
|
+
}
|
|
27003
|
+
};
|
|
27004
|
+
}
|
|
27005
|
+
async function openCurrentChildSpan(client, args, ctx) {
|
|
27006
|
+
try {
|
|
27007
|
+
const { currentSpan } = await import("braintrust");
|
|
27008
|
+
return asSpanParent(currentSpan()).startSpan(args);
|
|
27009
|
+
} catch (err) {
|
|
27010
|
+
client.recordError(err, ctx);
|
|
27011
|
+
return void 0;
|
|
27012
|
+
}
|
|
27013
|
+
}
|
|
27014
|
+
function buildPipelineCompletionLog(started, completed) {
|
|
27015
|
+
const startRecord = asRecord2(started) ?? {};
|
|
27016
|
+
const completeRecord = asRecord2(completed) ?? {};
|
|
27017
|
+
return {
|
|
27018
|
+
input: redact({
|
|
27019
|
+
step_name: readPipelineStep(started),
|
|
27020
|
+
step_prompt: readFirstString(startRecord, ["step_prompt", "stepPrompt", "prompt"]),
|
|
27021
|
+
plan_section: readFirstString(startRecord, ["plan_section", "planSection", "section"])
|
|
27022
|
+
}),
|
|
27023
|
+
output: redact({
|
|
27024
|
+
result_summary: readFirstString(completeRecord, [
|
|
27025
|
+
"result_summary",
|
|
27026
|
+
"resultSummary",
|
|
27027
|
+
"summary"
|
|
27028
|
+
]),
|
|
27029
|
+
files_changed: readFirstValue(completeRecord, ["files_changed", "filesChanged"]),
|
|
27030
|
+
success: completed.success
|
|
27031
|
+
}),
|
|
27032
|
+
scores: {
|
|
27033
|
+
passed: completed.success ? 1 : 0
|
|
27034
|
+
},
|
|
27035
|
+
metrics: buildPipelineMetrics(completed)
|
|
27036
|
+
};
|
|
27037
|
+
}
|
|
27038
|
+
function buildSuperintendentLog(role, result) {
|
|
27039
|
+
const record = asRecord2(result);
|
|
27040
|
+
const event = {
|
|
27041
|
+
input: redact(record?.input),
|
|
27042
|
+
output: redact(record?.output ?? result)
|
|
27043
|
+
};
|
|
27044
|
+
if (role === "inspector") {
|
|
27045
|
+
const satisfied = readSatisfiedScore(record);
|
|
27046
|
+
if (satisfied !== void 0) {
|
|
27047
|
+
event.scores = { satisfied };
|
|
27048
|
+
}
|
|
27049
|
+
}
|
|
27050
|
+
return event;
|
|
27051
|
+
}
|
|
27052
|
+
function buildExperimentLog(row, entry) {
|
|
27053
|
+
const entryRecord = asRecord2(entry) ?? {};
|
|
27054
|
+
const scores = buildExperimentScores(row.baseline, entry.scores);
|
|
27055
|
+
const metrics = { ...row.metrics };
|
|
27056
|
+
if (Number.isFinite(entry.durationMs)) {
|
|
27057
|
+
metrics.durationMs = entry.durationMs;
|
|
27058
|
+
}
|
|
27059
|
+
return {
|
|
27060
|
+
input: redact({
|
|
27061
|
+
brief: readFirstString(entryRecord, ["brief"]),
|
|
27062
|
+
baseline: row.baseline,
|
|
27063
|
+
agent: row.agent,
|
|
27064
|
+
iteration: row.iteration
|
|
27065
|
+
}),
|
|
27066
|
+
output: redact({
|
|
27067
|
+
diff_summary: entry.agentOutput,
|
|
27068
|
+
kept: entry.status === "keep"
|
|
27069
|
+
}),
|
|
27070
|
+
...Object.keys(scores).length === 0 ? {} : { scores },
|
|
27071
|
+
...Object.keys(metrics).length === 0 ? {} : { metrics },
|
|
27072
|
+
metadata: {
|
|
27073
|
+
...row.commit !== void 0 ? { commit: row.commit } : {},
|
|
27074
|
+
...row.reset !== void 0 ? { reset: row.reset } : {}
|
|
27075
|
+
}
|
|
27076
|
+
};
|
|
27077
|
+
}
|
|
27078
|
+
function buildPipelineMetrics(progress) {
|
|
27079
|
+
const metrics = {};
|
|
27080
|
+
const usage = progress.usage;
|
|
27081
|
+
if (usage !== void 0) {
|
|
27082
|
+
addMetric(metrics, "prompt_tokens", usage.inputTokens);
|
|
27083
|
+
addMetric(metrics, "completion_tokens", usage.outputTokens);
|
|
27084
|
+
addMetric(metrics, "tokens", usage.inputTokens + usage.outputTokens);
|
|
27085
|
+
addMetric(metrics, "prompt_cached_tokens", usage.cachedTokens);
|
|
27086
|
+
}
|
|
27087
|
+
addMetric(metrics, "durationMs", progress.durationMs);
|
|
27088
|
+
return metrics;
|
|
27089
|
+
}
|
|
27090
|
+
function buildExperimentScores(baseline, scores) {
|
|
27091
|
+
const result = { ...scores ?? {} };
|
|
27092
|
+
const delta = sumDelta(baseline, scores);
|
|
27093
|
+
if (delta !== void 0) {
|
|
27094
|
+
result.delta = delta;
|
|
27095
|
+
}
|
|
27096
|
+
return result;
|
|
27097
|
+
}
|
|
27098
|
+
function sumDelta(baseline, scores) {
|
|
27099
|
+
if (baseline === void 0 || scores === void 0) {
|
|
27100
|
+
return void 0;
|
|
27101
|
+
}
|
|
27102
|
+
let delta = 0;
|
|
27103
|
+
let count = 0;
|
|
27104
|
+
for (const [name, value] of Object.entries(scores)) {
|
|
27105
|
+
const base = baseline[name];
|
|
27106
|
+
if (Number.isFinite(value) && Number.isFinite(base)) {
|
|
27107
|
+
delta += value - base;
|
|
27108
|
+
count += 1;
|
|
27109
|
+
}
|
|
27110
|
+
}
|
|
27111
|
+
return count === 0 ? void 0 : delta;
|
|
27112
|
+
}
|
|
27113
|
+
function readSatisfiedScore(record) {
|
|
27114
|
+
const verdict = record?.verdict ?? record?.satisfied;
|
|
27115
|
+
if (typeof verdict === "boolean") {
|
|
27116
|
+
return verdict ? 1 : 0;
|
|
27117
|
+
}
|
|
27118
|
+
if (typeof verdict === "number" && Number.isFinite(verdict)) {
|
|
27119
|
+
return verdict > 0 ? 1 : 0;
|
|
27120
|
+
}
|
|
27121
|
+
if (typeof verdict !== "string") {
|
|
27122
|
+
return void 0;
|
|
27123
|
+
}
|
|
27124
|
+
const normalized = verdict.trim().toLowerCase();
|
|
27125
|
+
if (["satisfied", "satisfy", "pass", "passed", "approve", "approved", "true", "yes"].includes(normalized)) {
|
|
27126
|
+
return 1;
|
|
27127
|
+
}
|
|
27128
|
+
if (["unsatisfied", "fail", "failed", "reject", "rejected", "false", "no"].includes(normalized)) {
|
|
27129
|
+
return 0;
|
|
27130
|
+
}
|
|
27131
|
+
return void 0;
|
|
27132
|
+
}
|
|
27133
|
+
function pipelineKey(progress) {
|
|
27134
|
+
return [
|
|
27135
|
+
progress.taskId,
|
|
27136
|
+
progress.stepName ?? "",
|
|
27137
|
+
progress.phase ?? "",
|
|
27138
|
+
String(progress.taskIndex),
|
|
27139
|
+
String(progress.stepIndex ?? "")
|
|
27140
|
+
].join(":");
|
|
27141
|
+
}
|
|
27142
|
+
function readPipelineStep(progress) {
|
|
27143
|
+
return progress.stepName ?? progress.phase ?? progress.taskTitle;
|
|
27144
|
+
}
|
|
27145
|
+
function readPipelineIndex(progress) {
|
|
27146
|
+
return progress.stepIndex ?? progress.taskIndex;
|
|
27147
|
+
}
|
|
27148
|
+
function readLatestRow(rows) {
|
|
27149
|
+
let latest;
|
|
27150
|
+
for (const row of rows.values()) {
|
|
27151
|
+
if (latest === void 0 || row.iteration > latest.iteration) {
|
|
27152
|
+
latest = row;
|
|
27153
|
+
}
|
|
27154
|
+
}
|
|
27155
|
+
return latest;
|
|
27156
|
+
}
|
|
27157
|
+
function asSpanParent(value) {
|
|
27158
|
+
const span = value;
|
|
27159
|
+
if (span === void 0 || typeof span.startSpan !== "function") {
|
|
27160
|
+
throw new Error("Braintrust span parent unavailable");
|
|
27161
|
+
}
|
|
27162
|
+
return span;
|
|
27163
|
+
}
|
|
27164
|
+
function asRecord2(value) {
|
|
27165
|
+
return typeof value === "object" && value !== null ? value : void 0;
|
|
27166
|
+
}
|
|
27167
|
+
function readFirstString(record, keys) {
|
|
27168
|
+
const value = readFirstValue(record, keys);
|
|
27169
|
+
return typeof value === "string" ? value : void 0;
|
|
27170
|
+
}
|
|
27171
|
+
function readFirstValue(record, keys) {
|
|
27172
|
+
for (const key2 of keys) {
|
|
27173
|
+
if (Object.hasOwn(record, key2)) {
|
|
27174
|
+
return record[key2];
|
|
27175
|
+
}
|
|
27176
|
+
}
|
|
27177
|
+
return void 0;
|
|
27178
|
+
}
|
|
27179
|
+
function addMetric(metrics, key2, value) {
|
|
27180
|
+
if (value !== void 0 && Number.isFinite(value)) {
|
|
27181
|
+
metrics[key2] = value;
|
|
27182
|
+
}
|
|
27183
|
+
}
|
|
27184
|
+
|
|
27185
|
+
// packages/braintrust/src/adapters/experiment.ts
|
|
27186
|
+
function createExperimentCallbacks(client, experimentName) {
|
|
27187
|
+
const state = makeExperimentIterationState(client, experimentName);
|
|
27188
|
+
return {
|
|
27189
|
+
onExperimentStart(index, agent2) {
|
|
27190
|
+
void state.start(index, agent2);
|
|
27191
|
+
},
|
|
27192
|
+
onBaselineCollected(baseline) {
|
|
27193
|
+
state.baseline(baseline);
|
|
27194
|
+
},
|
|
27195
|
+
onMetricResult(metric, result) {
|
|
27196
|
+
if (result.score !== null) {
|
|
27197
|
+
state.metric(metric.name, result.score);
|
|
27198
|
+
}
|
|
27199
|
+
},
|
|
27200
|
+
onCommit(commitHash) {
|
|
27201
|
+
state.commit(commitHash);
|
|
27202
|
+
},
|
|
27203
|
+
onReset(targetHash) {
|
|
27204
|
+
state.reset(targetHash);
|
|
27205
|
+
},
|
|
27206
|
+
onExperimentComplete(index, entry) {
|
|
27207
|
+
void state.complete(index, entry);
|
|
27208
|
+
}
|
|
27209
|
+
};
|
|
27210
|
+
}
|
|
27211
|
+
|
|
27212
|
+
// packages/braintrust/src/adapters/pipeline.ts
|
|
27213
|
+
function createPipelineCallbacks(client) {
|
|
27214
|
+
const state = makePipelineRowState(client);
|
|
27215
|
+
return {
|
|
27216
|
+
onPlanResolved(summary) {
|
|
27217
|
+
void summary;
|
|
27218
|
+
},
|
|
27219
|
+
onTaskStart(progress) {
|
|
27220
|
+
state.start(progress);
|
|
27221
|
+
},
|
|
27222
|
+
onTaskComplete(progress) {
|
|
27223
|
+
state.complete(progress);
|
|
27224
|
+
},
|
|
27225
|
+
onLockStatusChange(status) {
|
|
27226
|
+
void status;
|
|
27227
|
+
}
|
|
27228
|
+
};
|
|
27229
|
+
}
|
|
27230
|
+
|
|
27231
|
+
// packages/braintrust/src/span-builder.ts
|
|
27232
|
+
async function logSpawnSession(client, ctx) {
|
|
27233
|
+
try {
|
|
27234
|
+
const { currentSpan } = await import("braintrust");
|
|
27235
|
+
const agentSpan = asSpanParent2(currentSpan()).startSpan({
|
|
27236
|
+
name: `agent:${ctx.agent}:${ctx.model ?? "?"}`,
|
|
27237
|
+
type: "task"
|
|
27238
|
+
});
|
|
27239
|
+
try {
|
|
27240
|
+
logToolSpans(agentSpan, ctx.events);
|
|
27241
|
+
agentSpan.log({
|
|
27242
|
+
input: redact({
|
|
27243
|
+
prompt: ctx.prompt,
|
|
27244
|
+
mode: ctx.mode,
|
|
27245
|
+
cwd: ctx.cwd
|
|
27246
|
+
}),
|
|
27247
|
+
output: redact(accumulateAgentOutput(ctx.events)),
|
|
27248
|
+
metadata: {
|
|
27249
|
+
sessionId: ctx.sessionId,
|
|
27250
|
+
threadId: ctx.threadId,
|
|
27251
|
+
...ctx.metadata
|
|
27252
|
+
},
|
|
27253
|
+
metrics: buildMetrics(ctx)
|
|
27254
|
+
});
|
|
27255
|
+
} finally {
|
|
27256
|
+
agentSpan.end();
|
|
27257
|
+
}
|
|
27258
|
+
} catch (err) {
|
|
27259
|
+
client.recordError(err, "log spawn session");
|
|
27260
|
+
}
|
|
27261
|
+
}
|
|
27262
|
+
function logToolSpans(agentSpan, events) {
|
|
27263
|
+
for (const [index, event] of events.entries()) {
|
|
27264
|
+
const toolCall = asToolCall(event);
|
|
27265
|
+
if (toolCall === void 0) {
|
|
27266
|
+
continue;
|
|
27267
|
+
}
|
|
27268
|
+
const toolSpan = agentSpan.startSpan({
|
|
27269
|
+
name: `tool_call:${readString6(toolCall.kind) ?? "unknown"}`,
|
|
27270
|
+
type: "tool"
|
|
27271
|
+
});
|
|
27272
|
+
try {
|
|
27273
|
+
const metadata = collectToolMeta(events, index, readString6(toolCall.toolCallId));
|
|
27274
|
+
toolSpan.log({
|
|
27275
|
+
input: redact(readToolInput(toolCall)),
|
|
27276
|
+
output: redact(assembleToolOutput(events, index, readString6(toolCall.toolCallId))),
|
|
27277
|
+
...metadata ? { metadata } : {}
|
|
27278
|
+
});
|
|
27279
|
+
} finally {
|
|
27280
|
+
toolSpan.end();
|
|
27281
|
+
}
|
|
27282
|
+
}
|
|
27283
|
+
}
|
|
27284
|
+
function collectToolMeta(events, toolCallIndex, toolCallId) {
|
|
27285
|
+
const merged = {};
|
|
27286
|
+
const startMeta = asRecord3(asRecord3(events[toolCallIndex])?._meta);
|
|
27287
|
+
if (startMeta) {
|
|
27288
|
+
for (const [key2, value] of Object.entries(startMeta)) {
|
|
27289
|
+
merged[key2 === "ts" ? "startTs" : key2] = value;
|
|
27290
|
+
}
|
|
27291
|
+
}
|
|
27292
|
+
for (const event of events.slice(toolCallIndex + 1)) {
|
|
27293
|
+
const update = asToolCallUpdate(event);
|
|
27294
|
+
if (update === void 0) continue;
|
|
27295
|
+
if (toolCallId !== void 0 && update.toolCallId !== toolCallId) continue;
|
|
27296
|
+
const updateMeta = asRecord3(update._meta);
|
|
27297
|
+
if (!updateMeta) continue;
|
|
27298
|
+
for (const [key2, value] of Object.entries(updateMeta)) {
|
|
27299
|
+
merged[key2 === "ts" ? "endTs" : key2] = value;
|
|
27300
|
+
}
|
|
27301
|
+
}
|
|
27302
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
27303
|
+
}
|
|
27304
|
+
function accumulateAgentOutput(events) {
|
|
27305
|
+
let output = "";
|
|
27306
|
+
for (const event of events) {
|
|
27307
|
+
const record = asRecord3(event);
|
|
27308
|
+
if (record === void 0) {
|
|
27309
|
+
continue;
|
|
27310
|
+
}
|
|
27311
|
+
if (record.event === "agent_message") {
|
|
27312
|
+
output += readString6(record.text) ?? "";
|
|
27313
|
+
continue;
|
|
27314
|
+
}
|
|
27315
|
+
if (record.sessionUpdate === "agent_message_chunk") {
|
|
27316
|
+
output += readContentText(record.content);
|
|
27317
|
+
}
|
|
27318
|
+
}
|
|
27319
|
+
return output;
|
|
27320
|
+
}
|
|
27321
|
+
function assembleToolOutput(events, toolCallIndex, toolCallId) {
|
|
27322
|
+
const outputs = [];
|
|
27323
|
+
let text4 = "";
|
|
27324
|
+
for (const event of events.slice(toolCallIndex + 1)) {
|
|
27325
|
+
const update = asToolCallUpdate(event);
|
|
27326
|
+
if (update === void 0) {
|
|
27327
|
+
continue;
|
|
27328
|
+
}
|
|
27329
|
+
if (toolCallId !== void 0 && update.toolCallId !== toolCallId) {
|
|
27330
|
+
continue;
|
|
27331
|
+
}
|
|
27332
|
+
if (Object.hasOwn(update, "rawOutput")) {
|
|
27333
|
+
outputs.push(update.rawOutput);
|
|
27334
|
+
}
|
|
27335
|
+
const contentText = readContentText(update.content);
|
|
27336
|
+
if (contentText.length > 0) {
|
|
27337
|
+
text4 += contentText;
|
|
27338
|
+
}
|
|
27339
|
+
}
|
|
27340
|
+
if (outputs.length === 0) {
|
|
27341
|
+
return text4;
|
|
27342
|
+
}
|
|
27343
|
+
if (text4.length > 0) {
|
|
27344
|
+
outputs.push(text4);
|
|
27345
|
+
}
|
|
27346
|
+
return outputs.length === 1 ? outputs[0] : outputs;
|
|
27347
|
+
}
|
|
27348
|
+
function buildMetrics(ctx) {
|
|
27349
|
+
const usage = ctx.usage;
|
|
27350
|
+
const metrics = {};
|
|
27351
|
+
const promptTokens = readNumber(usage.prompt_tokens) ?? readNumber(usage.inputTokens);
|
|
27352
|
+
const completionTokens = readNumber(usage.completion_tokens) ?? readNumber(usage.outputTokens);
|
|
27353
|
+
addMetric2(metrics, "prompt_tokens", promptTokens);
|
|
27354
|
+
addMetric2(metrics, "completion_tokens", completionTokens);
|
|
27355
|
+
addMetric2(
|
|
27356
|
+
metrics,
|
|
27357
|
+
"tokens",
|
|
27358
|
+
readNumber(usage.tokens) ?? sumIfPresent(promptTokens, completionTokens)
|
|
27359
|
+
);
|
|
27360
|
+
addMetric2(
|
|
27361
|
+
metrics,
|
|
27362
|
+
"prompt_cached_tokens",
|
|
27363
|
+
readNumber(usage.prompt_cached_tokens) ?? readNumber(usage.cachedTokens)
|
|
27364
|
+
);
|
|
27365
|
+
addMetric2(
|
|
27366
|
+
metrics,
|
|
27367
|
+
"prompt_cache_creation_tokens",
|
|
27368
|
+
readNumber(usage.prompt_cache_creation_tokens)
|
|
27369
|
+
);
|
|
27370
|
+
addMetric2(metrics, "durationMs", readNumber(usage.durationMs));
|
|
27371
|
+
return metrics;
|
|
27372
|
+
}
|
|
27373
|
+
function asSpanParent2(value) {
|
|
27374
|
+
const span = value;
|
|
27375
|
+
if (span === void 0 || typeof span.startSpan !== "function") {
|
|
27376
|
+
throw new Error("Braintrust current span unavailable");
|
|
27377
|
+
}
|
|
27378
|
+
return span;
|
|
27379
|
+
}
|
|
27380
|
+
function asToolCall(event) {
|
|
27381
|
+
const record = asRecord3(event);
|
|
27382
|
+
return record?.sessionUpdate === "tool_call" ? record : void 0;
|
|
27383
|
+
}
|
|
27384
|
+
function asToolCallUpdate(event) {
|
|
27385
|
+
const record = asRecord3(event);
|
|
27386
|
+
return record?.sessionUpdate === "tool_call_update" ? record : void 0;
|
|
27387
|
+
}
|
|
27388
|
+
function readToolInput(toolCall) {
|
|
27389
|
+
if (Object.hasOwn(toolCall, "input")) {
|
|
27390
|
+
return toolCall.input;
|
|
27391
|
+
}
|
|
27392
|
+
return toolCall.rawInput;
|
|
27393
|
+
}
|
|
27394
|
+
function readContentText(value) {
|
|
27395
|
+
if (Array.isArray(value)) {
|
|
27396
|
+
return value.map(readContentText).join("");
|
|
27397
|
+
}
|
|
27398
|
+
const record = asRecord3(value);
|
|
27399
|
+
if (record === void 0 || record.type !== "text") {
|
|
27400
|
+
return "";
|
|
27401
|
+
}
|
|
27402
|
+
return readString6(record.text) ?? "";
|
|
27403
|
+
}
|
|
27404
|
+
function asRecord3(value) {
|
|
27405
|
+
return typeof value === "object" && value !== null ? value : void 0;
|
|
27406
|
+
}
|
|
27407
|
+
function readString6(value) {
|
|
27408
|
+
return typeof value === "string" ? value : void 0;
|
|
27409
|
+
}
|
|
27410
|
+
function readNumber(value) {
|
|
27411
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
27412
|
+
}
|
|
27413
|
+
function addMetric2(metrics, key2, value) {
|
|
27414
|
+
if (value !== void 0) {
|
|
27415
|
+
metrics[key2] = value;
|
|
27416
|
+
}
|
|
27417
|
+
}
|
|
27418
|
+
function sumIfPresent(left, right) {
|
|
27419
|
+
return left !== void 0 && right !== void 0 ? left + right : void 0;
|
|
27420
|
+
}
|
|
27421
|
+
|
|
27422
|
+
// packages/braintrust/src/adapters/spawn.ts
|
|
27423
|
+
function createSpawnMiddleware(client) {
|
|
27424
|
+
return async (ctx, next) => {
|
|
27425
|
+
try {
|
|
27426
|
+
await next();
|
|
27427
|
+
} catch (err) {
|
|
27428
|
+
const metadataCtx = ctx;
|
|
27429
|
+
metadataCtx.metadata = {
|
|
27430
|
+
...metadataCtx.metadata,
|
|
27431
|
+
aborted: true
|
|
27432
|
+
};
|
|
27433
|
+
throw err;
|
|
27434
|
+
} finally {
|
|
27435
|
+
await logSpawnSession(client, ctx);
|
|
27436
|
+
}
|
|
27437
|
+
};
|
|
27438
|
+
}
|
|
27439
|
+
|
|
27440
|
+
// packages/braintrust/src/adapters/superintendent.ts
|
|
27441
|
+
function createSuperintendentCallbacks(client) {
|
|
27442
|
+
return {
|
|
27443
|
+
onBuilderComplete(result) {
|
|
27444
|
+
void logSuperintendentRole(client, "builder", result);
|
|
27445
|
+
},
|
|
27446
|
+
onBuilderFailed(error2) {
|
|
27447
|
+
void logFailedRole(client, "builder", error2);
|
|
27448
|
+
},
|
|
27449
|
+
onInspectorComplete(result) {
|
|
27450
|
+
void logSuperintendentRole(client, "inspector", result);
|
|
27451
|
+
},
|
|
27452
|
+
onInspectorFailed(name, error2) {
|
|
27453
|
+
void logFailedRole(client, "inspector", error2, name);
|
|
27454
|
+
},
|
|
27455
|
+
onSuperintendentComplete(result) {
|
|
27456
|
+
void logSuperintendentRole(client, "superintendent", result);
|
|
27457
|
+
},
|
|
27458
|
+
onOwnerComplete(result) {
|
|
27459
|
+
void logSuperintendentRole(client, "owner", result);
|
|
27460
|
+
}
|
|
27461
|
+
};
|
|
27462
|
+
}
|
|
27463
|
+
async function logFailedRole(client, role, error2, name) {
|
|
27464
|
+
try {
|
|
27465
|
+
const { currentSpan } = await import("braintrust");
|
|
27466
|
+
const span = asSpanParent3(currentSpan()).startSpan({
|
|
27467
|
+
name: name === void 0 ? `role:${role}:failed` : `role:${role}:${name}:failed`,
|
|
27468
|
+
type: "task"
|
|
27469
|
+
});
|
|
27470
|
+
try {
|
|
27471
|
+
span.log({
|
|
27472
|
+
metadata: {
|
|
27473
|
+
role,
|
|
27474
|
+
...name === void 0 ? {} : { name },
|
|
27475
|
+
error: error2.message
|
|
27476
|
+
},
|
|
27477
|
+
scores: {
|
|
27478
|
+
passed: 0
|
|
27479
|
+
}
|
|
27480
|
+
});
|
|
27481
|
+
} finally {
|
|
27482
|
+
span.end();
|
|
27483
|
+
}
|
|
27484
|
+
} catch (err) {
|
|
27485
|
+
client.recordError(err, `superintendent ${role} failed`);
|
|
27486
|
+
}
|
|
27487
|
+
}
|
|
27488
|
+
function asSpanParent3(value) {
|
|
27489
|
+
const span = value;
|
|
27490
|
+
if (span === void 0 || typeof span.startSpan !== "function") {
|
|
27491
|
+
throw new Error("Braintrust current span unavailable");
|
|
27492
|
+
}
|
|
27493
|
+
return span;
|
|
27494
|
+
}
|
|
27495
|
+
|
|
27496
|
+
// packages/braintrust/src/trace-run.ts
|
|
27497
|
+
function makeTraceRun(client) {
|
|
27498
|
+
return async (surface, name, fn) => {
|
|
27499
|
+
const spanName = `${surface}:${name}`;
|
|
27500
|
+
let callbackCompleted = false;
|
|
27501
|
+
let callbackFailed = false;
|
|
27502
|
+
let callbackError;
|
|
27503
|
+
let callbackValue;
|
|
27504
|
+
try {
|
|
27505
|
+
const rootLogger = await client.getRootLogger();
|
|
27506
|
+
if (rootLogger === void 0) {
|
|
27507
|
+
throw new Error("Braintrust logger unavailable");
|
|
27508
|
+
}
|
|
27509
|
+
const sdk = await client.getSdk();
|
|
27510
|
+
const traced = resolveTraced(sdk);
|
|
27511
|
+
const tracedValue = await traced(
|
|
27512
|
+
async () => {
|
|
27513
|
+
try {
|
|
27514
|
+
callbackValue = await fn();
|
|
27515
|
+
} catch (err) {
|
|
27516
|
+
callbackFailed = true;
|
|
27517
|
+
callbackError = err;
|
|
27518
|
+
throw err;
|
|
27519
|
+
}
|
|
27520
|
+
callbackCompleted = true;
|
|
27521
|
+
return callbackValue;
|
|
27522
|
+
},
|
|
27523
|
+
{
|
|
27524
|
+
name: spanName,
|
|
27525
|
+
type: "task",
|
|
27526
|
+
event: {
|
|
27527
|
+
tags: [`surface:${surface}`]
|
|
27528
|
+
}
|
|
27529
|
+
}
|
|
27530
|
+
);
|
|
27531
|
+
return tracedValue;
|
|
27532
|
+
} catch (err) {
|
|
27533
|
+
if (callbackFailed) {
|
|
27534
|
+
throw callbackError;
|
|
27535
|
+
}
|
|
27536
|
+
client.recordError(err, `trace ${spanName}`);
|
|
27537
|
+
if (callbackCompleted) {
|
|
27538
|
+
return callbackValue;
|
|
27539
|
+
}
|
|
27540
|
+
return fn();
|
|
27541
|
+
}
|
|
27542
|
+
};
|
|
27543
|
+
}
|
|
27544
|
+
function resolveTraced(sdk) {
|
|
27545
|
+
if (sdk === void 0) {
|
|
27546
|
+
throw new Error("Braintrust SDK unavailable");
|
|
27547
|
+
}
|
|
27548
|
+
if (typeof sdk.traced !== "function") {
|
|
27549
|
+
throw new Error("Braintrust SDK traced unavailable");
|
|
27550
|
+
}
|
|
27551
|
+
return sdk.traced;
|
|
27552
|
+
}
|
|
27553
|
+
|
|
27554
|
+
// packages/braintrust/src/load-integrations.ts
|
|
27555
|
+
async function loadIntegrations(config) {
|
|
27556
|
+
if (config.integrations?.braintrust?.enabled !== true) {
|
|
27557
|
+
return null;
|
|
27558
|
+
}
|
|
27559
|
+
return bootstrap(config.integrations.braintrust);
|
|
27560
|
+
}
|
|
27561
|
+
|
|
27562
|
+
// packages/braintrust/src/index.ts
|
|
27563
|
+
function bootstrap(options) {
|
|
27564
|
+
if (options?.enabled !== true) {
|
|
27565
|
+
return null;
|
|
27566
|
+
}
|
|
27567
|
+
validateBraintrustConfig(options);
|
|
27568
|
+
const client = createClient(options);
|
|
27569
|
+
return {
|
|
27570
|
+
spawnMiddleware: createSpawnMiddleware(client),
|
|
27571
|
+
pipelineCallbacks: createPipelineCallbacks(client),
|
|
27572
|
+
experimentCallbacks: createExperimentCallbacks(client, options.project),
|
|
27573
|
+
superintendentCallbacks: createSuperintendentCallbacks(client),
|
|
27574
|
+
status: () => client.status(),
|
|
27575
|
+
traceRun: makeTraceRun(client),
|
|
27576
|
+
shutdown() {
|
|
27577
|
+
return client.flush(5e3);
|
|
27578
|
+
}
|
|
27579
|
+
};
|
|
27580
|
+
}
|
|
27581
|
+
function validateBraintrustConfig(options) {
|
|
27582
|
+
requiredString(options.apiKey, "apiKey");
|
|
27583
|
+
requiredString(options.project, "project");
|
|
27584
|
+
}
|
|
27585
|
+
function requiredString(value, field) {
|
|
27586
|
+
if (typeof value === "string" && value.trim() !== "") {
|
|
27587
|
+
return;
|
|
27588
|
+
}
|
|
27589
|
+
throw new Error(`Braintrust integration is enabled but ${field} is missing`);
|
|
27590
|
+
}
|
|
27591
|
+
|
|
26702
27592
|
// packages/superintendent/src/config-scope.ts
|
|
26703
27593
|
init_src5();
|
|
26704
27594
|
var superintendentConfigScope = defineScope("superintendent", {
|