totemlayer 0.1.16 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -39
- package/bin/agent-runner.mjs +7 -2
- package/bin/automatic-runtime.mjs +22 -3
- package/bin/executor-readiness.mjs +48 -18
- package/bin/inbox-queue.mjs +23 -10
- package/bin/installation.mjs +35 -8
- package/bin/mission-infrastructure.mjs +0 -49
- package/bin/mission-worker.mjs +11 -129
- package/bin/model-adapter.mjs +28 -6
- package/bin/network-retry.mjs +26 -0
- package/bin/network-runtime.mjs +75 -22
- package/bin/runtime-diagnostics.mjs +33 -8
- package/bin/sdk/index.js +11 -40
- package/bin/totem.mjs +1 -1
- package/bin/worker-core/adapters.mjs +3 -1
- package/bin/worker-core/errors.mjs +5 -1
- package/package.json +1 -1
- package/scripts/prepare.mjs +20 -0
package/bin/mission-worker.mjs
CHANGED
|
@@ -21,7 +21,6 @@ import { setTimeout as sleep } from "node:timers/promises";
|
|
|
21
21
|
import { inspectLocalRepository } from "./repository.mjs";
|
|
22
22
|
import { localClientEnvironment } from "./platform-client.mjs";
|
|
23
23
|
import { localModelEndpoint } from "./local-model-provider.mjs";
|
|
24
|
-
import { customerInspector } from "./local-inspector.mjs";
|
|
25
24
|
import { externalModelReply, isExternalModelProvider } from "./model-adapter.mjs";
|
|
26
25
|
import { runDockerNodeTests } from "./docker-test-sandbox.mjs";
|
|
27
26
|
import { acquireMissionAgentSession, assertMissionAgentSession } from "./mission-agent-session.mjs";
|
|
@@ -753,10 +752,7 @@ export async function runCodexMissionAgent({
|
|
|
753
752
|
stage,
|
|
754
753
|
agentName,
|
|
755
754
|
agentAddress,
|
|
756
|
-
inspector = customerInspector(),
|
|
757
|
-
task,
|
|
758
755
|
}) {
|
|
759
|
-
await inspector.assertAllowed("input", prompt, { task, signal });
|
|
760
756
|
if (provider !== "ollama-local")
|
|
761
757
|
throw new Error(
|
|
762
758
|
"The legacy Codex adapter requires explicit provider: ollama-local. Use runLocalMissionAgent for native Qwen by default.",
|
|
@@ -842,7 +838,6 @@ export async function runCodexMissionAgent({
|
|
|
842
838
|
});
|
|
843
839
|
}
|
|
844
840
|
if (!final.trim()) throw new Error("The connected agent returned no final result.");
|
|
845
|
-
await inspector.assertAllowed("output", { text: final, commands }, { task, signal });
|
|
846
841
|
return { text: final.slice(0, 32000), commands: commands.slice(-30) };
|
|
847
842
|
}
|
|
848
843
|
|
|
@@ -909,14 +904,7 @@ export function redactRepositorySource(source) {
|
|
|
909
904
|
return output;
|
|
910
905
|
}
|
|
911
906
|
|
|
912
|
-
export async function repositoryEvidence(
|
|
913
|
-
checkout,
|
|
914
|
-
stage,
|
|
915
|
-
checkoutInfo,
|
|
916
|
-
inspector = customerInspector(),
|
|
917
|
-
task,
|
|
918
|
-
signal,
|
|
919
|
-
) {
|
|
907
|
+
export async function repositoryEvidence(checkout, stage, checkoutInfo) {
|
|
920
908
|
if (!["engineering", "review"].includes(stage)) return "";
|
|
921
909
|
const tracked = (await git(checkout, "ls-files"))
|
|
922
910
|
.split("\n")
|
|
@@ -947,9 +935,7 @@ export async function repositoryEvidence(
|
|
|
947
935
|
throw new Error(`The reviewer cannot inspect the complete changed file: ${path}`);
|
|
948
936
|
continue;
|
|
949
937
|
}
|
|
950
|
-
const
|
|
951
|
-
await inspector.assertAllowed("tool-result", { path, source: original }, { task, signal });
|
|
952
|
-
const source = redactRepositorySource(original);
|
|
938
|
+
const source = redactRepositorySource(await readFile(absolute, "utf8"));
|
|
953
939
|
const numbered = source
|
|
954
940
|
.split("\n")
|
|
955
941
|
.map((line, index) => `${index + 1}: ${line}`)
|
|
@@ -964,10 +950,7 @@ export async function repositoryEvidence(
|
|
|
964
950
|
return JSON.stringify({ trackedFiles: paths, changedFiles: changed, files });
|
|
965
951
|
}
|
|
966
952
|
|
|
967
|
-
async function codeRepositoryEvidence(
|
|
968
|
-
checkout,
|
|
969
|
-
{ repair = false, inspector = customerInspector(), task, signal } = {},
|
|
970
|
-
) {
|
|
953
|
+
async function codeRepositoryEvidence(checkout, { repair = false } = {}) {
|
|
971
954
|
const checkoutPath = await realpath(checkout);
|
|
972
955
|
const changed = repair
|
|
973
956
|
? [
|
|
@@ -1005,7 +988,6 @@ async function codeRepositoryEvidence(
|
|
|
1005
988
|
continue;
|
|
1006
989
|
}
|
|
1007
990
|
const content = await readFile(absolute, "utf8");
|
|
1008
|
-
await inspector.assertAllowed("tool-result", { path, content }, { task, signal });
|
|
1009
991
|
if (
|
|
1010
992
|
/-----BEGIN [A-Z ]*PRIVATE KEY-----|\b(?:sk-[A-Za-z0-9_-]{20,}|gh[pousr]_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16})\b/.test(
|
|
1011
993
|
content,
|
|
@@ -1587,10 +1569,7 @@ async function runStructuredMissionAgent({
|
|
|
1587
1569
|
reasoning = false,
|
|
1588
1570
|
rejectionEvidenceDirectory,
|
|
1589
1571
|
adapterSettings = {},
|
|
1590
|
-
inspector = customerInspector(),
|
|
1591
|
-
task,
|
|
1592
1572
|
}) {
|
|
1593
|
-
await inspector.assertAllowed("input", prompt, { task, signal });
|
|
1594
1573
|
const regressionFile =
|
|
1595
1574
|
stage === "code" && !repair ? await selectMissionRegressionTestFile(checkout) : null;
|
|
1596
1575
|
const properties =
|
|
@@ -1651,9 +1630,8 @@ async function runStructuredMissionAgent({
|
|
|
1651
1630
|
throw new Error("Specify the installed model for this provider.");
|
|
1652
1631
|
const evidence =
|
|
1653
1632
|
stage === "code"
|
|
1654
|
-
? await codeRepositoryEvidence(checkout, { repair
|
|
1655
|
-
: await repositoryEvidence(checkout, stage, checkoutInfo
|
|
1656
|
-
await inspector.assertAllowed("tool-result", evidence, { task, signal });
|
|
1633
|
+
? await codeRepositoryEvidence(checkout, { repair })
|
|
1634
|
+
: await repositoryEvidence(checkout, stage, checkoutInfo);
|
|
1657
1635
|
const regressionInstruction = regressionFile
|
|
1658
1636
|
? `\n\nWorker-selected regression file: ${JSON.stringify(regressionFile.path)}. Return exactly this one path in testChanges, as a new file. Do not edit, replace, or rename any existing test file in initial generation. The new .test.js file uses ${regressionFile.moduleType === "module" ? "ES modules: import test from 'node:test' and assert from 'node:assert/strict'" : "CommonJS: require('node:test') and require('node:assert/strict'); use an async test with dynamic import() if the source API is an ES module"}. Import the actual source API with paths relative to this file. Node --test discovers this filename. Baseline expectation changes for accepted new fields are reserved for exact-edit repair after actual test failure.`
|
|
1659
1637
|
: "";
|
|
@@ -1662,7 +1640,6 @@ async function runStructuredMissionAgent({
|
|
|
1662
1640
|
let correction = "";
|
|
1663
1641
|
const useReasoning = reasoning === true && ["code", "review"].includes(stage);
|
|
1664
1642
|
for (let attempt = 0; attempt < (stage === "code" ? 2 : 1); attempt += 1) {
|
|
1665
|
-
await inspector.assertAllowed("input", correction, { task, signal });
|
|
1666
1643
|
const generationLimit = useReasoning ? 8192 : stage === "code" ? 4096 : 2048;
|
|
1667
1644
|
const data = await requestModel(endpoint, {
|
|
1668
1645
|
method: "POST",
|
|
@@ -1709,25 +1686,15 @@ async function runStructuredMissionAgent({
|
|
|
1709
1686
|
provider !== "ollama-local" ? data?.choices?.[0]?.message?.content : data?.message?.content;
|
|
1710
1687
|
if (typeof result !== "string" || !result.trim())
|
|
1711
1688
|
throw new Error("The local model returned no structured result.");
|
|
1712
|
-
await inspector.assertAllowed("output", result, { task, signal: combined });
|
|
1713
1689
|
if (stage !== "code") return { text: result.slice(0, 32000), commands: [] };
|
|
1714
1690
|
try {
|
|
1715
1691
|
return repair
|
|
1716
|
-
? await
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
{ checkout, result },
|
|
1723
|
-
(input) =>
|
|
1724
|
-
applyModelFileChanges(input.checkout, input.result, {
|
|
1725
|
-
requireRegressionTests: true,
|
|
1726
|
-
requireSeparateTestChanges: true,
|
|
1727
|
-
newRegressionTestPath: regressionFile.path,
|
|
1728
|
-
}),
|
|
1729
|
-
{ task, signal: combined },
|
|
1730
|
-
);
|
|
1692
|
+
? await applyModelFileEdits(checkout, result)
|
|
1693
|
+
: await applyModelFileChanges(checkout, result, {
|
|
1694
|
+
requireRegressionTests: true,
|
|
1695
|
+
requireSeparateTestChanges: true,
|
|
1696
|
+
newRegressionTestPath: regressionFile.path,
|
|
1697
|
+
});
|
|
1731
1698
|
} catch (error) {
|
|
1732
1699
|
if (rejectionEvidenceDirectory) {
|
|
1733
1700
|
await mkdir(rejectionEvidenceDirectory, {
|
|
@@ -2419,7 +2386,6 @@ async function settleCompletedResult(client, missionId, workItemId, path, record
|
|
|
2419
2386
|
stage: record.stage,
|
|
2420
2387
|
result: record.result,
|
|
2421
2388
|
leaseToken: record.leaseToken,
|
|
2422
|
-
originalTask: record.originalTask,
|
|
2423
2389
|
});
|
|
2424
2390
|
}
|
|
2425
2391
|
|
|
@@ -2431,7 +2397,6 @@ async function recoverReadyCompletions(
|
|
|
2431
2397
|
signal,
|
|
2432
2398
|
emit,
|
|
2433
2399
|
limit = Infinity,
|
|
2434
|
-
inspector = customerInspector(),
|
|
2435
2400
|
) {
|
|
2436
2401
|
let handled = 0;
|
|
2437
2402
|
for (const name of await readdir(directory)) {
|
|
@@ -2479,7 +2444,6 @@ async function recoverReadyCompletions(
|
|
|
2479
2444
|
await reportBlockedCompletion(record);
|
|
2480
2445
|
continue;
|
|
2481
2446
|
}
|
|
2482
|
-
await inspector.assertAllowed("output", record.result, { task: record.originalTask, signal });
|
|
2483
2447
|
if (record.state === "result_settling")
|
|
2484
2448
|
await settleCompletedResult(client, missionId, workItemId, path, record, signal);
|
|
2485
2449
|
await client.missions.complete(
|
|
@@ -2557,26 +2521,7 @@ export async function runMissionWorker({
|
|
|
2557
2521
|
emit = () => {},
|
|
2558
2522
|
runAgent = runLocalMissionAgent,
|
|
2559
2523
|
adapterSettings = {},
|
|
2560
|
-
inspector,
|
|
2561
|
-
inspectorOptions,
|
|
2562
2524
|
}) {
|
|
2563
|
-
inspector = customerInspector({ inspector, inspectorOptions });
|
|
2564
|
-
const uncheckedRunAgent = runAgent;
|
|
2565
|
-
let originalTask;
|
|
2566
|
-
runAgent = async (input) => {
|
|
2567
|
-
const result = await inspector.runAction(
|
|
2568
|
-
{
|
|
2569
|
-
prompt: input.prompt,
|
|
2570
|
-
checkout: input.checkout,
|
|
2571
|
-
write: input.write ?? false,
|
|
2572
|
-
stage: input.stage,
|
|
2573
|
-
},
|
|
2574
|
-
(snapshot) => uncheckedRunAgent({ ...input, ...snapshot, inspector, task: originalTask }),
|
|
2575
|
-
{ task: originalTask, signal: input.signal },
|
|
2576
|
-
);
|
|
2577
|
-
await inspector.assertAllowed("output", result, { task: originalTask, signal: input.signal });
|
|
2578
|
-
return result;
|
|
2579
|
-
};
|
|
2580
2525
|
if (!UUID.test(missionId)) throw new Error("Provide a valid mission ID.");
|
|
2581
2526
|
missionModelEndpoint(provider, { ollamaUrl, qwenUrl, nimUrl });
|
|
2582
2527
|
model ??=
|
|
@@ -2655,7 +2600,6 @@ export async function runMissionWorker({
|
|
|
2655
2600
|
signal,
|
|
2656
2601
|
emit,
|
|
2657
2602
|
_singleCycle ? 1 : Infinity,
|
|
2658
|
-
inspector,
|
|
2659
2603
|
);
|
|
2660
2604
|
if (_singleCycle && recovered) return { activity: "recovery", handled: recovered };
|
|
2661
2605
|
const reconciled = await recoverKnownLocalTestFailures(
|
|
@@ -2668,7 +2612,6 @@ export async function runMissionWorker({
|
|
|
2668
2612
|
);
|
|
2669
2613
|
if (_singleCycle && reconciled) return { activity: "recovery", handled: reconciled };
|
|
2670
2614
|
const mission = await client.missions.get(missionId, { signal });
|
|
2671
|
-
originalTask = String(mission.goal ?? "");
|
|
2672
2615
|
if (
|
|
2673
2616
|
mission.status === "canceled" ||
|
|
2674
2617
|
mission.status === "completed" ||
|
|
@@ -2713,32 +2656,6 @@ export async function runMissionWorker({
|
|
|
2713
2656
|
)
|
|
2714
2657
|
continue;
|
|
2715
2658
|
processed += 1;
|
|
2716
|
-
try {
|
|
2717
|
-
await inspector.assertAllowed(
|
|
2718
|
-
"input",
|
|
2719
|
-
{
|
|
2720
|
-
goal: mission.goal,
|
|
2721
|
-
desiredOutput: mission.desiredOutput,
|
|
2722
|
-
repository: mission.repository,
|
|
2723
|
-
workItem,
|
|
2724
|
-
events: mission.events ?? [],
|
|
2725
|
-
artifacts: mission.artifacts ?? [],
|
|
2726
|
-
},
|
|
2727
|
-
{ task: originalTask, signal },
|
|
2728
|
-
);
|
|
2729
|
-
} catch (error) {
|
|
2730
|
-
if (!error.code?.startsWith("INSPECTION_")) throw error;
|
|
2731
|
-
if (!reportedAttention.has(workItem.id)) {
|
|
2732
|
-
await recordRecoveryBlocker(workItem, error.code);
|
|
2733
|
-
emit({
|
|
2734
|
-
type: "needs_attention",
|
|
2735
|
-
workItemId: workItem.id,
|
|
2736
|
-
code: error.code,
|
|
2737
|
-
message: "The customer-side inspector blocked this assignment before launch.",
|
|
2738
|
-
});
|
|
2739
|
-
}
|
|
2740
|
-
continue;
|
|
2741
|
-
}
|
|
2742
2659
|
let stage;
|
|
2743
2660
|
try {
|
|
2744
2661
|
stage = stageOf(workItem);
|
|
@@ -3091,16 +3008,6 @@ export async function runMissionWorker({
|
|
|
3091
3008
|
? ["repository.read"]
|
|
3092
3009
|
: [];
|
|
3093
3010
|
const authorizeLocalLaunch = async () => {
|
|
3094
|
-
await inspector.assertAllowed(
|
|
3095
|
-
"action",
|
|
3096
|
-
{
|
|
3097
|
-
stage,
|
|
3098
|
-
actions: repositoryActions,
|
|
3099
|
-
resource: mission.repository?.url ?? null,
|
|
3100
|
-
objective: workItem.objective,
|
|
3101
|
-
},
|
|
3102
|
-
{ task: originalTask, signal: combined },
|
|
3103
|
-
);
|
|
3104
3011
|
// Renewal rechecks mission state, current credential, membership, and the lease.
|
|
3105
3012
|
await client.missions.renew(missionId, workItem.id, claim.leaseToken, {
|
|
3106
3013
|
signal: combined,
|
|
@@ -3149,15 +3056,6 @@ export async function runMissionWorker({
|
|
|
3149
3056
|
for (const repositoryAction of repositoryActions) {
|
|
3150
3057
|
if (!mission.repository)
|
|
3151
3058
|
throw new Error("This assignment needs a selected repository.");
|
|
3152
|
-
await inspector.assertAllowed(
|
|
3153
|
-
"action",
|
|
3154
|
-
{
|
|
3155
|
-
action: repositoryAction,
|
|
3156
|
-
resource: mission.repository.url,
|
|
3157
|
-
objective: workItem.objective,
|
|
3158
|
-
},
|
|
3159
|
-
{ task: originalTask, signal: combined },
|
|
3160
|
-
);
|
|
3161
3059
|
const reservation =
|
|
3162
3060
|
(prior?.reservedEffects ?? []).find(
|
|
3163
3061
|
(effect) =>
|
|
@@ -3199,16 +3097,6 @@ export async function runMissionWorker({
|
|
|
3199
3097
|
effectIds,
|
|
3200
3098
|
});
|
|
3201
3099
|
for (const effectId of effectIds) {
|
|
3202
|
-
await inspector.assertAllowed(
|
|
3203
|
-
"action",
|
|
3204
|
-
{
|
|
3205
|
-
effectId,
|
|
3206
|
-
actions: repositoryActions,
|
|
3207
|
-
resource: mission.repository.url,
|
|
3208
|
-
objective: workItem.objective,
|
|
3209
|
-
},
|
|
3210
|
-
{ task: originalTask, signal: combined },
|
|
3211
|
-
);
|
|
3212
3100
|
const dispatch = await client.missions.dispatchEffect(
|
|
3213
3101
|
missionId,
|
|
3214
3102
|
effectId,
|
|
@@ -3548,7 +3436,6 @@ export async function runMissionWorker({
|
|
|
3548
3436
|
state: "result_settling",
|
|
3549
3437
|
stage,
|
|
3550
3438
|
result,
|
|
3551
|
-
originalTask,
|
|
3552
3439
|
leaseToken: claim.leaseToken,
|
|
3553
3440
|
effectIds: [...effectIds],
|
|
3554
3441
|
repositoryUrl: mission.repository.url,
|
|
@@ -3569,15 +3456,10 @@ export async function runMissionWorker({
|
|
|
3569
3456
|
state: "ready",
|
|
3570
3457
|
stage,
|
|
3571
3458
|
result,
|
|
3572
|
-
originalTask,
|
|
3573
3459
|
leaseToken: claim.leaseToken,
|
|
3574
3460
|
});
|
|
3575
3461
|
}
|
|
3576
3462
|
}
|
|
3577
|
-
await inspector.assertAllowed("output", result, {
|
|
3578
|
-
task: originalTask,
|
|
3579
|
-
signal: combined,
|
|
3580
|
-
});
|
|
3581
3463
|
await client.missions.complete(
|
|
3582
3464
|
missionId,
|
|
3583
3465
|
workItem.id,
|
package/bin/model-adapter.mjs
CHANGED
|
@@ -7,11 +7,32 @@ import { installedClientInvocation, localClientEnvironment } from "./platform-cl
|
|
|
7
7
|
import { WorkerError } from "./worker-core/errors.mjs";
|
|
8
8
|
|
|
9
9
|
export function providerFailureCode(text) {
|
|
10
|
-
if (
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
if (
|
|
10
|
+
if (
|
|
11
|
+
/unexpected argument|unknown option|unrecognized option|no such option|unknown flag/i.test(text)
|
|
12
|
+
)
|
|
13
|
+
return "AGENT_CLI_INCOMPATIBLE";
|
|
14
|
+
if (
|
|
15
|
+
/not logged in|authentication required|invalid[_ ]api[_ ]key|unauthorized|refresh token.*(?:expired|reused)|sign in.*required/i.test(
|
|
16
|
+
text,
|
|
17
|
+
)
|
|
18
|
+
)
|
|
19
|
+
return "AGENT_AUTH_REQUIRED";
|
|
20
|
+
if (
|
|
21
|
+
/usage limit|rate limit|quota.*(?:exceeded|insufficient)|insufficient_quota|credit balance/i.test(
|
|
22
|
+
text,
|
|
23
|
+
)
|
|
24
|
+
)
|
|
25
|
+
return "AGENT_QUOTA_EXCEEDED";
|
|
26
|
+
if (
|
|
27
|
+
/sandbox.*(?:setup failed|not available|not configured|unsupported|error)|CreateProcessAsUser|error 1385/i.test(
|
|
28
|
+
text,
|
|
29
|
+
)
|
|
30
|
+
)
|
|
31
|
+
return "AGENT_SANDBOX_UNAVAILABLE";
|
|
32
|
+
if (
|
|
33
|
+
/model.*(?:not found|not supported|does not exist|not have access)|model_not_found/i.test(text)
|
|
34
|
+
)
|
|
35
|
+
return "AGENT_MODEL_UNAVAILABLE";
|
|
15
36
|
return "AGENT_EXIT_FAILED";
|
|
16
37
|
}
|
|
17
38
|
|
|
@@ -123,7 +144,8 @@ export async function externalModelReply({
|
|
|
123
144
|
];
|
|
124
145
|
parseResult = (output) => {
|
|
125
146
|
const result = JSON.parse(output);
|
|
126
|
-
if (result.is_error || result.subtype !== "success")
|
|
147
|
+
if (result.is_error || result.subtype !== "success")
|
|
148
|
+
throw new WorkerError(providerFailureCode(String(result.result ?? "")));
|
|
127
149
|
return {
|
|
128
150
|
status: "completed",
|
|
129
151
|
output: result.structured_output
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { pause } from "./network.mjs";
|
|
2
|
+
|
|
3
|
+
export function transientNetworkError(error) {
|
|
4
|
+
return (
|
|
5
|
+
error?.status === 429 ||
|
|
6
|
+
error?.status >= 500 ||
|
|
7
|
+
["NETWORK_BUSY", "NETWORK_ERROR", "TIMEOUT"].includes(error?.code)
|
|
8
|
+
);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
// Only use for reads or mutations protected by a durable idempotency key.
|
|
12
|
+
export async function retryNetwork(
|
|
13
|
+
operation,
|
|
14
|
+
{ signal, onRetry = () => {}, delayMs = 1000, attempts = 4 } = {},
|
|
15
|
+
) {
|
|
16
|
+
for (let attempt = 0; ; attempt++) {
|
|
17
|
+
signal?.throwIfAborted();
|
|
18
|
+
try {
|
|
19
|
+
return await operation();
|
|
20
|
+
} catch (error) {
|
|
21
|
+
if (signal?.aborted || !transientNetworkError(error) || attempt + 1 >= attempts) throw error;
|
|
22
|
+
onRetry(error);
|
|
23
|
+
await pause(Math.min(delayMs * 2 ** attempt, 10000), signal);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
package/bin/network-runtime.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import { join } from "node:path";
|
|
|
5
5
|
import { pause } from "./network.mjs";
|
|
6
6
|
import { localModelEndpoint } from "./local-model-provider.mjs";
|
|
7
7
|
import { runInboxQueue } from "./inbox-queue.mjs";
|
|
8
|
+
import { retryNetwork, transientNetworkError } from "./network-retry.mjs";
|
|
8
9
|
import { customerInspector, inspectNetworkMessage } from "./local-inspector.mjs";
|
|
9
10
|
import { runtimeErrorCode } from "./runtime-diagnostics.mjs";
|
|
10
11
|
import {
|
|
@@ -406,23 +407,38 @@ async function generate({
|
|
|
406
407
|
new Error(
|
|
407
408
|
"A previous provider turn has an uncertain outcome. Inspect its private journal before retrying.",
|
|
408
409
|
),
|
|
409
|
-
{
|
|
410
|
+
{
|
|
411
|
+
code: state.externalFailure ?? "AGENT_OUTCOME_UNCERTAIN",
|
|
412
|
+
externalOutcomeUncertain: true,
|
|
413
|
+
},
|
|
410
414
|
);
|
|
411
415
|
state.externalPending = true;
|
|
412
416
|
await journal.save(key, state);
|
|
413
417
|
}
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
418
|
+
let response;
|
|
419
|
+
try {
|
|
420
|
+
response = await modelReply({
|
|
421
|
+
origin: settings.origin,
|
|
422
|
+
provider: settings.provider,
|
|
423
|
+
qwenUrl: settings.qwenUrl,
|
|
424
|
+
nimUrl: settings.nimUrl,
|
|
425
|
+
model: settings.model,
|
|
426
|
+
messages: state.messages,
|
|
427
|
+
tools,
|
|
428
|
+
signal,
|
|
429
|
+
timeoutMs: settings.modelTimeoutMs,
|
|
430
|
+
adapterSettings: settings.adapterSettings,
|
|
431
|
+
});
|
|
432
|
+
} catch (error) {
|
|
433
|
+
// Preserve a fixed diagnostic instead of hiding the original failure after restart.
|
|
434
|
+
// Keep externalPending: arbitrary external providers may have uncertain side effects.
|
|
435
|
+
if (isExternalModelProvider(settings.provider)) {
|
|
436
|
+
error.externalOutcomeUncertain = true;
|
|
437
|
+
state.externalFailure = runtimeErrorCode(error);
|
|
438
|
+
await journal.save(key, state);
|
|
439
|
+
}
|
|
440
|
+
throw error;
|
|
441
|
+
}
|
|
426
442
|
await settings.inspector.assertAllowed("output", response, { task: text, signal });
|
|
427
443
|
delete state.externalPending;
|
|
428
444
|
state.messages.push(response);
|
|
@@ -575,10 +591,15 @@ export async function runRuntime(settings) {
|
|
|
575
591
|
let fatalError;
|
|
576
592
|
let diagnosticsSupported = false;
|
|
577
593
|
let diagnostics = { phase: "starting" };
|
|
578
|
-
const packageVersion = JSON.parse(
|
|
594
|
+
const packageVersion = JSON.parse(
|
|
595
|
+
await readFile(new URL("../package.json", import.meta.url), "utf8"),
|
|
596
|
+
).version;
|
|
579
597
|
const report = (error, context = {}) => {
|
|
580
|
-
diagnostics = {
|
|
581
|
-
|
|
598
|
+
diagnostics = {
|
|
599
|
+
phase: "blocked",
|
|
600
|
+
errorCode: runtimeErrorCode(error),
|
|
601
|
+
...(typeof context.messageId === "string" ? { messageId: context.messageId } : {}),
|
|
602
|
+
};
|
|
582
603
|
if (settings.once) process.exitCode = 1;
|
|
583
604
|
emit({
|
|
584
605
|
type: "runtime_error",
|
|
@@ -599,9 +620,21 @@ export async function runRuntime(settings) {
|
|
|
599
620
|
try {
|
|
600
621
|
const beat = async () => {
|
|
601
622
|
const result = await client.runtime.heartbeat(
|
|
602
|
-
{
|
|
603
|
-
|
|
604
|
-
|
|
623
|
+
{
|
|
624
|
+
label: `${settings.provider}:${settings.model}`.slice(0, 100),
|
|
625
|
+
...(diagnosticsSupported
|
|
626
|
+
? {
|
|
627
|
+
diagnostics: {
|
|
628
|
+
...diagnostics,
|
|
629
|
+
packageVersion,
|
|
630
|
+
provider: settings.provider,
|
|
631
|
+
platform: ["darwin", "linux", "win32"].includes(process.platform)
|
|
632
|
+
? process.platform
|
|
633
|
+
: "other",
|
|
634
|
+
},
|
|
635
|
+
}
|
|
636
|
+
: {}),
|
|
637
|
+
},
|
|
605
638
|
{ signal },
|
|
606
639
|
);
|
|
607
640
|
diagnosticsSupported = result?.diagnosticsSupported === true;
|
|
@@ -642,6 +675,7 @@ export async function runRuntime(settings) {
|
|
|
642
675
|
pollMs: settings.pollMs,
|
|
643
676
|
concurrency: settings.messageConcurrency ?? 2,
|
|
644
677
|
once: settings.once,
|
|
678
|
+
onPollError: (error) => report(error),
|
|
645
679
|
shouldContinue: () => !settings.once || !humanDone,
|
|
646
680
|
messages: (signal) =>
|
|
647
681
|
client.messages.iterate({ direction: "inbound", unread: true, limit: 100 }, { signal }),
|
|
@@ -720,10 +754,12 @@ export async function runRuntime(settings) {
|
|
|
720
754
|
} catch (error) {
|
|
721
755
|
if (!signal.aborted) report(error, { messageId: message.id });
|
|
722
756
|
if (error.status === 401) throw error;
|
|
723
|
-
|
|
757
|
+
// Retry this durable message on the next poll without stopping other turns.
|
|
758
|
+
if (transientNetworkError(error)) return;
|
|
724
759
|
if (
|
|
725
760
|
(error.status >= 400 && error.status < 500) ||
|
|
726
|
-
error.code === "AGENT_OUTCOME_UNCERTAIN"
|
|
761
|
+
error.code === "AGENT_OUTCOME_UNCERTAIN" ||
|
|
762
|
+
error.externalOutcomeUncertain
|
|
727
763
|
)
|
|
728
764
|
await journal.save(`message-blocker:${message.id}`, {
|
|
729
765
|
code: error.code ?? "MESSAGE_BLOCKED",
|
|
@@ -737,7 +773,20 @@ export async function runRuntime(settings) {
|
|
|
737
773
|
}
|
|
738
774
|
});
|
|
739
775
|
do {
|
|
740
|
-
|
|
776
|
+
let commands;
|
|
777
|
+
try {
|
|
778
|
+
commands = await retryNetwork(() => client.chatCommands.list({ limit: 20 }, { signal }), {
|
|
779
|
+
signal,
|
|
780
|
+
delayMs: settings.pollMs,
|
|
781
|
+
onRetry: report,
|
|
782
|
+
});
|
|
783
|
+
} catch (error) {
|
|
784
|
+
if (signal.aborted) break;
|
|
785
|
+
if (!transientNetworkError(error)) throw error;
|
|
786
|
+
report(error);
|
|
787
|
+
await pause(Math.max(settings.pollMs, 100), signal);
|
|
788
|
+
continue;
|
|
789
|
+
}
|
|
741
790
|
for (const command of commands.data) {
|
|
742
791
|
if (signal.aborted) break;
|
|
743
792
|
let claim;
|
|
@@ -745,6 +794,10 @@ export async function runRuntime(settings) {
|
|
|
745
794
|
claim = await client.chatCommands.claim(command.id, { signal });
|
|
746
795
|
} catch (error) {
|
|
747
796
|
if (error.status === 409) continue;
|
|
797
|
+
if (transientNetworkError(error)) {
|
|
798
|
+
report(error);
|
|
799
|
+
continue;
|
|
800
|
+
}
|
|
748
801
|
throw error;
|
|
749
802
|
}
|
|
750
803
|
const leaseStop = new AbortController();
|
|
@@ -1,12 +1,37 @@
|
|
|
1
1
|
export const RUNTIME_ERROR_CODES = new Set([
|
|
2
|
-
"AGENT_START_FAILED",
|
|
3
|
-
"
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
"
|
|
7
|
-
"
|
|
8
|
-
"
|
|
9
|
-
"
|
|
2
|
+
"AGENT_START_FAILED",
|
|
3
|
+
"AGENT_EXIT_FAILED",
|
|
4
|
+
"AGENT_REPORTED_FAILURE",
|
|
5
|
+
"INVALID_AGENT_RESULT",
|
|
6
|
+
"AGENT_CLI_INCOMPATIBLE",
|
|
7
|
+
"AGENT_AUTH_REQUIRED",
|
|
8
|
+
"AGENT_QUOTA_EXCEEDED",
|
|
9
|
+
"AGENT_SANDBOX_UNAVAILABLE",
|
|
10
|
+
"AGENT_MODEL_UNAVAILABLE",
|
|
11
|
+
"AGENT_OUTCOME_UNCERTAIN",
|
|
12
|
+
"RUN_TIMEOUT",
|
|
13
|
+
"RUN_CANCELLED",
|
|
14
|
+
"OUTPUT_LIMIT_EXCEEDED",
|
|
15
|
+
"UNKNOWN_HTTP_OUTCOME",
|
|
16
|
+
"HTTP_AGENT_FAILED",
|
|
17
|
+
"INVALID_ADAPTER_CONFIG",
|
|
18
|
+
"INVALID_ADAPTER_ENV",
|
|
19
|
+
"INSPECTION_BLOCKED",
|
|
20
|
+
"INSPECTION_UNAVAILABLE",
|
|
21
|
+
"INSPECTION_UNSUPPORTED_FILE",
|
|
22
|
+
"AGENT_REPLY_TIMEOUT",
|
|
23
|
+
"INVALID_ASK",
|
|
24
|
+
"TOOL_NOT_ALLOWED",
|
|
25
|
+
"INVALID_PEER_REPLY",
|
|
26
|
+
"NETWORK_BUSY",
|
|
27
|
+
"NETWORK_ERROR",
|
|
28
|
+
"TIMEOUT",
|
|
29
|
+
"CANCELED",
|
|
30
|
+
"INVALID_NETWORK_CREDENTIAL",
|
|
31
|
+
"AGENT_IDENTITY_MISMATCH",
|
|
32
|
+
"MODEL_ERROR",
|
|
33
|
+
"MAX_STEPS",
|
|
34
|
+
"INTERNAL_ERROR",
|
|
10
35
|
]);
|
|
11
36
|
export function runtimeErrorCode(error) {
|
|
12
37
|
return RUNTIME_ERROR_CODES.has(error?.code) ? error.code : "INTERNAL_ERROR";
|
package/bin/sdk/index.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { TotemError } from "./errors.js";
|
|
2
|
-
import { createLocalInspector, InspectionError } from "./inspector.js";
|
|
3
2
|
import { idempotencyKey, sleep, Transport } from "./transport.js";
|
|
4
3
|
import { assertInputFields, assistantCheckOptions, signalOnlyOptions } from "./request-options.js";
|
|
5
4
|
export * from "./types.js";
|
|
@@ -113,11 +112,6 @@ export class Totem {
|
|
|
113
112
|
},
|
|
114
113
|
idempotencyKey: options.idempotencyKey ?? `sdk:reply:${id}`,
|
|
115
114
|
})).data,
|
|
116
|
-
replyContent: async (id, content, options = {}) => (await this.#transport.request("POST", `/messages/${encodeURIComponent(id)}/reply`, {
|
|
117
|
-
...options,
|
|
118
|
-
body: { content },
|
|
119
|
-
idempotencyKey: options.idempotencyKey ?? idempotencyKey(),
|
|
120
|
-
})).data,
|
|
121
115
|
ack: async (id, options) => (await this.#transport.request("POST", `/messages/${encodeURIComponent(id)}/ack`, {
|
|
122
116
|
...options,
|
|
123
117
|
body: {},
|
|
@@ -323,8 +317,6 @@ export class ConnectedAgent {
|
|
|
323
317
|
if (!Number.isFinite(interval) || interval < 10)
|
|
324
318
|
throw new TotemError("pollIntervalMs must be at least 10.", { code: "INVALID_CONFIG" });
|
|
325
319
|
const controller = new AbortController();
|
|
326
|
-
const inspector = options.inspector ?? createLocalInspector();
|
|
327
|
-
const blocked = new Set();
|
|
328
320
|
const report = (cause) => {
|
|
329
321
|
this.#lastError = cause instanceof Error ? cause : new Error(String(cause));
|
|
330
322
|
try {
|
|
@@ -340,42 +332,22 @@ export class ConnectedAgent {
|
|
|
340
332
|
for await (const message of this.client.messages.iterate({ direction: "inbound", unread: true, limit: 100 }, { signal: controller.signal })) {
|
|
341
333
|
if (controller.signal.aborted)
|
|
342
334
|
break;
|
|
343
|
-
|
|
344
|
-
|
|
335
|
+
const incoming = {
|
|
336
|
+
...message,
|
|
337
|
+
text: message.content.type === "text"
|
|
338
|
+
? message.content.text
|
|
339
|
+
: JSON.stringify(message.content.data),
|
|
340
|
+
reply: (text, writeOptions = {}) => this.client.messages.reply(message.id, text, {
|
|
341
|
+
...writeOptions,
|
|
342
|
+
idempotencyKey: writeOptions.idempotencyKey ?? `reply:${message.id}:0`,
|
|
343
|
+
}),
|
|
344
|
+
ack: () => this.client.messages.ack(message.id),
|
|
345
|
+
};
|
|
345
346
|
try {
|
|
346
|
-
if (message.content.type === "encrypted")
|
|
347
|
-
throw new TotemError("Use E2eeTotem and pinned endpoint keys to process encrypted messages.", { code: "E2EE_CLIENT_REQUIRED" });
|
|
348
|
-
await inspector.assertAllowed("input", { content: message.content, metadata: message.metadata ?? {} }, { signal: controller.signal });
|
|
349
|
-
const files = message.attachments ?? [];
|
|
350
|
-
const ids = new Set(files.map((file) => file.id));
|
|
351
|
-
if (message.content.type === "text" &&
|
|
352
|
-
message.content.attachments?.some((id) => !ids.has(id)))
|
|
353
|
-
throw new InspectionError("INSPECTION_UNSUPPORTED_FILE");
|
|
354
|
-
for (const file of files) {
|
|
355
|
-
const downloaded = await this.client.attachments.download(file.id, {
|
|
356
|
-
signal: controller.signal,
|
|
357
|
-
});
|
|
358
|
-
await inspector.inspectFile(file.name, file.mediaType, downloaded.bytes, {
|
|
359
|
-
signal: controller.signal,
|
|
360
|
-
});
|
|
361
|
-
}
|
|
362
|
-
const incoming = {
|
|
363
|
-
...message,
|
|
364
|
-
text: message.content.type === "text"
|
|
365
|
-
? message.content.text
|
|
366
|
-
: JSON.stringify(message.content.data),
|
|
367
|
-
reply: (text, writeOptions = {}) => inspector.runAction({ id: message.id, text }, (input) => this.client.messages.reply(input.id, input.text, {
|
|
368
|
-
...writeOptions,
|
|
369
|
-
idempotencyKey: writeOptions.idempotencyKey ?? `reply:${message.id}:0`,
|
|
370
|
-
}), { signal: controller.signal }),
|
|
371
|
-
ack: () => this.client.messages.ack(message.id),
|
|
372
|
-
};
|
|
373
347
|
await handler(incoming);
|
|
374
348
|
await incoming.ack();
|
|
375
349
|
}
|
|
376
350
|
catch (error) {
|
|
377
|
-
if (error instanceof InspectionError && error.code !== "INSPECTION_UNAVAILABLE")
|
|
378
|
-
blocked.add(message.id);
|
|
379
351
|
report(error);
|
|
380
352
|
}
|
|
381
353
|
}
|
|
@@ -413,4 +385,3 @@ export class ConnectedAgent {
|
|
|
413
385
|
/** Compatibility name for callers who prefer `new TotemAgent({ address, apiKey })`. */
|
|
414
386
|
export class TotemAgent extends Totem {
|
|
415
387
|
}
|
|
416
|
-
export { createLocalInspector, InspectionError };
|