totemlayer 0.1.17 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/bin/agent-runner.mjs +8 -2
- package/bin/automatic-runtime.mjs +22 -3
- package/bin/connected-agent-service.mjs +1 -0
- package/bin/executor-diagnostic.mjs +204 -0
- package/bin/model-adapter.mjs +8 -0
- package/bin/provider-diagnostics.mjs +46 -0
- package/bin/totem.mjs +6 -0
- package/bin/worker-core/adapters.mjs +17 -6
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -18,6 +18,8 @@ Totem detects an installed, signed-in Codex or Claude Code client, an installed
|
|
|
18
18
|
|
|
19
19
|
The listener runs independently of the terminal, restarts after failure, and resumes after login on macOS, Linux user-service hosts, and Windows. On headless hosts without a service manager, it starts a background supervisor. A destroyed host still needs persistent private storage or its environment's secret provisioning; an installed listener cannot keep a destroyed host alive.
|
|
20
20
|
|
|
21
|
+
If an executor fails, Totem keeps a redacted error privately on that computer. Retrieve it with `npx --yes totemlayer@latest diagnose --json`; no separate diagnostic download is needed. If no error has been saved yet, Windows runs the check through the installed task account. `--fresh` requests a new check, and `--profile YOUR_AGENT_UUID` selects a connection when more than one is installed. The diagnostic does not clear recovery journals or restart the existing listener.
|
|
22
|
+
|
|
21
23
|
```sh
|
|
22
24
|
npx totemlayer@latest status --profile YOUR_AGENT_UUID
|
|
23
25
|
npx totemlayer@latest stop --profile YOUR_AGENT_UUID
|
package/bin/agent-runner.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import { constants } from "node:fs";
|
|
4
|
-
import { lstat, mkdir, open, opendir, rename, unlink } from "node:fs/promises";
|
|
4
|
+
import { lstat, mkdir, open, opendir, rename, unlink, readFile } from "node:fs/promises";
|
|
5
5
|
import { dirname, isAbsolute, join, parse, resolve, sep } from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import { parseArgs } from "node:util";
|
|
@@ -11,6 +11,9 @@ import { localClientEnvironment } from "./platform-client.mjs";
|
|
|
11
11
|
import { Totem } from "./sdk/index.js";
|
|
12
12
|
import { RUNTIME_ERROR_CODES, runtimeErrorCode } from "./runtime-diagnostics.mjs";
|
|
13
13
|
|
|
14
|
+
const PACKAGE_VERSION = JSON.parse(
|
|
15
|
+
await readFile(new URL("../package.json", import.meta.url), "utf8"),
|
|
16
|
+
).version;
|
|
14
17
|
const CLI_PATH = fileURLToPath(new URL("./totem.mjs", import.meta.url));
|
|
15
18
|
const UUID = /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i;
|
|
16
19
|
const PHASES = new Set([
|
|
@@ -230,6 +233,7 @@ export async function readAgentRuntimeStatus(profile, { isAlive = alive, now = D
|
|
|
230
233
|
? record.phase
|
|
231
234
|
: "interrupted",
|
|
232
235
|
runId: record.runId,
|
|
236
|
+
...(typeof record.packageVersion === "string" ? { packageVersion: record.packageVersion } : {}),
|
|
233
237
|
updatedAt: record.updatedAt,
|
|
234
238
|
...(RUNTIME_ERROR_CODES.has(record.blockerCode) ? { blockerCode: record.blockerCode } : {}),
|
|
235
239
|
...(UUID.test(record.missionId ?? "") ? { missionId: record.missionId } : {}),
|
|
@@ -287,6 +291,7 @@ export async function runAgentSession(profile, options = {}) {
|
|
|
287
291
|
if (options.signal?.aborted) controller.abort();
|
|
288
292
|
let status = {
|
|
289
293
|
version: 1,
|
|
294
|
+
packageVersion: PACKAGE_VERSION,
|
|
290
295
|
runId,
|
|
291
296
|
pid: process.pid,
|
|
292
297
|
agentId: profile.agentId,
|
|
@@ -394,6 +399,7 @@ export async function runAgentSession(profile, options = {}) {
|
|
|
394
399
|
.filter((key) => profile[key] !== undefined)
|
|
395
400
|
.map((key) => [key, profile[key]]),
|
|
396
401
|
),
|
|
402
|
+
diagnosticDirectory: profile.stateDirectory,
|
|
397
403
|
ollamaUrl: profile.ollamaUrl,
|
|
398
404
|
nimUrl: profile.nimUrl,
|
|
399
405
|
qwenUrl: profile.qwenUrl,
|
|
@@ -442,7 +448,7 @@ function runnerOptions(args) {
|
|
|
442
448
|
return parsed.values;
|
|
443
449
|
}
|
|
444
450
|
|
|
445
|
-
async function requestStop(profile, runId) {
|
|
451
|
+
export async function requestStop(profile, runId) {
|
|
446
452
|
if (!UUID.test(runId)) throw new Error("Invalid agent run identifier.");
|
|
447
453
|
const directory = await runtimeDirectory(profile, true);
|
|
448
454
|
const path = join(directory, `${runId}.stop`);
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { opendir, rm } from "node:fs/promises";
|
|
1
|
+
import { opendir, rm, readFile } from "node:fs/promises";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
4
4
|
import { fileURLToPath } from "node:url";
|
|
5
5
|
import { readAgentProfile } from "./agent-profile.mjs";
|
|
6
|
-
import { agentStartMain, readAgentRuntimeStatus } from "./agent-runner.mjs";
|
|
6
|
+
import { agentStartMain, readAgentRuntimeStatus, requestStop } from "./agent-runner.mjs";
|
|
7
7
|
import {
|
|
8
8
|
privateDirectory,
|
|
9
9
|
privateRead,
|
|
@@ -12,6 +12,10 @@ import {
|
|
|
12
12
|
} from "./runtime-installation.mjs";
|
|
13
13
|
import { resumeDeviceLogin } from "./device-login.mjs";
|
|
14
14
|
|
|
15
|
+
const PACKAGE_VERSION = JSON.parse(
|
|
16
|
+
await readFile(new URL("../package.json", import.meta.url), "utf8"),
|
|
17
|
+
).version;
|
|
18
|
+
|
|
15
19
|
export async function setAutomaticEnabled(profile, enabled) {
|
|
16
20
|
await privateWrite(join(profile.stateDirectory, "automatic.json"), { enabled });
|
|
17
21
|
}
|
|
@@ -42,7 +46,22 @@ export async function superviseOnce(env = process.env, options = {}) {
|
|
|
42
46
|
const profile = await readAgentProfile(entry.name, { environment: env });
|
|
43
47
|
if (!(await automaticEnabled(profile))) continue;
|
|
44
48
|
const state = await (options.readStatus ?? readAgentRuntimeStatus)(profile);
|
|
45
|
-
if (state.running)
|
|
49
|
+
if (state.running) {
|
|
50
|
+
if (state.packageVersion === (options.packageVersion ?? PACKAGE_VERSION)) continue;
|
|
51
|
+
// Finish active work before upgrading. Never overlap two workers or clear journals.
|
|
52
|
+
if (!["connected", "waiting", "blocked", "reconnecting"].includes(state.phase)) continue;
|
|
53
|
+
await (options.stop ?? requestStop)(profile, state.runId);
|
|
54
|
+
const deadline = Date.now() + (options.stopTimeoutMs ?? 10000);
|
|
55
|
+
let stopped = false;
|
|
56
|
+
while (Date.now() < deadline) {
|
|
57
|
+
if (!(await (options.readStatus ?? readAgentRuntimeStatus)(profile)).running) {
|
|
58
|
+
stopped = true;
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
await sleep(100);
|
|
62
|
+
}
|
|
63
|
+
if (!stopped || !(await automaticEnabled(profile))) continue;
|
|
64
|
+
}
|
|
46
65
|
// The run-local journal decides whether any interrupted work can safely resume.
|
|
47
66
|
await (options.start ?? agentStartMain)(["--profile", entry.name, "--background", "--json"], {
|
|
48
67
|
env,
|
|
@@ -32,6 +32,7 @@ export async function runConnectedAgentService(settings, dependencies = {}) {
|
|
|
32
32
|
signal,
|
|
33
33
|
stateDir: settings.stateDirectory,
|
|
34
34
|
allowAgents: [],
|
|
35
|
+
adapterSettings: { ...settings.adapterSettings, diagnosticDirectory: settings.diagnosticDirectory },
|
|
35
36
|
modelTimeoutMs: 900000,
|
|
36
37
|
replyTimeoutMs: 90000,
|
|
37
38
|
pollMs: settings.pollMs ?? 5000,
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
// Standalone support probe: use the installed Totem adapter without reading
|
|
2
|
+
// its connection key, changing its listener, or touching its work journal.
|
|
3
|
+
import childProcess from "node:child_process";
|
|
4
|
+
import { syncBuiltinESMExports } from "node:module";
|
|
5
|
+
import { promisify } from "node:util";
|
|
6
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
7
|
+
import { readFile, readdir, copyFile, writeFile, rename, rm } from "node:fs/promises";
|
|
8
|
+
import { homedir } from "node:os";
|
|
9
|
+
import { join, resolve, dirname, basename } from "node:path";
|
|
10
|
+
import { pathToFileURL, fileURLToPath } from "node:url";
|
|
11
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
12
|
+
|
|
13
|
+
async function installedModules(root) {
|
|
14
|
+
const pointer = JSON.parse(await readFile(join(root, "runtime", "current.json"), "utf8"));
|
|
15
|
+
if (!/^\d+\.\d+\.\d+$/.test(pointer.version)) throw Error("Invalid installed version");
|
|
16
|
+
const bin = join(root, "runtime-packages", pointer.version, "node_modules", "totemlayer", "bin");
|
|
17
|
+
const helpers = await import(pathToFileURL(join(bin, "runtime-installation.mjs")));
|
|
18
|
+
return { pointer, bin, ...helpers };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async function selectedProfile(root, profileName) {
|
|
22
|
+
const { privateRead } = await installedModules(root);
|
|
23
|
+
const names = profileName ? [profileName] : (await readdir(join(root, "connections"), { withFileTypes: true }))
|
|
24
|
+
.filter(entry => entry.isDirectory()).map(entry => entry.name);
|
|
25
|
+
const profiles = [];
|
|
26
|
+
for (const name of names) {
|
|
27
|
+
if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(name)) throw Error("Invalid profile name");
|
|
28
|
+
profiles.push(await privateRead(join(root, "connections", name, "profile.json")));
|
|
29
|
+
}
|
|
30
|
+
if (profiles.length !== 1) throw Error("Choose exactly one connection with --profile NAME");
|
|
31
|
+
return profiles[0];
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Task Scheduler starts this one-shot probe with the installed service's account,
|
|
35
|
+
// rather than inheriting the working interactive terminal's provider environment.
|
|
36
|
+
export async function diagnoseBackground({
|
|
37
|
+
root = process.env.TOTEM_CONFIG_DIR ?? join(homedir(), ".totem"), profileName,
|
|
38
|
+
} = {}) {
|
|
39
|
+
if (process.platform !== "win32") throw Error("This probe requires Windows Task Scheduler");
|
|
40
|
+
const { privateDirectory, privateRead } = await installedModules(root);
|
|
41
|
+
const id = randomUUID();
|
|
42
|
+
const directory = join(root, "runtime", "executor-probes", id);
|
|
43
|
+
await privateDirectory(directory);
|
|
44
|
+
const entry = join(directory, "diagnose-executor.mjs");
|
|
45
|
+
const reportPath = join(directory, "report.json");
|
|
46
|
+
const receipt = join(directory, "registered.json");
|
|
47
|
+
await copyFile(fileURLToPath(import.meta.url), entry);
|
|
48
|
+
const task = `TotemExecutorProbe-${id}`;
|
|
49
|
+
const suffix = resolve(root) === resolve(join(homedir(), ".totem")) ? "" :
|
|
50
|
+
`.${createHash("sha256").update(root).digest("hex").slice(0, 12)}`;
|
|
51
|
+
const service = `TotemRuntime${suffix}`;
|
|
52
|
+
const literal = value => `'${String(value).replaceAll("'", "''")}'`;
|
|
53
|
+
const argumentsText = `"${entry}" --report-file "${reportPath}"` +
|
|
54
|
+
(profileName ? ` --profile "${profileName}"` : "");
|
|
55
|
+
if (profileName && !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(profileName)) throw Error("Invalid profile name");
|
|
56
|
+
const script = `$ErrorActionPreference='Stop'
|
|
57
|
+
$source=Get-ScheduledTask -TaskName ${literal(service)}
|
|
58
|
+
$owner=$source.Principal.UserId
|
|
59
|
+
if ($owner -like 'S-1-*') { $sid=[Security.Principal.SecurityIdentifier]::new($owner) }
|
|
60
|
+
else { $sid=[Security.Principal.NTAccount]::new($owner).Translate([Security.Principal.SecurityIdentifier]) }
|
|
61
|
+
if ($sid.Value -ne [Security.Principal.WindowsIdentity]::GetCurrent().User.Value) { throw 'The service belongs to another user' }
|
|
62
|
+
$action=New-ScheduledTaskAction -Execute ${literal(process.execPath)} -Argument ${literal(argumentsText)}
|
|
63
|
+
$settings=New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 2)
|
|
64
|
+
Register-ScheduledTask -TaskName ${literal(task)} -Action $action -Principal $source.Principal -Settings $settings | Out-Null
|
|
65
|
+
[IO.File]::WriteAllText(${literal(receipt)},'{}')
|
|
66
|
+
Start-ScheduledTask -TaskName ${literal(task)}`;
|
|
67
|
+
const run = promisify(childProcess.execFile);
|
|
68
|
+
const powershell = source => run("powershell.exe", ["-NoProfile", "-NonInteractive", "-EncodedCommand",
|
|
69
|
+
Buffer.from(source, "utf16le").toString("base64")], { windowsHide: true, timeout: 30000, maxBuffer: 65536 });
|
|
70
|
+
try {
|
|
71
|
+
await powershell(script);
|
|
72
|
+
const deadline = Date.now() + 90000;
|
|
73
|
+
while (Date.now() < deadline) {
|
|
74
|
+
try { return { context: "windows-scheduled-task", ...await privateRead(reportPath) }; }
|
|
75
|
+
catch (error) { if (error.code !== "ENOENT") throw error; }
|
|
76
|
+
await sleep(250);
|
|
77
|
+
}
|
|
78
|
+
return { ok: false, code: "BACKGROUND_PROBE_TIMEOUT" };
|
|
79
|
+
} finally {
|
|
80
|
+
try {
|
|
81
|
+
await readFile(receipt);
|
|
82
|
+
await powershell(`$ErrorActionPreference='Stop';Stop-ScheduledTask -TaskName ${literal(task)} -ErrorAction SilentlyContinue;Unregister-ScheduledTask -TaskName ${literal(task)} -Confirm:$false -ErrorAction SilentlyContinue`);
|
|
83
|
+
} catch { /* Never modify the installed service or any task without our receipt. */ }
|
|
84
|
+
await rm(directory, { recursive: true, force: true });
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export async function diagnoseExecutor({
|
|
89
|
+
root = process.env.TOTEM_CONFIG_DIR ?? join(homedir(), ".totem"),
|
|
90
|
+
profileName,
|
|
91
|
+
environment = process.env,
|
|
92
|
+
} = {}) {
|
|
93
|
+
const { pointer, bin, privateRead } = await installedModules(root);
|
|
94
|
+
const profile = await selectedProfile(root, profileName);
|
|
95
|
+
const provider = profile.client ?? profile.provider;
|
|
96
|
+
const model = profile.client && profile.provider === "qwen-local" ? "default" : profile.model;
|
|
97
|
+
let saved = {}, secrets = {};
|
|
98
|
+
for (const [file, target] of [["environment.json", saved], ["provider-environment.json", secrets]]) {
|
|
99
|
+
try { Object.assign(target, await privateRead(join(root, "runtime", file))); }
|
|
100
|
+
catch (error) { if (error.code !== "ENOENT") throw error; }
|
|
101
|
+
}
|
|
102
|
+
const providerEnvironment = { ...environment, ...saved, ...secrets };
|
|
103
|
+
const secretValues = Object.entries(providerEnvironment)
|
|
104
|
+
.filter(([key, value]) => /secret|token|password|api.?key|db.?pass/i.test(key) && typeof value === "string" && value.length >= 4)
|
|
105
|
+
.map(([, value]) => value);
|
|
106
|
+
const redact = value => {
|
|
107
|
+
let text = String(value);
|
|
108
|
+
for (const secret of secretValues) text = text.split(secret).join("[REDACTED]");
|
|
109
|
+
return text.replace(/\b(?:tn_(?:live|test)_[A-Za-z0-9_-]+|sk-[A-Za-z0-9_-]+|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)\b/g, "[REDACTED]")
|
|
110
|
+
.replace(/(Bearer\s+)[^\s"']+/gi, "$1[REDACTED]")
|
|
111
|
+
.replace(/(https?:\/\/)[^\s\/@]+@/gi, "$1[REDACTED]@")
|
|
112
|
+
.slice(-6000);
|
|
113
|
+
};
|
|
114
|
+
const originalSpawn = childProcess.spawn;
|
|
115
|
+
const launches = [];
|
|
116
|
+
childProcess.spawn = function (command, args, options) {
|
|
117
|
+
const record = { command: redact(command), stderr: "", stdout: "" };
|
|
118
|
+
launches.push(record);
|
|
119
|
+
const child = originalSpawn.call(this, command, args, options);
|
|
120
|
+
for (const name of ["stdout", "stderr"]) child[name]?.on("data", data => {
|
|
121
|
+
record[name] = (record[name] + data.toString("utf8")).slice(-32000);
|
|
122
|
+
});
|
|
123
|
+
child.once("close", code => { record.exitCode = code; });
|
|
124
|
+
return child;
|
|
125
|
+
};
|
|
126
|
+
syncBuiltinESMExports();
|
|
127
|
+
let ok = false, code;
|
|
128
|
+
try {
|
|
129
|
+
const { modelReply } = await import(pathToFileURL(join(bin, "network-runtime.mjs")));
|
|
130
|
+
const response = await modelReply({ provider, model, origin: profile.ollamaUrl ?? "http://127.0.0.1:11434",
|
|
131
|
+
qwenUrl: profile.qwenUrl, nimUrl: profile.nimUrl,
|
|
132
|
+
adapterSettings: { ...profile, environment: providerEnvironment }, timeoutMs: 60000,
|
|
133
|
+
signal: new AbortController().signal, tools: [],
|
|
134
|
+
messages: [{ role: "user", content: "Reply hi only. Do not use tools or read files." }] });
|
|
135
|
+
const answer = response.content;
|
|
136
|
+
ok = typeof answer === "string" && answer.trim().length > 0;
|
|
137
|
+
code = ok ? undefined : "EMPTY_REPLY";
|
|
138
|
+
} catch (error) { code = error.code ?? "PROBE_FAILED"; }
|
|
139
|
+
finally { childProcess.spawn = originalSpawn; syncBuiltinESMExports(); }
|
|
140
|
+
const failures = [];
|
|
141
|
+
if (!ok) for (const launch of launches) {
|
|
142
|
+
const errors = [];
|
|
143
|
+
for (const line of launch.stdout.split(/\r?\n/)) {
|
|
144
|
+
try { const event = JSON.parse(line); if (["error", "turn.failed"].includes(event.type))
|
|
145
|
+
errors.push(redact(event.error?.message ?? event.message ?? event.error ?? "Provider reported an error")); }
|
|
146
|
+
catch { /* Other stdout is excluded from the report. */ }
|
|
147
|
+
}
|
|
148
|
+
failures.push({ command: launch.command, exitCode: launch.exitCode,
|
|
149
|
+
stderr: redact(launch.stderr), errors });
|
|
150
|
+
}
|
|
151
|
+
return { version: pointer.version, profile: profile.profileName, provider, model,
|
|
152
|
+
ok, ...(code ? { code } : {}), launches: launches.map(({ command, exitCode }) => ({ command, exitCode })),
|
|
153
|
+
customProviderHome: { terminal: Boolean(environment.CODEX_HOME), saved: Boolean(saved.CODEX_HOME) },
|
|
154
|
+
...(failures.length ? { failures } : {}) };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export async function executorDiagnosticMain(inputArgs, options = {}) {
|
|
158
|
+
const output = options.stdout ?? process.stdout;
|
|
159
|
+
try {
|
|
160
|
+
const args = [...inputArgs];
|
|
161
|
+
let background = false, foreground = false, fresh = false, profileName, reportFile;
|
|
162
|
+
while (args.length) {
|
|
163
|
+
const flag = args.shift();
|
|
164
|
+
if (flag === "--background") background = true;
|
|
165
|
+
else if (flag === "--foreground") foreground = true;
|
|
166
|
+
else if (flag === "--fresh") fresh = true;
|
|
167
|
+
else if (flag === "--json") { /* The report is always safe structured JSON. */ }
|
|
168
|
+
else if (["--help", "-h"].includes(flag)) {
|
|
169
|
+
output.write("totem diagnose [--profile NAME] [--fresh] [--json]\nShow the captured background executor failure. If none is saved, check the executor through its Windows task account or local runtime.\n");
|
|
170
|
+
return 0;
|
|
171
|
+
}
|
|
172
|
+
else if (flag === "--profile" && args.length) profileName = args.shift();
|
|
173
|
+
else if (flag === "--report-file" && args.length) reportFile = args.shift();
|
|
174
|
+
else throw Error("Use --background or --profile NAME");
|
|
175
|
+
}
|
|
176
|
+
if (reportFile && background) throw Error("Invalid diagnostic mode");
|
|
177
|
+
if (foreground && background) throw Error("Choose one diagnostic context");
|
|
178
|
+
// Child reports may only be created in this probe's own temporary directory.
|
|
179
|
+
if (reportFile && (resolve(dirname(reportFile)) !== dirname(fileURLToPath(import.meta.url)) || basename(reportFile) !== "report.json"))
|
|
180
|
+
throw Error("Invalid report destination");
|
|
181
|
+
const probeRoot = reportFile ? resolve(dirname(reportFile), "../../..") : undefined;
|
|
182
|
+
const root = probeRoot ?? options.env?.TOTEM_CONFIG_DIR ?? process.env.TOTEM_CONFIG_DIR ?? join(homedir(), ".totem");
|
|
183
|
+
let report;
|
|
184
|
+
if (!fresh && !reportFile) {
|
|
185
|
+
const { readProviderDiagnostic } = await import(new URL("./provider-diagnostics.mjs", import.meta.url));
|
|
186
|
+
const profile = await selectedProfile(root, profileName);
|
|
187
|
+
const failure = await readProviderDiagnostic(profile);
|
|
188
|
+
if (failure) report = { context: "captured-background-failure", profile: profile.profileName, ok: false, ...failure };
|
|
189
|
+
}
|
|
190
|
+
if (!report) report = await (
|
|
191
|
+
background || (!reportFile && !foreground && (options.platform ?? process.platform) === "win32")
|
|
192
|
+
? (options.backgroundProbe ?? diagnoseBackground) : (options.foregroundProbe ?? diagnoseExecutor)
|
|
193
|
+
)({ root, profileName });
|
|
194
|
+
if (reportFile) {
|
|
195
|
+
await writeFile(reportFile + ".tmp", JSON.stringify(report), { mode: 0o600, flag: "wx" });
|
|
196
|
+
await rename(reportFile + ".tmp", reportFile);
|
|
197
|
+
} else output.write(JSON.stringify(report, null, 2) + "\n");
|
|
198
|
+
return report.ok ? 0 : 1;
|
|
199
|
+
} catch { output.write(JSON.stringify({ ok: false, code: "DIAGNOSTIC_SETUP_FAILED" }) + "\n"); return 1; }
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
|
203
|
+
process.exitCode = await executorDiagnosticMain(process.argv.slice(2));
|
|
204
|
+
}
|
package/bin/model-adapter.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import { isAbsolute, join } from "node:path";
|
|
|
5
5
|
import { createCommandAdapter, createHttpAdapter } from "./worker-core/adapters.mjs";
|
|
6
6
|
import { installedClientInvocation, localClientEnvironment } from "./platform-client.mjs";
|
|
7
7
|
import { WorkerError } from "./worker-core/errors.mjs";
|
|
8
|
+
import { saveProviderDiagnostic } from "./provider-diagnostics.mjs";
|
|
8
9
|
|
|
9
10
|
export function providerFailureCode(text) {
|
|
10
11
|
if (
|
|
@@ -83,6 +84,7 @@ export async function externalModelReply({
|
|
|
83
84
|
agentUrl,
|
|
84
85
|
agentTokenEnv,
|
|
85
86
|
environment = process.env,
|
|
87
|
+
diagnosticDirectory,
|
|
86
88
|
}) {
|
|
87
89
|
validateModelAdapter(provider, { command, commandArgs, agentUrl, agentTokenEnv });
|
|
88
90
|
// Provider tools must not operate on a Mission checkout. Totem applies and
|
|
@@ -185,12 +187,18 @@ export async function externalModelReply({
|
|
|
185
187
|
env,
|
|
186
188
|
timeoutMs,
|
|
187
189
|
parseResult,
|
|
190
|
+
captureFailure: Boolean(diagnosticDirectory),
|
|
188
191
|
...(["codex", "claude"].includes(provider) ? { classifyFailure: providerFailureCode } : {}),
|
|
189
192
|
});
|
|
190
193
|
}
|
|
191
194
|
const result = await adapter.run({ prompt, signal, runId: randomUUID() });
|
|
192
195
|
if (result.status !== "completed") throw new Error("The agent did not complete this turn.");
|
|
193
196
|
return result.output;
|
|
197
|
+
} catch (error) {
|
|
198
|
+
// Keep the real background error locally. Remote heartbeats and journal
|
|
199
|
+
// blockers retain fixed codes; provider logs never become model input.
|
|
200
|
+
await saveProviderDiagnostic(diagnosticDirectory, error, { environment, provider, model }).catch(() => {});
|
|
201
|
+
throw error;
|
|
194
202
|
} finally {
|
|
195
203
|
await rm(directory, { recursive: true, force: true });
|
|
196
204
|
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { join, isAbsolute } from "node:path";
|
|
2
|
+
import { privateRead, privateWrite } from "./runtime-installation.mjs";
|
|
3
|
+
import { runtimeErrorCode } from "./runtime-diagnostics.mjs";
|
|
4
|
+
|
|
5
|
+
export function providerDiagnostic(error, { environment = process.env, provider, model } = {}) {
|
|
6
|
+
// A stream truncated at its output bound could contain only part of a secret.
|
|
7
|
+
const captured = error.code === "OUTPUT_LIMIT_EXCEEDED" ? {} : error.privateProviderDiagnostic ?? {};
|
|
8
|
+
const secrets = Object.entries(environment)
|
|
9
|
+
.filter(([name, value]) => /secret|token|password|api.?key|db.?pass/i.test(name) && typeof value === "string" && value.length >= 4)
|
|
10
|
+
.map(([, value]) => value);
|
|
11
|
+
const redact = value => {
|
|
12
|
+
let text = String(value ?? "");
|
|
13
|
+
for (const secret of secrets) text = text.split(secret).join("[REDACTED]");
|
|
14
|
+
return text.replace(/\b(?:tn_(?:live|test)_[A-Za-z0-9_-]+|sk-[A-Za-z0-9_-]+|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)\b/g, "[REDACTED]")
|
|
15
|
+
.replace(/(Bearer\s+)[^\s"']+/gi, "$1[REDACTED]")
|
|
16
|
+
.replace(/(https?:\/\/)[^\s\/@]+@/gi, "$1[REDACTED]@")
|
|
17
|
+
.slice(-6000);
|
|
18
|
+
};
|
|
19
|
+
const errors = [];
|
|
20
|
+
for (const line of String(captured.stdout ?? "").split(/\r?\n/)) {
|
|
21
|
+
try {
|
|
22
|
+
const event = JSON.parse(line);
|
|
23
|
+
if (["error", "turn.failed"].includes(event.type))
|
|
24
|
+
errors.push(redact(event.error?.message ?? event.message ?? event.error));
|
|
25
|
+
} catch { /* Successful model output and ordinary stdout never become diagnostics. */ }
|
|
26
|
+
}
|
|
27
|
+
return { version: 1, checkedAt: new Date().toISOString(), provider, model,
|
|
28
|
+
code: runtimeErrorCode(error), command: redact(captured.command),
|
|
29
|
+
...(Number.isInteger(captured.exitCode) ? { exitCode: captured.exitCode } : {}),
|
|
30
|
+
stderr: redact(captured.stderr), errors: errors.slice(-8) };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export async function saveProviderDiagnostic(directory, error, context) {
|
|
34
|
+
if (typeof directory !== "string" || !isAbsolute(directory)) return;
|
|
35
|
+
await privateWrite(join(directory, "last-provider-failure.json"), providerDiagnostic(error, context));
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export async function readProviderDiagnostic(profile) {
|
|
39
|
+
try {
|
|
40
|
+
const value = await privateRead(join(profile.stateDirectory, "last-provider-failure.json"));
|
|
41
|
+
const model = profile.client && profile.provider === "qwen-local" ? "default" : profile.model;
|
|
42
|
+
if (value.version !== 1 || value.provider !== (profile.client ?? profile.provider) || value.model !== model || !Number.isFinite(Date.parse(value.checkedAt)))
|
|
43
|
+
return undefined;
|
|
44
|
+
return value;
|
|
45
|
+
} catch (error) { if (error.code !== "ENOENT") throw error; }
|
|
46
|
+
}
|
package/bin/totem.mjs
CHANGED
|
@@ -245,6 +245,7 @@ Connect once and receive messages and approved Mission work
|
|
|
245
245
|
Select Codex, Claude, a local model, or an owner-configured CLI/HTTP agent.
|
|
246
246
|
totem setup
|
|
247
247
|
totem status
|
|
248
|
+
totem diagnose [--profile NAME] [--fresh] [--json]
|
|
248
249
|
totem start --background
|
|
249
250
|
totem stop
|
|
250
251
|
|
|
@@ -1748,6 +1749,11 @@ async function runRfq(options, baseUrl) {
|
|
|
1748
1749
|
}
|
|
1749
1750
|
|
|
1750
1751
|
async function main() {
|
|
1752
|
+
if (process.argv[2] === "diagnose") {
|
|
1753
|
+
const { executorDiagnosticMain } = await import("./executor-diagnostic.mjs");
|
|
1754
|
+
process.exitCode = await executorDiagnosticMain(process.argv.slice(3));
|
|
1755
|
+
return;
|
|
1756
|
+
}
|
|
1751
1757
|
if (!process.argv[2] || ["login", "install"].includes(process.argv[2])) {
|
|
1752
1758
|
const { installationMain } = await import("./installation.mjs");
|
|
1753
1759
|
process.exitCode = await installationMain(process.argv.slice(3), {});
|
|
@@ -52,7 +52,8 @@ export function createCommandAdapter(config) {
|
|
|
52
52
|
!(config.args ?? []).every((arg) => typeof arg === "string" && !arg.includes("\0")) ||
|
|
53
53
|
typeof config.cwd !== "string" ||
|
|
54
54
|
!isAbsolute(config.cwd) ||
|
|
55
|
-
(config.classifyFailure !== undefined && typeof config.classifyFailure !== "function")
|
|
55
|
+
(config.classifyFailure !== undefined && typeof config.classifyFailure !== "function") ||
|
|
56
|
+
(config.captureFailure !== undefined && typeof config.captureFailure !== "boolean")
|
|
56
57
|
) {
|
|
57
58
|
throw new WorkerError("INVALID_ADAPTER_CONFIG");
|
|
58
59
|
}
|
|
@@ -78,6 +79,14 @@ export function createCommandAdapter(config) {
|
|
|
78
79
|
let failure;
|
|
79
80
|
let output = "";
|
|
80
81
|
let diagnosticOutput = "";
|
|
82
|
+
const diagnosticFailure = (error, exitCode) => {
|
|
83
|
+
if (config.captureFailure)
|
|
84
|
+
Object.defineProperty(error, "privateProviderDiagnostic", {
|
|
85
|
+
value: { command: config.command, exitCode, stderr: diagnosticOutput, stdout: output },
|
|
86
|
+
enumerable: false,
|
|
87
|
+
});
|
|
88
|
+
return error;
|
|
89
|
+
};
|
|
81
90
|
let bytes = 0;
|
|
82
91
|
let killTimer;
|
|
83
92
|
// Also remove surviving descendants if the parent exits before escalation.
|
|
@@ -135,7 +144,7 @@ export function createCommandAdapter(config) {
|
|
|
135
144
|
child.stderr.on("data", (chunk) => {
|
|
136
145
|
bytes += chunk.length;
|
|
137
146
|
if (bytes > maxOutputBytes) stop("OUTPUT_LIMIT_EXCEEDED");
|
|
138
|
-
else if (config.classifyFailure) diagnosticOutput += chunk.toString("utf8");
|
|
147
|
+
else if (config.classifyFailure || config.captureFailure) diagnosticOutput += chunk.toString("utf8");
|
|
139
148
|
});
|
|
140
149
|
child.stdin.on("error", () => {
|
|
141
150
|
/* A failed child can close stdin early. */
|
|
@@ -149,21 +158,23 @@ export function createCommandAdapter(config) {
|
|
|
149
158
|
signal?.removeEventListener("abort", cancel);
|
|
150
159
|
if (failure) {
|
|
151
160
|
kill("SIGKILL");
|
|
152
|
-
return reject(failure);
|
|
161
|
+
return reject(diagnosticFailure(failure, code));
|
|
153
162
|
}
|
|
154
163
|
if (code !== 0) {
|
|
155
164
|
try {
|
|
156
165
|
return reject(
|
|
157
|
-
new WorkerError(
|
|
166
|
+
diagnosticFailure(new WorkerError(
|
|
167
|
+
config.classifyFailure?.(`${diagnosticOutput}\n${output}`) ?? "AGENT_EXIT_FAILED",
|
|
168
|
+
), code),
|
|
158
169
|
);
|
|
159
170
|
} catch {
|
|
160
|
-
return reject(new WorkerError("AGENT_EXIT_FAILED"));
|
|
171
|
+
return reject(diagnosticFailure(new WorkerError("AGENT_EXIT_FAILED"), code));
|
|
161
172
|
}
|
|
162
173
|
}
|
|
163
174
|
try {
|
|
164
175
|
resolve(validateResult(parseResult(output)));
|
|
165
176
|
} catch (error) {
|
|
166
|
-
reject(error instanceof WorkerError ? error : new WorkerError("INVALID_AGENT_RESULT"));
|
|
177
|
+
reject(diagnosticFailure(error instanceof WorkerError ? error : new WorkerError("INVALID_AGENT_RESULT"), code));
|
|
167
178
|
}
|
|
168
179
|
});
|
|
169
180
|
// Prompts cannot become argv, executable names, paths, or shell syntax.
|