trantor 0.18.19 → 0.18.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +8 -0
- package/bin/crew-runner.mjs +26 -2
- package/bin/crew.sh +12 -7
- package/bin/doctor.mjs +50 -0
- package/bin/duty.mjs +157 -47
- package/bin/turn-watchdog.mjs +38 -0
- package/hooks/sessionstart.mjs +14 -2
- package/hooks/statusline.mjs +47 -0
- package/hub.mjs +61 -2
- package/lib/balances.mjs +37 -6
- package/package.json +2 -2
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "trantor",
|
|
3
|
-
"version": "0.18.
|
|
3
|
+
"version": "0.18.21",
|
|
4
4
|
"description": "Trantor \u2014 the hub-world for AI agent crews: live message bus, presence, project Kanban/flow board + crew orchestration for independent AI coding agents (Claude, Codex, Gemini, Kimi, DeepSeek)",
|
|
5
5
|
"mcpServers": {
|
|
6
6
|
"relay": {
|
package/README.md
CHANGED
|
@@ -230,6 +230,14 @@ project's workspace, so when you run several sessions each driving its own crew,
|
|
|
230
230
|
can't nuke another's. `trantor down <agent>` drops a single seat; `trantor down --all --yes` tears down
|
|
231
231
|
every project's crew.
|
|
232
232
|
|
|
233
|
+
Since 0.18.20, **seat trouble wakes the foreman instead of hoping someone looks**: a failing or
|
|
234
|
+
dead seat direct-messages the project's orchestrator (broadcasts wake nobody), a detached
|
|
235
|
+
watchdog reports a turn running silent past 15 minutes — once, without killing it — and the
|
|
236
|
+
failure classifier tells a provider backend error ("retry or swap") from real quota exhaustion
|
|
237
|
+
("wait the window out"). The duty seat itself now runs under a launchd keepalive, so the fleet's
|
|
238
|
+
janitor relaunches after a crash or reboot instead of dying silently, and the hub routes
|
|
239
|
+
escalations back to their senders whenever the janitor goes dark.
|
|
240
|
+
|
|
233
241
|
**One-time setup:**
|
|
234
242
|
- Install cmux — `brew install --cask cmux` (or grab it from **[cmux.com](https://cmux.com)**).
|
|
235
243
|
- Trantor drives cmux over its control socket, which is off to outside processes by default. Enable it in
|
package/bin/crew-runner.mjs
CHANGED
|
@@ -8,11 +8,11 @@
|
|
|
8
8
|
// over plain HTTP (zero tokens, doubles as a heartbeat), and when a message addressed to this
|
|
9
9
|
// agent arrives it RESUMES the CLI session (native resume = full context kept) with that
|
|
10
10
|
// message as the prompt. The model just works and ends its turn; the runner does the rest.
|
|
11
|
-
import { execSync, spawnSync } from "node:child_process";
|
|
11
|
+
import { execSync, spawnSync, spawn } from "node:child_process";
|
|
12
12
|
import { readFileSync, writeFileSync, unlinkSync, existsSync, appendFileSync, mkdirSync } from "node:fs";
|
|
13
13
|
import { join, basename } from "node:path";
|
|
14
14
|
import { homedir } from "node:os";
|
|
15
|
-
import { resolveProject, resolveHub, withEnvFiles } from "../lib/project.mjs";
|
|
15
|
+
import { resolveProject, resolveHub, withEnvFiles, hostId } from "../lib/project.mjs";
|
|
16
16
|
import { loadOrCreate } from "../lib/identity.mjs";
|
|
17
17
|
import { signedHeaders } from "../lib/signed-fetch.mjs";
|
|
18
18
|
import { ensureEnrolled } from "../lib/enroll.mjs";
|
|
@@ -314,6 +314,11 @@ const AUTH_MARKER_RE = /unauthor|401|403|forbidden|invalid[ _-]?api[ _-]?key|aut
|
|
|
314
314
|
function classifyFailure(exit, errText) {
|
|
315
315
|
const t = (errText || "").toLowerCase();
|
|
316
316
|
if (exit === 127) return "missing-cli";
|
|
317
|
+
// #5684: a provider BACKEND failure is not quota — it wants retry/swap, not a window wait.
|
|
318
|
+
// The specimen (#5683): codex's "unexpected status 404 Not Found … /responses/compact" was
|
|
319
|
+
// labelled "exhausted" and the operator was advised to wait out a window that did not exist.
|
|
320
|
+
// 401/403/429 deliberately fall through to the auth/exhausted branches below.
|
|
321
|
+
if (/unexpected status (404|408|410|5\d\d)|internal server error|bad gateway|service unavailable|gateway time.?out|econnrefused|connection refused|socket hang ?up|network is unreachable/.test(t)) return "backend-error";
|
|
317
322
|
// "reached your … limit" / "usage limit" catch the subscription CLIs (Claude's "You've reached
|
|
318
323
|
// your Fable 5 limit"), which say nothing about quota or credits and would otherwise read as a crash.
|
|
319
324
|
if (/quota|insufficient|credit|balance|payment required|402|429|too many requests|rate.?limit|exceeded your|reached your [^.\n]*limit|usage limit|out of (credit|quota)/.test(t)) return "exhausted";
|
|
@@ -329,6 +334,7 @@ async function reportFailure(exit, trigger, undelivered = 0) {
|
|
|
329
334
|
await api("/register", { session: SESSION, project: PROJ, status, llm: AGENT, model: MODEL }).catch(() => {});
|
|
330
335
|
const hint = reason === "exhausted" ? " — needs `trantor swap`"
|
|
331
336
|
: reason === "auth" ? " — check credentials"
|
|
337
|
+
: reason === "backend-error" ? " — provider backend error (NOT quota): retry, or `trantor swap` to another provider"
|
|
332
338
|
: reason === "missing-cli" ? " — CLI not on PATH" : "";
|
|
333
339
|
// The count of messages this seat is HOLDING is the operator-actionable half of a failure: a
|
|
334
340
|
// crashed pulse costs nothing, a crashed turn sitting on three escalations is someone waiting.
|
|
@@ -343,6 +349,12 @@ async function reportFailure(exit, trigger, undelivered = 0) {
|
|
|
343
349
|
if (state !== announced) {
|
|
344
350
|
announced = state;
|
|
345
351
|
await api("/send", { from: SESSION, to: "all", text, project: PROJ }).catch(() => {});
|
|
352
|
+
// #5684: a broadcast does not wake anyone — the incident is the operator spotting dead seats
|
|
353
|
+
// before the foreman did, twice in one morning. The same state-change event now goes DIRECT
|
|
354
|
+
// to the project's orchestrator (direct = wake), gated identically so a standing outage says
|
|
355
|
+
// it once. A seat that IS the orchestrator's own runner has nobody above it to wake.
|
|
356
|
+
const orch = `${hostId()}:${PROJ}`;
|
|
357
|
+
if (orch !== SESSION) await api("/send", { from: SESSION, to: orch, text, project: PROJ }).catch(() => {});
|
|
346
358
|
} else {
|
|
347
359
|
log(`still ${state} (${consecFails} fails) — already announced, staying quiet`);
|
|
348
360
|
}
|
|
@@ -420,6 +432,17 @@ function runTurn(prompt, isFirst, trigger = "kickoff") {
|
|
|
420
432
|
// `tee /dev/stderr`; the rest now tee straight into ERRF. A real pipeline (not a process
|
|
421
433
|
// substitution) so bash waits for tee to flush before we read the file back.
|
|
422
434
|
const inner = cli.sid ? `${cmd} | tee /dev/stderr` : `${cmd} | tee -a ${ERRF}`;
|
|
435
|
+
// #5684: runTurn is spawnSync, so the runner cannot watch its own turn — a DETACHED watchdog
|
|
436
|
+
// does. Armed by a stamp file, disarmed when the turn ends (stamp removed below); a turn past
|
|
437
|
+
// the window with no ERRF growth earns ONE direct stall report to the foreman, never a kill.
|
|
438
|
+
const WD_MS = Number(process.env.TRANTOR_TURN_WATCHDOG_MS || 15 * 60 * 1000);
|
|
439
|
+
const STAMPF = join(homedir(), ".agent-bus", `turnstamp-${AGENT}-${PROJ}.json`);
|
|
440
|
+
try {
|
|
441
|
+
writeFileSync(STAMPF, JSON.stringify({ turn: TURN, startedAt: Date.now() }));
|
|
442
|
+
const wd = spawn(process.execPath, [join(import.meta.dirname, "turn-watchdog.mjs"), STAMPF, ERRF, String(WD_MS), SESSION, PROJ, HUB],
|
|
443
|
+
{ detached: true, stdio: "ignore" });
|
|
444
|
+
wd.unref();
|
|
445
|
+
} catch {}
|
|
423
446
|
const r = spawnSync("/bin/bash", ["-c", `set -o pipefail; { ${inner} ; } 2> >(tee -a ${ERRF} >&2)`], {
|
|
424
447
|
cwd: TURN_DIR, encoding: "utf8", stdio: cli.sid ? ["ignore", "pipe", "inherit"] : "inherit",
|
|
425
448
|
env: { ...process.env, RELAY_URL: HUB, RELAY_AGENT: AGENT, RELAY_PROJECT: PROJ,
|
|
@@ -436,6 +459,7 @@ function runTurn(prompt, isFirst, trigger = "kickoff") {
|
|
|
436
459
|
TRANTOR_NO_HANDOFF_SPAWN: "1", TRANTOR_NO_BATON_SPAWN: "1" },
|
|
437
460
|
maxBuffer: 16 * 1024 * 1024,
|
|
438
461
|
});
|
|
462
|
+
try { unlinkSync(STAMPF); } catch {} // turn over — disarm the watchdog
|
|
439
463
|
try { lastErrText = readFileSync(ERRF, "utf8").slice(-4000); } catch { lastErrText = ""; }
|
|
440
464
|
if (cli.sid && r.stdout) { const m = r.stdout.match(cli.sid); if (m) sid = m[1]; }
|
|
441
465
|
const realExit = r.status;
|
package/bin/crew.sh
CHANGED
|
@@ -244,26 +244,31 @@ _orch_cmd() { # $1=dir $2=sid
|
|
|
244
244
|
}
|
|
245
245
|
|
|
246
246
|
# A recorded thread that HANDED OFF has ended: resuming it replays a dead conversation while the
|
|
247
|
-
# handoff waits for a successor (the 2026-08-27 seam — "Trantor resumes the wrong thread").
|
|
248
|
-
#
|
|
249
|
-
# FRESH id
|
|
250
|
-
# orch-sessions.txt
|
|
247
|
+
# handoff waits for a successor (the 2026-08-27 seam — "Trantor resumes the wrong thread").
|
|
248
|
+
# ANY unconsumed handoff for the project means a successor is OWED a fresh window — open must
|
|
249
|
+
# start a FRESH id so the sessionstart hook claims the baton (it then records the fresh id in
|
|
250
|
+
# orch-sessions.txt; single writer: the hook + adopt — this function writes nothing).
|
|
251
|
+
# The old predicate required the handoff to be written BY the recorded session (session_id match)
|
|
252
|
+
# — but MANUAL handoffs carry no session_id at all, so on 2026-08-31 the reboot flow resumed the
|
|
253
|
+
# same maxed-out conversation TWICE while its handoff sat unclaimed, and the injected recap
|
|
254
|
+
# banner made each resume LOOK like a clean takeover. Who wrote it doesn't matter; that it is
|
|
255
|
+
# unclaimed does.
|
|
251
256
|
_orch_takeover_sid() { # $1=project $2=recorded-sid → prints the sid open should use
|
|
252
257
|
local fresh
|
|
253
258
|
if node -e '
|
|
254
259
|
const fs=require("fs"),path=require("path"),os=require("os");
|
|
255
260
|
const dir=path.join(process.env.AGENT_BUS_DIR||process.env.RELAY_DATA_DIR||path.join(os.homedir(),".agent-bus"),"handoffs");
|
|
256
|
-
const [proj
|
|
261
|
+
const [proj]=process.argv.slice(1);
|
|
257
262
|
try{
|
|
258
263
|
const re=new RegExp("^"+proj.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")+"-(\\d+)\\.json$");
|
|
259
264
|
const files=fs.readdirSync(dir).map(f=>{const m=re.exec(f);return m?{f,s:Number(m[1])}:null}).filter(Boolean).sort((a,b)=>b.s-a.s);
|
|
260
265
|
for(const {f} of files){
|
|
261
266
|
const r=JSON.parse(fs.readFileSync(path.join(dir,f),"utf8"));
|
|
262
267
|
if(r.consumed) continue;
|
|
263
|
-
process.exit(
|
|
268
|
+
process.exit(0); // newest UNCONSUMED exists → a successor is owed a fresh window
|
|
264
269
|
}
|
|
265
270
|
}catch(e){}
|
|
266
|
-
process.exit(1);' "$1"
|
|
271
|
+
process.exit(1);' "$1" 2>/dev/null; then
|
|
267
272
|
fresh="$(uuidgen 2>/dev/null | tr 'A-Z' 'a-z')"
|
|
268
273
|
[ -n "$fresh" ] && { echo "— recorded session $2 handed off: starting fresh as $fresh to claim it —" >&2; printf '%s' "$fresh"; return 0; }
|
|
269
274
|
fi
|
package/bin/doctor.mjs
CHANGED
|
@@ -10,6 +10,7 @@ import { execSync } from "node:child_process";
|
|
|
10
10
|
import { fileURLToPath } from "node:url";
|
|
11
11
|
import { resolveProject, resolveHub, DEFAULT_HUB_URL } from "../lib/project.mjs";
|
|
12
12
|
import { loadOrCreate } from "../lib/identity.mjs";
|
|
13
|
+
import { sfetchJson } from "../lib/signed-fetch.mjs";
|
|
13
14
|
import { scan } from "../lib/splitbrain.mjs";
|
|
14
15
|
|
|
15
16
|
const H = homedir();
|
|
@@ -87,6 +88,55 @@ section("hub routing");
|
|
|
87
88
|
}
|
|
88
89
|
}
|
|
89
90
|
|
|
91
|
+
// ── duty seat: is the fleet's watcher actually alive? ────────────────────────────────────────
|
|
92
|
+
// The duty seat sat dead for four days (2026-08-27→31) while everything else reported green.
|
|
93
|
+
// A dead watcher raises no error of its own — it just stops producing nudges — so this row makes
|
|
94
|
+
// that state loud: process, keepalive, hub registration and the freshness of the seat's last hub
|
|
95
|
+
// beat, each with its fix.
|
|
96
|
+
section("duty seat (the fleet watcher)");
|
|
97
|
+
{
|
|
98
|
+
const BUSD = join(H, ".agent-bus");
|
|
99
|
+
const DUTY_PLIST = join(H, "Library", "LaunchAgents", "com.trantor.duty.plist");
|
|
100
|
+
const FIX_UP = "trantor duty up (installs the launchd keepalive com.trantor.duty, which relaunches the seat after a crash or reboot)";
|
|
101
|
+
let pid = 0;
|
|
102
|
+
try { pid = Number(readFileSync(join(BUSD, "duty.pid"), "utf8")) || 0; if (pid) process.kill(pid, 0); else pid = 0; } catch { pid = 0; }
|
|
103
|
+
const keepalive = existsSync(DUTY_PLIST);
|
|
104
|
+
// The seat watches the FLEET hub — the most common project hub in config, the exact pick
|
|
105
|
+
// bin/duty.mjs makes — not necessarily THIS project's hub.
|
|
106
|
+
const counts = new Map();
|
|
107
|
+
for (const u of Object.values(cfg.hubs || {})) counts.set(u, (counts.get(u) || 0) + 1);
|
|
108
|
+
const fleet = [...counts.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] || cfg.url || "";
|
|
109
|
+
const signed = cfg.ownerIdentity ? (() => { try { return loadOrCreate(cfg.ownerIdentity, "human"); } catch { return null; } })() : null;
|
|
110
|
+
const hubGet = (path) => sfetchJson(`${fleet}${path}`, { method: "GET", identity: signed, signal: AbortSignal.timeout(4000) })
|
|
111
|
+
.then((r) => (r?.ok ? r.json() : null)).catch(() => null);
|
|
112
|
+
if (!fleet) {
|
|
113
|
+
note("duty seat: no hub pinned — cannot check the fleet feed");
|
|
114
|
+
} else {
|
|
115
|
+
const st = await hubGet("/overseer/status");
|
|
116
|
+
const peers = await hubGet("/peers");
|
|
117
|
+
const dutySession = st?.dutySession || "";
|
|
118
|
+
const beat = dutySession ? (peers?.sessions || []).find((p) => p.session === dutySession)?.lastSeen || 0 : 0;
|
|
119
|
+
const ageMin = beat ? Math.floor((Date.now() - beat) / 60000) : null;
|
|
120
|
+
const age = ageMin == null ? "no beat yet" : ageMin < 1 ? "beat just now" : ageMin < 60 ? `last beat ${ageMin}m ago` : `last beat ${Math.floor(ageMin / 60)}h ago`;
|
|
121
|
+
if (!st || !peers) {
|
|
122
|
+
// The core section already flags a dead hub; here we only refuse to guess.
|
|
123
|
+
note(`duty seat: hub feed UNKNOWN — ${fleet} did not answer the duty read${cfg.ownerIdentity ? "" : " (no owner identity to sign with)"}`);
|
|
124
|
+
pid ? ok(`duty seat: process running (pid ${pid})`) : warn("duty seat: not running and its hub feed cannot be checked", FIX_UP);
|
|
125
|
+
} else if (pid && dutySession && ageMin != null && ageMin > 5) {
|
|
126
|
+
warn(`duty seat: process up (pid ${pid}) but the hub heard nothing for ${Math.floor(ageMin / 60) >= 1 ? Math.floor(ageMin / 60) + "h " : ""}${ageMin % 60}m — it is running deaf`, "trantor duty down && trantor duty up");
|
|
127
|
+
} else if (pid) {
|
|
128
|
+
ok(`duty seat: running (pid ${pid}${dutySession ? `, ${dutySession} on the fleet hub` : ""}, ${age})`);
|
|
129
|
+
if (process.platform === "darwin" && !keepalive) warn("duty seat: running WITHOUT a keepalive — a crash or reboot leaves it down", FIX_UP);
|
|
130
|
+
} else if (dutySession) {
|
|
131
|
+
warn(`duty seat: the hub still points at ${dutySession} but no seat process is running — escalations go into a hole`, FIX_UP);
|
|
132
|
+
} else if (keepalive) {
|
|
133
|
+
warn("duty seat: keepalive installed but the seat is down — launchd should have relaunched it", `launchctl list | grep com.trantor.duty then: trantor duty down && trantor duty up`);
|
|
134
|
+
} else {
|
|
135
|
+
warn("duty seat: none — nobody is watching the fleet (undelivered mail and dead seats go unnudged)", FIX_UP);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
90
140
|
// claude plugin
|
|
91
141
|
section("claude (the orchestrator)");
|
|
92
142
|
if (!has("claude")) warn("claude CLI not found", "install Claude Code: https://claude.com/claude-code");
|
package/bin/duty.mjs
CHANGED
|
@@ -6,14 +6,20 @@
|
|
|
6
6
|
// trantor duty down stop
|
|
7
7
|
// trantor duty status pid + last turns + presence
|
|
8
8
|
//
|
|
9
|
+
// Keepalive (the 4-day silent death, 2026-08-27→31): by default the seat runs HEADLESS under a
|
|
10
|
+
// launchd service (label com.trantor.duty, KeepAlive=true) — launchd relaunches it after a crash
|
|
11
|
+
// and at every login, so the watcher stops being one bad afternoon away from silently gone.
|
|
12
|
+
// `--window` opts into the visible cmux/Terminal surface instead; a window CANNOT be kept alive
|
|
13
|
+
// by launchd, so that mode says plainly that nothing will bring it back.
|
|
14
|
+
//
|
|
9
15
|
// Division of labor (the overseer doctrine, extended): DETECTION stays mechanical and hub-side —
|
|
10
16
|
// RELAY_DUTY_SESSION makes the hub DM this seat when a direct message sits undelivered past
|
|
11
17
|
// RELAY_DUTY_UNDELIVERED_MS or the overseer emits a warning. The SEAT only triages: relay, wake,
|
|
12
18
|
// annotate, and only involves the human when a real decision is needed. It runs under the same
|
|
13
19
|
// crew-runner that keeps crew seats alive (long-poll wake, turn telemetry, failure reporting) —
|
|
14
20
|
// just with a triage doctrine instead of "work your card" (RUNNER_RULES / CREW_KICKOFF).
|
|
15
|
-
import { spawn, execSync } from "node:child_process";
|
|
16
|
-
import { readFileSync, writeFileSync, existsSync, mkdirSync, openSync, rmSync } from "node:fs";
|
|
21
|
+
import { spawn, execSync, execFileSync } from "node:child_process";
|
|
22
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync, openSync, rmSync, unlinkSync } from "node:fs";
|
|
17
23
|
import { join, dirname } from "node:path";
|
|
18
24
|
import { homedir } from "node:os";
|
|
19
25
|
import { fileURLToPath } from "node:url";
|
|
@@ -25,6 +31,9 @@ const BUS = process.env.AGENT_BUS_DIR || join(homedir(), ".agent-bus");
|
|
|
25
31
|
const DIR = join(BUS, "trantor-duty"); // the seat's cwd, and therefore its bus id
|
|
26
32
|
const PIDF = join(BUS, "duty.pid");
|
|
27
33
|
const LOGF = join(BUS, "duty.log");
|
|
34
|
+
const LAUNCHER = join(BUS, "duty-launch.sh");
|
|
35
|
+
const DUTY_LABEL = "com.trantor.duty";
|
|
36
|
+
const DUTY_PLIST = join(homedir(), "Library", "LaunchAgents", `${DUTY_LABEL}.plist`);
|
|
28
37
|
|
|
29
38
|
const argv = process.argv.slice(2);
|
|
30
39
|
const cmd = argv[0] || "status";
|
|
@@ -46,8 +55,12 @@ function fleetHub() {
|
|
|
46
55
|
// a patrol script, send a templated nudge, post 280 chars. Nobody chose that; it was inherited.
|
|
47
56
|
// Precedence: --model flag > CREW_MODEL env > sonnet. `--model inherit` restores the old behaviour.
|
|
48
57
|
const DUTY_MODEL = val("model", "") || process.env.CREW_MODEL || "sonnet";
|
|
49
|
-
//
|
|
50
|
-
|
|
58
|
+
// Headless is the DEFAULT now, and it rides the launchd keepalive. The old default — a visible
|
|
59
|
+
// window, headless only as a silent fallback — is exactly how the seat died quietly on
|
|
60
|
+
// 2026-08-27: the osascript window-open failed, the fallback printed one line into a log nobody
|
|
61
|
+
// reads, and the fleet had no watcher for four days. Headless+keepalive is the honest default;
|
|
62
|
+
// `--window` is the explicit choice of a surface launchd cannot keep alive.
|
|
63
|
+
const WINDOW = argv.includes("--window") && process.platform === "darwin";
|
|
51
64
|
const AGENT = val("agent", "claude");
|
|
52
65
|
// Named, not inherited. It used to be "claude:fleet" purely because the seat's directory was
|
|
53
66
|
// called fleet and identity is derived from directory basename — the same identity-by-position
|
|
@@ -134,6 +147,71 @@ function cmuxBinary() {
|
|
|
134
147
|
return "";
|
|
135
148
|
}
|
|
136
149
|
|
|
150
|
+
// launchd, invoked by NAME so a drill can stub it on PATH (seats.mjs hardcodes /bin/launchctl,
|
|
151
|
+
// which is why its install has no drill). Every call swallows its error: on a machine without
|
|
152
|
+
// launchd the keepalive path is simply unavailable and the caller says so.
|
|
153
|
+
const bootoutDuty = () => { try { execFileSync("launchctl", ["bootout", `gui/${process.getuid()}/${DUTY_LABEL}`], { stdio: "ignore", timeout: 8000 }); return true; } catch { return false; } };
|
|
154
|
+
const bootstrapDuty = () => { try { execFileSync("launchctl", ["bootstrap", `gui/${process.getuid()}`, DUTY_PLIST], { stdio: "ignore", timeout: 8000 }); return true; } catch { return false; } };
|
|
155
|
+
const dutyLoaded = () => { try { return execFileSync("launchctl", ["list"], { encoding: "utf8", timeout: 8000 }).split("\n").some((l) => l.includes(DUTY_LABEL)); } catch { return false; } };
|
|
156
|
+
|
|
157
|
+
// Same service shape as the hub (deploy/com.trantor.hub.plist): the long-running process is the
|
|
158
|
+
// job, RunAtLoad + KeepAlive bring it back after a crash and at every login. One deliberate
|
|
159
|
+
// addition, ThrottleInterval=30: a seat that exits because the hub is down must not hot-loop —
|
|
160
|
+
// a KeepAlive job retrying every 10s forever is how this machine hit load 490 on 2026-08-21.
|
|
161
|
+
function keepalivePlistBody() {
|
|
162
|
+
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
163
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
164
|
+
<!-- trantor duty seat as an always-on launchd service. \`trantor duty up\` rewrites this;
|
|
165
|
+
\`trantor duty down\` removes it. -->
|
|
166
|
+
<plist version="1.0">
|
|
167
|
+
<dict>
|
|
168
|
+
<key>Label</key><string>${DUTY_LABEL}</string>
|
|
169
|
+
<key>ProgramArguments</key>
|
|
170
|
+
<array><string>/bin/bash</string><string>${LAUNCHER}</string></array>
|
|
171
|
+
<key>RunAtLoad</key><true/>
|
|
172
|
+
<key>KeepAlive</key><true/>
|
|
173
|
+
<key>ThrottleInterval</key><integer>30</integer>
|
|
174
|
+
<key>StandardOutPath</key><string>${LOGF}</string>
|
|
175
|
+
<key>StandardErrorPath</key><string>${LOGF}</string>
|
|
176
|
+
</dict>
|
|
177
|
+
</plist>
|
|
178
|
+
`;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// The rules are ~4KB of prose with backticks, quotes and $ in them, so they cannot ride a
|
|
182
|
+
// command line or an AppleScript string. A launcher script carries them instead. The values ride
|
|
183
|
+
// SINGLE QUOTES (apostrophes escaped as '"'"'): backticks, $( ), parens and newlines are then all
|
|
184
|
+
// literal. The previous form — $(cat <<'EOF' … EOF) — silently broke under macOS bash 3.2 the
|
|
185
|
+
// moment a value contained a backtick (the rules do): the export failed, the seat started without
|
|
186
|
+
// its kickoff text, and the only symptom was a syntax-error line in a log nobody reads.
|
|
187
|
+
function writeLauncher(env) {
|
|
188
|
+
const shquote = (v) => `'${String(v).replace(/'/g, `'\\''`)}'`;
|
|
189
|
+
const exports = Object.entries(env).map(([k, v]) => `export ${k}=${shquote(v)}`).join("\n");
|
|
190
|
+
writeFileSync(LAUNCHER, `#!/bin/bash\n# written by \`trantor duty up\` — safe to delete when the seat is down\n${exports}\nexec ${JSON.stringify(process.execPath)} ${JSON.stringify(join(ROOT, "bin", "crew-runner.mjs"))} ${AGENT} ${JSON.stringify(DIR)}\n`, { mode: 0o700 });
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// The old headless start: a detached child of whoever ran `up`. No keepalive — it dies with a
|
|
194
|
+
// reboot and nothing brings it back. Kept for non-darwin and as the loud last fallback.
|
|
195
|
+
function startDetached(env) {
|
|
196
|
+
const out = openSync(LOGF, "a");
|
|
197
|
+
const child = spawn(process.execPath, [join(ROOT, "bin", "crew-runner.mjs"), AGENT, DIR], {
|
|
198
|
+
detached: true, stdio: ["ignore", out, out], env: { ...process.env, ...env },
|
|
199
|
+
});
|
|
200
|
+
child.unref();
|
|
201
|
+
return child.pid;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// Find the runner's pid once a surface (window or launchd) should have started it, and park it
|
|
205
|
+
// in the pidfile `down`/`status` read. Terminal/launchd take a moment, so poll briefly.
|
|
206
|
+
function pollRunnerPid(max = 25) {
|
|
207
|
+
let pid = 0;
|
|
208
|
+
for (let i = 0; i < max && !pid; i++) {
|
|
209
|
+
try { pid = Number(execSync(`pgrep -f "crew-runner.mjs ${AGENT} ${DIR}" | head -1`, { encoding: "utf8" }).trim()) || 0; } catch {}
|
|
210
|
+
if (!pid) execSync("sleep 0.2");
|
|
211
|
+
}
|
|
212
|
+
return pid;
|
|
213
|
+
}
|
|
214
|
+
|
|
137
215
|
/** Close every workspace this seat owns. No-op when cmux is absent or its socket is off. */
|
|
138
216
|
function closeDutyWorkspace() {
|
|
139
217
|
const bin = cmuxBinary();
|
|
@@ -166,77 +244,94 @@ if (cmd === "up") {
|
|
|
166
244
|
if (!(await ensureFleetIdentity(hub))) process.exit(1);
|
|
167
245
|
const env = (() => {
|
|
168
246
|
const e = { RELAY_URL: hub, RUNNER_RULES: RULES, CREW_KICKOFF: KICKOFF,
|
|
169
|
-
RUNNER_TITLE: "Trantor Duty Agent", RUNNER_ABOUT: ABOUT
|
|
247
|
+
RUNNER_TITLE: "Trantor Duty Agent", RUNNER_ABOUT: ABOUT,
|
|
248
|
+
// launchd starts jobs with a MINIMAL Path — the resurrected seat could not find
|
|
249
|
+
// `claude` and every turn died exit 127 "missing-cli" (found live 2026-08-31,
|
|
250
|
+
// duty's own triage caught it). Bake the operator's PATH from `up` time into the
|
|
251
|
+
// launcher, exactly like every other value it carries.
|
|
252
|
+
PATH: process.env.PATH || "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin" };
|
|
170
253
|
if (DUTY_MODEL !== "inherit") e.CREW_MODEL = DUTY_MODEL;
|
|
171
254
|
return e;
|
|
172
255
|
})();
|
|
173
256
|
|
|
257
|
+
writeLauncher(env);
|
|
258
|
+
|
|
174
259
|
let pid = 0;
|
|
260
|
+
let how = "";
|
|
261
|
+
|
|
175
262
|
if (WINDOW) {
|
|
176
|
-
//
|
|
177
|
-
//
|
|
178
|
-
//
|
|
179
|
-
//
|
|
180
|
-
//
|
|
181
|
-
// The rules are ~4KB of prose with backticks, quotes and $ in them, so they cannot ride a
|
|
182
|
-
// command line or an AppleScript string. A launcher script carries them instead: a quoted
|
|
183
|
-
// heredoc means the shell expands nothing, and osascript only ever sees the path.
|
|
184
|
-
const launcher = join(BUS, "duty-launch.sh");
|
|
185
|
-
const exports = Object.entries(env).map(([k, v]) =>
|
|
186
|
-
`export ${k}=$(cat <<'TRANTOR_${k}_EOF'\n${v}\nTRANTOR_${k}_EOF\n)`).join("\n");
|
|
187
|
-
writeFileSync(launcher, `#!/bin/bash\n# written by \`trantor duty up\` — safe to delete when the seat is down\n${exports}\nexec ${JSON.stringify(process.execPath)} ${JSON.stringify(join(ROOT, "bin", "crew-runner.mjs"))} ${AGENT} ${JSON.stringify(DIR)}\n`, { mode: 0o700 });
|
|
188
|
-
// PREFER CMUX. Terminal.app was the only surface here, and a plain window is stacking by
|
|
189
|
-
// construction: every `duty up` opens another one and nothing closes the last, so restarts
|
|
190
|
-
// accumulate windows that all look like live duty agents. cmux gives the seat ONE named
|
|
191
|
-
// workspace that gets REPLACED on each up — the same "replace, never stack" rule bin/crew.sh
|
|
192
|
-
// already applies to crew seats, which is why they never pile up and this did.
|
|
193
|
-
//
|
|
194
|
-
// Terminal remains the fallback: no cmux, or its control socket off, and nothing changes.
|
|
263
|
+
// --window: the visible surface. cmux first (ONE named workspace, replaced on each up — the
|
|
264
|
+
// same "replace, never stack" rule bin/crew.sh applies to crew seats), Terminal as fallback.
|
|
265
|
+
// A window cannot be kept alive by launchd, so `how` says plainly that nothing will bring
|
|
266
|
+
// the seat back if it dies.
|
|
195
267
|
const cmuxBin = cmuxBinary();
|
|
196
|
-
|
|
197
|
-
let openedInCmux = false;
|
|
268
|
+
let opened = false;
|
|
198
269
|
if (cmuxBin) {
|
|
199
270
|
try {
|
|
200
271
|
// Replace, never stack: take the previous duty workspace away before opening this one.
|
|
201
272
|
// Closing first is safe here (unlike a crew pane swap) — the seat is a single surface with
|
|
202
273
|
// nothing to preserve.
|
|
203
274
|
closeDutyWorkspace();
|
|
204
|
-
execSync(`${cmuxBin} new-workspace --name ${JSON.stringify(CMUX_WS_NAME)} --cwd ${JSON.stringify(DIR)} --command ${JSON.stringify(`bash ${
|
|
275
|
+
execSync(`${cmuxBin} new-workspace --name ${JSON.stringify(CMUX_WS_NAME)} --cwd ${JSON.stringify(DIR)} --command ${JSON.stringify(`bash ${LAUNCHER}`)} --focus false`,
|
|
205
276
|
{ stdio: "ignore", timeout: 8000, env: { ...process.env, CMUX_QUIET: "1" } });
|
|
206
|
-
|
|
277
|
+
opened = true;
|
|
207
278
|
} catch (e) {
|
|
208
279
|
console.error(`cmux launch failed (${e?.message || e}) — falling back to a Terminal window`);
|
|
209
280
|
}
|
|
210
281
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
282
|
+
if (!opened) {
|
|
283
|
+
const osa = `tell application "Terminal"\n do script ${JSON.stringify(`bash ${LAUNCHER}`)}\n activate\nend tell\n`;
|
|
284
|
+
try { execSync(`osascript -e ${JSON.stringify(osa)}`, { stdio: "ignore", timeout: 8000 }); opened = true; }
|
|
285
|
+
catch (e) {
|
|
286
|
+
// The 2026-08-27 incident: this printed one quiet line, fell back headless, and the fleet
|
|
287
|
+
// had no watcher for four days. Now it is loud, and the keepalive path below takes over.
|
|
288
|
+
console.error(` ⚠️ could not open a window (${e?.message || e}) — falling back to the headless launchd keepalive`);
|
|
289
|
+
}
|
|
216
290
|
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
if (!pid) execSync("sleep 0.2");
|
|
291
|
+
if (opened) {
|
|
292
|
+
pid = pollRunnerPid();
|
|
293
|
+
writeFileSync(PIDF, String(pid));
|
|
294
|
+
how = "in a window (NO keepalive — if it dies or the Mac reboots, it stays down)";
|
|
222
295
|
}
|
|
223
296
|
}
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
297
|
+
|
|
298
|
+
if (!how) {
|
|
299
|
+
if (process.platform === "darwin") {
|
|
300
|
+
// The honest default: headless under launchd, so a crashed seat relaunches itself instead
|
|
301
|
+
// of dying silently (the seat sat dead 2026-08-27→31 before anyone noticed).
|
|
302
|
+
mkdirSync(dirname(DUTY_PLIST), { recursive: true }); // a fresh machine has no LaunchAgents dir yet
|
|
303
|
+
writeFileSync(DUTY_PLIST, keepalivePlistBody());
|
|
304
|
+
bootoutDuty();
|
|
305
|
+
if (bootstrapDuty()) {
|
|
306
|
+
pid = pollRunnerPid(15);
|
|
307
|
+
writeFileSync(PIDF, String(pid));
|
|
308
|
+
how = `headless under the launchd keepalive ${DUTY_LABEL} (relaunched after a crash or reboot)`;
|
|
309
|
+
if (!pid) console.error(` ⚠️ keepalive installed but no runner seen after 3s — check: launchctl list | grep ${DUTY_LABEL}`);
|
|
310
|
+
} else {
|
|
311
|
+
console.error(" ⚠️ launchctl bootstrap failed — starting a plain headless seat with NO keepalive");
|
|
312
|
+
pid = startDetached(env);
|
|
313
|
+
writeFileSync(PIDF, String(pid));
|
|
314
|
+
how = "headless, NO keepalive (launchd refused the job)";
|
|
315
|
+
}
|
|
316
|
+
} else {
|
|
317
|
+
pid = startDetached(env);
|
|
318
|
+
writeFileSync(PIDF, String(pid));
|
|
319
|
+
how = `headless, NO keepalive (launchd is macOS-only; ${process.platform} gets a plain background seat)`;
|
|
320
|
+
}
|
|
231
321
|
}
|
|
232
|
-
|
|
233
|
-
console.log(`— duty agent up: ${SESSION} (pid ${pid})${
|
|
322
|
+
|
|
323
|
+
console.log(`— duty agent up: ${SESSION} (pid ${pid}) ${how} on ${DUTY_MODEL === "inherit" ? "the CLI default model" : DUTY_MODEL} watching ${hub} — log: ${LOGF}`);
|
|
234
324
|
const fed = await registerDutySeat(hub, SESSION);
|
|
235
325
|
if (fed) console.log(` hub feeds it: undelivered DMs (>${Math.round(Number(process.env.RELAY_DUTY_UNDELIVERED_MS || 600000) / 60000)}m) + overseer warnings.`);
|
|
236
326
|
process.exit(0); // the seat IS up; a hub that won't feed it is a warning, not a failed start
|
|
237
327
|
}
|
|
238
328
|
|
|
239
329
|
if (cmd === "down") {
|
|
330
|
+
const hadKeepalive = existsSync(DUTY_PLIST);
|
|
331
|
+
// Unload the keepalive FIRST (bootout kills the job's process), then remove the plist so a
|
|
332
|
+
// reboot cannot resurrect the seat behind a `down`. `up` rewrites both.
|
|
333
|
+
bootoutDuty();
|
|
334
|
+
if (hadKeepalive) { try { unlinkSync(DUTY_PLIST); } catch {} }
|
|
240
335
|
const pid = alivePid();
|
|
241
336
|
if (pid) { try { process.kill(pid); } catch {} console.log(`— duty seat stopped (pid ${pid}) —`); }
|
|
242
337
|
else console.log("no duty seat running");
|
|
@@ -249,6 +344,18 @@ if (cmd === "down") {
|
|
|
249
344
|
// Clear the hub's pointer too — escalations aimed at a seat that no longer exists are messages
|
|
250
345
|
// sent into a hole, and the hub has no other way to learn the seat went away.
|
|
251
346
|
await registerDutySeat(fleetHub(), "");
|
|
347
|
+
// GOING LOUD. A quiet `down` is how the seat sat dead for four days (2026-08-27→31) while every
|
|
348
|
+
// other surface reported green: nothing errors when the watcher is gone, it just stops watching.
|
|
349
|
+
// Anyone turning the watcher off must see, in that moment, exactly what they are leaving dark.
|
|
350
|
+
if (pid || hadKeepalive) {
|
|
351
|
+
console.log(`
|
|
352
|
+
────────────────────────────────────────────────────────────
|
|
353
|
+
⚠️ DUTY IS DOWN. Nobody is watching the fleet now.
|
|
354
|
+
Undelivered mail and dead seats will go unnudged — last time
|
|
355
|
+
that silence lasted four days (Aug 27→31) before anyone noticed.
|
|
356
|
+
Bring it back: trantor duty up
|
|
357
|
+
────────────────────────────────────────────────────────────`);
|
|
358
|
+
}
|
|
252
359
|
process.exit(0);
|
|
253
360
|
}
|
|
254
361
|
|
|
@@ -263,6 +370,9 @@ if (cmd === "down") {
|
|
|
263
370
|
console.log(" and never edits your project files. Stop it with: trantor duty down");
|
|
264
371
|
console.log("");
|
|
265
372
|
console.log(pid ? `RUNNING (pid ${pid}) as ${SESSION}` : "NOT running");
|
|
373
|
+
console.log(existsSync(DUTY_PLIST)
|
|
374
|
+
? `keepalive: installed (${DUTY_LABEL}${process.platform === "darwin" ? (dutyLoaded() ? ", loaded" : ", not loaded in this session") : ""}) — launchd relaunches the seat after a crash or reboot`
|
|
375
|
+
: "keepalive: NOT installed — a crash or reboot leaves the seat down (trantor duty up installs it)");
|
|
266
376
|
// A running seat the hub isn't feeding looks identical to a working one from the outside — which
|
|
267
377
|
// is the whole failure mode this command exists to make visible. So ask the hub, don't assume.
|
|
268
378
|
const hub = fleetHub();
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Turn watchdog (#5684). runTurn is spawnSync — the runner cannot watch its own turn — so this
|
|
3
|
+
// DETACHED helper does: armed at turn start, disarmed by turn end (the stamp file vanishes or
|
|
4
|
+
// its turn number moves on). A turn that runs past the window with NO output growth earns ONE
|
|
5
|
+
// direct stall report to the foreman (episode, never a timer storm), and the turn is never
|
|
6
|
+
// killed — reporting is the whole job. The operator's 2026-08-31 complaint is the incident:
|
|
7
|
+
// seats sat visibly dead in their panes while every signal channel stayed quiet.
|
|
8
|
+
//
|
|
9
|
+
// node bin/turn-watchdog.mjs <stampFile> <errFile> <windowMs> <session> <project> <hubUrl>
|
|
10
|
+
import { readFileSync, existsSync, statSync } from "node:fs";
|
|
11
|
+
import { hostId } from "../lib/project.mjs";
|
|
12
|
+
import { signedPost } from "../hooks/lib/api.mjs";
|
|
13
|
+
|
|
14
|
+
const [stampFile, errFile, windowMsRaw, session, project, hub] = process.argv.slice(2);
|
|
15
|
+
const windowMs = Number(windowMsRaw) || 15 * 60 * 1000; // no floor: drills pass tiny windows, and one report per turn caps the damage anyway
|
|
16
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
17
|
+
|
|
18
|
+
const readStamp = () => { try { return JSON.parse(readFileSync(stampFile, "utf8")); } catch { return null; } };
|
|
19
|
+
const errSize = () => { try { return statSync(errFile).size; } catch { return 0; } };
|
|
20
|
+
|
|
21
|
+
const armed = readStamp();
|
|
22
|
+
if (!armed) process.exit(0);
|
|
23
|
+
let baseline = errSize();
|
|
24
|
+
|
|
25
|
+
for (;;) {
|
|
26
|
+
await sleep(windowMs);
|
|
27
|
+
const s = readStamp();
|
|
28
|
+
if (!s || s.turn !== armed.turn) process.exit(0); // turn ended — nothing to say
|
|
29
|
+
const size = errSize();
|
|
30
|
+
if (size > baseline + 200) { baseline = size; continue; } // producing output: working, re-arm
|
|
31
|
+
const mins = Math.round((Date.now() - (s.startedAt || Date.now())) / 60000);
|
|
32
|
+
const orch = `${hostId()}:${project}`;
|
|
33
|
+
const text = `⏱ ${session} turn STALLED — running ${mins}m with no output (turn ${s.turn}). Not killed; check its pane, or \`trantor swap\`.`;
|
|
34
|
+
// Direct = wake. The foreman first; if this seat IS the foreman's own runner, say it to all.
|
|
35
|
+
const to = orch === session ? "all" : orch;
|
|
36
|
+
try { await signedPost(`${hub}/send`, { from: session, to, text, project }, { session }); } catch {}
|
|
37
|
+
process.exit(0); // one report per turn, by construction
|
|
38
|
+
}
|
package/hooks/sessionstart.mjs
CHANGED
|
@@ -441,6 +441,11 @@ try {
|
|
|
441
441
|
// The hold LAPSES (default 30m) so an unclaimed baton never strands every other session.
|
|
442
442
|
// Any other handoff keeps first-fresh-session-wins.
|
|
443
443
|
const isCompact = source === "compact";
|
|
444
|
+
// A RESUMED session must never claim a handoff (2026-08-31, twice in one afternoon): it
|
|
445
|
+
// already carries its whole history, so claiming injects the takeover banner into the same
|
|
446
|
+
// maxed-out context — the recap LOOKS right while the window never reset, which is worse
|
|
447
|
+
// than failing loudly. The successor is whatever fresh session `trantor open` starts.
|
|
448
|
+
const isResume = source === "resume";
|
|
444
449
|
const orchEnv = process.env.TRANTOR_ORCH || "";
|
|
445
450
|
const isOrchPane = !!orchEnv && (orchEnv === "1" || orchEnv === project); // project-matched: a child claude in another dir must not inherit the badge
|
|
446
451
|
const orchSid = readOrchSession(project);
|
|
@@ -450,12 +455,12 @@ try {
|
|
|
450
455
|
const ageMs = peek ? Date.now() - (Number(peek.stamp) || 0) * 1000 : 0;
|
|
451
456
|
const held = orchOrigin && !isOrchPane && !isCompact && ageMs < holdMs;
|
|
452
457
|
const handoff = !peek ? null
|
|
453
|
-
: (isCompact || held) ? peek
|
|
458
|
+
: (isCompact || held || isResume) ? peek
|
|
454
459
|
: loadPendingHandoff(basename(projectDir), {
|
|
455
460
|
claim: true,
|
|
456
461
|
freshSession: { session_id: stdinObj.session_id || "", transcript_path: stdinObj.transcript_path || "" },
|
|
457
462
|
});
|
|
458
|
-
const claimed = !!handoff && !isCompact && !held;
|
|
463
|
+
const claimed = !!handoff && !isCompact && !held && !isResume;
|
|
459
464
|
// Follow the thread: claiming the orchestrator's baton makes THIS session the orchestrator
|
|
460
465
|
// thread, so the map `trantor open` resumes and the app's chat reads moves with it. The pane
|
|
461
466
|
// also records itself on every fresh start — that keeps the map honest even if a future
|
|
@@ -468,6 +473,13 @@ try {
|
|
|
468
473
|
additionalContext += `<trantor-handoff-held id="${sanitize(handoff.id)}" from="${sanitize(handoff.machine)}">\n`;
|
|
469
474
|
additionalContext += `🔒 A handoff from this project's ORCHESTRATOR thread is pending, and it is being HELD for the Trantor orchestrator pane (\`trantor open\` claims it on start). This session did NOT claim it and does not have its content — do not act as the successor. If the user wants THIS session to take over instead: \`trantor adopt\`, then restart this session. Unclaimed, the hold lapses in ~${mins} minute(s) and the handoff becomes first-come.\n`;
|
|
470
475
|
additionalContext += `</trantor-handoff-held>\n`;
|
|
476
|
+
} else if (handoff && isResume) {
|
|
477
|
+
// No takeover banner here — injecting it into a resumed session is exactly the failure this
|
|
478
|
+
// guard exists for: the recap reads like a clean handoff while the context never reset.
|
|
479
|
+
process.stderr.write(`[trantor] pending handoff ${handoff.id} NOT claimed — this session was RESUMED, not started fresh\n`);
|
|
480
|
+
additionalContext += `<trantor-handoff-unclaimed id="${sanitize(handoff.id)}" reason="resumed-session">\n`;
|
|
481
|
+
additionalContext += `⚠️ An unclaimed handoff (${sanitize(handoff.id)}) is waiting for this project, but THIS session was RESUMED (\`--resume\`) — it already carries its own history and is NOT the successor. Do not act on the handoff. Tell the user plainly: a fresh session must claim it — \`trantor open\` starts one. If they want THIS resumed session to continue instead, they can ignore the handoff or clear it.\n`;
|
|
482
|
+
additionalContext += `</trantor-handoff-unclaimed>\n`;
|
|
471
483
|
} else if (handoff) {
|
|
472
484
|
process.stderr.write(`[trantor] ${isCompact ? "showing (not claiming, compact)" : "loaded"} pending handoff ${handoff.id}\n`);
|
|
473
485
|
// Baton claimed → supersede every OTHER instance of this durable identity (instance-keys
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// USAGE v2 — the Claude statusline sidechannel (docs/RESEARCH-orca-usage.md §1.1).
|
|
3
|
+
// Claude Code >=2.1.80 pipes a JSON blob (with `rate_limits`) into the statusLine command on
|
|
4
|
+
// every turn, piggybacked on the Messages API response — live usage that costs zero API budget.
|
|
5
|
+
// This forwarder reads that stdin, POSTs the windows to the hub's /usage/claude (signed), and
|
|
6
|
+
// prints NOTHING: it is designed to be tee'd ahead of the operator's real statusline command,
|
|
7
|
+
// never to be one. Every failure is swallowed — a usage forwarder must never break a statusline.
|
|
8
|
+
//
|
|
9
|
+
// Floor: one POST per session per 15s (stamp file) — the statusline ticks ~3x/sec while
|
|
10
|
+
// streaming, and the hub dedupes same-value posts inside 30s anyway.
|
|
11
|
+
import { readFileSync, writeFileSync, statSync, mkdirSync } from "node:fs";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
import { homedir } from "node:os";
|
|
14
|
+
|
|
15
|
+
const FLOOR_MS = 15_000;
|
|
16
|
+
|
|
17
|
+
async function main() {
|
|
18
|
+
let raw = "";
|
|
19
|
+
for await (const c of process.stdin) raw += c;
|
|
20
|
+
const j = JSON.parse(raw);
|
|
21
|
+
const rl = j.rate_limits;
|
|
22
|
+
if (!rl || typeof rl !== "object") return; // most ticks carry none — free exit
|
|
23
|
+
const sid = String(j.session_id || j.sessionId || "nosession").replace(/[^A-Za-z0-9_.-]/g, "_").slice(0, 80);
|
|
24
|
+
const dir = join(homedir(), ".agent-bus");
|
|
25
|
+
const stamp = join(dir, `usage-claude-${sid}.stamp`);
|
|
26
|
+
try { if (Date.now() - statSync(stamp).mtimeMs < FLOOR_MS) return; } catch {}
|
|
27
|
+
try { mkdirSync(dir, { recursive: true }); writeFileSync(stamp, ""); } catch {}
|
|
28
|
+
|
|
29
|
+
const payload = {
|
|
30
|
+
configDir: process.env.CLAUDE_CONFIG_DIR || null,
|
|
31
|
+
fiveHour: rl.five_hour ?? rl.fiveHour ?? null,
|
|
32
|
+
sevenDay: rl.seven_day ?? rl.sevenDay ?? null,
|
|
33
|
+
// The fable/model-scoped window normally arrives via the OAuth poller, but accept the
|
|
34
|
+
// statusline-shaped variants too — schema drift degrades instead of going dark.
|
|
35
|
+
fable: rl.fable_weekly ?? rl.fable ?? null,
|
|
36
|
+
};
|
|
37
|
+
if (!payload.fiveHour && !payload.sevenDay && !payload.fable) return;
|
|
38
|
+
// PRIMARY: the local live cache — lib/balances.mjs merges it and skips the OAuth poll while
|
|
39
|
+
// it is fresh (Orca's isLiveClaudeUsageFresh, docs/RESEARCH-orca-usage.md §2). This is the
|
|
40
|
+
// path the app's footer actually reads (it shells the local CLI, not the hub).
|
|
41
|
+
try { writeFileSync(join(dir, "usage-claude-live.json"), JSON.stringify({ ts: Date.now(), ...payload })); } catch {}
|
|
42
|
+
// SECONDARY, best-effort: the hub copy, for dashboard surfaces that read hub state.
|
|
43
|
+
const { signedPost } = await import("./lib/api.mjs");
|
|
44
|
+
await signedPost("/usage/claude", payload, { timeoutMs: 2500 });
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
main().catch(() => {}).finally(() => process.exit(0));
|
package/hub.mjs
CHANGED
|
@@ -437,6 +437,26 @@ setTimeout(overseerTick, 2000).unref?.();
|
|
|
437
437
|
let DUTY_SESSION = String(process.env.RELAY_DUTY_SESSION || state.dutySession || "");
|
|
438
438
|
const DUTY_UNDELIVERED_MS = Number(process.env.RELAY_DUTY_UNDELIVERED_MS || 10 * 60 * 1000);
|
|
439
439
|
const dutyEscalated = new Set();
|
|
440
|
+
// #5686: the janitor died 08-27 and NOTHING noticed for 4 days — the hub kept escalating to a
|
|
441
|
+
// corpse. Duty liveness is now a first-class state: dark = configured but no heartbeat inside
|
|
442
|
+
// DUTY_DARK_MS. Episode semantics (one event per transition, a standing flag on /health), and
|
|
443
|
+
// while dark, escalations go to the party owed the reply instead of the dead seat.
|
|
444
|
+
const DUTY_DARK_MS = Number(process.env.RELAY_DUTY_DARK_MS || 10 * 60 * 1000);
|
|
445
|
+
let dutyDarkSince = 0;
|
|
446
|
+
// A freshly appointed seat has no heartbeat yet and is NOT a corpse: the dark clock starts at
|
|
447
|
+
// appointment (boot or POST /overseer/duty), so a newborn gets one full window to first-poll.
|
|
448
|
+
let dutySeenFloor = Date.now();
|
|
449
|
+
function dutyLiveness() {
|
|
450
|
+
if (!DUTY_SESSION) return { configured: false, online: false, lastSeenMs: 0 };
|
|
451
|
+
const seen = Math.max(state.peers[DUTY_SESSION]?.lastSeen || 0, dutySeenFloor);
|
|
452
|
+
const lastSeenMs = now() - seen;
|
|
453
|
+
return { configured: true, online: lastSeenMs < DUTY_DARK_MS, lastSeenMs: Math.max(0, lastSeenMs) };
|
|
454
|
+
}
|
|
455
|
+
function dutyQueuedEscalations() {
|
|
456
|
+
if (!DUTY_SESSION) return 0;
|
|
457
|
+
const upTo = state.peers[DUTY_SESSION]?.deliveredUpTo || 0;
|
|
458
|
+
return state.messages.reduce((n, m) => n + (m.to === DUTY_SESSION && m.id > upTo ? 1 : 0), 0);
|
|
459
|
+
}
|
|
440
460
|
function hubSend(to, text, project) {
|
|
441
461
|
const msg = { id: ++state.seq, ts: now(), from: "hub:duty", to, text: String(text).slice(0, 2000), project: String(project || "").slice(0, 80) };
|
|
442
462
|
state.messages.push(msg); if (state.messages.length > 5000) state.messages.splice(0, 1000);
|
|
@@ -446,6 +466,15 @@ function hubSend(to, text, project) {
|
|
|
446
466
|
}
|
|
447
467
|
function dutyTick() {
|
|
448
468
|
if (!DUTY_SESSION) return;
|
|
469
|
+
// #5686: track the dark episode BEFORE escalating, so this tick already routes around a corpse.
|
|
470
|
+
const live = dutyLiveness();
|
|
471
|
+
if (!live.online && !dutyDarkSince) {
|
|
472
|
+
dutyDarkSince = now();
|
|
473
|
+
appendEvent("duty-dark", "", "hub:duty", { text: `duty seat ${DUTY_SESSION} has no heartbeat — seat trouble is not being triaged (trantor duty up)` });
|
|
474
|
+
} else if (live.online && dutyDarkSince) {
|
|
475
|
+
appendEvent("duty-back", "", "hub:duty", { text: `duty seat ${DUTY_SESSION} is back after ${Math.round((now() - dutyDarkSince) / 60000)}m dark` });
|
|
476
|
+
dutyDarkSince = 0;
|
|
477
|
+
}
|
|
449
478
|
const cutoff = now() - DUTY_UNDELIVERED_MS;
|
|
450
479
|
const floor = now() - 24 * 3600 * 1000; // never escalate ancient history
|
|
451
480
|
for (const m of state.messages) {
|
|
@@ -460,7 +489,10 @@ function dutyTick() {
|
|
|
460
489
|
if (dutyEscalated.has(m.id)) continue;
|
|
461
490
|
if ((state.peers[m.to]?.deliveredUpTo || 0) >= m.id) continue;
|
|
462
491
|
dutyEscalated.add(m.id);
|
|
463
|
-
|
|
492
|
+
// #5686: a dark janitor must not eat escalations. Route to the SENDER — the party who
|
|
493
|
+
// believes they were heard and are owed the reply — with the duty outage named, so the
|
|
494
|
+
// failure is visible to someone who can act instead of queued on a corpse.
|
|
495
|
+
hubSend(dutyDarkSince ? m.from : DUTY_SESSION,
|
|
464
496
|
`⚠️ UNDELIVERED for ${Math.round((now() - m.ts) / 60000)}m: #${m.id} ${m.from} -> ${m.to} — "${String(m.text).slice(0, 280)}" — the recipient has not been handed this (recipient last seen ${state.peers[m.to]?.lastSeen ? Math.round((now() - state.peers[m.to].lastSeen) / 60000) + "m ago" : "never"}). Triage: is the recipient's session idle, deaf (wrong hub / old hooks), or gone? Relay, wake, or note it on their board.`,
|
|
465
497
|
m.project || "");
|
|
466
498
|
}
|
|
@@ -1549,6 +1581,8 @@ const server = http.createServer(async (req, res) => {
|
|
|
1549
1581
|
const session = String(b.session).slice(0, 120);
|
|
1550
1582
|
DUTY_SESSION = session;
|
|
1551
1583
|
state.dutySession = session;
|
|
1584
|
+
dutySeenFloor = Date.now(); // #5686: appointment restarts the dark clock — a newborn is not a corpse
|
|
1585
|
+
if (dutyDarkSince) { dutyDarkSince = 0; } // fresh seat, fresh episode accounting
|
|
1552
1586
|
dirty = true;
|
|
1553
1587
|
return json(res, 200, { ok: true, dutySession: DUTY_SESSION });
|
|
1554
1588
|
}
|
|
@@ -1608,6 +1642,29 @@ const server = http.createServer(async (req, res) => {
|
|
|
1608
1642
|
if (ts >= (state.balances?.ts || 0)) { state.balances = { ts, by: String(b.by || "").slice(0, 120), entries }; dirty = true; }
|
|
1609
1643
|
return json(res, 200, { ok: true });
|
|
1610
1644
|
}
|
|
1645
|
+
// USAGE v2: the Claude statusline sidechannel. Claude Code >=2.1.80 pipes rate_limits into
|
|
1646
|
+
// the statusLine command on every turn; hooks/statusline.mjs forwards it here (floored 15s
|
|
1647
|
+
// client-side). The live windows PATCH the cached balances snapshot — free usage between
|
|
1648
|
+
// `trantor balances` runs, and the poller can skip Claude while liveTs is fresh (Orca's
|
|
1649
|
+
// lesson, docs/RESEARCH-orca-usage.md §1.1: the OAuth endpoint 429s under polling).
|
|
1650
|
+
if (req.method === "POST" && P === "/usage/claude") {
|
|
1651
|
+
const b = await body(req);
|
|
1652
|
+
const win = (w, name) => (w && (w.used_percentage ?? w.utilization) != null)
|
|
1653
|
+
? { name, usedPct: Math.round(Number(w.used_percentage ?? w.utilization)), resetsAt: w.resets_at ?? null } : null;
|
|
1654
|
+
const wins = [["fiveHour", "5h"], ["sevenDay", "7d"], ["fable", "Fable"]]
|
|
1655
|
+
.map(([k, n]) => win(b[k], n)).filter(Boolean);
|
|
1656
|
+
if (!wins.length) return json(res, 400, { error: "no usable windows" });
|
|
1657
|
+
state.balances ||= { ts: 0, by: "", entries: [] };
|
|
1658
|
+
let e = state.balances.entries.find(x => x.provider === "claude");
|
|
1659
|
+
if (!e) { e = { provider: "claude", label: "Claude", kind: "windows", ok: true, windows: [] }; state.balances.entries.push(e); }
|
|
1660
|
+
// Same-value posts inside 30s are dropped (the statusline ticks ~3x/sec while streaming).
|
|
1661
|
+
const sig = JSON.stringify(wins);
|
|
1662
|
+
if (e._liveSig === sig && now() - (e.liveTs || 0) < 30_000) return json(res, 200, { ok: true, deduped: true });
|
|
1663
|
+
for (const w of wins) { const cur = (e.windows ||= []).find(x => x.name === w.name); if (cur) Object.assign(cur, w); else e.windows.push(w); }
|
|
1664
|
+
e.ok = true; e.liveTs = now(); e._liveSig = sig; e.liveSource = "statusline";
|
|
1665
|
+
dirty = true;
|
|
1666
|
+
return json(res, 200, { ok: true, windows: wins.length });
|
|
1667
|
+
}
|
|
1611
1668
|
if (req.method === "GET" && P === "/balances") {
|
|
1612
1669
|
let cfg = {}; try { cfg = JSON.parse(readFileSync(join(homedir(), ".agent-bus", "config.json"), "utf8")); } catch {}
|
|
1613
1670
|
const low = { USD: 5, CNY: 35, EUR: 5, ...(cfg.lowBalance || {}) };
|
|
@@ -2646,7 +2703,9 @@ const server = http.createServer(async (req, res) => {
|
|
|
2646
2703
|
if (req.method === "GET" && (P === "/" || P === "/ui")) {
|
|
2647
2704
|
res.writeHead(200, { "content-type": "text/html; charset=utf-8" }); return res.end(UI || "<h1>trantor</h1><p>dashboard unavailable</p>");
|
|
2648
2705
|
}
|
|
2649
|
-
if (P === "/health") return json(res, 200, { ok: true, authMode: AUTH_MODE, peers: Object.keys(state.peers).length, messages: state.messages.length, streams: streams.length
|
|
2706
|
+
if (P === "/health") return json(res, 200, { ok: true, authMode: AUTH_MODE, peers: Object.keys(state.peers).length, messages: state.messages.length, streams: streams.length,
|
|
2707
|
+
// #5686: duty liveness rides /health so the app's Home strip and doctor read one truth.
|
|
2708
|
+
duty: { ...dutyLiveness(), darkSinceMs: dutyDarkSince ? now() - dutyDarkSince : 0, queuedEscalations: dutyQueuedEscalations() } });
|
|
2650
2709
|
json(res, 404, { error: "not found" });
|
|
2651
2710
|
} catch (e) { json(res, 500, { error: String(e?.message || e) }); }
|
|
2652
2711
|
});
|
package/lib/balances.mjs
CHANGED
|
@@ -11,6 +11,10 @@
|
|
|
11
11
|
// ARCHITECTURE NOTE: the hub runs under launchd with a minimal env (no keys), so it can't fetch these
|
|
12
12
|
// itself. The env-having clients fetch + POST /balances; the hub caches + serves the dashboard.
|
|
13
13
|
|
|
14
|
+
import { readFileSync } from "node:fs";
|
|
15
|
+
import { join } from "node:path";
|
|
16
|
+
import { homedir } from "node:os";
|
|
17
|
+
|
|
14
18
|
const TIMEOUT = 8000;
|
|
15
19
|
|
|
16
20
|
async function getJSON(url, key, extraHeaders = {}) {
|
|
@@ -40,13 +44,40 @@ export const ADAPTERS = [
|
|
|
40
44
|
provider: "claude", label: "Claude", kind: "windows", match: ["claude", "anthropic"], envKeys: [],
|
|
41
45
|
keyless: true,
|
|
42
46
|
async fetch() {
|
|
47
|
+
// USAGE v2: the statusline sidechannel (hooks/statusline.mjs) keeps a live local cache —
|
|
48
|
+
// Claude Code pipes rate_limits to the statusLine every turn, free. When the OAuth
|
|
49
|
+
// endpoint fails (it 429s under polling — Orca's lesson, docs/RESEARCH-orca-usage.md
|
|
50
|
+
// §1.1), a fresh live capture answers instead of an error row. OAuth stays primary
|
|
51
|
+
// because only it carries the model-scoped (Fable) window.
|
|
52
|
+
const live = (() => {
|
|
53
|
+
try {
|
|
54
|
+
const l = JSON.parse(readFileSync(join(homedir(), ".agent-bus", "usage-claude-live.json"), "utf8"));
|
|
55
|
+
return l && Date.now() - (l.ts || 0) < 15 * 60 * 1000 ? l : null;
|
|
56
|
+
} catch { return null; }
|
|
57
|
+
})();
|
|
58
|
+
const liveWin = (w, name) => (w && (w.used_percentage ?? w.utilization) != null)
|
|
59
|
+
? { name, usedPct: Math.round(Number(w.used_percentage ?? w.utilization)), resetsAt: w.resets_at || null, locked: null }
|
|
60
|
+
: null;
|
|
61
|
+
const liveWindows = () => [liveWin(live?.fiveHour, "5h"), liveWin(live?.sevenDay, "7d")].filter(Boolean);
|
|
43
62
|
const tok = await claudeOAuthToken();
|
|
44
|
-
if (!tok)
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
63
|
+
if (!tok) {
|
|
64
|
+
if (live && liveWindows().length) return { windows: liveWindows(), live: true };
|
|
65
|
+
throw new Error("no Claude Code OAuth token found");
|
|
66
|
+
}
|
|
67
|
+
let r;
|
|
68
|
+
try {
|
|
69
|
+
r = await fetch("https://api.anthropic.com/api/oauth/usage", {
|
|
70
|
+
headers: { authorization: `Bearer ${tok}`, "anthropic-beta": "oauth-2025-04-20" },
|
|
71
|
+
signal: AbortSignal.timeout(8000),
|
|
72
|
+
});
|
|
73
|
+
} catch (e) {
|
|
74
|
+
if (live && liveWindows().length) return { windows: liveWindows(), live: true };
|
|
75
|
+
throw e;
|
|
76
|
+
}
|
|
77
|
+
if (!r.ok) {
|
|
78
|
+
if (live && liveWindows().length) return { windows: liveWindows(), live: true };
|
|
79
|
+
throw new Error(`usage endpoint ${r.status}`);
|
|
80
|
+
}
|
|
50
81
|
const d = await r.json();
|
|
51
82
|
const win = (w, name) => (w && w.utilization != null)
|
|
52
83
|
? { name, usedPct: Math.round(w.utilization), resetsAt: w.resets_at || null, locked: w.locked_reason || null }
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "trantor",
|
|
3
|
-
"version": "0.18.
|
|
3
|
+
"version": "0.18.21",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"bin": {
|
|
6
6
|
"trantor": "bin/cli.mjs"
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"zod": "^4.4.3"
|
|
12
12
|
},
|
|
13
13
|
"scripts": {
|
|
14
|
-
"test": "node bin/slop-gate.mjs --surface desktop/src && node test.mjs && node test-scenarios.mjs && node test-failure.mjs && node test-redelivery.mjs && node test-crew-completion.mjs && node test-contracts.mjs && node test-autonomy.mjs && node test-integrate.mjs && node test-handoff.mjs && node test-handoff-summarizer.mjs && node test-baton-turn-boundary.mjs && node test-agents.mjs && node test-update.mjs && node test-handoff-guard.mjs && node test-baton-latest.mjs && node test-balances.mjs && node test-subagent-cost.mjs && node test-inbox.mjs && node test-inflight.mjs && node test-notify.mjs && node test-focus.mjs && node test-cardlog.mjs && node test-checklist.mjs && node test-relay-note.mjs && node test-reaper.mjs && node test-events.mjs && node test-proposals.mjs && node test-scrub.mjs && node test-store-delta.mjs && node test-message-re.mjs && node test-claims.mjs && node test-adopt.mjs && node test-summarize.mjs && node test-identity-core.mjs && node test-identity.mjs && node test-identity-instances.mjs && node test-duty.mjs && node test-duty-seat.mjs && node test-discovery.mjs && node test-doctor.mjs && node test-crew-env.mjs && node test-desktop-transport.mjs && node test-crew-worktree.mjs && bash test-crew-herdr.sh && npm --prefix desktop run test --silent && node test-splitbrain.mjs && node test-overseer.mjs && node test-overseer-lib.mjs && node test-overseer-warn.mjs && node test-provider-keys.mjs && node test-inbox-delivery.mjs && node test-identity-drift.mjs && node test-inbox-staleness.mjs && node test-seats.mjs && node test-seat-identity.mjs && node test-hub-routing.mjs && node test-hook-routing.mjs && node test-relay-wait.mjs && node test-dsh-seat.mjs && node test-kimi-bridge.mjs && node test-kimi-events.mjs && python3 engine/test-routing.py && node test-reconcile.mjs && node test-resources.mjs && node test-patrol.mjs && node test-bridge.mjs && bash test-crew.sh"
|
|
14
|
+
"test": "node bin/slop-gate.mjs --surface desktop/src && node test.mjs && node test-scenarios.mjs && node test-failure.mjs && node test-redelivery.mjs && node test-crew-completion.mjs && node test-contracts.mjs && node test-autonomy.mjs && node test-integrate.mjs && node test-handoff.mjs && node test-handoff-summarizer.mjs && node test-baton-turn-boundary.mjs && node test-agents.mjs && node test-update.mjs && node test-handoff-guard.mjs && node test-baton-latest.mjs && node test-balances.mjs && node test-usage-live.mjs && node test-subagent-cost.mjs && node test-inbox.mjs && node test-inflight.mjs && node test-notify.mjs && node test-focus.mjs && node test-cardlog.mjs && node test-checklist.mjs && node test-relay-note.mjs && node test-reaper.mjs && node test-events.mjs && node test-proposals.mjs && node test-scrub.mjs && node test-store-delta.mjs && node test-message-re.mjs && node test-claims.mjs && node test-adopt.mjs && node test-summarize.mjs && node test-identity-core.mjs && node test-identity.mjs && node test-identity-instances.mjs && node test-duty.mjs && node test-duty-dark.mjs && node test-duty-seat.mjs && node test-discovery.mjs && node test-doctor.mjs && node test-crew-env.mjs && node test-desktop-transport.mjs && node test-crew-worktree.mjs && bash test-crew-herdr.sh && npm --prefix desktop run test --silent && node test-splitbrain.mjs && node test-overseer.mjs && node test-overseer-lib.mjs && node test-overseer-warn.mjs && node test-provider-keys.mjs && node test-inbox-delivery.mjs && node test-identity-drift.mjs && node test-inbox-staleness.mjs && node test-seats.mjs && node test-seat-identity.mjs && node test-hub-routing.mjs && node test-hook-routing.mjs && node test-relay-wait.mjs && node test-dsh-seat.mjs && node test-kimi-bridge.mjs && node test-kimi-events.mjs && python3 engine/test-routing.py && node test-reconcile.mjs && node test-resources.mjs && node test-patrol.mjs && node test-bridge.mjs && bash test-crew.sh"
|
|
15
15
|
},
|
|
16
16
|
"description": "The hub-world for AI agent crews \u2014 orchestrate Claude Code, Codex, GLM, Kimi, DeepSeek & any OpenRouter model as live crews with a plan-aware Advisor, a Kanban/flow command center, a testing gate, and an economics brain (Scrooge).",
|
|
17
17
|
"files": [
|