trantor 0.18.82 → 0.18.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/bin/cli.mjs +2 -0
- package/bin/crew/state.mjs +17 -2
- package/bin/crew-runner.mjs +142 -31
- package/bin/cut-snapshot.mjs +37 -0
- package/bin/gate.mjs +30 -0
- package/bin/turn-watchdog.mjs +14 -9
- package/bin/wake-nudge.mjs +21 -2
- package/hooks/file-claim.mjs +19 -7
- package/hooks/file-hold.mjs +49 -0
- package/hooks/heartbeat.mjs +2 -1
- package/hooks/hooks.json +10 -1
- package/hooks/lib/git-checkout.mjs +10 -0
- package/hooks/lib/overseer-level-cache.mjs +25 -0
- package/hooks/overseer-warn.mjs +1 -1
- package/hub/auth.mjs +6 -51
- package/hub/overseer.mjs +100 -34
- package/hub/routes/admin.mjs +58 -12
- package/hub/store.mjs +3 -2
- package/lib/duty-nudges.mjs +12 -0
- package/lib/nudge-ledger.mjs +95 -0
- package/lib/overseer.mjs +82 -24
- package/lib/remote-overlap.mjs +193 -0
- package/lib/seat-why.mjs +6 -1
- package/lib/session-liveness.mjs +81 -0
- package/lib/store-pg.mjs +2 -0
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "trantor",
|
|
3
|
-
"version": "0.18.
|
|
3
|
+
"version": "0.18.85",
|
|
4
4
|
"description": "Trantor — the hub-world for AI agent crews: live message bus, presence, project Kanban/flow board + crew orchestration for independent AI coding agents (Claude, Codex, Gemini, Kimi, DeepSeek)",
|
|
5
5
|
"mcpServers": {
|
|
6
6
|
"relay": {
|
package/bin/cli.mjs
CHANGED
|
@@ -103,6 +103,7 @@ switch (cmd) {
|
|
|
103
103
|
case "watch": run("bin/relay-watch.mjs"); break;
|
|
104
104
|
case "catchup": run("bin/catchup.mjs"); break;
|
|
105
105
|
case "agents": run("bin/agents.mjs"); break;
|
|
106
|
+
case "gate": run("bin/gate.mjs"); break;
|
|
106
107
|
case "gates": run("bin/gates.mjs"); break;
|
|
107
108
|
case "backfill": run("bin/git-backfill.mjs"); break;
|
|
108
109
|
case "sweep": run("bin/sweep.mjs"); break;
|
|
@@ -250,6 +251,7 @@ switch (cmd) {
|
|
|
250
251
|
trantor app the DESKTOP app: status | install | update — pulls the latest DMG from GitHub Releases
|
|
251
252
|
trantor catchup "where are we?" — the continuous board + git, with a synthesized brief
|
|
252
253
|
trantor agents what this session's sub-agents did (task · returned? · files written · survived on disk) — [<sessionId>] [--json]
|
|
254
|
+
trantor gate go|nogo <id> decide a conflicting file edit
|
|
253
255
|
trantor gates verification gates: "must verify before shipping" claims that survive handoffs — [--all] [--json]
|
|
254
256
|
trantor backfill card past GIT work onto the board (solo commits that were never carded) — [--since "14 days ago"] [--dry-run]
|
|
255
257
|
trantor init-hooks install a git post-commit hook so EVERY commit auto-cards on the board (reliable solo-work backstop) — [--uninstall]
|
package/bin/crew/state.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { existsSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
1
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
2
3
|
import { seatProcesses, signalProcesses } from "./processes.mjs";
|
|
3
4
|
import { call, listPids, run } from "./core.mjs";
|
|
4
5
|
import { hostId } from "../../lib/project.mjs";
|
|
@@ -142,7 +143,21 @@ export async function down(ctx, args, adapters) {
|
|
|
142
143
|
const removed = new Set(scoped.map(row => `${row.project}|${row.kind}|${row.agent}|${row.handle}`));
|
|
143
144
|
const kept = rows.filter(row => !removed.has(`${row.project}|${row.kind}|${row.agent}|${row.handle}`) || (orchProjects.has(row.project) && ["herdrws", "cmuxws", "orch"].includes(row.kind)));
|
|
144
145
|
writeRows(ctx, kept);
|
|
145
|
-
|
|
146
|
+
const seatRows = scoped.filter(row => !["attach", "cmuxws", "herdrws", "orch"].includes(row.kind));
|
|
147
|
+
if (!ctx.dry) {
|
|
148
|
+
const logDir = join(dirname(ctx.statePath), "logs");
|
|
149
|
+
mkdirSync(logDir, { recursive: true });
|
|
150
|
+
for (const row of new Map(seatRows.map(row => [`${row.project}/${row.agent}`, row])).values()) {
|
|
151
|
+
appendFileSync(join(logDir, `${row.agent}-${row.project}.jsonl`),
|
|
152
|
+
`${JSON.stringify({ ts: Date.now(), agent: row.agent, project: row.project, stopped: true })}\n`);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
if (options.agents.length) {
|
|
156
|
+
const stopped = [...new Set(seatRows.map(row => row.agent))];
|
|
157
|
+
const remaining = [...new Set(kept.filter(row => row.project === ctx.project &&
|
|
158
|
+
!["attach", "cmuxws", "herdrws", "orch"].includes(row.kind)).map(row => row.agent))];
|
|
159
|
+
console.log(`— ${stopped.join(", ") || options.agents.join(", ")} stopped${remaining.length ? ` (${remaining.join(", ")} still running)` : ""} (${scope})`);
|
|
160
|
+
} else console.log(`— crew torn down (${scope})`);
|
|
146
161
|
return 0;
|
|
147
162
|
}
|
|
148
163
|
|
package/bin/crew-runner.mjs
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// trantor crew runner — keeps a crew agent alive without burning tokens: it long-polls the bus
|
|
3
3
|
// (zero tokens) and resumes the CLI with each message. Usage: node crew-runner.mjs <agent> [dir]
|
|
4
4
|
import { execSync, spawnSync, spawn } from "node:child_process";
|
|
5
|
-
import { readFileSync, writeFileSync, unlinkSync, existsSync, appendFileSync, mkdirSync, realpathSync } from "node:fs";
|
|
5
|
+
import { readFileSync, writeFileSync, unlinkSync, existsSync, appendFileSync, mkdirSync, realpathSync, statSync } from "node:fs";
|
|
6
6
|
import { join, basename } from "node:path";
|
|
7
7
|
import { homedir } from "node:os";
|
|
8
8
|
import { resolveProject, resolveHub, withEnvFiles, hostId } from "../lib/project.mjs";
|
|
@@ -26,12 +26,14 @@ import {
|
|
|
26
26
|
senderProjectOf, isLinkedProject, stateSkipReason, isMessageCardTitle, OPEN_CARD_STATUSES,
|
|
27
27
|
CUT_CHAIN_PARK_MIN, cutChainEvidence, isBoundedPark, askDuringTurn, askedVerdict,
|
|
28
28
|
} from "../lib/turn-policy.mjs";
|
|
29
|
+
import { snapshotBeforeCut } from "./cut-snapshot.mjs";
|
|
29
30
|
import { ocTurnUsage, dshTurnUsage, usageTotal } from "../lib/turn-usage.mjs";
|
|
30
31
|
import { listContainers, newContainers, recordContainers, statePathFor as dockerStatePathFor, sweep as dockerSweep } from "../lib/docker-janitor.mjs";
|
|
31
32
|
import {
|
|
32
33
|
auditDutyNudges, claimDutyNudges, claudeTranscriptDir, dutyEscalations, dutyNudgeDirective,
|
|
33
|
-
observedDutyNudgeIds, requeueMissingWakeMessages, shedExpiredHubAlerts,
|
|
34
|
+
observedDutyNudgeIds, requeueMissingWakeMessages, releaseDutyNudgeClaims, shedExpiredHubAlerts,
|
|
34
35
|
} from "../lib/duty-nudges.mjs";
|
|
36
|
+
import { reserveNudgeIds, releaseNudgeIds } from "../lib/nudge-ledger.mjs";
|
|
35
37
|
import { dutyRecipientResolver } from "../lib/duty-recipient.mjs";
|
|
36
38
|
import { cliEffortFlag } from "../lib/model-catalog.mjs";
|
|
37
39
|
import {
|
|
@@ -43,7 +45,35 @@ import { resolveStateFlags } from "../lib/state/flags.mjs";
|
|
|
43
45
|
|
|
44
46
|
import { awaitSwapRelease } from "./crew/stage.mjs";
|
|
45
47
|
|
|
46
|
-
|
|
48
|
+
const releaseState = process.env.CREW_RELEASE_STATE ? JSON.parse(process.env.CREW_RELEASE_STATE) : null;
|
|
49
|
+
delete process.env.CREW_RELEASE_STATE;
|
|
50
|
+
const packagePath = new URL("../package.json", import.meta.url);
|
|
51
|
+
function installedVersion() {
|
|
52
|
+
try {
|
|
53
|
+
statSync(packagePath);
|
|
54
|
+
return JSON.parse(readFileSync(packagePath, "utf8")).version;
|
|
55
|
+
} catch { return ""; }
|
|
56
|
+
}
|
|
57
|
+
const bootVersion = installedVersion();
|
|
58
|
+
const attemptedVersions = new Set(releaseState?.attempted || []);
|
|
59
|
+
function newerVersion(next, previous) {
|
|
60
|
+
const parse = value => /^(\d+)\.(\d+)\.(\d+)(?:-([\w.-]+))?(?:\+[\w.-]+)?$/.exec(value || "");
|
|
61
|
+
const a = parse(next), b = parse(previous);
|
|
62
|
+
if (!a || !b) return false;
|
|
63
|
+
for (let i = 1; i <= 3; i++) if (+a[i] !== +b[i]) return +a[i] > +b[i];
|
|
64
|
+
if (!a[4] || !b[4]) return !!b[4] && !a[4];
|
|
65
|
+
const ap = a[4].split("."), bp = b[4].split(".");
|
|
66
|
+
for (let i = 0; i < Math.max(ap.length, bp.length); i++) {
|
|
67
|
+
if (ap[i] === bp[i]) continue;
|
|
68
|
+
if (ap[i] === undefined || bp[i] === undefined) return bp[i] === undefined;
|
|
69
|
+
const an = /^\d+$/.test(ap[i]), bn = /^\d+$/.test(bp[i]);
|
|
70
|
+
if (an && bn) return +ap[i] > +bp[i];
|
|
71
|
+
if (an !== bn) return bn;
|
|
72
|
+
return ap[i] > bp[i];
|
|
73
|
+
}
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
if (!releaseState) await awaitSwapRelease();
|
|
47
77
|
const AGENT = process.env.CREW_SWAP_AGENT || process.argv[2];
|
|
48
78
|
const DIR = process.argv[3] || process.cwd();
|
|
49
79
|
// Crew agents MUST share the orchestrator's project key (one repo = one lane).
|
|
@@ -126,10 +156,10 @@ process.on("unhandledRejection", (e) => { console.log(`\x1b[31m[runner] UNHANDLE
|
|
|
126
156
|
const log = (s) => console.log(`\x1b[38;5;43m[runner]\x1b[0m ${redactKeys(String(s))}`);
|
|
127
157
|
const LOGDIR = join(homedir(), ".agent-bus", "logs");
|
|
128
158
|
try { mkdirSync(LOGDIR, { recursive: true }); } catch {}
|
|
129
|
-
let TURN = 0;
|
|
159
|
+
let TURN = releaseState?.turn || 0;
|
|
130
160
|
const telemetry = (rec) => { try { appendFileSync(join(LOGDIR, `${AGENT}-${PROJ}.jsonl`), JSON.stringify(rec) + "\n"); } catch {} };
|
|
131
161
|
// The boot line records the HUB this runner bound to, so a split-brain is diagnosable from disk.
|
|
132
|
-
telemetry({ ts: Date.now(), agent: AGENT, project: PROJ, boot: true, hub: HUB });
|
|
162
|
+
telemetry({ ts: Date.now(), agent: AGENT, project: PROJ, boot: true, hub: HUB, pid: process.pid, version: bootVersion });
|
|
133
163
|
// A seat can open a terminal window on a machine whose owner never asked for one and does not know
|
|
134
164
|
// what they are looking at. "◤ CLAUDE ◢ trantor crew · fleet" tells that person nothing: not what
|
|
135
165
|
// started, not what it will do, not how to stop it. RUNNER_TITLE names it in full and RUNNER_ABOUT
|
|
@@ -258,7 +288,7 @@ const EFFORT_FLAG = EFFORT ? cliEffortFlag(DRIVER, EFFORT) : { flag: "", text: "
|
|
|
258
288
|
|
|
259
289
|
// RUNNER_RULES / RUNNER_KICKOFF env overrides: the runner is also the substrate for non-crew
|
|
260
290
|
// always-on seats (the fleet DUTY agent, bin/duty.mjs) whose doctrine is not "work your card".
|
|
261
|
-
const RULES = process.env.RUNNER_RULES || `Rules: you are ${SESSION} on the trantor crew. Before starting a card, read YOUR card: relay_board with card:<id> (the card, its deps, its notes, and the last five done cards whose title shares a word); never the whole board. If your wake names no card id (or cites only a message-card), make relay_board with mine:true your FIRST call — it lists the cards assigned to you (doing/testing/todo, newest first, full card shape); work your newest one (#7763). Work your assigned file(s), report on the bus (relay_send, <280 chars), move your Kanban card as you go with a NOTE saying what you did (doing -> testing, then STOP: you never close your own card to done — the orchestrator runs the card's drill on the built artifact and closes it, and the hub refuses a move to done on a card with no drill line anyway; in 'testing' run YOUR OWN test file — never the full npm test, suites collide across seats — plus \`node bin/slop-gate.mjs\` when the repo has one: it lints ONLY your changed files against the anti-slop rules, and a card must not reach testing with slop-gate failing; use 'failed' + a report if anything breaks). If your work starts containers or a dev server (\`supabase start\`, \`docker run\`, a watch process), stop them before moving your card to testing and name them in the note — the runner also stops containers it saw your turn create when the card closes, but the first responsibility is yours (#9778). If a contract omits a fact you cannot proceed without, ASK — never invent the value: relay_ask(<card>, <question>) blocks the card with your question, keeps the turn owed (no park, no failure), and resumes you when the assigner's answer lands; an invented value that reads as reasoned is the worst outcome this crew ships (#7756). If you need something from another session, message THAT SESSION (relay_peers to find its id, relay_send to reach it) — never ask the human to pass it along; carrying messages between agents is the job this bus exists to remove. When your work for THIS message is finished, END YOUR TURN — do NOT park, do NOT loop relay_wait; the runner waits for you and will wake you with the next message. Path discipline: build/test from your worktree root ${TURN_DIR} with absolute paths or --manifest-path/--prefix instead of cd-ing into subdirs, and put anything that must land outside the repo under ${TURN_DIR}/.agent-bus-out/ (gitignored) — never ~/.agent-bus. Realigning your seat branch after the orchestrator harvested your commits is \`trantor sync\` run from your worktree: it reads the harvest receipts and refuses when an unharvested commit would be lost, so never reset or rebase onto main by hand. A contract's \`base: <sha>\` line is the integration head: start your seat branch at that sha (\`git reset --hard <sha>\` on the fresh branch), never at origin/main, which trails the orchestrator's unpushed integration commits; the object is already here as the ref \`main\`. If \`git cat-file -e <sha>\` fails, say \`cannot resolve base <sha>\` on the bus and move the card to blocked instead of reasoning from origin/main. Every testing/done note names the sha you verified against as \`verified at <sha>\`; a note without it is flagged HOLLOW. Cross-project action is a breach: never \`trantor up\` a crew, register a seat, or send a card/contract into a project other than ${PROJ} unless the operator ran \`trantor policy link ${PROJ} <other> --reason "<why>"\` first — the hub, the CLI and this runner all refuse it mechanically, so ask the operator to link the projects instead of routing around the refusal.`;
|
|
291
|
+
const RULES = process.env.RUNNER_RULES || `Rules: you are ${SESSION} on the trantor crew. Before starting a card, read YOUR card: relay_board with card:<id> (the card, its deps, its notes, and the last five done cards whose title shares a word); never the whole board. If your wake names no card id (or cites only a message-card), make relay_board with mine:true your FIRST call — it lists the cards assigned to you (doing/testing/todo, newest first, full card shape); work your newest one (#7763). After long read-only recon, commit a WIP note with findings so the turn leaves evidence. Work your assigned file(s), report on the bus (relay_send, <280 chars), move your Kanban card as you go with a NOTE saying what you did (doing -> testing, then STOP: you never close your own card to done — the orchestrator runs the card's drill on the built artifact and closes it, and the hub refuses a move to done on a card with no drill line anyway; in 'testing' run YOUR OWN test file — never the full npm test, suites collide across seats — plus \`node bin/slop-gate.mjs\` when the repo has one: it lints ONLY your changed files against the anti-slop rules, and a card must not reach testing with slop-gate failing; use 'failed' + a report if anything breaks). If your work starts containers or a dev server (\`supabase start\`, \`docker run\`, a watch process), stop them before moving your card to testing and name them in the note — the runner also stops containers it saw your turn create when the card closes, but the first responsibility is yours (#9778). If a contract omits a fact you cannot proceed without, ASK — never invent the value: relay_ask(<card>, <question>) blocks the card with your question, keeps the turn owed (no park, no failure), and resumes you when the assigner's answer lands; an invented value that reads as reasoned is the worst outcome this crew ships (#7756). If you need something from another session, message THAT SESSION (relay_peers to find its id, relay_send to reach it) — never ask the human to pass it along; carrying messages between agents is the job this bus exists to remove. When your work for THIS message is finished, END YOUR TURN — do NOT park, do NOT loop relay_wait; the runner waits for you and will wake you with the next message. Path discipline: build/test from your worktree root ${TURN_DIR} with absolute paths or --manifest-path/--prefix instead of cd-ing into subdirs, and put anything that must land outside the repo under ${TURN_DIR}/.agent-bus-out/ (gitignored) — never ~/.agent-bus. Realigning your seat branch after the orchestrator harvested your commits is \`trantor sync\` run from your worktree: it reads the harvest receipts and refuses when an unharvested commit would be lost, so never reset or rebase onto main by hand. A contract's \`base: <sha>\` line is the integration head: start your seat branch at that sha (\`git reset --hard <sha>\` on the fresh branch), never at origin/main, which trails the orchestrator's unpushed integration commits; the object is already here as the ref \`main\`. If \`git cat-file -e <sha>\` fails, say \`cannot resolve base <sha>\` on the bus and move the card to blocked instead of reasoning from origin/main. Every testing/done note names the sha you verified against as \`verified at <sha>\`; a note without it is flagged HOLLOW. Cross-project action is a breach: never \`trantor up\` a crew, register a seat, or send a card/contract into a project other than ${PROJ} unless the operator ran \`trantor policy link ${PROJ} <other> --reason "<why>"\` first — the hub, the CLI and this runner all refuse it mechanically, so ask the operator to link the projects instead of routing around the refusal.`;
|
|
262
292
|
|
|
263
293
|
// ---- the pulse --------------------------------------------------------------
|
|
264
294
|
// RUNNER_PULSE_MS re-runs an orchestrator seat's mission note on a cadence when the bus is silent;
|
|
@@ -713,9 +743,9 @@ let cutChain = [];
|
|
|
713
743
|
const PARK_WINDOW_MS = Math.max(0, Number(process.env.TRANTOR_PARK_WINDOW_MS || 15 * 60 * 1000));
|
|
714
744
|
// The card the CURRENT CLI session belongs to (#6134). 0 = the kickoff session, which belongs to
|
|
715
745
|
// no card, so the first contract that names one starts a session of its own.
|
|
716
|
-
let sessionCard = 0;
|
|
746
|
+
let sessionCard = releaseState?.sessionCard || 0;
|
|
717
747
|
|
|
718
|
-
let sid = "";
|
|
748
|
+
let sid = releaseState?.sid || "";
|
|
719
749
|
// #6206: the watchdog is DETACHED, so a runner that dies without ending it leaves an orphan
|
|
720
750
|
// sleeping toward a false alarm against whatever runner comes next.
|
|
721
751
|
// Every exit path therefore kills it, and the stamp carries this runner's instance id so any
|
|
@@ -755,11 +785,12 @@ async function runTurn(prompt, isFirst, trigger = "kickoff", opts = {}) {
|
|
|
755
785
|
// #10197: the card status snapshot the turn is judged against at its end — a move to
|
|
756
786
|
// testing/done under the turn is delivery even without a commit. null = the hub read failed:
|
|
757
787
|
// the delivery check stands down rather than claim no-delivery on a transport error.
|
|
758
|
-
let cardStatusBefore = null;
|
|
788
|
+
let cardStatusBefore = null, cardLogBefore = [];
|
|
759
789
|
if (!opts.state && sessionCard > 0) {
|
|
760
790
|
try {
|
|
761
791
|
const { task } = await api(`/card?project=${encodeURIComponent(PROJ)}&id=${sessionCard}`);
|
|
762
792
|
cardStatusBefore = task ? String(task.status || "") : null;
|
|
793
|
+
cardLogBefore = task?.log || [];
|
|
763
794
|
} catch { cardStatusBefore = null; }
|
|
764
795
|
}
|
|
765
796
|
// #9778: the docker snapshot the turn is judged against — containers the CLI spawns are recorded
|
|
@@ -833,7 +864,11 @@ async function runTurn(prompt, isFirst, trigger = "kickoff", opts = {}) {
|
|
|
833
864
|
const DRAINF = join(homedir(), ".agent-bus", `turndrain-${AGENT}-${PROJ}`);
|
|
834
865
|
try { unlinkSync(DRAINF); } catch {}
|
|
835
866
|
try {
|
|
836
|
-
|
|
867
|
+
const activity = cli.pinned || ["kimi", "claude"].includes(AGENT) ? {
|
|
868
|
+
kind: cli.pinned ? "opencode" : AGENT, sid, db: OC_DB, workDir: TURN_DIR,
|
|
869
|
+
home: homedir(), startedAt: t0, transcriptDir: TRANSCRIPT_DIR,
|
|
870
|
+
} : undefined;
|
|
871
|
+
writeFileSync(STAMPF, JSON.stringify({ turn: TURN, startedAt: Date.now(), runner: RUNNER_ID, box: boxPlan, activity }));
|
|
837
872
|
const wd = spawn(process.execPath, [join(import.meta.dirname, "turn-watchdog.mjs"), STAMPF, ERRF, String(WD_MS), SESSION, PROJ, HUB, TRANSCRIPT_DIR, TURN_DIR,
|
|
838
873
|
// #7752: the stall marker only exists when a box does — with no box there is no sweep to
|
|
839
874
|
// end the turn, so the watchdog stays report-only (reporting is the whole job, boxless).
|
|
@@ -851,6 +886,8 @@ async function runTurn(prompt, isFirst, trigger = "kickoff", opts = {}) {
|
|
|
851
886
|
// #7752: the box POLLS instead of one long sleep — the watchdog's stall marker (a silent
|
|
852
887
|
// turn) breaks the loop at the stall window, the deadline breaks it at the full box, and only
|
|
853
888
|
// the deadline path writes CUTF so the runner can tell the two cuts apart.
|
|
889
|
+
const shellArg = value => "'" + String(value).replaceAll("'", "'\"'\"'") + "'";
|
|
890
|
+
const snapshotCmd = [process.execPath, join(import.meta.dirname, "cut-snapshot.mjs"), TURN_DIR].map(shellArg).join(" ");
|
|
854
891
|
const box = TURN_MAX_MS ? `
|
|
855
892
|
${sweep}
|
|
856
893
|
( trap 'kill "$sleeppid" 2>/dev/null; wait "$sleeppid" 2>/dev/null; exit 0' TERM
|
|
@@ -863,12 +900,25 @@ ${sweep}
|
|
|
863
900
|
wait "$sleeppid"
|
|
864
901
|
done
|
|
865
902
|
kill -0 $job 2>/dev/null || exit 0
|
|
903
|
+
reason=stall
|
|
904
|
+
[ -f "${STALLF}" ] || reason=box
|
|
905
|
+
${snapshotCmd} "$reason" 2>> ${ERRF}
|
|
866
906
|
[ -f "${STALLF}" ] || : > ${CUTF}
|
|
867
907
|
sweep $job
|
|
868
908
|
) >/dev/null 2>&1 & boxpid=$!` : "\nboxpid=";
|
|
909
|
+
const backstop = TURN_MAX_MS ? `
|
|
910
|
+
${sweep}
|
|
911
|
+
backstop() {
|
|
912
|
+
[ -n "$boxpid" ] && kill $boxpid 2>/dev/null
|
|
913
|
+
${snapshotCmd} box 2>> ${ERRF}
|
|
914
|
+
: > ${CUTF}
|
|
915
|
+
sweep $job
|
|
916
|
+
exit 137
|
|
917
|
+
}
|
|
918
|
+
trap backstop TERM` : "";
|
|
869
919
|
const shell = `set -o pipefail
|
|
870
920
|
{ ${inner} ; } 2> >(${SCRUB} --tee2 ${ERRF}; : >> "${DRAINF}") &
|
|
871
|
-
job=$!${box}
|
|
921
|
+
job=$!${box}${backstop}
|
|
872
922
|
wait $job; turn_exit=$?
|
|
873
923
|
[ -n "$boxpid" ] && kill $boxpid 2>/dev/null
|
|
874
924
|
wait
|
|
@@ -890,10 +940,9 @@ exit $turn_exit`;
|
|
|
890
940
|
TRANTOR_TRANSCRIPT_DIR: TRANSCRIPT_DIR },
|
|
891
941
|
maxBuffer: 16 * 1024 * 1024,
|
|
892
942
|
};
|
|
893
|
-
//
|
|
894
|
-
//
|
|
895
|
-
|
|
896
|
-
if (TURN_MAX_MS) { spawnOpts.timeout = (TURN_EXTENSIONS_MAX ? TURN_CEILING_MS : TURN_MAX_MS) + 30000; spawnOpts.killSignal = "SIGKILL"; }
|
|
943
|
+
// The timeout signals the shell to snapshot before sweeping, just like the normal box.
|
|
944
|
+
// Its 30s margin keeps it out of the normal cut path.
|
|
945
|
+
if (TURN_MAX_MS) { spawnOpts.timeout = (TURN_EXTENSIONS_MAX ? TURN_CEILING_MS : TURN_MAX_MS) + 30000; spawnOpts.killSignal = "SIGTERM"; }
|
|
897
946
|
const r = spawnSync("/bin/bash", ["-c", shell], spawnOpts);
|
|
898
947
|
// The shell's box leaves the marker; the backstop leaves an ETIMEDOUT. Either way the turn was
|
|
899
948
|
// cut, not merely failed. A stall marker (#7752) outranks the box marker: a turn silent for
|
|
@@ -918,6 +967,8 @@ exit $turn_exit`;
|
|
|
918
967
|
killWatchdog(); // #6206: turn over — the watchdog dies NOW, it does not sleep on
|
|
919
968
|
try { unlinkSync(STAMPF); } catch {} // disarm any survivor: the stamp is gone
|
|
920
969
|
if (cut) {
|
|
970
|
+
const saved = snapshotBeforeCut(TURN_DIR, stallCut ? "stall" : "box");
|
|
971
|
+
if (!saved.ok) log(`cut WIP snapshot failed: ${saved.error}`);
|
|
921
972
|
// Belt and braces after the shell's descendant sweep: anything still sharing the turn's group.
|
|
922
973
|
if (r.pid) { try { process.kill(-r.pid, "SIGKILL"); } catch {} }
|
|
923
974
|
try { unlinkSync(CUTF); } catch {}
|
|
@@ -1041,17 +1092,24 @@ exit $turn_exit`;
|
|
|
1041
1092
|
// turn started, or the card moved to testing/done under it. The ibkr shape (exit 0, output, no
|
|
1042
1093
|
// commit, card still todo) is NO-DELIVERY: the ledger names it and the contract stays owed.
|
|
1043
1094
|
// Judged on hub reads of the card status; a failed read keeps today's label, never no-delivery.
|
|
1044
|
-
let cardStatusAtEnd = "", noDelivery = false;
|
|
1095
|
+
let cardStatusAtEnd = "", noDelivery = false, blockedVerdict = "";
|
|
1045
1096
|
if (!cut && !opts.state && realExit === 0 && effExit === 0 && sessionCard > 0 && cardStatusBefore !== null) {
|
|
1046
1097
|
let endStatus = null;
|
|
1047
1098
|
try {
|
|
1048
1099
|
const { task } = await api(`/card?project=${encodeURIComponent(PROJ)}&id=${sessionCard}`);
|
|
1049
1100
|
endStatus = task ? String(task.status || "") : null;
|
|
1101
|
+
const freshNote = (task?.log || []).filter(n => n.by === SESSION && n.ts >= t0
|
|
1102
|
+
&& !cardLogBefore.some(old => old.ts === n.ts && old.by === n.by && old.text === n.text)).at(-1);
|
|
1103
|
+
if (endStatus === "blocked" && (cardStatusBefore !== "blocked" || freshNote)) {
|
|
1104
|
+
const note = freshNote || (task?.log || []).at(-1);
|
|
1105
|
+
blockedVerdict = `blocked: ${String(note?.text || "").split(/\r?\n/)[0]}`;
|
|
1106
|
+
lastEmptyTurn = false;
|
|
1107
|
+
}
|
|
1050
1108
|
} catch { endStatus = null; }
|
|
1051
1109
|
if (endStatus !== null) {
|
|
1052
1110
|
cardStatusAtEnd = endStatus;
|
|
1053
1111
|
const moved = ["testing", "done"].includes(endStatus) && !["testing", "done"].includes(cardStatusBefore);
|
|
1054
|
-
noDelivery = !newCommit && !moved && !askVerdict;
|
|
1112
|
+
noDelivery = !newCommit && !moved && !askVerdict && !blockedVerdict;
|
|
1055
1113
|
}
|
|
1056
1114
|
}
|
|
1057
1115
|
lastNoDelivery = noDelivery;
|
|
@@ -1063,6 +1121,7 @@ exit $turn_exit`;
|
|
|
1063
1121
|
// reads as the sweep's SIGKILL (#7099); a zero-usage turn names its own verdict too (#9724).
|
|
1064
1122
|
const verdict = stallCut ? stallVerdict() : zeroUsage ? zeroUsageVerdict()
|
|
1065
1123
|
: askVerdict ? askVerdict
|
|
1124
|
+
: blockedVerdict ? blockedVerdict
|
|
1066
1125
|
: noDelivery ? noDeliveryVerdict(sessionCard, cardStatusAtEnd || "unknown")
|
|
1067
1126
|
: verdictFor(realExit, effExit, lastEmptyOutput, ownOut, lastEmptyTurn, cut);
|
|
1068
1127
|
// #6289: every ledger row names in ONE field what happened to the turn — cut, stalled (#7752),
|
|
@@ -1070,7 +1129,7 @@ exit $turn_exit`;
|
|
|
1070
1129
|
// binds the row to the card the turn worked (0 = kickoff/pulse) so the seat record attributes
|
|
1071
1130
|
// empty/stalled turns to the card that produced nothing. `cut` stays too: the drills read it.
|
|
1072
1131
|
const outcome = cut ? (stallCut ? "stalled" : "cut") : (effExit !== 0 ? "api-error" : noDelivery ? "no-delivery" : lastEmptyTurn ? "empty" : "completed");
|
|
1073
|
-
const finalOutcome = askVerdict ? "asked" : outcome;
|
|
1132
|
+
const finalOutcome = askVerdict ? "asked" : blockedVerdict ? "blocked" : outcome;
|
|
1074
1133
|
const telemetryRow = { ts: Date.now(), agent: AGENT, project: PROJ, turn: TURN, trigger, card: sessionCard || 0, model: MODEL || "cli-default", duration_ms: Date.now() - t0, exit: realExit, effExit, authFailed: effExit !== realExit, emptyOutput: lastEmptyOutput, emptyTurn: lastEmptyTurn, verdict, outcome: finalOutcome, tokens };
|
|
1075
1134
|
if (usage) telemetryRow.usage = usage;
|
|
1076
1135
|
if (dockerNew.length) telemetryRow.containers = dockerNew.map(c => c.name || c.id);
|
|
@@ -1311,7 +1370,8 @@ async function resolveWakeCard(messages, { session }) {
|
|
|
1311
1370
|
await loadLessons();
|
|
1312
1371
|
// start cursor at the CURRENT tip so we don't replay history
|
|
1313
1372
|
let cursor = 0;
|
|
1314
|
-
if (
|
|
1373
|
+
if (releaseState) cursor = releaseState.cursor;
|
|
1374
|
+
else if (process.env.CREW_SWAP_STAGE) cursor = JSON.parse(readFileSync(`${process.env.CREW_SWAP_STAGE}.release`, "utf8")).cursor;
|
|
1315
1375
|
else try { const r = await api(`/inbox?session=${encodeURIComponent(SESSION)}&since=0`); cursor = r.cursor || 0; } catch {}
|
|
1316
1376
|
// kind "agent" on every beat (#6075): the peer row's kind is the hub's OWN record of what a
|
|
1317
1377
|
// session is — the overseer's declared-crew exemption reads it, and on the remote hub there is
|
|
@@ -1337,7 +1397,7 @@ async function resolveWakeCard(messages, { session }) {
|
|
|
1337
1397
|
// #7914: a park belongs to the runner that made it. This process is starting fresh and will
|
|
1338
1398
|
// re-park if the CLI is still cutting, so a record left by the runner before it is stale state —
|
|
1339
1399
|
// and stale state in a health read is the same failure as no state at all.
|
|
1340
|
-
try { unlinkSync(PARKF); } catch {}
|
|
1400
|
+
if (!releaseState) { try { unlinkSync(PARKF); } catch {} }
|
|
1341
1401
|
seenLedger = restored.seen;
|
|
1342
1402
|
if (seenLedger.length) log(`\x1b[33m${seenLedger.length} consumed message id(s) restored from the pending file — the poll keeps standing down on them\x1b[0m`);
|
|
1343
1403
|
if (restored.ask) {
|
|
@@ -1357,25 +1417,54 @@ async function resolveWakeCard(messages, { session }) {
|
|
|
1357
1417
|
// disagreeing makes a health check lie.
|
|
1358
1418
|
savePending(pendingWake, pendingBcast);
|
|
1359
1419
|
}
|
|
1360
|
-
let retryAt = 0; // 0 = deliver at the next opportunity
|
|
1361
|
-
let deliveryFails = 0; // consecutive failed attempts at the SAME pending batch
|
|
1362
|
-
let noDeliveryRetry = false;
|
|
1420
|
+
let retryAt = releaseState?.retryAt || 0; // 0 = deliver at the next opportunity
|
|
1421
|
+
let deliveryFails = releaseState?.deliveryFails || 0; // consecutive failed attempts at the SAME pending batch
|
|
1422
|
+
let noDeliveryRetry = releaseState?.noDeliveryRetry || false;
|
|
1423
|
+
askReleased = releaseState?.askReleased || false;
|
|
1363
1424
|
if (pendingWake.length) log(`\x1b[33m${pendingWake.length} message(s) survived from a previous run — redelivering\x1b[0m`);
|
|
1364
1425
|
|
|
1365
1426
|
// #7060: the turn the boot line was read as a promise about. It is a transcript turn by
|
|
1366
1427
|
// construction and now says so, in the same breath as the line that armed the mode.
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1428
|
+
if (!releaseState) {
|
|
1429
|
+
stateSkip("kickoff");
|
|
1430
|
+
const ec0 = await runTurn(composedTurn({ base: KICKOFF, lessons: pickLessons(LESSONS_RAW, "") }), true, "kickoff");
|
|
1431
|
+
if (ec0) await reportFailure(ec0, "kickoff", pendingWake.length); // a failed kickoff = the "fired up, died, nobody knew" case
|
|
1432
|
+
// #7778: the kickoff is a turn like any other — a message that landed mid-kickoff and was read
|
|
1433
|
+
// through the session's own inbox is consumed by a successful kickoff, and the first poll must
|
|
1434
|
+
// not wake on it. Same reconcile, same clean-boundary rule as deliverWake below.
|
|
1435
|
+
else await reconcileSessionReads(cursor);
|
|
1436
|
+
} else {
|
|
1437
|
+
const text = `runner restarted onto ${bootVersion}`;
|
|
1438
|
+
log(text);
|
|
1439
|
+
telemetry({ ts: Date.now(), text, version: bootVersion });
|
|
1440
|
+
await api("/send", { from: SESSION, to: "all", project: PROJ, kind: "status", text }).catch(() => {});
|
|
1441
|
+
}
|
|
1442
|
+
let lastTurnAt = releaseState?.lastTurnAt || Date.now();
|
|
1443
|
+
function restartForRelease() {
|
|
1444
|
+
const version = installedVersion();
|
|
1445
|
+
if (!newerVersion(version, bootVersion) || attemptedVersions.has(version)) return;
|
|
1446
|
+
attemptedVersions.add(version);
|
|
1447
|
+
// SAFETY: older supported Node releases lack in-place exec; never replace it with a duplicate runner.
|
|
1448
|
+
// oxlint-disable-next-line anti-slop/no-runtime-typeof
|
|
1449
|
+
if (typeof process.execve !== "function") {
|
|
1450
|
+
log(`runner upgrade to ${version} requires Node with process.execve; restart manually`);
|
|
1451
|
+
return;
|
|
1452
|
+
}
|
|
1453
|
+
try {
|
|
1454
|
+
// Persist before exec: a held contract and a released ask both survive the boundary.
|
|
1455
|
+
writeFileSync(PENDF, JSON.stringify({ agent: AGENT, project: PROJ, ts: Date.now(),
|
|
1456
|
+
wake: pendingWake, bcast: pendingBcast, seen: seenLedger, ask: awaitingAsk?.id || 0, askTo: awaitingAsk?.to || "" }));
|
|
1457
|
+
const state = { cursor, turn: TURN, sid, sessionCard, retryAt, deliveryFails,
|
|
1458
|
+
noDeliveryRetry, askReleased, lastTurnAt, attempted: [...attemptedVersions] };
|
|
1459
|
+
process.execve(process.execPath, [process.execPath, ...process.execArgv, ...process.argv.slice(1)],
|
|
1460
|
+
{ ...process.env, CREW_RELEASE_STATE: JSON.stringify(state) });
|
|
1461
|
+
} catch (error) { log(`runner restart failed: ${error.message}`); }
|
|
1462
|
+
}
|
|
1375
1463
|
if (PULSE_MS) log(`pulse armed — mission re-read every ${Math.round(PULSE_MS / 1000)}s (${MISSION_FILE})`);
|
|
1376
1464
|
log(`parked — long-polling the bus as ${SESSION} (free; this poll is also the heartbeat)`);
|
|
1377
1465
|
|
|
1378
1466
|
while (true) {
|
|
1467
|
+
restartForRelease();
|
|
1379
1468
|
// pulse first: a due mission beat runs even on a silent bus. Measured from the END of the
|
|
1380
1469
|
// last turn, so a long turn doesn't stack an immediate pulse on top of itself.
|
|
1381
1470
|
if (PULSE_MS && Date.now() - lastTurnAt >= PULSE_MS) {
|
|
@@ -1545,6 +1634,7 @@ async function resolveWakeCard(messages, { session }) {
|
|
|
1545
1634
|
} catch { return 0; }
|
|
1546
1635
|
}
|
|
1547
1636
|
async function deliverWake() {
|
|
1637
|
+
restartForRelease();
|
|
1548
1638
|
noDeliveryRetry = false;
|
|
1549
1639
|
// #7756: captured and cleared HERE so an early return below never leaks the flag into a
|
|
1550
1640
|
// later turn — only the turn the answer released reads it.
|
|
@@ -1570,6 +1660,22 @@ async function resolveWakeCard(messages, { session }) {
|
|
|
1570
1660
|
},
|
|
1571
1661
|
})
|
|
1572
1662
|
: { items: [], targets: [], owner: "" };
|
|
1663
|
+
// #11355: the shared pre-send ledger — the wake daemon stamps the same files, so of the two
|
|
1664
|
+
// orchestrator-wake nudgers exactly one send per unread id lands; skipped ids release this
|
|
1665
|
+
// path's claim and never reach the seat, so the existing consumed-without-a-wake branch fires.
|
|
1666
|
+
const dutyOwner = dutyPlan.owner;
|
|
1667
|
+
const skippedNudgeIds = [];
|
|
1668
|
+
for (const target of dutyPlan.targets) {
|
|
1669
|
+
const reserve = await reserveNudgeIds({ recipient: target.recipient, ids: target.ids, owner: dutyOwner });
|
|
1670
|
+
target.ids = reserve.allowed;
|
|
1671
|
+
skippedNudgeIds.push(...reserve.skipped.map(s => s.id));
|
|
1672
|
+
}
|
|
1673
|
+
if (skippedNudgeIds.length) {
|
|
1674
|
+
const skippedSet = new Set(skippedNudgeIds);
|
|
1675
|
+
dutyPlan.items = dutyPlan.items.filter(item => !skippedSet.has(item.id));
|
|
1676
|
+
dutyPlan.targets = dutyPlan.targets.filter(target => target.ids.length);
|
|
1677
|
+
await releaseDutyNudgeClaims({ ids: skippedNudgeIds, statePath: DUTY_NUDGE_STATE, owner: dutyOwner });
|
|
1678
|
+
}
|
|
1573
1679
|
const claimedIds = new Set(dutyPlan.items.map(item => item.id));
|
|
1574
1680
|
const wakeForTurn = DUTY_NUDGES
|
|
1575
1681
|
? wake.filter(message => {
|
|
@@ -1692,6 +1798,11 @@ async function resolveWakeCard(messages, { session }) {
|
|
|
1692
1798
|
},
|
|
1693
1799
|
});
|
|
1694
1800
|
skippedNudges = audit.missing;
|
|
1801
|
+
// #11355: the ledger mirrors the claim release — a duty turn that ended without the send
|
|
1802
|
+
// must not silence the wake daemon for the TTL.
|
|
1803
|
+
for (const target of audit.missing) {
|
|
1804
|
+
await releaseNudgeIds({ recipient: target.recipient, ids: target.ids, owner: dutyOwner });
|
|
1805
|
+
}
|
|
1695
1806
|
}
|
|
1696
1807
|
if (!ec && skippedNudges.length) {
|
|
1697
1808
|
deliveryFails++;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// The shell calls this before every destructive turn sweep (#10497).
|
|
3
|
+
import { spawnSync } from "node:child_process";
|
|
4
|
+
import { realpathSync } from "node:fs";
|
|
5
|
+
import { pathToFileURL } from "node:url";
|
|
6
|
+
|
|
7
|
+
export function snapshotBeforeCut(workDir, reason = "stall") {
|
|
8
|
+
const git = args => {
|
|
9
|
+
const r = spawnSync("git", args, { cwd: workDir, encoding: "utf8", timeout: 20000 });
|
|
10
|
+
if (r.status !== 0) throw new Error(String(r.stderr || r.error || "git failed").trim());
|
|
11
|
+
return r.stdout;
|
|
12
|
+
};
|
|
13
|
+
try {
|
|
14
|
+
// Never capture a parent checkout when a CLI was launched in a nested scratch directory.
|
|
15
|
+
const top = git(["rev-parse", "--show-toplevel"]).trim();
|
|
16
|
+
if (realpathSync(top) !== realpathSync(workDir)) return { ok: true, files: 0 };
|
|
17
|
+
const branch = git(["symbolic-ref", "--short", "HEAD"]).trim();
|
|
18
|
+
const status = git(["status", "--porcelain=v1", "-z", "--untracked-files=all"]);
|
|
19
|
+
if (!status) return { ok: true, files: 0 };
|
|
20
|
+
if (!branch.startsWith("seat/")) return { ok: false, error: `dirty branch ${branch} is not a seat branch` };
|
|
21
|
+
let files = 0;
|
|
22
|
+
const rows = status.split("\0");
|
|
23
|
+
for (let i = 0; i < rows.length; i++) {
|
|
24
|
+
if (!rows[i]) continue;
|
|
25
|
+
files++;
|
|
26
|
+
if (/^[RC]|^.[RC]/.test(rows[i])) i++;
|
|
27
|
+
}
|
|
28
|
+
git(["add", "-A"]);
|
|
29
|
+
git(["-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false", "commit", "-m", `wip: cut at ${reason}, ${files} files`]);
|
|
30
|
+
return { ok: true, files, sha: git(["rev-parse", "HEAD"]).trim() };
|
|
31
|
+
} catch (e) { return { ok: false, error: e.message }; }
|
|
32
|
+
}
|
|
33
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
34
|
+
const result = snapshotBeforeCut(process.argv[2], process.argv[3]);
|
|
35
|
+
if (!result.ok) console.error(`cut snapshot failed: ${result.error}`);
|
|
36
|
+
process.exit(result.ok ? 0 : 1);
|
|
37
|
+
}
|
package/bin/gate.mjs
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { homedir } from "node:os";
|
|
5
|
+
import { loadOrCreate } from "../lib/identity.mjs";
|
|
6
|
+
import { sfetchJson } from "../lib/signed-fetch.mjs";
|
|
7
|
+
import { resolveProject } from "../lib/project.mjs";
|
|
8
|
+
import { relayUrl } from "../hooks/lib/api.mjs";
|
|
9
|
+
|
|
10
|
+
const [status, rawId] = process.argv.slice(2);
|
|
11
|
+
const id = Number(rawId);
|
|
12
|
+
if (!["go", "nogo"].includes(status) || !Number.isInteger(id) || id <= 0) {
|
|
13
|
+
console.error("usage: trantor gate go|nogo <id>");
|
|
14
|
+
process.exit(1);
|
|
15
|
+
}
|
|
16
|
+
let config = {};
|
|
17
|
+
try { config = JSON.parse(readFileSync(join(process.env.AGENT_BUS_DIR || join(homedir(), ".agent-bus"), "config.json"), "utf8")); } catch {}
|
|
18
|
+
const identity = loadOrCreate(config.ownerIdentity || "admin", "human");
|
|
19
|
+
try {
|
|
20
|
+
const project = resolveProject(process.cwd());
|
|
21
|
+
const r = await sfetchJson(`${relayUrl(project)}/hold/decide`, {
|
|
22
|
+
method: "POST", identity, payload: { id, status, project }, signal: AbortSignal.timeout(8000),
|
|
23
|
+
});
|
|
24
|
+
const result = await r.json();
|
|
25
|
+
if (!r.ok) throw new Error(result.error || `HTTP ${r.status}`);
|
|
26
|
+
console.log(`#${id}: ${status} — ${result.hold.file}`);
|
|
27
|
+
} catch (error) {
|
|
28
|
+
console.error(error.message);
|
|
29
|
+
process.exit(1);
|
|
30
|
+
}
|
package/bin/turn-watchdog.mjs
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// node bin/turn-watchdog.mjs <stampFile> <errFile> <windowMs> <session> <project> <hubUrl> <transcriptDir> <workDir> [stallFile] [turnStateFile]
|
|
6
6
|
import { readFileSync, writeFileSync, appendFileSync, statSync, readdirSync } from "node:fs";
|
|
7
7
|
import { join } from "node:path";
|
|
8
|
+
import { sessionActivityReader } from "../lib/session-liveness.mjs";
|
|
8
9
|
import { hostId } from "../lib/project.mjs";
|
|
9
10
|
import { signedPost } from "../hooks/lib/api.mjs";
|
|
10
11
|
import { refreshTurnLiveness } from "../lib/turnstate.mjs";
|
|
@@ -41,6 +42,7 @@ function newestMtime(dir) {
|
|
|
41
42
|
|
|
42
43
|
const armed = readStamp();
|
|
43
44
|
if (!armed) process.exit(0);
|
|
45
|
+
const sessionActivity = armed.activity ? sessionActivityReader(armed.activity) : () => 0;
|
|
44
46
|
// The runner instance that armed us (pid + boot ts, so a recycled pid cannot impersonate it).
|
|
45
47
|
// A stamp without a runner id (only possible mid-upgrade) skips the liveness check; the id
|
|
46
48
|
// match in the loop still guards it.
|
|
@@ -56,6 +58,7 @@ const describeLast = (b) => {
|
|
|
56
58
|
const parts = [];
|
|
57
59
|
if (b.wk) parts.push(`worktree ${ago(b.wk)}`);
|
|
58
60
|
if (b.tr) parts.push(`transcript ${ago(b.tr)}`);
|
|
61
|
+
if (b.cli) parts.push(`CLI session ${ago(b.cli)}`);
|
|
59
62
|
return parts.length ? parts.join(", ") : "nothing";
|
|
60
63
|
};
|
|
61
64
|
let baseErr = errSize();
|
|
@@ -109,24 +112,25 @@ if (stallFile) {
|
|
|
109
112
|
const size = errSize();
|
|
110
113
|
if (size > baseErr + 200) { baseErr = size; lastErrAt = now; } // new bytes: alive
|
|
111
114
|
const freshCut = Math.max(armedAt, now - windowMs - SLACK);
|
|
112
|
-
const tr = transcriptDir ? newestMtime(transcriptDir) : 0;
|
|
115
|
+
const tr = transcriptDir && !armed.activity ? newestMtime(transcriptDir) : 0;
|
|
116
|
+
const cli = sessionActivity();
|
|
113
117
|
const wk = workDir ? newestMtime(workDir) : 0;
|
|
114
|
-
reportLiveness(now, tr);
|
|
115
|
-
if (tr > freshCut || wk > freshCut || now - lastErrAt < windowMs) { // producing work: alive
|
|
118
|
+
reportLiveness(now, Math.max(tr, cli));
|
|
119
|
+
if (tr > freshCut || cli > freshCut || wk > freshCut || now - lastErrAt < windowMs) { // producing work: alive
|
|
116
120
|
if (box && extensions < Number(box.extensionsMax) && now + poll + SLACK >= deadline) {
|
|
117
121
|
extensions++;
|
|
118
122
|
deadline += Number(box.extendMs);
|
|
119
123
|
try { writeFileSync(box.deadlineFile, String(Math.floor(deadline / 1000))); } catch {}
|
|
120
124
|
try { appendFileSync(box.extFile, JSON.stringify({ n: extensions, at: now, until: deadline }) + "\n"); } catch {}
|
|
121
125
|
const card = Number(box.card) > 0 ? ` on #${box.card}` : "";
|
|
122
|
-
await tellExtension(`⏳ ${session} turn extended +${secsOrMins(Number(box.extendMs))} (${extensions}/${box.extensionsMax})${card} — alive: ${describeLast({ tr, wk })}${now - lastErrAt < windowMs ? ", stderr moving" : ""}; box now ${secsOrMins(deadline - armedAt)} of a ${secsOrMins(Number(box.ceilingMs))} ceiling`);
|
|
126
|
+
await tellExtension(`⏳ ${session} turn extended +${secsOrMins(Number(box.extendMs))} (${extensions}/${box.extensionsMax})${card} — alive: ${describeLast({ tr, wk, cli })}${now - lastErrAt < windowMs ? ", stderr moving" : ""}; box now ${secsOrMins(deadline - armedAt)} of a ${secsOrMins(Number(box.ceilingMs))} ceiling`);
|
|
123
127
|
}
|
|
124
128
|
continue;
|
|
125
129
|
}
|
|
126
130
|
try { writeFileSync(stallFile, ""); } catch {}
|
|
127
131
|
const mins = Math.round((now - armedAt) / 60000);
|
|
128
132
|
const orch = `${hostId()}:${project}`;
|
|
129
|
-
const text = `⏱ ${session} turn STALLED — ${mins}m with no activity (turn ${s.turn}; last seen: ${describeLast({ tr, wk })}, stderr ${baseErr > 0 ? `${baseErr}B` : "silent"}); ending the turn at the stall window, not the box.`;
|
|
133
|
+
const text = `⏱ ${session} turn STALLED — ${mins}m with no activity (turn ${s.turn}; last seen: ${describeLast({ tr, wk, cli })}, stderr ${baseErr > 0 ? `${baseErr}B` : "silent"}); ending the turn at the stall window, not the box.`;
|
|
130
134
|
// Direct = wake. The foreman first; if this seat IS the foreman's own runner, say it to all.
|
|
131
135
|
const to = orch === session ? "all" : orch;
|
|
132
136
|
try { await signedPost(`${hub}/send`, { from: session, to, text, project }, { session }); } catch {}
|
|
@@ -144,13 +148,14 @@ for (;;) {
|
|
|
144
148
|
// count (they predate arm), and absolute freshness cannot drift the way a probe-to-probe
|
|
145
149
|
// delta does when the seat writes coarsely or the machine loads (the +1130ms false alarm).
|
|
146
150
|
const freshCut = Math.max(armedAt, Date.now() - windowMs - SLACK);
|
|
147
|
-
const tr = transcriptDir ? newestMtime(transcriptDir) : 0;
|
|
151
|
+
const tr = transcriptDir && !armed.activity ? newestMtime(transcriptDir) : 0;
|
|
152
|
+
const cli = sessionActivity();
|
|
148
153
|
const wk = workDir ? newestMtime(workDir) : 0;
|
|
149
|
-
reportLiveness(Date.now(), tr);
|
|
150
|
-
if (errSize() > baseErr + 200 || tr > freshCut || wk > freshCut) continue; // producing work: alive, re-arm
|
|
154
|
+
reportLiveness(Date.now(), Math.max(tr, cli));
|
|
155
|
+
if (errSize() > baseErr + 200 || tr > freshCut || cli > freshCut || wk > freshCut) continue; // producing work: alive, re-arm
|
|
151
156
|
const mins = Math.round((Date.now() - armedAt) / 60000);
|
|
152
157
|
const orch = `${hostId()}:${project}`;
|
|
153
|
-
const text = `⏱ ${session} turn STALLED — running ${mins}m with no activity (turn ${s.turn}; last seen: ${describeLast({ tr, wk })}, stderr ${baseErr > 0 ? `${baseErr}B` : "silent"}). Not killed; check its pane, or \`trantor swap\`.`;
|
|
158
|
+
const text = `⏱ ${session} turn STALLED — running ${mins}m with no activity (turn ${s.turn}; last seen: ${describeLast({ tr, wk, cli })}, stderr ${baseErr > 0 ? `${baseErr}B` : "silent"}). Not killed; check its pane, or \`trantor swap\`.`;
|
|
154
159
|
// Direct = wake. The foreman first; if this seat IS the foreman's own runner, say it to all.
|
|
155
160
|
const to = orch === session ? "all" : orch;
|
|
156
161
|
try { await signedPost(`${hub}/send`, { from: session, to, text, project }, { session }); } catch {}
|
package/bin/wake-nudge.mjs
CHANGED
|
@@ -23,7 +23,8 @@ import { load } from "../lib/identity.mjs";
|
|
|
23
23
|
import { sfetchJson } from "../lib/signed-fetch.mjs";
|
|
24
24
|
import { busDir, hostId, readConfig } from "../lib/project.mjs";
|
|
25
25
|
import { ledgerPaths } from "../hooks/lib/inbox-ledger.mjs";
|
|
26
|
-
import { dutyEscalations, claimDutyNudges, auditDutyNudges } from "../lib/duty-nudges.mjs";
|
|
26
|
+
import { dutyEscalations, claimDutyNudges, auditDutyNudges, releaseDutyNudgeClaims } from "../lib/duty-nudges.mjs";
|
|
27
|
+
import { reserveNudgeIds, releaseNudgeIds } from "../lib/nudge-ledger.mjs";
|
|
27
28
|
import { recipientPanes } from "../lib/duty-recipient.mjs";
|
|
28
29
|
|
|
29
30
|
const SELF = fileURLToPath(import.meta.url);
|
|
@@ -107,7 +108,7 @@ export function postNudge({ socketPath, token, sid }, ids, timeoutMs = 1500) {
|
|
|
107
108
|
});
|
|
108
109
|
}
|
|
109
110
|
|
|
110
|
-
export async function wakeOnce({ api, bus = busDir(), resolver = recipient => resolveRecipient(recipient, { bus }), verifyMs = 10000, now = Date.now(), attempted = new Set(), cursor = { id: 0, ts: 0 } }) {
|
|
111
|
+
export async function wakeOnce({ api, bus = busDir(), resolver = recipient => resolveRecipient(recipient, { bus }), verifyMs = 10000, now = Date.now(), attempted = new Set(), cursor = { id: 0, ts: 0 }, log = m => console.error(m) }) {
|
|
111
112
|
const feed = await api(`/events?type=message&by=hub%3Aduty&since=${cursor.id}${cursor.id ? "" : "&limit=2000"}`);
|
|
112
113
|
if (Number.isFinite(feed.latest) && feed.latest < cursor.id) {
|
|
113
114
|
cursor.id = 0; cursor.ts = 0;
|
|
@@ -132,6 +133,18 @@ export async function wakeOnce({ api, bus = busDir(), resolver = recipient => re
|
|
|
132
133
|
const peer = await api(`/peer?session=${encodeURIComponent(recipient)}`);
|
|
133
134
|
return Number(peer.deliveredUpTo || 0) >= Number(id);
|
|
134
135
|
} });
|
|
136
|
+
// #11355: the shared pre-send ledger — the duty path stamps the same files, so of the two
|
|
137
|
+
// orchestrator-wake nudgers exactly one send per unread id lands; the loser logs and stands down.
|
|
138
|
+
const skipped = [];
|
|
139
|
+
for (const target of plan.targets) {
|
|
140
|
+
const reserve = await reserveNudgeIds({ recipient: target.recipient, ids: target.ids, owner: `wake:${process.pid}`, bus, log });
|
|
141
|
+
target.ids = reserve.allowed;
|
|
142
|
+
skipped.push(...reserve.skipped.map(s => s.id));
|
|
143
|
+
}
|
|
144
|
+
plan.items = plan.items.filter(item => !skipped.includes(item.id));
|
|
145
|
+
plan.targets = plan.targets.filter(target => target.ids.length);
|
|
146
|
+
await releaseDutyNudgeClaims({ ids: skipped, statePath, owner: plan.owner });
|
|
147
|
+
if (!plan.targets.length) return { nudged: [], missing: [] };
|
|
135
148
|
const observedIds = new Set();
|
|
136
149
|
let result;
|
|
137
150
|
try {
|
|
@@ -150,6 +163,12 @@ export async function wakeOnce({ api, bus = busDir(), resolver = recipient => re
|
|
|
150
163
|
await sleep(100);
|
|
151
164
|
}
|
|
152
165
|
}));
|
|
166
|
+
// #11355: a tick that ends with unverified sends releases their stamps — the ledger must
|
|
167
|
+
// mirror the claim release below, or a failed send would silence the duty path for the TTL.
|
|
168
|
+
for (const target of plan.targets) {
|
|
169
|
+
const unverified = target.ids.filter(id => !observedIds.has(id));
|
|
170
|
+
if (unverified.length) await releaseNudgeIds({ recipient: target.recipient, ids: unverified, owner: plan.owner, bus });
|
|
171
|
+
}
|
|
153
172
|
} finally {
|
|
154
173
|
// #7429: release unverified claims so the duty seat retains its triage path.
|
|
155
174
|
result = await auditDutyNudges({ plan, observedIds, statePath, reportFailure: async () => {} });
|