trantor 0.18.59 → 0.18.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +8 -0
- package/bin/connect.mjs +62 -55
- package/bin/crew/preflight.mjs +70 -0
- package/bin/crew-payload.mjs +19 -10
- package/bin/crew-runner.mjs +114 -108
- package/bin/crew.mjs +8 -5
- package/bin/turn-watchdog.mjs +38 -4
- package/hooks/lib/hollow-move.mjs +12 -1
- package/lib/classify-failure.mjs +19 -0
- package/lib/project.mjs +34 -9
- package/lib/seat-worktree.mjs +203 -0
- package/mcp.mjs +18 -10
- package/package.json +1 -1
package/bin/crew-runner.mjs
CHANGED
|
@@ -10,12 +10,13 @@ import { loadOrCreate } from "../lib/identity.mjs";
|
|
|
10
10
|
import { signedHeaders } from "../lib/signed-fetch.mjs";
|
|
11
11
|
import { ensureEnrolled } from "../lib/enroll.mjs";
|
|
12
12
|
import { redactKeys } from "../lib/redact.mjs";
|
|
13
|
+
import { applyWorktreeDeclaration, ensureSeatWorktree, provisioningLines, readWorktreeDeclaration } from "../lib/seat-worktree.mjs";
|
|
13
14
|
import {
|
|
14
15
|
AUTH_MARKER_RE, classifyFailure, looksLikeAuthDeath,
|
|
15
|
-
verdictFor, substantiveOutput,
|
|
16
|
+
verdictFor, substantiveOutput, stallVerdict, cutSignalFor,
|
|
16
17
|
readPromptText, stripPromptEcho,
|
|
17
18
|
} from "../lib/classify-failure.mjs";
|
|
18
|
-
import { capWake, capBcast, pickLessons, composePrompt } from "./crew-payload.mjs";
|
|
19
|
+
import { capWake, capBcast, pickLessons, composePrompt, contractBase, baseLine } from "./crew-payload.mjs";
|
|
19
20
|
import {
|
|
20
21
|
cardRefs, wakeCard, carriesWork, parseTurnTokens, parseResetAt, reasonWithBalances, quotaResetAt, PARKING_REASONS,
|
|
21
22
|
senderProjectOf, isLinkedProject, stateSkipReason,
|
|
@@ -39,80 +40,33 @@ const DIR = process.argv[3] || process.cwd();
|
|
|
39
40
|
// fork the host's "builtbetter.ai" into a separate "builtbetter" lane.
|
|
40
41
|
const PROJ = process.env.RELAY_PROJECT || resolveProject(DIR);
|
|
41
42
|
|
|
42
|
-
function safePathSegment(s) {
|
|
43
|
-
return String(s).replace(/\.{2,}/g, "_").replace(/[^A-Za-z0-9_.-]/g, "_");
|
|
44
|
-
}
|
|
45
|
-
|
|
46
43
|
function gitOut(args, cwd = DIR) {
|
|
47
44
|
const r = spawnSync("git", args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 8000 });
|
|
48
45
|
return r.status === 0 ? String(r.stdout || "").trim() : "";
|
|
49
46
|
}
|
|
50
47
|
|
|
51
|
-
function
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
if (ok) {
|
|
62
|
-
// #5403: a worktree created once builds against THAT day's main forever — every wave since
|
|
63
|
-
// has needed a hand fast-forward. Refresh only when it is CLEAN: a dirty tree is a seat's
|
|
64
|
-
// unintegrated work and a diverged branch is a decision, and refreshing must never eat
|
|
65
|
-
// either. Failure to refresh is loud but non-fatal: stale beats broken.
|
|
66
|
-
const dirty = gitOut(["-C", seatDir, "status", "--porcelain"], seatDir);
|
|
67
|
-
if (dirty === "") {
|
|
68
|
-
const head = gitOut(["-C", root, "rev-parse", "HEAD"], root);
|
|
69
|
-
const ff = head && spawnSync("git", ["-C", seatDir, "merge", "--ff-only", head], { stdio: "ignore", timeout: 15000 });
|
|
70
|
-
if (ff && ff.status === 0) console.log(`\x1b[2m[runner]\x1b[0m ${branch} worktree refreshed to ${head.slice(0, 7)}`);
|
|
71
|
-
else console.log(`\x1b[33m[runner]\x1b[0m ${branch} worktree diverged from main HEAD — left as-is (integrate or reset it)`);
|
|
72
|
-
} else {
|
|
73
|
-
console.log(`\x1b[33m[runner]\x1b[0m ${branch} worktree has uncommitted work — not refreshed`);
|
|
74
|
-
}
|
|
75
|
-
return seatDir;
|
|
76
|
-
}
|
|
77
|
-
console.log(`\x1b[33m[runner]\x1b[0m worktree path exists but is not a git worktree: ${seatDir} — using ${sourceDir}`);
|
|
78
|
-
return sourceDir;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
try { mkdirSync(join(homedir(), ".agent-bus", "worktrees", safePathSegment(PROJ)), { recursive: true }); } catch {}
|
|
82
|
-
|
|
83
|
-
// Fast-forward the base branch before branching: the worktree should build
|
|
84
|
-
// against the latest main, not a stale checkout. (#5403)
|
|
85
|
-
const base = gitOut(["-C", root, "rev-parse", "--abbrev-ref", "HEAD"], root);
|
|
86
|
-
if (base) {
|
|
87
|
-
const remote = gitOut(["-C", root, "rev-parse", "--abbrev-ref", `${base}@{upstream}`], root);
|
|
88
|
-
if (remote) {
|
|
89
|
-
const ff = spawnSync("git", ["-C", root, "merge", "--ff-only", remote], { stdio: "ignore", timeout: 15000 });
|
|
90
|
-
if (ff && ff.status === 0) console.log(`\x1b[2m[runner]\x1b[0m ${base} fast-forwarded to ${remote}`);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
const r = spawnSync("git", ["-C", root, "worktree", "add", "--no-track", "-B", branch, seatDir, "HEAD"], {
|
|
95
|
-
encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 30000,
|
|
96
|
-
});
|
|
97
|
-
if (r.status === 0) {
|
|
98
|
-
// Persist the base branch so the Review lens can name the real base instead
|
|
99
|
-
// of guessing via merge-base. Stale metadata is unset, not trusted. (#5403)
|
|
100
|
-
if (base) {
|
|
101
|
-
spawnSync("git", ["-C", seatDir, "config", `branch.${branch}.base`, base], { stdio: "ignore", timeout: 5000 });
|
|
48
|
+
function seatWorktree(sourceDir) {
|
|
49
|
+
const wt = ensureSeatWorktree({ sourceDir, project: PROJ, agent: AGENT });
|
|
50
|
+
// #7760: a worktree is a git checkout — it lacks the gitignored files the build needs and cannot
|
|
51
|
+
// resolve relative sibling packages. The project's declaration provides both before the first turn.
|
|
52
|
+
if (wt.root) {
|
|
53
|
+
const decl = readWorktreeDeclaration(wt.root);
|
|
54
|
+
if (decl) {
|
|
55
|
+
const applied = applyWorktreeDeclaration(decl, { root: wt.root, seatDir: wt.dir });
|
|
56
|
+
const lines = wt.created ? provisioningLines(applied) : provisioningLines({ linked: [], provisioned: [], operator: applied.operator, problems: applied.problems });
|
|
57
|
+
for (const l of lines) console.log(`\x1b[2m[runner]\x1b[0m ${l}`);
|
|
102
58
|
}
|
|
103
|
-
// Set push.autoSetupRemote so a plain git push creates and sets upstream on
|
|
104
|
-
// first push (git >= 2.37, older clients ignore it). (#5403)
|
|
105
|
-
const pushAuto = gitOut(["-C", seatDir, "config", "--get", "push.autoSetupRemote"], seatDir);
|
|
106
|
-
if (!pushAuto) {
|
|
107
|
-
spawnSync("git", ["-C", seatDir, "config", "push.autoSetupRemote", "true"], { stdio: "ignore", timeout: 5000 });
|
|
108
|
-
}
|
|
109
|
-
return seatDir;
|
|
110
59
|
}
|
|
111
|
-
|
|
112
|
-
return sourceDir;
|
|
60
|
+
return wt.dir;
|
|
113
61
|
}
|
|
114
62
|
|
|
115
|
-
const TURN_DIR =
|
|
63
|
+
const TURN_DIR = seatWorktree(DIR);
|
|
64
|
+
|
|
65
|
+
// #7754: the integration head is the orchestrator's LOCAL main, which this linked worktree already
|
|
66
|
+
// holds as the ref `main`; origin/main trails it whenever integration commits are unpushed.
|
|
67
|
+
const localMainHead = () => gitOut(["rev-parse", "--verify", "--quiet", "main^{commit}"], TURN_DIR);
|
|
68
|
+
const resolvesHere = (sha) =>
|
|
69
|
+
spawnSync("git", ["cat-file", "-e", `${sha}^{commit}`], { cwd: TURN_DIR, stdio: "ignore", timeout: 8000 }).status === 0;
|
|
116
70
|
|
|
117
71
|
// #6154: opencode records each session with its directory; the newest row for OUR worktree is the
|
|
118
72
|
// only session a resume may pin. Fail-open: no row means a fresh turn, never a stranger's session.
|
|
@@ -284,7 +238,7 @@ if (!CLI[AGENT]) log(`'${AGENT}' is not a built-in seat — running it as an ope
|
|
|
284
238
|
|
|
285
239
|
// RUNNER_RULES / RUNNER_KICKOFF env overrides: the runner is also the substrate for non-crew
|
|
286
240
|
// always-on seats (the fleet DUTY agent, bin/duty.mjs) whose doctrine is not "work your card".
|
|
287
|
-
const RULES = process.env.RUNNER_RULES || `Rules: you are ${SESSION} on the trantor crew. Before starting a card, read YOUR card: relay_board with card:<id> (the card, its deps, its notes, and the last five done cards whose title shares a word); never the whole board. Work your assigned file(s), report on the bus (relay_send, <280 chars), move your Kanban card as you go with a NOTE saying what you did (doing -> testing -> done; in 'testing' run YOUR OWN test file — never the full npm test, suites collide across seats — plus \`node bin/slop-gate.mjs\` when the repo has one: it lints ONLY your changed files against the anti-slop rules, and a card must not reach done with slop-gate failing; use 'failed' + a report if anything breaks). If a contract omits a fact you cannot proceed without, ASK — never invent the value: relay_ask(<card>, <question>) blocks the card with your question, keeps the turn owed (no park, no failure), and resumes you when the assigner's answer lands; an invented value that reads as reasoned is the worst outcome this crew ships (#7756). If you need something from another session, message THAT SESSION (relay_peers to find its id, relay_send to reach it) — never ask the human to pass it along; carrying messages between agents is the job this bus exists to remove. When your work for THIS message is finished, END YOUR TURN — do NOT park, do NOT loop relay_wait; the runner waits for you and will wake you with the next message. Path discipline: build/test from your worktree root ${TURN_DIR} with absolute paths or --manifest-path/--prefix instead of cd-ing into subdirs, and put anything that must land outside the repo under ${TURN_DIR}/.agent-bus-out/ (gitignored) — never ~/.agent-bus. Realigning your seat branch after the orchestrator harvested your commits is \`trantor sync\` run from your worktree: it reads the harvest receipts and refuses when an unharvested commit would be lost, so never reset or rebase onto main by hand. Cross-project action is a breach: never \`trantor up\` a crew, register a seat, or send a card/contract into a project other than ${PROJ} unless the operator ran \`trantor policy link ${PROJ} <other> --reason "<why>"\` first — the hub, the CLI and this runner all refuse it mechanically, so ask the operator to link the projects instead of routing around the refusal.`;
|
|
241
|
+
const RULES = process.env.RUNNER_RULES || `Rules: you are ${SESSION} on the trantor crew. Before starting a card, read YOUR card: relay_board with card:<id> (the card, its deps, its notes, and the last five done cards whose title shares a word); never the whole board. Work your assigned file(s), report on the bus (relay_send, <280 chars), move your Kanban card as you go with a NOTE saying what you did (doing -> testing -> done; in 'testing' run YOUR OWN test file — never the full npm test, suites collide across seats — plus \`node bin/slop-gate.mjs\` when the repo has one: it lints ONLY your changed files against the anti-slop rules, and a card must not reach done with slop-gate failing; use 'failed' + a report if anything breaks). If a contract omits a fact you cannot proceed without, ASK — never invent the value: relay_ask(<card>, <question>) blocks the card with your question, keeps the turn owed (no park, no failure), and resumes you when the assigner's answer lands; an invented value that reads as reasoned is the worst outcome this crew ships (#7756). If you need something from another session, message THAT SESSION (relay_peers to find its id, relay_send to reach it) — never ask the human to pass it along; carrying messages between agents is the job this bus exists to remove. When your work for THIS message is finished, END YOUR TURN — do NOT park, do NOT loop relay_wait; the runner waits for you and will wake you with the next message. Path discipline: build/test from your worktree root ${TURN_DIR} with absolute paths or --manifest-path/--prefix instead of cd-ing into subdirs, and put anything that must land outside the repo under ${TURN_DIR}/.agent-bus-out/ (gitignored) — never ~/.agent-bus. Realigning your seat branch after the orchestrator harvested your commits is \`trantor sync\` run from your worktree: it reads the harvest receipts and refuses when an unharvested commit would be lost, so never reset or rebase onto main by hand. A contract's \`base: <sha>\` line is the integration head: start your seat branch at that sha (\`git reset --hard <sha>\` on the fresh branch), never at origin/main, which trails the orchestrator's unpushed integration commits; the object is already here as the ref \`main\`. If \`git cat-file -e <sha>\` fails, say \`cannot resolve base <sha>\` on the bus and move the card to blocked instead of reasoning from origin/main. Every testing/done note names the sha you verified against as \`verified at <sha>\`; a note without it is flagged HOLLOW. Cross-project action is a breach: never \`trantor up\` a crew, register a seat, or send a card/contract into a project other than ${PROJ} unless the operator ran \`trantor policy link ${PROJ} <other> --reason "<why>"\` first — the hub, the CLI and this runner all refuse it mechanically, so ask the operator to link the projects instead of routing around the refusal.`;
|
|
288
242
|
|
|
289
243
|
// ---- the pulse --------------------------------------------------------------
|
|
290
244
|
// RUNNER_PULSE_MS re-runs an orchestrator seat's mission note on a cadence when the bus is silent;
|
|
@@ -406,6 +360,7 @@ async function reportFailure(exit, trigger, undelivered = 0, reasonOverride = ""
|
|
|
406
360
|
const status = down ? `down: ${reason} · ${consecFails} fails` : `errored: ${reason}`;
|
|
407
361
|
await api("/register", { session: SESSION, project: PROJ, status, llm: AGENT, model: MODEL, kind: "agent" }).catch(() => {});
|
|
408
362
|
const hint = reason === "exhausted" ? " — needs `trantor swap`"
|
|
363
|
+
: reason === "stalled" ? " — CLI silent for the whole watchdog window: check the provider, or `trantor swap`"
|
|
409
364
|
: reason === "auth" ? " — check credentials"
|
|
410
365
|
: reason === "backend-error" ? " — provider backend error (NOT quota): retry, or `trantor swap` to another provider"
|
|
411
366
|
: reason === "missing-cli" ? " — CLI not on PATH"
|
|
@@ -454,7 +409,10 @@ async function parkSeat(reason, undelivered, resetHint = 0) {
|
|
|
454
409
|
const when = resetAt ? new Date(resetAt).toLocaleString() : "";
|
|
455
410
|
if (!parkAnnounced) {
|
|
456
411
|
parkAnnounced = true;
|
|
457
|
-
|
|
412
|
+
// #7752: a stalled park names the CLI and its model — the swap decision needs both, and this
|
|
413
|
+
// line is the one place the room reads them together.
|
|
414
|
+
const named = reason === "stalled" ? ` — ${AGENT}${MODEL ? `, model ${MODEL}` : " (cli default model)"}` : "";
|
|
415
|
+
const text = redactKeys(`⛔ ${SESSION} PARKED (${reason}${named}) — holding ${undelivered} message(s), redelivery stopped ${when ? `until ${when}` : `until \`trantor up ${AGENT}\``}`);
|
|
458
416
|
await api("/send", { from: SESSION, to: "all", text, project: PROJ, kind: "status" }).catch(() => {});
|
|
459
417
|
const orch = `${hostId()}:${PROJ}`;
|
|
460
418
|
if (orch !== SESSION) await api("/send", { from: SESSION, to: orch, text, project: PROJ, kind: "alert" }).catch(() => {});
|
|
@@ -648,6 +606,10 @@ let inFollowUp = false;
|
|
|
648
606
|
// overwrites it with its own state, so a caller reading it after the chain sees the state of the
|
|
649
607
|
// LAST turn — which is what decides whether a failed chain died to the box or to the API.
|
|
650
608
|
let lastTurnCut = false;
|
|
609
|
+
// #7752: whether the turn that just ended was a SILENT cut — the watchdog's stall marker (every
|
|
610
|
+
// liveness channel quiet for the whole window), not the box. Same overwrite semantics as
|
|
611
|
+
// lastTurnCut: the follow-up's own ending is what the ladder reads.
|
|
612
|
+
let lastTurnStalled = false;
|
|
651
613
|
// The card the CURRENT CLI session belongs to (#6134). 0 = the kickoff session, which belongs to
|
|
652
614
|
// no card, so the first contract that names one starts a session of its own.
|
|
653
615
|
let sessionCard = 0;
|
|
@@ -728,13 +690,21 @@ async function runTurn(prompt, isFirst, trigger = "kickoff", opts = {}) {
|
|
|
728
690
|
// CLI can write lines the watchdog sees. CUTF is written by the shell's time box: cut, not crashed.
|
|
729
691
|
const CUTF = join(homedir(), ".agent-bus", `turncut-${AGENT}-${PROJ}`);
|
|
730
692
|
try { unlinkSync(CUTF); } catch {}
|
|
693
|
+
// #7752: written by the watchdog's kill mode when EVERY liveness channel has been quiet for
|
|
694
|
+
// the whole window; the box sweeps on the marker, so a silent turn ends at the stall window
|
|
695
|
+
// instead of burning the box. Unlinked at turn start and again with the cut cleanup.
|
|
696
|
+
const STALLF = join(homedir(), ".agent-bus", `turnstall-${AGENT}-${PROJ}`);
|
|
697
|
+
try { unlinkSync(STALLF); } catch {}
|
|
731
698
|
// Touched by the stderr scrubber as its LAST act (the shell below); node waits for it after
|
|
732
699
|
// spawnSync before reading ERRF — see the drain note at the spawnSync call.
|
|
733
700
|
const DRAINF = join(homedir(), ".agent-bus", `turndrain-${AGENT}-${PROJ}`);
|
|
734
701
|
try { unlinkSync(DRAINF); } catch {}
|
|
735
702
|
try {
|
|
736
703
|
writeFileSync(STAMPF, JSON.stringify({ turn: TURN, startedAt: Date.now(), runner: RUNNER_ID }));
|
|
737
|
-
const wd = spawn(process.execPath, [join(import.meta.dirname, "turn-watchdog.mjs"), STAMPF, ERRF, String(WD_MS), SESSION, PROJ, HUB, TRANSCRIPT_DIR, TURN_DIR
|
|
704
|
+
const wd = spawn(process.execPath, [join(import.meta.dirname, "turn-watchdog.mjs"), STAMPF, ERRF, String(WD_MS), SESSION, PROJ, HUB, TRANSCRIPT_DIR, TURN_DIR,
|
|
705
|
+
// #7752: the stall marker only exists when a box does — with no box there is no sweep to
|
|
706
|
+
// end the turn, so the watchdog stays report-only (reporting is the whole job, boxless).
|
|
707
|
+
TURN_MAX_MS ? STALLF : ""],
|
|
738
708
|
{ detached: true, stdio: "ignore" });
|
|
739
709
|
WD_CHILD = wd;
|
|
740
710
|
wd.unref();
|
|
@@ -743,13 +713,21 @@ async function runTurn(prompt, isFirst, trigger = "kickoff", opts = {}) {
|
|
|
743
713
|
// (setsid escapes a group signal, never its parent). The marker file tells node "cut", not "crashed".
|
|
744
714
|
const sweep = `sweep() { local p; for p in $(pgrep -P $1 2>/dev/null); do sweep $p; done; kill -KILL $1 2>/dev/null; }`;
|
|
745
715
|
// #7742: the box must neither hold the sid capture pipe nor orphan its sleep on turn exit.
|
|
716
|
+
// #7752: the box POLLS instead of one long sleep — the watchdog's stall marker (a silent
|
|
717
|
+
// turn) breaks the loop at the stall window, the deadline breaks it at the full box, and only
|
|
718
|
+
// the deadline path writes CUTF so the runner can tell the two cuts apart.
|
|
746
719
|
const box = TURN_MAX_MS ? `
|
|
747
720
|
${sweep}
|
|
748
721
|
( trap 'kill "$sleeppid" 2>/dev/null; wait "$sleeppid" 2>/dev/null; exit 0' TERM
|
|
749
|
-
|
|
750
|
-
|
|
722
|
+
deadline=$(( $(date +%s) + ${Math.ceil(TURN_MAX_MS / 1000)} ))
|
|
723
|
+
while :; do
|
|
724
|
+
[ -f "${STALLF}" ] && break
|
|
725
|
+
[ "$(date +%s)" -ge "$deadline" ] && break
|
|
726
|
+
sleep 1 & sleeppid=$!
|
|
727
|
+
wait "$sleeppid"
|
|
728
|
+
done
|
|
751
729
|
kill -0 $job 2>/dev/null || exit 0
|
|
752
|
-
: > ${CUTF}
|
|
730
|
+
[ -f "${STALLF}" ] || : > ${CUTF}
|
|
753
731
|
sweep $job
|
|
754
732
|
) >/dev/null 2>&1 & boxpid=$!` : "\nboxpid=";
|
|
755
733
|
const shell = `set -o pipefail
|
|
@@ -782,9 +760,11 @@ exit $turn_exit`;
|
|
|
782
760
|
if (TURN_MAX_MS) { spawnOpts.timeout = TURN_MAX_MS + 30000; spawnOpts.killSignal = "SIGKILL"; }
|
|
783
761
|
const r = spawnSync("/bin/bash", ["-c", shell], spawnOpts);
|
|
784
762
|
// The shell's box leaves the marker; the backstop leaves an ETIMEDOUT. Either way the turn was
|
|
785
|
-
// cut, not merely failed.
|
|
763
|
+
// cut, not merely failed. A stall marker (#7752) outranks the box marker: a turn silent for
|
|
764
|
+
// the whole window is a stall however the sweep found it.
|
|
786
765
|
const boxed = existsSync(CUTF);
|
|
787
|
-
const
|
|
766
|
+
const stallCut = existsSync(STALLF);
|
|
767
|
+
const cut = !!TURN_MAX_MS && (boxed || stallCut || r.error?.code === "ETIMEDOUT");
|
|
788
768
|
// DRAIN before classifying, never on a CUT turn (the sweep killed the scrubber, its marker never
|
|
789
769
|
// comes). bash 3.2 `wait` skips process substitutions, so wait for DRAINF, bounded.
|
|
790
770
|
if (!cut) {
|
|
@@ -798,9 +778,12 @@ exit $turn_exit`;
|
|
|
798
778
|
// Belt and braces after the shell's descendant sweep: anything still sharing the turn's group.
|
|
799
779
|
if (r.pid) { try { process.kill(-r.pid, "SIGKILL"); } catch {} }
|
|
800
780
|
try { unlinkSync(CUTF); } catch {}
|
|
801
|
-
|
|
781
|
+
try { unlinkSync(STALLF); } catch {}
|
|
782
|
+
if (stallCut) log(`\x1b[33mturn STALLED — no bytes on either stream and no transcript advance for ${Math.round(WD_MS / 60000)}m; ended at the stall window, not the ${Math.round(TURN_MAX_MS / 1000)}s box\x1b[0m`);
|
|
783
|
+
else log(`\x1b[33mturn cut at the ${Math.round(TURN_MAX_MS / 1000)}s time box — CLI and every descendant ended${boxed ? "" : " (node backstop: bash itself was wedged)"}\x1b[0m`);
|
|
802
784
|
}
|
|
803
785
|
lastTurnCut = cut;
|
|
786
|
+
lastTurnStalled = cut && stallCut;
|
|
804
787
|
// #5869: scrub AT REST, synchronously, before anything reads the file back. The explicit shell
|
|
805
788
|
// wait above drains the live stderr scrubber first; this pass is defense in depth for redaction.
|
|
806
789
|
try { writeFileSync(ERRF, redactKeys(readFileSync(ERRF, "utf8"))); } catch {}
|
|
@@ -854,7 +837,9 @@ exit $turn_exit`;
|
|
|
854
837
|
if (lastEmptyTurn) log("\x1b[33mexit 0 but the turn was EMPTY — no worktree change, no substantive output, no bus activity\x1b[0m");
|
|
855
838
|
// #5868: the verdict rides the telemetry row so a classification survives the pane scrolling
|
|
856
839
|
// away — the same "classified X because Y" shape the runner logs, in the seat's jsonl forever.
|
|
857
|
-
|
|
840
|
+
// A silent cut names its own verdict (#7752): the exit under it is the sweep's SIGPIPE, never
|
|
841
|
+
// a provider failure.
|
|
842
|
+
const verdict = stallCut ? stallVerdict() : verdictFor(realExit, effExit, lastEmptyOutput, ownOut, lastEmptyTurn);
|
|
858
843
|
// #6134: what the turn COST, from the CLI's own usage line. Zero means this CLI printed none —
|
|
859
844
|
// never that the turn was free. `trantor seat-why` totals these into today's spend per seat.
|
|
860
845
|
let tokens = parseTurnTokens(ownOut);
|
|
@@ -864,10 +849,11 @@ exit $turn_exit`;
|
|
|
864
849
|
// actually carried counts.
|
|
865
850
|
if (c) tokens = c.input + c.output + c.cache_read + c.cache_creation;
|
|
866
851
|
}
|
|
867
|
-
// #6289: every ledger row names in ONE field what happened to the turn — cut (the box ended
|
|
868
|
-
// api-error (the CLI failed), completed —
|
|
869
|
-
//
|
|
870
|
-
|
|
852
|
+
// #6289: every ledger row names in ONE field what happened to the turn — cut (the box ended
|
|
853
|
+
// it), stalled (the watchdog window ended it, #7752), api-error (the CLI failed), completed —
|
|
854
|
+
// and what it cost in tokens, even when this CLI printed no usage line (0 means "not
|
|
855
|
+
// reported", never "free"). `cut` stays too: the drills read it.
|
|
856
|
+
const outcome = cut ? (stallCut ? "stalled" : "cut") : (effExit !== 0 ? "api-error" : lastEmptyTurn ? "empty" : "completed");
|
|
871
857
|
// #7756: a clean turn that ASKED its assigner is demoted-but-owed, not "completed". The judge
|
|
872
858
|
// (deliverWake's /contracts read) renames the ledger row, so "asked" is what the log keeps.
|
|
873
859
|
let finalOutcome = outcome;
|
|
@@ -876,6 +862,10 @@ exit $turn_exit`;
|
|
|
876
862
|
}
|
|
877
863
|
const telemetryRow = { ts: Date.now(), agent: AGENT, project: PROJ, turn: TURN, trigger, model: MODEL || "cli-default", duration_ms: Date.now() - t0, exit: realExit, effExit, authFailed: effExit !== realExit, emptyOutput: lastEmptyOutput, emptyTurn: lastEmptyTurn, verdict, outcome: finalOutcome, tokens };
|
|
878
864
|
if (cut) telemetryRow.cut = true;
|
|
865
|
+
if (stallCut) telemetryRow.stalled = true;
|
|
866
|
+
// #7752: a cut turn's 141 is the sweep's SIGPIPE, recorded as the cut signal — never read as
|
|
867
|
+
// a quota or crash pattern downstream.
|
|
868
|
+
if (cut) { const sig = cutSignalFor(realExit); if (sig) telemetryRow.cutSignal = sig; }
|
|
879
869
|
telemetry(telemetryRow);
|
|
880
870
|
log(`turn ended (exit ${realExit}${effExit !== realExit ? ` → effective ${effExit} (${lastEmptyOutput ? "empty-output" : "auth"})` : ""}, ${((Date.now() - t0) / 1000).toFixed(0)}s)`);
|
|
881
871
|
if (realExit === 0 && effExit === 0) { cmuxStatus("idle", "#8a94a6", "robot"); herdrAgent("idle"); } // finished this turn, waiting for the next
|
|
@@ -1175,12 +1165,10 @@ function askedExcerpt(message) {
|
|
|
1175
1165
|
// reply-linked outcomes, and the old stable marker before direct-address logic sees them. Status
|
|
1176
1166
|
// broadcasts are presence chatter and are dropped rather than saved as future prompt context.
|
|
1177
1167
|
msgs = msgs.filter(m => !isReceipt(m) && !isStatusBroadcast(m));
|
|
1178
|
-
// #7778: ids in the seen-set
|
|
1179
|
-
//
|
|
1180
|
-
//
|
|
1181
|
-
//
|
|
1182
|
-
// Filtering BEFORE the wake/bcast split keeps a dupe out of the demoted-sender notice too:
|
|
1183
|
-
// the seat DID turn on that message, and #7766's notice must not say otherwise.
|
|
1168
|
+
// #7778: ids in the seen-set already reached the model through the session's own inbox read
|
|
1169
|
+
// mid-turn (recorded at the last clean turn boundary). The poll filters by THIS runner's cursor,
|
|
1170
|
+
// not the shared ledger, so the second reader stands down here and the cursor adopts the consumed
|
|
1171
|
+
// ids. Filtering BEFORE the wake/bcast split keeps a dupe out of #7766's demoted-sender notice.
|
|
1184
1172
|
if (seenLedger.length) {
|
|
1185
1173
|
const seenIds = new Set(seenLedger.map(e => e.id));
|
|
1186
1174
|
const dupes = msgs.filter(m => seenIds.has(m.id));
|
|
@@ -1258,17 +1246,10 @@ function askedExcerpt(message) {
|
|
|
1258
1246
|
log("parked — waiting for the next message");
|
|
1259
1247
|
}
|
|
1260
1248
|
|
|
1261
|
-
//
|
|
1262
|
-
//
|
|
1263
|
-
//
|
|
1264
|
-
//
|
|
1265
|
-
// read and answered MID-TURN was re-polled after the turn and woken AGAIN (seen live: a seat
|
|
1266
|
-
// answered inside its turn, then a second turn 16s later re-quoted the same message verbatim).
|
|
1267
|
-
// The hub's deliveredUpTo is the one ledger both paths already write, so at a CLEAN turn
|
|
1268
|
-
// boundary every id above our poll cursor but at-or-below it was necessarily consumed by the
|
|
1269
|
-
// session-side path. Those ids go into the persisted seen-set; the poll filter stands the
|
|
1270
|
-
// second reader down. A failed, parked or hollow turn reconciles NOTHING — dedupe must never
|
|
1271
|
-
// swallow a message the seat still owes, so only full success is trusted to mark history read.
|
|
1249
|
+
// The pending batch is cleared ONLY on exit 0; any other outcome leaves it queued on disk with a
|
|
1250
|
+
// backoff. #7778: the session's inbox read and this /poll are two readers of one bus with no shared
|
|
1251
|
+
// cursor, so a message answered mid-turn was woken AGAIN. The hub's deliveredUpTo is the ledger both
|
|
1252
|
+
// write: at a CLEAN boundary the ids it covers join the seen-set; a failed or parked turn marks nothing.
|
|
1272
1253
|
async function reconcileSessionReads(upToCursor) {
|
|
1273
1254
|
try {
|
|
1274
1255
|
const peer = await api(`/peer?session=${encodeURIComponent(SESSION)}`);
|
|
@@ -1343,6 +1324,22 @@ function askedExcerpt(message) {
|
|
|
1343
1324
|
// #6134: ONE SESSION PER CARD; a different card starts a fresh CLI session and the seat is told.
|
|
1344
1325
|
// #7061: bound by SHAPE, not position, so an order opening with what shipped binds the right card.
|
|
1345
1326
|
const card = wakeCard(wakeForTurn, { session: SESSION });
|
|
1327
|
+
// #7754: the contract carries the integration head as `base: <sha>` — the wake's own line if it
|
|
1328
|
+
// has one, else local main. A sha this worktree cannot resolve is never worked from origin/main:
|
|
1329
|
+
// the card goes to blocked and the assigner is told, and no model turn is spent on it.
|
|
1330
|
+
const base = contractBase(wakeForTurn) || localMainHead();
|
|
1331
|
+
if (base && card && !resolvesHere(base)) {
|
|
1332
|
+
const text = `cannot resolve base ${base} in ${TURN_DIR} (git cat-file -e failed) — #${card} moved to blocked; push or fetch that sha, or name one this worktree can reach`;
|
|
1333
|
+
log(text);
|
|
1334
|
+
await api("/task/update", { id: card, status: "blocked", note: text, by: SESSION }).catch(() => {});
|
|
1335
|
+
const owed = assigners.map(a => a.from).filter(f => f && f !== SESSION);
|
|
1336
|
+
for (const to of (owed.length ? owed : ["all"]))
|
|
1337
|
+
await api("/send", { from: SESSION, to, text, project: PROJ, kind: "alert" }).catch(() => {});
|
|
1338
|
+
pendingWake = [];
|
|
1339
|
+
savePending([], pendingBcast);
|
|
1340
|
+
return;
|
|
1341
|
+
}
|
|
1342
|
+
const baseText = base ? `\n${baseLine(base)}\n` : "";
|
|
1346
1343
|
const fresh = card > 0 && card !== sessionCard;
|
|
1347
1344
|
if (card) sessionCard = card;
|
|
1348
1345
|
const cited = [...new Set(wakeForTurn.flatMap(m => cardRefs(m.text)))];
|
|
@@ -1354,7 +1351,7 @@ function askedExcerpt(message) {
|
|
|
1354
1351
|
? `\n(This turn is card #${card} — the wake cites ${cited.length} cards; the rest are context.)\n`
|
|
1355
1352
|
: "");
|
|
1356
1353
|
const prompt = composedTurn({
|
|
1357
|
-
wakeText, ctxText, againText: againText + freshText + dutyNudgeDirective(dutyPlan),
|
|
1354
|
+
wakeText, ctxText, againText: againText + freshText + baseText + dutyNudgeDirective(dutyPlan),
|
|
1358
1355
|
tailText: "\nAct on what's addressed to you, then end your turn.\n\n",
|
|
1359
1356
|
rulesText: RULES, lessons,
|
|
1360
1357
|
});
|
|
@@ -1370,7 +1367,7 @@ function askedExcerpt(message) {
|
|
|
1370
1367
|
// broadcasts, the redelivery note, and any rejection the last step earned. None of it
|
|
1371
1368
|
// may reach the preamble, or the prefix stops being byte-identical and the cache claim
|
|
1372
1369
|
// dies quietly (§4.6).
|
|
1373
|
-
observation: [stateObservation, wakeText, ctxText, againText + freshText].filter(Boolean).join("\n"),
|
|
1370
|
+
observation: [stateObservation, wakeText, ctxText, againText + freshText + baseText].filter(Boolean).join("\n"),
|
|
1374
1371
|
})
|
|
1375
1372
|
: await runTurn(prompt, fresh, deliveryFails ? `${trigger} (redelivery)` : trigger, { judgeOutcome: askJudge });
|
|
1376
1373
|
}
|
|
@@ -1414,11 +1411,15 @@ function askedExcerpt(message) {
|
|
|
1414
1411
|
}
|
|
1415
1412
|
if (ec) {
|
|
1416
1413
|
deliveryFails++;
|
|
1417
|
-
// #
|
|
1418
|
-
//
|
|
1419
|
-
|
|
1414
|
+
// #7752: a SILENT cut is a stall, whatever the captured text contains — the qwen
|
|
1415
|
+
// specimen's err file carried a "rate-limit" line and the seat was parked exhausted on a
|
|
1416
|
+
// turn the box itself had killed. The wake stays owed (#7759's shape): the queue is kept,
|
|
1417
|
+
// the ladder retries, and the second silent chain parks as "stalled".
|
|
1418
|
+
let reason = lastTurnStalled ? "stalled" : classify(ec);
|
|
1420
1419
|
let quotaReset = 0;
|
|
1421
1420
|
if (reason === "empty-output") {
|
|
1421
|
+
// #6131: a silent turn on a seat whose plan reads spent is exhaustion wearing a crash's
|
|
1422
|
+
// clothes. Only that one reason is ever re-read, and only from the seat's balance rows.
|
|
1422
1423
|
const rows = await balanceRows();
|
|
1423
1424
|
reason = reasonWithBalances(reason, rows);
|
|
1424
1425
|
// The rows that just proved the plan is spent also carry when it lifts — a park that names
|
|
@@ -1427,9 +1428,10 @@ function askedExcerpt(message) {
|
|
|
1427
1428
|
}
|
|
1428
1429
|
savePending(pendingWake, pendingBcast);
|
|
1429
1430
|
await reportFailure(ec, "message", pendingWake.length, reason);
|
|
1430
|
-
// #6289: TWO consecutive
|
|
1431
|
-
// died to cuts, api-error otherwise
|
|
1432
|
-
|
|
1431
|
+
// #6289: TWO consecutive failed turns on one contract PARK the seat — "stalled" for a
|
|
1432
|
+
// silent chain (#7752), time-box when the chain died to cuts, api-error otherwise —
|
|
1433
|
+
// holding the queue until `trantor up`.
|
|
1434
|
+
const parkReason = lastTurnStalled ? "stalled" : (PARKING_REASONS.has(reason) ? reason : (lastTurnCut ? "time-box" : "api-error"));
|
|
1433
1435
|
if (PARKING_REASONS.has(reason) || deliveryFails >= 2) {
|
|
1434
1436
|
retryAt = await parkSeat(parkReason, pendingWake.length, quotaReset);
|
|
1435
1437
|
// RUNNER_PARK_MAX_MS is set only by `trantor duty up` (launchd keepalive): past the ceiling,
|
|
@@ -1444,7 +1446,9 @@ function askedExcerpt(message) {
|
|
|
1444
1446
|
}, wakeIn).unref?.();
|
|
1445
1447
|
}
|
|
1446
1448
|
await notifyAssigners(assigners,
|
|
1447
|
-
|
|
1449
|
+
reason === "stalled"
|
|
1450
|
+
? `⛔ your contract is PARKED on ${SESSION} (stalled: two turns silent for the whole watchdog window) — not retrying · asked: "${asked}"`
|
|
1451
|
+
: `⛔ your contract is PARKED on ${SESSION} (${parkReason}) — not retrying · asked: "${asked}"`);
|
|
1448
1452
|
lastTurnAt = Date.now();
|
|
1449
1453
|
return;
|
|
1450
1454
|
}
|
|
@@ -1452,7 +1456,9 @@ function askedExcerpt(message) {
|
|
|
1452
1456
|
retryAt = Date.now() + wait;
|
|
1453
1457
|
// The room hears the broadcast above; the one who is actually blocked hears it directly.
|
|
1454
1458
|
await notifyAssigners(assigners,
|
|
1455
|
-
|
|
1459
|
+
reason === "stalled"
|
|
1460
|
+
? `🫥 your contract STALLED on ${SESSION} (no output for the whole watchdog window; ended at the window, not the box) · wake stays owed · retrying in ${Math.round(wait / 1000)}s · asked: "${asked}"`
|
|
1461
|
+
: `⚠️ your contract FAILED on ${SESSION} (exit ${ec}, ${reason}) · retrying in ${Math.round(wait / 1000)}s · asked: "${asked}"`);
|
|
1456
1462
|
log(`\x1b[31m${pendingWake.length} message(s) still UNDELIVERED — next attempt in ${Math.round(wait / 1000)}s\x1b[0m`);
|
|
1457
1463
|
} else if (turnAsk) {
|
|
1458
1464
|
// #7756: a clean exit that ASKED is not "done". The wake stays owed — queued on disk with
|
package/bin/crew.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import { down, prune } from "./crew/state.mjs";
|
|
|
9
9
|
import { spawnTerminal, spawnTmux } from "./crew/tmux.mjs";
|
|
10
10
|
import { epochMs, failedAgents, verifyCrew } from "./crew/verify.mjs";
|
|
11
11
|
import { guardCrossProjectUp } from "./crew/worktrees.mjs";
|
|
12
|
+
import { preflightFirstSeat } from "./crew/preflight.mjs";
|
|
12
13
|
import { resolveAgentLaunchSpecs } from "../lib/agent-preferences.mjs";
|
|
13
14
|
import { readConfig } from "../lib/project.mjs";
|
|
14
15
|
|
|
@@ -51,7 +52,7 @@ function connect() {
|
|
|
51
52
|
call(process.execPath, [join(ctx.root, "bin/connect.mjs")], { env: ctx.env });
|
|
52
53
|
}
|
|
53
54
|
|
|
54
|
-
function runUp(args, verify = true) {
|
|
55
|
+
async function runUp(args, verify = true) {
|
|
55
56
|
const options = parseUpArgs(args);
|
|
56
57
|
const requested = resolveAgentLaunchSpecs(options.specs, readConfig());
|
|
57
58
|
options.specs = requested.specs;
|
|
@@ -67,6 +68,8 @@ function runUp(args, verify = true) {
|
|
|
67
68
|
connect();
|
|
68
69
|
console.log(`[crew] hub for ${ctx.project}: ${ctx.hub} (baked into every seat; CREW_HUB=<url> overrides)`);
|
|
69
70
|
console.log(`— bringing up crew for ${ctx.project} (${ctx.mux === "terminal" ? "Terminal windows" : ctx.mux}) —`);
|
|
71
|
+
// #7760: build once in the first seat's worktree before any seat gets a contract.
|
|
72
|
+
if (!ctx.dry) await preflightFirstSeat(ctx, options.specs[0].split(":")[0]);
|
|
70
73
|
const skipped = [];
|
|
71
74
|
const started = epochMs();
|
|
72
75
|
spawnCrew(options, skipped);
|
|
@@ -92,7 +95,7 @@ function runUp(args, verify = true) {
|
|
|
92
95
|
return reportSkipped(skipped);
|
|
93
96
|
}
|
|
94
97
|
|
|
95
|
-
function swap(args) {
|
|
98
|
+
async function swap(args) {
|
|
96
99
|
const [oldAgent, replacement, ...flags] = args;
|
|
97
100
|
if (!oldAgent || !replacement) {
|
|
98
101
|
console.log("usage: trantor swap <oldAgent> <newAgent[:provider[/model]]> [--task K --difficulty D]");
|
|
@@ -101,7 +104,7 @@ function swap(args) {
|
|
|
101
104
|
console.log(`— tearing down old seat '${oldAgent}' in ${ctx.project} —`);
|
|
102
105
|
down(ctx, [oldAgent], adapters);
|
|
103
106
|
console.log(`— spawning replacement: ${replacement} —`);
|
|
104
|
-
const code = runUp([...flags, replacement]);
|
|
107
|
+
const code = await runUp([...flags, replacement]);
|
|
105
108
|
if (!code) console.log(`— swapped. RESEND the contract to '${replacement.split(":")[0]}' (it joined fresh with no context). —`);
|
|
106
109
|
return code;
|
|
107
110
|
}
|
|
@@ -110,7 +113,7 @@ let code = 0;
|
|
|
110
113
|
if (command === "down") code = down(ctx, rawArgs, adapters);
|
|
111
114
|
else if (command === "prune") { prune(ctx, adapters); console.log(`— pruned dead crew rows (${ctx.statePath}) —`); }
|
|
112
115
|
else if (command === "open") code = openOrchestrator(ctx, rawArgs);
|
|
113
|
-
else if (command === "swap") code = swap(rawArgs);
|
|
114
|
-
else if (command === "up") code = guardCrossProjectUp(ctx) ? runUp(rawArgs) : 1;
|
|
116
|
+
else if (command === "swap") code = await swap(rawArgs);
|
|
117
|
+
else if (command === "up") code = guardCrossProjectUp(ctx) ? await runUp(rawArgs) : 1;
|
|
115
118
|
else { usage(); code = 1; }
|
|
116
119
|
process.exit(code);
|
package/bin/turn-watchdog.mjs
CHANGED
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
// Turn watchdog (#5684, reworked #6206). runTurn is spawnSync — the runner cannot watch its own
|
|
3
3
|
// turn — so this DETACHED helper does: armed at turn start, disarmed by turn end. A turn past
|
|
4
4
|
// the window with NO new activity earns ONE direct stall report to the foreman (episode, never a
|
|
5
|
-
// timer storm)
|
|
5
|
+
// timer storm). With no stall file (report-only mode) the turn is never killed — reporting is
|
|
6
|
+
// the whole job. With one (#7752, kill mode) a turn silent on EVERY channel for the whole
|
|
7
|
+
// window is also ENDED at the window: the marker tells the runner's shell box to sweep early,
|
|
8
|
+
// so the kill itself still lives in exactly one place — the shell that owns $job.
|
|
6
9
|
//
|
|
7
10
|
// #6206: stdout silence is NOT a stall — `claude -p` prints nothing until the turn ends by
|
|
8
11
|
// design, so a seat editing five files was reported STALLED while its transcript advanced.
|
|
@@ -12,13 +15,13 @@
|
|
|
12
15
|
// instance id, so a survivor of a replaced runner exits on mismatch or runner death instead of
|
|
13
16
|
// re-matching the NEW runner's turn number (the 09:47 false alarm was exactly that orphan).
|
|
14
17
|
//
|
|
15
|
-
// node bin/turn-watchdog.mjs <stampFile> <errFile> <windowMs> <session> <project> <hubUrl> <transcriptDir> <workDir>
|
|
16
|
-
import { readFileSync, statSync, readdirSync } from "node:fs";
|
|
18
|
+
// node bin/turn-watchdog.mjs <stampFile> <errFile> <windowMs> <session> <project> <hubUrl> <transcriptDir> <workDir> [stallFile]
|
|
19
|
+
import { readFileSync, writeFileSync, statSync, readdirSync } from "node:fs";
|
|
17
20
|
import { join } from "node:path";
|
|
18
21
|
import { hostId } from "../lib/project.mjs";
|
|
19
22
|
import { signedPost } from "../hooks/lib/api.mjs";
|
|
20
23
|
|
|
21
|
-
const [stampFile, errFile, windowMsRaw, session, project, hub, transcriptDir = "", workDir = ""] = process.argv.slice(2);
|
|
24
|
+
const [stampFile, errFile, windowMsRaw, session, project, hub, transcriptDir = "", workDir = "", stallFile = ""] = process.argv.slice(2);
|
|
22
25
|
// SAFETY: the 10-minute floor lives in crew-runner.mjs (the default when TRANTOR_TURN_WATCHDOG_MS
|
|
23
26
|
// is unset); this fallback only covers a missing argument. Drills pass tiny windows on purpose.
|
|
24
27
|
const windowMs = Number(windowMsRaw) || 10 * 60 * 1000;
|
|
@@ -71,6 +74,37 @@ let baseErr = errSize();
|
|
|
71
74
|
const armedAt = armed.startedAt || Date.now();
|
|
72
75
|
const SLACK = 2000; // timestamp granularity + scheduler drift under load
|
|
73
76
|
|
|
77
|
+
// #7752 kill mode: poll liveness continuously and, on a whole window of silence on EVERY
|
|
78
|
+
// channel, write the stall marker, report once, exit. The runner's shell box sweeps the CLI
|
|
79
|
+
// when it sees the marker, so a silent turn ends at the window instead of burning the box.
|
|
80
|
+
// stderr liveness is measured ROLLING here (growth since the last poll, not since arm): the
|
|
81
|
+
// question is "any bytes in this window", not "any bytes ever".
|
|
82
|
+
if (stallFile) {
|
|
83
|
+
const poll = Math.max(250, Math.min(windowMs / 4, 10000));
|
|
84
|
+
let lastErrAt = armedAt;
|
|
85
|
+
for (;;) {
|
|
86
|
+
await sleep(poll);
|
|
87
|
+
const s = readStamp();
|
|
88
|
+
if (!s || s.turn !== armed.turn || (armed.runner && s.runner !== armed.runner)) process.exit(0); // turn ended, or a NEWER runner owns the stamp now
|
|
89
|
+
if (!runnerAlive()) process.exit(0); // our runner is gone — never speak for it
|
|
90
|
+
const now = Date.now();
|
|
91
|
+
const size = errSize();
|
|
92
|
+
if (size > baseErr + 200) { baseErr = size; lastErrAt = now; } // new bytes: alive
|
|
93
|
+
const freshCut = Math.max(armedAt, now - windowMs - SLACK);
|
|
94
|
+
const tr = transcriptDir ? newestMtime(transcriptDir) : 0;
|
|
95
|
+
const wk = workDir ? newestMtime(workDir) : 0;
|
|
96
|
+
if (tr > freshCut || wk > freshCut || now - lastErrAt < windowMs) continue; // producing work: alive
|
|
97
|
+
try { writeFileSync(stallFile, ""); } catch {}
|
|
98
|
+
const mins = Math.round((now - armedAt) / 60000);
|
|
99
|
+
const orch = `${hostId()}:${project}`;
|
|
100
|
+
const text = `⏱ ${session} turn STALLED — ${mins}m with no activity (turn ${s.turn}; last seen: ${describeLast({ tr, wk })}, stderr ${baseErr > 0 ? `${baseErr}B` : "silent"}); ending the turn at the stall window, not the box.`;
|
|
101
|
+
// Direct = wake. The foreman first; if this seat IS the foreman's own runner, say it to all.
|
|
102
|
+
const to = orch === session ? "all" : orch;
|
|
103
|
+
try { await signedPost(`${hub}/send`, { from: session, to, text, project }, { session }); } catch {}
|
|
104
|
+
process.exit(0); // one report per turn, by construction
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
74
108
|
for (;;) {
|
|
75
109
|
await sleep(windowMs);
|
|
76
110
|
const s = readStamp();
|
|
@@ -12,8 +12,14 @@ const TEST_CMD_RE = /(node\s+test\/|npm\s+(run\s+)?test|pnpm\s+test|yarn\s+test|
|
|
|
12
12
|
// that says one of these is NEVER flagged, however empty its worktree is.
|
|
13
13
|
const NOCODE_RE = /\b(no[- ]code|docs?[- ]only|investigat\w*|read[- ]only|research[- ]only|analysis[- ]only|answered (on|in) the bus|refus\w+|declin\w+|won'?t fix)\b/i;
|
|
14
14
|
|
|
15
|
+
// #7754: the note names the sha it verified against, or a gate cannot tell green-on-the-wrong-base
|
|
16
|
+
// from green. Required on its own, not only when every other piece of evidence is missing too.
|
|
17
|
+
const VERIFIED_AT_RE = /\bverified at\s+([0-9a-f]{7,40})\b/i;
|
|
18
|
+
const VERIFIED_AT_MISSING = "verified-at sha in the note";
|
|
19
|
+
|
|
15
20
|
export const hasTestCommand = (note) => TEST_CMD_RE.test(String(note || ""));
|
|
16
21
|
export const declaresNoCode = (note) => NOCODE_RE.test(String(note || ""));
|
|
22
|
+
export const namesVerifiedSha = (note) => VERIFIED_AT_RE.test(String(note || ""));
|
|
17
23
|
|
|
18
24
|
function git(cwd, args) {
|
|
19
25
|
return execFileSync("git", args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim();
|
|
@@ -48,6 +54,11 @@ export function hollowVerdict(cwd, id, note, checklist) {
|
|
|
48
54
|
} catch { return { checked: false, hollow: false, missing: [] }; }
|
|
49
55
|
if (!(checklist || []).some((c) => c.done)) missing.push("ticked checklist items");
|
|
50
56
|
if (!hasTestCommand(note)) missing.push("test command in the note");
|
|
51
|
-
const
|
|
57
|
+
const evidenceless = missing.length === 4;
|
|
58
|
+
const unanchored = !namesVerifiedSha(note);
|
|
59
|
+
// The flag names only what tripped it: the four evidence gaps together, or the missing sha alone.
|
|
60
|
+
if (!evidenceless) missing.length = 0;
|
|
61
|
+
if (unanchored) missing.push(VERIFIED_AT_MISSING);
|
|
62
|
+
const hollow = (evidenceless || unanchored) && !declaresNoCode(note);
|
|
52
63
|
return { checked: true, hollow, missing };
|
|
53
64
|
}
|
package/lib/classify-failure.mjs
CHANGED
|
@@ -37,6 +37,20 @@ export function stripPromptEcho(errText, promptText) {
|
|
|
37
37
|
* printed its whole failure in under a hundred characters and produced nothing else. */
|
|
38
38
|
export const OWN_OUTPUT_ANSWER_MIN = 400;
|
|
39
39
|
|
|
40
|
+
/** #7752: what a CUT turn's exit says about WHY the CLI died. 141 is 128+13 — SIGPIPE: the time
|
|
41
|
+
* box's sweep ends a CLI still holding the turn's pipes, so the pipe death is the box's
|
|
42
|
+
* signature, recorded in the ledger row as the cut signal. Any other exit is not a signal. */
|
|
43
|
+
export function cutSignalFor(exit) {
|
|
44
|
+
return Number(exit) === 141 ? "SIGPIPE" : "";
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** #7752: the verdict line for a SILENT cut — a turn that put bytes on neither stream and
|
|
48
|
+
* advanced no transcript for the whole watchdog window, ended by the runner at the window
|
|
49
|
+
* instead of the box. Named here so the drill pins the exact string the ledger carries. */
|
|
50
|
+
export function stallVerdict() {
|
|
51
|
+
return "classified stalled because no bytes on either stream and no transcript advance for the whole watchdog window";
|
|
52
|
+
}
|
|
53
|
+
|
|
40
54
|
// ---- the hollow turn (#7759): exit 0, no worktree change, no substantive output ---------------
|
|
41
55
|
// A CLI banner is bytes on a stream, not work. ANSI stripped, a line is CHROME when it is a pure
|
|
42
56
|
// separator, decorative box/block art, or a label a CLI prints about itself (session, model,
|
|
@@ -87,6 +101,11 @@ export function classifyFailure(exit, errText, emptyOutput = false, emptyTurn =
|
|
|
87
101
|
if (emptyOutput) return { reason: "empty-output", matched: "exit 0 with no output on either stream" };
|
|
88
102
|
const t = String(errText || "").toLowerCase();
|
|
89
103
|
if (exit === 127) return { reason: "missing-cli", matched: "exit 127 — command not found" };
|
|
104
|
+
// #7099/#7752: 141 is the box's own signature (SIGPIPE — the sweep ended a CLI still holding
|
|
105
|
+
// the turn's pipes), recorded as the cut signal. The provider's answer never landed, so no
|
|
106
|
+
// quota or crash pattern may match it: a captured "rate-limit" line once parked a silent
|
|
107
|
+
// qwen seat as exhausted on a turn the box itself had killed.
|
|
108
|
+
if (Number(exit) === 141) return { reason: "cut-signal", matched: "exit 141 — SIGPIPE from the time box sweep, not a provider failure" };
|
|
90
109
|
// #5684: a provider BACKEND failure is not quota — it wants retry/swap, not a window wait.
|
|
91
110
|
// The specimen (#5683): codex's "unexpected status 404 Not Found … /responses/compact" was
|
|
92
111
|
// labelled "exhausted" and the operator was advised to wait out a window that did not exist.
|