switchroom 0.18.9 → 0.18.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +1 -0
- package/dist/auth-broker/index.js +198 -13
- package/dist/cli/notion-write-pretool.mjs +1 -0
- package/dist/cli/switchroom.js +28 -4
- package/dist/host-control/main.js +3 -2
- package/dist/vault/approvals/kernel-server.js +2 -1
- package/dist/vault/broker/server.js +2 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +119 -37
- package/profiles/_shared/dev-protocol.md.hbs +42 -0
- package/skills/dev-protocol/SKILL.md +131 -0
- package/telegram-plugin/README.md +2 -1
- package/telegram-plugin/admin-commands/dispatch.test.ts +40 -2
- package/telegram-plugin/admin-commands/index.ts +6 -1
- package/telegram-plugin/bridge/bridge.ts +23 -1
- package/telegram-plugin/bridge/crash-breadcrumb.ts +42 -0
- package/telegram-plugin/chat-lock.ts +13 -0
- package/telegram-plugin/dist/bridge/bridge.js +24 -1
- package/telegram-plugin/dist/gateway/gateway.js +1831 -263
- package/telegram-plugin/dist/server.js +29 -2
- package/telegram-plugin/fallback-card-collapse.ts +131 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +546 -0
- package/telegram-plugin/gateway/effort-command.ts +47 -3
- package/telegram-plugin/gateway/gateway.ts +1435 -211
- package/telegram-plugin/gateway/model-command.ts +94 -8
- package/telegram-plugin/gateway/pending-session-command.ts +365 -0
- package/telegram-plugin/gateway/permission-timeout.ts +25 -0
- package/telegram-plugin/gateway/resume-inbound-builder.ts +23 -3
- package/telegram-plugin/gateway/session-model-file.ts +166 -23
- package/telegram-plugin/gateway/stop-command.ts +56 -0
- package/telegram-plugin/photo-precheck.ts +201 -0
- package/telegram-plugin/quota-watch.ts +141 -2
- package/telegram-plugin/registry/subagents-schema.ts +26 -3
- package/telegram-plugin/registry/subagents.test.ts +67 -0
- package/telegram-plugin/retry-api-call.ts +31 -0
- package/telegram-plugin/subagent-watcher.ts +392 -1
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +576 -0
- package/telegram-plugin/tests/buffer-gate-broadened.test.ts +11 -5
- package/telegram-plugin/tests/chat-lock-unhandled-rejection.test.ts +101 -0
- package/telegram-plugin/tests/crash-breadcrumb.test.ts +57 -0
- package/telegram-plugin/tests/effort-command.test.ts +59 -2
- package/telegram-plugin/tests/fallback-card-collapse.test.ts +104 -0
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +124 -0
- package/telegram-plugin/tests/gateway-secret-detect.test.ts +7 -1
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +19 -11
- package/telegram-plugin/tests/model-command.test.ts +46 -3
- package/telegram-plugin/tests/pending-session-command.test.ts +322 -0
- package/telegram-plugin/tests/permission-timeout.test.ts +26 -0
- package/telegram-plugin/tests/permission-verdict-resume-guard.test.ts +16 -0
- package/telegram-plugin/tests/photo-dimension-fallback.test.ts +129 -0
- package/telegram-plugin/tests/photo-precheck.test.ts +240 -0
- package/telegram-plugin/tests/photo-reroute-wiring.test.ts +85 -0
- package/telegram-plugin/tests/quota-watch.test.ts +225 -0
- package/telegram-plugin/tests/session-model-file.test.ts +101 -2
- package/telegram-plugin/tests/stop-command.test.ts +234 -0
- package/telegram-plugin/tests/subagent-watcher-env-thresholds.test.ts +27 -9
- package/telegram-plugin/tests/subagent-watcher-resurrection.test.ts +398 -0
- package/telegram-plugin/tests/subagent-watcher-stall-terminal.test.ts +172 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +37 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +18 -4
- package/telegram-plugin/welcome-text.ts +4 -3
- package/telegram-plugin/worker-activity-feed.ts +27 -0
|
@@ -183,6 +183,20 @@ export interface ReapStuckRunningArgs {
|
|
|
183
183
|
ttlMs: number
|
|
184
184
|
/** Current time (DI for tests). */
|
|
185
185
|
now: number
|
|
186
|
+
/**
|
|
187
|
+
* Optional liveness cross-check against the in-memory file-discovery
|
|
188
|
+
* registry. Called with a candidate row's `jsonl_agent_id` (null when
|
|
189
|
+
* linkage never happened); return `true` if the watcher is actively
|
|
190
|
+
* tailing a live worker for that id. When it returns true the row is
|
|
191
|
+
* NOT reaped — the DB's `last_activity_at` is a *stale* liveness signal
|
|
192
|
+
* (it's only bumped when the JSONL is linked, and it freezes during a
|
|
193
|
+
* long in-flight tool call), so a row the watcher knows is alive must
|
|
194
|
+
* not be independently classified as dead here. The watcher owns that
|
|
195
|
+
* worker's terminal transition. Incident 2026-07-10: a live, actively-
|
|
196
|
+
* card-editing worker was reaped as terminal because this cross-check
|
|
197
|
+
* didn't exist. Omit to preserve the pre-fix unconditional behaviour.
|
|
198
|
+
*/
|
|
199
|
+
isLive?: (jsonlAgentId: string | null) => boolean
|
|
186
200
|
}
|
|
187
201
|
|
|
188
202
|
export interface ReapStuckRunningResult {
|
|
@@ -524,14 +538,23 @@ export function reapStuckRunningRows(
|
|
|
524
538
|
args: ReapStuckRunningArgs,
|
|
525
539
|
): ReapStuckRunningResult {
|
|
526
540
|
const cutoff = args.now - args.ttlMs
|
|
527
|
-
const
|
|
541
|
+
const rows = db
|
|
528
542
|
.prepare(`
|
|
529
|
-
SELECT id FROM subagents
|
|
543
|
+
SELECT id, jsonl_agent_id FROM subagents
|
|
530
544
|
WHERE status = 'running'
|
|
531
545
|
AND background = 1
|
|
532
546
|
AND COALESCE(last_activity_at, started_at) < ?
|
|
533
547
|
`)
|
|
534
|
-
.all(cutoff) as Array<{ id: string }>
|
|
548
|
+
.all(cutoff) as Array<{ id: string; jsonl_agent_id: string | null }>
|
|
549
|
+
|
|
550
|
+
// Cross-check each stale-by-DB candidate against the live in-memory
|
|
551
|
+
// registry. A worker the watcher is actively tailing is NOT dead just
|
|
552
|
+
// because its DB `last_activity_at` is stale/NULL — the watcher owns
|
|
553
|
+
// its terminal transition. Skipping these prevents the incident where a
|
|
554
|
+
// live, actively-card-editing worker was reaped as terminal.
|
|
555
|
+
const candidates = args.isLive
|
|
556
|
+
? rows.filter((r) => !args.isLive!(r.jsonl_agent_id))
|
|
557
|
+
: rows
|
|
535
558
|
|
|
536
559
|
for (const row of candidates) {
|
|
537
560
|
recordSubagentStall(db, {
|
|
@@ -665,6 +665,73 @@ describe('reapStuckRunningRows', () => {
|
|
|
665
665
|
expect(result.reaped).toBe(5)
|
|
666
666
|
expect(result.ids.sort()).toEqual(['sa-0', 'sa-1', 'sa-2', 'sa-3', 'sa-4'])
|
|
667
667
|
})
|
|
668
|
+
|
|
669
|
+
// Incident 2026-07-10: a live, actively-card-editing worker was reaped as
|
|
670
|
+
// terminal because the DB `last_activity_at` was stale (linkage never
|
|
671
|
+
// bumped it) while the in-memory file-discovery registry knew it was
|
|
672
|
+
// alive. The reaper must cross-check the live registry before reaping.
|
|
673
|
+
it('does NOT reap a row the live registry reports as an active worker', () => {
|
|
674
|
+
const db = openFreshSubagentsDbInMemory()
|
|
675
|
+
recordSubagentStart(db, {
|
|
676
|
+
id: 'sa-live',
|
|
677
|
+
background: true,
|
|
678
|
+
startedAt: 1000,
|
|
679
|
+
jsonlAgentId: 'a3c5b2d5765810114',
|
|
680
|
+
})
|
|
681
|
+
// DB liveness is stale (past ttl) — but the watcher is actively tailing
|
|
682
|
+
// this worker, so isLive returns true and the row is spared.
|
|
683
|
+
const liveIds = new Set(['a3c5b2d5765810114'])
|
|
684
|
+
const result = reapStuckRunningRows(db, {
|
|
685
|
+
ttlMs: 500,
|
|
686
|
+
now: 5000,
|
|
687
|
+
isLive: (jid) => jid != null && liveIds.has(jid),
|
|
688
|
+
})
|
|
689
|
+
expect(result.reaped).toBe(0)
|
|
690
|
+
expect(getSubagent(db, 'sa-live')!.status).toBe('running')
|
|
691
|
+
})
|
|
692
|
+
|
|
693
|
+
it('still reaps a stale row that is NOT in the live registry (genuine orphan)', () => {
|
|
694
|
+
const db = openFreshSubagentsDbInMemory()
|
|
695
|
+
recordSubagentStart(db, {
|
|
696
|
+
id: 'sa-orphan',
|
|
697
|
+
background: true,
|
|
698
|
+
startedAt: 1000,
|
|
699
|
+
jsonlAgentId: 'dead-worker',
|
|
700
|
+
})
|
|
701
|
+
// Also a live one to prove the predicate discriminates per-row.
|
|
702
|
+
recordSubagentStart(db, {
|
|
703
|
+
id: 'sa-alive',
|
|
704
|
+
background: true,
|
|
705
|
+
startedAt: 1000,
|
|
706
|
+
jsonlAgentId: 'live-worker',
|
|
707
|
+
})
|
|
708
|
+
const liveIds = new Set(['live-worker'])
|
|
709
|
+
const result = reapStuckRunningRows(db, {
|
|
710
|
+
ttlMs: 500,
|
|
711
|
+
now: 5000,
|
|
712
|
+
isLive: (jid) => jid != null && liveIds.has(jid),
|
|
713
|
+
})
|
|
714
|
+
expect(result.reaped).toBe(1)
|
|
715
|
+
expect(result.ids).toEqual(['sa-orphan'])
|
|
716
|
+
expect(getSubagent(db, 'sa-orphan')!.status).toBe('stalled')
|
|
717
|
+
expect(getSubagent(db, 'sa-alive')!.status).toBe('running')
|
|
718
|
+
})
|
|
719
|
+
|
|
720
|
+
it('reaps a stale row with NULL jsonl_agent_id (the watcher predicate never matches an unlinked row)', () => {
|
|
721
|
+
const db = openFreshSubagentsDbInMemory()
|
|
722
|
+
recordSubagentStart(db, { id: 'sa-unlinked', background: true, startedAt: 1000 })
|
|
723
|
+
// Mirror the production predicate shape: a null jsonl_agent_id can't
|
|
724
|
+
// key into the registry, so it always reports not-live and the reaper
|
|
725
|
+
// (the safety net for exactly this unlinked-orphan case) still fires.
|
|
726
|
+
const liveIds = new Set<string>()
|
|
727
|
+
const result = reapStuckRunningRows(db, {
|
|
728
|
+
ttlMs: 500,
|
|
729
|
+
now: 5000,
|
|
730
|
+
isLive: (jid) => jid != null && liveIds.has(jid),
|
|
731
|
+
})
|
|
732
|
+
expect(result.reaped).toBe(1)
|
|
733
|
+
expect(getSubagent(db, 'sa-unlinked')!.status).toBe('stalled')
|
|
734
|
+
})
|
|
668
735
|
})
|
|
669
736
|
|
|
670
737
|
// ---------------------------------------------------------------------------
|
|
@@ -371,3 +371,34 @@ export function isMessageTooLongError(err: unknown): boolean {
|
|
|
371
371
|
d.includes('text is too long')
|
|
372
372
|
)
|
|
373
373
|
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* True when Telegram rejected a `sendPhoto` / `sendMediaGroup` because the
|
|
377
|
+
* image is unusable AS A PHOTO — dimensions out of range (Telegram caps
|
|
378
|
+
* photos at width+height ≤ 10000 and aspect ratio ≤ 20), the file can't be
|
|
379
|
+
* saved as a photo, or it exceeds the photo-path size ceiling (~10MB;
|
|
380
|
+
* documents allow ~50MB). A tall phone screenshot is the canonical trigger
|
|
381
|
+
* (PHOTO_INVALID_DIMENSIONS in the #klanker 2026-07-10 incident).
|
|
382
|
+
*
|
|
383
|
+
* These 400s are deliberately NOT swallowed or retried by `retryApiCall`
|
|
384
|
+
* (only not-modified / not-found / thread-not-found are) — they surface to
|
|
385
|
+
* the caller, which recovers by re-sending the SAME file as a document
|
|
386
|
+
* (`sendDocument`) so the user still receives it. Same "caller-level
|
|
387
|
+
* fallback" shape as the THREAD_NOT_FOUND and isHtmlParseRejectError
|
|
388
|
+
* contracts above.
|
|
389
|
+
*/
|
|
390
|
+
export function isPhotoDimensionRejectError(err: unknown): boolean {
|
|
391
|
+
if (!(err instanceof GrammyError) || err.error_code !== 400) return false
|
|
392
|
+
const d = (err.description || '').toLowerCase()
|
|
393
|
+
return (
|
|
394
|
+
d.includes('photo_invalid_dimensions') ||
|
|
395
|
+
d.includes('photo_save_file_invalid') ||
|
|
396
|
+
d.includes('photo dimensions') ||
|
|
397
|
+
d.includes('image_process_failed') ||
|
|
398
|
+
// size-driven rejections of the photo path
|
|
399
|
+
d.includes('photo is too big') ||
|
|
400
|
+
d.includes('too big for a photo') ||
|
|
401
|
+
d.includes('image is too big') ||
|
|
402
|
+
d.includes('file is too big')
|
|
403
|
+
)
|
|
404
|
+
}
|
|
@@ -140,6 +140,21 @@ export interface WorkerEntry {
|
|
|
140
140
|
* the renderer only ever shows the latest.
|
|
141
141
|
*/
|
|
142
142
|
lastTool: { name: string; sanitisedArg: string } | null
|
|
143
|
+
/**
|
|
144
|
+
* Tool-use ids for tool calls this worker has STARTED but not yet
|
|
145
|
+
* finished — a `sub_agent_tool_use` was observed with no matching
|
|
146
|
+
* `sub_agent_tool_result` yet. A non-empty set means the worker is
|
|
147
|
+
* currently *inside* a tool call (e.g. a long-running `Bash` frame-
|
|
148
|
+
* capture loop that can legally run 10+ minutes with zero JSONL
|
|
149
|
+
* growth). The silent-stall terminal synthesis (checkStalls Pass 2)
|
|
150
|
+
* MUST NOT fire while this is non-empty: a frozen `lastActivityAt`
|
|
151
|
+
* during an in-flight tool call is expected, not a dead worker.
|
|
152
|
+
* Cleared on the matching `sub_agent_tool_result` and on
|
|
153
|
+
* `sub_agent_turn_end` (belt-and-braces). Tool-use lines with a null
|
|
154
|
+
* `toolUseId` can't be paired, so they don't participate — Bash and
|
|
155
|
+
* every real long-runner always carries a `toolu_…` id.
|
|
156
|
+
*/
|
|
157
|
+
inflightToolUseIds: Set<string>
|
|
143
158
|
/**
|
|
144
159
|
* True if the underlying JSONL file existed before the watcher started.
|
|
145
160
|
* Historical entries are tracked for late state transitions but are
|
|
@@ -300,6 +315,21 @@ export interface SubagentWatcherConfig {
|
|
|
300
315
|
* synthesis; tests use a tiny value to exercise the path.
|
|
301
316
|
*/
|
|
302
317
|
silentStallTerminalMs?: number
|
|
318
|
+
/**
|
|
319
|
+
* Upper bound (ms of total JSONL idle) on the in-flight tool-call
|
|
320
|
+
* deferral of terminal synthesis. While a tool call is in flight
|
|
321
|
+
* (tool_use seen, matching tool_result not yet), synthesis is deferred —
|
|
322
|
+
* a long `Bash` legitimately freezes the JSONL for 10+ min (incident
|
|
323
|
+
* 2026-07-10). But a worker that DIES mid-tool (process killed; JSONL
|
|
324
|
+
* stops growing but is never deleted) would otherwise defer forever, and
|
|
325
|
+
* the reaper's isLive cross-check would shield it from the DB net too —
|
|
326
|
+
* wedged for the life of the gateway. Past this cap the deferral ends
|
|
327
|
+
* and synthesis proceeds. Default 45 min
|
|
328
|
+
* (DEFAULT_INFLIGHT_TERMINAL_CAP_MS) — above any legitimate single tool
|
|
329
|
+
* call, below the 1h reaper TTL. Env override
|
|
330
|
+
* `SWITCHROOM_SUBAGENT_INFLIGHT_TERMINAL_CAP_MS`.
|
|
331
|
+
*/
|
|
332
|
+
inflightTerminalCapMs?: number
|
|
303
333
|
/**
|
|
304
334
|
* Freshness window (ms) for promoting a running-at-boot worker file to
|
|
305
335
|
* live. A file whose last write (mtime) is older than this is treated as
|
|
@@ -409,6 +439,34 @@ export interface SubagentWatcherConfig {
|
|
|
409
439
|
* (the handback) regardless. Retained as an unwired hook.
|
|
410
440
|
*/
|
|
411
441
|
onStallTerminal?: (agentId: string, description: string) => void
|
|
442
|
+
/**
|
|
443
|
+
* Issue #3023 (card resurrection). Fires when a worker whose card was
|
|
444
|
+
* FALSELY finalised — its terminal state came from silent-stall synthesis
|
|
445
|
+
* (`onStallTerminal` above), NOT a real `sub_agent_turn_end` — resumes
|
|
446
|
+
* writing to its JSONL after the synthesis. The synthesis was wrong: the
|
|
447
|
+
* worker is still alive, so the operator invariant ("active work must
|
|
448
|
+
* always be visible") requires its progress surface to come back. The
|
|
449
|
+
* watcher re-registers the worker as a LIVE, non-historical entry (so
|
|
450
|
+
* `onProgress` / stall-detection / the real handback resume), and fires
|
|
451
|
+
* this so the gateway can revive the worker's activity card (clear the
|
|
452
|
+
* feed's finalized gate → repaint on the next progress tick).
|
|
453
|
+
*
|
|
454
|
+
* GUARD (at-most-once per false finish + bounded chain): a given false
|
|
455
|
+
* finish is resurrected at most once. A worker that is resurrected and
|
|
456
|
+
* then falsely finalised AGAIN is NOT resurrected a second time — it is
|
|
457
|
+
* named-as-lost via `onWorkerLost` instead, so a pathological worker
|
|
458
|
+
* can't loop finish→resurrect forever.
|
|
459
|
+
*/
|
|
460
|
+
onResurrect?: (agentId: string, description: string) => void
|
|
461
|
+
/**
|
|
462
|
+
* Issue #3023 (bounded resurrection chain). Fires when a worker that was
|
|
463
|
+
* already resurrected once is falsely finalised a second time and its
|
|
464
|
+
* JSONL resumes yet again. Rather than resurrect it forever, the watcher
|
|
465
|
+
* declares it LOST (a single log line names it) and stops resurrecting.
|
|
466
|
+
* The gateway may surface this however it likes; the invariant is only
|
|
467
|
+
* that the chain is bounded, not that a lost worker gets a fresh card.
|
|
468
|
+
*/
|
|
469
|
+
onWorkerLost?: (agentId: string, description: string) => void
|
|
412
470
|
/**
|
|
413
471
|
* Called exactly once per sub-agent when its watcher observes a terminal
|
|
414
472
|
* transition (`done` or `failed`). Mirrors the existing `sub_agent_started`
|
|
@@ -536,6 +594,12 @@ const DEFAULT_SILENT_SYNTHESIS_STALL_THRESHOLD_MS = 300_000
|
|
|
536
594
|
* ceiling that closed-out cards used to wait on.
|
|
537
595
|
*/
|
|
538
596
|
const DEFAULT_SILENT_STALL_TERMINAL_MS = 300_000
|
|
597
|
+
// Upper bound on the in-flight tool-call deferral of terminal synthesis.
|
|
598
|
+
// 45 min: comfortably above any legitimate single tool call (Bash caps at
|
|
599
|
+
// 10 min per call; the incident loop ran ~10 min) but below the 1h DB
|
|
600
|
+
// reaper TTL, so the watcher — not the reaper — still owns the terminal
|
|
601
|
+
// transition for a worker that died mid-tool.
|
|
602
|
+
const DEFAULT_INFLIGHT_TERMINAL_CAP_MS = 45 * 60_000
|
|
539
603
|
|
|
540
604
|
/**
|
|
541
605
|
* Tools that legitimately run for minutes with ZERO intervening JSONL
|
|
@@ -553,6 +617,21 @@ const DEFAULT_SILENT_STALL_TERMINAL_MS = 300_000
|
|
|
553
617
|
*/
|
|
554
618
|
const LONG_RUNNING_TOOLS: ReadonlySet<string> = new Set(['Bash'])
|
|
555
619
|
|
|
620
|
+
/**
|
|
621
|
+
* Issue #3023 (bounded resurrection chain). Maximum number of times a single
|
|
622
|
+
* worker may have its falsely-finalised card resurrected. Once a worker has
|
|
623
|
+
* been resurrected this many times and is falsely finalised AGAIN, the next
|
|
624
|
+
* post-terminal JSONL resumption names it LOST rather than resurrecting it —
|
|
625
|
+
* so a pathological finish→resurrect→finish loop is bounded, not infinite.
|
|
626
|
+
* One resurrection is enough to recover the real 2026-07-10 incident (a
|
|
627
|
+
* single false finish from one over-long tool call); a second false finish on
|
|
628
|
+
* the same worker is a signal the heuristics can't track it, so we stop.
|
|
629
|
+
*/
|
|
630
|
+
const MAX_RESURRECTIONS = 1
|
|
631
|
+
/** Cap on the false-finish tracker so it can't grow unbounded over a
|
|
632
|
+
* long-lived gateway; oldest record is evicted FIFO past this. */
|
|
633
|
+
const FALSE_FINISH_TRACKER_CAP = 512
|
|
634
|
+
|
|
556
635
|
/** True when the tool named legitimately runs quiet for minutes (see
|
|
557
636
|
* LONG_RUNNING_TOOLS). Null/undefined tool name → false. */
|
|
558
637
|
function isLongRunningTool(name: string | null | undefined): boolean {
|
|
@@ -1144,6 +1223,14 @@ export function readSubTail(
|
|
|
1144
1223
|
entry.lastReplyText = ev.input.text as string
|
|
1145
1224
|
}
|
|
1146
1225
|
entry.toolCount++
|
|
1226
|
+
// Track this as an IN-FLIGHT tool call so the silent-stall
|
|
1227
|
+
// terminal synthesis (checkStalls Pass 2) doesn't misread the
|
|
1228
|
+
// frozen JSONL of a long-running tool (a 10-min `Bash` loop) as
|
|
1229
|
+
// a dead worker. Only pairable (non-null id) tool_uses count —
|
|
1230
|
+
// Bash + every real long-runner always carries a `toolu_…` id.
|
|
1231
|
+
if (ev.toolUseId != null && ev.toolUseId !== '') {
|
|
1232
|
+
entry.inflightToolUseIds.add(ev.toolUseId)
|
|
1233
|
+
}
|
|
1147
1234
|
// P0 of #662: surface the most recent tool name + sanitised
|
|
1148
1235
|
// arg so the driver's fleet-state shadow can render the
|
|
1149
1236
|
// last-tool column on the v2 status card. Sanitiser lives in
|
|
@@ -1197,6 +1284,16 @@ export function readSubTail(
|
|
|
1197
1284
|
}
|
|
1198
1285
|
}
|
|
1199
1286
|
} else if (ev.kind === 'sub_agent_nested_spawn') {
|
|
1287
|
+
// A nested Agent/Task dispatch is the same frozen-JSONL shape as
|
|
1288
|
+
// a long tool call: a FOREGROUND nested child blocks this worker
|
|
1289
|
+
// until it returns, with the matching tool_result only landing
|
|
1290
|
+
// then — so gate terminal synthesis on it too. The existing
|
|
1291
|
+
// `sub_agent_tool_result` handler clears the id (a background
|
|
1292
|
+
// nested dispatch's "launched" result lands almost immediately,
|
|
1293
|
+
// so it barely defers). Same cap applies.
|
|
1294
|
+
if (ev.toolUseId != null && ev.toolUseId.length > 0) {
|
|
1295
|
+
entry.inflightToolUseIds.add(ev.toolUseId)
|
|
1296
|
+
}
|
|
1200
1297
|
// Nested (depth-2+) dispatch keying: this worker just dispatched a
|
|
1201
1298
|
// sub-agent of its own. The PreToolUse hook can't attribute it (the
|
|
1202
1299
|
// main turn's turn-active.json marker is long gone for a background
|
|
@@ -1254,7 +1351,17 @@ export function readSubTail(
|
|
|
1254
1351
|
fireNarrativeProgress() // prior pending was pure narration → SHOW
|
|
1255
1352
|
}
|
|
1256
1353
|
entry.pendingNarrative = { text: ev.text }
|
|
1354
|
+
} else if (ev.kind === 'sub_agent_tool_result') {
|
|
1355
|
+
// The tool call completed — clear it from the in-flight set so
|
|
1356
|
+
// the terminal-synthesis gate re-opens. Idempotent: a result
|
|
1357
|
+
// whose tool_use we never tracked (null id, parallel spill) is
|
|
1358
|
+
// a harmless no-op delete.
|
|
1359
|
+
if (ev.toolUseId != null && ev.toolUseId !== '') {
|
|
1360
|
+
entry.inflightToolUseIds.delete(ev.toolUseId)
|
|
1361
|
+
}
|
|
1257
1362
|
} else if (ev.kind === 'sub_agent_turn_end') {
|
|
1363
|
+
// Belt-and-braces: a turn boundary means nothing is in flight.
|
|
1364
|
+
entry.inflightToolUseIds.clear()
|
|
1258
1365
|
// Narrative-dedup gate step 3: a trailing sub_agent_text block with
|
|
1259
1366
|
// nothing after it. SUPPRESS only when it drafts the foreground
|
|
1260
1367
|
// sub-agent's delivered reply (entry.lastReplyText, set above on a
|
|
@@ -1346,6 +1453,10 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
1346
1453
|
config.silentStallTerminalMs
|
|
1347
1454
|
?? parseEnvMs('SWITCHROOM_SUBAGENT_STALL_TERMINAL_MS')
|
|
1348
1455
|
?? DEFAULT_SILENT_STALL_TERMINAL_MS
|
|
1456
|
+
const inflightTerminalCapMs =
|
|
1457
|
+
config.inflightTerminalCapMs
|
|
1458
|
+
?? parseEnvMs('SWITCHROOM_SUBAGENT_INFLIGHT_TERMINAL_CAP_MS')
|
|
1459
|
+
?? DEFAULT_INFLIGHT_TERMINAL_CAP_MS
|
|
1349
1460
|
const inflightPromoteMaxAgeMs =
|
|
1350
1461
|
config.inflightPromoteMaxAgeMs
|
|
1351
1462
|
?? parseEnvMs('SWITCHROOM_SUBAGENT_INFLIGHT_MAX_AGE_MS')
|
|
@@ -1428,6 +1539,39 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
1428
1539
|
* terminal JSONLs as a no-op.
|
|
1429
1540
|
*/
|
|
1430
1541
|
const terminatedAgentIds = new Set<string>()
|
|
1542
|
+
/**
|
|
1543
|
+
* Issue #3023 (card resurrection). Per-worker record of a FALSE terminal
|
|
1544
|
+
* finish — a terminal state produced by silent-stall synthesis (NOT a real
|
|
1545
|
+
* `sub_agent_turn_end`). Keyed by agentId; SURVIVES `cleanupTerminalAgent`
|
|
1546
|
+
* (which drops the registry entry) precisely so a post-terminal JSONL
|
|
1547
|
+
* resumption can be detected after the entry is gone.
|
|
1548
|
+
*
|
|
1549
|
+
* A genuine completion (`turn_end`) or a genuine boot-time historical
|
|
1550
|
+
* rediscovery NEVER writes here — only synthesis does — so those paths can
|
|
1551
|
+
* never be resurrected. This map IS the discriminator between "old
|
|
1552
|
+
* completed worker rediscovered at boot" (no record → stays suppressed)
|
|
1553
|
+
* and "worker I finalised moments ago whose JSONL just grew again"
|
|
1554
|
+
* (record present → resurrect).
|
|
1555
|
+
*
|
|
1556
|
+
* - `synthesisedAt` — wall-clock ms the false finish fired.
|
|
1557
|
+
* - `sizeAtSynthesis`— JSONL byte size at that moment; post-terminal
|
|
1558
|
+
* growth past this proves the worker resumed.
|
|
1559
|
+
* - `resurrectionCount` — times this worker has been resurrected. Bounds
|
|
1560
|
+
* the chain: once it reaches MAX, the next false
|
|
1561
|
+
* finish is named-as-lost, not resurrected.
|
|
1562
|
+
* - `resurrectedForThisFinish` — at-most-once guard for the CURRENT false
|
|
1563
|
+
* finish; reset when a new synthesis records over it.
|
|
1564
|
+
* - `lost` — the chain bound was hit; no further resurrection.
|
|
1565
|
+
*/
|
|
1566
|
+
interface FalseFinishRecord {
|
|
1567
|
+
filePath: string
|
|
1568
|
+
synthesisedAt: number
|
|
1569
|
+
sizeAtSynthesis: number
|
|
1570
|
+
resurrectionCount: number
|
|
1571
|
+
resurrectedForThisFinish: boolean
|
|
1572
|
+
lost: boolean
|
|
1573
|
+
}
|
|
1574
|
+
const falseFinishTracker = new Map<string, FalseFinishRecord>()
|
|
1431
1575
|
/**
|
|
1432
1576
|
* True while the initial boot scan is running. During this window every
|
|
1433
1577
|
* newly discovered file is added to historicalFiles.
|
|
@@ -1463,6 +1607,7 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
1463
1607
|
lastResultText: '',
|
|
1464
1608
|
lastProgressBucketIdx: null,
|
|
1465
1609
|
lastTool: null,
|
|
1610
|
+
inflightToolUseIds: new Set<string>(),
|
|
1466
1611
|
historical: isHistorical,
|
|
1467
1612
|
}
|
|
1468
1613
|
registry.set(agentId, entry)
|
|
@@ -1778,6 +1923,188 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
1778
1923
|
log?.(`subagent-watcher: cleaned up terminal agent ${agentId}`)
|
|
1779
1924
|
}
|
|
1780
1925
|
|
|
1926
|
+
// ─── Card resurrection (issue #3023) ─────────────────────────────────────
|
|
1927
|
+
|
|
1928
|
+
/**
|
|
1929
|
+
* Record a FALSE terminal finish (silent-stall synthesis). Preserves any
|
|
1930
|
+
* carried-over `resurrectionCount` from a prior false finish on the same
|
|
1931
|
+
* worker (bounded-chain guard) while re-arming the per-finish at-most-once
|
|
1932
|
+
* guard. Best-effort file-size snapshot: an unreadable stat records 0, so
|
|
1933
|
+
* ANY later readable growth still counts as a resumption.
|
|
1934
|
+
*/
|
|
1935
|
+
function recordFalseFinish(agentId: string, filePath: string, n: number): void {
|
|
1936
|
+
let size = 0
|
|
1937
|
+
try {
|
|
1938
|
+
size = fs.statSync(filePath).size
|
|
1939
|
+
} catch { /* unreadable → 0, any later growth still trips resumption */ }
|
|
1940
|
+
const prior = falseFinishTracker.get(agentId)
|
|
1941
|
+
if (prior == null && falseFinishTracker.size >= FALSE_FINISH_TRACKER_CAP) {
|
|
1942
|
+
const oldest = falseFinishTracker.keys().next().value
|
|
1943
|
+
if (oldest != null) falseFinishTracker.delete(oldest)
|
|
1944
|
+
}
|
|
1945
|
+
falseFinishTracker.set(agentId, {
|
|
1946
|
+
filePath,
|
|
1947
|
+
synthesisedAt: n,
|
|
1948
|
+
sizeAtSynthesis: size,
|
|
1949
|
+
resurrectionCount: prior?.resurrectionCount ?? 0,
|
|
1950
|
+
resurrectedForThisFinish: false,
|
|
1951
|
+
// `lost` is sticky: once a worker is named-lost it stays lost.
|
|
1952
|
+
lost: prior?.lost ?? false,
|
|
1953
|
+
})
|
|
1954
|
+
}
|
|
1955
|
+
|
|
1956
|
+
/**
|
|
1957
|
+
* Bring a falsely-finalised worker back to LIVE. Cancels any pending
|
|
1958
|
+
* terminal cleanup and clears the terminated/historical suppression, then
|
|
1959
|
+
* revives via one of two branches so `onProgress`, stall-detection and a
|
|
1960
|
+
* genuine future handback all resume:
|
|
1961
|
+
*
|
|
1962
|
+
* 1. IN-PLACE REVIVE (registry entry still present — cleanup grace hadn't
|
|
1963
|
+
* elapsed): flip the existing entry back to `running` and reset its
|
|
1964
|
+
* stall/completion flags. The entry KEEPS its existing tail cursor and
|
|
1965
|
+
* FSWatcher, so the card repaints INCREMENTALLY from where it left off —
|
|
1966
|
+
* it does NOT re-read from cursor 0.
|
|
1967
|
+
* 2. SWEPT RE-REGISTER (entry already dropped by cleanupTerminalAgent):
|
|
1968
|
+
* `registerAgent` re-registers the JSONL as a fresh non-historical
|
|
1969
|
+
* entry, re-reading from cursor 0 to rebuild the worker's activity so
|
|
1970
|
+
* the revived card catches up instead of showing a frozen stub.
|
|
1971
|
+
*/
|
|
1972
|
+
function resurrectAgent(agentId: string, filePath: string): void {
|
|
1973
|
+
const pc = pendingCloses.get(agentId)
|
|
1974
|
+
if (pc != null) {
|
|
1975
|
+
clearT(pc)
|
|
1976
|
+
pendingCloses.delete(agentId)
|
|
1977
|
+
}
|
|
1978
|
+
terminatedAgentIds.delete(agentId)
|
|
1979
|
+
historicalFiles.delete(filePath)
|
|
1980
|
+
|
|
1981
|
+
const existing = registry.get(agentId)
|
|
1982
|
+
if (existing != null) {
|
|
1983
|
+
// Cleanup grace hadn't elapsed — revive the live entry in place so the
|
|
1984
|
+
// existing tail/FSWatcher keeps feeding it.
|
|
1985
|
+
existing.state = 'running'
|
|
1986
|
+
existing.historical = false
|
|
1987
|
+
existing.stallNotified = false
|
|
1988
|
+
existing.stalledAt = null
|
|
1989
|
+
existing.stallTerminalSynthesised = false
|
|
1990
|
+
// Re-arm the completion notification INTENTIONALLY (issue #3023). The
|
|
1991
|
+
// false synthesized finish already fired `onFinish` once, delivering a
|
|
1992
|
+
// (possibly wrong / incomplete) synthesized handback to the parent.
|
|
1993
|
+
// Clearing this lets a genuine future `sub_agent_turn_end` fire `onFinish`
|
|
1994
|
+
// a SECOND time — and that is the desired behaviour: the corrected REAL
|
|
1995
|
+
// result must reach the parent, superseding the false one. This does NOT
|
|
1996
|
+
// double-deliver harmfully: the gateway's handback spool dedups only
|
|
1997
|
+
// CONCURRENTLY-LIVE envelopes for the same worker (inbound-spool.ts:
|
|
1998
|
+
// `s:handback:<agentId>` — `live.has(id)` guard, no permanent tombstone),
|
|
1999
|
+
// so the first (synthesized) handback drains + acks as a normal turn, is
|
|
2000
|
+
// removed from the live set, and the later real handback is then
|
|
2001
|
+
// delivered as a fresh turn rather than being suppressed. See the
|
|
2002
|
+
// "resurrected worker's real turn_end re-fires onFinish" test.
|
|
2003
|
+
existing.completionNotified = false
|
|
2004
|
+
existing.errored = false
|
|
2005
|
+
existing.errorDetail = undefined
|
|
2006
|
+
// The worker just proved it is alive (its JSONL grew), so restart the
|
|
2007
|
+
// stall clock from now — otherwise the pre-synthesis idle would carry
|
|
2008
|
+
// over and a fresh stall/synthesis could fire almost immediately.
|
|
2009
|
+
existing.lastActivityAt = nowFn()
|
|
2010
|
+
knownFiles.add(filePath)
|
|
2011
|
+
// KNOWN LIMITATION (issue #3023): we revive the IN-MEMORY registry entry
|
|
2012
|
+
// to `running`, but the subagents DB row stays `completed`/`failed` (the
|
|
2013
|
+
// stall-synthesis path wrote a terminal row via recordSubagentEnd). We do
|
|
2014
|
+
// NOT flip it back: `recordSubagentResume` only reverses `stalled→running`
|
|
2015
|
+
// by design ("terminal beats both stalled and running" — see its doc),
|
|
2016
|
+
// and there is no terminal→running edge because the stuck-row reaper and
|
|
2017
|
+
// audit consistency both lean on terminal being final. Consequence:
|
|
2018
|
+
// `countRunningBackgroundSubagents` (WHERE status='running') UNDERCOUNTS a
|
|
2019
|
+
// resurrected worker until its genuine `turn_end` re-terminates the entry.
|
|
2020
|
+
// Impact is bounded and benign — that count only gates the deferred-👍
|
|
2021
|
+
// reaction promotion (reaction-defer.ts), so at worst a 👍 promotes one
|
|
2022
|
+
// resurrection-window early; it is never a delivery/correctness path. Left
|
|
2023
|
+
// as a known limitation rather than punching a terminal→running hole in
|
|
2024
|
+
// the schema invariant.
|
|
2025
|
+
return
|
|
2026
|
+
}
|
|
2027
|
+
// Entry was already swept — re-register from scratch as a live worker.
|
|
2028
|
+
knownFiles.add(filePath)
|
|
2029
|
+
registerAgent(filePath, agentId)
|
|
2030
|
+
}
|
|
2031
|
+
|
|
2032
|
+
/**
|
|
2033
|
+
* Detect post-terminal JSONL resumption for falsely-finalised workers and
|
|
2034
|
+
* either resurrect the card (once per false finish) or name the worker lost
|
|
2035
|
+
* (bounded chain). Called on every poll tick.
|
|
2036
|
+
*/
|
|
2037
|
+
function checkResurrections(): void {
|
|
2038
|
+
const n = nowFn()
|
|
2039
|
+
for (const [agentId, rec] of falseFinishTracker) {
|
|
2040
|
+
if (rec.lost) continue
|
|
2041
|
+
if (rec.resurrectedForThisFinish) continue
|
|
2042
|
+
let size: number
|
|
2043
|
+
try {
|
|
2044
|
+
size = fs.statSync(rec.filePath).size
|
|
2045
|
+
} catch {
|
|
2046
|
+
continue // file vanished / unreadable — nothing to resurrect
|
|
2047
|
+
}
|
|
2048
|
+
// Growth past the synthesis snapshot is the proof the worker resumed.
|
|
2049
|
+
// (The tracker itself already excludes genuine completions and boot-time
|
|
2050
|
+
// historical rediscoveries — they never record here — so size growth is
|
|
2051
|
+
// a sufficient signal; no mtime dependence needed.)
|
|
2052
|
+
//
|
|
2053
|
+
// BOUNDED FALSE-POSITIVE (issue #3023, accepted): ANY byte growth past
|
|
2054
|
+
// the snapshot trips resurrection, including a late buffered `turn_end`
|
|
2055
|
+
// flush from a genuinely-done worker. If the silent-stall synthesis fired
|
|
2056
|
+
// moments before Claude Code finally flushed the worker's real
|
|
2057
|
+
// `sub_agent_turn_end` line, that trailing write grows the JSONL and we
|
|
2058
|
+
// resurrect a worker that is actually finished — a benign
|
|
2059
|
+
// resurrect→immediate-refinish flicker that burns ONE unit of the
|
|
2060
|
+
// resurrection budget (MAX_RESURRECTIONS). We accept this because (a) the
|
|
2061
|
+
// flicker is self-healing: the very next poll sees the terminal line,
|
|
2062
|
+
// fires a real `turn_end`, and re-finalises the card; (b) the corrected
|
|
2063
|
+
// real handback still reaches the parent (see the completionNotified
|
|
2064
|
+
// reset in resurrectAgent); and (c) the alternative — parsing the tail to
|
|
2065
|
+
// distinguish a real turn_end flush from live resumption — is far more
|
|
2066
|
+
// fragile than a bounded, harmless re-finish. The chain bound guarantees
|
|
2067
|
+
// this can never loop. Covered by the "late turn_end after synthesis"
|
|
2068
|
+
// resurrection test.
|
|
2069
|
+
if (size <= rec.sizeAtSynthesis) continue
|
|
2070
|
+
|
|
2071
|
+
const entry = registry.get(agentId)
|
|
2072
|
+
const description = entry?.description ?? 'sub-agent'
|
|
2073
|
+
|
|
2074
|
+
if (rec.resurrectionCount >= MAX_RESURRECTIONS) {
|
|
2075
|
+
// Bounded-chain guard: this worker was already resurrected once and
|
|
2076
|
+
// has now falsely finished again. Do NOT resurrect forever — name it
|
|
2077
|
+
// lost and stop. One log line names the worker for the operator.
|
|
2078
|
+
rec.lost = true
|
|
2079
|
+
log?.(`subagent-watcher: worker ${agentId} FALSELY finalised again after a prior resurrection (resurrectionCount=${rec.resurrectionCount} >= ${MAX_RESURRECTIONS}) — NAMED AS LOST, not resurrecting again (bounded resurrection chain, issue #3023)`)
|
|
2080
|
+
if (config.onWorkerLost != null) {
|
|
2081
|
+
try {
|
|
2082
|
+
config.onWorkerLost(agentId, description)
|
|
2083
|
+
} catch (cbErr) {
|
|
2084
|
+
log?.(`subagent-watcher: onWorkerLost callback error ${agentId}: ${(cbErr as Error).message}`)
|
|
2085
|
+
}
|
|
2086
|
+
}
|
|
2087
|
+
continue
|
|
2088
|
+
}
|
|
2089
|
+
|
|
2090
|
+
// Resurrect: at-most-once for THIS false finish.
|
|
2091
|
+
rec.resurrectionCount++
|
|
2092
|
+
rec.resurrectedForThisFinish = true
|
|
2093
|
+
log?.(`subagent-watcher: RESURRECTING worker ${agentId} — JSONL resumed growing (${rec.sizeAtSynthesis} → ${size} bytes) after a false terminal synthesis; the worker is still alive, reviving its card (resurrection #${rec.resurrectionCount}, issue #3023)`)
|
|
2094
|
+
// Fire onResurrect FIRST so the gateway clears the feed's finalized
|
|
2095
|
+
// gate before the re-registration below replays onProgress ticks —
|
|
2096
|
+
// otherwise those first ticks would be swallowed by the finalized gate.
|
|
2097
|
+
if (config.onResurrect != null) {
|
|
2098
|
+
try {
|
|
2099
|
+
config.onResurrect(agentId, description)
|
|
2100
|
+
} catch (cbErr) {
|
|
2101
|
+
log?.(`subagent-watcher: onResurrect callback error ${agentId}: ${(cbErr as Error).message}`)
|
|
2102
|
+
}
|
|
2103
|
+
}
|
|
2104
|
+
resurrectAgent(agentId, rec.filePath)
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2107
|
+
|
|
1781
2108
|
// ─── Stall detection ────────────────────────────────────────────────────
|
|
1782
2109
|
|
|
1783
2110
|
function checkStalls(): void {
|
|
@@ -1862,11 +2189,57 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
1862
2189
|
if (entry.stallTerminalSynthesised) continue
|
|
1863
2190
|
if (entry.stalledAt == null) continue
|
|
1864
2191
|
if (n - entry.stalledAt < silentStallTerminalMs) continue
|
|
2192
|
+
// In-flight tool-call gate (incident 2026-07-10): a worker whose
|
|
2193
|
+
// transcript ends in a `tool_use` with no matching `tool_result`
|
|
2194
|
+
// yet is NOT silent — it's *inside* a long tool call (a `Bash`
|
|
2195
|
+
// frame-capture loop legally runs 10+ minutes with zero JSONL
|
|
2196
|
+
// growth). Synthesising `sub_agent_turn_end` here finalises a live
|
|
2197
|
+
// worker's card while it keeps running (real incident: card 16201
|
|
2198
|
+
// reaped at t+13min, worker resumed 92ms later with no card).
|
|
2199
|
+
//
|
|
2200
|
+
// The deferral is CAPPED, not unconditional (design reconciliation
|
|
2201
|
+
// with #2777/#2782, whose contract is "a bg worker JSONL that
|
|
2202
|
+
// legitimately lacks turn_end must still release the completion
|
|
2203
|
+
// gate"): a worker that DIES mid-tool (process killed; JSONL frozen
|
|
2204
|
+
// but never deleted) would otherwise defer forever, and the reaper's
|
|
2205
|
+
// isLive cross-check would shield it from the DB net too. So: while
|
|
2206
|
+
// a tool call is in flight, defer synthesis up to
|
|
2207
|
+
// `inflightTerminalCapMs` of total JSONL idle (default 45 min — far
|
|
2208
|
+
// above any legitimate single tool call, below the 1h reaper TTL);
|
|
2209
|
+
// past the cap, synthesis proceeds. A tool_result landing at any
|
|
2210
|
+
// point drains the set and re-arms normal detection via the
|
|
2211
|
+
// un-stall path.
|
|
2212
|
+
if (entry.inflightToolUseIds.size > 0) {
|
|
2213
|
+
const totalIdleMs = n - entry.lastActivityAt
|
|
2214
|
+
if (totalIdleMs < inflightTerminalCapMs) {
|
|
2215
|
+
log?.(`subagent-watcher: silent-stall terminal synthesis deferred for ${entry.agentId} — ${entry.inflightToolUseIds.size} tool call(s) still in flight, ${Math.floor(totalIdleMs / 1000)}s idle < ${Math.floor(inflightTerminalCapMs / 1000)}s cap (long-running tool, not a dead worker)`)
|
|
2216
|
+
continue
|
|
2217
|
+
}
|
|
2218
|
+
log?.(`subagent-watcher: in-flight deferral cap reached for ${entry.agentId} (${Math.floor(totalIdleMs / 1000)}s idle >= ${Math.floor(inflightTerminalCapMs / 1000)}s cap with ${entry.inflightToolUseIds.size} tool call(s) still unresolved) — treating as died-mid-tool, proceeding with terminal synthesis`)
|
|
2219
|
+
}
|
|
2220
|
+
// TODO(#3023/PR #3029): the cap-reached path above is a SECOND source of
|
|
2221
|
+
// possibly-false terminal synthesis (a worker mid-very-long-tool that is
|
|
2222
|
+
// NOT dead gets finalised here). PR #3029 adds `recordFalseFinish(...)`
|
|
2223
|
+
// to this synthesis block so a later JSONL resumption can resurrect the
|
|
2224
|
+
// card. When #3029 lands, make sure the merge resolution keeps
|
|
2225
|
+
// `recordFalseFinish(entry.agentId, entry.filePath, n)` covering BOTH
|
|
2226
|
+
// the plain silent-stall path and this cap-reached fall-through (they
|
|
2227
|
+
// share this block, so a clean merge does — verify at conflict time).
|
|
2228
|
+
// Intentionally NOT wired here to keep this PR standalone (no
|
|
2229
|
+
// cross-PR dependency; recordFalseFinish does not exist on this branch).
|
|
1865
2230
|
entry.stallTerminalSynthesised = true
|
|
1866
2231
|
entry.state = 'done'
|
|
1867
2232
|
const postStallSec = Math.floor((n - entry.stalledAt) / 1000)
|
|
1868
2233
|
const totalIdleSec = Math.floor((n - entry.lastActivityAt) / 1000)
|
|
1869
2234
|
log?.(`subagent-watcher: silent-stall terminal synthesis for ${entry.agentId} (stalled ${postStallSec}s post-notify, ${totalIdleSec}s total idle) — bg worker JSONL lacks turn_end; synthesising sub_agent_turn_end so deferred-completion gate releases`)
|
|
2235
|
+
// Issue #3023: this terminal state is SYNTHESISED, not a real
|
|
2236
|
+
// `turn_end` — it may be wrong (the worker could still be alive). Record
|
|
2237
|
+
// a false-finish so a later JSONL resumption can resurrect the card. The
|
|
2238
|
+
// record survives cleanupTerminalAgent (keyed by agentId, in its own
|
|
2239
|
+
// map). A carried-over resurrectionCount from a PRIOR false finish is
|
|
2240
|
+
// preserved so the chain stays bounded; resurrectedForThisFinish resets
|
|
2241
|
+
// to arm the at-most-once guard for THIS finish.
|
|
2242
|
+
recordFalseFinish(entry.agentId, entry.filePath, n)
|
|
1870
2243
|
// Persist completion to the registry DB so reaper / audit paths
|
|
1871
2244
|
// see the same terminal state as the JSONL-driven path.
|
|
1872
2245
|
if (db != null) {
|
|
@@ -2087,6 +2460,10 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
2087
2460
|
|
|
2088
2461
|
// Stall detection
|
|
2089
2462
|
checkStalls()
|
|
2463
|
+
|
|
2464
|
+
// Issue #3023: revive any worker whose falsely-finalised card's JSONL has
|
|
2465
|
+
// resumed growing (or name it lost if the resurrection chain is spent).
|
|
2466
|
+
checkResurrections()
|
|
2090
2467
|
}
|
|
2091
2468
|
|
|
2092
2469
|
// Initial boot scan: discover pre-existing files and mark them historical
|
|
@@ -2103,7 +2480,21 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
2103
2480
|
function runReaper(): void {
|
|
2104
2481
|
if (db == null) return
|
|
2105
2482
|
try {
|
|
2106
|
-
const result = reapStuckRunningRows(db, {
|
|
2483
|
+
const result = reapStuckRunningRows(db, {
|
|
2484
|
+
ttlMs: reaperTtlMs,
|
|
2485
|
+
now: nowFn(),
|
|
2486
|
+
// Liveness cross-check: never reap a row whose worker the watcher
|
|
2487
|
+
// is actively tailing. A live entry is `running`, non-historical,
|
|
2488
|
+
// and still in the in-memory registry (cleanupTerminalAgent drops
|
|
2489
|
+
// it on real termination). The DB `last_activity_at` freezes during
|
|
2490
|
+
// a long in-flight tool call and is NULL when linkage failed — both
|
|
2491
|
+
// would otherwise false-positive a live worker as terminal.
|
|
2492
|
+
isLive: (jsonlAgentId) => {
|
|
2493
|
+
if (jsonlAgentId == null) return false
|
|
2494
|
+
const entry = registry.get(jsonlAgentId)
|
|
2495
|
+
return entry != null && entry.state === 'running' && !entry.historical
|
|
2496
|
+
},
|
|
2497
|
+
})
|
|
2107
2498
|
if (result.reaped > 0) {
|
|
2108
2499
|
log?.(`subagent-watcher: reaper transitioned ${result.reaped} stuck-running row(s) to stalled (ttl=${Math.round(reaperTtlMs / 60_000)}min)`)
|
|
2109
2500
|
}
|