switchroom 0.18.9 → 0.18.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/agent-scheduler/index.js +1 -0
  2. package/dist/auth-broker/index.js +198 -13
  3. package/dist/cli/notion-write-pretool.mjs +1 -0
  4. package/dist/cli/switchroom.js +28 -4
  5. package/dist/host-control/main.js +3 -2
  6. package/dist/vault/approvals/kernel-server.js +2 -1
  7. package/dist/vault/broker/server.js +2 -1
  8. package/package.json +1 -1
  9. package/profiles/_base/start.sh.hbs +119 -37
  10. package/profiles/_shared/dev-protocol.md.hbs +42 -0
  11. package/skills/dev-protocol/SKILL.md +131 -0
  12. package/telegram-plugin/README.md +2 -1
  13. package/telegram-plugin/admin-commands/dispatch.test.ts +40 -2
  14. package/telegram-plugin/admin-commands/index.ts +6 -1
  15. package/telegram-plugin/bridge/bridge.ts +23 -1
  16. package/telegram-plugin/bridge/crash-breadcrumb.ts +42 -0
  17. package/telegram-plugin/chat-lock.ts +13 -0
  18. package/telegram-plugin/dist/bridge/bridge.js +24 -1
  19. package/telegram-plugin/dist/gateway/gateway.js +1831 -263
  20. package/telegram-plugin/dist/server.js +29 -2
  21. package/telegram-plugin/fallback-card-collapse.ts +131 -0
  22. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +546 -0
  23. package/telegram-plugin/gateway/effort-command.ts +47 -3
  24. package/telegram-plugin/gateway/gateway.ts +1435 -211
  25. package/telegram-plugin/gateway/model-command.ts +94 -8
  26. package/telegram-plugin/gateway/pending-session-command.ts +365 -0
  27. package/telegram-plugin/gateway/permission-timeout.ts +25 -0
  28. package/telegram-plugin/gateway/resume-inbound-builder.ts +23 -3
  29. package/telegram-plugin/gateway/session-model-file.ts +166 -23
  30. package/telegram-plugin/gateway/stop-command.ts +56 -0
  31. package/telegram-plugin/photo-precheck.ts +201 -0
  32. package/telegram-plugin/quota-watch.ts +141 -2
  33. package/telegram-plugin/registry/subagents-schema.ts +26 -3
  34. package/telegram-plugin/registry/subagents.test.ts +67 -0
  35. package/telegram-plugin/retry-api-call.ts +31 -0
  36. package/telegram-plugin/subagent-watcher.ts +392 -1
  37. package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +576 -0
  38. package/telegram-plugin/tests/buffer-gate-broadened.test.ts +11 -5
  39. package/telegram-plugin/tests/chat-lock-unhandled-rejection.test.ts +101 -0
  40. package/telegram-plugin/tests/crash-breadcrumb.test.ts +57 -0
  41. package/telegram-plugin/tests/effort-command.test.ts +59 -2
  42. package/telegram-plugin/tests/fallback-card-collapse.test.ts +104 -0
  43. package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +124 -0
  44. package/telegram-plugin/tests/gateway-secret-detect.test.ts +7 -1
  45. package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +19 -11
  46. package/telegram-plugin/tests/model-command.test.ts +46 -3
  47. package/telegram-plugin/tests/pending-session-command.test.ts +322 -0
  48. package/telegram-plugin/tests/permission-timeout.test.ts +26 -0
  49. package/telegram-plugin/tests/permission-verdict-resume-guard.test.ts +16 -0
  50. package/telegram-plugin/tests/photo-dimension-fallback.test.ts +129 -0
  51. package/telegram-plugin/tests/photo-precheck.test.ts +240 -0
  52. package/telegram-plugin/tests/photo-reroute-wiring.test.ts +85 -0
  53. package/telegram-plugin/tests/quota-watch.test.ts +225 -0
  54. package/telegram-plugin/tests/session-model-file.test.ts +101 -2
  55. package/telegram-plugin/tests/stop-command.test.ts +234 -0
  56. package/telegram-plugin/tests/subagent-watcher-env-thresholds.test.ts +27 -9
  57. package/telegram-plugin/tests/subagent-watcher-resurrection.test.ts +398 -0
  58. package/telegram-plugin/tests/subagent-watcher-stall-terminal.test.ts +172 -0
  59. package/telegram-plugin/tests/worker-activity-feed.test.ts +37 -0
  60. package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +18 -4
  61. package/telegram-plugin/welcome-text.ts +4 -3
  62. package/telegram-plugin/worker-activity-feed.ts +27 -0
@@ -183,6 +183,20 @@ export interface ReapStuckRunningArgs {
183
183
  ttlMs: number
184
184
  /** Current time (DI for tests). */
185
185
  now: number
186
+ /**
187
+ * Optional liveness cross-check against the in-memory file-discovery
188
+ * registry. Called with a candidate row's `jsonl_agent_id` (null when
189
+ * linkage never happened); return `true` if the watcher is actively
190
+ * tailing a live worker for that id. When it returns true the row is
191
+ * NOT reaped — the DB's `last_activity_at` is a *stale* liveness signal
192
+ * (it's only bumped when the JSONL is linked, and it freezes during a
193
+ * long in-flight tool call), so a row the watcher knows is alive must
194
+ * not be independently classified as dead here. The watcher owns that
195
+ * worker's terminal transition. Incident 2026-07-10: a live, actively-
196
+ * card-editing worker was reaped as terminal because this cross-check
197
+ * didn't exist. Omit to preserve the pre-fix unconditional behaviour.
198
+ */
199
+ isLive?: (jsonlAgentId: string | null) => boolean
186
200
  }
187
201
 
188
202
  export interface ReapStuckRunningResult {
@@ -524,14 +538,23 @@ export function reapStuckRunningRows(
524
538
  args: ReapStuckRunningArgs,
525
539
  ): ReapStuckRunningResult {
526
540
  const cutoff = args.now - args.ttlMs
527
- const candidates = db
541
+ const rows = db
528
542
  .prepare(`
529
- SELECT id FROM subagents
543
+ SELECT id, jsonl_agent_id FROM subagents
530
544
  WHERE status = 'running'
531
545
  AND background = 1
532
546
  AND COALESCE(last_activity_at, started_at) < ?
533
547
  `)
534
- .all(cutoff) as Array<{ id: string }>
548
+ .all(cutoff) as Array<{ id: string; jsonl_agent_id: string | null }>
549
+
550
+ // Cross-check each stale-by-DB candidate against the live in-memory
551
+ // registry. A worker the watcher is actively tailing is NOT dead just
552
+ // because its DB `last_activity_at` is stale/NULL — the watcher owns
553
+ // its terminal transition. Skipping these prevents the incident where a
554
+ // live, actively-card-editing worker was reaped as terminal.
555
+ const candidates = args.isLive
556
+ ? rows.filter((r) => !args.isLive!(r.jsonl_agent_id))
557
+ : rows
535
558
 
536
559
  for (const row of candidates) {
537
560
  recordSubagentStall(db, {
@@ -665,6 +665,73 @@ describe('reapStuckRunningRows', () => {
665
665
  expect(result.reaped).toBe(5)
666
666
  expect(result.ids.sort()).toEqual(['sa-0', 'sa-1', 'sa-2', 'sa-3', 'sa-4'])
667
667
  })
668
+
669
+ // Incident 2026-07-10: a live, actively-card-editing worker was reaped as
670
+ // terminal because the DB `last_activity_at` was stale (linkage never
671
+ // bumped it) while the in-memory file-discovery registry knew it was
672
+ // alive. The reaper must cross-check the live registry before reaping.
673
+ it('does NOT reap a row the live registry reports as an active worker', () => {
674
+ const db = openFreshSubagentsDbInMemory()
675
+ recordSubagentStart(db, {
676
+ id: 'sa-live',
677
+ background: true,
678
+ startedAt: 1000,
679
+ jsonlAgentId: 'a3c5b2d5765810114',
680
+ })
681
+ // DB liveness is stale (past ttl) — but the watcher is actively tailing
682
+ // this worker, so isLive returns true and the row is spared.
683
+ const liveIds = new Set(['a3c5b2d5765810114'])
684
+ const result = reapStuckRunningRows(db, {
685
+ ttlMs: 500,
686
+ now: 5000,
687
+ isLive: (jid) => jid != null && liveIds.has(jid),
688
+ })
689
+ expect(result.reaped).toBe(0)
690
+ expect(getSubagent(db, 'sa-live')!.status).toBe('running')
691
+ })
692
+
693
+ it('still reaps a stale row that is NOT in the live registry (genuine orphan)', () => {
694
+ const db = openFreshSubagentsDbInMemory()
695
+ recordSubagentStart(db, {
696
+ id: 'sa-orphan',
697
+ background: true,
698
+ startedAt: 1000,
699
+ jsonlAgentId: 'dead-worker',
700
+ })
701
+ // Also a live one to prove the predicate discriminates per-row.
702
+ recordSubagentStart(db, {
703
+ id: 'sa-alive',
704
+ background: true,
705
+ startedAt: 1000,
706
+ jsonlAgentId: 'live-worker',
707
+ })
708
+ const liveIds = new Set(['live-worker'])
709
+ const result = reapStuckRunningRows(db, {
710
+ ttlMs: 500,
711
+ now: 5000,
712
+ isLive: (jid) => jid != null && liveIds.has(jid),
713
+ })
714
+ expect(result.reaped).toBe(1)
715
+ expect(result.ids).toEqual(['sa-orphan'])
716
+ expect(getSubagent(db, 'sa-orphan')!.status).toBe('stalled')
717
+ expect(getSubagent(db, 'sa-alive')!.status).toBe('running')
718
+ })
719
+
720
+ it('reaps a stale row with NULL jsonl_agent_id (the watcher predicate never matches an unlinked row)', () => {
721
+ const db = openFreshSubagentsDbInMemory()
722
+ recordSubagentStart(db, { id: 'sa-unlinked', background: true, startedAt: 1000 })
723
+ // Mirror the production predicate shape: a null jsonl_agent_id can't
724
+ // key into the registry, so it always reports not-live and the reaper
725
+ // (the safety net for exactly this unlinked-orphan case) still fires.
726
+ const liveIds = new Set<string>()
727
+ const result = reapStuckRunningRows(db, {
728
+ ttlMs: 500,
729
+ now: 5000,
730
+ isLive: (jid) => jid != null && liveIds.has(jid),
731
+ })
732
+ expect(result.reaped).toBe(1)
733
+ expect(getSubagent(db, 'sa-unlinked')!.status).toBe('stalled')
734
+ })
668
735
  })
669
736
 
670
737
  // ---------------------------------------------------------------------------
@@ -371,3 +371,34 @@ export function isMessageTooLongError(err: unknown): boolean {
371
371
  d.includes('text is too long')
372
372
  )
373
373
  }
374
+
375
+ /**
376
+ * True when Telegram rejected a `sendPhoto` / `sendMediaGroup` because the
377
+ * image is unusable AS A PHOTO — dimensions out of range (Telegram caps
378
+ * photos at width+height ≤ 10000 and aspect ratio ≤ 20), the file can't be
379
+ * saved as a photo, or it exceeds the photo-path size ceiling (~10MB;
380
+ * documents allow ~50MB). A tall phone screenshot is the canonical trigger
381
+ * (PHOTO_INVALID_DIMENSIONS in the #klanker 2026-07-10 incident).
382
+ *
383
+ * These 400s are deliberately NOT swallowed or retried by `retryApiCall`
384
+ * (only not-modified / not-found / thread-not-found are) — they surface to
385
+ * the caller, which recovers by re-sending the SAME file as a document
386
+ * (`sendDocument`) so the user still receives it. Same "caller-level
387
+ * fallback" shape as the THREAD_NOT_FOUND and isHtmlParseRejectError
388
+ * contracts above.
389
+ */
390
+ export function isPhotoDimensionRejectError(err: unknown): boolean {
391
+ if (!(err instanceof GrammyError) || err.error_code !== 400) return false
392
+ const d = (err.description || '').toLowerCase()
393
+ return (
394
+ d.includes('photo_invalid_dimensions') ||
395
+ d.includes('photo_save_file_invalid') ||
396
+ d.includes('photo dimensions') ||
397
+ d.includes('image_process_failed') ||
398
+ // size-driven rejections of the photo path
399
+ d.includes('photo is too big') ||
400
+ d.includes('too big for a photo') ||
401
+ d.includes('image is too big') ||
402
+ d.includes('file is too big')
403
+ )
404
+ }
@@ -140,6 +140,21 @@ export interface WorkerEntry {
140
140
  * the renderer only ever shows the latest.
141
141
  */
142
142
  lastTool: { name: string; sanitisedArg: string } | null
143
+ /**
144
+ * Tool-use ids for tool calls this worker has STARTED but not yet
145
+ * finished — a `sub_agent_tool_use` was observed with no matching
146
+ * `sub_agent_tool_result` yet. A non-empty set means the worker is
147
+ * currently *inside* a tool call (e.g. a long-running `Bash` frame-
148
+ * capture loop that can legally run 10+ minutes with zero JSONL
149
+ * growth). The silent-stall terminal synthesis (checkStalls Pass 2)
150
+ * MUST NOT fire while this is non-empty: a frozen `lastActivityAt`
151
+ * during an in-flight tool call is expected, not a dead worker.
152
+ * Cleared on the matching `sub_agent_tool_result` and on
153
+ * `sub_agent_turn_end` (belt-and-braces). Tool-use lines with a null
154
+ * `toolUseId` can't be paired, so they don't participate — Bash and
155
+ * every real long-runner always carries a `toolu_…` id.
156
+ */
157
+ inflightToolUseIds: Set<string>
143
158
  /**
144
159
  * True if the underlying JSONL file existed before the watcher started.
145
160
  * Historical entries are tracked for late state transitions but are
@@ -300,6 +315,21 @@ export interface SubagentWatcherConfig {
300
315
  * synthesis; tests use a tiny value to exercise the path.
301
316
  */
302
317
  silentStallTerminalMs?: number
318
+ /**
319
+ * Upper bound (ms of total JSONL idle) on the in-flight tool-call
320
+ * deferral of terminal synthesis. While a tool call is in flight
321
+ * (tool_use seen, matching tool_result not yet), synthesis is deferred —
322
+ * a long `Bash` legitimately freezes the JSONL for 10+ min (incident
323
+ * 2026-07-10). But a worker that DIES mid-tool (process killed; JSONL
324
+ * stops growing but is never deleted) would otherwise defer forever, and
325
+ * the reaper's isLive cross-check would shield it from the DB net too —
326
+ * wedged for the life of the gateway. Past this cap the deferral ends
327
+ * and synthesis proceeds. Default 45 min
328
+ * (DEFAULT_INFLIGHT_TERMINAL_CAP_MS) — above any legitimate single tool
329
+ * call, below the 1h reaper TTL. Env override
330
+ * `SWITCHROOM_SUBAGENT_INFLIGHT_TERMINAL_CAP_MS`.
331
+ */
332
+ inflightTerminalCapMs?: number
303
333
  /**
304
334
  * Freshness window (ms) for promoting a running-at-boot worker file to
305
335
  * live. A file whose last write (mtime) is older than this is treated as
@@ -409,6 +439,34 @@ export interface SubagentWatcherConfig {
409
439
  * (the handback) regardless. Retained as an unwired hook.
410
440
  */
411
441
  onStallTerminal?: (agentId: string, description: string) => void
442
+ /**
443
+ * Issue #3023 (card resurrection). Fires when a worker whose card was
444
+ * FALSELY finalised — its terminal state came from silent-stall synthesis
445
+ * (`onStallTerminal` above), NOT a real `sub_agent_turn_end` — resumes
446
+ * writing to its JSONL after the synthesis. The synthesis was wrong: the
447
+ * worker is still alive, so the operator invariant ("active work must
448
+ * always be visible") requires its progress surface to come back. The
449
+ * watcher re-registers the worker as a LIVE, non-historical entry (so
450
+ * `onProgress` / stall-detection / the real handback resume), and fires
451
+ * this so the gateway can revive the worker's activity card (clear the
452
+ * feed's finalized gate → repaint on the next progress tick).
453
+ *
454
+ * GUARD (at-most-once per false finish + bounded chain): a given false
455
+ * finish is resurrected at most once. A worker that is resurrected and
456
+ * then falsely finalised AGAIN is NOT resurrected a second time — it is
457
+ * named-as-lost via `onWorkerLost` instead, so a pathological worker
458
+ * can't loop finish→resurrect forever.
459
+ */
460
+ onResurrect?: (agentId: string, description: string) => void
461
+ /**
462
+ * Issue #3023 (bounded resurrection chain). Fires when a worker that was
463
+ * already resurrected once is falsely finalised a second time and its
464
+ * JSONL resumes yet again. Rather than resurrect it forever, the watcher
465
+ * declares it LOST (a single log line names it) and stops resurrecting.
466
+ * The gateway may surface this however it likes; the invariant is only
467
+ * that the chain is bounded, not that a lost worker gets a fresh card.
468
+ */
469
+ onWorkerLost?: (agentId: string, description: string) => void
412
470
  /**
413
471
  * Called exactly once per sub-agent when its watcher observes a terminal
414
472
  * transition (`done` or `failed`). Mirrors the existing `sub_agent_started`
@@ -536,6 +594,12 @@ const DEFAULT_SILENT_SYNTHESIS_STALL_THRESHOLD_MS = 300_000
536
594
  * ceiling that closed-out cards used to wait on.
537
595
  */
538
596
  const DEFAULT_SILENT_STALL_TERMINAL_MS = 300_000
597
+ // Upper bound on the in-flight tool-call deferral of terminal synthesis.
598
+ // 45 min: comfortably above any legitimate single tool call (Bash caps at
599
+ // 10 min per call; the incident loop ran ~10 min) but below the 1h DB
600
+ // reaper TTL, so the watcher — not the reaper — still owns the terminal
601
+ // transition for a worker that died mid-tool.
602
+ const DEFAULT_INFLIGHT_TERMINAL_CAP_MS = 45 * 60_000
539
603
 
540
604
  /**
541
605
  * Tools that legitimately run for minutes with ZERO intervening JSONL
@@ -553,6 +617,21 @@ const DEFAULT_SILENT_STALL_TERMINAL_MS = 300_000
553
617
  */
554
618
  const LONG_RUNNING_TOOLS: ReadonlySet<string> = new Set(['Bash'])
555
619
 
620
+ /**
621
+ * Issue #3023 (bounded resurrection chain). Maximum number of times a single
622
+ * worker may have its falsely-finalised card resurrected. Once a worker has
623
+ * been resurrected this many times and is falsely finalised AGAIN, the next
624
+ * post-terminal JSONL resumption names it LOST rather than resurrecting it —
625
+ * so a pathological finish→resurrect→finish loop is bounded, not infinite.
626
+ * One resurrection is enough to recover the real 2026-07-10 incident (a
627
+ * single false finish from one over-long tool call); a second false finish on
628
+ * the same worker is a signal the heuristics can't track it, so we stop.
629
+ */
630
+ const MAX_RESURRECTIONS = 1
631
+ /** Cap on the false-finish tracker so it can't grow unbounded over a
632
+ * long-lived gateway; oldest record is evicted FIFO past this. */
633
+ const FALSE_FINISH_TRACKER_CAP = 512
634
+
556
635
  /** True when the tool named legitimately runs quiet for minutes (see
557
636
  * LONG_RUNNING_TOOLS). Null/undefined tool name → false. */
558
637
  function isLongRunningTool(name: string | null | undefined): boolean {
@@ -1144,6 +1223,14 @@ export function readSubTail(
1144
1223
  entry.lastReplyText = ev.input.text as string
1145
1224
  }
1146
1225
  entry.toolCount++
1226
+ // Track this as an IN-FLIGHT tool call so the silent-stall
1227
+ // terminal synthesis (checkStalls Pass 2) doesn't misread the
1228
+ // frozen JSONL of a long-running tool (a 10-min `Bash` loop) as
1229
+ // a dead worker. Only pairable (non-null id) tool_uses count —
1230
+ // Bash + every real long-runner always carries a `toolu_…` id.
1231
+ if (ev.toolUseId != null && ev.toolUseId !== '') {
1232
+ entry.inflightToolUseIds.add(ev.toolUseId)
1233
+ }
1147
1234
  // P0 of #662: surface the most recent tool name + sanitised
1148
1235
  // arg so the driver's fleet-state shadow can render the
1149
1236
  // last-tool column on the v2 status card. Sanitiser lives in
@@ -1197,6 +1284,16 @@ export function readSubTail(
1197
1284
  }
1198
1285
  }
1199
1286
  } else if (ev.kind === 'sub_agent_nested_spawn') {
1287
+ // A nested Agent/Task dispatch is the same frozen-JSONL shape as
1288
+ // a long tool call: a FOREGROUND nested child blocks this worker
1289
+ // until it returns, with the matching tool_result only landing
1290
+ // then — so gate terminal synthesis on it too. The existing
1291
+ // `sub_agent_tool_result` handler clears the id (a background
1292
+ // nested dispatch's "launched" result lands almost immediately,
1293
+ // so it barely defers). Same cap applies.
1294
+ if (ev.toolUseId != null && ev.toolUseId.length > 0) {
1295
+ entry.inflightToolUseIds.add(ev.toolUseId)
1296
+ }
1200
1297
  // Nested (depth-2+) dispatch keying: this worker just dispatched a
1201
1298
  // sub-agent of its own. The PreToolUse hook can't attribute it (the
1202
1299
  // main turn's turn-active.json marker is long gone for a background
@@ -1254,7 +1351,17 @@ export function readSubTail(
1254
1351
  fireNarrativeProgress() // prior pending was pure narration → SHOW
1255
1352
  }
1256
1353
  entry.pendingNarrative = { text: ev.text }
1354
+ } else if (ev.kind === 'sub_agent_tool_result') {
1355
+ // The tool call completed — clear it from the in-flight set so
1356
+ // the terminal-synthesis gate re-opens. Idempotent: a result
1357
+ // whose tool_use we never tracked (null id, parallel spill) is
1358
+ // a harmless no-op delete.
1359
+ if (ev.toolUseId != null && ev.toolUseId !== '') {
1360
+ entry.inflightToolUseIds.delete(ev.toolUseId)
1361
+ }
1257
1362
  } else if (ev.kind === 'sub_agent_turn_end') {
1363
+ // Belt-and-braces: a turn boundary means nothing is in flight.
1364
+ entry.inflightToolUseIds.clear()
1258
1365
  // Narrative-dedup gate step 3: a trailing sub_agent_text block with
1259
1366
  // nothing after it. SUPPRESS only when it drafts the foreground
1260
1367
  // sub-agent's delivered reply (entry.lastReplyText, set above on a
@@ -1346,6 +1453,10 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
1346
1453
  config.silentStallTerminalMs
1347
1454
  ?? parseEnvMs('SWITCHROOM_SUBAGENT_STALL_TERMINAL_MS')
1348
1455
  ?? DEFAULT_SILENT_STALL_TERMINAL_MS
1456
+ const inflightTerminalCapMs =
1457
+ config.inflightTerminalCapMs
1458
+ ?? parseEnvMs('SWITCHROOM_SUBAGENT_INFLIGHT_TERMINAL_CAP_MS')
1459
+ ?? DEFAULT_INFLIGHT_TERMINAL_CAP_MS
1349
1460
  const inflightPromoteMaxAgeMs =
1350
1461
  config.inflightPromoteMaxAgeMs
1351
1462
  ?? parseEnvMs('SWITCHROOM_SUBAGENT_INFLIGHT_MAX_AGE_MS')
@@ -1428,6 +1539,39 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
1428
1539
  * terminal JSONLs as a no-op.
1429
1540
  */
1430
1541
  const terminatedAgentIds = new Set<string>()
1542
+ /**
1543
+ * Issue #3023 (card resurrection). Per-worker record of a FALSE terminal
1544
+ * finish — a terminal state produced by silent-stall synthesis (NOT a real
1545
+ * `sub_agent_turn_end`). Keyed by agentId; SURVIVES `cleanupTerminalAgent`
1546
+ * (which drops the registry entry) precisely so a post-terminal JSONL
1547
+ * resumption can be detected after the entry is gone.
1548
+ *
1549
+ * A genuine completion (`turn_end`) or a genuine boot-time historical
1550
+ * rediscovery NEVER writes here — only synthesis does — so those paths can
1551
+ * never be resurrected. This map IS the discriminator between "old
1552
+ * completed worker rediscovered at boot" (no record → stays suppressed)
1553
+ * and "worker I finalised moments ago whose JSONL just grew again"
1554
+ * (record present → resurrect).
1555
+ *
1556
+ * - `synthesisedAt` — wall-clock ms the false finish fired.
1557
+ * - `sizeAtSynthesis`— JSONL byte size at that moment; post-terminal
1558
+ * growth past this proves the worker resumed.
1559
+ * - `resurrectionCount` — times this worker has been resurrected. Bounds
1560
+ * the chain: once it reaches MAX, the next false
1561
+ * finish is named-as-lost, not resurrected.
1562
+ * - `resurrectedForThisFinish` — at-most-once guard for the CURRENT false
1563
+ * finish; reset when a new synthesis records over it.
1564
+ * - `lost` — the chain bound was hit; no further resurrection.
1565
+ */
1566
+ interface FalseFinishRecord {
1567
+ filePath: string
1568
+ synthesisedAt: number
1569
+ sizeAtSynthesis: number
1570
+ resurrectionCount: number
1571
+ resurrectedForThisFinish: boolean
1572
+ lost: boolean
1573
+ }
1574
+ const falseFinishTracker = new Map<string, FalseFinishRecord>()
1431
1575
  /**
1432
1576
  * True while the initial boot scan is running. During this window every
1433
1577
  * newly discovered file is added to historicalFiles.
@@ -1463,6 +1607,7 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
1463
1607
  lastResultText: '',
1464
1608
  lastProgressBucketIdx: null,
1465
1609
  lastTool: null,
1610
+ inflightToolUseIds: new Set<string>(),
1466
1611
  historical: isHistorical,
1467
1612
  }
1468
1613
  registry.set(agentId, entry)
@@ -1778,6 +1923,188 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
1778
1923
  log?.(`subagent-watcher: cleaned up terminal agent ${agentId}`)
1779
1924
  }
1780
1925
 
1926
+ // ─── Card resurrection (issue #3023) ─────────────────────────────────────
1927
+
1928
+ /**
1929
+ * Record a FALSE terminal finish (silent-stall synthesis). Preserves any
1930
+ * carried-over `resurrectionCount` from a prior false finish on the same
1931
+ * worker (bounded-chain guard) while re-arming the per-finish at-most-once
1932
+ * guard. Best-effort file-size snapshot: an unreadable stat records 0, so
1933
+ * ANY later readable growth still counts as a resumption.
1934
+ */
1935
+ function recordFalseFinish(agentId: string, filePath: string, n: number): void {
1936
+ let size = 0
1937
+ try {
1938
+ size = fs.statSync(filePath).size
1939
+ } catch { /* unreadable → 0, any later growth still trips resumption */ }
1940
+ const prior = falseFinishTracker.get(agentId)
1941
+ if (prior == null && falseFinishTracker.size >= FALSE_FINISH_TRACKER_CAP) {
1942
+ const oldest = falseFinishTracker.keys().next().value
1943
+ if (oldest != null) falseFinishTracker.delete(oldest)
1944
+ }
1945
+ falseFinishTracker.set(agentId, {
1946
+ filePath,
1947
+ synthesisedAt: n,
1948
+ sizeAtSynthesis: size,
1949
+ resurrectionCount: prior?.resurrectionCount ?? 0,
1950
+ resurrectedForThisFinish: false,
1951
+ // `lost` is sticky: once a worker is named-lost it stays lost.
1952
+ lost: prior?.lost ?? false,
1953
+ })
1954
+ }
1955
+
1956
+ /**
1957
+ * Bring a falsely-finalised worker back to LIVE. Cancels any pending
1958
+ * terminal cleanup and clears the terminated/historical suppression, then
1959
+ * revives via one of two branches so `onProgress`, stall-detection and a
1960
+ * genuine future handback all resume:
1961
+ *
1962
+ * 1. IN-PLACE REVIVE (registry entry still present — cleanup grace hadn't
1963
+ * elapsed): flip the existing entry back to `running` and reset its
1964
+ * stall/completion flags. The entry KEEPS its existing tail cursor and
1965
+ * FSWatcher, so the card repaints INCREMENTALLY from where it left off —
1966
+ * it does NOT re-read from cursor 0.
1967
+ * 2. SWEPT RE-REGISTER (entry already dropped by cleanupTerminalAgent):
1968
+ * `registerAgent` re-registers the JSONL as a fresh non-historical
1969
+ * entry, re-reading from cursor 0 to rebuild the worker's activity so
1970
+ * the revived card catches up instead of showing a frozen stub.
1971
+ */
1972
+ function resurrectAgent(agentId: string, filePath: string): void {
1973
+ const pc = pendingCloses.get(agentId)
1974
+ if (pc != null) {
1975
+ clearT(pc)
1976
+ pendingCloses.delete(agentId)
1977
+ }
1978
+ terminatedAgentIds.delete(agentId)
1979
+ historicalFiles.delete(filePath)
1980
+
1981
+ const existing = registry.get(agentId)
1982
+ if (existing != null) {
1983
+ // Cleanup grace hadn't elapsed — revive the live entry in place so the
1984
+ // existing tail/FSWatcher keeps feeding it.
1985
+ existing.state = 'running'
1986
+ existing.historical = false
1987
+ existing.stallNotified = false
1988
+ existing.stalledAt = null
1989
+ existing.stallTerminalSynthesised = false
1990
+ // Re-arm the completion notification INTENTIONALLY (issue #3023). The
1991
+ // false synthesized finish already fired `onFinish` once, delivering a
1992
+ // (possibly wrong / incomplete) synthesized handback to the parent.
1993
+ // Clearing this lets a genuine future `sub_agent_turn_end` fire `onFinish`
1994
+ // a SECOND time — and that is the desired behaviour: the corrected REAL
1995
+ // result must reach the parent, superseding the false one. This does NOT
1996
+ // double-deliver harmfully: the gateway's handback spool dedups only
1997
+ // CONCURRENTLY-LIVE envelopes for the same worker (inbound-spool.ts:
1998
+ // `s:handback:<agentId>` — `live.has(id)` guard, no permanent tombstone),
1999
+ // so the first (synthesized) handback drains + acks as a normal turn, is
2000
+ // removed from the live set, and the later real handback is then
2001
+ // delivered as a fresh turn rather than being suppressed. See the
2002
+ // "resurrected worker's real turn_end re-fires onFinish" test.
2003
+ existing.completionNotified = false
2004
+ existing.errored = false
2005
+ existing.errorDetail = undefined
2006
+ // The worker just proved it is alive (its JSONL grew), so restart the
2007
+ // stall clock from now — otherwise the pre-synthesis idle would carry
2008
+ // over and a fresh stall/synthesis could fire almost immediately.
2009
+ existing.lastActivityAt = nowFn()
2010
+ knownFiles.add(filePath)
2011
+ // KNOWN LIMITATION (issue #3023): we revive the IN-MEMORY registry entry
2012
+ // to `running`, but the subagents DB row stays `completed`/`failed` (the
2013
+ // stall-synthesis path wrote a terminal row via recordSubagentEnd). We do
2014
+ // NOT flip it back: `recordSubagentResume` only reverses `stalled→running`
2015
+ // by design ("terminal beats both stalled and running" — see its doc),
2016
+ // and there is no terminal→running edge because the stuck-row reaper and
2017
+ // audit consistency both lean on terminal being final. Consequence:
2018
+ // `countRunningBackgroundSubagents` (WHERE status='running') UNDERCOUNTS a
2019
+ // resurrected worker until its genuine `turn_end` re-terminates the entry.
2020
+ // Impact is bounded and benign — that count only gates the deferred-👍
2021
+ // reaction promotion (reaction-defer.ts), so at worst a 👍 promotes one
2022
+ // resurrection-window early; it is never a delivery/correctness path. Left
2023
+ // as a known limitation rather than punching a terminal→running hole in
2024
+ // the schema invariant.
2025
+ return
2026
+ }
2027
+ // Entry was already swept — re-register from scratch as a live worker.
2028
+ knownFiles.add(filePath)
2029
+ registerAgent(filePath, agentId)
2030
+ }
2031
+
2032
+ /**
2033
+ * Detect post-terminal JSONL resumption for falsely-finalised workers and
2034
+ * either resurrect the card (once per false finish) or name the worker lost
2035
+ * (bounded chain). Called on every poll tick.
2036
+ */
2037
+ function checkResurrections(): void {
2038
+ const n = nowFn()
2039
+ for (const [agentId, rec] of falseFinishTracker) {
2040
+ if (rec.lost) continue
2041
+ if (rec.resurrectedForThisFinish) continue
2042
+ let size: number
2043
+ try {
2044
+ size = fs.statSync(rec.filePath).size
2045
+ } catch {
2046
+ continue // file vanished / unreadable — nothing to resurrect
2047
+ }
2048
+ // Growth past the synthesis snapshot is the proof the worker resumed.
2049
+ // (The tracker itself already excludes genuine completions and boot-time
2050
+ // historical rediscoveries — they never record here — so size growth is
2051
+ // a sufficient signal; no mtime dependence needed.)
2052
+ //
2053
+ // BOUNDED FALSE-POSITIVE (issue #3023, accepted): ANY byte growth past
2054
+ // the snapshot trips resurrection, including a late buffered `turn_end`
2055
+ // flush from a genuinely-done worker. If the silent-stall synthesis fired
2056
+ // moments before Claude Code finally flushed the worker's real
2057
+ // `sub_agent_turn_end` line, that trailing write grows the JSONL and we
2058
+ // resurrect a worker that is actually finished — a benign
2059
+ // resurrect→immediate-refinish flicker that burns ONE unit of the
2060
+ // resurrection budget (MAX_RESURRECTIONS). We accept this because (a) the
2061
+ // flicker is self-healing: the very next poll sees the terminal line,
2062
+ // fires a real `turn_end`, and re-finalises the card; (b) the corrected
2063
+ // real handback still reaches the parent (see the completionNotified
2064
+ // reset in resurrectAgent); and (c) the alternative — parsing the tail to
2065
+ // distinguish a real turn_end flush from live resumption — is far more
2066
+ // fragile than a bounded, harmless re-finish. The chain bound guarantees
2067
+ // this can never loop. Covered by the "late turn_end after synthesis"
2068
+ // resurrection test.
2069
+ if (size <= rec.sizeAtSynthesis) continue
2070
+
2071
+ const entry = registry.get(agentId)
2072
+ const description = entry?.description ?? 'sub-agent'
2073
+
2074
+ if (rec.resurrectionCount >= MAX_RESURRECTIONS) {
2075
+ // Bounded-chain guard: this worker was already resurrected once and
2076
+ // has now falsely finished again. Do NOT resurrect forever — name it
2077
+ // lost and stop. One log line names the worker for the operator.
2078
+ rec.lost = true
2079
+ log?.(`subagent-watcher: worker ${agentId} FALSELY finalised again after a prior resurrection (resurrectionCount=${rec.resurrectionCount} >= ${MAX_RESURRECTIONS}) — NAMED AS LOST, not resurrecting again (bounded resurrection chain, issue #3023)`)
2080
+ if (config.onWorkerLost != null) {
2081
+ try {
2082
+ config.onWorkerLost(agentId, description)
2083
+ } catch (cbErr) {
2084
+ log?.(`subagent-watcher: onWorkerLost callback error ${agentId}: ${(cbErr as Error).message}`)
2085
+ }
2086
+ }
2087
+ continue
2088
+ }
2089
+
2090
+ // Resurrect: at-most-once for THIS false finish.
2091
+ rec.resurrectionCount++
2092
+ rec.resurrectedForThisFinish = true
2093
+ log?.(`subagent-watcher: RESURRECTING worker ${agentId} — JSONL resumed growing (${rec.sizeAtSynthesis} → ${size} bytes) after a false terminal synthesis; the worker is still alive, reviving its card (resurrection #${rec.resurrectionCount}, issue #3023)`)
2094
+ // Fire onResurrect FIRST so the gateway clears the feed's finalized
2095
+ // gate before the re-registration below replays onProgress ticks —
2096
+ // otherwise those first ticks would be swallowed by the finalized gate.
2097
+ if (config.onResurrect != null) {
2098
+ try {
2099
+ config.onResurrect(agentId, description)
2100
+ } catch (cbErr) {
2101
+ log?.(`subagent-watcher: onResurrect callback error ${agentId}: ${(cbErr as Error).message}`)
2102
+ }
2103
+ }
2104
+ resurrectAgent(agentId, rec.filePath)
2105
+ }
2106
+ }
2107
+
1781
2108
  // ─── Stall detection ────────────────────────────────────────────────────
1782
2109
 
1783
2110
  function checkStalls(): void {
@@ -1862,11 +2189,57 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
1862
2189
  if (entry.stallTerminalSynthesised) continue
1863
2190
  if (entry.stalledAt == null) continue
1864
2191
  if (n - entry.stalledAt < silentStallTerminalMs) continue
2192
+ // In-flight tool-call gate (incident 2026-07-10): a worker whose
2193
+ // transcript ends in a `tool_use` with no matching `tool_result`
2194
+ // yet is NOT silent — it's *inside* a long tool call (a `Bash`
2195
+ // frame-capture loop legally runs 10+ minutes with zero JSONL
2196
+ // growth). Synthesising `sub_agent_turn_end` here finalises a live
2197
+ // worker's card while it keeps running (real incident: card 16201
2198
+ // reaped at t+13min, worker resumed 92ms later with no card).
2199
+ //
2200
+ // The deferral is CAPPED, not unconditional (design reconciliation
2201
+ // with #2777/#2782, whose contract is "a bg worker JSONL that
2202
+ // legitimately lacks turn_end must still release the completion
2203
+ // gate"): a worker that DIES mid-tool (process killed; JSONL frozen
2204
+ // but never deleted) would otherwise defer forever, and the reaper's
2205
+ // isLive cross-check would shield it from the DB net too. So: while
2206
+ // a tool call is in flight, defer synthesis up to
2207
+ // `inflightTerminalCapMs` of total JSONL idle (default 45 min — far
2208
+ // above any legitimate single tool call, below the 1h reaper TTL);
2209
+ // past the cap, synthesis proceeds. A tool_result landing at any
2210
+ // point drains the set and re-arms normal detection via the
2211
+ // un-stall path.
2212
+ if (entry.inflightToolUseIds.size > 0) {
2213
+ const totalIdleMs = n - entry.lastActivityAt
2214
+ if (totalIdleMs < inflightTerminalCapMs) {
2215
+ log?.(`subagent-watcher: silent-stall terminal synthesis deferred for ${entry.agentId} — ${entry.inflightToolUseIds.size} tool call(s) still in flight, ${Math.floor(totalIdleMs / 1000)}s idle < ${Math.floor(inflightTerminalCapMs / 1000)}s cap (long-running tool, not a dead worker)`)
2216
+ continue
2217
+ }
2218
+ log?.(`subagent-watcher: in-flight deferral cap reached for ${entry.agentId} (${Math.floor(totalIdleMs / 1000)}s idle >= ${Math.floor(inflightTerminalCapMs / 1000)}s cap with ${entry.inflightToolUseIds.size} tool call(s) still unresolved) — treating as died-mid-tool, proceeding with terminal synthesis`)
2219
+ }
2220
+ // TODO(#3023/PR #3029): the cap-reached path above is a SECOND source of
2221
+ // possibly-false terminal synthesis (a worker mid-very-long-tool that is
2222
+ // NOT dead gets finalised here). PR #3029 adds `recordFalseFinish(...)`
2223
+ // to this synthesis block so a later JSONL resumption can resurrect the
2224
+ // card. When #3029 lands, make sure the merge resolution keeps
2225
+ // `recordFalseFinish(entry.agentId, entry.filePath, n)` covering BOTH
2226
+ // the plain silent-stall path and this cap-reached fall-through (they
2227
+ // share this block, so a clean merge does — verify at conflict time).
2228
+ // Intentionally NOT wired here to keep this PR standalone (no
2229
+ // cross-PR dependency; recordFalseFinish does not exist on this branch).
1865
2230
  entry.stallTerminalSynthesised = true
1866
2231
  entry.state = 'done'
1867
2232
  const postStallSec = Math.floor((n - entry.stalledAt) / 1000)
1868
2233
  const totalIdleSec = Math.floor((n - entry.lastActivityAt) / 1000)
1869
2234
  log?.(`subagent-watcher: silent-stall terminal synthesis for ${entry.agentId} (stalled ${postStallSec}s post-notify, ${totalIdleSec}s total idle) — bg worker JSONL lacks turn_end; synthesising sub_agent_turn_end so deferred-completion gate releases`)
2235
+ // Issue #3023: this terminal state is SYNTHESISED, not a real
2236
+ // `turn_end` — it may be wrong (the worker could still be alive). Record
2237
+ // a false-finish so a later JSONL resumption can resurrect the card. The
2238
+ // record survives cleanupTerminalAgent (keyed by agentId, in its own
2239
+ // map). A carried-over resurrectionCount from a PRIOR false finish is
2240
+ // preserved so the chain stays bounded; resurrectedForThisFinish resets
2241
+ // to arm the at-most-once guard for THIS finish.
2242
+ recordFalseFinish(entry.agentId, entry.filePath, n)
1870
2243
  // Persist completion to the registry DB so reaper / audit paths
1871
2244
  // see the same terminal state as the JSONL-driven path.
1872
2245
  if (db != null) {
@@ -2087,6 +2460,10 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
2087
2460
 
2088
2461
  // Stall detection
2089
2462
  checkStalls()
2463
+
2464
+ // Issue #3023: revive any worker whose falsely-finalised card's JSONL has
2465
+ // resumed growing (or name it lost if the resurrection chain is spent).
2466
+ checkResurrections()
2090
2467
  }
2091
2468
 
2092
2469
  // Initial boot scan: discover pre-existing files and mark them historical
@@ -2103,7 +2480,21 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
2103
2480
  function runReaper(): void {
2104
2481
  if (db == null) return
2105
2482
  try {
2106
- const result = reapStuckRunningRows(db, { ttlMs: reaperTtlMs, now: nowFn() })
2483
+ const result = reapStuckRunningRows(db, {
2484
+ ttlMs: reaperTtlMs,
2485
+ now: nowFn(),
2486
+ // Liveness cross-check: never reap a row whose worker the watcher
2487
+ // is actively tailing. A live entry is `running`, non-historical,
2488
+ // and still in the in-memory registry (cleanupTerminalAgent drops
2489
+ // it on real termination). The DB `last_activity_at` freezes during
2490
+ // a long in-flight tool call and is NULL when linkage failed — both
2491
+ // would otherwise false-positive a live worker as terminal.
2492
+ isLive: (jsonlAgentId) => {
2493
+ if (jsonlAgentId == null) return false
2494
+ const entry = registry.get(jsonlAgentId)
2495
+ return entry != null && entry.state === 'running' && !entry.historical
2496
+ },
2497
+ })
2107
2498
  if (result.reaped > 0) {
2108
2499
  log?.(`subagent-watcher: reaper transitioned ${result.reaped} stuck-running row(s) to stalled (ttl=${Math.round(reaperTtlMs / 60_000)}min)`)
2109
2500
  }