switchroom 0.7.15 → 0.10.0

package/telegram-plugin/subagent-watcher.ts

@@ -40,7 +40,7 @@ import {
 } from 'fs'
 import { basename, join } from 'path'
 import { homedir } from 'os'
-import { projectSubagentLine } from './session-tail.js'
+import { projectSubagentLine, sanitizeCwdToProjectName } from './session-tail.js'
 import { sanitiseToolArg } from './fleet-state.js'
 import { escapeHtml, truncate } from './card-format.js'
 import { bumpSubagentActivity, recordSubagentStall, recordSubagentResume, recordSubagentEnd, reapStuckRunningRows } from './registry/subagents-schema.js'
@@ -83,8 +83,26 @@ export interface WorkerEntry {
   toolCount: number
   /** True once a stall notification has been sent (suppresses repeat). */
   stallNotified: boolean
+  /**
+   * Wall-clock ms when `stallNotified` flipped true. Null until then.
+   * Used by the post-stall terminal-synthesis path (RFC §Bug 6) to
+   * measure the post-stall window: when `now - stalledAt >=
+   * silentStallTerminalMs` the watcher synthesises a terminal
+   * transition for the entry. Workers whose JSONL never writes an
+   * explicit `sub_agent_turn_end` (e.g. background `Agent` dispatches
+   * in some Claude Code versions) would otherwise sit forever in
+   * `running` despite their real worker process having exited.
+   */
+  stalledAt: number | null
   /** True once a completion notification has been sent. */
   completionNotified: boolean
+  /**
+   * True once the post-stall terminal synthesis has fired so we don't
+   * re-synthesise on every poll tick after the silentStallTerminalMs
+   * window elapses. Paired with `stalledAt` — when synthesis runs it
+   * sets both `state='done'` and this flag.
+   */
+  stallTerminalSynthesised: boolean
   /** Short summary from last completed tool / narrative, for completion message. */
   lastSummaryLine: string
   /**
@@ -109,6 +127,16 @@ export interface SubagentWatcherConfig {
    * Used to derive `.claude/projects/<cwd>/` dirs to watch.
    */
   agentDir: string
+  /**
+   * Agent's working directory — used to compute the project-dir slug the
+   * watcher should restrict its enumeration to (Claude Code keys project
+   * dirs off the cwd at first launch via `sanitizeCwdToProjectName`).
+   * When omitted, the watcher walks every subdir of
+   * `<agentDir>/.claude/projects/` (legacy behaviour; see issue #1116
+   * for why this is unsafe — a foreign agent's stale project dir under
+   * an agent's home pollutes the watcher with phantom registrations).
+   */
+  agentCwd?: string
   /**
    * Send a fresh (non-edit) Telegram message. For stall / completion
    * state-transition notifications.
@@ -137,6 +165,16 @@ export interface SubagentWatcherConfig {
    * Both can be overridden for tests.
    */
   silentSynthesisStallThresholdMs?: number
+  /**
+   * RFC §Bug 6: how long after `stallNotified` fires the watcher waits
+   * before synthesising a terminal `sub_agent_turn_end` for the entry
+   * (ms). Default 300_000 (5 min) — sympathetic to legitimately-paused
+   * workers but tight enough that the progress card releases its
+   * deferred-completion gate well before the 30-min `maxIdleMs`
+   * ceiling. Set to a very large number (e.g. `Infinity`) to disable
+   * synthesis; tests use a tiny value to exercise the path.
+   */
+  silentStallTerminalMs?: number
   /**
    * Reaper TTL (ms): background rows in `status='running'` whose
    * `last_activity_at` (or `started_at` if liveness never wrote) is older
@@ -197,6 +235,21 @@ export interface SubagentWatcherConfig {
    * later in the same lifetime is detected (and reported) again.
    */
   onUnstall?: (agentId: string, description: string) => void
+  /**
+   * RFC §Bug 6: fires when the watcher synthesises a terminal transition
+   * for a stalled sub-agent (no explicit `sub_agent_turn_end` line in
+   * the JSONL after `silentStallTerminalMs` past the stall notification).
+   * Wired in gateway.ts to push a synthetic
+   * `{kind:'sub_agent_turn_end', agentId}` event into the progress
+   * driver so the pinned card can release its deferred-completion gate
+   * for the background dispatch.
+   *
+   * Idempotent: each sub-agent triggers this at most once per lifetime
+   * (guarded by `entry.stallTerminalSynthesised`). Fires *before* the
+   * existing `onFinish` callback so the driver-side state mutation
+   * lands first; the audit-log surface then sees a consistent fleet.
+   */
+  onStallTerminal?: (agentId: string, description: string) => void
   /**
    * Called exactly once per sub-agent when its watcher observes a terminal
    * transition (`done` or `failed`). Mirrors the existing `sub_agent_started`
@@ -257,6 +310,33 @@ const DEFAULT_STALL_THRESHOLD_MS = 60_000
  *  before emitting their first event — the 60s active-loop threshold
  *  misfires on those and freezes the card at ⚠. */
 const DEFAULT_SILENT_SYNTHESIS_STALL_THRESHOLD_MS = 300_000
+/**
+ * RFC §Bug 6 — post-stall terminal-synthesis window. 5min past the
+ * stall notification before the watcher synthesises a
+ * `sub_agent_turn_end` for the entry. Generous enough that a worker
+ * paused on an external dependency (operator unblocking, slow API)
+ * isn't reported done prematurely; tight enough that the pinned card's
+ * deferred-completion gate releases well before the 30-min `maxIdleMs`
+ * ceiling that closed-out cards used to wait on.
+ */
+const DEFAULT_SILENT_STALL_TERMINAL_MS = 300_000
+
+/**
+ * Resolve a threshold-knob env var (e.g.
+ * `SWITCHROOM_SUBAGENT_STALL_TERMINAL_MS`) to a positive integer ms
+ * value. Returns null when unset, empty, or unparseable so the caller
+ * falls through to the compile-time default. Negative/zero/NaN values
+ * are treated as "invalid" rather than "disable" — a real "disable"
+ * needs an explicit config-arg, not an env override (don't let a
+ * stray `=0` silently kill the watcher's stall-detection in prod).
+ */
+function parseEnvMs(varName: string): number | null {
+  const raw = process.env[varName]
+  if (raw == null || raw === '') return null
+  const n = Number(raw)
+  if (!Number.isFinite(n) || n <= 0) return null
+  return n
+}
 const DEFAULT_REAPER_TTL_MS = 60 * 60_000          // 1 hour
 const DEFAULT_REAPER_INTERVAL_MS = 15 * 60_000     // 15 minutes
 /**
@@ -458,6 +538,11 @@ function readSubTail(
         // driver to clear its render-time badge.
         if (entry.stallNotified) {
           entry.stallNotified = false
+          // Clear the stall timestamp so a subsequent re-stall starts
+          // the post-stall terminal-synthesis clock from scratch
+          // (RFC §Bug 6). Without this, a stall→resume→stall sequence
+          // could prematurely synthesise terminal on the second stall.
+          entry.stalledAt = null
           if (db != null) {
             try {
               const rowRef = db
@@ -533,9 +618,36 @@ function readSubTail(
 
 export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWatcherHandle {
   const agentDir = config.agentDir
-  const stallThresholdMs = config.stallThresholdMs ?? DEFAULT_STALL_THRESHOLD_MS
+  // Issue #1116: when agentCwd is supplied, restrict project-dir
+  // enumeration to the slug Claude Code would mint for that cwd.
+  // Foreign-slug shadow dirs (a sibling agent's stale project tree
+  // left over from a wayward CLAUDE_PROJECT_DIR or a past boot) are
+  // skipped — pre-#1116 they caused ENOENT log spam and false stalls.
+  // When agentCwd is null/undefined, fall back to the legacy walk-
+  // every-subdir behaviour (preserves tests that don't care about
+  // multi-slug isolation).
+  const expectedProjectSlug = config.agentCwd != null
+    ? sanitizeCwdToProjectName(config.agentCwd)
+    : null
+  // One-shot logging: warn the first time a foreign slug is observed
+  // so silent regressions are visible without re-running with debug.
+  const warnedForeignSlugs = new Set<string>()
+  // Threshold knobs resolve in this order: explicit config arg →
+  // env-var override → compile-time default. Env-vars exist so the
+  // UAT scenario (which times out at 120s) can compress the watcher's
+  // 60s-stall + 300s-synth window down to a few seconds without
+  // having to plumb config through every spinUp() caller. Production
+  // gateways don't set these — the defaults are tuned for live use.
+  const stallThresholdMs =
+    config.stallThresholdMs ?? parseEnvMs('SWITCHROOM_SUBAGENT_STALL_MS') ?? DEFAULT_STALL_THRESHOLD_MS
   const silentSynthesisStallThresholdMs =
-    config.silentSynthesisStallThresholdMs ?? DEFAULT_SILENT_SYNTHESIS_STALL_THRESHOLD_MS
+    config.silentSynthesisStallThresholdMs
+    ?? parseEnvMs('SWITCHROOM_SUBAGENT_SILENT_SYNTH_STALL_MS')
+    ?? DEFAULT_SILENT_SYNTHESIS_STALL_THRESHOLD_MS
+  const silentStallTerminalMs =
+    config.silentStallTerminalMs
+    ?? parseEnvMs('SWITCHROOM_SUBAGENT_STALL_TERMINAL_MS')
+    ?? DEFAULT_SILENT_STALL_TERMINAL_MS
   const reaperTtlMs = config.reaperTtlMs ?? DEFAULT_REAPER_TTL_MS
   const reaperIntervalMs = config.reaperIntervalMs ?? DEFAULT_REAPER_INTERVAL_MS
   const rescanMs = config.rescanMs ?? DEFAULT_RESCAN_MS
@@ -595,6 +707,20 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
    * when they eventually report done — that transition is meaningful.
    */
   const historicalFiles = new Set<string>()
+  /**
+   * AgentIds that have transitioned to a terminal state and been swept
+   * out of `registry` by `cleanupTerminalAgent`. Issue #1116 (Bug B):
+   * the JSONL file outlives the registry entry — Claude Code leaves
+   * the file on disk after the sub-agent finishes. Without this guard,
+   * the next `rescanSubagentDirs` poll re-discovered the file, called
+   * `registerAgent`, the fresh entry read the terminal `turn_duration`
+   * line, and `maybySendStateTransition` fired a duplicate "Worker done"
+   * notification — looping forever every grace-window.
+   *
+   * `scanSubagentsDir` consults this set and treats re-discovered
+   * terminal JSONLs as a no-op.
+   */
+  const terminatedAgentIds = new Set<string>()
   /**
    * True while the initial boot scan is running. During this window every
    * newly discovered file is added to historicalFiles.
@@ -620,7 +746,9 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
       lastActivityAt: n,
       toolCount: 0,
       stallNotified: false,
+      stalledAt: null,
       completionNotified: false,
+      stallTerminalSynthesised: false,
       lastSummaryLine: '',
       lastTool: null,
       historical: isHistorical,
@@ -788,6 +916,10 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
       knownFiles.delete(entry.filePath)
     }
     registry.delete(agentId)
+    // Issue #1116 (Bug B): record that this agent has been fully
+    // processed so a rescan that rediscovers the still-present JSONL
+    // doesn't re-register and re-notify.
+    terminatedAgentIds.add(agentId)
     log?.(`subagent-watcher: cleaned up terminal agent ${agentId}`)
   }
 
@@ -795,6 +927,9 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
 
   function checkStalls(): void {
     const n = nowFn()
+    // Pass 1: stall detection (existing behaviour). A running sub-agent
+    // with no JSONL growth for `threshold` ms transitions to "stalled"
+    // and notifies subscribers (badge on card, DB row update).
     for (const entry of registry.values()) {
       if (entry.state !== 'running') continue
       if (entry.historical) continue
@@ -812,6 +947,7 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
         : stallThresholdMs
       if (idleMs >= threshold) {
         entry.stallNotified = true
+        entry.stalledAt = n
         const desc = escapeHtml(truncate(entry.description, 80))
         const idleSec = Math.floor(idleMs / 1000)
         log?.(`subagent-watcher: stall detected for ${entry.agentId} (idle ${idleSec}s): ${desc}`)
@@ -842,6 +978,63 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
         }
       }
     }
+
+    // Pass 2 (RFC §Bug 6): post-stall terminal synthesis. Background
+    // `Agent` dispatches in some Claude Code versions write a JSONL
+    // that ends with the worker's last `sub_agent_tool_result` and
+    // never emits an explicit `system + turn_duration` line — so the
+    // canonical `sub_agent_turn_end` event never fires. Without
+    // synthesis the entry stays `running` until the 30-min
+    // `maxIdleMs` ceiling, and the pinned card's deferred-completion
+    // gate never releases.
+    //
+    // Wait `silentStallTerminalMs` past the stall notification before
+    // synthesising: a genuinely-paused worker (e.g. waiting on an
+    // external API the operator has to unblock) shouldn't be reported
+    // done immediately at the stall threshold.
+    for (const entry of registry.values()) {
+      if (entry.state !== 'running') continue
+      if (!entry.stallNotified) continue
+      if (entry.stallTerminalSynthesised) continue
+      if (entry.stalledAt == null) continue
+      if (n - entry.stalledAt < silentStallTerminalMs) continue
+      entry.stallTerminalSynthesised = true
+      entry.state = 'done'
+      const postStallSec = Math.floor((n - entry.stalledAt) / 1000)
+      const totalIdleSec = Math.floor((n - entry.lastActivityAt) / 1000)
+      log?.(`subagent-watcher: silent-stall terminal synthesis for ${entry.agentId} (stalled ${postStallSec}s post-notify, ${totalIdleSec}s total idle) — bg worker JSONL lacks turn_end; synthesising sub_agent_turn_end so deferred-completion gate releases`)
+      // Persist completion to the registry DB so reaper / audit paths
+      // see the same terminal state as the JSONL-driven path.
+      if (db != null) {
+        try {
+          const rowRef = db
+            .prepare('SELECT id FROM subagents WHERE jsonl_agent_id = ?')
+            .get(entry.agentId) as { id: string } | null
+          if (rowRef != null) {
+            recordSubagentEnd(db, {
+              id: rowRef.id,
+              endedAt: n,
+              status: 'completed',
+            })
+          }
+        } catch (dbErr) {
+          log?.(`subagent-watcher: stall-synth DB write error ${entry.agentId}: ${(dbErr as Error).message}`)
+        }
+      }
+      // Push a synthetic sub_agent_turn_end into the progress driver
+      // BEFORE the audit-log surface so the card mutation lands first.
+      if (config.onStallTerminal != null) {
+        try {
+          config.onStallTerminal(entry.agentId, entry.description)
+        } catch (cbErr) {
+          log?.(`subagent-watcher: onStallTerminal callback error ${entry.agentId}: ${(cbErr as Error).message}`)
+        }
+      }
+      // Fire the existing terminal-transition path (onFinish +
+      // deferred cleanup). state==='done' was set above so
+      // maybySendStateTransition flows through its happy path.
+      maybySendStateTransition(entry.agentId)
+    }
   }
 
   // ─── Subagents dir scanner ───────────────────────────────────────────────
@@ -865,6 +1058,16 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
     } catch { return }
 
     for (const pDir of projectDirs) {
+      // Issue #1116: filter to the agent's own slug. Skip foreign
+      // project dirs so their stale subagent JSONLs (which Claude
+      // Code reaps mid-session) don't pollute the watcher's registry.
+      if (expectedProjectSlug != null && pDir !== expectedProjectSlug) {
+        if (!warnedForeignSlugs.has(pDir)) {
+          warnedForeignSlugs.add(pDir)
+          log?.(`subagent-watcher: skipping foreign project dir ${pDir} (expected ${expectedProjectSlug})`)
+        }
+        continue
+      }
       const projectPath = join(projectsRoot, pDir)
       let sessionDirs: string[]
       try {
@@ -910,6 +1113,12 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
       if (!e.startsWith('agent-') || !e.endsWith('.jsonl')) continue
       const filePath = join(subagentsPath, e)
       if (knownFiles.has(filePath)) continue
+      const agentId = e.slice('agent-'.length, -'.jsonl'.length)
+      // Issue #1116 (Bug B): skip JSONLs whose agent already completed
+      // and was swept by cleanupTerminalAgent. Re-adding to knownFiles
+      // here would let a subsequent rescan re-register, fire a duplicate
+      // "Worker done", and loop forever every grace-window.
+      if (terminatedAgentIds.has(agentId)) continue
       knownFiles.add(filePath)
       // During the initial boot scan, mark every discovered file as
       // historical so stall-detection and completion notifications are
@@ -918,7 +1127,6 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
       if (bootScanInProgress) {
         historicalFiles.add(filePath)
       }
-      const agentId = e.slice('agent-'.length, -'.jsonl'.length)
       registerAgent(filePath, agentId)
     }
   }
@@ -1003,6 +1211,7 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
       tails.clear()
       registry.clear()
       knownFiles.clear()
+      terminatedAgentIds.clear()
     },
 
     getRegistry(): ReadonlyMap<string, WorkerEntry> {