switchroom 0.18.18 → 0.18.20

package/telegram-plugin/worker-activity-feed.ts

@@ -69,7 +69,20 @@ export function isWorkerActivityFeedEnabled(envVal: string | undefined): boolean
   return envVal !== '0'
 }
 
-export type WorkerActivityState = 'running' | 'done' | 'failed'
+/**
+ * Terminal states a worker card can render.
+ *   - 'running'    — live.
+ *   - 'done'       — clean finish WITH a result (✅).
+ *   - 'failed'     — clean finish reporting a failure / crash observed in the
+ *                    transcript (⚠️).
+ *   - 'incomplete' — force-reaped / vanished / crashed WITHOUT any clean finish
+ *                    (the TTL sweep or the watcher's authoritative
+ *                    `onTerminalCleanup` synthesised the terminal because
+ *                    neither `onFinish` nor a turn_end ever delivered a result).
+ *                    Renders `incomplete · …` so a reaped worker NEVER reads as
+ *                    "done" — the truthful "ended without result" terminal.
+ */
+export type WorkerActivityState = 'running' | 'done' | 'failed' | 'incomplete'
 
 /** The render-relevant snapshot of a worker at one instant. */
 export interface WorkerActivityView {
@@ -141,7 +154,7 @@ const DESC_MAX = 80
  */
 export function renderWorkerActivity(v: WorkerActivityView, liveSuffix = ''): string {
   const desc = truncate(stripMarkdown(v.description).trim() || 'background task', DESC_MAX)
-  const finished = v.state === 'done' || v.state === 'failed'
+  const finished = v.state === 'done' || v.state === 'failed' || v.state === 'incomplete'
 
   // Raw narrative steps (unstripped/unescaped) — the unified renderer runs the
   // full per-line pipeline (stripMarkdown → collapse ws → clip → escape).
@@ -169,8 +182,15 @@ export function renderWorkerActivity(v: WorkerActivityView, liveSuffix = ''): st
 
   // Terminal: latestSummary carries the worker's final result text (gateway
   // onFinish), distinct from the running narrative steps. Pass it as `result`.
+  //
+  // Truthful-no-result invariant (deterministic control, not caller-discipline):
+  // an `incomplete` worker produced NO result, so it must NEVER render a result
+  // block regardless of whatever `latestSummary` happens to carry. The current
+  // call site (terminateWorker) always sets latestSummary:'' for incomplete, but
+  // enforce the invariant HERE so any future/direct caller can't fabricate a
+  // `⚠️`-prefixed result paragraph out of stray summary text.
   let result: { emoji: string; text: string } | undefined
-  if (finished) {
+  if (finished && v.state !== 'incomplete') {
     const text = cleanWorkerResultParagraph(v.latestSummary)
     if (text.length > 0) result = { emoji: v.state === 'done' ? '✅' : '⚠️', text }
   }
@@ -257,6 +277,27 @@ export interface WorkerActivityFeedOpts {
    * `channels.telegram.worker_feed.max_rows` via the config cascade.
    */
   maxRows?: number
+  /**
+   * Backstop TTL (ms): a worker row that has received no `update()` cue for
+   * longer than this AND is not already finished is force-terminated by the
+   * heartbeat sweep — its row is removed, and when it was the last live worker
+   * the shared card collapses to its terminal summary and unpins. This is the
+   * durable guard against an immortal card if BOTH terminal signals are missed
+   * (the gateway's `onFinish` AND the watcher's `onTerminalCleanup` sweep).
+   *
+   * The gateway DERIVES this in code from the watcher's effective in-flight
+   * terminal cap (`resolveInflightTerminalCapMs()` — the same env/default the
+   * watcher resolves) plus a margin, so the invariant "never reap a row the
+   * watcher still considers live" holds even if an operator raises the cap via
+   * `SWITCHROOM_SUBAGENT_INFLIGHT_TERMINAL_CAP_MS`. A worker mid-very-long tool
+   * can go silent up to the cap before the watcher declares it terminal, so any
+   * row still present past cap + margin is a definitive leak — the watcher would
+   * already have swept a live worker.
+   *
+   * Fallback default (this module, for direct / non-gateway callers) 50 min.
+   * Tests inject a small value.
+   */
+  staleWorkerTtlMs?: number
   /**
    * Group-level status-pin reconcile hook (#3207 review). Because workers now
    * COALESCE into one shared message, the pin MUST follow the GROUP lifecycle,
@@ -309,6 +350,15 @@ interface WorkerRow {
    * stable sort order within the combined feed.
    */
   dispatchAtMs: number | null
+  /**
+   * Wall-clock ms of the most recent `update()` cue for this worker (its last
+   * observed liveness). The heartbeat's backstop TTL sweep force-terminates a
+   * row whose `lastUpdateAt` is older than `staleWorkerTtlMs` — the durable
+   * guard that no card can go immortal even if every terminal signal (onFinish
+   * AND the onTerminalCleanup sweep) is somehow missed. Stamped on row creation
+   * and on every `update()`.
+   */
+  lastUpdateAt: number
   /**
    * Wall-clock ms the CURRENT step started — stamped whenever a NEW narrative
    * line lands (the `→` line changes). The heartbeat's step suffix shows the
@@ -432,6 +482,17 @@ export interface WorkerActivityFeed {
    *  in the group — force the terminal recap edit. No-op if the worker was
    *  never tracked. */
   finish(agentId: string, view: WorkerActivityView): Promise<void>
+  /**
+   * Force a worker terminal from an AUTHORITATIVE watcher sweep
+   * (`onTerminalCleanup`) or the backstop TTL, WITHOUT an external result view:
+   * the terminal recap is synthesised from the worker's own last-known row
+   * state (state → `done`, no fabricated result text — the real result reaches
+   * the user via the separate handback, if one ran). When it was the last live
+   * worker the shared card collapses to its terminal summary and unpins; with
+   * siblings, its row is dropped and the combined body re-renders. Idempotent —
+   * a no-op if the worker was already removed (e.g. `onFinish` fired first).
+   */
+  terminate(agentId: string): Promise<void>
   /** Forget a worker's state without a recap edit (e.g. error path); re-renders
    *  the group so the dropped worker disappears from the combined body. */
   drop(agentId: string): void
@@ -461,6 +522,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
   const firstPaintMin = opts.firstPaintMinMs ?? 8000
   const heartbeatTickMs = opts.heartbeatTickMs ?? 6000
   const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8))
+  const staleWorkerTtlMs = Math.max(1, Math.floor(opts.staleWorkerTtlMs ?? 50 * 60_000))
   const reconcilePinFn = opts.reconcilePin ?? (() => {})
   const setIntervalFn =
     opts.setInterval ??
@@ -616,6 +678,11 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
         elapsedMs: elapsedFor(r),
         toolCount: v.toolCount,
         currentStep,
+        // Full per-worker rolling history (oldest→newest) so the combined feed
+        // can paint an adaptive-depth ✓/→ trail, not just the latest line. The
+        // renderer clamps depth to the shared body-line budget; when a worker
+        // has no narrative yet this is empty and it falls back to currentStep.
+        historyLines: r.narrative.length > 0 ? [...r.narrative] : undefined,
         model: v.model,
       }
     })
@@ -785,10 +852,110 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
     }
   }
 
+  /**
+   * Shared terminal-finalize path for a worker whose row is still tracked.
+   * Latches the row terminal synchronously (so a late `running` cue on the
+   * chain can't resurrect it), then on the chain either drops the row + re-
+   * renders the surviving siblings, or — when it was the last live worker —
+   * finalizes the shared message to its terminal recap. Used by `finish` (with
+   * the gateway's onFinish view) AND by `terminate` (authoritative watcher
+   * sweep / TTL backstop, view synthesised from the row's own last state).
+   */
+  function finalizeWorker(group: FeedGroup, agentId: string, row: WorkerRow, view: WorkerActivityView): Promise<void> {
+    row.finished = true
+    // Preserve the truthful terminal state through to the row (done / failed /
+    // incomplete) — never collapse a reaped 'incomplete' into 'done'. view.state
+    // is always terminal on this path (finalize is only reached for a finishing
+    // worker), so threading it verbatim is correct.
+    row.state = view.state
+    markFinalized(agentId)
+    group.chain = group.chain
+      .then(() => {
+        const others = runningRows(group).filter((w) => w.agentId !== agentId)
+        if (others.length > 0) {
+          // Siblings still live → drop this row from the combined body and re-
+          // render the running set. The group pin STAYS (siblings still need
+          // the shared message).
+          removeWorker(group, agentId)
+          syncPin(group)
+          return doRender(group, { force: true })
+        }
+        // Last live worker → finalize the shared message to its terminal recap.
+        const recap: WorkerActivityView = { ...view, narrativeLines: [...row.narrative] }
+        return doRender(group, { force: true, terminalRecap: recap, finishingAgentId: agentId })
+      })
+      .catch((err) => {
+        log(`worker-feed: finalize chain error ${agentId}: ${(err as Error).message}`)
+      })
+    return group.chain
+  }
+
+  /**
+   * Force a worker terminal without an external result view (authoritative
+   * `onTerminalCleanup` sweep or the TTL backstop). Synthesises the recap from
+   * the row's own last-known state — state `incomplete` (NOT `done`), NO
+   * fabricated result text (the real result, if any, reaches the user via the
+   * separate handback). Reaching here means NO clean finish arrived: a clean
+   * `onFinish` removes the row FIRST (this call is then a no-op), and an
+   * errored worker goes through `onFinish(outcome:'failed')` → `finish` — so a
+   * row still present at terminate time genuinely ended WITHOUT a result. It
+   * must therefore render `incomplete · …`, never "done" (a crash/vanish read
+   * as "done" was the residual this fixes). Idempotent: a no-op once the worker
+   * was already removed (onFinish first). Depth-generic: operates on the
+   * generic worker row by agentId, identical at every nesting level.
+   */
+  function terminateWorker(agentId: string): Promise<void> {
+    const g = groupOfAgent(agentId)
+    const row = g?.workers.get(agentId)
+    if (g == null || row == null) {
+      // Already gone (onFinish removed it) — latch finalized so a late cue can't
+      // resurrect, and no-op.
+      markFinalized(agentId)
+      return Promise.resolve()
+    }
+    if (row.finished) {
+      // Terminal already latched; the chain will settle removal. No-op.
+      return g.chain
+    }
+    const lv = row.lastView
+    const view: WorkerActivityView = {
+      description: lv?.description ?? 'background task',
+      lastTool: null,
+      toolCount: lv?.toolCount ?? 0,
+      // No fabricated result paragraph — an authoritative sweep can't know what
+      // the worker returned; the terminal card shows the header struck-through
+      // as `incomplete`, and the handback (if it ran) carries the actual result.
+      latestSummary: '',
+      elapsedMs: liveElapsed(row, nowFn()),
+      state: 'incomplete',
+      model: lv?.model,
+    }
+    return finalizeWorker(g, agentId, row, view)
+  }
+
   // Arm the heartbeat once at construction. The real timer is `.unref()`'d so
   // it never keeps the process alive; tests inject setInterval/clearInterval.
   function heartbeatTick(): void {
     const now = nowFn()
+    // Backstop TTL sweep: force-terminate any worker row that has gone silent
+    // past `staleWorkerTtlMs` (no `update()` cue AND not already finished). This
+    // is the durable guard against an immortal card if BOTH terminal signals
+    // are missed — the gateway's `onFinish` AND the watcher's authoritative
+    // `onTerminalCleanup` sweep. Collect the stale agent ids first (terminate
+    // mutates the group's worker map), then terminate each through its chain so
+    // the render/unpin happens under the normal cooldown/flood guards.
+    const staleAgentIds: string[] = []
+    for (const g of groups.values()) {
+      for (const row of g.workers.values()) {
+        if (!row.finished && now - row.lastUpdateAt >= staleWorkerTtlMs) {
+          staleAgentIds.push(row.agentId)
+        }
+      }
+    }
+    for (const agentId of staleAgentIds) {
+      log(`worker-feed: TTL reap agent=${agentId} — no update in ${Math.floor((now - (groupOfAgent(agentId)?.workers.get(agentId)?.lastUpdateAt ?? now)) / 1000)}s (>= ${Math.floor(staleWorkerTtlMs / 1000)}s); force-terminating leaked row`)
+      void terminateWorker(agentId)
+    }
     for (const g of [...groups.values()]) {
       // Deferred-finalize re-drive: a terminal edit that hit a cooldown/flood
       // window was staged on `pendingFinalize`. Re-drive it once the cooldown
@@ -888,12 +1055,16 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
           lastView: null,
           state: 'running',
           finished: false,
+          lastUpdateAt: nowFn(),
           dispatchAtMs: null,
           stepStartedAtMs: null,
         }
         g.workers.set(agentId, row)
         agentIndex.set(agentId, feedKey)
       }
+      // Stamp liveness for the backstop TTL sweep (this is the worker's most
+      // recent observed activity cue).
+      row.lastUpdateAt = nowFn()
       // Accumulate before the gate so a throttled tick still grows the
       // narrative — it surfaces on the next edit that does fire.
       accumulateNarrative(row, view)
@@ -916,33 +1087,13 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
         markFinalized(agentId)
         return Promise.resolve()
       }
-      // Latch synchronously so a late `running` cue on the chain can't resurrect.
-      row.finished = true
-      row.state = view.state === 'failed' ? 'failed' : 'done'
-      markFinalized(agentId)
-
-      const group = g
-      group.chain = group.chain
-        .then(() => {
-          const others = runningRows(group).filter((w) => w.agentId !== agentId)
-          if (others.length > 0) {
-            // Siblings still live → drop this row from the combined body and
-            // re-render the running set. The result reaches the user via the
-            // separate handback, never folded into this cosmetic edit. The
-            // group pin STAYS (siblings still need the shared message) — a
-            // per-worker unpin here was the #3207 review blocker.
-            removeWorker(group, agentId)
-            syncPin(group)
-            return doRender(group, { force: true })
-          }
-          // Last live worker → finalize the shared message to its terminal recap.
-          const recap: WorkerActivityView = { ...view, narrativeLines: [...row.narrative] }
-          return doRender(group, { force: true, terminalRecap: recap, finishingAgentId: agentId })
-        })
-        .catch((err) => {
-          log(`worker-feed: finish chain error ${agentId}: ${(err as Error).message}`)
-        })
-      return group.chain
+      // Latch + finalize via the shared path (drop-with-siblings / terminal
+      // recap on the last worker). Synchronous latch inside finalizeWorker
+      // stops a late `running` cue on the chain from resurrecting the row.
+      return finalizeWorker(g, agentId, row, view)
+    },
+    terminate(agentId) {
+      return terminateWorker(agentId)
     },
     drop(agentId) {
       // A dropped worker is also done — mark finalized so a late tick can't