switchroom 0.18.23 → 0.18.25

package/telegram-plugin/worker-activity-feed.ts

@@ -92,6 +92,14 @@ export interface WorkerActivityView {
   lastTool: { name: string; sanitisedArg: string } | null
   /** Number of tool calls observed so far. */
   toolCount: number
+  /**
+   * Running TOTAL tokens across the worker's assistant messages so far
+   * (input + output + cache_creation, deduped by message.id in the watcher;
+   * cache_read is excluded — replayed cached context, not new work). Rendered
+   * as `· {N} tok` on the card's metrics line. Omitted /
+   * 0 → no token segment (a worker that emitted no usage).
+   */
+  totalTokens?: number
   /** The worker's latest narrative line, if any (already capped upstream). */
   latestSummary: string
   /**
@@ -178,6 +186,7 @@ export function renderWorkerActivity(v: WorkerActivityView, liveSuffix = ''): st
     toolCount: v.toolCount,
     state: v.state,
     model: v.model,
+    totalTokens: v.totalTokens,
   }
 
   // Terminal: latestSummary carries the worker's final result text (gateway
@@ -298,6 +307,59 @@ export interface WorkerActivityFeedOpts {
    * Tests inject a small value.
    */
   staleWorkerTtlMs?: number
+  /**
+   * ABSOLUTE row-lifetime cap (ms), measured from a row's creation — NOT from
+   * its last `update()`. The heartbeat force-terminates (and, when it was the
+   * last live worker, unpins) ANY row this old, whether or not it is marked
+   * finished and no matter how recently it updated.
+   *
+   * Why this exists AND is distinct from `staleWorkerTtlMs` (Carrie 5h zombie
+   * pin, v0.18.23): the `staleWorkerTtlMs` backstop is keyed off `lastUpdateAt`,
+   * so it only bites a row that goes SILENT. A leaked row that never transitions
+   * to finished AND keeps receiving `update()` cues every heartbeat (~6s) resets
+   * `lastUpdateAt` on every tick, so the silence sweep can NEVER match — the row
+   * (and its group pin) lives forever. This cap is immune to that reset: it is
+   * anchored to `createdAtMs` (immutable), so an immortal-but-updating row is
+   * still reaped past an absolute age. The primary fix (the watcher's
+   * `onTerminalCleanup` → `terminate` wiring) drives the clean path; this is the
+   * deterministic last-resort guarantee that no card can outlive the cap even if
+   * that signal is missed or the row is spuriously re-driven.
+   *
+   * The gateway DERIVES this from the watcher's effective in-flight terminal cap
+   * (`resolveInflightTerminalCapMs()`) times a multiple, matching the
+   * `staleWorkerTtlMs` derivation pattern (never a bare magic number). It is set
+   * comfortably above any legitimate single worker's lifetime so it can only
+   * bite a genuine leak; if a real long worker ever hit it, its cosmetic feed
+   * card collapses to `incomplete` and unpins — its actual result still reaches
+   * the user via the separate handback reply.
+   *
+   * Fallback default (this module, for direct / non-gateway callers) 6 h.
+   * Tests inject a small value.
+   */
+  absoluteRowLifetimeCapMs?: number
+  /**
+   * ABSOLUTE reused-group-MESSAGE lifetime cap (ms), measured from the moment
+   * the shared message was posted (`messageCreatedAtMs`), NOT from a worker
+   * row's age. When workers overlap continuously the group is NEVER reset to a
+   * fresh message (the only messageId reset is the group-reuse-after-terminal
+   * branch, gated on `!hasLiveWorker`), so ONE message can live for hours. If
+   * that message loses its pin out-of-band (a raw unpin somewhere else, or
+   * Telegram pin-stacking burying it), the steady-state edit path re-pins via
+   * `syncPin`; but a STALE in-memory pin claim (the claim still names this id, so
+   * `decidePinAction` no-ops on equal id) can only be cleared by rotating to a
+   * NEW message id. This cap force-rotates the shared message past an absolute
+   * age WHILE live workers remain, so the pin surface is periodically re-
+   * established through the first-paint `syncPin` path — the deterministic
+   * defense-in-depth that bounds how long a buried card can stay invisible
+   * (invisible-worker-cards incident, 2026-07-15; mirrors the immutable-anchor
+   * absolute-cap pattern of `absoluteRowLifetimeCapMs` / #3239's row cap).
+   *
+   * Conservative by design: set well above a normal edit cadence so a healthy
+   * pinned card is not needlessly churned. Rotation costs ONE unpin (old claim)
+   * + ONE pin (fresh message) per cap interval — never a burst. Fallback default
+   * (this module) 60 min; tests inject a small value.
+   */
+  groupMessageLifetimeCapMs?: number
   /**
    * Group-level status-pin reconcile hook (#3207 review). Because workers now
    * COALESCE into one shared message, the pin MUST follow the GROUP lifecycle,
@@ -359,6 +421,19 @@ interface WorkerRow {
    * and on every `update()`.
    */
   lastUpdateAt: number
+  /**
+   * Wall-clock ms this row was first CREATED (its first `update()` cue).
+   * IMMUTABLE after creation — unlike `lastUpdateAt`, it is NEVER re-stamped by
+   * later updates. The heartbeat's ABSOLUTE row-lifetime cap force-terminates a
+   * row whose `createdAtMs` is older than `absoluteRowLifetimeCapMs` regardless
+   * of how recently it updated. This closes the immortal-card leak the
+   * `lastUpdateAt` backstop cannot: a row that keeps receiving `update()` cues
+   * (a zombie whose watcher entry re-registers, or any spurious re-drive) resets
+   * `lastUpdateAt` every tick, so the silence-TTL sweep NEVER fires — but the
+   * absolute cap is immune to that reset (Carrie 5h zombie pin, re-edited 3000+
+   * times, only cleared by the restart-time dm-pin-sweep).
+   */
+  createdAtMs: number
   /**
    * Wall-clock ms the CURRENT step started — stamped whenever a NEW narrative
    * line lands (the `→` line changes). The heartbeat's step suffix shows the
@@ -382,6 +457,15 @@ interface FeedGroup {
   chatId: string
   threadId?: number
   messageId: number | null
+  /**
+   * Wall-clock ms the CURRENT shared message was posted (first paint / re-paint).
+   * IMMUTABLE for the lifetime of a given `messageId`: stamped when a message id
+   * is assigned and NEVER re-stamped by later edits, so the group-message
+   * lifetime cap (`groupMessageLifetimeCapMs`) measures the reused message's true
+   * absolute age and cannot be reset by a continuous edit stream. Reset to 0
+   * whenever `messageId` goes null (rotation / stale-message drop).
+   */
+  messageCreatedAtMs: number
   lastBody: string | null
   lastEditAt: number
   cooldownUntil: number
@@ -417,6 +501,17 @@ interface FeedGroup {
 
 const COOLDOWN_JITTER_MS = 500
 
+/**
+ * Static body the retired shared message is finalized to when the group-message
+ * lifetime cap rotates to a fresh card (invisible-worker-cards review, FIX 1).
+ * Without this the abandoned message stays frozen showing live-styled worker
+ * rows and reads like a stuck worker. A single best-effort edit collapses it to
+ * an honest "moved" note — issued once per rotation (≥ cap interval), never per
+ * tick, so it adds no edit churn / pin storm. Plain voice, no em dash.
+ */
+const WORKER_CARD_SUPERSEDED_BODY =
+  '🛠 **Worker** · _continued_\n\n_Live progress moved to a fresh card to stay pinned._'
+
 function extractRetryAfterSecs(err: unknown): number | null {
   if (err == null || typeof err !== 'object') return null
   const e = err as { error_code?: unknown; parameters?: { retry_after?: unknown } }
@@ -523,6 +618,22 @@ export interface WorkerActivityFeed {
    * stays visible. Idempotent; a no-op if the worker was never finalized.
    */
   resurrect(agentId: string): void
+  /**
+   * Boot / reconnect purge — reconcile the ENTIRE feed to empty and release
+   * EVERY group's pin, unconditionally.
+   *
+   * Why this is a hard invariant, not a TTL: every tracked worker row is a live
+   * sub-agent that was a CHILD PROCESS of the gateway. It cannot outlive a
+   * gateway restart, and it cannot survive a bridge reconnect that reconstructs
+   * the feed either — so any row still present when the feed is (re)initialised
+   * is definitionally dead. Left alone, the OLD feed instance's `wk:group:` pins
+   * are orphaned: the replacement feed is empty and never knew those groups, so
+   * it never unpins them (the reconnect-orphaned-pin leak — no full-boot pin
+   * sweep runs on a bare reconnect). This finalizes+removes every row and drives
+   * `reconcilePin(messageId: null)` for every group so the coalesced card is
+   * unpinned at once. Idempotent; a no-op on an already-empty feed.
+   */
+  purgeAllOnBoot(): void
   /** Clear the heartbeat interval (gateway shutdown). Idempotent. */
   stop(): void
   /** Manually fire one heartbeat tick (test hook). */
@@ -540,6 +651,8 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
   const heartbeatTickMs = opts.heartbeatTickMs ?? 6000
   const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8))
   const staleWorkerTtlMs = Math.max(1, Math.floor(opts.staleWorkerTtlMs ?? 50 * 60_000))
+  const absoluteRowLifetimeCapMs = Math.max(1, Math.floor(opts.absoluteRowLifetimeCapMs ?? 6 * 60 * 60_000))
+  const groupMessageLifetimeCapMs = Math.max(1, Math.floor(opts.groupMessageLifetimeCapMs ?? 60 * 60_000))
   const reconcilePinFn = opts.reconcilePin ?? (() => {})
   const setIntervalFn =
     opts.setInterval ??
@@ -694,6 +807,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
         description: v.description,
         elapsedMs: elapsedFor(r),
         toolCount: v.toolCount,
+        totalTokens: v.totalTokens,
         currentStep,
         // Full per-worker rolling history (oldest→newest) so the combined feed
         // can paint an adaptive-depth ✓/→ trail, not just the latest line. The
@@ -843,6 +957,9 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
           return
         }
         g.messageId = sent.message_id
+        // Stamp the reused-message birth time (immutable for this id) so the
+        // group-message lifetime cap measures the message's true absolute age.
+        g.messageCreatedAtMs = now
         g.lastBody = body
         g.lastEditAt = now
         // A fresh message is a live running paint, never a terminal recap.
@@ -891,6 +1008,16 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
           `worker-feed: edit feed=${g.feedKey} chat=${g.chatId} ` +
             `thread=${g.threadId ?? '-'} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`,
         )
+        // Re-assert the group pin on the steady-state edit path (invisible-
+        // worker-cards fix). `syncPin` was previously called ONLY at lifecycle
+        // edges (first-paint / terminal / GC), so once a long-lived reused
+        // message lost its pin out-of-band it was edited live forever but never
+        // re-pinned → scroll-buried. This closes that gap deterministically. It
+        // is a NO-OP when the pin is already correct: `decidePinAction` returns
+        // `noop` on equal claim id (status-pin.ts), so this makes ZERO Telegram
+        // calls on the common path and cannot create a pin storm — when the
+        // claim was dropped (variant 1) it re-pins the current message.
+        syncPin(g)
       }
       if (isTerminal) clearStaged()
     } catch (err) {
@@ -911,6 +1038,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
         // stale message id; a fresh first-paint re-establishes one if workers
         // are still live. On a terminal finalize, clear the staged repaint.
         g.messageId = null
+        g.messageCreatedAtMs = 0
         g.lastBody = null
         // The pinned message no longer exists → release the group pin claim
         // (clearStaged already re-syncs on the terminal path).
@@ -994,6 +1122,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
       description: lv?.description ?? 'background task',
       lastTool: null,
       toolCount: lv?.toolCount ?? 0,
+      totalTokens: lv?.totalTokens,
       // No fabricated result paragraph — an authoritative sweep can't know what
       // the worker returned; the terminal card shows the header struck-through
       // as `incomplete`, and the handback (if it ran) carries the actual result.
@@ -1016,14 +1145,29 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
     // `onTerminalCleanup` sweep. Collect the stale agent ids first (terminate
     // mutates the group's worker map), then terminate each through its chain so
     // the render/unpin happens under the normal cooldown/flood guards.
-    const staleAgentIds: string[] = []
-    const staleFinished: Array<{ g: FeedGroup; agentId: string }> = []
+    //
+    // TWO independent triggers, OR'd per row:
+    //   1. SILENCE TTL (`staleWorkerTtlMs`, keyed off `lastUpdateAt`) — a row
+    //      that stopped receiving cues.
+    //   2. ABSOLUTE row-lifetime cap (`absoluteRowLifetimeCapMs`, keyed off the
+    //      IMMUTABLE `createdAtMs`) — a row past an absolute age NO MATTER how
+    //      recently it updated. This is the durable guard against the immortal-
+    //      but-updating card the silence sweep can never catch: a leaked row
+    //      that keeps getting `update()` cues every heartbeat resets
+    //      `lastUpdateAt` forever, so only an age anchor immune to that reset can
+    //      reap it (Carrie 5h zombie pin, re-edited 3000+ times).
+    const staleAgentIds: Array<{ agentId: string; reason: 'silence' | 'absolute' }> = []
+    const staleFinished: Array<{ g: FeedGroup; agentId: string; reason: 'silence' | 'absolute' }> = []
     for (const g of groups.values()) {
       for (const row of g.workers.values()) {
-        if (now - row.lastUpdateAt >= staleWorkerTtlMs) {
-          if (row.finished) staleFinished.push({ g, agentId: row.agentId })
-          else staleAgentIds.push(row.agentId)
-        }
+        const silent = now - row.lastUpdateAt >= staleWorkerTtlMs
+        const tooOld = now - row.createdAtMs >= absoluteRowLifetimeCapMs
+        if (!silent && !tooOld) continue
+        // Attribute the reap to the absolute cap only when silence alone would
+        // NOT have fired — so the log names the trigger that actually caught it.
+        const reason: 'silence' | 'absolute' = silent ? 'silence' : 'absolute'
+        if (row.finished) staleFinished.push({ g, agentId: row.agentId, reason })
+        else staleAgentIds.push({ agentId: row.agentId, reason })
       }
     }
     // GC any FINISHED row still lingering past the TTL (#3207: finished rows
@@ -1032,14 +1176,25 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
     // normal finalize path drops the row immediately now, but reap directly
     // here as a durable backstop: `terminate()` no-ops on a finished row, so
     // remove it and release the pin without routing through it.
-    for (const { g, agentId } of staleFinished) {
-      log(`worker-feed: TTL GC finished row agent=${agentId} feed=${g.feedKey} — reaping leaked finished row`)
+    for (const { g, agentId, reason } of staleFinished) {
+      if (reason === 'absolute') {
+        const age = Math.floor((now - (g.workers.get(agentId)?.createdAtMs ?? now)) / 1000)
+        log(`worker-feed: ABSOLUTE cap GC finished row agent=${agentId} feed=${g.feedKey} — age ${age}s (>= ${Math.floor(absoluteRowLifetimeCapMs / 1000)}s); reaping immortal finished row`)
+      } else {
+        log(`worker-feed: TTL GC finished row agent=${agentId} feed=${g.feedKey} — reaping leaked finished row`)
+      }
       g.pendingFinalize.delete(agentId)
       removeWorker(g, agentId)
       syncPin(g)
     }
-    for (const agentId of staleAgentIds) {
-      log(`worker-feed: TTL reap agent=${agentId} — no update in ${Math.floor((now - (groupOfAgent(agentId)?.workers.get(agentId)?.lastUpdateAt ?? now)) / 1000)}s (>= ${Math.floor(staleWorkerTtlMs / 1000)}s); force-terminating leaked row`)
+    for (const { agentId, reason } of staleAgentIds) {
+      const row = groupOfAgent(agentId)?.workers.get(agentId)
+      if (reason === 'absolute') {
+        const age = Math.floor((now - (row?.createdAtMs ?? now)) / 1000)
+        log(`worker-feed: ABSOLUTE cap reap agent=${agentId} — row age ${age}s (>= ${Math.floor(absoluteRowLifetimeCapMs / 1000)}s); force-terminating immortal row (survives lastUpdateAt reset)`)
+      } else {
+        log(`worker-feed: TTL reap agent=${agentId} — no update in ${Math.floor((now - (row?.lastUpdateAt ?? now)) / 1000)}s (>= ${Math.floor(staleWorkerTtlMs / 1000)}s); force-terminating leaked row`)
+      }
       void terminateWorker(agentId)
     }
     for (const g of [...groups.values()]) {
@@ -1066,6 +1221,39 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
       const running = runningRows(g)
       if (running.length === 0) continue
 
+      // ABSOLUTE group-message lifetime cap (invisible-worker-cards fix, defense
+      // in depth). While workers overlap continuously the shared message is
+      // never reset, so one message can live for hours; if its pin was lost
+      // out-of-band and the in-memory claim went STALE (claim still names this
+      // id → `decidePinAction` no-ops on equal id), `syncPin` alone can never
+      // re-pin it. Force-rotate to a fresh message past an absolute age WHILE
+      // live workers remain: drop the id (unpinning the stale claim via
+      // `syncPin`, which clears it to null), then fall through to the first-
+      // paint path, whose `syncPin` re-pins the NEW message from a null claim.
+      // Costs exactly one unpin + one pin per cap interval — never a burst.
+      if (g.messageId != null && now - g.messageCreatedAtMs >= groupMessageLifetimeCapMs) {
+        const age = Math.floor((now - g.messageCreatedAtMs) / 1000)
+        const retiredId = g.messageId
+        log(
+          `worker-feed: group-message lifetime cap rotate feed=${g.feedKey} ` +
+            `msgId=${retiredId} — age ${age}s (>= ${Math.floor(groupMessageLifetimeCapMs / 1000)}s); ` +
+            `rotating to a fresh message to re-establish the pin surface`,
+        )
+        g.messageId = null
+        g.messageCreatedAtMs = 0
+        g.lastBody = null
+        // Clear the (possibly stale) pin claim for the retired message; the
+        // fresh first-paint below re-pins the new one from a null claim.
+        syncPin(g)
+        // FIX 1: collapse the retired message to an honest "moved" note so it
+        // doesn't sit frozen showing live-styled rows (mistakable for a stuck
+        // worker). ONE best-effort edit per rotation (≥ cap interval), fired
+        // off-chain and swallowing errors — never per-tick, never a burst.
+        void opts.bot
+          .editMessageText(g.chatId, retiredId, WORKER_CARD_SUPERSEDED_BODY, sendOptsFor(g))
+          .catch(() => {})
+      }
+
       // First-paint path: no message yet and some worker has now crossed
       // firstPaintMin (a prose-silent worker's single early tick was held).
       if (g.messageId == null) {
@@ -1132,6 +1320,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
           chatId,
           threadId,
           messageId: null,
+          messageCreatedAtMs: 0,
           lastBody: null,
           lastEditAt: 0,
           cooldownUntil: 0,
@@ -1151,6 +1340,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
       // live worker remains AND the group last went terminal.
       if (!g.workers.has(agentId) && g.terminalPainted && !hasLiveWorker(g)) {
         g.messageId = null
+        g.messageCreatedAtMs = 0
         g.lastBody = null
         g.pendingFinalize.clear()
         g.terminalPainted = false
@@ -1166,6 +1356,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
           state: 'running',
           finished: false,
           lastUpdateAt: nowFn(),
+          createdAtMs: nowFn(),
           dispatchAtMs: null,
           stepStartedAtMs: null,
         }
@@ -1240,6 +1431,27 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
         log(`worker-feed: resurrect agent=${agentId} — cleared finalized gate; card will repaint on next running cue`)
       }
     },
+    purgeAllOnBoot(): void {
+      for (const g of [...groups.values()]) {
+        // Release the group pin unconditionally: the shared card is orphaned
+        // after a restart/reconnect (its workers are dead child processes), so
+        // it must be unpinned regardless of `hasLiveWorker` — NOT gated like
+        // `syncPin`, which would keep the pin for a row we are about to drop.
+        if (g.messageId != null) {
+          reconcilePinFn({ feedKey: g.feedKey, chatId: g.chatId, threadId: g.threadId, messageId: null })
+        }
+        for (const agentId of [...g.workers.keys()]) {
+          // Latch finalized so a late/inflight cue on the OLD chain can't
+          // resurrect a row on a feed we are tearing down.
+          markFinalized(agentId)
+          agentIndex.delete(agentId)
+        }
+        g.workers.clear()
+        g.pendingFinalize.clear()
+        groups.delete(g.feedKey)
+      }
+      log('worker-feed: purgeAllOnBoot — reconciled feed to empty and released all group pins')
+    },
     heartbeatTick,
     stop() {
       if (heartbeatTimer != null) {