switchroom 0.18.10 → 0.18.12

package/telegram-plugin/shared/bot-runtime.ts

@@ -29,8 +29,9 @@ import { createHash } from 'crypto'
 import { AsyncLocalStorage } from 'async_hooks'
 import { clearStaleTelegramPollingState } from '../startup-reset.js'
 import { createRetryApiCall } from '../retry-api-call.js'
-import { makeFloodWaitRecorder } from '../flood-circuit-breaker.js'
+import { makeFloodWaitRecorder, makeFloodWaitProbe } from '../flood-circuit-breaker.js'
 import { RICH_MESSAGE_MAX_CHARS } from '../format.js'
+import { shouldEmitTgPost } from './gw-trace-gate.js'
 
 // ─── tg-post tag plumbing ─────────────────────────────────────────────────
 
@@ -115,9 +116,14 @@ export function installTgPostLogger(bot: Bot): void {
     const tagSuffix = formatTgPostTags(_getTgPostTags())
     try {
       const res = await prev(method, payload, signal)
-      process.stderr.write(
-        `tg-post method=${method} chat=${chat} thread=${thread} parse_mode=${parseMode} bytes=${bytes} hash=${hash} status=ok err=- code=- desc=-${tagSuffix}\n`,
-      )
+      // #3025: suppress zero-signal per-poll heartbeats (getUpdates/getMe
+      // status=ok, one line per ~30s long-poll tick) unless the operator
+      // set SWITCHROOM_GW_TRACE. Errors and all other methods still log.
+      if (shouldEmitTgPost(method, 'ok')) {
+        process.stderr.write(
+          `tg-post method=${method} chat=${chat} thread=${thread} parse_mode=${parseMode} bytes=${bytes} hash=${hash} status=ok err=- code=- desc=-${tagSuffix}\n`,
+        )
+      }
       return res
     } catch (err) {
       const errClass = err instanceof GrammyError
@@ -156,8 +162,14 @@ export function createRobustApiCall(opts: { floodStatePath?: string } = {}) {
     // #2923: persist every observed 429 flood-wait window so a restart during
     // the ban can suppress non-essential sends (boot cards) instead of feeding
     // the per-bot flood counter and prolonging the ban.
+    // #3084: and refuse to issue a call while that window is still open —
+    // otherwise the card heartbeats re-drive a request into the ban every
+    // few seconds once the policy stops sleeping long waits.
     ...(opts.floodStatePath
-      ? { onFloodWait: makeFloodWaitRecorder(opts.floodStatePath) }
+      ? {
+          onFloodWait: makeFloodWaitRecorder(opts.floodStatePath),
+          floodWaitRemainingMs: makeFloodWaitProbe(opts.floodStatePath),
+        }
       : {}),
   })
 }

package/telegram-plugin/shared/gw-trace-gate.ts

@@ -0,0 +1,105 @@
+/**
+ * Gateway trace verbosity gate (#3025).
+ *
+ * The gateway emits two families of high-frequency stderr trace lines that
+ * are pure per-poll noise on a healthy, idle agent:
+ *
+ *   - `tg-post method=getUpdates ... status=ok` / `method=getMe ... status=ok`
+ *     — one line per long-poll tick (~every 30s) from the tg-post
+ *     observability transformer in `bot-runtime.ts`.
+ *   - `gw-trace shadow event=tick effects=[] global=bridge_alive_idle ...`
+ *     — one line per delivery-machine TTL tick (~every 30s) from
+ *     `inbound-delivery-machine-shadow.ts`.
+ *
+ * With no log rotation these ungated lines grew gateway-supervisor.log to
+ * 555MB on clerk / 162MB on klanker and filled the host disk. They carry
+ * zero signal when nothing happened: a successful empty long-poll and an
+ * idle no-op tick.
+ *
+ * This module centralises the "should I emit a zero-signal trace line?"
+ * decision behind a single debug env flag so operators can turn the full
+ * firehose back on when actually debugging the delivery/poll path.
+ *
+ * Flag: set `SWITCHROOM_GW_TRACE=1` (or `true`/`yes`/`on`, any case) to
+ * restore every trace line unconditionally. Unset / any other value → zero-signal polling
+ * success + idle-tick lines are suppressed, while ALL error lines, all
+ * non-poll methods (sendMessage, editMessageText, ...), and every tick or
+ * event that produced a real effect or state change are still emitted.
+ *
+ * Read once at module init — the flag is a boot-time debug switch, not a
+ * hot-reloadable knob, so a cached read keeps the per-line cost at a
+ * single boolean check.
+ */
+
+/**
+ * Pure env-flag parse — exported so tests can exercise the parsing
+ * directly. Kept separate from the module-init read because the dual
+ * vitest/bun runner boundary means module-cache tricks
+ * (`vi.resetModules()` + re-import) don't work under bun's vitest shim;
+ * flag variants must be testable via explicit arguments.
+ */
+export function computeGwTraceVerbose(flag: string | undefined): boolean {
+  const v = (flag ?? '').trim().toLowerCase()
+  return v === '1' || v === 'true' || v === 'yes' || v === 'on'
+}
+
+/** True when the operator opted into the full gateway trace firehose. */
+export const gwTraceVerbose: boolean = computeGwTraceVerbose(
+  process.env.SWITCHROOM_GW_TRACE,
+)
+
+/**
+ * Bot API methods whose successful long-poll ticks are zero-signal:
+ * `getUpdates` is the long-poll itself and `getMe` is the periodic
+ * identity refresh. A `status=ok` on either says only "polling is alive",
+ * which the gateway already reports once at startup. Errors on these
+ * methods are NOT suppressed (see `shouldEmitTgPost`).
+ */
+const ZERO_SIGNAL_POLL_METHODS = new Set(['getUpdates', 'getMe'])
+
+/**
+ * Decide whether a `tg-post` line should be written.
+ *
+ * Suppressed (unless `SWITCHROOM_GW_TRACE` is set):
+ *   - `status=ok` for `getUpdates` / `getMe` — the per-poll heartbeat.
+ *
+ * Always emitted:
+ *   - any `status=err` (real failures — the whole point of the log),
+ *   - every other method's `ok` line (sendMessage/editMessageText/... are
+ *     genuine outbound observability, #656/#657).
+ */
+export function shouldEmitTgPost(
+  method: string,
+  status: 'ok' | 'err',
+  verbose: boolean = gwTraceVerbose,
+): boolean {
+  if (verbose) return true
+  if (status === 'err') return true
+  return !ZERO_SIGNAL_POLL_METHODS.has(method)
+}
+
+/**
+ * Decide whether a `gw-trace shadow` line should be written.
+ *
+ * Suppressed (unless `SWITCHROOM_GW_TRACE` is set):
+ *   - `event=tick` that produced NO effects AND left the machine in an
+ *     idle global state (`bridge_alive_idle` / `bridge_dead`) — the
+ *     no-op TTL heartbeat.
+ *
+ * Always emitted:
+ *   - any non-tick event (real inbound/turn/bridge activity),
+ *   - any tick that produced effects (e.g. a TTL turn expiry) or that
+ *     landed the machine in `bridge_alive_in_turn`.
+ */
+export function shouldEmitShadowTrace(
+  eventKind: string,
+  effectCount: number,
+  globalKind: string,
+  verbose: boolean = gwTraceVerbose,
+): boolean {
+  if (verbose) return true
+  if (eventKind !== 'tick') return true
+  if (effectCount > 0) return true
+  const idle = globalKind === 'bridge_alive_idle' || globalKind === 'bridge_dead'
+  return !idle
+}

package/telegram-plugin/status-pin-driver.ts

@@ -29,8 +29,8 @@
  * job spec.
  */
 
-import type { PinState, DesiredPin } from './status-pin.js'
-import { decidePinAction } from './status-pin.js'
+import type { PinState, DesiredPin, PinRightsCache } from './status-pin.js'
+import { decidePinAction, isPinRightsError } from './status-pin.js'
 
 /** Minimal subset of grammy's `bot.api` the pin driver depends on.
  *  Lets tests swap in a fake without dragging in the full Bot type. */
@@ -55,6 +55,16 @@ export interface ReconcilePinArgs {
   desired: DesiredPin
   /** Optional API-failure observer. Default: silent. */
   onError?: (phase: 'pin' | 'unpin', err: unknown) => void
+  /** Optional per-process rights-aware negative cache (issue #3024). When a
+   *  pin attempt fails with the permanent "not enough rights" 400, the chat is
+   *  recorded and all subsequent pin attempts in it are skipped (no API call,
+   *  no log). Omit to disable the cache entirely (the pre-#3024 behaviour). */
+  rightsCache?: PinRightsCache
+  /** Called EXACTLY ONCE per chat, the first time that chat is recorded as
+   *  pin-incapable. Lets the caller emit a single warn line instead of the
+   *  per-attempt `status-pin pin failed` spam. Only fires when `rightsCache`
+   *  is supplied. */
+  onPinRightsDisabled?: (chatId: string) => void
 }
 
 /**
@@ -77,10 +87,26 @@ export async function reconcilePin(
   if (action.kind === 'noop') return args.prevState
 
   if (action.kind === 'unpin') {
-    try {
-      await args.api.unpinChatMessage(args.chatId, action.messageId)
-    } catch (err) {
-      args.onError?.('unpin', err)
+    // Skip the unpin API call in a chat the bot can't manage pins in — the
+    // call would fail with the same rights 400 and spam the log. The claim is
+    // dropped either way (below), so skipping is safe.
+    if (!args.rightsCache?.isBlocked(args.chatId)) {
+      try {
+        await args.api.unpinChatMessage(args.chatId, action.messageId)
+      } catch (err) {
+        // Symmetric with the pin path below: a permanent rights 400 on UNPIN
+        // (rights revoked mid-session after we pinned) also enters the
+        // negative cache and logs once via onPinRightsDisabled — otherwise
+        // every later unpin attempt would burn an API call and spam
+        // `status-pin unpin failed` per attempt, the exact class this cache
+        // exists to kill (#3073 review finding). Claim is dropped regardless.
+        if (args.rightsCache && isPinRightsError(err)) {
+          const firstTime = args.rightsCache.block(args.chatId)
+          if (firstTime) args.onPinRightsDisabled?.(args.chatId)
+        } else {
+          args.onError?.('unpin', err)
+        }
+      }
     }
     // Drop the claim regardless of the unpin outcome. A stuck claim would
     // leave a permanent pin on a crash / out-of-band unpin — the exact
@@ -89,14 +115,33 @@ export async function reconcilePin(
   }
 
   // action.kind === 'pin' — pin an EXISTING message, silently.
+  // Rights-aware negative cache (issue #3024): if a prior attempt in this chat
+  // already failed with the permanent "not enough rights" 400, skip silently —
+  // no API call, no log. Don't claim the message (the pin never happened), so a
+  // later reconcile after a restart (cache cleared) can retry.
+  if (args.rightsCache?.isBlocked(args.chatId)) {
+    return args.prevState
+  }
   try {
     await args.api.pinChatMessage(args.chatId, action.messageId, {
       disable_notification: true,
     })
   } catch (err) {
-    args.onError?.('pin', err)
+    // A permanent pin-rights failure enters the negative cache and logs ONCE
+    // (via onPinRightsDisabled) — every subsequent attempt in this chat is then
+    // skipped above. Transient failures (429 / 5xx / network) are NOT cached
+    // and route through onError as before, preserving retry behaviour.
+    if (args.rightsCache && isPinRightsError(err)) {
+      const firstTime = args.rightsCache.block(args.chatId)
+      if (firstTime) args.onPinRightsDisabled?.(args.chatId)
+    } else {
+      args.onError?.('pin', err)
+    }
     // Don't claim a message we failed to pin — the next reconcile retries.
     return args.prevState
   }
+  // Pin succeeded — if this chat was previously cached as pin-incapable, rights
+  // were granted since; forget it so we resume normal behaviour immediately.
+  args.rightsCache?.clear(args.chatId)
   return { messageId: action.messageId }
 }

package/telegram-plugin/status-pin.ts

@@ -74,3 +74,84 @@ export function decidePinAction(
   // re-pins the new one on the next reconcile.
   return { kind: 'unpin', messageId: prev.messageId }
 }
+
+/** Extract a lowercased human description from any thrown value, preferring
+ *  grammy's structured `.description` (the wire text Telegram returned) and
+ *  falling back to `.message` / String(). Kept dependency-free — matches on
+ *  the wire text, never on a grammy class import. */
+function errorDescription(err: unknown): string {
+  if (err != null && typeof err === 'object') {
+    const o = err as { description?: unknown; message?: unknown }
+    if (typeof o.description === 'string' && o.description.length > 0) {
+      return o.description.toLowerCase()
+    }
+    if (typeof o.message === 'string' && o.message.length > 0) {
+      return o.message.toLowerCase()
+    }
+  }
+  return String(err).toLowerCase()
+}
+
+/**
+ * True when a pin/unpin failure is the PERMANENT "the bot lacks pin rights in
+ * this chat" class — Telegram returns this as a 400 (not a 403):
+ * "not enough rights to manage pinned messages in the chat". This is the one
+ * error class that will NOT self-heal on retry: the bot is simply not an admin
+ * (or lacks the "Pin messages" right) in that supergroup, and every subsequent
+ * pin attempt in the same chat fails identically until an admin grants the
+ * right (which only takes effect after a gateway restart re-reads chat perms).
+ *
+ * Transient classes (429 flood-wait, 5xx, network HttpError) are deliberately
+ * EXCLUDED — they must keep the existing retry behaviour, never enter the
+ * negative cache. Only this permanent-rights class is cacheable.
+ *
+ * The single source of truth for the pin-rights concept: the gateway's
+ * `unhandled-rejection-policy.ts` and the various edit-error classifiers each
+ * fold `not enough rights` into a broader boolean for their own purpose; this
+ * is the one helper dedicated to the pin negative-cache decision.
+ */
+export function isPinRightsError(err: unknown): boolean {
+  return errorDescription(err).includes('not enough rights')
+}
+
+/**
+ * Per-process, per-chat negative cache for chats the bot cannot pin in.
+ *
+ * When an auto status-pin attempt fails with the permanent pin-rights 400
+ * (`isPinRightsError`), the chat is recorded here as pin-incapable so the
+ * driver skips every subsequent auto-pin attempt in that chat — no wasted API
+ * call, no repeated `status-pin pin failed` log line (issue #3024: marko logged
+ * 41 identical pin-rights rejections in 48h, one per attempt).
+ *
+ * Deliberately IN-MEMORY / per-boot only: pin rights may be granted later, and
+ * Telegram surfaces the new permission to the bot only on a fresh chat-member
+ * fetch — a gateway restart. Clearing the cache on restart is therefore the
+ * correct re-enable trigger. An explicit `pin_message` tool success in a chat
+ * also clears its entry (rights were granted mid-session).
+ *
+ * Scope: the AUTO status-pin path only. The explicit `pin_message` MCP tool
+ * never consults this cache — it always attempts and surfaces the error to the
+ * agent as a normal tool error.
+ */
+export class PinRightsCache {
+  private readonly blocked = new Set<string>()
+
+  /** True when auto-pin should be skipped for this chat (already known bad). */
+  isBlocked(chatId: string): boolean {
+    return this.blocked.has(chatId)
+  }
+
+  /** Record a chat as pin-incapable. Returns `true` only the FIRST time a chat
+   *  is added, so the caller can log the warn line exactly once per chat. */
+  block(chatId: string): boolean {
+    if (this.blocked.has(chatId)) return false
+    this.blocked.add(chatId)
+    return true
+  }
+
+  /** Forget a chat (rights re-granted, e.g. an explicit pin later succeeded).
+   *  Returns `true` if an entry was actually removed. */
+  clear(chatId: string): boolean {
+    return this.blocked.delete(chatId)
+  }
+}

package/telegram-plugin/subagent-watcher.ts

@@ -155,6 +155,25 @@ export interface WorkerEntry {
    * every real long-runner always carries a `toolu_…` id.
    */
   inflightToolUseIds: Set<string>
+  /**
+   * Wall-clock ms of the most recent EMITTED "terminal synthesis deferred"
+   * log line for this entry, or null if the entry is not currently in the
+   * deferred state (never entered it, or resumed out of it). Drives the
+   * per-worker log rate-limit (#3092): null ⇒ this is a fresh entry into the
+   * deferred state and MUST log; non-null ⇒ log only once
+   * `deferralLogIntervalMs` has elapsed. Reset to null on the un-stall path
+   * so a stall → resume → stall cycle logs its first deferral again.
+   */
+  deferralLoggedAt: number | null
+  /**
+   * Count of deferral ticks whose log line was SUPPRESSED by the rate-limit
+   * since this entry entered the deferred state (#3092). Carried into the
+   * next emitted line (and into the resume / cap-crossing transition lines)
+   * so the suppression is itself visible and the operator can see the true
+   * tick volume without reading 1,547 lines. Reset alongside
+   * `deferralLoggedAt`.
+   */
+  deferralSuppressedTicks: number
   /**
    * True if the underlying JSONL file existed before the watcher started.
    * Historical entries are tracked for late state transitions but are
@@ -330,6 +349,22 @@ export interface SubagentWatcherConfig {
    * `SWITCHROOM_SUBAGENT_INFLIGHT_TERMINAL_CAP_MS`.
    */
   inflightTerminalCapMs?: number
+  /**
+   * Rate-limit (ms) on the repeated "terminal synthesis deferred" log line
+   * for a single worker (#3092). The deferral decision is re-evaluated on
+   * every ~1s rescan tick; without this gate each re-evaluation logged,
+   * emitting ~1 line/sec for as long as the worker stayed blocked (1,547
+   * lines in 35 min on the live overlord gateway, drowning the concurrent
+   * 429 flood-ban lines of #3084).
+   *
+   * Semantics: the FIRST tick that enters the deferred state always logs;
+   * subsequent ticks log only once this many ms have passed since the last
+   * emitted line. State CHANGES are never suppressed — the cap-crossing
+   * ("proceeding with terminal synthesis") and the resume/un-stall lines
+   * fire regardless. Default 60_000 (1 min). Env override
+   * `SWITCHROOM_SUBAGENT_DEFERRAL_LOG_INTERVAL_MS`.
+   */
+  deferralLogIntervalMs?: number
   /**
    * Freshness window (ms) for promoting a running-at-boot worker file to
    * live. A file whose last write (mtime) is older than this is treated as
@@ -600,6 +635,16 @@ const DEFAULT_SILENT_STALL_TERMINAL_MS = 300_000
 // reaper TTL, so the watcher — not the reaper — still owns the terminal
 // transition for a worker that died mid-tool.
 const DEFAULT_INFLIGHT_TERMINAL_CAP_MS = 45 * 60_000
+// Minimum wall-clock gap between two "terminal synthesis deferred" log lines
+// for the SAME worker (#3092). `checkStalls` runs on the ~1s rescan tick, and
+// the deferral is re-evaluated (and, before this gate, re-logged) on every one
+// of them. A single worker legitimately blocked inside a long tool call
+// produced 1,547 near-identical lines in 35 minutes on the overlord gateway,
+// burying the concurrent Telegram 429 flood-ban lines (#3084). The DECISION is
+// re-made every tick — only the LOGGING is rate-limited: first entry into the
+// deferred state logs immediately, then at most once per this interval, and
+// every state CHANGE (resume, cap-crossing) logs unconditionally.
+const DEFAULT_DEFERRAL_LOG_INTERVAL_MS = 60_000
 
 /**
  * Tools that legitimately run for minutes with ZERO intervening JSONL
@@ -1183,6 +1228,18 @@ export function readSubTail(
               log?.(`subagent-watcher: onUnstall callback error ${entry.agentId}: ${(cbErr as Error).message}`)
             }
           }
+          // STATE CHANGE out of the deferred state (#3092) — never rate-
+          // limited. A worker that was deferring terminal synthesis behind an
+          // in-flight tool call has RESURRECTED; that transition is the
+          // payoff of the deferral and must be visible even though the
+          // intervening per-tick deferral lines were suppressed. Report the
+          // suppressed volume so the quiet window is accounted for, then
+          // re-arm so a subsequent re-stall logs its first deferral again.
+          if (entry.deferralLoggedAt != null) {
+            log?.(`subagent-watcher: in-flight deferral resolved for ${entry.agentId} (worker resumed after ${idleSecBeforeBump}s idle — deferral was correct, not a dead worker; ${entry.deferralSuppressedTicks} deferral tick(s) suppressed since the last deferral line)`)
+            entry.deferralLoggedAt = null
+            entry.deferralSuppressedTicks = 0
+          }
           log?.(`subagent-watcher: stall cleared for ${entry.agentId} (activity resumed after ${idleSecBeforeBump}s — re-arming detection)`)
         }
         if (ev.kind === 'sub_agent_model') {
@@ -1457,6 +1514,10 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
     config.inflightTerminalCapMs
     ?? parseEnvMs('SWITCHROOM_SUBAGENT_INFLIGHT_TERMINAL_CAP_MS')
     ?? DEFAULT_INFLIGHT_TERMINAL_CAP_MS
+  const deferralLogIntervalMs =
+    config.deferralLogIntervalMs
+    ?? parseEnvMs('SWITCHROOM_SUBAGENT_DEFERRAL_LOG_INTERVAL_MS')
+    ?? DEFAULT_DEFERRAL_LOG_INTERVAL_MS
   const inflightPromoteMaxAgeMs =
     config.inflightPromoteMaxAgeMs
     ?? parseEnvMs('SWITCHROOM_SUBAGENT_INFLIGHT_MAX_AGE_MS')
@@ -1603,6 +1664,8 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
       stalledAt: null,
       completionNotified: false,
       stallTerminalSynthesised: false,
+      deferralLoggedAt: null,
+      deferralSuppressedTicks: 0,
       lastSummaryLine: '',
       lastResultText: '',
       lastProgressBucketIdx: null,
@@ -1987,6 +2050,19 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
       existing.stallNotified = false
       existing.stalledAt = null
       existing.stallTerminalSynthesised = false
+      // Re-arm the deferral log gate alongside its stall siblings (#3092).
+      // Unreachable with stale state TODAY (both routes out of the deferred
+      // state to `done` — the cap-crossing and the un-stall drain — already
+      // null `deferralLoggedAt`), so this is belt-and-braces, not a live fix.
+      // Reset it here anyway: a resurrected entry must be indistinguishable
+      // from a fresh one, so a genuinely NEW stall always logs its FIRST
+      // deferral line immediately. Leaving these to a reachability argument
+      // makes the gate fragile by construction — a later reorder of the drain,
+      // or a third path to `done`, would silently suppress that first line for
+      // up to `deferralLogIntervalMs`, losing the exact signal this gate
+      // exists to preserve.
+      existing.deferralLoggedAt = null
+      existing.deferralSuppressedTicks = 0
       // Re-arm the completion notification INTENTIONALLY (issue #3023). The
       // false synthesized finish already fired `onFinish` once, delivering a
       // (possibly wrong / incomplete) synthesized handback to the parent.
@@ -2212,10 +2288,34 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
       if (entry.inflightToolUseIds.size > 0) {
         const totalIdleMs = n - entry.lastActivityAt
         if (totalIdleMs < inflightTerminalCapMs) {
-          log?.(`subagent-watcher: silent-stall terminal synthesis deferred for ${entry.agentId} — ${entry.inflightToolUseIds.size} tool call(s) still in flight, ${Math.floor(totalIdleMs / 1000)}s idle < ${Math.floor(inflightTerminalCapMs / 1000)}s cap (long-running tool, not a dead worker)`)
+          // Log-rate gate (#3092). The deferral DECISION above is re-made on
+          // every ~1s rescan tick and is correct; only its LOGGING is gated.
+          // Emit on first entry into the deferred state, then at most once per
+          // `deferralLogIntervalMs`. Everything else is counted, not printed —
+          // the suppressed count rides the next emitted line so no volume is
+          // lost, just the 1/sec repetition (1,547 lines / 35 min on overlord,
+          // burying the concurrent #3084 flood-ban lines).
+          const lastLoggedAt = entry.deferralLoggedAt
+          const firstEntry = lastLoggedAt == null
+          if (firstEntry || n - lastLoggedAt >= deferralLogIntervalMs) {
+            const suppressed = entry.deferralSuppressedTicks
+            const suffix = firstEntry
+              ? ` — further deferral ticks for this worker are rate-limited to 1 line / ${Math.floor(deferralLogIntervalMs / 1000)}s until it resumes or crosses the cap`
+              : ` — still deferred (${suppressed} tick(s) suppressed since the last line)`
+            log?.(`subagent-watcher: silent-stall terminal synthesis deferred for ${entry.agentId} — ${entry.inflightToolUseIds.size} tool call(s) still in flight, ${Math.floor(totalIdleMs / 1000)}s idle < ${Math.floor(inflightTerminalCapMs / 1000)}s cap (long-running tool, not a dead worker)${suffix}`)
+            entry.deferralLoggedAt = n
+            entry.deferralSuppressedTicks = 0
+          } else {
+            entry.deferralSuppressedTicks++
+          }
           continue
         }
-        log?.(`subagent-watcher: in-flight deferral cap reached for ${entry.agentId} (${Math.floor(totalIdleMs / 1000)}s idle >= ${Math.floor(inflightTerminalCapMs / 1000)}s cap with ${entry.inflightToolUseIds.size} tool call(s) still unresolved) — treating as died-mid-tool, proceeding with terminal synthesis`)
+        // STATE CHANGE — never rate-limited. The deferral ends here and a
+        // terminal synthesis fires; this is the transition an operator needs.
+        const suppressedTotal = entry.deferralSuppressedTicks
+        log?.(`subagent-watcher: in-flight deferral cap reached for ${entry.agentId} (${Math.floor(totalIdleMs / 1000)}s idle >= ${Math.floor(inflightTerminalCapMs / 1000)}s cap with ${entry.inflightToolUseIds.size} tool call(s) still unresolved) — treating as died-mid-tool, proceeding with terminal synthesis (${suppressedTotal} deferral tick(s) suppressed since the last deferral line)`)
+        entry.deferralLoggedAt = null
+        entry.deferralSuppressedTicks = 0
       }
       // TODO(#3023/PR #3029): the cap-reached path above is a SECOND source of
       // possibly-false terminal synthesis (a worker mid-very-long-tool that is

package/telegram-plugin/tests/activity-card-wiring.test.ts

@@ -66,14 +66,27 @@ describe('activity-card durability wiring', () => {
   })
 
   it('the boot reaper runs ONLY after the startup mutex is won (never at import time)', () => {
-    // Both invocation sites (mutex-won + mutex-fallback) sit AFTER
-    // acquireStartupLock, alongside statusPinBootCleanup.
+    // #3026: the reaper is now invoked via the mutex-gated orchestrator
+    // runBootPinCleanupAndDmSweep() (which awaits it alongside
+    // statusPinBootCleanup + queuedCardBootReaper, then runs the DM
+    // stale-pin sweep). Invariant unchanged: reachable only post-lock.
+    // (1) The orchestrator awaits the reaper.
+    const orchestrator = between(
+      gatewaySrc,
+      'async function runBootPinCleanupAndDmSweep()',
+      'dmPinSweepEligible = true',
+    )
+    expect(orchestrator).toMatch(/await activityCardBootReaper\(\)/)
+    // (2) Both orchestrator invocation sites (mutex-won + mutex-fallback)
+    // sit AFTER acquireStartupLock.
     const afterLock = between(gatewaySrc, 'await acquireStartupLock({', 'catch (err)')
-    expect(afterLock).toMatch(/void activityCardBootReaper\(\)/)
-    // And it must NOT be invoked at module import time — the same losing-double-
-    // boot hazard the NOTE at statusPinBootCleanup documents.
+    expect(afterLock).toMatch(/void runBootPinCleanupAndDmSweep\(\)/)
+    // (3) Neither the reaper nor the orchestrator is invoked at module import
+    // time — the same losing-double-boot hazard the NOTE at
+    // statusPinBootCleanup documents.
     const beforeMain = gatewaySrc.split('await acquireStartupLock({')[0] ?? ''
     expect(beforeMain).not.toMatch(/^\s*void activityCardBootReaper\(\)/m)
+    expect(beforeMain).not.toMatch(/^\s*void runBootPinCleanupAndDmSweep\(\)/m)
   })
 
   it('the reaper wrapper counts a benign-400 as vanished, not finalized (honest boot log)', () => {