switchroom 0.21.3 → 0.21.5

package/telegram-plugin/gateway/gateway.ts

@@ -302,7 +302,7 @@ import {
 import { installEditFloodFuse, editFloodFuseConfigFromEnv } from '../edit-flood-fuse.js'
 import { createSendGate, sendGateConfigFromEnv, isSendGateShed } from '../send-gate.js'
 import { createStatsLogger, createFloodWindowObserver } from '../send-gate-observability.js'
-import { installTgPostLogger, installRichMarkdownGuard, withTgPostTags } from '../shared/bot-runtime.js'
+import { installTgPostLogger, installRichMarkdownGuard, withTgPostTags, withTgSendContext, installSystemMessageObserver } from '../shared/bot-runtime.js'
 import { installSentTextCapture } from '../shared/sent-text-capture.js'
 import {
   floodStatePath,
@@ -396,8 +396,9 @@ import {
   pruneMessagesOlderThanDays,
   hasOutboundDeliveredSince,
   hasOutboundWithText,
-  recordSystemOutbound, updateSystemOutboundText,
+  recordSystemOutbound, updateSystemOutboundText, reopenHistory, getHistoryReopenFailure,
 } from '../history.js'
+import { startOrphanedDbSweep } from './orphaned-db-sweep.js'
 import { makeSystemMessageObserver } from './system-message-observer.js'
 import {
   runRegistryReaper,
@@ -2162,6 +2163,7 @@ if (isGatewayMain) runHistoryReaperNow('boot')
 if (isGatewayMain && !STATIC) {
   setInterval(() => runHistoryReaperNow('periodic'), REGISTRY_REAPER_INTERVAL_MS).unref()
 }
+if (isGatewayMain && !STATIC) startOrphanedDbSweep({ stateDir: STATE_DIR, reopenHistory: HISTORY_ENABLED ? () => reopenHistory(STATE_DIR, HISTORY_ACCESS.historyRetentionDays ?? 30) : undefined, historyReopenFailure: HISTORY_ENABLED ? getHistoryReopenFailure : undefined, log: (l) => process.stderr.write(l) }) // own 5-min tick, NOT the 6h reaper: bounds silent data loss from a deleted-inode DB handle to one interval (gateway/orphaned-db-sweep.ts). Runs regardless of HISTORY_ENABLED — registry.db opens whenever isGatewayMain.
 
 // ─── Approval polling ─────────────────────────────────────────────────────
 function checkApprovals(): void {
@@ -5601,28 +5603,25 @@ const rawRobustApiCall = createRetryApiCall({
   floodWaitRemainingMs: probeFloodWaitRemainingMs,
 })
 
-// #4571 — card/system-surface history lane. Every card the gateway posts
-// (activity card, status pin, approval/boot/issues/worker-feed cards, progress
-// lines, notices) goes through `robustApiCall`, so observing its RESOLVED
-// result is the one chokepoint that makes every posted message id resolvable
-// when the operator quote-replies to it. Never throws; see
-// system-message-observer.ts for the send-vs-edit and throttling contract.
-// Gated on the SAME condition as `initHistory` above — a non-main gateway
-// process never opens the DB, so an observer there would be pure noise.
-// The empty-body alarm (#4576) is the observer's own default — see
-// `defaultEmptyCardTextWarning` in system-message-observer.ts.
+// #4571 — card/system-surface history lane: every card the gateway posts must leave
+// a row so its id resolves when the operator quote-replies to it. #4599 moved the
+// hook off `robustApiCall` (which the `ctx.replyWithRichMessage` slash-command card
+// path bypasses entirely) onto the grammy transformer layer — see
+// `installSystemMessageObserver`, wired at bot construction, and
+// system-message-observer.ts for the send-vs-edit / throttling / alarm contract.
+// Gated as `initHistory` is: a non-main gateway never opens the DB.
 const observeSentMessage = isGatewayMain && HISTORY_ENABLED
   ? makeSystemMessageObserver({ insert: recordSystemOutbound, updateText: updateSystemOutboundText })
   : undefined
 
+// `withTgSendContext` publishes {chat_id, threadId, verb} down to the transformer layer
+// where the observer runs — how a card keeps its `kind`. Sends outside this wrapper
+// still record, just without a verb.
 const robustApiCall = <T>(
   fn: () => Promise<T>,
   opts?: Parameters<typeof rawRobustApiCall<T>>[1],
-): Promise<T> => {
-  const p = sendGate.gate(() => rawRobustApiCall(fn, opts), opts)
-  if (observeSentMessage == null) return p
-  return p.then((res) => { observeSentMessage(res, opts); return res })
-}
+): Promise<T> =>
+  sendGate.gate(() => withTgSendContext(opts, () => rawRobustApiCall(fn, opts)), opts)
 
 // Fire-and-forget wrapper for outbound surfaces that previously had
 // `.catch(() => {})` directly on `bot.api.*` calls. Resolves to undefined
@@ -15059,11 +15058,11 @@ export async function handleInbound(
   } = replyForwardCtx
 
   // Reply-to buffer fallback (post-reset continuity, resolveReplyToFromBuffer).
-  // On a native reply to the BOT's OWN message, Telegram delivers
-  // reply_to_message.message_id but NOT its .text — so the live reply text is
-  // empty even though we authored (and, via recordOutbound, persisted) that
-  // message to history.db (role='assistant', 30-day retention). Recover it so
-  // the antecedent survives a session reset. Fills replyToText (raw, so the
+  // On a native reply to a PLAIN bot-sent message, Telegram delivers
+  // reply_to_message.message_id but NOT its .text (a RICH parent — every card —
+  // carries rich_message and is resolved LIVE upstream since #4598), so the live
+  // reply text is empty even though we persisted it (role='assistant', 30-day
+  // retention). Recover it so it survives a reset. Fills replyToText (raw, so the
   // recordInbound write below persists it — envelope-only would leave the row
   // NULL and starve future briefings), replyToTextEscaped (channel meta), and
   // replyToRole (the reply_to_role attribute). Only when the live text is
@@ -22885,6 +22884,9 @@ async function initGatewayBot(): Promise<void> {
   // on the resolved Message for the shapes a response can't supply. After the fmt
   // guard so it composes OUTSIDE it. See sent-text-capture.ts.
   installSentTextCapture(bot)
+  // #4599 card-history lane: AFTER sent-text-capture so it composes outside it and
+  // can read the request-side stamp. The seam `ctx.replyWithRichMessage` can't bypass.
+  if (observeSentMessage != null) installSystemMessageObserver(bot, observeSentMessage)
   // #3620 flood fuse — installed LAST so it composes OUTERMOST: the one seam no
   // outbound call can bypass (grammY has no route to the network that skips the
   // transformer stack). Kill-switch SWITCHROOM_EDIT_FUSE=0; see edit-flood-fuse.ts.

package/telegram-plugin/gateway/inbound-router.ts

@@ -35,6 +35,7 @@ import {
 } from './inbound-interceptors.js'
 import type { InboundMessage } from './ipc-protocol.js'
 import { deriveTurnId } from './derive-turn-id.js'
+import { extractRichMessageText } from './rich-message-handler.js'
 import { formatReplyToText } from '../steering.js'
 import { fmtLocalStamp, resolveEnvTimezone } from '../shared/local-time.js'
 import { safeResolvePersonName, type PersonDirectory } from './resolve-person.js'
@@ -160,10 +161,33 @@ export function buildReplyForwardContext(p: ReplyForwardContextParams): {
   // gateway runs when neither is present. Accessed defensively in case the
   // installed grammy/@grammyjs/types predate `TextQuote`.
   const quoteText = p.ctx.message?.quote?.text
+  // Rich-message parents (#4598). Every card the gateway posts goes out via
+  // Bot API 10.1 `sendRichMessage`, so a native reply to one delivers a
+  // `reply_to_message` whose body lives under `rich_message.blocks` with
+  // `text` / `caption` ABSENT. Reading only `.text ?? .caption` therefore
+  // yielded `undefined` for 100% of card parents and pushed every such reply
+  // onto the history-buffer fallback — which cannot help when no row was ever
+  // recorded (a send while the gateway was down, or one that bypassed the
+  // recording chokepoint).
+  //
+  // Measured on the wire, not inferred: a user reply to a `sendRichMessage`
+  // card yields `reply_to_message` keys
+  // `[message_id, from, chat, date, rich_message]`, `text`/`caption` absent,
+  // `rich_message.blocks` fully populated. The older "Telegram omits the
+  // parent body" comments on this path predate rich messages.
+  //
+  // Flattened by the SAME renderer the inbound rich-message handler and the
+  // outbound card observer use, so all three agree on what a card "says".
+  // Returns `undefined` (never `''`) for an unrenderable block tree, so
+  // `resolveReplyToFromBuffer`'s `liveTextEmpty` gate still falls through to
+  // the buffer instead of pinning an empty antecedent.
+  const richParentText = extractRichMessageText(
+    (replyToMsg as { rich_message?: unknown } | undefined)?.rich_message,
+  )
   const replyToTextRaw = (quoteText != null && quoteText.length > 0)
     ? quoteText
     : replyToMsg
-      ? (replyToMsg.text ?? replyToMsg.caption ?? undefined)
+      ? (replyToMsg.text ?? replyToMsg.caption ?? richParentText ?? undefined)
       : undefined
   const replyToText = replyToTextRaw != null
     ? (replyToTextRaw.length > p.replyToTextMax
@@ -198,8 +222,11 @@ export function buildReplyForwardContext(p: ReplyForwardContextParams): {
 export interface ReplyToBufferFallbackParams {
   /** From {@link buildReplyForwardContext}. */
   replyToMessageId: number | undefined
-  /** Raw reply text off the live update (for the SQLite write); empty when the
-   *  reply target is the bot's own message (Telegram omits its text). */
+  /** Raw reply text off the live update (for the SQLite write). Empty only
+   *  when the live update carried NO readable parent body at all — a plain
+   *  `sendMessage` parent (Telegram omits `.text` on the bot's own plain
+   *  messages), or a rich parent whose blocks render to nothing. A normal
+   *  card parent now arrives populated off `rich_message` (#4598). */
   replyToText: string | undefined
   /** XML-escaped reply text for the channel meta; empty in the same case. */
   replyToTextEscaped: string | undefined
@@ -219,21 +246,36 @@ export interface ReplyToBufferFallbackParams {
 }
 
 /**
- * Reply-to buffer fallback (post-reset continuity). On a native reply to the
- * BOT's OWN message, Telegram delivers `reply_to_message.message_id` but NOT
- * its `.text` — so the live reply text is empty even though the gateway
- * authored (and, via `recordOutbound`, persisted) that message. This recovers
- * the antecedent from the local history buffer so it survives a session reset
- * (`resume_mode: handoff`), where the transcript is gone.
+ * Reply-to buffer fallback (post-reset continuity). On a native reply to a
+ * PLAIN message the bot itself sent, Telegram delivers
+ * `reply_to_message.message_id` but NOT its `.text` — so the live reply text
+ * is empty even though the gateway authored (and, via `recordOutbound`,
+ * persisted) that message. This recovers the antecedent from the local history
+ * buffer so it survives a session reset (`resume_mode: handoff`), where the
+ * transcript is gone.
+ *
+ * Since #4598 this is the SECOND line of defence FOR THE TEXT, not the first:
+ * a reply to a RICH parent (every card) now carries its body on
+ * `reply_to_message.rich_message` and is resolved live in
+ * {@link buildReplyForwardContext}, so a live-resolved body is never
+ * overwritten from the buffer.
+ *
+ * The LOOKUP itself, however, still runs on every reply while history is on.
+ * `replyToRole` and `replyToKind` have no live-update source at all, so
+ * skipping the lookup whenever the live text resolved would strip
+ * `reply_to_role` / `reply_to_kind` from precisely the recorded-card replies
+ * the #4571 kind lane was built for.
  *
  * Returns updated `replyToText` (raw, for the SQLite `recordInbound` write —
  * envelope-only would leave the row NULL and starve future handoff briefings),
  * `replyToTextEscaped` (for the channel-meta `reply_to_text`), and the
  * recovered `replyToRole` ('assistant' = the bot's own message, disambiguating
  * the "you're replying to yourself" case). Pure except for the injected
- * `lookup`. Only fills in when the live reply text is empty — never overwrites
- * a non-empty live value (a partial-quote or a reply to a person's message).
- * Degrades silently to the id-only inputs on any lookup failure.
+ * `lookup`. Only fills in the TEXT when the live reply text is empty — never
+ * overwrites a non-empty live value (a partial-quote, a reply to a person's
+ * message, or a live-rendered rich parent). Role and kind are filled in from
+ * any buffer hit regardless. Degrades silently to the id-only inputs on any
+ * lookup failure.
  */
 export function resolveReplyToFromBuffer(p: ReplyToBufferFallbackParams): {
   replyToText: string | undefined
@@ -248,23 +290,32 @@ export function resolveReplyToFromBuffer(p: ReplyToBufferFallbackParams): {
   let replyToRole: 'user' | 'assistant' | 'system' | undefined
   let replyToKind: string | undefined
   const liveTextEmpty = replyToTextEscaped == null || replyToTextEscaped.length === 0
-  if (p.historyEnabled && p.replyToMessageId != null && liveTextEmpty) {
+  if (p.historyEnabled && p.replyToMessageId != null) {
     try {
       const recovered = p.lookup(p.replyToMessageId)
-      if (recovered != null && recovered.role === 'system' && recovered.kind) {
-        replyToKind = recovered.kind
-      }
-      if (recovered && recovered.text.length > 0) {
-        replyToText =
-          recovered.text.length > p.replyToTextMax
-            ? recovered.text.slice(0, p.replyToTextMax - 1) + '…'
-            : recovered.text
-        replyToTextEscaped = formatReplyToText(recovered.text, p.replyToTextMax)
-        replyToRole = recovered.role
-      } else if (recovered) {
-        // Row exists (authorship known) but text is empty/redacted — still
-        // surface the role so the model knows whose message it is replying to.
+      if (recovered != null) {
+        // Role and kind are produced ONLY here — there is no live-update
+        // source for either. So the lookup runs on EVERY reply, not just the
+        // ones whose text the live update failed to carry: gating it on
+        // `liveTextEmpty` would mean a full reply to a recorded card resolved
+        // its body live (#4598) and silently lost `reply_to_role="system"` +
+        // `reply_to_kind` (#4571) — exactly the case the kind lane exists for,
+        // and the thing #4599 goes to AsyncLocalStorage lengths to record.
+        // Costs one SQLite point-read per reply.
         replyToRole = recovered.role
+        if (recovered.role === 'system' && recovered.kind) {
+          replyToKind = recovered.kind
+        }
+        // The TEXT is still second line of defence: never overwrite a
+        // non-empty live value (a partial quote, a person's message, or a
+        // rich parent rendered off the wire).
+        if (liveTextEmpty && recovered.text.length > 0) {
+          replyToText =
+            recovered.text.length > p.replyToTextMax
+              ? recovered.text.slice(0, p.replyToTextMax - 1) + '…'
+              : recovered.text
+          replyToTextEscaped = formatReplyToText(recovered.text, p.replyToTextMax)
+        }
       }
     } catch {
       // History disabled mid-run / requireDb throws / row missing — degrade

package/telegram-plugin/gateway/orphaned-db-sweep.ts

@@ -0,0 +1,315 @@
+/**
+ * Orphaned-DB-fd sweep — detect and recover from a SQLite handle that is
+ * still writing into DELETED inodes.
+ *
+ * THE FAILURE MODE
+ * ----------------
+ * The gateway holds `history.db` (bun:sqlite, WAL) open for the process
+ * lifetime. A FOREIGN process rw-opened the same DB, and on exit — as the
+ * last connection — SQLite checkpointed and UNLINKED `history.db-wal` and
+ * `history.db-shm`. Our long-lived connection kept the deleted inodes mapped
+ * and kept writing into them for 3h06m, logging success on every insert. The
+ * rows were never on disk; they vanished at the next restart. The signature is
+ * visible from the process itself:
+ *
+ *     /proc/<pid>/fd/13 -> /…/history.db-wal (deleted)
+ *
+ * Nothing in the write path can notice this: `INSERT` returns success, the
+ * boot-time `verifyHistoryWritable()` self-check already ran hours earlier, and
+ * a WAL checkpoint through the stale mapping "succeeds" too. The only in-process
+ * evidence is the fd table, so that is what we poll.
+ *
+ * WHAT THIS DOES
+ * --------------
+ * Every 5 minutes, walk `/proc/self/fd` and look for a link whose target is a
+ * `*.db` (or `-wal`/`-shm`/`-journal` sidecar) file inside the gateway's state
+ * dir marked `(deleted)`. On a hit:
+ *
+ *   - Log LOUDLY that rows written since the last checkpoint are LOST. Silent
+ *     data loss becomes a visible operator signal bounded to one sweep interval.
+ *   - `history.db`: hard-close-drop-reopen via `reopenHistory()`. Read the long
+ *     note there before touching it — a plain `close()` is NOT enough (it does
+ *     not release the fds, and the reopened connection then throws on the first
+ *     WRITE), and there is deliberately no salvage checkpoint through the
+ *     orphaned handle. If the reopen throws, we say so and ask for a restart
+ *     rather than claim a recovery that did not happen.
+ *   - `registry.db`: alarm only, RESTART REQUIRED. The registry handle
+ *     (`turnsDb`) is captured BY VALUE into long-lived wiring in `gateway.ts`
+ *     (the subagent-watcher options object among others), so a close-and-
+ *     reassign would leave those consumers holding a CLOSED handle — strictly
+ *     worse than the orphaned one. Detection without action is the honest
+ *     behaviour here; the operator restarts.
+ *   - anything else `*.db` in the state dir: alarm only, RESTART REQUIRED. No
+ *     lane owns it, so the honest answer is to name the file and say so rather
+ *     than raise a data-loss alarm with no instruction attached.
+ *
+ * AND ONE THING THAT IS NOT FD-DRIVEN
+ * -----------------------------------
+ * A reopen that hard-closes the old handle and then fails to re-init leaves
+ * history NULL and the orphaned fds GONE — so fd detection can never fire
+ * again, and a "FAILED to reopen" line printed once at 03:00 would be the only
+ * record of a permanent outage. Every tick therefore also checks the sticky
+ * `getHistoryReopenFailure()` flag, alarms on it, and retries the reopen,
+ * independently of what the fd table says.
+ *
+ * WHY THE SCAN IS ASYNC
+ * ---------------------
+ * The walk is O(open fds) — measured at 10-13ms against a live gateway holding
+ * 3366 fds, and `RLIMIT_NOFILE` on the fleet is 524288, so the cost is
+ * unbounded by anything we control. A synchronous readdir+readlink loop of that
+ * shape blocks the event loop, i.e. stalls inbound Telegram handling, for a
+ * check that finds nothing 99.99% of the time. `fs/promises` `opendir` +
+ * `readlink` yields between entries, so a long scan costs latency on the sweep
+ * (which nobody is waiting for) instead of on the gateway. The alternative —
+ * probing only the known DB paths — was rejected: it cannot tell "the file is
+ * gone" from "we still hold the gone file open", which is the entire signal.
+ *
+ * Dependency-free on purpose (`fs` + `path` only) so it loads identically under
+ * `bun test` and vitest, and so the recovery path cannot itself be broken by a
+ * transitive import that touches the DB.
+ */
+
+import { realpathSync } from 'fs'
+import { opendir, readlink } from 'fs/promises'
+import { basename } from 'path'
+
+/** A `/proc/self/fd` entry pointing at a deleted DB file in the state dir. */
+export interface OrphanedFd {
+  fd: number
+  /** The raw readlink target, including the trailing ` (deleted)` marker. */
+  target: string
+}
+
+/** Linux marks an unlinked-but-open fd's readlink target with this suffix. */
+const DELETED_SUFFIX = ' (deleted)'
+
+/**
+ * A SQLite database file or one of its sidecars, and nothing else.
+ *
+ * The previous `basename.includes('.db')` test also matched `notes.dbg`,
+ * `dump.dbf`, `history.db.bak` and `x.dbus` — an unrelated deleted temp file in
+ * the state dir would have raised a "rows are LOST" alarm. Anchored to the end
+ * of the basename so only a real `*.db` / `*.db-wal` / `*.db-shm` /
+ * `*.db-journal` matches.
+ */
+const DB_BASENAME_RE = /\.db(-wal|-shm|-journal)?$/
+
+/**
+ * Canonicalise the state dir for prefix matching against `/proc` targets.
+ *
+ * `/proc/self/fd` targets are ALWAYS fully resolved, so comparing them against
+ * a `TELEGRAM_STATE_DIR` that is a symlink (or relative) makes every
+ * `startsWith` fail and disables detection permanently — with no error, no log,
+ * and a sweep that reports "healthy" forever. Returns null when the path cannot
+ * be resolved at all, which callers surface as a warning rather than silence.
+ */
+export function resolveStateDirPrefix(stateDir: string): string | null {
+  let resolved: string
+  try {
+    resolved = realpathSync(stateDir)
+  } catch {
+    return null
+  }
+  return resolved.endsWith('/') ? resolved : resolved + '/'
+}
+
+/**
+ * Scan `/proc/self/fd` for handles onto deleted DB files under `stateDir`.
+ *
+ * Returns `[]` — never throws — on non-Linux (no `/proc`), on an unreadable or
+ * unresolvable `stateDir`, on an unreadable `/proc/self/fd`, and for any
+ * individual fd that races closed between the directory read and the
+ * `readlink`.
+ *
+ * A match requires ALL THREE of:
+ *   1. the target is inside the CANONICAL `stateDir` (so an unrelated deleted
+ *      DB elsewhere on the box is not our problem),
+ *   2. the target ends with ` (deleted)` (a healthy open WAL is NOT an orphan),
+ *   3. the basename is a SQLite file or sidecar (an unrelated deleted temp file
+ *      in the state dir must not raise a data-loss alarm).
+ */
+export async function detectOrphanedDbFds(stateDir: string): Promise<OrphanedFd[]> {
+  if (process.platform !== 'linux') return []
+  const prefix = resolveStateDirPrefix(stateDir)
+  if (prefix == null) return []
+  const found: OrphanedFd[] = []
+  try {
+    const dir = await opendir('/proc/self/fd')
+    for await (const entry of dir) {
+      let target: string
+      try {
+        target = await readlink(`/proc/self/fd/${entry.name}`)
+      } catch {
+        // The fd closed underneath us (including the directory handle's own
+        // fd). Not an orphan; just gone.
+        continue
+      }
+      // Parse FIRST, then filter. Doing the `slice` inside the deleted-check
+      // would make the basename test below silently do the deleted-check's job
+      // too (slicing 10 chars off a healthy `…/history.db-wal` yields `…/hist`,
+      // which fails the DB test by accident) — and an accidental guard is one
+      // nobody can mutation-test or safely refactor.
+      const deleted = target.endsWith(DELETED_SUFFIX)
+      const bare = deleted ? target.slice(0, -DELETED_SUFFIX.length) : target
+      if (!bare.startsWith(prefix)) continue
+      if (!deleted) continue
+      if (!DB_BASENAME_RE.test(basename(bare))) continue
+      found.push({ fd: Number(entry.name), target })
+    }
+  } catch {
+    return found
+  }
+  return found
+}
+
+/** Strip the ` (deleted)` marker and return the bare file name. */
+function orphanBasename(target: string): string {
+  return basename(target.endsWith(DELETED_SUFFIX) ? target.slice(0, -DELETED_SUFFIX.length) : target)
+}
+
+export interface OrphanedDbSweepOptions {
+  /** The gateway state dir whose DB files we own. */
+  stateDir: string
+  /**
+   * Recovery for `history.db`. Omit when history is disabled — detection and
+   * the loud log still run, only the reopen is skipped.
+   */
+  reopenHistory?: () => void
+  /**
+   * The sticky "history is dead" flag (`history.getHistoryReopenFailure`).
+   * Checked on EVERY tick, not only when an orphaned fd is found: once a
+   * reopen has closed the old handle the fds are released, so fd detection can
+   * no longer see the outage it caused. Omit only where history is not wired.
+   */
+  historyReopenFailure?: () => string | null
+  /** Log sink. The gateway passes `(l) => process.stderr.write(l)`. */
+  log: (line: string) => void
+}
+
+/**
+ * One sweep tick: detect, alarm, and recover. Returns the orphans found (empty
+ * on the healthy path, which is every tick but the incident one).
+ *
+ * Never throws — a failed reopen is logged and the next tick retries. That
+ * promise is only meaningful because of the sticky-failure lane below: a reopen
+ * whose close succeeded but whose re-init failed releases the very fds that
+ * would have triggered the next retry.
+ */
+export async function runOrphanedDbSweepTick(
+  opts: OrphanedDbSweepOptions,
+): Promise<OrphanedFd[]> {
+  if (process.platform === 'linux' && resolveStateDirPrefix(opts.stateDir) == null) {
+    opts.log(
+      `telegram gateway: orphaned-db-sweep cannot resolve stateDir=${opts.stateDir} —`
+      + ` deleted-inode DB detection is DISABLED until it exists and is readable.\n`,
+    )
+  }
+  const orphans = await detectOrphanedDbFds(opts.stateDir)
+  let historyHandled = false
+
+  if (orphans.length > 0) {
+    const names = orphans.map((o) => orphanBasename(o.target))
+    opts.log(
+      `telegram gateway: orphaned-db-sweep DETECTED ${orphans.length} deleted-inode DB handle(s): `
+      + orphans.map((o) => `fd=${o.fd} ${o.target}`).join(', ')
+      + ` — another process unlinked these files while we held them open; every row written`
+      + ` since the last checkpoint is LOST and further writes would be lost too.\n`,
+    )
+
+    if (names.some((n) => n.startsWith('history.db'))) {
+      historyHandled = true
+      if (opts.reopenHistory) {
+        attemptHistoryReopen(opts, 'reopened history.db')
+      } else {
+        opts.log(
+          `telegram gateway: orphaned-db-sweep found an orphaned history.db handle but no reopen`
+          + ` is wired (history disabled) — RESTART the gateway to recover.\n`,
+        )
+      }
+    }
+
+    if (names.some((n) => n.startsWith('registry.db'))) {
+      opts.log(
+        `telegram gateway: orphaned-db-sweep found an orphaned registry.db handle. An in-process`
+        + ` reopen is NOT safe here — the turnsDb handle is captured by value into long-lived`
+        + ` wiring, so closing it would leave those consumers on a closed handle. RESTART the`
+        + ` gateway to recover; subagent/turn rows written since the last checkpoint are LOST.\n`,
+      )
+    }
+
+    // Anything else under the state dir has no lane. Say so explicitly: an
+    // unnamed data-loss alarm with no recovery instruction is worse than none.
+    const unowned = [...new Set(names.filter(
+      (n) => !n.startsWith('history.db') && !n.startsWith('registry.db'),
+    ))]
+    if (unowned.length > 0) {
+      opts.log(
+        `telegram gateway: orphaned-db-sweep found orphaned handle(s) on ${unowned.join(', ')},`
+        + ` which no recovery lane owns — the gateway cannot reopen them in place. RESTART the`
+        + ` gateway to recover; rows written to those files since the last checkpoint are LOST.\n`,
+      )
+    }
+  }
+
+  // The fd table cannot see a history DB that is already closed and failed to
+  // re-open, so this lane runs whether or not anything was detected.
+  if (!historyHandled) {
+    const stuck = opts.historyReopenFailure?.()
+    if (stuck != null && stuck !== '') {
+      opts.log(
+        `telegram gateway: orphaned-db-sweep history.db is CLOSED and a previous reopen failed`
+        + ` (${stuck}) — every history read and write is dead and no fd evidence remains.`
+        + ` Retrying the reopen; RESTART the gateway if this keeps repeating.\n`,
+      )
+      if (opts.reopenHistory) attemptHistoryReopen(opts, 'recovered history.db')
+    }
+  }
+
+  return orphans
+}
+
+/**
+ * Run the wired reopen, logging honestly either way. `successVerb` distinguishes
+ * the first-detection recovery from the sticky-failure retry in the log.
+ */
+function attemptHistoryReopen(opts: OrphanedDbSweepOptions, successVerb: string): void {
+  try {
+    opts.reopenHistory?.()
+    opts.log(
+      `telegram gateway: orphaned-db-sweep ${successVerb} — writes are durable again`
+      + ` (proved by the post-reopen writer self-check); rows written since the last`
+      + ` checkpoint are NOT recoverable.\n`,
+    )
+  } catch (err) {
+    opts.log(
+      `telegram gateway: orphaned-db-sweep FAILED to reopen history.db: ${(err as Error).message}`
+      + ` — history writes are NOT durable; RESTART the gateway.\n`,
+    )
+  }
+}
+
+/** Default cadence: bounds silent loss to 5 minutes without polling cost. */
+const DEFAULT_INTERVAL_MS = 5 * 60_000
+
+/**
+ * Start the periodic sweep. Returns a `stop()` so tests can tear it down;
+ * production never stops it (matches every sibling gateway interval).
+ *
+ * `unref()`s so the timer cannot hold the process alive past shutdown. Ticks do
+ * not overlap: the scan is async, so a slow `/proc` walk on a process holding
+ * hundreds of thousands of fds must not stack up behind itself.
+ */
+export function startOrphanedDbSweep(
+  opts: OrphanedDbSweepOptions & { intervalMs?: number },
+): () => void {
+  let running = false
+  const timer = setInterval(() => {
+    if (running) return
+    running = true
+    void runOrphanedDbSweepTick(opts)
+      .catch(() => { /* a sweep must never take the gateway down; next tick retries */ })
+      .finally(() => { running = false })
+  }, opts.intervalMs ?? DEFAULT_INTERVAL_MS)
+  timer.unref?.()
+  return () => clearInterval(timer)
+}

package/telegram-plugin/gateway/system-message-observer.ts

@@ -24,10 +24,29 @@
  * Rather than add a `recordSystemOutbound(...)` call to each of the ~110 raw
  * send sites (which is exactly the kind of per-call-site discipline that
  * decays — the `reply` path was the only site anyone remembered), this hooks
- * the ONE chokepoint every gateway outbound already goes through:
- * `gateway.ts`'s `robustApiCall` (chat-lock → send-gate → retry policy). Every
- * card send in the gateway is routed through it, enforced by the
- * `check-bot-api-wrapping` lint guard.
+ * ONE chokepoint and observes the resolved response there.
+ *
+ * WHICH chokepoint took two goes, and the first answer was wrong (#4599). This
+ * docblock used to say the hook sat on `gateway.ts`'s `robustApiCall` — "the
+ * ONE chokepoint every gateway outbound already goes through … enforced by the
+ * `check-bot-api-wrapping` lint guard". Both halves were false from the day it
+ * merged. grammY's `ctx.*` sugar builds the payload and calls `bot.api.*`
+ * itself, so `switchroomReply` — the helper every SLASH-COMMAND card answers
+ * through (`/usage`, `/model`, `/auth`, `/approvals`, `/start`, `/help`) — sends
+ * via `ctx.replyWithRichMessage` and never touches `robustApiCall`; and the
+ * lint guard could not have caught that, because its verb pattern matched only
+ * `(bot|lockedBot|ctx)\.api\.<verb>`, never mentioned `sendRichMessage`, and
+ * structurally cannot match a `ctx.replyWith*` call at all. Measured, not
+ * inferred: a live agent's `/usage` card at id 20938 left NO row while the
+ * `tg-post` transformer logged its `sendRichMessage` POST.
+ *
+ * The hook therefore lives at the grammy API TRANSFORMER layer
+ * (`installSystemMessageObserver` in `shared/bot-runtime.ts`), which grammy
+ * resolves immediately around the HTTP POST — below every helper, `ctx.*`
+ * shorthand, `lockedBot`, and `bot.api.raw`. No call shape reaches Telegram
+ * without passing through it, so no future verb can opt out the way
+ * `switchroomReply` silently did. `robustApiCall` still publishes its `verb`
+ * down to that layer (`withTgSendContext`) so cards keep their `kind`.
  *
  * The observer reads the Telegram RESPONSE, which buys three things for free:
  *   - the real `message_id` (the only thing a reply can point at),
@@ -90,7 +109,7 @@ export interface SentMessageLike {
   rich_message?: unknown
 }
 
-/** The subset of `robustApiCall`'s opts the observer reads. */
+/** The call-site metadata the observer reads (`bot-runtime.ts`'s `TgSendContext`). */
 export interface ObservedCallOpts {
   chat_id?: string
   threadId?: number
@@ -293,7 +312,7 @@ type TrackedState = { lane: 'system' | 'foreign'; lastStoredMs: number; storedLe
 
 /**
  * Build the observer. The returned function is called with the RESOLVED result
- * of every `robustApiCall` and never throws.
+ * of every outbound Bot API call (from the transformer layer) and never throws.
  */
 export function makeSystemMessageObserver(
   deps: SystemMessageObserverDeps,