switchroom 0.19.29 → 0.19.31

package/telegram-plugin/gateway/status-pin-retarget.ts

@@ -79,6 +79,15 @@ export interface StatusPinClaim {
   messageId: number
   /** Chat the pin lives in. Never empty — the reconcile refuses an empty chat. */
   chatId: string
+  /**
+   * Forum topic the pinned message lives in, when known.
+   *
+   * The claim is keyed by `(chat, thread)`. It is NOT needed to unpin this one
+   * message (`unpinChatMessage` takes no `message_thread_id`), but it is the
+   * only thing that lets a sweep aim `unpinAllForumTopicMessages` at the right
+   * topic when a crash leaves a chat-wide stack of orphans behind.
+   */
+  threadId?: number
   /** Wall-clock ms the claim was FIRST taken (survives a same-id re-pin). */
   pinnedAt: number
 }
@@ -86,6 +95,9 @@ export interface StatusPinClaim {
 export interface StatusPinReconcileArgs {
   pinKey: string
   chatId: string
+  /** Forum topic the pinned message lives in, when the caller knows it. Carried
+   *  into both the in-memory claim and the durable row (see StatusPinClaim). */
+  threadId?: number
   /** The claim held for this key when the reconcile started. */
   prev: PinState | null
   /** What the caller wants pinned for this key now. */
@@ -116,7 +128,7 @@ export interface StatusPinReconcileArgs {
  * is read once by the caller and both legs run inside that critical section.
  */
 export async function runStatusPinReconcile(args: StatusPinReconcileArgs): Promise<void> {
-  const { pinKey, chatId, prev, desired, persist, runPin, claims } = args
+  const { pinKey, chatId, threadId, prev, desired, persist, runPin, claims } = args
   const now = args.now ?? Date.now
 
   // Publish a leg's outcome into the single claim registry. `pinnedAt` is
@@ -128,7 +140,7 @@ export async function runStatusPinReconcile(args: StatusPinReconcileArgs): Promi
       return
     }
     const pinnedAt = claims.get(pinKey)?.pinnedAt ?? now()
-    claims.set(pinKey, { messageId: next.messageId, chatId, pinnedAt })
+    claims.set(pinKey, { messageId: next.messageId, chatId, threadId, pinnedAt })
   }
 
   // One leg, with the persist ordering its action requires. Only a fresh `pin`
@@ -152,6 +164,7 @@ export async function runStatusPinReconcile(args: StatusPinReconcileArgs): Promi
       fs: persist.fs,
       pinKey,
       chatId,
+      threadId,
       op,
       applyPin: () => runPin(legAction, from),
       now: now(),

package/telegram-plugin/gateway/status-pin-store.ts

@@ -55,6 +55,23 @@ export interface StatusPinStoreFsSeam {
 export interface PersistedStatusPin {
   pinKey: string
   chatId: string
+  /**
+   * Forum topic the pinned message lives in, when the caller knows it.
+   *
+   * The durable claim is keyed by `(chat, thread)`, not by chat alone. Telegram's
+   * pin stack is CHAT-WIDE — a "topic pin" is a chat-level pin whose message
+   * happens to sit in that thread, and `pinChatMessage`/`unpinChatMessage` take
+   * no `message_thread_id` at all. So the thread is NOT needed to unpin one
+   * known message; it is needed to DRAIN a topic, because the only topic-scoped
+   * pin verb a bot has is `unpinAllForumTopicMessages(chat_id, message_thread_id)`.
+   *
+   * Without this field a row records "there were orphan pins in forum chat X"
+   * and nothing more, so a boot after a crash cannot aim the one verb that would
+   * clear them — which is how a chat-wide stack grows on every restart. Optional
+   * so a v1–v3 snapshot still loads; a row without it degrades to the
+   * chat-level path.
+   */
+  threadId?: number
   messageId: number
   /** True while the pin API call is in-flight / unconfirmed (see above). */
   pending?: boolean
@@ -102,16 +119,17 @@ export const BOOT_UNPIN_MAX_ATTEMPTS = 5
 
 /** Envelope version. v1 had no `pending` field; a v1 row loads as a confirmed
  *  pin (pending undefined). v2 adds the optional `pending` flag. v3 adds the
- *  optional `pinnedAt` claim timestamp (#3810). All load fail-open — an
- *  unknown/newer version yields []. Every field added since v1 is optional, so
- *  the versions are mutually readable and a downgrade degrades rather than
- *  breaks. */
+ *  optional `pinnedAt` claim timestamp (#3810). v4 adds the optional `threadId`
+ *  so the claim is keyed by (chat, thread) and a forum topic can be drained
+ *  after a crash. All load fail-open — an unknown/newer version yields [].
+ *  Every field added since v1 is optional, so the versions are mutually
+ *  readable and a downgrade degrades rather than breaks. */
 interface SnapshotEnvelope {
-  v: 1 | 2 | 3
+  v: 1 | 2 | 3 | 4
   pins: PersistedStatusPin[]
 }
 
-const SNAPSHOT_VERSIONS = new Set([1, 2, 3])
+const SNAPSHOT_VERSIONS = new Set([1, 2, 3, 4])
 
 function isPinRow(x: unknown): x is PersistedStatusPin {
   if (x == null || typeof x !== 'object') return false
@@ -121,6 +139,7 @@ function isPinRow(x: unknown): x is PersistedStatusPin {
     o.pinKey.length > 0 &&
     typeof o.chatId === 'string' &&
     o.chatId.length > 0 &&
+    (o.threadId === undefined || typeof o.threadId === 'number') &&
     typeof o.messageId === 'number' &&
     (o.pending === undefined || typeof o.pending === 'boolean') &&
     (o.expiresAt === undefined || typeof o.expiresAt === 'number') &&
@@ -170,7 +189,7 @@ export function persistStatusPins(
   snapshot: readonly PersistedStatusPin[],
   log: (line: string) => void = (l) => process.stderr.write(l),
 ): void {
-  const env: SnapshotEnvelope = { v: 3, pins: [...snapshot] }
+  const env: SnapshotEnvelope = { v: 4, pins: [...snapshot] }
   const tmp = path + '.tmp'
   try {
     fs.writeFileSync(tmp, JSON.stringify(env))
@@ -450,6 +469,9 @@ export function reconcileAndPersistStatusPin(args: {
   fs: StatusPinStoreFsSeam
   pinKey: string
   chatId: string
+  /** Forum topic the pinned message lives in, when known — persisted so a boot
+   *  after a crash can aim `unpinAllForumTopicMessages` at the right topic. */
+  threadId?: number
   op: StatusPinPersistOp
   /** Execute the real pin/unpin; returns the confirmed message id (pin) or
    *  null (cleared). Must never throw — API errors are swallowed inside. */
@@ -458,7 +480,7 @@ export function reconcileAndPersistStatusPin(args: {
   now?: number
   log?: (line: string) => void
 }): Promise<{ messageId: number } | null> {
-  const { path, fs, pinKey, chatId, op } = args
+  const { path, fs, pinKey, chatId, threadId, op } = args
   const log = args.log ?? ((l: string) => process.stderr.write(l))
   const now = args.now ?? Date.now()
 
@@ -489,7 +511,7 @@ export function reconcileAndPersistStatusPin(args: {
         path,
         fs,
         pinKey,
-        { pinKey, chatId, messageId: op.messageId, pending: true, pinnedAt },
+        { pinKey, chatId, threadId, messageId: op.messageId, pending: true, pinnedAt },
         log,
       )
       const next = await args.applyPin()
@@ -504,7 +526,7 @@ export function reconcileAndPersistStatusPin(args: {
         path,
         fs,
         pinKey,
-        { pinKey, chatId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
+        { pinKey, chatId, threadId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
         log,
       )
       return next
@@ -541,7 +563,7 @@ export function reconcileAndPersistStatusPin(args: {
         path,
         fs,
         pinKey,
-        { pinKey, chatId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
+        { pinKey, chatId, threadId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
         log,
       )
     }

package/telegram-plugin/registry/turns-schema.ts

@@ -240,7 +240,27 @@ const PHASE4_MIGRATIONS = [
 
 function applySchema(db: SqliteDatabase): void {
   db.exec('PRAGMA journal_mode = WAL')
-  db.exec('PRAGMA synchronous = NORMAL')
+  // FULL, not NORMAL. This registry is the crash-recovery ledger: the turn row
+  // is what tells the boot-time resume protocol that a turn was in flight when
+  // the process died. Under WAL, `synchronous = NORMAL` deliberately does NOT
+  // fsync the WAL on commit — it syncs only at checkpoint — so a container kill
+  // is survived but a HOST POWER CUT can lose the last commits, including the
+  // whole turn row. The interrupted turn then becomes invisible to resume:
+  // silently unrecoverable, which is exactly the case this ledger exists for.
+  //
+  // Cost is bounded and measured, not assumed: on this fleet's ext4/NVMe volume
+  // a FULL commit costs ~0.87ms mean (p99 1.7ms) vs ~0.011ms at NORMAL, i.e. a
+  // ~1150 commits/s ceiling. Steady state is ~330 commits/day, and the hot path
+  // is `bumpSubagentActivity` (subagents-schema.ts) at ~1Hz per running worker
+  // — ~4 commits/s during fan-out, a ~0.35% duty cycle.
+  //
+  // `synchronous` is per-connection and NOT persisted in the DB file (unlike
+  // `journal_mode`), so it binds only connections that set it. The PreToolUse
+  // subagent tracker (`hooks/subagent-tracker-pretool.mjs`) opens this same
+  // registry.db without the pragma and has therefore been running at SQLite's
+  // default FULL on the tool-call hot path since it shipped; the gateway's
+  // NORMAL was the outlier.
+  db.exec('PRAGMA synchronous = FULL')
   // Concurrency: multiple writers contend on this registry (the PreToolUse
   // subagent-tracker hook, the gateway's subagent-watcher backfill, the turns
   // writer) — especially when several sub-agents dispatch at once. Without a

package/telegram-plugin/retry-api-call.ts

@@ -92,13 +92,49 @@ export interface RetryCallOpts {
   priorityClass?: 'critical' | 'useful' | 'cosmetic'
 }
 
+/**
+ * The three benign Telegram 400s this policy swallows (contract table above).
+ */
+export type BenignTelegram400Kind = 'not_modified' | 'message_not_found' | 'delete_not_found'
+
+/**
+ * Single source of truth for "is this Telegram 400 a benign no-op?".
+ *
+ * Benign here means exactly what the retry policy already treats as a
+ * non-event: the edit was a no-op because the content is identical, or the
+ * target message vanished before the edit/delete landed. Both are
+ * high-volume on a healthy agent (every coalesced card repaint can produce
+ * one) and neither means anything is wrong — so any surface that reports
+ * Telegram failures needs to tell them apart from a real rejection WITHOUT
+ * maintaining a second copy of the description list (#3927). The retry
+ * policy's own swallow branches call this, so there is one list.
+ *
+ * Returns the benign kind, or `null` for everything else — including 429,
+ * 403, and any 400 not on this deliberately narrow list. A new benign
+ * description belongs here, not in a caller.
+ */
+export function classifyBenignTelegram400(
+  errorCode: number | undefined,
+  description: string | undefined,
+): BenignTelegram400Kind | null {
+  if (errorCode !== 400) return null
+  const desc = (description ?? '').toLowerCase()
+  // "message is not modified" / "message was not modified" — Telegram's
+  // no-op-on-equal-text.
+  if (desc.includes('not modified')) return 'not_modified'
+  // "message to edit/delete not found" — the target vanished.
+  if (desc.includes('message to edit not found')) return 'message_not_found'
+  if (desc.includes('message to delete not found')) return 'delete_not_found'
+  return null
+}
+
 export interface RetryObserver {
   /** Fires just before sleeping for a retry. */
   onRetry?(info: { attempt: number; reason: 'flood_wait' | 'network'; delayMs: number }): void
   /** Fires when max retries is reached and the wrapper gives up. */
   onGiveUp?(info: { attempts: number; error: unknown }): void
   /** Fires for each benign error we swallowed (not-modified, not-found). */
-  onBenign?(info: { kind: 'not_modified' | 'message_not_found' | 'delete_not_found' }): void
+  onBenign?(info: { kind: BenignTelegram400Kind }): void
 }
 
 export interface RetryApiCallConfig {
@@ -399,26 +435,15 @@ export function createRetryApiCall(
           continue
         }
 
-        // Swallow "message is not modified" — Telegram's no-op-on-equal-text.
-        if (
-          isGrammyErr &&
-          (err as GrammyError).error_code === 400 &&
-          desc.includes('not modified')
-        ) {
-          observer?.onBenign?.({ kind: 'not_modified' })
-          return undefined as unknown as T
-        }
-
-        // Swallow "message to edit/delete not found" — target vanished.
-        if (
-          isGrammyErr &&
-          (err as GrammyError).error_code === 400 &&
-          (desc.includes('message to edit not found') ||
-            desc.includes('message to delete not found'))
-        ) {
-          observer?.onBenign?.({
-            kind: desc.includes('edit') ? 'message_not_found' : 'delete_not_found',
-          })
+        // Swallow the benign 400s — "message is not modified" (Telegram's
+        // no-op-on-equal-text) and "message to edit/delete not found" (the
+        // target vanished). Classification lives in `classifyBenignTelegram400`
+        // so the tg-post logger reports the same family identically (#3927).
+        const benignKind = isGrammyErr
+          ? classifyBenignTelegram400((err as GrammyError).error_code, desc)
+          : null
+        if (benignKind !== null) {
+          observer?.onBenign?.({ kind: benignKind })
           return undefined as unknown as T
         }
 

package/telegram-plugin/shared/bot-runtime.ts

@@ -28,10 +28,10 @@ import { execFileSync, spawnSync } from 'child_process'
 import { createHash } from 'crypto'
 import { AsyncLocalStorage } from 'async_hooks'
 import { clearStaleTelegramPollingState } from '../startup-reset.js'
-import { createRetryApiCall } from '../retry-api-call.js'
+import { createRetryApiCall, classifyBenignTelegram400 } from '../retry-api-call.js'
 import { makeFloodWaitRecorder, makeFloodWaitProbe } from '../flood-circuit-breaker.js'
 import { RICH_MESSAGE_MAX_CHARS } from '../format.js'
-import { shouldEmitTgPost } from './gw-trace-gate.js'
+import { shouldEmitTgPost, type TgPostStatus } from './gw-trace-gate.js'
 import { guardAccidentalFormatting } from '../rich-send.js'
 
 // ─── tg-post tag plumbing ─────────────────────────────────────────────────
@@ -80,6 +80,17 @@ function formatTgPostTags(tags: TgPostTags | undefined): string {
 
 // ─── tg-post observability transformer ────────────────────────────────────
 
+/**
+ * Sanitise a Telegram error description for single-line log output —
+ * collapse whitespace, strip newlines, cap at 80 chars, `-` when absent.
+ * PII-safe: Telegram error descriptions are server-generated and don't
+ * echo the request body.
+ */
+function shortDesc(raw: string | undefined): string {
+  if (!raw) return '-'
+  return raw.replace(/\s+/g, ' ').slice(0, 80).replace(/[\r\n]/g, ' ') || '-'
+}
+
 /**
  * Installs an API transformer on the bot that emits one stderr line per
  * outbound Telegram Bot API POST. This is the single catchment point for
@@ -91,7 +102,30 @@ function formatTgPostTags(tags: TgPostTags | undefined): string {
  *
  * Log shape (one line per POST, on both success and failure):
  *
- *   tg-post method=<m> chat=<id> thread=<id|-> parse_mode=<HTML|MarkdownV2|none> bytes=<n> hash=<sha1-12> status=<ok|err> err=<class-or--> code=<http-or--> desc=<short|-->
+ *   tg-post method=<m> chat=<id> thread=<id|-> parse_mode=<HTML|MarkdownV2|none> bytes=<n> hash=<sha1-12> status=<ok|benign|err> err=<class-or--> code=<http-or--> desc=<short|-->
+ *
+ * TRUTHFULNESS (#3927): grammY resolves the transformer chain with the raw
+ * `ApiResponse`, and only converts `{ok:false}` into a thrown `GrammyError`
+ * AFTER the chain returns — verified in the pinned grammy 1.44.0,
+ * `out/core/client.js:95-100`:
+ *
+ *     const data = await this.call(method, payload, signal)   // chain
+ *     if (data.ok) return data.result
+ *     else throw toGrammyError(data, method, payload)          // OUTSIDE
+ *
+ * So EVERY Telegram-level rejection (429 flood-wait, 400, 403) arrives here
+ * as a RESOLVED response, and the `catch` below only ever sees transport
+ * (`HttpError`) failures. Before this was fixed the logger reported every
+ * one of them as `status=ok err=- code=- desc=-` — a rate-limited
+ * `unpinAllChatMessages` logged as SUCCESS 3ms before the retry policy
+ * logged `429 rate limited, waiting 3s`. We therefore inspect the resolved
+ * body and report `status=err err=telegram_<code>` from it.
+ *
+ * The narrow set of benign 400s the retry policy already swallows
+ * ("message is not modified", "message to edit/delete not found" —
+ * classified by the shared `classifyBenignTelegram400`) gets `status=benign`
+ * instead, so promoting real rejections out of `status=ok` doesn't bury
+ * `grep status=err` under high-volume card-repaint no-ops.
  *
  * Body content is never logged — only its length and a 12-char sha1 prefix
  * so we can recognise repeated identical sends without leaking PII. The
@@ -115,16 +149,34 @@ export function installTgPostLogger(bot: Bot): void {
       ? createHash('sha1').update(text).digest('hex').slice(0, 12)
       : '-'
     const tagSuffix = formatTgPostTags(_getTgPostTags())
-    try {
-      const res = await prev(method, payload, signal)
+    const emit = (status: TgPostStatus, errClass: string, code: string, desc: string) => {
       // #3025: suppress zero-signal per-poll heartbeats (getUpdates/getMe
       // status=ok, one line per ~30s long-poll tick) unless the operator
       // set SWITCHROOM_GW_TRACE. Errors and all other methods still log.
-      if (shouldEmitTgPost(method, 'ok')) {
-        process.stderr.write(
-          `tg-post method=${method} chat=${chat} thread=${thread} parse_mode=${parseMode} bytes=${bytes} hash=${hash} status=ok err=- code=- desc=-${tagSuffix}\n`,
+      if (!shouldEmitTgPost(method, status)) return
+      process.stderr.write(
+        `tg-post method=${method} chat=${chat} thread=${thread} parse_mode=${parseMode} bytes=${bytes} hash=${hash} status=${status} err=${errClass} code=${code} desc=${desc}${tagSuffix}\n`,
+      )
+    }
+    try {
+      const res = await prev(method, payload, signal)
+      // A RESOLVED response can still be a Telegram-level rejection — grammy
+      // converts `{ok:false}` to a thrown GrammyError only after this chain
+      // returns (see the docblock). Same defensive shape as the edit fuse's
+      // `runObserved` (edit-flood-fuse.ts).
+      const r = res as unknown as { ok?: boolean; error_code?: number; description?: string }
+      if (r != null && typeof r === 'object' && r.ok === false) {
+        const code = r.error_code
+        const benign = classifyBenignTelegram400(code, r.description)
+        emit(
+          benign !== null ? 'benign' : 'err',
+          `telegram_${code ?? 'unknown'}`,
+          code != null ? String(code) : '-',
+          shortDesc(r.description),
         )
+        return res
       }
+      emit('ok', '-', '-', '-')
       return res
     } catch (err) {
       const errClass = err instanceof GrammyError
@@ -134,15 +186,7 @@ export function installTgPostLogger(bot: Bot): void {
       const rawDesc = err instanceof GrammyError
         ? (err as GrammyError).description
         : (err instanceof Error ? err.message : '')
-      // Sanitise the description for single-line log output — collapse
-      // whitespace, strip newlines, cap at 80 chars. PII-safe: Telegram
-      // error descriptions are server-generated and don't echo body.
-      const desc = rawDesc
-        ? rawDesc.replace(/\s+/g, ' ').slice(0, 80).replace(/[\r\n]/g, ' ') || '-'
-        : '-'
-      process.stderr.write(
-        `tg-post method=${method} chat=${chat} thread=${thread} parse_mode=${parseMode} bytes=${bytes} hash=${hash} status=err err=${errClass} code=${code} desc=${desc}${tagSuffix}\n`,
-      )
+      emit('err', errClass, code, shortDesc(rawDesc))
       throw err
     }
   })

package/telegram-plugin/shared/gw-trace-gate.ts

@@ -57,6 +57,20 @@ export const gwTraceVerbose: boolean = computeGwTraceVerbose(
  */
 const ZERO_SIGNAL_POLL_METHODS = new Set(['getUpdates', 'getMe'])
 
+/**
+ * The status tiers a `tg-post` line can carry.
+ *
+ * `benign` (#3927) is a REJECTED call that is nonetheless a non-event —
+ * the narrow set of 400s the retry policy already swallows ("message is
+ * not modified", "message to edit/delete not found"), classified by
+ * `classifyBenignTelegram400`. It exists so that promoting resolved
+ * `ok:false` responses out of `status=ok` doesn't drown the newly-truthful
+ * `status=err` signal in high-volume card-repaint no-ops: `grep status=err`
+ * stays a list of real failures, and the benign lines are still there,
+ * honestly labelled, for anyone who wants them.
+ */
+export type TgPostStatus = 'ok' | 'benign' | 'err'
+
 /**
  * Decide whether a `tg-post` line should be written.
  *
@@ -66,11 +80,13 @@ const ZERO_SIGNAL_POLL_METHODS = new Set(['getUpdates', 'getMe'])
  * Always emitted:
  *   - any `status=err` (real failures — the whole point of the log),
  *   - every other method's `ok` line (sendMessage/editMessageText/... are
- *     genuine outbound observability, #656/#657).
+ *     genuine outbound observability, #656/#657),
+ *   - `status=benign` on any non-poll method — same volume as the
+ *     `status=ok` line it replaces, just no longer mislabelled as success.
  */
 export function shouldEmitTgPost(
   method: string,
-  status: 'ok' | 'err',
+  status: TgPostStatus,
   verbose: boolean = gwTraceVerbose,
 ): boolean {
   if (verbose) return true

package/telegram-plugin/tests/activity-card-wiring.test.ts

@@ -119,17 +119,17 @@ describe('activity-card durability wiring', () => {
 
   it('the boot reaper runs ONLY after the startup mutex is won (never at import time)', () => {
     // #3026: the reaper is now invoked via the mutex-gated orchestrator
-    // runBootPinCleanupAndDmSweep() (which runs it alongside
-    // statusPinBootCleanup + queuedCardBootReaper, then runs the DM
-    // stale-pin sweep). Invariant unchanged: reachable only post-lock.
+    // runBootPinCleanupAndStalePinSweep() (which runs it alongside
+    // statusPinBootCleanup + queuedCardBootReaper, then runs the stale-pin
+    // stack sweep). Invariant unchanged: reachable only post-lock.
     // (1) The orchestrator runs the reaper. Since the #3664 S2 salvage the
     // steps are INJECTED into runBootPinSweepSteps (each individually
-    // absorbed, so a throwing reaper cannot strand the DM sweep behind it)
+    // absorbed, so a throwing reaper cannot strand the sweep behind it)
     // instead of being awaited inline — so the marker is the step binding.
     const orchestrator = between(
       gatewaySrc,
-      'function runBootPinCleanupAndDmSweep()',
-      'dmPinSweepEligible = true',
+      'function runBootPinCleanupAndStalePinSweep()',
+      'stalePinSweepEligible = true',
     )
     expect(orchestrator).toMatch(/activityCardReaper: activityCardBootReaper,/)
     // (2) Both orchestrator invocation sites (mutex-won + mutex-fallback)
@@ -146,7 +146,7 @@ describe('activity-card durability wiring', () => {
     // statusPinBootCleanup documents.
     const beforeMain = gatewaySrc.split('await acquireStartupLock({')[0] ?? ''
     expect(beforeMain).not.toMatch(/^\s*void activityCardBootReaper\(\)/m)
-    expect(beforeMain).not.toMatch(/^\s*void runBootPinCleanupAndDmSweep\(\)/m)
+    expect(beforeMain).not.toMatch(/^\s*void runBootPinCleanupAndStalePinSweep\(\)/m)
     expect(beforeMain).not.toMatch(/^\s*bootPinSweepGate\.(arm|botReady)\(\)/m)
   })