switchroom 0.19.17 → 0.19.19

package/telegram-plugin/gateway/pending-card-store.ts

@@ -32,11 +32,24 @@
  *
  * File format: JSON array of PersistedApprovalCard objects. Written
  * synchronously (mode 0o600) to avoid interleaving on concurrent card posts;
- * production rate is a handful of cards, so the file stays tiny.
+ * production rate is a handful of cards, so the file stays tiny. The write
+ * goes through `atomicWriteFileSync` (tmp + fsync + rename) so a crash
+ * mid-persist can never leave a torn file behind — the destination holds
+ * either the whole previous array or the whole new one. A file that IS
+ * corrupt (from a pre-fix write, or anything else) is quarantined and logged
+ * loudly rather than silently read as "no pending cards" — see
+ * `store-file.ts`. NOTE: atomic REPLACEMENT only — whole-old-or-whole-new, not
+ * power-loss durability; the missing parent-directory fsync is tracked in #3603.
  */
 
-import { readFileSync, writeFileSync, unlinkSync, chmodSync } from 'node:fs'
+import { unlinkSync } from 'node:fs'
 import { join } from 'node:path'
+import { atomicWriteFileSync } from '../../src/util/atomic.js'
+import {
+  preserveUnreadableStoreFile,
+  quarantineCorruptStoreFile,
+  readStoreJsonSync,
+} from './store-file.js'
 
 /** The four agent-initiated approval-card families we persist. */
 export type ApprovalCardFamily =
@@ -111,30 +124,47 @@ export interface PendingCardStore {
   clear(): void
 }
 
-export function createPendingCardStore(stateDir: string): PendingCardStore {
+export function createPendingCardStore(
+  stateDir: string,
+  /** Log sink — defaults to stderr (the gateway's runtime log). */
+  log: (line: string) => void = l => process.stderr.write(l),
+): PendingCardStore {
   const filePath = join(stateDir, 'pending-approval-cards.json')
 
+  /** Set when the last read failed for a non-ENOENT reason — the next write
+   * must preserve the file it could not read instead of clobbering it. */
+  let unreadable = false
+
   function read(): PersistedApprovalCard[] {
-    try {
-      const raw = readFileSync(filePath, 'utf-8')
-      const parsed = JSON.parse(raw)
-      return Array.isArray(parsed) ? (parsed as PersistedApprovalCard[]) : []
-    } catch {
+    const result = readStoreJsonSync(filePath, 'pending-card-store', log)
+    unreadable = result.status === 'unreadable'
+    if (result.status !== 'ok') return []
+    if (!Array.isArray(result.value)) {
+      quarantineCorruptStoreFile(
+        filePath,
+        'pending-card-store',
+        'parsed to a non-array — not a pending-card file',
+        log,
+      )
       return []
     }
+    return result.value as PersistedApprovalCard[]
   }
 
   function write(entries: PersistedApprovalCard[]): void {
     try {
-      writeFileSync(filePath, JSON.stringify(entries), { encoding: 'utf-8', mode: 0o600 })
-      // `mode` only applies when writeFileSync CREATES the file; an existing
-      // file keeps its prior perms. Re-assert 0600 on every write so the file
-      // can never stay laxer than intended.
-      chmodSync(filePath, 0o600)
+      // Fail closed: never let an overwrite be what destroys state we merely
+      // failed to READ (flaky mount, transient EACCES) — throws if it can't.
+      if (unreadable) {
+        preserveUnreadableStoreFile(filePath, 'pending-card-store', log)
+        unreadable = false
+      }
+      // tmp + fsync + rename, mode pinned to 0600 on the tempfile fd (so an
+      // existing file can't keep laxer perms, and a crash mid-write leaves
+      // the previous good file untouched).
+      atomicWriteFileSync(filePath, JSON.stringify(entries), 0o600)
     } catch (err) {
-      process.stderr.write(
-        `telegram gateway: pending-card-store write failed: ${(err as Error).message}\n`,
-      )
+      log(`telegram gateway: pending-card-store write failed: ${(err as Error).message}\n`)
     }
   }
 

package/telegram-plugin/gateway/resume-inbound-builder.ts

@@ -458,9 +458,13 @@ export function buildResumeDeferredReportInbound(
  * the gateway calls this with the classified `ended_via` so the
  * report-vs-resume policy lives in one testable place.
  *
- *   - 'timeout'                         → 'report'  (watchdog kill)
- *   - 'restart' | 'sigterm' | 'unknown' → 'resume'  (clean interrupt)
- *   - 'stop'                            → null      (finished; nothing to do)
+ *   - 'timeout'                                → 'report'  (watchdog kill)
+ *   - 'restart'|'reaped_stale'|'sigterm'|'unknown' → 'resume' (clean interrupt)
+ *   - 'stop'                                   → null      (finished; nothing to do)
+ *
+ * #3555: `'reaped_stale'` (mid-session stale-row sweep) is treated exactly
+ * like `'restart'` here — it is the same clean interrupt, just no longer
+ * mislabelled as a restart in the data.
  */
 export function selectResumeBuilder(
   endedVia: TurnEndedVia | null,
@@ -473,7 +477,12 @@ export function selectResumeBuilder(
 ): 'resume' | 'report' | null {
   let kind: 'resume' | 'report' | null
   if (endedVia === 'timeout') kind = 'report'
-  else if (endedVia === 'restart' || endedVia === 'sigterm' || endedVia === 'unknown') kind = 'resume'
+  else if (
+    endedVia === 'restart' ||
+    endedVia === 'reaped_stale' ||
+    endedVia === 'sigterm' ||
+    endedVia === 'unknown'
+  ) kind = 'resume'
   else if (endedVia == null) kind = 'resume' // still-open at boot = killed mid-flight
   else kind = null
   if (

package/telegram-plugin/gateway/scoped-grant-store.ts

@@ -8,7 +8,11 @@
  * immediately. It reads as "my approval didn't stick."
  *
  * Fix: mirror the store to a tiny JSON file in STATE_DIR (same shape as
- * permission-card-store.ts — synchronous writes, mode 0o600, one small file).
+ * permission-card-store.ts — synchronous ATOMIC writes (tmp + fsync + rename),
+ * mode 0o600, one small file; a corrupt file is quarantined and logged loudly
+ * rather than silently read as "no grants" — see store-file.ts). NOTE: atomic
+ * REPLACEMENT only — whole-old-or-whole-new, not power-loss durability; the
+ * missing parent-directory fsync is tracked in #3603.
  * Write-through on every grant and on sweep-expiry removal; reload at boot,
  * dropping entries already past their ABSOLUTE expiry.
  *
@@ -23,8 +27,13 @@
  * and never write (any pre-existing file is ignored).
  */
 
-import { readFileSync, writeFileSync } from 'node:fs'
 import { join } from 'node:path'
+import { atomicWriteFileSync } from '../../src/util/atomic.js'
+import {
+  preserveUnreadableStoreFile,
+  quarantineCorruptStoreFile,
+  readStoreJsonSync,
+} from './store-file.js'
 import {
   serializeScopedGrants,
   deserializeScopedGrants,
@@ -50,18 +59,31 @@ export function scopedGrantPersistEnabled(
 export function createScopedGrantStore(
   stateDir: string,
   env: Record<string, string | undefined> = process.env,
+  /** Log sink — defaults to stderr (the gateway's runtime log). */
+  log: (line: string) => void = l => process.stderr.write(l),
 ): ScopedGrantPersistence {
   const filePath = join(stateDir, 'scoped-grants.json')
   const enabled = scopedGrantPersistEnabled(env)
 
+  /** Set when the last read failed for a non-ENOENT reason — the next write
+   * must preserve the file it could not read instead of clobbering it. */
+  let unreadable = false
+
   function read(): unknown[] {
-    try {
-      const raw = readFileSync(filePath, 'utf-8')
-      const parsed = JSON.parse(raw)
-      return Array.isArray(parsed) ? parsed : []
-    } catch {
+    const result = readStoreJsonSync(filePath, 'scoped-grant-store', log)
+    unreadable = result.status === 'unreadable'
+    if (result.status !== 'ok') return []
+    const parsed = result.value
+    if (!Array.isArray(parsed)) {
+      quarantineCorruptStoreFile(
+        filePath,
+        'scoped-grant-store',
+        'parsed to a non-array — not a scoped-grants file',
+        log,
+      )
       return []
     }
+    return parsed
   }
 
   return {
@@ -75,14 +97,17 @@ export function createScopedGrantStore(
     save(store) {
       if (!enabled) return
       try {
-        writeFileSync(filePath, JSON.stringify(serializeScopedGrants(store)), {
-          encoding: 'utf-8',
-          mode: 0o600,
-        })
+        // Fail closed: never let an overwrite be what destroys grants we
+        // merely failed to READ (flaky mount, transient EACCES).
+        if (unreadable) {
+          preserveUnreadableStoreFile(filePath, 'scoped-grant-store', log)
+          unreadable = false
+        }
+        // tmp + fsync + rename — a crash mid-persist leaves the previous
+        // grant set intact rather than a torn file that reads as "no grants".
+        atomicWriteFileSync(filePath, JSON.stringify(serializeScopedGrants(store)), 0o600)
       } catch (err) {
-        process.stderr.write(
-          `telegram gateway: scoped-grant-store write failed: ${(err as Error).message}\n`,
-        )
+        log(`telegram gateway: scoped-grant-store write failed: ${(err as Error).message}\n`)
       }
     },
   }

package/telegram-plugin/gateway/store-file.ts

@@ -0,0 +1,244 @@
+/**
+ * Shared read side for the gateway's small JSON state stores
+ * (`pending-card-store.ts`, `scoped-grant-store.ts`,
+ * `missed-approvals-store.ts`, `always-allow-persist-queue.ts`).
+ *
+ * Problem: each of those stores read its backing file with a bare
+ * `try { JSON.parse(readFileSync(...)) } catch { return [] }`. Combined
+ * with a non-atomic `writeFileSync` straight over the destination (fixed
+ * separately — every writer now goes through `atomicWriteFileSync`), a
+ * crash between truncate and write left a TORN file. On the next boot the
+ * parse threw, the catch swallowed it, and the store came up EMPTY: every
+ * pending approval card and every live scoped grant silently forgotten,
+ * with nothing in the logs. The operator just saw dead buttons.
+ *
+ * Losing security state must be OBSERVABLE. So a parse failure here is
+ * never silent:
+ *   - the corrupt bytes are preserved by renaming them aside to
+ *     `<file>.corrupt-<epoch-ms>-<seq>-<rand>` (never deleted while they are the
+ *     newest few — they're the forensic record of what was lost, and the
+ *     only chance of manual recovery);
+ *   - a loud `telegram gateway: <store> CORRUPT …` line goes to stderr,
+ *     the same channel the surrounding stores already log write failures
+ *     on (the gateway's stderr is captured in the agent's runtime log);
+ *   - the caller gets a non-`ok` result and starts from an empty store —
+ *     the process still boots, it just no longer does so in silence.
+ *
+ * Three read outcomes, deliberately distinguished:
+ *   - `missing`     — the normal cold start. Silent, no quarantine.
+ *   - `corrupt`     — unparseable JSON (or a shape the store rejects, via
+ *                     `quarantineCorruptStoreFile`). Quarantine + loud log.
+ *   - `unreadable`  — any other fs error (EACCES, EIO, EISDIR, …). Loud log,
+ *                     but NO quarantine: renaming a file we merely failed to
+ *                     read would destroy good state over a transient fault.
+ *                     Instead the caller FAILS CLOSED on its next write —
+ *                     see `preserveUnreadableStoreFile` below.
+ *
+ * ── Known hazard: two writers ──────────────────────────────────────────
+ * Quarantine is a destructive-looking move (it renames the destination
+ * away), and it is NOT safe against a second process writing the same
+ * file: process A can parse-fail on a torn file, process B can then
+ * atomically rename GOOD state into place, and A's quarantine would move
+ * B's good file aside. Pre-fix a parse failure was inert, so this hazard is
+ * introduced here. It is accepted because the gateway is (near-)singleton
+ * per agent — see the `withLock` note in `always-allow-persist-queue.ts`
+ * for why "near", and note the window is the few microseconds between the
+ * failed parse and the rename. If a genuine multi-writer arrangement ever
+ * appears, quarantine must become an flock-guarded compare-and-rename.
+ */
+
+import { randomBytes } from 'node:crypto'
+import { readFileSync, readdirSync, renameSync, rmSync, statSync } from 'node:fs'
+import { basename, dirname, join } from 'node:path'
+
+/** Default sink — matches the stores' existing `process.stderr.write` use. */
+const defaultLog = (line: string): void => {
+  process.stderr.write(line)
+}
+
+/**
+ * How many `<file>.corrupt-*` forensic copies to keep per store file.
+ * They're tiny, but a store that corrupts on a loop must not fill the
+ * state dir — so the oldest are reaped past this cap. Newest wins.
+ */
+export const MAX_QUARANTINED_COPIES = 5
+
+/** Outcome of a store-file read. See the module docblock. */
+export type StoreReadResult =
+  | { status: 'ok'; value: unknown }
+  | { status: 'missing' }
+  | { status: 'corrupt' }
+  | { status: 'unreadable' }
+
+/**
+ * Monotonic within this process. Purely a name-uniqueness aid now — the
+ * reaper orders by mtime, NOT by name (see `reapOldQuarantines`), because a
+ * name-based order is only monotonic within one process: a restart resets
+ * this counter to 0, and a fresh copy written in the same epoch-ms as a
+ * prior process's burst would sort as the OLDEST and be reaped first.
+ */
+let quarantineSeq = 0
+
+/**
+ * `<file>.corrupt-<epoch-ms>-<seq>-<rand>`.
+ *
+ * All three tail components exist to make the name UNIQUE: two quarantines
+ * in the same millisecond, or from two processes, would otherwise collide
+ * and the second `renameSync` would silently clobber the first forensic copy
+ * — same reasoning as the tempfile naming in `src/util/atomic.ts`. The name
+ * is deliberately NOT the ordering key; mtime is.
+ */
+function quarantinePath(filePath: string): string {
+  const seq = String(quarantineSeq++).padStart(6, '0')
+  return `${filePath}.corrupt-${Date.now()}-${seq}-${randomBytes(4).toString('hex')}`
+}
+
+/**
+ * Reap all but the newest {@link MAX_QUARANTINED_COPIES} forensic copies.
+ *
+ * Ordered by the filesystem's own mtime (nanosecond resolution via the
+ * bigint stat) — unlike the embedded `<epoch-ms>-<seq>`, whose sequence
+ * restarts at 0 on every boot and would make a fresh copy sort as the oldest
+ * whenever a restart lands in the same millisecond as the previous process's
+ * burst. The name is used only as a stable tie-break.
+ *
+ * Where the guarantee ends: this is exact only on filesystems that record
+ * mtime at (sub-)nanosecond granularity — ext4, xfs, apfs, btrfs all do. On a
+ * coarse-granularity filesystem (older ext3, some network/FUSE mounts, where
+ * mtime rounds to a whole second) a burst of more than MAX_QUARANTINED_COPIES
+ * quarantines inside one second all tie on `mtimeNs`, the sort falls through
+ * to the name, and the restart-reset ordering above comes back: the prior
+ * process burns seq 000001-000005 in second T, a restart quarantines at
+ * 000000, all six tie, and the freshest copy is reaped first. That costs a
+ * forensic copy on a rare filesystem and never live state, so it is not
+ * defended against here — but do not read this reaper as unconditionally
+ * restart-safe.
+ */
+function reapOldQuarantines(filePath: string, log: (line: string) => void): void {
+  const dir = dirname(filePath)
+  const prefix = `${basename(filePath)}.corrupt-`
+  try {
+    const copies = readdirSync(dir)
+      .filter(f => f.startsWith(prefix))
+      .map(name => {
+        let mtimeNs = 0n
+        try {
+          mtimeNs = statSync(join(dir, name), { bigint: true }).mtimeNs
+        } catch {
+          // Vanished under us (a concurrent reap) — sorts oldest; the rmSync
+          // below is `force`, so a missing file is a no-op either way.
+        }
+        return { name, mtimeNs }
+      })
+      .sort((a, b) => (a.mtimeNs === b.mtimeNs ? a.name.localeCompare(b.name) : a.mtimeNs < b.mtimeNs ? -1 : 1))
+    for (const stale of copies.slice(0, Math.max(0, copies.length - MAX_QUARANTINED_COPIES))) {
+      rmSync(join(dir, stale.name), { force: true })
+    }
+  } catch (err) {
+    // Reaping is opportunistic — never let it mask the corruption itself.
+    log(`telegram gateway: quarantine reap failed dir=${dir}: ${(err as Error).message}\n`)
+  }
+}
+
+/**
+ * Move a corrupt/unusable store file aside so its bytes survive, and log
+ * loudly. Exported for stores whose corruption is a SHAPE failure (valid
+ * JSON, wrong structure) that only the store itself can detect.
+ */
+export function quarantineCorruptStoreFile(
+  filePath: string,
+  store: string,
+  reason: string,
+  log: (line: string) => void = defaultLog,
+): void {
+  const target = quarantinePath(filePath)
+  let preserved = target
+  try {
+    renameSync(filePath, target)
+  } catch (err) {
+    preserved = `NOT preserved (${(err as Error).message})`
+  }
+  log(
+    `telegram gateway: ${store} CORRUPT — ${reason}. ` +
+      `Persisted state was LOST and the store is starting EMPTY; ` +
+      `corrupt file ${preserved}\n`,
+  )
+  reapOldQuarantines(filePath, log)
+}
+
+/**
+ * FAIL-CLOSED guard for the `unreadable` read outcome.
+ *
+ * A read that failed for a non-ENOENT reason (a flaky mount returning EIO,
+ * a transient EACCES) leaves the store in memory EMPTY while the on-disk
+ * file may still hold perfectly good state. The next `save()` would then
+ * rename a fresh, near-empty file over it and destroy that state
+ * permanently — quietly, and for a fault that may have lasted one syscall.
+ *
+ * So the stores call this immediately BEFORE their first write following an
+ * unreadable read: the existing bytes are renamed aside (same
+ * `.corrupt-<ts>-<seq>-<rand>` forensic convention) so the overwrite can never be
+ * the thing that loses them. If the rename itself fails the write is
+ * ABORTED by throwing — better a loud failed write than a silent
+ * destruction of the only copy.
+ */
+export function preserveUnreadableStoreFile(
+  filePath: string,
+  store: string,
+  log: (line: string) => void = defaultLog,
+): void {
+  const target = quarantinePath(filePath)
+  try {
+    renameSync(filePath, target)
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === 'ENOENT') {
+      // Nothing there to lose — the read failure wasn't about existing bytes.
+      return
+    }
+    log(
+      `telegram gateway: ${store} REFUSING to overwrite an unreadable file at ` +
+        `${filePath} — could not preserve it first: ${(err as Error).message}\n`,
+    )
+    throw err
+  }
+  log(
+    `telegram gateway: ${store} could not READ its file but is about to write it. ` +
+      `Preserved the previous (unreadable) bytes as ${target} rather than ` +
+      `overwriting them; the store continues from EMPTY\n`,
+  )
+  reapOldQuarantines(filePath, log)
+}
+
+/**
+ * Read + parse a store's backing JSON file. See {@link StoreReadResult} and
+ * the module docblock for what each outcome means and what it triggers.
+ */
+export function readStoreJsonSync(
+  filePath: string,
+  store: string,
+  log: (line: string) => void = defaultLog,
+): StoreReadResult {
+  let raw: string
+  try {
+    raw = readFileSync(filePath, 'utf-8')
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === 'ENOENT') return { status: 'missing' }
+    log(
+      `telegram gateway: ${store} read FAILED path=${filePath}: ` +
+        `${(err as Error).message} — starting EMPTY for this read; the next write ` +
+        `will preserve the unreadable file rather than overwrite it\n`,
+    )
+    return { status: 'unreadable' }
+  }
+  try {
+    return { status: 'ok', value: JSON.parse(raw) }
+  } catch (err) {
+    quarantineCorruptStoreFile(
+      filePath,
+      store,
+      `unparseable JSON (${(err as Error).message}) — likely a torn write from a crash mid-persist`,
+      log,
+    )
+    return { status: 'corrupt' }
+  }
+}

package/telegram-plugin/gateway/stream-render.ts

@@ -372,10 +372,6 @@ export function handleSessionEvent(deps: StreamRenderDeps, ev: SessionEvent): vo
         // `activityMessageId` + `activityEverOpened` so `renderActivityFeed`
         // EDITS the existing card instead of opening a second one, and so the
         // turn's own end-of-turn `clearActivitySummary` finalizes it (lever 3).
-        // The handback turn also gets the turn-long typing loop it never had —
-        // whether or not a card was painted (the debounce may not have fired) —
-        // stopped by the canonical turn-end (`purgeReactionTracking →
-        // stopTurnTypingLoop`).
         if (HANDBACK_PRETURN_ENABLED) {
           const handbackAdoption = handbackPreturnSignal.tryAdopt(turnId)
           if (handbackAdoption != null) {
@@ -383,9 +379,32 @@ export function handleSessionEvent(deps: StreamRenderDeps, ev: SessionEvent): vo
               next.activityMessageId = handbackAdoption.activityMessageId
               next.activityEverOpened = true
             }
-            startTurnTypingLoop(ev.chatId, enqThreadIdNum ?? null)
+            // Observability (#3544): adoption is the rare, previously-silent
+            // branch — one line per adopted handback turn, not per turn.
+            process.stderr.write(
+              `telegram gateway: handback pre-turn adopted turnId=${turnId} ` +
+                `key=${handbackAdoption.statusKey} card=${handbackAdoption.activityMessageId ?? 'none'}\n`,
+            )
           }
         }
+        // #3544 — arm the turn-long `typing…` loop for EVERY minted turn,
+        // unconditionally. It used to hang off the handback ADOPTION above,
+        // which misses whenever the pre-turn entry was deduped (parallel
+        // workers on one topic), had no derivable turn id, or was already
+        // reaped — and the whole compose window went dark. Only the real-inbound
+        // path (`turn-start-surfaces.ts`) armed a loop, so a synthetic turn
+        // (handback / cron / wake) could have none at all. Unconditional is safe
+        // and costs nothing extra on the wire:
+        //   - `turnTypingLoop.start` is restart-safe (stops any prior loop on
+        //     the key first, so a real inbound's loop is replaced, not doubled);
+        //   - every send goes through the SHARED per-chat-key emitter floor
+        //     (`typing-emitter.ts`, TYPING_FLOOR_MS) so N arms on one chat still
+        //     cost at most one chat action per floor window — the 2026-07-11
+        //     flood-ban guard is what makes arming more loops free;
+        //   - `turn-end.ts` (`purgeReactionTracking → stopTurnTypingLoop`) is
+        //     already the single stop-owner for ALL turns, and a start is
+        //     self-healing anyway, so this cannot leak an interval.
+        startTurnTypingLoop(ev.chatId, enqThreadIdNum ?? null)
         // PR-4e — route the turn-SET through the keyed accessor: flag-OFF assigns
         // the singleton (byte-identical to `currentTurn = next`); flag-ON sets the
         // per-topic `byKey[statusKey]` entry AND the most-recent mirror. The key is