switchroom 0.19.18 → 0.19.22

package/telegram-plugin/gateway/always-allow-persist-queue.ts

@@ -32,11 +32,18 @@
  * {@link MAX_QUEUE_SIZE} below, and the tests that pin them.
  *
  * File format mirrors `missed-approvals-store.ts`: a single bounded JSON
- * array, written synchronously, mode 0o600.
+ * array, written synchronously and ATOMICALLY (tmp + fsync + rename),
+ * mode 0o600 — a crash mid-persist leaves the previous queue intact
+ * instead of a torn file. NOTE: atomic REPLACEMENT only —
+ * whole-old-or-whole-new, not power-loss durability; the missing
+ * parent-directory fsync is tracked in #3603.
  *
  * Failure semantics (hardened post-#2973 adversarial review): a failed
- * READ degrades to an empty list — a corrupt/missing queue file is not
- * fatal, it just means "nothing queued yet". A failed WRITE is a
+ * READ degrades to an empty list so the gateway still boots — but a
+ * CORRUPT file is no longer silent: the bytes are quarantined to
+ * `<file>.corrupt-<ts>` and a loud line goes to the log (see
+ * `store-file.ts`), because "queue silently came up empty" is exactly
+ * how queued retries disappeared unnoticed. A failed WRITE is a
  * different story: silently swallowing it would mean `enqueue()` tells
  * its caller "queued for retry" when nothing was actually persisted to
  * disk, and a concurrent `recordAttempt()`/`remove()` would silently
@@ -51,8 +58,14 @@
  * success.
  */
 
-import { readFileSync, writeFileSync, unlinkSync } from 'node:fs'
+import { writeFileSync, unlinkSync } from 'node:fs'
 import { join } from 'node:path'
+import { atomicWriteFileSync } from '../../src/util/atomic.js'
+import {
+  preserveUnreadableStoreFile,
+  quarantineCorruptStoreFile,
+  readStoreJsonSync,
+} from './store-file.js'
 
 /** Hard cap on retry attempts per entry — never retry indefinitely. */
 export const MAX_ATTEMPTS = 5
@@ -161,13 +174,33 @@ export function computeBackoffMs(attempts: number, retryAfterMs?: number): numbe
   return exp
 }
 
+/**
+ * The real writer: `atomicWriteFileSync` (tmp + fsync + rename) behind a
+ * `writeFileSync`-shaped signature, so the injectable seam below keeps its
+ * existing type and the fault-injection tests are unaffected. Only `mode`
+ * from the options bag is meaningful here; the store always passes 0o600.
+ */
+export const atomicWriteSeam = ((path, data, opts) => {
+  const mode = typeof opts === 'object' && opts !== null && typeof opts.mode === 'number' ? opts.mode : 0o600
+  atomicWriteFileSync(path as string, data as string, mode)
+}) as typeof writeFileSync
+
 export function createAlwaysAllowPersistQueue(
   stateDir: string,
   /** Injectable for tests to force a write failure (disk full / permissions /
    * read-only fs) without real filesystem faults — we run as root in CI/
    * containers, so chmod-based permission tricks don't reliably fail, and
-   * bun's test runner doesn't support mocking node:fs built-ins. */
-  writeFileSyncFn: typeof writeFileSync = writeFileSync,
+   * bun's test runner doesn't support mocking node:fs built-ins.
+   *
+   * CAUTION: a test that injects a seam replaces the ATOMIC writer. Such a
+   * test proves failure PROPAGATION, never atomicity — the injected function
+   * is whatever the test supplies (typically a plain `writeFileSync`, which
+   * is exactly the non-atomic writer this store moved off). Tests that mean
+   * to exercise the real write path must either leave this defaulted or wrap
+   * the exported {@link atomicWriteSeam}. */
+  writeFileSyncFn: typeof writeFileSync = atomicWriteSeam,
+  /** Log sink — defaults to stderr (the gateway's runtime log). */
+  log: (line: string) => void = l => process.stderr.write(l),
 ): AlwaysAllowPersistQueue {
   const filePath = join(stateDir, 'always-allow-persist-queue.json')
 
@@ -183,6 +216,26 @@ export function createAlwaysAllowPersistQueue(
   // single promise chain, so at most one is ever in flight at a time,
   // regardless of how many callers invoke enqueue/recordAttempt/remove
   // "concurrently".
+  //
+  // SCOPE (verified, not assumed): this is an IN-PROCESS promise chain, not
+  // an OS file lock. It serializes callers inside ONE gateway process only.
+  // Two gateway processes sharing a STATE_DIR WOULD still lose updates to
+  // each other, and that is not impossible — only rare. `startup-mutex.ts`
+  // makes concurrent gateways UNLIKELY, not unreachable: its bootMismatch
+  // path steals the lock with NO liveness check when the holder's bootId
+  // differs from the current one (exactly the restart-overlap case on a
+  // shared STATE_DIR — see the `boot.lock_stale_recovered_boot_mismatch`
+  // revert referenced at gateway.ts), `readCurrentBootId()` returns null
+  // off-Linux which disables the gate entirely, the lock is taken once at
+  // boot and never revalidated, and `isGatewayMain` lets harnesses bypass
+  // it. So: rare, not guaranteed.
+  //
+  // Cross-process mutual exclusion is deliberately OUT OF SCOPE for this
+  // change (which is about torn writes, not lost updates), and every write
+  // here is now atomic so an overlap can lose an update but can never
+  // corrupt the file. A real `flock` (cf. src/vault/flock-concurrent.test.ts)
+  // is the durable fix — tracked as follow-up. Do not read the startup mutex
+  // as a hard singleton invariant.
   let lock: Promise<unknown> = Promise.resolve()
   function withLock<T>(fn: () => T): Promise<T> {
     const result = lock.then(fn, fn) // run fn even if the previous link rejected
@@ -192,14 +245,39 @@ export function createAlwaysAllowPersistQueue(
     return result
   }
 
+  /** Set when the last read failed for a non-ENOENT reason — the next write
+   * must preserve the file it could not read instead of clobbering it. */
+  let unreadable = false
+
   function read(): FileShape {
-    try {
-      const raw = readFileSync(filePath, 'utf-8')
-      const parsed = JSON.parse(raw) as Partial<FileShape>
-      return { entries: Array.isArray(parsed?.entries) ? parsed.entries : [] }
-    } catch {
+    const result = readStoreJsonSync(filePath, 'always-allow-persist-queue', log)
+    unreadable = result.status === 'unreadable'
+    if (result.status !== 'ok') return { entries: [] }
+    const parsed = result.value as Partial<FileShape>
+    if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+      quarantineCorruptStoreFile(
+        filePath,
+        'always-allow-persist-queue',
+        'parsed to a non-object — not a persist-queue file',
+        log,
+      )
       return { entries: [] }
     }
+    // A PRESENT-but-non-array `entries` is corruption, not "empty queue".
+    // Coercing it to [] would resurrect the silent-loss bug: a half-written
+    // `{"entries": {}}` parses fine, so quarantine would never fire and the
+    // queued retries would vanish unnoticed. An ABSENT `entries` is the
+    // legitimate cold-start/partial-shape case and stays silent.
+    if (parsed.entries !== undefined && !Array.isArray(parsed.entries)) {
+      quarantineCorruptStoreFile(
+        filePath,
+        'always-allow-persist-queue',
+        '`entries` is present but not an array — truncated or malformed write',
+        log,
+      )
+      return { entries: [] }
+    }
+    return { entries: parsed.entries ?? [] }
   }
 
   /** Unlike `read()`, a write failure is NOT swallowed — it propagates so
@@ -207,6 +285,14 @@ export function createAlwaysAllowPersistQueue(
    * not actually land on disk (disk full, permissions, etc.) instead of
    * silently proceeding as if it had. */
   function write(f: FileShape): void {
+    // Fail closed: never let an overwrite be what destroys a queue we merely
+    // failed to READ (flaky mount, transient EACCES) — this throws if the
+    // previous bytes can't be preserved, and that throw is exactly the
+    // propagate-don't-swallow contract above.
+    if (unreadable) {
+      preserveUnreadableStoreFile(filePath, 'always-allow-persist-queue', log)
+      unreadable = false
+    }
     writeFileSyncFn(filePath, JSON.stringify(f), { encoding: 'utf-8', mode: 0o600 })
   }
 

package/telegram-plugin/gateway/boot-sweep-gate.ts

@@ -0,0 +1,164 @@
+/**
+ * boot-sweep-gate.ts — control flow for the boot pin sweep (#3664): WHEN it may
+ * start (`createBootSweepGate`, the two-condition arming gate) and HOW its steps
+ * are sequenced once it does (`runBootPinSweepSteps`, per-step isolation).
+ *
+ * Why this exists
+ * ---------------
+ * The boot orphan sweep (`runBootPinCleanupAndDmSweep` → `statusPinBootCleanup`
+ * → `unpinChatMessage`) has TWO independent preconditions, and they are
+ * satisfied at two different points of the gateway's boot:
+ *
+ *   1. **Ownership** — this process won the startup mutex, so it is the sole
+ *      live owner of the shared per-agent pin state. Established during module
+ *      evaluation (`acquireStartupLock`, plus the non-atomic fallback path).
+ *      A LOSING double-boot must never unpin the live holder's pins.
+ *   2. **A Telegram handle** — `lockedBot` is assigned, so an unpin can
+ *      actually reach Telegram. Established much later, inside
+ *      `initGatewayBot()` at boot.
+ *
+ * Before this gate the sweep was kicked off fire-and-forget the moment (1) was
+ * satisfied, i.e. at module-eval time — thousands of lines before (2). Every
+ * boot-cleanup unpin therefore threw `undefined is not an object (evaluating
+ * 'lockedBot.api')` before touching the network, the row was retained with an
+ * incremented attempt counter, and after `BOOT_UNPIN_MAX_ATTEMPTS` the only
+ * record of a still-pinned message was forfeited. That was a regression from
+ * `caa4d7568` (#3310), which moved the Bot construction — and with it the
+ * `lockedBot` assignment — out of module scope into `initGatewayBot()` while
+ * leaving the sweep's kick-off where it was.
+ *
+ * The gate makes the ordering DETERMINISTIC rather than a race: the sweep runs
+ * exactly once, when BOTH signals have arrived, in whichever order they arrive.
+ * It cannot regress by code motion — moving either signal site around cannot
+ * make the sweep fire before the other signal.
+ *
+ * Deliberately dependency-free and side-effect-free so it is provable in
+ * isolation (`telegram-plugin/tests/boot-sweep-gate.test.ts`); the gateway owns
+ * the wiring.
+ */
+
+export interface BootSweepGate {
+  /** Signal (1): this gateway owns the shared per-agent pin state. */
+  arm(): void
+  /** Signal (2): the Telegram bot handle exists — API calls can be made. */
+  botReady(): void
+  /** True once the sweep has been dispatched. Test/introspection seam. */
+  hasRun(): boolean
+}
+
+/**
+ * Build a gate that dispatches `run` exactly once, after BOTH `arm()` and
+ * `botReady()` have been called. Both signals are idempotent and
+ * order-independent. `run` is dispatched fire-and-forget; a rejection is routed
+ * to `onError` so it can never surface as an unhandled rejection (the sweep is
+ * best-effort and must never take the gateway down).
+ */
+export function createBootSweepGate(args: {
+  run: () => Promise<void>
+  onError?: (err: unknown) => void
+}): BootSweepGate {
+  let armed = false
+  let ready = false
+  let started = false
+
+  const maybeRun = (): void => {
+    if (started || !armed || !ready) return
+    started = true
+    void (async () => {
+      try {
+        await args.run()
+      } catch (err) {
+        args.onError?.(err)
+      }
+    })()
+  }
+
+  return {
+    arm(): void {
+      armed = true
+      maybeRun()
+    },
+    botReady(): void {
+      ready = true
+      maybeRun()
+    },
+    hasRun(): boolean {
+      return started
+    },
+  }
+}
+
+/** Dependencies of {@link runBootPinSweepSteps}. All side effects are injected
+ *  so the sequencing contract is provable without a gateway. */
+export interface BootPinSweepSteps {
+  /** Pure fs scan for DM chat ids with a prior-session pin record. MUST run
+   *  BEFORE the reapers, which empty those same stores. */
+  scanDmChatIds: () => string[]
+  statusPinCleanup: () => Promise<unknown>
+  activityCardReaper: () => Promise<unknown>
+  queuedCardReaper: () => Promise<unknown>
+  /** Flips the flag authorising the DM unpin-all path — for this sweep AND for
+   *  later lazy first-inbound sweeps. */
+  enableDmSweep: () => void
+  sweepDm: (chatId: string) => Promise<unknown>
+  log?: (line: string) => void
+}
+
+/**
+ * Run one sweep step, absorbing (and logging) a throw. See
+ * {@link runBootPinSweepSteps} for why isolation is mandatory here.
+ *
+ * Absorbs unconditionally: a non-`Error` throw is stringified rather than
+ * logged as `undefined`, and a `log` that itself throws (a closed stderr on a
+ * dying process) is swallowed too — otherwise the one function whose whole job
+ * is "never let a step take the sweep down" could take the sweep down.
+ */
+async function step(name: string, fn: () => Promise<unknown>, log: (line: string) => void) {
+  try {
+    await fn()
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err)
+    try {
+      log(`telegram gateway: boot pin sweep step '${name}' failed: ${msg}\n`)
+    } catch {
+      /* stderr is gone — nothing useful left to do */
+    }
+  }
+}
+
+/**
+ * Sequence the boot sweep's steps, each individually absorbed.
+ *
+ * Why isolation is mandatory (#3664 salvage S2). The steps used to run as a
+ * bare sequential `await` chain. `runStatusPinBootCleanup` absorbs its own
+ * per-row throws, but the two card reapers issue real Bot API calls and can
+ * reject outright — and a rejection there did not merely skip the later
+ * reapers, it skipped `enableDmSweep()`. That flag authorises the DM
+ * unpin-all path for the WHOLE SESSION (the boot sweep AND every later lazy
+ * first-inbound sweep), so one throwing reaper silently disabled DM stale-pin
+ * cleanup entirely.
+ *
+ * That was unreachable before #3664 only because the sweep never actually ran
+ * — every step died on `undefined is not an object (evaluating
+ * 'lockedBot.api')` before doing anything. Making the sweep work for the first
+ * time is precisely what makes an earlier step's throw newly reachable, so the
+ * isolation lands with the fix rather than after it.
+ *
+ * Never rejects: every step is best-effort and the caller fire-and-forgets it.
+ */
+export async function runBootPinSweepSteps(deps: BootPinSweepSteps): Promise<void> {
+  const log = deps.log ?? ((l: string) => process.stderr.write(l))
+
+  let dmChatIds: string[] = []
+  await step('dm-chat-scan', async () => {
+    dmChatIds = deps.scanDmChatIds()
+  }, log)
+
+  await step('status-pin-cleanup', deps.statusPinCleanup, log)
+  await step('activity-card-reaper', deps.activityCardReaper, log)
+  await step('queued-card-reaper', deps.queuedCardReaper, log)
+
+  // Unconditional: reached even when every step above threw. See the docblock.
+  await step('enable-dm-sweep', async () => deps.enableDmSweep(), log)
+  for (const id of dmChatIds) await step(`dm-sweep:${id}`, () => deps.sweepDm(id), log)
+}