switchroom 0.5.0

package/telegram-plugin/gateway/startup-mutex.ts

@@ -0,0 +1,285 @@
+/**
+ * Startup mutex for the telegram gateway — atomic single-writer guarantee
+ * on the per-agent PID file.
+ *
+ * Why this exists (2026-04-23 incident):
+ * --------------------------------------
+ * Two pollers can race on Telegram's getUpdates long-poll. When the OLD
+ * gateway hasn't fully released its long-poll TCP connection before the
+ * NEW gateway boots, the new one gets `409: Conflict: terminated by other
+ * getUpdates request`. The gateway then enters an exponential-backoff
+ * retry loop. Earlier today we saw the OLD clerk-gateway looping on retry
+ * attempt 13 with 10–12s backoffs — i.e. two gateway processes alive at
+ * the same time, both polling.
+ *
+ * The PRs #45–#50 stack added a PID-file probe and a SIGTERM marker but
+ * did NOT add a real startup mutex. This module closes that gap.
+ *
+ * Algorithm (POSIX-portable, no fcntl):
+ *   1. Write our record to a uniquely-named tmp file (pid + nanoseconds).
+ *   2. fs.link(tmp, canonical) — atomic on every POSIX filesystem.
+ *      - If link succeeds: we hold the lock. Unlink the tmp.
+ *      - If link fails with EEXIST: another holder. Read the existing
+ *        file, check whether the recorded PID is still alive
+ *        (process.kill(pid, 0)). If alive → blocked. If dead → unlink
+ *        the stale canonical and retry the link once.
+ *
+ * Releases happen on shutdown (SIGTERM/SIGINT/uncaught error) by
+ * unlinking the canonical path. We log every state transition; do NOT
+ * silently swallow filesystem errors.
+ */
+import {
+  link as linkAsync,
+  unlink as unlinkAsync,
+  writeFile as writeFileAsync,
+  readFile as readFileAsync,
+} from "node:fs/promises";
+
+export interface MutexRecord {
+  pid: number;
+  startedAtMs: number;
+}
+
+export type AcquireOutcome =
+  | {
+      status: "acquired";
+      record: MutexRecord;
+      /** True if a stale prior record was cleaned up before acquiring. */
+      recoveredFrom?: MutexRecord;
+    }
+  | {
+      status: "blocked";
+      holder: MutexRecord;
+      holderAgeSec: number;
+    };
+
+export interface AcquireOptions {
+  /** Canonical path of the lock file (e.g. .../telegram/gateway.pid.json). */
+  path: string;
+  /** Record we'll try to write. */
+  record: MutexRecord;
+  /**
+   * PID liveness probe. Defaults to process.kill(pid, 0) semantics.
+   * Injectable so tests can simulate dead/alive PIDs without forking.
+   */
+  isPidAlive?: (pid: number) => boolean;
+  /**
+   * Logger. Defaults to process.stderr.write. Lines are pre-formatted
+   * with the `telegram gateway:` prefix to match journalctl style.
+   */
+  log?: (line: string) => void;
+  /**
+   * Agent name to include in log lines for journalctl filtering.
+   */
+  agentName?: string;
+}
+
+const DEFAULT_LOG = (line: string): void => {
+  process.stderr.write(line.endsWith("\n") ? line : line + "\n");
+};
+
+const DEFAULT_IS_ALIVE = (pid: number): boolean => {
+  if (!Number.isFinite(pid) || pid <= 0) return false;
+  if (pid === process.pid) return true;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    // EPERM means the process exists but we can't signal it (different
+    // user). Treat as alive — we'd rather block than collide with an
+    // unkillable holder.
+    if (code === "EPERM") return true;
+    return false;
+  }
+};
+
+function fmtAgent(agentName: string | undefined): string {
+  return agentName ? ` agent=${agentName}` : "";
+}
+
+function tmpPath(canonical: string, pid: number): string {
+  // hrtime.bigint() guarantees uniqueness even under same-millisecond
+  // double-boot races (which is exactly what we're defending against).
+  return `${canonical}.tmp-${pid}-${process.hrtime.bigint().toString(36)}`;
+}
+
+async function tryReadRecord(path: string): Promise<MutexRecord | null> {
+  try {
+    const raw = await readFileAsync(path, "utf-8");
+    const parsed = JSON.parse(raw) as Partial<MutexRecord>;
+    if (
+      typeof parsed.pid === "number" &&
+      typeof parsed.startedAtMs === "number" &&
+      Number.isFinite(parsed.pid) &&
+      Number.isFinite(parsed.startedAtMs)
+    ) {
+      return { pid: parsed.pid, startedAtMs: parsed.startedAtMs };
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Atomically attempt to acquire the lock. Resolves with either
+ * `acquired` (we own the file now) or `blocked` (a live holder exists).
+ *
+ * Throws only on unrecoverable filesystem errors (ENOSPC, EROFS, EACCES
+ * on a directory that should be writable). The caller MUST treat thrown
+ * errors as fatal — the gateway should exit non-zero so systemd can
+ * apply its restart-burst backoff.
+ */
+export async function acquireStartupLock(
+  opts: AcquireOptions,
+): Promise<AcquireOutcome> {
+  const log = opts.log ?? DEFAULT_LOG;
+  const isAlive = opts.isPidAlive ?? DEFAULT_IS_ALIVE;
+  const { path, record, agentName } = opts;
+  const agentTag = fmtAgent(agentName);
+
+  const tmp = tmpPath(path, record.pid);
+  const payload = JSON.stringify(record);
+
+  // Write the tmp file first. If this throws, the canonical isn't
+  // touched — caller can retry on a fresh boot.
+  await writeFileAsync(tmp, payload, { encoding: "utf-8", mode: 0o600 });
+
+  let recoveredFrom: MutexRecord | undefined;
+
+  // Try the atomic link. ONE retry after stale-recovery; we don't loop
+  // forever — if the second link also fails EEXIST we treat as blocked
+  // (something else won the race in the gap, and that something else is
+  // now the legitimate holder).
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      await linkAsync(tmp, path);
+      // Won the race. Drop the tmp; canonical is the live lock now.
+      await unlinkAsync(tmp).catch(() => {
+        /* tmp cleanup is best-effort */
+      });
+      log(
+        `telegram gateway: boot.lock_acquired pid=${record.pid} started_at=${new Date(
+          record.startedAtMs,
+        ).toISOString()}${agentTag}`,
+      );
+      return recoveredFrom
+        ? { status: "acquired", record, recoveredFrom }
+        : { status: "acquired", record };
+    } catch (err) {
+      const code = (err as NodeJS.ErrnoException).code;
+      if (code !== "EEXIST") {
+        // Unrecoverable. Clean up tmp before propagating.
+        await unlinkAsync(tmp).catch(() => {});
+        throw err;
+      }
+
+      // EEXIST → inspect holder.
+      const holder = await tryReadRecord(path);
+      if (holder == null) {
+        // File exists but unreadable / corrupt. Treat as stale and
+        // unlink it. (Better than blocking forever on a garbage file.)
+        log(
+          `telegram gateway: boot.lock_corrupt_recovered path=${path}${agentTag}`,
+        );
+        await unlinkAsync(path).catch(() => {});
+        continue;
+      }
+
+      if (isAlive(holder.pid)) {
+        // Live holder. Drop tmp and report blocked.
+        await unlinkAsync(tmp).catch(() => {});
+        const ageSec = Math.max(
+          0,
+          Math.round((Date.now() - holder.startedAtMs) / 1000),
+        );
+        log(
+          `telegram gateway: boot.lock_blocked holder_pid=${holder.pid} holder_started_at=${new Date(
+            holder.startedAtMs,
+          ).toISOString()} holder_age_sec=${ageSec}${agentTag}`,
+        );
+        return { status: "blocked", holder, holderAgeSec: ageSec };
+      }
+
+      // Stale holder (PID dead). Unlink and loop once to retry the
+      // link. Log before unlink so the recovery is auditable.
+      log(
+        `telegram gateway: boot.lock_stale_recovered prior_pid=${holder.pid} prior_started_at=${new Date(
+          holder.startedAtMs,
+        ).toISOString()}${agentTag}`,
+      );
+      try {
+        await unlinkAsync(path);
+      } catch (unlinkErr) {
+        const unlinkCode = (unlinkErr as NodeJS.ErrnoException).code;
+        if (unlinkCode !== "ENOENT") {
+          // Couldn't clean up — propagate. Tmp gets cleaned by finalizer.
+          await unlinkAsync(tmp).catch(() => {});
+          throw unlinkErr;
+        }
+        // ENOENT means someone else cleaned it; that's fine, retry link.
+      }
+
+      recoveredFrom = holder;
+    }
+  }
+
+  // Reached only if both link attempts failed EEXIST. The second EEXIST
+  // implies a live process won the race after we cleared the stale
+  // file. Re-read and report as blocked.
+  await unlinkAsync(tmp).catch(() => {});
+  const finalHolder = await tryReadRecord(path);
+  if (finalHolder != null && isAlive(finalHolder.pid)) {
+    const ageSec = Math.max(
+      0,
+      Math.round((Date.now() - finalHolder.startedAtMs) / 1000),
+    );
+    log(
+      `telegram gateway: boot.lock_blocked holder_pid=${finalHolder.pid} holder_started_at=${new Date(
+        finalHolder.startedAtMs,
+      ).toISOString()} holder_age_sec=${ageSec}${agentTag}`,
+    );
+    return { status: "blocked", holder: finalHolder, holderAgeSec: ageSec };
+  }
+  // Edge: file vanished between attempts. Treat as blocked-with-unknown
+  // rather than spinning further; systemd backoff is the right answer.
+  const fallback: MutexRecord = finalHolder ?? { pid: 0, startedAtMs: 0 };
+  log(
+    `telegram gateway: boot.lock_blocked_unknown_holder${agentTag}`,
+  );
+  return { status: "blocked", holder: fallback, holderAgeSec: 0 };
+}
+
+/**
+ * Release the lock by unlinking the canonical file. Logs and swallows
+ * ENOENT (already gone — fine). Other errors are logged but NOT thrown:
+ * shutdown paths run in finally blocks where throwing would skip
+ * subsequent cleanup steps.
+ */
+export async function releaseStartupLock(opts: {
+  path: string;
+  pid: number;
+  log?: (line: string) => void;
+  agentName?: string;
+}): Promise<void> {
+  const log = opts.log ?? DEFAULT_LOG;
+  const agentTag = fmtAgent(opts.agentName);
+  try {
+    await unlinkAsync(opts.path);
+    log(`telegram gateway: shutdown.lock_released pid=${opts.pid}${agentTag}`);
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    if (code === "ENOENT") {
+      // Already gone — log at info, not error. Common after a crash
+      // where someone else cleaned up.
+      log(
+        `telegram gateway: shutdown.lock_release_noop pid=${opts.pid}${agentTag}`,
+      );
+      return;
+    }
+    log(
+      `telegram gateway: shutdown.lock_release_failed pid=${opts.pid} code=${code ?? "unknown"} err=${(err as Error).message}${agentTag}`,
+    );
+  }
+}

package/telegram-plugin/gateway/startup-network-retry.ts

@@ -0,0 +1,142 @@
+/**
+ * Bounded exponential-backoff retry for gateway startup network errors.
+ *
+ * On 2026-04-29 all five switchroom gateways silently broke at boot because
+ * `api.telegram.org` was unreachable for ~27 minutes after system boot (the
+ * network stack wasn't fully usable when `network-online.target` fired).
+ * Grammy threw `HttpError: Network request for 'deleteWebhook'/'getMe' failed!`
+ * and the gateway's catch block logged the error and **returned** — leaving the
+ * process alive but not polling. No crash, so systemd's `Restart=always` never
+ * fired. Telegram → agent delivery was dead until manual restarts.
+ *
+ * This module provides:
+ *
+ *   `isBootNetworkError(err)`  — recognises network-layer errors thrown by
+ *       grammy's HttpError wrapper and by raw fetch/Node network failures.
+ *
+ *   `STARTUP_RETRY_DELAYS_MS`  — the chosen backoff schedule.
+ *
+ *   `gatewayStartupRetry(fn, opts)` — drives the retry loop. Calls `fn()` up to
+ *       `maxAttempts` times with delays from `delaysMs`. On success it resolves.
+ *       On exhaustion it calls `opts.onExhausted()` (default: `process.exit(1)`)
+ *       so systemd's `Restart=always` can restart the unit cleanly.
+ *
+ * The function is extracted from `gateway.ts`'s top-level IIFE so it can be
+ * unit-tested without spinning up the full bot runtime.
+ */
+
+export interface StartupRetryOpts {
+  /**
+   * Delay schedule in milliseconds. Each attempt waits the corresponding
+   * element before the NEXT attempt. Length determines max extra attempts
+   * (total = delays.length + 1 initial attempt).
+   *
+   * Defaults to `STARTUP_RETRY_DELAYS_MS` (~2 min budget).
+   */
+  delaysMs?: number[]
+
+  /** Inject a sleep helper so tests can use fake timers. */
+  sleep?: (ms: number) => Promise<void>
+
+  /**
+   * Called when all attempts are exhausted. Should NOT return (exit/throw).
+   * Defaults to `process.exit(1)`.
+   */
+  onExhausted?: (lastError: unknown) => never
+
+  /** Log sink for retry progress messages. Defaults to process.stderr.write. */
+  log?: (line: string) => void
+}
+
+/**
+ * Default backoff schedule: 1 s, 2 s, 4 s, 8 s, 16 s, 32 s, 64 s.
+ * Total budget including 8 attempts: ~2 min 7 s. Chosen so a typical
+ * post-boot network settle (empirically <90 s) is covered with headroom.
+ */
+export const STARTUP_RETRY_DELAYS_MS: number[] = [
+  1_000,
+  2_000,
+  4_000,
+  8_000,
+  16_000,
+  32_000,
+  64_000,
+]
+
+const DEFAULT_SLEEP = (ms: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, ms))
+
+/**
+ * Returns true if `err` is a transient network-level failure that the startup
+ * retry loop should absorb. Covers:
+ *
+ * - Grammy's `HttpError` (name === 'HttpError'), which wraps fetch/ECONN errors
+ *   during `deleteWebhook` and `getMe`.
+ * - Raw Node/fetch errors: ECONNRESET, ETIMEDOUT, ENOTFOUND, ECONNREFUSED,
+ *   fetch failed, etc.
+ */
+export function isBootNetworkError(err: unknown): boolean {
+  if (!(err instanceof Error)) return false
+  // Grammy wraps network errors in HttpError (name is set in the constructor)
+  if (err.name === 'HttpError') return true
+  const msg = err.message
+  return (
+    msg.includes('ECONNRESET') ||
+    msg.includes('ETIMEDOUT') ||
+    msg.includes('ENOTFOUND') ||
+    msg.includes('ECONNREFUSED') ||
+    msg.includes('fetch failed') ||
+    msg.includes('Network request')
+  )
+}
+
+/**
+ * Attempt `fn()` and retry on `isBootNetworkError` failures using the
+ * provided delay schedule.
+ *
+ * - On success: returns whatever `fn()` resolved to.
+ * - On non-network error: re-throws immediately (not a transient boot issue).
+ * - On exhausted retries: calls `opts.onExhausted(lastError)` which must not
+ *   return (it should exit or throw). The default is `process.exit(1)` so
+ *   systemd's `Restart=always` picks up the dead unit.
+ */
+export async function gatewayStartupRetry<T>(
+  fn: () => Promise<T>,
+  opts: StartupRetryOpts = {},
+): Promise<T> {
+  const delays = opts.delaysMs ?? STARTUP_RETRY_DELAYS_MS
+  const sleep = opts.sleep ?? DEFAULT_SLEEP
+  const onExhausted: (err: unknown) => never =
+    opts.onExhausted ??
+    ((err: unknown) => {
+      process.stderr.write(
+        `telegram gateway: startup failed after ${delays.length + 1} attempts — exiting so systemd can restart: ${err}\n`,
+      )
+      process.exit(1)
+    })
+  const log =
+    opts.log ??
+    ((line: string) => {
+      process.stderr.write(line.endsWith('\n') ? line : line + '\n')
+    })
+
+  const maxAttempts = delays.length + 1
+  let lastError: unknown
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      return await fn()
+    } catch (err) {
+      if (!isBootNetworkError(err)) throw err
+      lastError = err
+      if (attempt >= maxAttempts) break
+      const delayMs = delays[attempt - 1]
+      log(
+        `telegram gateway: startup network error (attempt ${attempt}/${maxAttempts}), retrying in ${delayMs / 1000}s: ${err}`,
+      )
+      await sleep(delayMs)
+    }
+  }
+
+  return onExhausted(lastError)
+}

package/telegram-plugin/gateway/turn-active-marker.ts

@@ -0,0 +1,176 @@
+/**
+ * Turn-active liveness marker (#412).
+ *
+ * Writes `<STATE_DIR>/turn-active.json` on turn_start, touches its mtime
+ * on every tool_use, removes it on turn_complete. The watchdog
+ * (bin/bridge-watchdog.sh) reads the mtime: if the file exists AND its
+ * mtime is older than TURN_HANG_SECS (default 300s = 5min), the agent
+ * is wedged mid-turn and the watchdog restarts.
+ *
+ * Why this exists: PR #410 raised the journal-silence detector to 4000s
+ * to kill false positives on chat-cadence agents that legitimately
+ * idle for hours between turns. That left a gap — Stop-hook deadlocks
+ * (the original failure mode #116 tracked) are no longer caught under
+ * default thresholds.
+ *
+ * The distinguisher is "in-turn-and-silent" vs "between-turns-and-silent":
+ * the former is a wedge, the latter is healthy idle. This marker exists
+ * exactly during in-turn windows, so its staleness uniquely indicates
+ * the wedge.
+ *
+ * Pure file I/O. The actual hang-detection-and-restart loop lives in the
+ * bash watchdog, where it composes with the existing
+ * Restart=on-failure / journal-silence / bridge-disconnect detectors.
+ */
+
+import {
+  closeSync,
+  existsSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  statSync,
+  unlinkSync,
+  utimesSync,
+  writeFileSync,
+} from "node:fs";
+import { join } from "node:path";
+
+export const TURN_ACTIVE_MARKER_FILE = "turn-active.json";
+
+export interface TurnActiveMarker {
+  turnKey: string;
+  chatId: string;
+  threadId?: string | null;
+  startedAt: number;
+}
+
+/**
+ * Write the marker file at turn-start. Idempotent — if the file
+ * already exists from a stale prior turn (unlikely; turn_complete
+ * removes it), the new write wins.
+ */
+export function writeTurnActiveMarker(stateDir: string, marker: TurnActiveMarker): void {
+  try {
+    mkdirSync(stateDir, { recursive: true });
+    writeFileSync(
+      join(stateDir, TURN_ACTIVE_MARKER_FILE),
+      JSON.stringify(marker, null, 2) + "\n",
+      { mode: 0o600 },
+    );
+  } catch {
+    // Best-effort: marker file is a watchdog optimisation, not a
+    // correctness requirement. Don't break the turn-start path on
+    // disk-full, ENOSPC, etc.
+  }
+}
+
+/**
+ * Touch the marker file's mtime. Called on every tool_use event so an
+ * agent doing real work continually advances the mtime. The watchdog's
+ * threshold compares against this mtime.
+ */
+export function touchTurnActiveMarker(stateDir: string): void {
+  const path = join(stateDir, TURN_ACTIVE_MARKER_FILE);
+  if (!existsSync(path)) return;
+  const now = new Date();
+  try {
+    utimesSync(path, now, now);
+  } catch {
+    // utimesSync can fail on some filesystems; fall back to a tiny
+    // open-close cycle to bump the mtime via writes from the kernel side.
+    try {
+      const fd = openSync(path, "r+");
+      closeSync(fd);
+    } catch {
+      /* swallow — best-effort */
+    }
+  }
+}
+
+/**
+ * Remove the marker file at turn_complete. Absence of the file is the
+ * watchdog's signal that no turn is in flight (legitimate idle, no
+ * reason to suspect a hang).
+ */
+export function removeTurnActiveMarker(stateDir: string): void {
+  try {
+    unlinkSync(join(stateDir, TURN_ACTIVE_MARKER_FILE));
+  } catch {
+    // ENOENT is fine (already removed); other errors don't justify
+    // breaking the turn-end path.
+  }
+}
+
+/**
+ * Sweep a stale marker file. Defence-in-depth backstop for #550 — when
+ * the primary `turn_end` removal path is silently skipped (e.g. SDK
+ * killed before the JSONL turn_duration record is written, or the
+ * progress-card driver's `forceCompleteTurn` no-ops because the card
+ * was already torn down), the marker leaks across restarts and the
+ * watchdog reads it as a hung turn.
+ *
+ * Removes the marker if EITHER:
+ *   - mtime is older than `idleSweepMs` AND the caller asserts that no
+ *     turn is currently in flight (`turnInFlight=false`), OR
+ *   - mtime is older than `hardTtlMs` unconditionally (the absolute
+ *     ceiling — anything older than this can't be a real turn).
+ *
+ * Both conditions are best-effort and idempotent. Returns true if the
+ * marker was removed, false otherwise.
+ */
+export function sweepStaleTurnActiveMarker(
+  stateDir: string,
+  opts: {
+    turnInFlight: boolean;
+    idleSweepMs: number;
+    hardTtlMs: number;
+    now?: number;
+    /**
+     * Optional diagnostic callback invoked when the marker is removed.
+     * Best-effort: keeps the function pure (no logger coupling) while
+     * letting the caller emit a structured journalctl line. Must not
+     * throw — exceptions from this callback are swallowed.
+     */
+    onRemove?: (info: {
+      ageMs: number;
+      reason: "idle-stale" | "hard-ttl";
+      payload: string | null;
+    }) => void;
+  },
+): boolean {
+  const path = join(stateDir, TURN_ACTIVE_MARKER_FILE);
+  if (!existsSync(path)) return false;
+  const now = opts.now ?? Date.now();
+  try {
+    const st = statSync(path);
+    const ageMs = now - st.mtimeMs;
+    const hardExpired = ageMs > opts.hardTtlMs;
+    const idleExpired = !opts.turnInFlight && ageMs > opts.idleSweepMs;
+    if (!hardExpired && !idleExpired) return false;
+    // Best-effort read so the diagnostic callback can include the
+    // payload (turnKey, chatId, startedAt) for forensic logging.
+    let payload: string | null = null;
+    try {
+      payload = readFileSync(path, "utf8");
+    } catch {
+      /* unreadable — still drop the marker */
+    }
+    unlinkSync(path);
+    if (opts.onRemove) {
+      try {
+        opts.onRemove({
+          ageMs,
+          reason: hardExpired ? "hard-ttl" : "idle-stale",
+          payload,
+        });
+      } catch {
+        /* swallow — diagnostics must never break the sweep */
+      }
+    }
+    return true;
+  } catch {
+    // ENOENT race or stat failure — nothing actionable.
+    return false;
+  }
+}

package/telegram-plugin/gateway/unhandled-rejection-policy.ts

@@ -0,0 +1,78 @@
+/**
+ * Discriminating policy for the gateway's `unhandledRejection` handler.
+ *
+ * Background: gateway.ts crashes the process on every unhandledRejection
+ * (it calls `shutdown()` from the handler). Some Telegram API errors
+ * surface here as benign 400s — "message is not modified", "message to
+ * edit not found" — and crashing the gateway over them creates restart
+ * loops (issue #99 + lawgpt's 11:36 crash family).
+ *
+ * Pure helper so it can be tested without spinning up the gateway.
+ */
+
+import { GrammyError } from 'grammy'
+
+export type RejectionAction = 'shutdown' | 'log_only'
+
+export interface RejectionPolicyOptions {
+  /** Allow tests to inject error type detection without depending on grammy. */
+  isGrammyError?: (err: unknown) => boolean
+}
+
+/**
+ * Decide whether an unhandledRejection should crash the gateway.
+ *
+ * Returns:
+ *   - `'log_only'` for benign Telegram 400s the bot already tolerates
+ *     elsewhere (see retry-api-call.ts). Logging surfaces the leak; not
+ *     crashing prevents restart loops.
+ *   - `'shutdown'` for everything else. Genuine bugs still crash, which
+ *     systemd will surface as a restart and we want that signal.
+ *
+ * The set of benign descriptions is intentionally narrow — only the
+ * specific 400s the wrapper already swallows. Any other 400 still
+ * triggers shutdown so we don't silently mask new bugs.
+ */
+export function classifyRejection(
+  err: unknown,
+  opts: RejectionPolicyOptions = {},
+): RejectionAction {
+  const isGrammy =
+    opts.isGrammyError != null
+      ? opts.isGrammyError(err)
+      : err instanceof GrammyError
+
+  if (!isGrammy) return 'shutdown'
+
+  const e = err as { error_code?: number; description?: string }
+  if (e.error_code !== 400) return 'shutdown'
+
+  const desc = (e.description ?? '').toLowerCase()
+  if (
+    desc.includes('message is not modified') ||
+    desc.includes('message to edit not found') ||
+    desc.includes('message to delete not found') ||
+    // HTML parse errors (e.g. formatDuration sub-second output like "<1s"
+    // interpreted as a tag). These are transient render bugs — log the
+    // failure so we can fix the root cause, but don't crash the gateway
+    // into a restart loop (issue #101).
+    desc.includes("can't parse entities") ||
+    desc.includes('unsupported start tag') ||
+    // 'chat not found' fires when an allowlisted chat id (typically a
+    // group in access.json/groups{}) is no longer reachable — bot was
+    // removed, group was deleted, or the id was stale config from a
+    // prior bot pairing. Boot-time pin sweep + various other checks
+    // call getChat against every allowlisted chat; a single unreachable
+    // entry was crashing the gateway into restart loops on ziggy
+    // (2026-05-02 12:05) even though boot-probe-failed already logged
+    // the failure structurally. The wrapped sweep handles the visible
+    // case; this policy entry covers any leaked rejection from the
+    // same family so a single bad chat id can't restart-loop the
+    // process. The visible boot-probe-failed log is the primary
+    // diagnostic; this is the can't-restart-loop guarantee.
+    desc.includes('chat not found')
+  ) {
+    return 'log_only'
+  }
+  return 'shutdown'
+}