switchroom 0.5.0

package/telegram-plugin/gateway/restart-watchdog.ts

@@ -0,0 +1,200 @@
+/**
+ * restart-watchdog.ts — polls systemd's NRestarts counter for the agent
+ * service and emits `agent-restarted-unexpectedly` when it ticks up
+ * without a corresponding planned-restart marker. Closes #30 task 4 and
+ * the 2026-04-21 lessons-learned loop where IPC flaps falsely triggered
+ * the gateway's recovery banner.
+ *
+ * Pure module — no telegram or grammy deps. The poller takes injected
+ * `execShow` and `emit` callbacks so the decision function can be tested
+ * with no systemd present.
+ *
+ * Detection model:
+ *   t0: snapshot NRestarts at gateway boot.
+ *   t1+: every pollIntervalMs, re-read. If NRestarts increased AND no
+ *        recent restart-pending marker, emit one event. Update snapshot.
+ *
+ * The "no recent restart-pending" check is the signal that this restart
+ * was unplanned. A user-issued /restart or `agent restart <name>` from
+ * the CLI writes restart-pending.json, which the gateway would have
+ * already consumed (clearRestartMarker) by the time this watchdog ticks.
+ * The marker is NOT a perfect signal — but combined with the cooldown
+ * inside the operator-events pipeline (5min per agent+kind), spurious
+ * emissions are bounded.
+ */
+
+export interface SystemdShowResult {
+  /** Cumulative restart count since unit was loaded. */
+  nRestarts: number
+  /** Microseconds since epoch — systemd's monotonic timestamp for the
+   *  current activation. Used for telemetry only; the decision is
+   *  driven by the NRestarts delta. */
+  activeEnterTimestampMonotonic: number
+}
+
+/** Output of `systemctl --user show <unit> -p NRestarts,ActiveEnterTimestampMonotonic`.
+ *  Format is one `Key=Value` per line. Returns `null` if either field is
+ *  missing or unparsable — the caller treats that as "skip this tick". */
+export function parseSystemdShowOutput(raw: string): SystemdShowResult | null {
+  const lines = raw.split(/\r?\n/)
+  let nRestarts: number | null = null
+  let activeEnter: number | null = null
+  for (const line of lines) {
+    const eq = line.indexOf('=')
+    if (eq < 0) continue
+    const key = line.slice(0, eq)
+    const value = line.slice(eq + 1)
+    if (key === 'NRestarts') {
+      const n = Number(value)
+      if (Number.isFinite(n) && n >= 0) nRestarts = n
+    } else if (key === 'ActiveEnterTimestampMonotonic') {
+      const n = Number(value)
+      if (Number.isFinite(n) && n >= 0) activeEnter = n
+    }
+  }
+  if (nRestarts == null || activeEnter == null) return null
+  return { nRestarts, activeEnterTimestampMonotonic: activeEnter }
+}
+
+export interface WatchdogTickInput {
+  /** Most recent reading from systemctl. */
+  current: SystemdShowResult
+  /** Reading from the previous tick (or boot snapshot). null on first tick. */
+  previous: SystemdShowResult | null
+  /** True if a restart-pending marker is on disk right now. The gateway
+   *  consumes the marker on boot, so a fresh marker means the restart
+   *  was user-initiated and shouldn't fire the operator event. */
+  recentPlannedRestart: boolean
+}
+
+export interface WatchdogTickDecision {
+  /** Should the operator-event emit fire? */
+  emit: boolean
+  /** Detail string passed through to the OperatorEvent renderer. */
+  detail: string
+  /** New snapshot to retain for the next tick (always `current`). */
+  nextSnapshot: SystemdShowResult
+}
+
+/**
+ * Decide whether this tick should fire an `agent-restarted-unexpectedly`
+ * event. Pure function — no I/O, no clock reads.
+ */
+export function decideWatchdogTick(input: WatchdogTickInput): WatchdogTickDecision {
+  const { current, previous, recentPlannedRestart } = input
+  // First tick after boot — no baseline to compare. Just record.
+  if (previous == null) {
+    return { emit: false, detail: '', nextSnapshot: current }
+  }
+  const delta = current.nRestarts - previous.nRestarts
+  if (delta <= 0) {
+    return { emit: false, detail: '', nextSnapshot: current }
+  }
+  // NRestarts went up — was it user-initiated?
+  if (recentPlannedRestart) {
+    return {
+      emit: false,
+      detail: `planned restart (delta=${delta})`,
+      nextSnapshot: current,
+    }
+  }
+  return {
+    emit: true,
+    detail: delta === 1
+      ? 'systemd auto-restarted the agent service unexpectedly'
+      : `systemd auto-restarted the agent service ${delta} times since last poll`,
+    nextSnapshot: current,
+  }
+}
+
+export interface WatchdogConfig {
+  /** Agent name to monitor. The systemd unit is `switchroom-<agent>`. */
+  agentName: string
+  /** Poll cadence (ms). Default 30s. Set to 0 to disable. */
+  pollIntervalMs?: number
+  /** Injected systemctl show — returns the raw stdout string, or throws
+   *  on failure (the watchdog skips that tick on error). */
+  execShow: (unit: string) => string
+  /** Returns true if a restart-pending marker is on disk. */
+  isPlannedRestartFresh: () => boolean
+  /** Called when the watchdog decides to emit. */
+  emit: (detail: string) => void
+  /** Optional log sink. */
+  log?: (msg: string) => void
+}
+
+export interface WatchdogHandle {
+  stop(): void
+}
+
+/**
+ * Start the watchdog poller. Returns a handle whose `stop()` clears the
+ * timer. Safe to call before the IPC server is up — the first tick just
+ * establishes a baseline.
+ */
+export function startRestartWatchdog(config: WatchdogConfig): WatchdogHandle {
+  const {
+    agentName,
+    pollIntervalMs = 30_000,
+    execShow,
+    isPlannedRestartFresh,
+    emit,
+    log,
+  } = config
+
+  if (pollIntervalMs <= 0) {
+    log?.(`restart-watchdog: disabled (pollIntervalMs=${pollIntervalMs})`)
+    return { stop() {} }
+  }
+
+  const unit = `switchroom-${agentName}`
+  let snapshot: SystemdShowResult | null = null
+
+  function tick(): void {
+    let raw: string
+    try {
+      raw = execShow(unit)
+    } catch (err) {
+      log?.(`restart-watchdog: systemctl show failed unit=${unit}: ${(err as Error).message}`)
+      return
+    }
+    const parsed = parseSystemdShowOutput(raw)
+    if (!parsed) {
+      log?.(`restart-watchdog: could not parse systemctl output unit=${unit}`)
+      return
+    }
+    const decision = decideWatchdogTick({
+      current: parsed,
+      previous: snapshot,
+      recentPlannedRestart: isPlannedRestartFresh(),
+    })
+    snapshot = decision.nextSnapshot
+    if (decision.emit) {
+      log?.(`restart-watchdog: detected unexpected restart agent=${agentName} ${decision.detail}`)
+      try {
+        emit(decision.detail)
+      } catch (err) {
+        log?.(`restart-watchdog: emit threw: ${(err as Error).message}`)
+      }
+    } else if (decision.detail) {
+      log?.(`restart-watchdog: tick agent=${agentName} ${decision.detail}`)
+    }
+  }
+
+  // Establish the baseline immediately so a planned restart that lands
+  // in the first poll interval doesn't get attributed to "before this
+  // gateway started." Then schedule the recurring poll.
+  tick()
+  const timer = setInterval(tick, pollIntervalMs)
+  // unref so the watchdog doesn't keep the gateway process alive past
+  // shutdown.
+  if (typeof timer.unref === 'function') timer.unref()
+
+  log?.(`restart-watchdog: started agent=${agentName} unit=${unit} interval=${pollIntervalMs}ms`)
+
+  return {
+    stop() {
+      clearInterval(timer)
+    },
+  }
+}

package/telegram-plugin/gateway/session-marker.ts

@@ -0,0 +1,83 @@
+/**
+ * Gateway session marker — "who/when am I?" identity file recording the
+ * current gateway process's PID and startedAt.
+ *
+ * 2026-04-22 incident: the crash-recovery banner ("⚡ Recovered from
+ * unexpected restart. (down ~Ns)") fired every time grammY restarted
+ * its long-poll, not every time the gateway process actually restarted.
+ * Same PID across all banner-fires; systemd recorded zero lifecycle
+ * events.
+ *
+ * Fix: on startup, write {pid, startedAtMs} to a marker file. Before
+ * firing the banner, compare the stored marker to the current process:
+ *   - no marker         → true first boot, fire
+ *   - PID differs       → real restart, fire
+ *   - startedAt differs → same PID reused (rare), still a new process,
+ *                         fire
+ *   - exact match       → we've been running continuously, suppress
+ *                         (this is a grammY poll-restart, not a process
+ *                         restart).
+ */
+
+import { writeFileSync, readFileSync, renameSync, unlinkSync } from "node:fs";
+
+export interface SessionMarker {
+  pid: number;
+  startedAtMs: number;
+}
+
+export function writeSessionMarker(path: string, marker: SessionMarker): void {
+  const tmp = `${path}.tmp-${process.pid}-${Date.now()}`;
+  writeFileSync(tmp, JSON.stringify(marker), "utf-8");
+  renameSync(tmp, path);
+}
+
+export function readSessionMarker(path: string): SessionMarker | null {
+  try {
+    const raw = readFileSync(path, "utf-8");
+    const parsed = JSON.parse(raw) as Partial<SessionMarker>;
+    if (
+      typeof parsed.pid === "number" &&
+      typeof parsed.startedAtMs === "number" &&
+      Number.isFinite(parsed.pid) &&
+      Number.isFinite(parsed.startedAtMs)
+    ) {
+      return { pid: parsed.pid, startedAtMs: parsed.startedAtMs };
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+export function clearSessionMarker(path: string): void {
+  try {
+    unlinkSync(path);
+  } catch {
+    /* best effort */
+  }
+}
+
+/**
+ * Pure decision: should the "recovered from unexpected restart" banner
+ * fire for the current process?
+ *
+ * Fire when:
+ *   - no stored marker (true first boot), OR
+ *   - stored PID != current PID (real process restart), OR
+ *   - stored startedAt != current startedAt (same PID reused).
+ *
+ * Suppress when stored === current: we are the same process we were
+ * last time the banner logic looked, so any "restart" is happening
+ * below the process level (e.g. grammY poll-restart on 409).
+ */
+export function shouldFireRestartBanner(input: {
+  stored: SessionMarker | null;
+  current: SessionMarker;
+}): boolean {
+  const { stored, current } = input;
+  if (stored === null) return true;
+  if (stored.pid !== current.pid) return true;
+  if (stored.startedAtMs !== current.startedAtMs) return true;
+  return false;
+}

package/telegram-plugin/gateway/shutdown-drain.ts

@@ -0,0 +1,162 @@
+/**
+ * SIGTERM drain coordinator for the telegram gateway.
+ *
+ * On shutdown we want to give the long-poll a real chance to die before
+ * the next gateway process boots and races it on Telegram's getUpdates
+ * channel. The 2026-04-23 incident showed that a 3-second hard exit was
+ * NOT enough — the kernel hadn't FIN'd the long-poll TCP socket before
+ * the new process started, and Telegram returned 409 Conflict to both
+ * pollers for 13+ retries (10–12s backoffs each).
+ *
+ * Budget: ~35 seconds. systemd's TimeoutStopSec defaults to 90s on user
+ * units (we now set it explicitly to 45s in generateGatewayUnit), so
+ * there's headroom for the drain plus systemd's own grace before SIGKILL.
+ *
+ * Phases:
+ *   1. stop_polling — call the supplied stopPolling() (bot.stop /
+ *      runner.stop). This halts new long-polls and lets the in-flight
+ *      one return naturally.
+ *   2. await in-flight — poll a counter (handlers, IPC calls) until it
+ *      reaches 0 or the budget expires.
+ *   3. report — single log line with elapsed_ms, in_flight_remaining,
+ *      and timed_out boolean.
+ *
+ * The lock release is the caller's responsibility — typically:
+ *
+ *   await drainShutdown({ ... })
+ *   await releaseStartupLock({ ... })
+ *   process.exit(0)
+ *
+ * keeping this module focused on the drain itself for testability.
+ */
+
+export interface DrainOptions {
+  /** Why we're shutting down (SIGTERM / SIGINT / etc) — for the log line. */
+  signal: string;
+  /**
+   * Halts the long-poll. Implementations should await until the polling
+   * loop has actually returned. Errors are caught and logged — drain
+   * continues with the in-flight wait.
+   */
+  stopPolling: () => Promise<void>;
+  /**
+   * Returns the current count of in-flight handlers / IPC calls. Drain
+   * polls this until it reaches 0 or the budget expires.
+   */
+  inFlight: () => number;
+  /** Total budget in ms. Defaults to 35_000. */
+  budgetMs?: number;
+  /** Poll interval for the in-flight wait. Defaults to 100ms. */
+  pollIntervalMs?: number;
+  /** Logger; defaults to process.stderr.write with a trailing newline. */
+  log?: (line: string) => void;
+  /** Agent name for journalctl-friendly tagging. */
+  agentName?: string;
+  /**
+   * Sleep function — injectable for tests. Defaults to setTimeout-based.
+   * Receives the poll interval, not absolute time.
+   */
+  sleep?: (ms: number) => Promise<void>;
+  /**
+   * Monotonic clock — injectable for tests. Defaults to performance.now().
+   */
+  now?: () => number;
+}
+
+export interface DrainResult {
+  elapsedMs: number;
+  inFlightRemaining: number;
+  timedOut: boolean;
+}
+
+const DEFAULT_LOG = (line: string): void => {
+  process.stderr.write(line.endsWith("\n") ? line : line + "\n");
+};
+
+const DEFAULT_SLEEP = (ms: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, ms));
+
+const DEFAULT_NOW = (): number =>
+  typeof performance !== "undefined" && typeof performance.now === "function"
+    ? performance.now()
+    : Date.now();
+
+function fmtAgent(agentName: string | undefined): string {
+  return agentName ? ` agent=${agentName}` : "";
+}
+
+/**
+ * Run the SIGTERM drain. Always resolves (never throws) — shutdown
+ * paths must not throw. Errors from stopPolling are logged and the
+ * in-flight wait continues; if the budget elapses we return with
+ * `timedOut: true` so the caller can decide whether to force-exit.
+ */
+export async function drainShutdown(opts: DrainOptions): Promise<DrainResult> {
+  const log = opts.log ?? DEFAULT_LOG;
+  const sleep = opts.sleep ?? DEFAULT_SLEEP;
+  const now = opts.now ?? DEFAULT_NOW;
+  const budgetMs = opts.budgetMs ?? 35_000;
+  const pollIntervalMs = opts.pollIntervalMs ?? 100;
+  const agentTag = fmtAgent(opts.agentName);
+
+  const start = now();
+  const startInFlight = safeCount(opts.inFlight);
+
+  log(
+    `telegram gateway: shutdown.drain_start signal=${opts.signal} in_flight=${startInFlight}${agentTag}`,
+  );
+
+  // Phase 1: halt polling. We await stopPolling() but bound it to the
+  // overall budget — if it hangs we still want to drain in-flight and
+  // exit. Use Promise.race with a deadline.
+  const stopDeadline = budgetMs;
+  try {
+    await Promise.race([
+      opts.stopPolling(),
+      sleep(stopDeadline).then(() => {
+        // Mark the timeout via a sentinel — we don't throw; we let the
+        // in-flight loop observe the elapsed time and report timed_out.
+      }),
+    ]);
+  } catch (err) {
+    log(
+      `telegram gateway: shutdown.stop_polling_error err=${(err as Error).message}${agentTag}`,
+    );
+  }
+
+  // Phase 2: poll inFlight() until 0 or budget exhausted.
+  while (true) {
+    const elapsed = now() - start;
+    const remaining = safeCount(opts.inFlight);
+    if (remaining <= 0) {
+      const finalElapsed = Math.round(now() - start);
+      log(
+        `telegram gateway: shutdown.drain_complete elapsed_ms=${finalElapsed} in_flight_remaining=0 timed_out=false${agentTag}`,
+      );
+      return { elapsedMs: finalElapsed, inFlightRemaining: 0, timedOut: false };
+    }
+    if (elapsed >= budgetMs) {
+      const finalElapsed = Math.round(elapsed);
+      log(
+        `telegram gateway: shutdown.drain_complete elapsed_ms=${finalElapsed} in_flight_remaining=${remaining} timed_out=true${agentTag}`,
+      );
+      return {
+        elapsedMs: finalElapsed,
+        inFlightRemaining: remaining,
+        timedOut: true,
+      };
+    }
+    // Don't oversleep past the budget.
+    const wait = Math.min(pollIntervalMs, Math.max(1, budgetMs - elapsed));
+    await sleep(wait);
+  }
+}
+
+function safeCount(fn: () => number): number {
+  try {
+    const n = fn();
+    return Number.isFinite(n) && n > 0 ? Math.floor(n) : 0;
+  } catch {
+    return 0;
+  }
+}