switchroom 0.18.7 → 0.18.8

package/telegram-plugin/flood-circuit-breaker.ts

@@ -0,0 +1,123 @@
+/**
+ * Telegram per-bot flood-wait circuit breaker (issue #2923).
+ *
+ * When a burst of outbound sends trips Telegram's per-bot-token flood limit,
+ * the API returns `429 { retry_after: N }` — a SERVER-SIDE ban on the bot
+ * token that no client-side lever clears early (observed retry_after ~4116s,
+ * ~68 min). The failure is misleading: the container is `Up`, the gateway is
+ * polling, inbound works — but every outbound send is rejected. Worse, each
+ * `docker restart` posts a fresh boot/config card = another send INTO the
+ * open window, which can reset/extend the flood counter. A local, recoverable
+ * disk-full problem thereby amplifies into a remote, unrecoverable ban.
+ *
+ * This breaker persists the flood-wait window to disk so that:
+ *   - `retryApiCall`'s `onFloodWait` hook records it the moment a 429 is seen;
+ *   - a restart-time NON-ESSENTIAL send (boot card, config summary) consults
+ *     `isFloodWaitActive` and SKIPS while the ban is open, so a restart during
+ *     a flood-wait doesn't feed the counter and prolong the ban.
+ *
+ * The state file is tiny JSON under the agent's telegram state dir. All the
+ * logic is pure + injectable so it unit tests without a real clock or bot.
+ */
+
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+
+export interface FloodWaitState {
+  /** Epoch ms at which the flood-wait window expires. */
+  untilTs: number
+  /** The retry_after (seconds) Telegram reported, for diagnostics. */
+  retryAfterSec: number
+  /** Epoch ms the window was (re)recorded. */
+  recordedTs: number
+}
+
+/** Default marker filename inside the telegram state dir. */
+export const FLOOD_STATE_FILE = 'flood-wait.json'
+
+/**
+ * Resolve the flood-wait marker path from a telegram state dir. Kept as a
+ * helper so callers share one location.
+ */
+export function floodStatePath(stateDir: string): string {
+  return join(stateDir, FLOOD_STATE_FILE)
+}
+
+/**
+ * Compute the flood-wait state for an observed `retry_after`. Extends (never
+ * shrinks) an existing window: if a fresh 429 reports a shorter remaining ban
+ * than we already recorded, we keep the longer expiry — the server is the
+ * authority and being conservative avoids sending back into an open window.
+ */
+export function computeFloodWait(
+  prior: FloodWaitState | null,
+  retryAfterSec: number,
+  now: number,
+): FloodWaitState {
+  const candidate = now + Math.max(0, retryAfterSec) * 1000
+  const untilTs = prior && prior.untilTs > candidate ? prior.untilTs : candidate
+  return { untilTs, retryAfterSec, recordedTs: now }
+}
+
+/** Remaining ban time in ms (0 when no active window). */
+export function floodWaitRemainingMs(state: FloodWaitState | null, now: number): number {
+  if (!state) return 0
+  return Math.max(0, state.untilTs - now)
+}
+
+/** True while the flood-wait ban is still open. */
+export function isFloodWaitActive(state: FloodWaitState | null, now: number): boolean {
+  return floodWaitRemainingMs(state, now) > 0
+}
+
+/** Read persisted flood state; null on absence / parse failure. */
+export function readFloodState(path: string): FloodWaitState | null {
+  try {
+    if (!existsSync(path)) return null
+    const raw = JSON.parse(readFileSync(path, 'utf-8')) as Partial<FloodWaitState>
+    if (typeof raw.untilTs !== 'number') return null
+    return {
+      untilTs: raw.untilTs,
+      retryAfterSec: typeof raw.retryAfterSec === 'number' ? raw.retryAfterSec : 0,
+      recordedTs: typeof raw.recordedTs === 'number' ? raw.recordedTs : 0,
+    }
+  } catch {
+    return null
+  }
+}
+
+/** Persist flood state (best-effort — a write failure must not crash the send path). */
+export function writeFloodState(path: string, state: FloodWaitState): void {
+  try {
+    mkdirSync(dirname(path), { recursive: true })
+    writeFileSync(path, JSON.stringify(state), { mode: 0o600 })
+  } catch {
+    /* best-effort */
+  }
+}
+
+/**
+ * Build the `onFloodWait` callback for `createRetryApiCall`, wired to persist
+ * (and extend) the window at `path`. Reads current state, merges the new
+ * retry_after, writes it back.
+ */
+export function makeFloodWaitRecorder(
+  path: string,
+  now: () => number = Date.now,
+): (retryAfterSec: number) => void {
+  return (retryAfterSec: number) => {
+    const t = now()
+    const next = computeFloodWait(readFloodState(path), retryAfterSec, t)
+    writeFloodState(path, next)
+  }
+}
+
+/**
+ * Decide whether a NON-ESSENTIAL restart-time send (boot card, config
+ * summary) should be suppressed because a flood-wait is active. Returns the
+ * remaining ms when suppressed (>0), or 0 to proceed. Reads state fresh so a
+ * concurrently-updated window is honoured.
+ */
+export function suppressNonEssentialSendMs(path: string, now: number): number {
+  return floodWaitRemainingMs(readFloodState(path), now)
+}

package/telegram-plugin/gateway/activity-card-store.ts

@@ -43,6 +43,11 @@
  * failure than one frozen orphan. A benign-400 (message already
  * deleted/vanished, or "not modified") is NOT counted as a finalize — nothing
  * was delivered — see the `vanished` tally in `runActivityCardBootReaper`.
+ *
+ * The UNPIN half is different (#3001): unpinning is idempotent, so it gets
+ * AT-LEAST-ONCE (capped) semantics — a failed unpin re-persists the record
+ * with `finalizeAttempted: true` + an `unpinAttempts` counter so the next boot
+ * retries ONLY the unpin (never the edit), up to BOOT_UNPIN_MAX_ATTEMPTS.
  */
 
 export interface ActivityCardStoreFsSeam {
@@ -73,8 +78,21 @@ export interface ActivityCardRecord {
    *  forever). Optional so a v1-shape record (pre-unpin-tracking) still
    *  loads and degrades to "don't attempt an unpin", never a crash. */
   pinned?: boolean
+  /** Set when the boot reaper retains a record ONLY to retry its failed
+   *  unpin (#3001). The finalizing edit stays AT-MOST-ONCE: a retained
+   *  record's edit was already attempted, so the next boot skips the edit
+   *  and only retries the (idempotent) unpin. */
+  finalizeAttempted?: boolean
+  /** Boot-reaper unpin retry counter (#3001); the record is forfeited at
+   *  BOOT_UNPIN_MAX_ATTEMPTS. Absent = 0. */
+  unpinAttempts?: number
 }
 
+/** Cap on cross-boot unpin retries — same rationale as the status-pin
+ *  store's BOOT_UNPIN_MAX_ATTEMPTS (bound a permanently-undeliverable
+ *  unpin; unpins themselves are idempotent so retrying is safe). */
+export const BOOT_UNPIN_MAX_ATTEMPTS = 5
+
 interface SnapshotEnvelope {
   v: 1
   cards: ActivityCardRecord[]
@@ -91,7 +109,9 @@ function isCardRow(x: unknown): x is ActivityCardRecord {
     (o.threadId === null || typeof o.threadId === 'number') &&
     typeof o.activityMessageId === 'number' &&
     typeof o.startedAt === 'number' &&
-    (o.pinned === undefined || typeof o.pinned === 'boolean')
+    (o.pinned === undefined || typeof o.pinned === 'boolean') &&
+    (o.finalizeAttempted === undefined || typeof o.finalizeAttempted === 'boolean') &&
+    (o.unpinAttempts === undefined || typeof o.unpinAttempts === 'number')
   )
 }
 
@@ -245,32 +265,57 @@ export async function runActivityCardBootReaper(args: {
     // turn that upserted a fresh card under the same turnKey mid-reap keeps
     // its own (different-id) record.
     clearActivityCardRecord(args.path, args.fs, record.turnKey, record.activityMessageId, log)
-    try {
-      // Count a finalize ONLY when the edit actually landed. robustApiCall
-      // resolves to undefined on a benign-400 (the card was already deleted or
-      // the chat is gone) — that delivered nothing, so it is a `vanished`
-      // orphan, not a `finalized` one. Counting it as finalized (the pre-fix
-      // behaviour) over-reported the guarantee in the boot log.
-      const res = await args.finalizeCard(record)
-      if (res != null) finalized++
-      else vanished++
-    } catch (err) {
-      log(
-        `activity-card-store: boot reaper finalize failed ` +
-          `(chat=${record.chatId} msg=${record.activityMessageId}): ` +
-          `${(err as Error).message}\n`,
-      )
+    // The finalizing EDIT stays at-most-once: skip it for a record retained by
+    // a prior boot purely to retry its failed unpin (finalizeAttempted).
+    if (!record.finalizeAttempted) {
+      try {
+        // Count a finalize ONLY when the edit actually landed. robustApiCall
+        // resolves to undefined on a benign-400 (the card was already deleted or
+        // the chat is gone) — that delivered nothing, so it is a `vanished`
+        // orphan, not a `finalized` one. Counting it as finalized (the pre-fix
+        // behaviour) over-reported the guarantee in the boot log.
+        const res = await args.finalizeCard(record)
+        if (res != null) finalized++
+        else vanished++
+      } catch (err) {
+        log(
+          `activity-card-store: boot reaper finalize failed ` +
+            `(chat=${record.chatId} msg=${record.activityMessageId}): ` +
+            `${(err as Error).message}\n`,
+        )
+      }
     }
     if (record.pinned) {
       try {
         await args.unpinCard(record)
         unpinned++
       } catch (err) {
+        // Retry-safe unpin (#3001): unlike the edit (at-most-once by design),
+        // an unpin is idempotent — so a failed one is RE-PERSISTED with an
+        // attempt counter and retried on the next boot, up to the cap, instead
+        // of being forfeited. `finalizeAttempted` guarantees the retained
+        // record can never re-run its edit.
+        const attempts = (record.unpinAttempts ?? 0) + 1
         log(
           `activity-card-store: boot reaper unpin failed ` +
-            `(chat=${record.chatId} msg=${record.activityMessageId}): ` +
-            `${(err as Error).message}\n`,
+            `(chat=${record.chatId} msg=${record.activityMessageId} ` +
+            `attempt=${attempts}): ${(err as Error).message}\n`,
         )
+        if (attempts < BOOT_UNPIN_MAX_ATTEMPTS) {
+          writeActivityCardRecord(
+            args.path,
+            args.fs,
+            { ...record, finalizeAttempted: true, unpinAttempts: attempts },
+            log,
+          )
+        } else {
+          log(
+            `activity-card-store: boot reaper FORFEITING card unpin after ` +
+              `${attempts} failed attempts ` +
+              `(chat=${record.chatId} msg=${record.activityMessageId}) — ` +
+              `will not retry again\n`,
+          )
+        }
       }
     }
   }

package/telegram-plugin/gateway/boot-card.ts

@@ -69,6 +69,7 @@ import {
 } from './config-snapshot.js'
 import { join } from 'path'
 import { bootCardChatKey, loadBootCardMsgId, saveBootCardMsgId } from './boot-card-msgid.js'
+import { suppressNonEssentialSendMs } from '../flood-circuit-breaker.js'
 import { loadConfig as _loadSwitchroomConfig } from '../../src/config/loader.js'
 import { resolveAgentConfig as _resolveAgentConfig } from '../../src/config/merge.js'
 
@@ -605,6 +606,16 @@ export interface RunProbesOpts {
    * behaviour.
    */
   bootCardStatePath?: string
+  /**
+   * #2923 flood-wait circuit breaker. Path to the persisted flood-wait
+   * marker. When a Telegram per-bot flood ban is active, posting a boot card
+   * on restart is a NON-ESSENTIAL send straight into the open window that can
+   * reset/extend the ban — so if this path shows an active flood-wait, the
+   * boot card is SUPPRESSED (logged, not sent). Omit to always post.
+   */
+  floodStatePath?: string
+  /** Injectable clock for the flood-wait check (tests). Defaults to Date.now. */
+  nowMs?: () => number
 }
 
 /** Run all six probes concurrently with their own per-probe timeouts.
@@ -652,6 +663,22 @@ export async function startBootCard(
   const setTimeoutFn = opts.setTimeoutImpl ?? setTimeout
   const settleMs = opts.settleWindowMs ?? SETTLE_WINDOW_MS
 
+  // #2923 circuit breaker: if a per-bot Telegram flood ban is active, do NOT
+  // post the boot card. It's a non-essential restart-time send straight into
+  // the open flood window — the exact thing that resets/extends the ban and
+  // keeps the agent mute for longer. Skip loudly (log) and return a no-op.
+  if (opts.floodStatePath != null) {
+    const now = (opts.nowMs ?? Date.now)()
+    const remainingMs = suppressNonEssentialSendMs(opts.floodStatePath, now)
+    if (remainingMs > 0) {
+      logger(
+        `telegram gateway: boot-card: SUPPRESSED — Telegram flood-wait active for ~${Math.round(remainingMs / 1000)}s; ` +
+          `not posting a restart card into the open ban window (issue #2923)\n`,
+      )
+      return { messageId: -1, complete: () => {} }
+    }
+  }
+
   // Render and post the bare ack line immediately. The user gets
   // confirmation that the agent is back without waiting on probes.
   const ackText = renderBootCard({

package/telegram-plugin/gateway/busy-ack.ts

@@ -0,0 +1,106 @@
+// #2995 — mid-flight busy ack.
+//
+// A mid-turn inbound that is neither a steer nor an interrupt is buffered
+// until the running turn goes idle (`buffer-until-idle`). When the turn is
+// sitting inside ONE long blocking tool call (`gh pr checks --watch`, a
+// long `sleep`, a slow build), that wait can be minutes — from the phone
+// the question reads as ignored. The user's message got only a 👀 reaction
+// and then silence until the blocking step returned (observed live
+// 2026-07-10: a trivial "are any agents working?" waited 2 minutes behind
+// a `--watch`).
+//
+// This module is the PURE half of the fix: a deterministic, model-free
+// decision — should the gateway post a silent "⏳ Queued — currently
+// inside <tool> …" card into the inbound's own chat/topic right now? No
+// tokens, no model calls; the gateway supplies live readings (delivery-gate
+// decision, tool-flight state, current step age, dedupe state) and this
+// module answers. Extracted so the policy is unit-testable without the
+// gateway IIFE (same pattern as `interrupt-defer.ts` / `feed-reopen-gate.ts`).
+//
+// Wording contract: the buffered-path card MUST say "Queued" — the
+// steer-or-queue job spec (`reference/jobs/steer-or-queue-mid-flight.md`)
+// requires the chosen classification to be visible in the chat, never
+// inferred. A steer-path card must NOT say "Queued" (it wasn't queued);
+// it says the steer was noted and will fold in at the step boundary.
+
+/** How the inbound was classified/handled by the delivery gate. Only the
+ *  two mid-turn shapes are ack-eligible; a plain fresh-turn `deliver`
+ *  needs no busy ack (the turn starts immediately). */
+export type BusyAckGateDecision = 'buffer-until-idle' | 'steer' | 'deliver'
+
+/**
+ * Minimum age of the CURRENT tool step before a busy ack fires. Below
+ * this the step is about to return anyway and the buffered inbound will
+ * flush in a beat — an ack would fire when the agent was seconds from
+ * answering (the job spec's explicit Bad bullet). 12s sits well above the
+ * common fast-tool envelope (reads/greps/short bashes finish in <5s) and
+ * well below the "reads as ignored" horizon (~30s+); it also matches the
+ * UAT latency promise (visible ack <10s of the ping, for a step that has
+ * already proven itself long).
+ */
+export const BUSY_ACK_STEP_AGE_THRESHOLD_MS = 12_000
+
+export interface BusyAckDecisionInput {
+  /** Delivery-gate outcome for this inbound ('steer' when it was
+   *  delivered mid-turn as a steering amend). */
+  gateDecision: BusyAckGateDecision
+  /** Live `ToolFlightTracker.isMidToolCall()` reading — at least one
+   *  top-level tool call currently open. */
+  midToolCall: boolean
+  /** Age (ms) of the LONGEST-running in-flight tool step, or null when no
+   *  step is tracked (e.g. the turn is thinking between tools). */
+  stepAgeMs: number | null
+  /** True when a busy-ack/queued-status card already exists (or was
+   *  already posted this turn) for the inbound's chat/topic — at most one
+   *  card per turn per chat/topic; a second ping gets no second card. */
+  alreadyAcked: boolean
+}
+
+/**
+ * Pure decision: post the busy-ack card now?
+ *
+ *  - only for mid-turn shapes (buffered, or delivered as a steer)
+ *  - only while genuinely mid-tool-call
+ *  - only once the current step is older than the threshold (a young
+ *    step returns soon; the normal flush covers it)
+ *  - at most once per turn per chat/topic (dedupe)
+ */
+export function shouldPostBusyAck(input: BusyAckDecisionInput): boolean {
+  if (input.gateDecision !== 'buffer-until-idle' && input.gateDecision !== 'steer') return false
+  if (!input.midToolCall) return false
+  if (input.stepAgeMs == null || input.stepAgeMs < BUSY_ACK_STEP_AGE_THRESHOLD_MS) return false
+  if (input.alreadyAcked) return false
+  return true
+}
+
+export interface BusyAckTextInput {
+  gateDecision: 'buffer-until-idle' | 'steer'
+  /** Bare tool name as tracked (e.g. "Bash"). Null when unknown. */
+  toolName: string | null
+  /** Natural-language descriptor from the PreToolUse sidecar's
+   *  `toolLabel()` (e.g. `sleep 90`), or null. */
+  toolLabel: string | null
+}
+
+/**
+ * Render the card text. Deterministic, no model. The buffered variant
+ * says "Queued" (classification-visibility invariant); the steer variant
+ * says the steer is noted — never "Queued".
+ *
+ * Deliberately carries NO elapsed figure: the card is posted once and
+ * never re-rendered while the blocking step runs, so any point-in-time
+ * number ("2m elapsed") would silently go stale on screen — a decaying
+ * claim on a card whose whole point is honesty. The activity name alone
+ * is time-invariant.
+ */
+export function formatBusyAckText(input: BusyAckTextInput): string {
+  const name = input.toolName ?? 'a long-running step'
+  const activity =
+    input.toolLabel != null && input.toolLabel.length > 0
+      ? `${name}: ${input.toolLabel}`
+      : name
+  if (input.gateDecision === 'steer') {
+    return `⏳ Steer noted — currently inside \`${activity}\`; I'll fold it in when this step finishes.`
+  }
+  return `⏳ Queued — currently inside \`${activity}\`; I'll answer when this step finishes.`
+}