switchroom 0.15.44 → 0.16.4

package/telegram-plugin/fleet-fallback-resume.ts

@@ -0,0 +1,131 @@
+/**
+ * Resume-the-turn-after-swap gate (THE load-bearing half of the
+ * auth-failover-stall fix).
+ *
+ * Background — the stall this kills:
+ *   When a Claude account hits a rate limit MID-TURN (HTTP 429, including a
+ *   *session* cap like "You've hit your session limit · resets 5pm"), the
+ *   failover machinery correctly detects it (session-tail → model-unavailable
+ *   → gateway.fireFleetAutoFallback → doFireFleetAutoFallback) and swaps the
+ *   fleet to a spare account. But the swap only takes effect on the NEXT
+ *   claude invocation — the turn that died on the 429 is never resumed. The
+ *   agent goes silent until the user manually sends a fresh message
+ *   ("Resume"). Symptom: a mid-conversation stall that needs a manual re-poke.
+ *
+ * The fix: after a SUCCESSFUL swap (outcome.kind === 'switched'), the gateway
+ * re-runs the dead turn on the freshly-active account.
+ *
+ * Mechanism choice — restart, NOT redeliver:
+ *   The inbound that died on the 429 was already DELIVERED (the turn started,
+ *   then the model rejected it). It is therefore NOT sitting in the
+ *   pending-inbound buffer (`redeliverBufferedInbound` only drains inbounds
+ *   that never reached a live bridge). The canonical way to re-run an
+ *   already-delivered, interrupted turn is `triggerSelfRestart`: on the next
+ *   boot the gateway's boot-resume path re-injects the LATEST interrupted turn
+ *   from the turn registry — exactly the turn the 429 killed — and the fresh
+ *   claude process picks up the swapped-to account. `redeliverBufferedInbound`
+ *   would have nothing to redeliver here, so it is the wrong seam.
+ *
+ * This module owns ONLY the decision + the guards, kept pure so it is testable
+ * without restarting a process (the gateway module-state pattern mirrors
+ * `fleet-fallback-gate.ts`). The gateway wires the actual `triggerSelfRestart`
+ * call to `decide()` returning `'resume'`.
+ *
+ * Guards (all REQUIRED by the fix spec):
+ *   1. all-blocked / no-swap no-op — `decide()` is only ever consulted by the
+ *      caller on `outcome.kind === 'switched'`; on any other outcome the caller
+ *      does not call us, preserving the existing all-blocked cooldown path.
+ *   2. Single-flight — once a resume is armed, a second swap (a 429 storm, a
+ *      repeated quota event) within the same window does NOT re-arm. Only ONE
+ *      restart fires per swap. Without this a swap-storm could loop-restart the
+ *      agent.
+ *      NOTE: this latch is IN-MEMORY and does NOT survive the self-restart it
+ *      arms — so it only dedups multiple swaps within a SINGLE process
+ *      lifetime; it cannot by itself bound a cross-restart loop. Cross-restart
+ *      loops are bounded separately, broker-side, by account exhaustion:
+ *      `markExhausted` persists across restarts, so once every spare is benched
+ *      the outcome becomes all-blocked, which does NOT restart.
+ *   3. Staleness failsafe — an ancient interrupted turn is NOT resurrected. If
+ *      the failed turn started longer ago than `maxAgeMs` (default 3h, matching
+ *      the boot-resume RESUME_MAX_AGE_MS failsafe), `decide()` returns
+ *      `'skip-stale'` so a day-old buffered turn isn't replayed unprompted.
+ *      `null` failedTurnStartedAtMs (no turn timestamp known) is treated as
+ *      "resume" — the boot-resume path applies its own 3h guard as a second
+ *      line of defence.
+ */
+
+export interface FleetFallbackResumeOptions {
+  /** Max age of the failed turn before a resume is suppressed as stale.
+   *  Default 3h — mirrors the gateway boot-resume RESUME_MAX_AGE_MS. */
+  maxAgeMs?: number;
+  /** Time source (overridable in tests). */
+  nowFn?: () => number;
+  /** Cooldown after an armed resume during which a second swap will NOT
+   *  re-arm. Dedups a 429 storm WITHIN one process lifetime (this latch is
+   *  in-memory and does not survive the restart it arms — cross-restart loops
+   *  are bounded broker-side by persisted account exhaustion, not here).
+   *  Default 60s — comfortably longer than a restart + boot takes, so the
+   *  in-flight restart settles before another resume can arm. */
+  singleFlightMs?: number;
+}
+
+export type ResumeDecision =
+  /** Fire the resume (restart so the boot path replays the dead turn). */
+  | 'resume'
+  /** Suppressed: a resume is already in flight (single-flight guard). */
+  | 'skip-inflight'
+  /** Suppressed: the failed turn is older than maxAgeMs (staleness guard). */
+  | 'skip-stale';
+
+export interface FleetFallbackResumeGate {
+  /**
+   * Decide whether to resume the dead turn after a successful swap. Call ONLY
+   * when the swap outcome was `'switched'`. Records the arm time on a 'resume'
+   * verdict so a follow-on swap within `singleFlightMs` is suppressed.
+   *
+   * @param failedTurnStartedAtMs epoch-ms the failed turn began, or null when
+   *        unknown (then the staleness guard is deferred to the boot path).
+   */
+  decide(failedTurnStartedAtMs: number | null): ResumeDecision;
+  /** Test seam — reset to fresh state. Production code should not call this. */
+  reset(): void;
+  /** Test/debug — current internal state. */
+  inspect(): { lastResumedAtMs: number };
+}
+
+export const DEFAULT_RESUME_MAX_AGE_MS = 10_800_000; // 3h
+export const DEFAULT_RESUME_SINGLE_FLIGHT_MS = 60_000; // 60s
+
+export function createFleetFallbackResumeGate(
+  opts: FleetFallbackResumeOptions = {},
+): FleetFallbackResumeGate {
+  const nowFn = opts.nowFn ?? (() => Date.now());
+  const maxAgeMs = opts.maxAgeMs ?? DEFAULT_RESUME_MAX_AGE_MS;
+  const singleFlightMs = opts.singleFlightMs ?? DEFAULT_RESUME_SINGLE_FLIGHT_MS;
+  // -Infinity = never resumed. A concrete value arms the single-flight window.
+  let lastResumedAtMs = Number.NEGATIVE_INFINITY;
+
+  function decide(failedTurnStartedAtMs: number | null): ResumeDecision {
+    const now = nowFn();
+    // Guard 2 — single-flight. A second swap within the window does not
+    // re-arm; the in-flight restart owns the resume.
+    if (now - lastResumedAtMs < singleFlightMs) return 'skip-inflight';
+    // Guard 3 — staleness. A known-old failed turn is not resurrected. An
+    // unknown (null) timestamp defers to the boot-resume 3h failsafe.
+    if (failedTurnStartedAtMs != null && now - failedTurnStartedAtMs > maxAgeMs) {
+      return 'skip-stale';
+    }
+    lastResumedAtMs = now;
+    return 'resume';
+  }
+
+  return {
+    decide,
+    reset() {
+      lastResumedAtMs = Number.NEGATIVE_INFINITY;
+    },
+    inspect() {
+      return { lastResumedAtMs };
+    },
+  };
+}

package/telegram-plugin/format.ts

@@ -563,34 +563,71 @@ function escapeUnescapedEntities(inner: string): string {
  * The message then ships to Telegram intact and the user sees literal
  * `\n\n` in the chat instead of paragraph breaks.
  *
- * Heuristic: if the text contains ZERO real newlines AND has at least one
- * literal `\n`, `\r`, or `\t` escape sequence, the caller almost certainly
- * intended those as real whitespace and the client serializer ate them.
- * Unescape them (also `\\` and `\"`). If the text has any real newline,
- * trust the caller exactly as given and do nothing — legitimate content
- * may contain a literal `\n` inside a shell snippet or regex.
+ * Unescape literal `\n`, `\r`, `\t`, and `\"` sequences everywhere EXCEPT
+ * inside code spans (inline backtick spans and fenced ``` blocks). Those
+ * regions are masked with placeholders before the unescape pass so that a
+ * literal `\n` a user typed inside a shell snippet or regex is preserved
+ * verbatim. The genuine escaped-backslash sequence `\\n` (which the user
+ * intended as a literal backslash + n, not a newline) is handled by
+ * protecting `\\` before touching `\n`.
  *
- * This is intentionally narrow: it only fires on the clear bug signature
- * (multi-line-looking content collapsed to one physical line). False
- * positives on a single-line message that legitimately contains `\n` are
- * possible but rare — users writing single-line shell snippets typically
- * wrap them in backticks, and this runs before markdown→HTML so the
- * unescape has no effect on text inside fenced code blocks if it already
- * has real newlines around them.
+ * This deliberately fires even when the message contains real newlines —
+ * the old whole-message heuristic ("bail if any real newline exists") was
+ * too broad and prevented repair of mixed messages that had both real
+ * newlines and stray literal `\n` escape sequences outside code spans.
  */
 export function repairEscapedWhitespace(text: string): string {
-  if (text.includes('\n') || text.includes('\r')) return text
   if (!/\\[nrt"\\]/.test(text)) return text
-  // Order matters: protect existing `\\` first so `\\n` stays as `\n`
-  // literal and doesn't become a newline.
-  const BACKSLASH_PH = '\x00BKSL\x00'
-  return text
+
+  // Per-call random nonce prevents sentinel collision: if user text happens to
+  // contain our placeholder bytes, the restore step would look up an out-of-range
+  // index and produce "undefined" in the Telegram output. A nonce that is unique
+  // per invocation makes the sentinel statistically impossible to collide with.
+  const nonce = Math.random().toString(36).slice(2)
+  const CODE_MASK_PH = `\x00RM${nonce}_`
+  const BACKSLASH_PH = `\x00BK${nonce}_`
+
+  // Mask fenced code blocks (``` ... ```) and inline code spans (` ... `)
+  // so the unescape pass never touches their content.
+  //
+  // Fenced blocks are extracted first. Only CLOSED fenced blocks (with a
+  // matching closing ```) are masked — an unclosed fence is left as-is so the
+  // inline-code pass below won't misparse the two leading backticks as an empty
+  // inline span and expose the block's interior.
+  //
+  // Inline code uses `[^\`\n]+` (one or more non-backtick, non-newline chars)
+  // matching the same definition that markdownToHtml uses, so the masked regions
+  // are consistent with what the downstream pipeline treats as code.
+  const codeMasks: string[] = []
+
+  const masked = text
+    // Closed fenced code blocks only (``` ... ``` with a matching closer).
+    .replace(/```[\s\S]*?```/g, (m) => {
+      const idx = codeMasks.length
+      codeMasks.push(m)
+      return `${CODE_MASK_PH}${idx}\x00`
+    })
+    // Inline code spans: at least one character between backticks, no embedded
+    // backtick or newline (matches markdownToHtml's /`([^`\n]+)`/ definition).
+    .replace(/`[^`\n]+`/g, (m) => {
+      const idx = codeMasks.length
+      codeMasks.push(m)
+      return `${CODE_MASK_PH}${idx}\x00`
+    })
+
+  // Order matters: protect existing `\\` first so `\\n` stays as a literal
+  // backslash + n and doesn't become a newline.
+  const unescaped = masked
     .replace(/\\\\/g, BACKSLASH_PH)
     .replace(/\\n/g, '\n')
     .replace(/\\r/g, '\r')
     .replace(/\\t/g, '\t')
     .replace(/\\"/g, '"')
-    .replace(new RegExp(BACKSLASH_PH, 'g'), '\\')
+    .replace(new RegExp(BACKSLASH_PH.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), '\\')
+
+  // Restore masked code spans verbatim.
+  const restoreRe = new RegExp(`${CODE_MASK_PH.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(\\d+)\x00`, 'g')
+  return unescaped.replace(restoreRe, (_m, idx) => codeMasks[Number(idx)] ?? _m)
 }
 
 // ---------------------------------------------------------------------------