switchroom 0.19.48 → 0.20.1

package/telegram-plugin/gateway/user-failure-notices.ts

@@ -0,0 +1,172 @@
+/**
+ * user-failure-notices.ts — gateway-side handling of the plain-language
+ * "couldn't complete that" user notice, plus the `transport-transient`
+ * classifier kind (a mid-response stream abort reaching Anthropic).
+ *
+ * Two concerns, one subsystem (the plain user-failure-notice send side):
+ *
+ *  1. `emitTransportTransientEvent` — the calm path for a `transport-transient`
+ *     operator event. Claude Code emits `error: "server_error"` with NO HTTP
+ *     status on a mid-response stream abort; that is a transport failure, not an
+ *     account/auth/quota fault, and Claude retries it internally. So there is NO
+ *     broadcast card (a Reauth card would be the wrong remedy and reach every
+ *     chat). Instead: record for `/status` history, and DEFER one plain user
+ *     notice to the turn-end gate (`pending-user-notice.ts`) so a genuinely
+ *     reply-less dead turn still gets one calm line while a recovered retry
+ *     sends nothing. A BURST — >=3 terminal transport-transient events for one
+ *     agent within ~10min — escalates to exactly ONE operator-only, Dismiss-only
+ *     card ("repeated stream failures reaching Anthropic"), so honest
+ *     suppression never hides a genuinely degraded pipe.
+ *
+ *  2. `flushDeferredUserNotices` — the turn-end send loop for notices the gate
+ *     released (moved here from the gateway; see #3293 finding 1).
+ *
+ * Pure orchestration: every side effect (record, schedule, resolve, send, log,
+ * clock) crosses in through `UserFailureNoticeDeps`, so the outcomes are
+ * unit-testable with fakes and no gateway/grammy/IPC coupling leaks in.
+ */
+
+import { escapeMarkdown } from '../format.js'
+import type { InlineKeyboardMarkup, OperatorEvent } from '../operator-events.js'
+import type { PendingUserNotice } from '../pending-user-notice.js'
+
+// ─── transport-transient escalation counter (pure) ───────────────────────────
+
+const ESCALATION_WINDOW_MS = 10 * 60_000
+const ESCALATION_THRESHOLD = 3
+
+/** Per-agent sliding window of terminal transport-transient event timestamps. */
+const transportEscalation = new Map<string, number[]>()
+
+/**
+ * Record one terminal transport-transient event for `agent` at `now` and decide
+ * whether it crosses the escalation threshold. Returns `true` at most once per
+ * burst: crossing the threshold RESETS the window so a sustained outage emits
+ * one operator card per ~`windowMs`, not one per event.
+ */
+export function noteTransportTransientAndShouldEscalate(
+  agent: string,
+  now: number,
+  opts?: { windowMs?: number; threshold?: number },
+): boolean {
+  const windowMs = opts?.windowMs ?? ESCALATION_WINDOW_MS
+  const threshold = opts?.threshold ?? ESCALATION_THRESHOLD
+  const recent = (transportEscalation.get(agent) ?? []).filter((t) => now - t < windowMs)
+  recent.push(now)
+  if (recent.length >= threshold) {
+    transportEscalation.set(agent, [])
+    return true
+  }
+  transportEscalation.set(agent, recent)
+  return false
+}
+
+/** Test/reauth-recovery helper: forget the escalation window (all or one agent). */
+export function resetTransportTransientEscalation(agent?: string): void {
+  if (agent == null) transportEscalation.clear()
+  else transportEscalation.delete(agent)
+}
+
+/**
+ * The ONE operator-only card a transport-transient BURST escalates to. Dismiss-
+ * only — no Reauth (a dropped stream is not a credential fault), no failover
+ * (every account hits the same degraded pipe).
+ */
+export function renderTransportEscalationCard(agent: string): {
+  text: string
+  keyboard: InlineKeyboardMarkup
+} {
+  const a = escapeMarkdown(agent)
+  return {
+    text: [
+      `🔌 **Repeated stream failures** reaching Anthropic for **${a}**.`,
+      `3+ mid-response aborts within ~10 min. Turns retry automatically; if this persists, Anthropic's API may be degraded.`,
+    ].join('\n'),
+    keyboard: {
+      inline_keyboard: [
+        [{ text: '❌ Dismiss', callback_data: `op:dismiss:${encodeURIComponent(agent)}` }],
+      ],
+    },
+  }
+}
+
+// ─── Deps + orchestration ────────────────────────────────────────────────────
+
+export interface UserFailureNoticeDeps {
+  /** Injectable clock. */
+  now(): number
+  /** Allowlist chats (broadcast audience). */
+  allowFrom(): readonly string[]
+  /** Topic key of the live turn, or undefined (agent-level / between turns). */
+  liveTurnKey(): string | undefined
+  /** Persist to /status history (best-effort — must swallow its own errors). */
+  record(event: OperatorEvent): void
+  /** Schedule the turn-end-gated plain user notice for `chatIds`. */
+  scheduleUserNotice(input: {
+    chatIds: string[]
+    agent: string
+    kind: string
+    key: string | undefined
+    atMs: number
+  }): void
+  /** Release the notices resolved by this turn end (may be empty). */
+  resolveNotices(turnDeliveredReply: boolean, turnKey: string): PendingUserNotice[]
+  /** Topic-aware raw send (fire-and-forget). `keyboard` omitted → plain message. */
+  send(chatId: string, text: string, keyboard?: InlineKeyboardMarkup): void
+  /** stderr line (the gateway prefixes "telegram gateway: "). */
+  log(msg: string): void
+}
+
+/**
+ * Handle a `transport-transient` operator event: record it, defer the plain
+ * user notice to turn-end, and — on a burst — post exactly one operator-only
+ * escalation card. Never sends a broadcast card.
+ */
+export function emitTransportTransientEvent(
+  event: OperatorEvent,
+  deps: UserFailureNoticeDeps,
+): void {
+  const now = deps.now()
+  deps.record(event)
+  const escalate = noteTransportTransientAndShouldEscalate(event.agent, now)
+  deps.log(
+    `transport-transient agent=${event.agent} escalate=${escalate} ` +
+      `(no broadcast card; user-notice deferred to turn-end)`,
+  )
+  const allowFrom = deps.allowFrom()
+  if (allowFrom.length === 0) return
+  deps.scheduleUserNotice({
+    chatIds: [...allowFrom],
+    agent: event.agent,
+    kind: event.kind,
+    key: deps.liveTurnKey(),
+    atMs: now,
+  })
+  if (escalate) {
+    const card = renderTransportEscalationCard(event.agent)
+    deps.send(allowFrom[0], card.text, card.keyboard)
+  }
+}
+
+/**
+ * Turn-end resolution of deferred user failure notices (#3293 finding 1, moved
+ * out of gateway.ts). `turnDeliveredReply` is `finalAnswerDelivered ||
+ * replyCalled` — a delivered reply DROPS the notices (turn recovered); a
+ * reply-less end flushes them (turn genuinely died). `turnKey` scopes resolution
+ * to the ending turn's topic under keyed liveness.
+ */
+export function flushDeferredUserNotices(
+  turnDeliveredReply: boolean,
+  turnKey: string,
+  deps: UserFailureNoticeDeps,
+): void {
+  const notices = deps.resolveNotices(turnDeliveredReply, turnKey)
+  if (notices.length === 0) return
+  for (const notice of notices) {
+    deps.log(
+      `user-notice flush (turn died reply-less) agent=${notice.agent} ` +
+        `kind=${notice.kind} chats=${notice.chatIds.length}`,
+    )
+    for (const chatId of notice.chatIds) deps.send(chatId, notice.text)
+  }
+}

package/telegram-plugin/history.ts

@@ -373,6 +373,21 @@ export function verifyHistoryWritable(): { ok: boolean; error?: string } {
   }
 }
 
+/**
+ * Read-only handle for the gateway boot-briefing builder
+ * (`gateway/boot-briefing-wiring.ts`). Exposes only the `prepare(...).all`
+ * subset the builder's `BriefingDb` seam needs; returns null before
+ * `initHistory` (or when history is disabled) so the briefing degrades to
+ * empty instead of throwing. Do NOT use for writes — every write path goes
+ * through the record* functions above so redaction and validity checks
+ * cannot be bypassed.
+ */
+export function getHistoryDbForBriefing(): {
+  prepare(sql: string): { all(...params: unknown[]): unknown[] }
+} | null {
+  return db
+}
+
 /**
  * For tests — close the singleton and forget it. Production code never
  * needs this; the DB is held open for the lifetime of the process.

package/telegram-plugin/llm-error-present.ts

@@ -252,10 +252,15 @@ function classifyKindAndSource(text: string): { kind: LlmErrorKind; source: LlmE
   if (claudeKind === 'rate-limited') {
     return { kind: 'rate_limit', source: 'anthropic' }
   }
-  if (claudeKind === 'unknown-5xx') {
-    return { kind: 'overload_529', source: 'anthropic' }
-  }
-
+  // NOTE: we deliberately do NOT map the taxonomy's `unknown-5xx` to
+  // `overload_529` here. `classifyClaudeError` is called above with only
+  // {message, type} (no HTTP status), so `unknown-5xx` at this point is the
+  // taxonomy's neutral no-status DEFAULT — not evidence of a genuine 529/5xx.
+  // A real Anthropic overload/5xx always carries recognizable wording and is
+  // resolved earlier by detectModelUnavailable (step 3). Fabricating
+  // "Anthropic is overloaded (529) — retrying automatically" for an
+  // unrecognized error would be a false diagnosis, so it falls through to the
+  // honest `unknown` kind.
   return { kind: 'unknown', source: 'anthropic' }
 }
 

package/telegram-plugin/model-unavailable.ts

@@ -474,6 +474,10 @@ export function detectModelUnavailable(
     'socket hang up',
     'request timed out',
     'connection refused',
+    // Mid-response stream aborts reaching Anthropic surface these markers
+    // (a transport failure, not an account/quota fault) — treat as network.
+    'connection closed',
+    'mid-response',
     'getaddrinfo',
   ]
   if (networkSignals.some(s => lower.includes(s))) {

package/telegram-plugin/operator-events.fixtures.json

@@ -119,18 +119,6 @@
       "_source": "Bare 403 with no recognised type",
       "status": 403,
       "message": "Forbidden"
-    },
-    {
-      "_source": "null input — always unknown-4xx",
-      "_value": null
-    },
-    {
-      "_source": "Completely empty object",
-      "_value": {}
-    },
-    {
-      "_source": "Non-object string",
-      "_value": "something went wrong"
     }
   ],
   "unknown-5xx": [
@@ -146,6 +134,18 @@
         "type": "service_unavailable",
         "message": "Service temporarily unavailable"
       }
+    },
+    {
+      "_source": "null input — no status, neutral 5xx default (never a fabricated 4xx)",
+      "_value": null
+    },
+    {
+      "_source": "Completely empty object — no status → unknown-5xx",
+      "_value": {}
+    },
+    {
+      "_source": "Non-object string — no status → unknown-5xx",
+      "_value": "something went wrong"
     }
   ]
 }

package/telegram-plugin/operator-events.ts

@@ -73,6 +73,17 @@ export const OPERATOR_EVENT_KINDS = [
   'mcp-dependency-blocked',
   'quota-exhausted',
   'rate-limited',
+  /**
+   * A TRANSPORT / network failure reaching Anthropic — Claude Code emits
+   * `error: "server_error"` (or `api_error`, which Anthropic documents as an
+   * HTTP 500) with NO `apiErrorStatus` on a mid-response stream abort. It is a
+   * pipe/transport failure, NOT an account, auth, or quota fault: retryable, and
+   * recovered by Claude's own internal retry. Handled by the gateway's dedicated
+   * calm path (no broadcast card; record for /status + a turn-end-gated plain
+   * user notice; a burst escalates to ONE operator-only card) — deliberately
+   * NEVER a Reauth card. See `gateway/user-failure-notices.ts`.
+   */
+  'transport-transient',
   'agent-crashed',
   'agent-restarted-unexpectedly',
   'unknown-4xx',
@@ -110,19 +121,21 @@ export interface InlineKeyboardMarkup {
  * Classify an error value from any source (Anthropic SDK throw, JSONL error
  * field, etc.) into an OperatorEventKind.
  *
- * CONTRACT: never throws. Unfamiliar shapes fall through to unknown-4xx or
- * unknown-5xx based on HTTP status, defaulting to unknown-4xx.
+ * CONTRACT: never throws. Unfamiliar shapes fall through to unknown-5xx or
+ * unknown-4xx based on HTTP status. With NO status the neutral default is
+ * unknown-5xx — an unrecognized shape with no HTTP status must NOT fabricate a
+ * "4xx" label (whose card historically carried a wrong Reauth remedy).
  */
 export function classifyClaudeError(raw: unknown): OperatorEventKind {
   try {
     return classifyInner(raw)
   } catch {
-    return 'unknown-4xx'
+    return 'unknown-5xx'
   }
 }
 
 function classifyInner(raw: unknown): OperatorEventKind {
-  if (raw == null) return 'unknown-4xx'
+  if (raw == null) return 'unknown-5xx'
 
   // Extract common fields defensively — never throw on bad shapes.
   const obj = typeof raw === 'object' ? (raw as Record<string, unknown>) : {}
@@ -258,13 +271,48 @@ function classifyInner(raw: unknown): OperatorEventKind {
     return 'agent-restarted-unexpectedly'
   }
 
+  // TRANSPORT / network failure reaching Anthropic. Claude Code emits
+  // `error: "server_error"` (and `api_error`, documented as HTTP 500) with NO
+  // `apiErrorStatus` on a mid-response stream abort — a pipe/transport failure,
+  // not an account/auth/quota fault. Without this branch it fell through to the
+  // status fallback below and FABRICATED `unknown-4xx`, whose card carried a
+  // wrong Reauth remedy and broadcast to every chat. It is retryable (Claude
+  // retries internally), so it maps to the calm `transport-transient` kind.
+  //
+  // Placed AFTER the auth-wording matches above ON PURPOSE: a genuine auth
+  // failure that upstream mislabels `server_error` still classifies as a
+  // credential fault. Matched only on EXACT type/code equality (never a message
+  // substring) so an unrelated error whose text merely mentions "server error"
+  // is untouched.
+  //
+  // GUARDED to status-less OR 5xx shapes (`status == null || status >= 500`): a
+  // mid-response abort carries NO status, and a genuine 500 is server-side. But
+  // LiteLLM stamps a generic `type: "api_error"` onto WRAPPED upstream faults
+  // that DO carry a 4xx status (e.g. a 401), and session-tail threads that
+  // `apiErrorStatus` through as `status`. Without this guard such a 401 would be
+  // swallowed as transport-transient (silent, no card) when it must reach the
+  // status fallback below and surface an operator card. The status wins.
+  if (
+    (status == null || status >= 500) &&
+    (errorType === 'server_error' ||
+      errorCode === 'server_error' ||
+      sdkCode === 'server_error' ||
+      errorType === 'api_error' ||
+      errorCode === 'api_error' ||
+      sdkCode === 'api_error')
+  ) {
+    return 'transport-transient'
+  }
+
   // Fallback: HTTP status-based.
   if (status != null) {
     if (status >= 400 && status < 500) return 'unknown-4xx'
     if (status >= 500 && status < 600) return 'unknown-5xx'
   }
 
-  return 'unknown-4xx'
+  // No recognized shape and NO status: neutral 5xx default. Do NOT claim 4xx —
+  // a fabricated client-error label mis-routed a Reauth card (the bug this fixes).
+  return 'unknown-5xx'
 }
 
 function extractString(obj: Record<string, unknown>, key: string): string | null {
@@ -508,6 +556,12 @@ export function renderOperatorEvent(ev: OperatorEvent): RenderResult {
         },
       }
 
+    // Dismiss-only, matching unknown-5xx / proxy-misconfig. The Reauth button
+    // was REMOVED: unknown-4xx is a catch-all for unrecognized client-error
+    // shapes, and re-auth is the wrong remedy for almost all of them. Post-RFC-H
+    // the auth-broker is the SOLE writer of `.credentials.json`, so a per-agent
+    // reauth is re-mirrored over anyway. Reauth buttons now live ONLY on the
+    // credentials-* cards, where the diagnosis IS a login fault.
     case 'unknown-4xx':
       return {
         text: [
@@ -518,10 +572,28 @@ export function renderOperatorEvent(ev: OperatorEvent): RenderResult {
           .join('\n'),
         keyboard: {
           inline_keyboard: [
-            [
-              { text: '🔐 Reauth', callback_data: `op:reauth:${encodeURIComponent(ev.agent)}` },
-              { text: '❌ Dismiss', callback_data: `op:dismiss:${encodeURIComponent(ev.agent)}` },
-            ],
+            [{ text: '❌ Dismiss', callback_data: `op:dismiss:${encodeURIComponent(ev.agent)}` }],
+          ],
+        },
+      }
+
+    // transport-transient is normally handled by the gateway's dedicated calm
+    // path (no broadcast card; a burst escalates via renderTransportEscalationCard
+    // in gateway/user-failure-notices.ts). This case exists so the switch stays
+    // exhaustive and any incidental render is calm + Dismiss-only — NEVER a
+    // Reauth button (a dropped stream is not a credential fault).
+    case 'transport-transient':
+      return {
+        text: [
+          `🔌 **Transient transport error** for **${agent}**.`,
+          detail ? `\`${detail}\`` : '',
+          `A mid-response stream to Anthropic dropped. Will retry automatically.`,
+        ]
+          .filter(Boolean)
+          .join('\n'),
+        keyboard: {
+          inline_keyboard: [
+            [{ text: '❌ Dismiss', callback_data: `op:dismiss:${encodeURIComponent(ev.agent)}` }],
           ],
         },
       }

package/telegram-plugin/session-tail.ts

@@ -1075,7 +1075,13 @@ export function detectErrorInTranscriptLine(
   // with no retry state is ambiguous → treat as in-flight and suppress
   // (the silence-poke covers a genuinely stuck turn; a false card is
   // the bug we are fixing, a missed ambiguous card costs nothing).
-  const transient = kind === 'rate-limited'
+  // `transport-transient` joins the transient family so Claude's retry
+  // annotations gate `terminal`: a mid-response abort Claude is still retrying
+  // (retryAttempt < maxRetries) is in-flight, not terminal, and must NOT count
+  // toward the operator escalation counter (which is bounded to ≥3 TERMINAL
+  // events). The primary bug shape (`isApiErrorMessage`) returns earlier with
+  // terminal:true by construction, so it is unaffected by this.
+  const transient = kind === 'rate-limited' || kind === 'transport-transient'
   const retry = extractRetryState(obj)
   const terminal = !transient
     ? true