switchroom 0.18.29 → 0.18.30

package/telegram-plugin/gateway/session-model-file.ts

@@ -38,6 +38,15 @@ import { isValidModelArg } from './model-command.js'
 
 export const SESSION_MODEL_FILE = '.session-model'
 export const CONFIGURED_DEFAULT_MODEL_FILE = '.configured-default-model'
+/**
+ * Bounded-retry attempt counter for the session-model carrier (#3284).
+ * start.sh increments this on every boot that reads `.session-model` and
+ * applies it, and gives up (reverts + alerts) once it exceeds its bound. The
+ * gateway clears BOTH this file and the carrier on a healthy boot (see
+ * consumeSessionModelCarrierOnHealthyBoot) — the "healthy" signal a wedged
+ * boot cannot fake. Kept in sync with start.sh.hbs.
+ */
+export const SESSION_MODEL_BOOT_ATTEMPTS_FILE = '.session-model-boot-attempts'
 
 export interface SessionModelRecord {
   model: string
@@ -125,6 +134,37 @@ export function clearSessionModelFile(agentDir: string): void {
   }
 }
 
+/**
+ * Consume the session-model carrier on a HEALTHY boot (#3284).
+ *
+ * Called once this gateway has ACQUIRED the boot lock (boot.lock_acquired) —
+ * the deterministic signal that THIS boot is the surviving healthy session,
+ * which a boot that wedges before lock-acquire cannot fake. Deletes BOTH the
+ * consume-once carrier and the bounded-retry attempt counter, so:
+ *   - the next ordinary restart (deploy, /restart, crash) finds no carrier and
+ *     reverts to the configured default (session-scoped semantics preserved);
+ *   - a transient wedge BEFORE this point leaves the carrier in place, so the
+ *     retry boot re-applies the intended model instead of silently reverting.
+ *
+ * This is what moved out of start.sh's old delete-before-apply: start.sh no
+ * longer consumes the carrier itself, it only APPLIES it and lets this healthy
+ * signal do the consume. Best-effort — a failure here just means the carrier is
+ * re-read (and re-applied, harmlessly) on the next boot, still bounded by the
+ * counter. Idempotent.
+ */
+export function consumeSessionModelCarrierOnHealthyBoot(agentDir: string): void {
+  try {
+    rmSync(join(agentDir, SESSION_MODEL_FILE), { force: true })
+  } catch {
+    /* best-effort */
+  }
+  try {
+    rmSync(join(agentDir, SESSION_MODEL_BOOT_ATTEMPTS_FILE), { force: true })
+  } catch {
+    /* best-effort */
+  }
+}
+
 /** Restore a rollback snapshot taken with readSessionModelFileRaw. */
 export function restoreSessionModelFileRaw(agentDir: string, raw: string | null): void {
   if (raw == null) {

package/telegram-plugin/gateway/unhandled-message.ts

@@ -0,0 +1,177 @@
+/**
+ * Terminal catch-all + diagnostic tap for inbound Telegram messages (#3300).
+ *
+ * THE CLASS OF BUG THIS CLOSES: the gateway registers only content-specific
+ * handlers (`bot.on('message:text')`, `:photo`, … `:paid_media`). grammy
+ * ^1.44 routes `bot.on` as filtering middleware — internally
+ * `on → filter(pred, handler) → branch(pred, handler, pass)` — and a
+ * `message` update matching NONE of the registered predicates falls through
+ * every branch and is SILENTLY discarded: no log, no ack, no history row.
+ *
+ * Live signature (2026-07-16, klanker DM): message_id 19090 was allocated
+ * between an outbound reply (19089, 21:51:11Z) and the next inbound (19091,
+ * 21:59:31Z) with ZERO gateway trace — no early_ack, no gw-trace inbound, no
+ * gate-deny, no error — while polling stayed healthy on both sides. Exactly
+ * the zero-observability failure this module makes impossible: every update
+ * is now logged on receipt (tap) and every message either produces a turn or
+ * an explicit log-only line (catch-all).
+ *
+ * Extracted into its own module (rather than inline in gateway.ts) so tests
+ * can drive the REAL registration path on a real grammy Bot — gateway.ts is
+ * a side-effecting module that cannot be imported into a unit test.
+ */
+
+import type { Bot, Context } from 'grammy'
+
+/**
+ * Top-level `message` envelope fields — identity/routing metadata, excluded
+ * when surfacing the CONTENT keys of a message (the fields that determine
+ * which `bot.on('message:*')` handler should match).
+ */
+export const MESSAGE_ENVELOPE_KEYS: ReadonlySet<string> = new Set<string>([
+  'message_id', 'message_thread_id', 'date', 'chat', 'from', 'sender_chat',
+  'forward_origin', 'reply_to_message', 'external_reply', 'quote',
+  'reply_to_story', 'edit_date', 'media_group_id', 'author_signature',
+  'is_topic_message', 'is_automatic_forward', 'via_bot', 'sender_boost_count',
+  'business_connection_id', 'effect_id', 'has_protected_content',
+  'is_from_offline', 'link_preview_options', 'show_caption_above_media',
+  'entities', 'caption_entities', 'paid_star_count',
+])
+
+/**
+ * Known-noise SERVICE message content keys: chat/topic lifecycle events that
+ * carry no user intent for the agent. These are LOGGED (never silently
+ * dropped — the whole point of #3300) but do NOT become agent turns:
+ * forum-topic lifecycle events are a recurring spam vector in supergroups,
+ * and each spurious turn costs tokens and attention.
+ *
+ * Anything NOT in this set that reaches the catch-all is delivered as a turn
+ * (fail-toward-delivery: an unknown content type plausibly carries user
+ * intent; better a placeholder turn than a lost message).
+ */
+export const SERVICE_NOISE_KEYS: ReadonlySet<string> = new Set<string>([
+  'new_chat_members', 'left_chat_member', 'new_chat_title', 'new_chat_photo',
+  'delete_chat_photo', 'group_chat_created', 'supergroup_chat_created',
+  'channel_chat_created', 'message_auto_delete_timer_changed',
+  'migrate_to_chat_id', 'migrate_from_chat_id',
+  'forum_topic_created', 'forum_topic_edited', 'forum_topic_closed',
+  'forum_topic_reopened', 'general_forum_topic_hidden',
+  'general_forum_topic_unhidden',
+  'video_chat_scheduled', 'video_chat_started', 'video_chat_ended',
+  'video_chat_participants_invited',
+  'giveaway_created', 'giveaway', 'giveaway_winners', 'giveaway_completed',
+  'boost_added', 'chat_background_set', 'write_access_allowed',
+  'proximity_alert_triggered', 'chat_set_theme',
+  'connected_website', 'direct_message_price_changed',
+])
+
+/** The CONTENT keys of a message — top-level keys minus envelope metadata. */
+export function messageContentKeys(msg: Record<string, unknown>): string[] {
+  return Object.keys(msg).filter(k => !MESSAGE_ENVELOPE_KEYS.has(k))
+}
+
+export type UnhandledMessagePlan =
+  | { action: 'turn'; text: string; contentKeys: string[] }
+  | { action: 'log-only'; contentKeys: string[] }
+
+/**
+ * Decide what the catch-all does with a message no specific handler consumed:
+ * known-noise service messages → log-only (no turn); everything else → a turn
+ * with best-effort text (`text ?? caption ?? placeholder naming the type`).
+ */
+export function planUnhandledMessage(msg: Record<string, unknown>): UnhandledMessagePlan {
+  const contentKeys = messageContentKeys(msg)
+  if (contentKeys.length > 0 && contentKeys.every(k => SERVICE_NOISE_KEYS.has(k))) {
+    return { action: 'log-only', contentKeys }
+  }
+  const contentType = contentKeys[0] ?? 'unknown'
+  const text =
+    (typeof msg.text === 'string' ? msg.text : undefined) ??
+    (typeof msg.caption === 'string' ? msg.caption : undefined) ??
+    `(unhandled message content: ${contentType})`
+  return { action: 'turn', text, contentKeys }
+}
+
+/** One compact diagnostic line per update; cap prevents log flooding. */
+export const TAP_MAX_LINES_PER_MINUTE = 300
+
+/**
+ * Install the diagnostic update tap: pass-through middleware (`bot.use` →
+ * always calls next()) logging every received update's type + content keys
+ * (never payload bodies — privacy). Rate-limited to TAP_MAX_LINES_PER_MINUTE
+ * per wall-clock minute; on rollover a single summary line reports how many
+ * were suppressed, so even a flood is never invisible.
+ *
+ * MUST be installed before all `bot.on` handlers so it observes every update.
+ */
+export function installUpdateTap(
+  bot: Pick<Bot, 'use'>,
+  log: (line: string) => void,
+  nowMs: () => number = Date.now,
+): void {
+  let windowStart = 0
+  let windowCount = 0
+  let suppressed = 0
+  bot.use(async (ctx, next) => {
+    try {
+      const now = nowMs()
+      if (now - windowStart >= 60_000) {
+        if (suppressed > 0) {
+          log(`telegram gateway: rx tap suppressed ${suppressed} update lines in the last minute (cap ${TAP_MAX_LINES_PER_MINUTE}/min)\n`)
+        }
+        windowStart = now
+        windowCount = 0
+        suppressed = 0
+      }
+      if (windowCount < TAP_MAX_LINES_PER_MINUTE) {
+        windowCount++
+        const upd = ctx.update as unknown as Record<string, unknown>
+        const updateType = Object.keys(upd).find(k => k !== 'update_id') ?? 'unknown'
+        const msg = ctx.message as unknown as Record<string, unknown> | undefined
+        const detail = msg ? ` content=[${messageContentKeys(msg).join(',')}]` : ''
+        log(`telegram gateway: rx update_id=${ctx.update.update_id} type=${updateType}${detail}\n`)
+      } else {
+        suppressed++
+      }
+    } catch {
+      // The diagnostic tap must never break the inbound pipeline.
+    }
+    await next()
+  })
+}
+
+/**
+ * Install the terminal catch-all. MUST be registered LAST among the
+ * `message`/`message:*` handlers: grammy leaf handlers never call `next()`,
+ * so a matched specific handler stops the chain (specific handler always
+ * wins) and this only fires for messages no specific handler consumed —
+ * registration order IS the no-double-handling guarantee.
+ *
+ * `onInbound` is the normal inbound pipeline (gateway.ts passes
+ * handleInboundCoalesced), which applies the same access gating and
+ * forward-origin parsing as every other content handler.
+ */
+export function installUnhandledMessageCatchAll(
+  bot: Pick<Bot, 'on'>,
+  onInbound: (ctx: Context, text: string) => Promise<void>,
+  log: (line: string) => void,
+): void {
+  bot.on('message', async ctx => {
+    try {
+      const msg = ctx.message as unknown as Record<string, unknown>
+      const plan = planUnhandledMessage(msg)
+      // Log KEYS + ids only — never the payload bodies (privacy).
+      log(
+        `telegram gateway: catch-all inbound (no specific handler) ` +
+          `update_id=${ctx.update.update_id} chat_id=${ctx.chat?.id ?? '?'} ` +
+          `message_id=${ctx.message?.message_id ?? '?'} ` +
+          `content_keys=[${plan.contentKeys.join(',')}] action=${plan.action}\n`,
+      )
+      if (plan.action === 'turn') {
+        await onInbound(ctx, plan.text)
+      }
+    } catch (err) {
+      log(`telegram gateway: catch-all handler error: ${(err as Error).message}\n`)
+    }
+  })
+}

package/telegram-plugin/llm-error-present.ts

@@ -30,6 +30,7 @@
 import {
   detectModelUnavailable,
   isLitellmProxyLocal429,
+  isLitellmProxyAuthMisconfig,
   parseResetTime,
 } from './model-unavailable.js'
 import { classify429Detail } from './throttle-tier.js'
@@ -46,6 +47,7 @@ export type LlmErrorKind =
   | 'overload_529'
   | 'quota_wall'
   | 'auth'
+  | 'infra_misconfig'
   | 'transient'
   | 'unknown'
 
@@ -150,8 +152,22 @@ export function parseLlmError(
 function classifyKindAndSource(text: string): { kind: LlmErrorKind; source: LlmErrorSource } {
   const lower = text.toLowerCase()
 
+  // 0. LiteLLM-proxy AUTH misconfig — checked BEFORE the auth branch. The
+  //    proxy's internal fallback re-dispatched WITHOUT the OAuth header onto a
+  //    keyless deployment, so Anthropic 401'd it ("x-api-key header is
+  //    required"). This is an OPERATOR-only infra fault, NOT an end-user login
+  //    wall — never the user-facing 'auth' kind (which renders a re-auth card a
+  //    non-operator user cannot act on). Source is the local proxy, not
+  //    Anthropic. See isLitellmProxyAuthMisconfig for provenance.
+  if (isLitellmProxyAuthMisconfig(text)) {
+    return { kind: 'infra_misconfig', source: 'litellm-local' }
+  }
+
   // 1. Auth — always terminal, always actionable.
   const claudeKind = classifyClaudeError({ message: text, type: text })
+  if (claudeKind === 'proxy-misconfig') {
+    return { kind: 'infra_misconfig', source: 'litellm-local' }
+  }
   if (claudeKind === 'credentials-expired' || claudeKind === 'credentials-invalid') {
     return { kind: 'auth', source: 'anthropic' }
   }
@@ -212,6 +228,8 @@ function buildCoreText(kind: LlmErrorKind, source: LlmErrorSource): string {
       return 'Usage limit reached on this Claude subscription.'
     case 'auth':
       return 'Claude login needs re-authentication.'
+    case 'infra_misconfig':
+      return 'Local model-gateway auth misconfig (proxy fallback dropped the OAuth header).'
     case 'transient':
       return source === 'network'
         ? "Couldn't reach Anthropic (network) — retrying automatically."
@@ -280,6 +298,10 @@ function buildRecommendation(parsed: ParsedLlmError, tz: string): string | undef
   switch (parsed.kind) {
     case 'auth':
       return '→ Re-authenticate this account to continue.'
+    case 'infra_misconfig':
+      // Operator-facing — NO re-auth wording (the login is fine). Points at the
+      // real remedy: the local LiteLLM proxy fallback config.
+      return '→ Fix the LiteLLM proxy fallback config (deployment missing OAuth passthrough).'
     case 'quota_wall': {
       const reset = formatResetClock(parsed.resetAt, tz)
       return reset
@@ -353,6 +375,8 @@ function kindEmoji(kind: LlmErrorKind): string {
       return '⚠️'
     case 'auth':
       return '🔑'
+    case 'infra_misconfig':
+      return '🛠️'
     case 'transient':
       return '🌐'
     case 'unknown':

package/telegram-plugin/model-unavailable.ts

@@ -195,6 +195,61 @@ export function isLitellmProxyLocal429(text: string): boolean {
   return litellmV3LimiterSignalPair.every(s => lower.includes(s))
 }
 
+/**
+ * True when `text` is a LiteLLM-proxy-LOCAL AUTH misconfiguration — the
+ * proxy-fallback keyless-401 class, NOT a genuine end-user credential wall.
+ *
+ * PROVENANCE (incident, 2026-07-16). LiteLLM's internal model-fallback chain
+ * (`gpt-oss-20b -> [gpt-oss-20b-openrouter, claude-sonnet-5]`) re-dispatches a
+ * failed primary WITHOUT forwarding the client's OAuth `Authorization` header.
+ * The `claude-sonnet-5` deployment is deliberately keyless (passthrough auth —
+ * it expects the forwarded OAuth), so Anthropic rejects the header-less
+ * fallback with a 401 `authentication_error` whose message is
+ * "x-api-key header is required". Switchroom agents authenticate the unmodified
+ * `claude` CLI with an OAuth Bearer token and NEVER an `x-api-key`, so that
+ * demand can only originate from the proxy dropping the header on a keyless
+ * fallback deployment — a host-side infra misconfig for the OPERATOR to fix,
+ * never a login problem an end user (or even the operator's OAuth) can act on.
+ *
+ * Misclassifying it (as `classifyClaudeError` did → `credentials-invalid` →
+ * a "🔑 re-authenticate" card) is doubly wrong: wrong AUDIENCE (a non-operator
+ * user like Lisa cannot re-auth anything) and wrong DIAGNOSIS (the login is
+ * fine; the proxy fallback config is not). Detecting it here lets both
+ * classifiers route it to the operator-only infra surface instead.
+ *
+ * Detection (deterministic wording, mirrors `isLitellmProxyLocal429`):
+ *   - the definitive "x-api-key header is required" marker (impossible for our
+ *     OAuth flow — only the keyless-proxy path emits it), OR
+ *   - an `authentication_error` CO-OCCURRING with the proxy-FALLBACK-specific
+ *     structural pair: "fallback" re-dispatch provenance AND an explicit
+ *     "x-api-key" mention.
+ *
+ * PRECISION over recall (#3293 review finding 2): an earlier draft matched any
+ * `authentication_error` + (litellm|proxy) + (fallback|x-api-key|api_key) —
+ * broad enough that a GENUINE OAuth expiry wrapped in a proxy envelope that
+ * merely mentions "api_key" would misdiagnose as proxy-misconfig (still
+ * operator-visible, but the recommendation would point at the proxy config
+ * instead of a re-auth). Both classes route operator-only now, so an ambiguous
+ * envelope deliberately KEEPS the credentials-invalid/-expired diagnosis;
+ * only the unambiguous keyless-fallback signature classifies as misconfig.
+ * Never throws on weird input.
+ */
+export function isLitellmProxyAuthMisconfig(text: string): boolean {
+  if (typeof text !== 'string' || text.length === 0) return false
+  const sample = text.length > 16_384 ? text.slice(0, 16_384) : text
+  const lower = sample.toLowerCase()
+  // Definitive marker — see provenance above.
+  if (lower.includes('x-api-key header is required')) return true
+  // Structural signature: an authentication_error carrying BOTH the fallback
+  // re-dispatch provenance AND an explicit x-api-key mention. A proxy envelope
+  // that merely mentions litellm/proxy/api_key stays on the credentials
+  // diagnosis (ambiguous → not misconfig).
+  const isAuthErr =
+    lower.includes('authentication_error') || lower.includes('authenticationerror')
+  if (!isAuthErr) return false
+  return lower.includes('fallback') && lower.includes('x-api-key')
+}
+
 /**
  * Best-effort extraction of the limit detail LiteLLM embeds in its
  * proxy-local 429 bodies, for instrumentation (the `rate_limit_429_classified`

package/telegram-plugin/operator-events.ts

@@ -14,12 +14,14 @@
 
 import { escapeMarkdown } from './format.js'
 import { stripRawErrorBytes } from './raw-error-scrub.js'
+import { isLitellmProxyAuthMisconfig } from './model-unavailable.js'
 
 // ─── Taxonomy ────────────────────────────────────────────────────────────────
 
 export type OperatorEventKind =
   | 'credentials-expired'
   | 'credentials-invalid'
+  | 'proxy-misconfig'
   | 'credit-exhausted'
   | 'quota-exhausted'
   | 'rate-limited'
@@ -92,6 +94,20 @@ function classifyInner(raw: unknown): OperatorEventKind {
   // Anthropic SDK: error_code field (newer SDK shape)
   const sdkCode = extractString(obj, 'error_code') ?? ''
 
+  // LiteLLM-proxy AUTH misconfig — checked BEFORE the generic
+  // authentication_error branch. The proxy's internal fallback chain
+  // re-dispatches without the client's OAuth Authorization header onto a
+  // keyless deployment, so Anthropic returns a 401 authentication_error
+  // ("x-api-key header is required"). That is a HOST-side infra misconfig for
+  // the operator to fix — NOT an end-user login wall. Mapping it to
+  // credentials-invalid mis-fired a "🔑 re-authenticate" card at non-operator
+  // users who cannot re-auth anything (incident 2026-07-16). Route it to the
+  // operator-only infra kind instead. Scanned across type/code/message so the
+  // marker is caught wherever the proxy stamps it.
+  if (isLitellmProxyAuthMisconfig(`${errorType}\n${errorCode}\n${sdkCode}\n${message}`)) {
+    return 'proxy-misconfig'
+  }
+
   // Map known Anthropic error types/codes first.
   // Source: https://docs.anthropic.com/en/api/errors
   if (
@@ -263,6 +279,27 @@ export function renderOperatorEvent(ev: OperatorEvent): RenderResult {
         },
       }
 
+    // Operator-only infra fault. Deliberately NO "Reauth" button and NO
+    // login language: the OAuth credential is fine — the local LiteLLM proxy
+    // re-dispatched an internal fallback without forwarding the OAuth header
+    // onto a keyless deployment, so Anthropic 401'd it. The fix is in the
+    // proxy fallback config (host-side), not a re-auth. Dismiss-only.
+    case 'proxy-misconfig':
+      return {
+        text: [
+          `🛠️ **Model-gateway auth misconfig** for **${agent}**.`,
+          detail ? `_${detail}_` : '',
+          `The local LiteLLM proxy re-dispatched a fallback without forwarding the OAuth header (keyless deployment → Anthropic 401). Fix the proxy fallback config — this is NOT a login problem.`,
+        ]
+          .filter(Boolean)
+          .join('\n'),
+        keyboard: {
+          inline_keyboard: [
+            [{ text: '❌ Dismiss', callback_data: `op:dismiss:${encodeURIComponent(ev.agent)}` }],
+          ],
+        },
+      }
+
     case 'credit-exhausted':
       return {
         text: [
@@ -487,4 +524,80 @@ export function resetAllCooldowns(): void {
   cooldownMap.clear()
 }
 
+// ─── Audience routing (Ken's deterministic error-surfacing policy) ────────────
+
+/**
+ * Kinds that are OPERATOR-ACTIONABLE and NOT user-actionable: a credential /
+ * infra fault whose remedy (re-auth, switch account slot, fix the proxy
+ * fallback config) only the operator can perform. Per Ken's standing policy
+ * (2026-07): raw or misleading auth/infra errors must not reach non-operator
+ * end users — they cannot act on them and the diagnosis is often wrong for
+ * them (e.g. a proxy misconfig rendered as "your login expired"). These route
+ * to the operator surface ONLY; a non-operator user gets at most a brief
+ * plain-language "couldn't complete, it's on our side" notice.
+ *
+ * NOTE: `quota-exhausted` / `rate-limited` are deliberately EXCLUDED — they
+ * carry their own auto-fallback UX + card-collapse machinery and are handled
+ * upstream; this set is scoped to the credential/infra fault classes.
+ */
+export const OPERATOR_ACTIONABLE_KINDS: ReadonlySet<OperatorEventKind> = new Set<OperatorEventKind>([
+  'credentials-expired',
+  'credentials-invalid',
+  'credit-exhausted',
+  'proxy-misconfig',
+])
+
+export function isOperatorActionableKind(kind: OperatorEventKind): boolean {
+  return OPERATOR_ACTIONABLE_KINDS.has(kind)
+}
+
+export interface OperatorEventAudience {
+  /** Chats that receive the full operator card (with any action buttons). */
+  operatorChats: string[]
+  /** Non-operator chats that receive only the plain-language failure notice. */
+  userNoticeChats: string[]
+}
+
+/**
+ * Split an operator-event's allowlist audience per Ken's routing policy.
+ *
+ * - Non-operator-actionable kinds (transient rate-limit, 5xx, crash, config
+ *   warning, …) keep their existing broadcast: every allowlist chat is an
+ *   `operatorChat`. Behavior unchanged.
+ * - Operator-actionable kinds (see {@link OPERATOR_ACTIONABLE_KINDS}) go to the
+ *   OPERATOR chat only. `operatorChatId` is the operator (in switchroom the
+ *   allowlist HEAD — `allowFrom[0]`); it stays an `operatorChat` even in a DM
+ *   agent where the operator is their own user (so the operator NEVER has an
+ *   error hidden from their own DM). Every other allowlist chat becomes a
+ *   `userNoticeChat`.
+ *
+ * Pure — no IPC, no bot. `allowFrom` order is preserved.
+ */
+export function decideOperatorEventAudience(
+  kind: OperatorEventKind,
+  allowFrom: readonly string[],
+  operatorChatId: string | undefined,
+): OperatorEventAudience {
+  if (!isOperatorActionableKind(kind)) {
+    return { operatorChats: [...allowFrom], userNoticeChats: [] }
+  }
+  const operator =
+    operatorChatId != null && allowFrom.includes(operatorChatId)
+      ? operatorChatId
+      : allowFrom[0]
+  const operatorChats = operator != null ? [operator] : []
+  const userNoticeChats = allowFrom.filter((c) => c !== operator)
+  return { operatorChats, userNoticeChats }
+}
+
+/**
+ * The ONE brief, plain-language failure notice a non-operator user gets when a
+ * turn genuinely can't be served because of an operator-actionable fault.
+ * Deliberately carries NO diagnosis, NO agent internals, NO re-auth / config
+ * instructions, and NO raw error text — just an honest "it's on our side".
+ */
+export function renderUserFacingFailureNotice(): string {
+  return "⚠️ Sorry — I couldn't complete that just now. It's a problem on our side, not anything you did. Please try again shortly."
+}
+
 // ─── Markdown escape (#2669) ──────────────────────────────────────────────────

package/telegram-plugin/pending-user-notice.ts

@@ -0,0 +1,88 @@
+/**
+ * pending-user-notice.ts — deterministic turn-outcome gate for the
+ * non-operator user failure notice (#3293 review finding 1).
+ *
+ * THE PROBLEM this closes: `emitGatewayOperatorEvent` fires whenever
+ * session-tail sees an api_error line, with no knowledge of whether the turn
+ * ultimately RECOVERS (e.g. the LiteLLM fallback 401s but a retry / another
+ * deployment serves the turn). Sending the "couldn't complete that — it's on
+ * our side" notice at error time would tell users a turn failed that actually
+ * completed. The 5-min per-kind operator-event cooldown debounces retry SPAM
+ * but cannot know the turn's outcome.
+ *
+ * THE MECHANISM: the notice is never sent at error time. It is SCHEDULED here,
+ * and resolved at the gateway's single turn-end funnel (`endCurrentTurnAtomic`):
+ *   - turn ended WITH a delivered reply  → the turn recovered → DROP the notice
+ *   - turn ended WITHOUT a delivered reply → the turn genuinely died → SEND it
+ * Operator cards are NOT gated — they stay immediate (the operator must see
+ * the infra fault even when the turn recovers).
+ *
+ * CONSERVATIVE TTL: if no turn end resolves a scheduled notice within
+ * {@link PENDING_USER_NOTICE_TTL_MS} (error arrived between turns, or the turn
+ * record was lost), the notice is silently discarded — a missed notice costs a
+ * user some confusion; a FALSE "couldn't complete" on a served turn costs
+ * trust. Bias to silence.
+ *
+ * Pure module: no IPC, no bot, no FS. Injectable `now` throughout.
+ */
+
+export interface PendingUserNotice {
+  /** Non-operator allowlist chats that should receive the notice. */
+  chatIds: string[]
+  /** The plain-language notice text (renderUserFacingFailureNotice()). */
+  text: string
+  agent: string
+  /** The operator-event kind that produced it (log/debug context only). */
+  kind: string
+  /** When the notice was scheduled (ms epoch). */
+  atMs: number
+}
+
+/** How long a scheduled notice may wait for a resolving turn end. */
+export const PENDING_USER_NOTICE_TTL_MS = 10 * 60_000
+
+export class PendingUserNoticeGate {
+  private pending: PendingUserNotice[] = []
+
+  /**
+   * Schedule a notice for turn-end resolution. Collapses per agent — a burst
+   * of error lines within one turn holds ONE pending notice, not a stack.
+   */
+  schedule(notice: PendingUserNotice): void {
+    this.prune(notice.atMs)
+    this.pending = this.pending.filter((p) => p.agent !== notice.agent)
+    this.pending.push(notice)
+  }
+
+  /**
+   * Resolve at turn end. `turnDeliveredReply` is the turn's outcome signal
+   * (the gateway passes `finalAnswerDelivered || replyCalled`):
+   *   - true  → the turn recovered; every pending notice is dropped, [] returned.
+   *   - false → the turn died without a reply; the un-expired pending notices
+   *     are returned EXACTLY ONCE for the caller to send.
+   * Either way the ledger is cleared (a notice never survives its turn end).
+   */
+  resolveTurnEnd(turnDeliveredReply: boolean, now: number = Date.now()): PendingUserNotice[] {
+    this.prune(now)
+    const out = turnDeliveredReply ? [] : [...this.pending]
+    this.pending = []
+    return out
+  }
+
+  /** True when at least one un-expired notice is pending (does not mutate). */
+  hasPending(now: number = Date.now()): boolean {
+    return this.pending.some((p) => now - p.atMs < PENDING_USER_NOTICE_TTL_MS)
+  }
+
+  private prune(now: number): void {
+    this.pending = this.pending.filter((p) => now - p.atMs < PENDING_USER_NOTICE_TTL_MS)
+  }
+
+  /** Test-only: forget everything. */
+  reset(): void {
+    this.pending = []
+  }
+}
+
+/** The process-wide gate the gateway consults. */
+export const pendingUserNoticeGate = new PendingUserNoticeGate()

package/telegram-plugin/shared/local-time.ts

@@ -123,3 +123,46 @@ export function fmtLocalStamp(ms: number, tz: string): string {
     return new Date(ms).toISOString()
   }
 }
+
+/**
+ * Leading ISO-8601-Z timestamp at the start of a log line, e.g.
+ * `2026-07-16T04:09:00.123456789Z` or `2026-07-16T04:09:00Z`. Docker log
+ * lines (as surfaced by `switchroom agent logs`) carry one of these per line.
+ * Anchored at line start; the trailing capture is the rest of the line.
+ */
+const LEADING_ISO_Z = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)(\s|$)/
+
+/**
+ * DISPLAY-ONLY: rewrite a leading UTC ISO-8601-Z timestamp on each line of
+ * `text` into the caller's LOCAL am/pm wall clock via {@link fmtLocalStamp},
+ * so `/logs` output reads `Thursday 2026-07-16 02:09 PM AEST …` instead of a
+ * raw `…T04:09:00Z`. Applied at send time; never mutates stored logs.
+ *
+ * The leading stamp comes from `docker logs --timestamps` (requested by the
+ * /logs path via `switchroom agent logs --timestamps`) — docker's stamp is
+ * ALWAYS UTC ISO-Z, which is what makes this conversion deterministic. App-
+ * emitted stamps inside the line body (e.g. Python's `%H:%M:%S,mmm`) are
+ * deliberately NOT converted: post-#3275 containers run with local TZ baked,
+ * so those are already local wall clock — re-shifting them as UTC would be
+ * wrong. The `tz` passed by /logs is the GATEWAY/operator zone
+ * (`resolveEnvTimezone` of the gateway process), not the target agent's
+ * zone — intended, since /logs is an operator-facing surface.
+ *
+ * Pure / total — matches ONLY a well-formed leading ISO-Z stamp and preserves
+ * every other line (and any line whose timestamp doesn't parse) verbatim, so a
+ * non-timestamped or partial log line is passed through untouched. `\r`
+ * line endings are preserved.
+ */
+export function renderLogTimestampsLocal(text: string, tz: string): string {
+  if (!text) return text
+  return text
+    .split('\n')
+    .map((line) => {
+      const m = LEADING_ISO_Z.exec(line)
+      if (!m) return line
+      const ms = Date.parse(m[1])
+      if (Number.isNaN(ms)) return line
+      return `${fmtLocalStamp(ms, tz)}${m[2] === '' ? '' : ' '}${line.slice(m[0].length)}`
+    })
+    .join('\n')
+}