switchroom 0.5.0

package/telegram-plugin/model-unavailable.ts

@@ -0,0 +1,325 @@
+/**
+ * model-unavailable.ts — graceful UX for the "model is down" failure modes.
+ *
+ * Issue #394 (Fix 2). When a user message hits Claude and the model is
+ * unreachable — quota exhausted, overloaded / 429-storm, billing dead, or
+ * the network simply timed out — the bridge used to relay Anthropic's raw
+ * stderr verbatim ("You're out of extra usage · resets May 3, 11am"). The
+ * desired UX is a clean ⚠️ card naming what failed and pointing at the
+ * three actions that actually move the needle:
+ *
+ *   - /authfallback — auto-switch to next slot
+ *   - /auth add     — attach another subscription
+ *   - /usage        — full quota breakdown
+ *
+ * This module owns:
+ *   1. `detectModelUnavailable(stderr)` — pattern-matches a raw error
+ *      string into one of three structured kinds (overload / quota_exhausted
+ *      / network), pulling out a reset Date when the source mentions one.
+ *      Returns null on lines that don't look like a model-down event so
+ *      callers can fall through to their default rendering.
+ *   2. `formatModelUnavailableCard(detection, agent)` — renders the HTML
+ *      card. Reset-time formatting routes through quota-check.ts's
+ *      `formatResetRelative` so "/usage" and this card speak the same
+ *      countdown dialect.
+ *
+ * Pure module: no IPC, no bot, no FS. Trivially unit-testable.
+ */
+
+import { formatResetRelative } from './quota-check.js'
+
+// ─── Public types ────────────────────────────────────────────────────────────
+
+export type ModelUnavailableKind = 'overload' | 'quota_exhausted' | 'network'
+
+export interface ModelUnavailableDetection {
+  kind: ModelUnavailableKind
+  /** When the source mentions a reset window, parsed best-effort to a Date. */
+  resetAt?: Date
+  /** The raw stderr string that triggered the detection. */
+  raw: string
+}
+
+// ─── Detection ───────────────────────────────────────────────────────────────
+
+/**
+ * Inspect a raw stderr / error-message string for one of the known
+ * model-unavailable patterns. Returns null when the line doesn't look
+ * like one — never throws on weird input.
+ *
+ * Detection rules (matched in priority order):
+ *   1. Quota / billing-style strings ("out of extra usage", "credit_balance_too_low",
+ *      "usage limit", "quota exhausted") → quota_exhausted
+ *   2. Overload / 429 / 5xx signals ("overloaded_error", "rate_limit_error",
+ *      "HTTP 429", "Service Unavailable", "503", "529") → overload
+ *   3. Network-layer failures (DNS, ECONNREFUSED, ETIMEDOUT, "fetch failed",
+ *      "network error", "socket hang up") → network
+ *
+ * Quota strings can also carry a reset-time hint ("resets May 3, 11am",
+ * "resets in 2h 15m", "retry after 60 seconds", ISO 8601). When present
+ * and parseable, the Date is attached.
+ */
+export function detectModelUnavailable(
+  stderr: string,
+): ModelUnavailableDetection | null {
+  if (typeof stderr !== 'string' || stderr.length === 0) return null
+  // Defend against pathological input — anything beyond a few KB is almost
+  // certainly not a clean error string and risks a regex stall.
+  const sample = stderr.length > 16_384 ? stderr.slice(0, 16_384) : stderr
+  const lower = sample.toLowerCase()
+
+  // ── 1. Quota / billing exhaustion ──────────────────────────────────────
+  const quotaSignals = [
+    'out of extra usage',
+    'extra usage',
+    'credit_balance_too_low',
+    'credit balance too low',
+    'usage limit',
+    'usage_limit',
+    'quota exhausted',
+    'quota_exhausted',
+    'plan limit',
+    'subscription limit',
+  ]
+  if (quotaSignals.some(s => lower.includes(s))) {
+    const resetAt = parseResetTime(sample)
+    return resetAt !== undefined
+      ? { kind: 'quota_exhausted', resetAt, raw: stderr }
+      : { kind: 'quota_exhausted', raw: stderr }
+  }
+
+  // ── 2. Overload / 429 / 5xx ────────────────────────────────────────────
+  const overloadSignals = [
+    'overloaded_error',
+    'overloaded',
+    'rate_limit_error',
+    'rate limit',
+    'rate-limited',
+    'http 429',
+    '"status":429',
+    'status: 429',
+    ' 429 ',
+    '503 service',
+    'service unavailable',
+    '"status":529',
+    'http 529',
+    ' 529 ',
+  ]
+  if (overloadSignals.some(s => lower.includes(s))) {
+    const resetAt = parseResetTime(sample)
+    return resetAt !== undefined
+      ? { kind: 'overload', resetAt, raw: stderr }
+      : { kind: 'overload', raw: stderr }
+  }
+
+  // ── 3. Network-layer failure ───────────────────────────────────────────
+  const networkSignals = [
+    'econnrefused',
+    'econnreset',
+    'etimedout',
+    'enotfound',
+    'eai_again',
+    'fetch failed',
+    'network error',
+    'socket hang up',
+    'request timed out',
+    'connection refused',
+    'getaddrinfo',
+  ]
+  if (networkSignals.some(s => lower.includes(s))) {
+    return { kind: 'network', raw: stderr }
+  }
+
+  return null
+}
+
+/**
+ * Best-effort reset-time extraction. Tries, in order:
+ *   - "retry after N seconds" / "retry-after: N"
+ *   - "resets in 2h 15m" (relative — anchored at parseTimeNow)
+ *   - "resets May 3, 11am" / "resets at May 3 11:00"
+ *   - bare ISO-8601 timestamp anywhere in the string
+ *
+ * Returns undefined when nothing parseable is found. The `parseTimeNow`
+ * arg lets tests pin the relative-clock anchor; production callers omit
+ * it to use Date.now().
+ */
+function parseResetTime(text: string, parseTimeNow: Date = new Date()): Date | undefined {
+  const lower = text.toLowerCase()
+
+  // "retry after 60 seconds" / "retry-after: 60"
+  const retryAfter = lower.match(/retry[\s-]*after[:\s]+(\d+)\s*(seconds?|s\b|minutes?|m\b|hours?|h\b)?/)
+  if (retryAfter) {
+    const n = Number(retryAfter[1])
+    if (Number.isFinite(n) && n > 0 && n < 7 * 24 * 3600) {
+      const unit = (retryAfter[2] ?? 'seconds').toLowerCase()
+      const ms = unit.startsWith('h')
+        ? n * 3600_000
+        : unit.startsWith('m')
+        ? n * 60_000
+        : n * 1000
+      return new Date(parseTimeNow.getTime() + ms)
+    }
+  }
+
+  // "resets in 2h 15m" / "resets in 30 minutes"
+  const relReset = lower.match(/resets?\s+in\s+([0-9hms\s]+)/)
+  if (relReset) {
+    const ms = parseRelativeDuration(relReset[1])
+    if (ms != null) return new Date(parseTimeNow.getTime() + ms)
+  }
+
+  // ISO-8601 timestamp anywhere in the text
+  const iso = text.match(/\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?\b/)
+  if (iso) {
+    const d = new Date(iso[0])
+    if (!Number.isNaN(d.getTime())) return d
+  }
+
+  // "resets May 3, 11am" / "resets May 3 at 11:00"
+  // Conservative regex — avoid greedy backtracking on long strings.
+  const calReset = text.match(
+    /resets?\s+(?:at\s+)?([A-Z][a-z]{2,8}\s+\d{1,2}(?:,?\s*(?:\d{1,2}(?::\d{2})?\s*(?:am|pm|AM|PM)?))?)/,
+  )
+  if (calReset) {
+    // Anchor to the current year — Anthropic's user-facing strings omit it.
+    const candidate = `${calReset[1]} ${parseTimeNow.getUTCFullYear()}`
+    const d = new Date(candidate)
+    if (!Number.isNaN(d.getTime())) return d
+  }
+
+  return undefined
+}
+
+function parseRelativeDuration(s: string): number | null {
+  // "2h 15m" / "30m" / "45 seconds"
+  let total = 0
+  let matched = false
+  const re = /(\d+)\s*(h|hours?|m|minutes?|s|seconds?)/g
+  let m: RegExpExecArray | null
+  while ((m = re.exec(s)) != null) {
+    matched = true
+    const n = Number(m[1])
+    const unit = m[2].toLowerCase()
+    if (unit.startsWith('h')) total += n * 3600_000
+    else if (unit.startsWith('m')) total += n * 60_000
+    else total += n * 1000
+  }
+  return matched && total > 0 ? total : null
+}
+
+// ─── Card rendering ──────────────────────────────────────────────────────────
+
+export interface FormatCardOptions {
+  /** Slot the agent was using when the failure happened — when known,
+   *  named in the card so the user can act precisely. */
+  slot?: string | null
+  /** Anchor for relative-time formatting. Tests pin this; prod omits it. */
+  now?: Date
+}
+
+/**
+ * Render the actionable ⚠️ card for a detected model-unavailable event.
+ * HTML-formatted for Telegram. Stable shape so snapshot tests remain
+ * meaningful when the suggestion list shifts.
+ *
+ *   ⚠️ <b>Model unavailable</b> on agent <b>name</b>
+ *   Reason: quota exhausted (resets in 5h)
+ *
+ *   <b>What to try</b>
+ *   • <code>/authfallback</code> — switch to the next account slot
+ *   • <code>/auth add</code> — attach another subscription
+ *   • <code>/usage</code> — show quota breakdown
+ */
+export function formatModelUnavailableCard(
+  detection: ModelUnavailableDetection,
+  agent: string,
+  opts: FormatCardOptions = {},
+): string {
+  const now = opts.now ?? new Date()
+  const slotPart = opts.slot ? ` (slot <b>${escHtml(opts.slot)}</b>)` : ''
+  const reason = formatReason(detection, now)
+  const lines = [
+    `⚠️ <b>Model unavailable</b> on agent <b>${escHtml(agent)}</b>${slotPart}`,
+    `Reason: ${reason}`,
+    '',
+    '<b>What to try</b>',
+    '• <code>/authfallback</code> — switch to the next account slot',
+    '• <code>/auth add</code> — attach another subscription',
+    '• <code>/usage</code> — show quota breakdown',
+  ]
+  return lines.join('\n')
+}
+
+function formatReason(d: ModelUnavailableDetection, now: Date): string {
+  const reset = d.resetAt ? ` (${formatResetRelative(d.resetAt, now)})` : ''
+  switch (d.kind) {
+    case 'quota_exhausted':
+      return `quota exhausted${reset}`
+    case 'overload':
+      return `model overloaded${reset}`
+    case 'network':
+      return 'network unreachable'
+  }
+}
+
+// ─── Operator-event bridge ───────────────────────────────────────────────────
+
+/**
+ * Minimal shape for the operator-event input — kept structural to avoid a
+ * runtime dependency on `operator-events.ts`. The gateway passes its real
+ * `OperatorEvent` here; tests can pass a hand-rolled object with just the
+ * `kind` and `detail` fields.
+ *
+ * The string union covers exactly the kinds that are model-availability-
+ * relevant. Any kind outside that set falls through to text-pattern
+ * detection on `detail`.
+ */
+export interface OperatorEventLike {
+  kind: string
+  detail: string
+}
+
+/**
+ * Decide whether an operator event represents a model-unavailable failure.
+ * Returns null when it's something else (auth issue, agent crash, etc.) so
+ * the caller can fall back to its default per-kind renderer.
+ *
+ * Used by the gateway's `emitGatewayOperatorEvent` to decide whether to
+ * suppress the raw stderr-style `detail` and post the actionable card
+ * instead. Lives here (not in the gateway) so it's pure-testable without
+ * spinning up the bot.
+ *
+ * Decision order:
+ *   1. If the kind is one of the known model-unavailable kinds, build a
+ *      synthetic detection from kind + detail (passing detail through
+ *      `detectModelUnavailable` first to pick up reset-time hints).
+ *   2. Otherwise, run pattern detection on `detail` — covers cases where
+ *      a generic 4xx/5xx slipped past the upstream classifier carrying
+ *      a quota/overload message in its body.
+ */
+export function resolveModelUnavailableFromOperatorEvent(
+  ev: OperatorEventLike,
+): ModelUnavailableDetection | null {
+  const detail = typeof ev.detail === 'string' ? ev.detail : ''
+  if (ev.kind === 'quota-exhausted') {
+    return detectModelUnavailable(detail) ?? { kind: 'quota_exhausted', raw: detail }
+  }
+  if (ev.kind === 'rate-limited') {
+    return detectModelUnavailable(detail) ?? { kind: 'overload', raw: detail }
+  }
+  if (ev.kind === 'unknown-5xx') {
+    return detectModelUnavailable(detail) ?? { kind: 'overload', raw: detail }
+  }
+  return detectModelUnavailable(detail)
+}
+
+// ─── HTML escape (mirrors operator-events.ts) ────────────────────────────────
+
+function escHtml(text: string): string {
+  return text
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+}

package/telegram-plugin/node_modules/.vite/vitest/da39a3ee5e6b4b0d3255bfef95601890afd80709/results.json

@@ -0,0 +1 @@
+{"version":"3.2.4","results":[[":tests/progress-card-driver.test.ts",{"duration":82.55005700000004,"failed":false}],[":tests/progress-card.test.ts",{"duration":50.04072000000002,"failed":false}],[":tests/telegram-format.test.ts",{"duration":0,"failed":false}],[":tests/stream-reply-handler.test.ts",{"duration":0,"failed":false}],[":tests/progress-card-harness.test.ts",{"duration":4955.245276000001,"failed":false}],[":tests/streaming-orchestration.test.ts",{"duration":0,"failed":false}],[":tests/races.test.ts",{"duration":0,"failed":false}],[":tests/subagent-watcher.test.ts",{"duration":0,"failed":false}],[":tests/gateway-bridge.test.ts",{"duration":0,"failed":true}],[":tests/answer-stream.test.ts",{"duration":0,"failed":false}],[":tests/progress-card-pin-manager.test.ts",{"duration":25.365711999999974,"failed":false}],[":tests/pty-tail.test.ts",{"duration":0,"failed":false}],[":tests/turn-end-regressions.test.ts",{"duration":0,"failed":false}],[":tests/session-tail.test.ts",{"duration":0,"failed":false}],[":tests/gateway-clean-shutdown-marker.test.ts",{"duration":0,"failed":true}],[":tests/e2e.test.ts",{"duration":0,"failed":false}],[":tests/setup-flow.test.ts",{"duration":0,"failed":false}],[":tests/tool-labels.test.ts",{"duration":0,"failed":false}],[":tests/registry-turns.test.ts",{"duration":0,"failed":true}],[":tests/welcome-text.test.ts",{"duration":0,"failed":false}],[":tests/secret-detect-oauth-code.test.ts",{"duration":0,"failed":false}],[":tests/draft-stream.test.ts",{"duration":0,"failed":false}],[":tests/progress-card-stuck-warning.test.ts",{"duration":21.23773799999998,"failed":false}],[":tests/history.test.ts",{"duration":0,"failed":false}],[":tests/streaming-e2e.test.ts",{"duration":0,"failed":false}],[":tests/foreman-handlers.test.ts",{"duration":0,"failed":false}],[":tests/foreman-create-flow.test.ts",{"duration":0,"failed":false}],[":tests/operator-events.test.ts",{"duration":0,"failed":false}],[":tests/vault-grants-revoke.test.ts",{"duration":0,"failed":false}],[":tests/boot-probes.test.ts",{"duration":0,"failed":true}],[":tests/pty-partial-handler.test.ts",{"duration":0,"failed":false}],[":tests/auth-slot-commands.test.ts",{"duration":0,"failed":false}],[":tests/vault-subcommands.test.ts",{"duration":0,"failed":false}],[":tests/auth-dashboard.test.ts",{"duration":0,"failed":false}],[":tests/active-pins-sweep.test.ts",{"duration":0,"failed":false}],[":tests/turns-writer.test.ts",{"duration":0,"failed":true}],[":tests/retry-api-call.test.ts",{"duration":0,"failed":false}],[":tests/steering.test.ts",{"duration":0,"failed":false}],[":tests/outbound-ordering.test.ts",{"duration":0,"failed":false}],[":tests/ipc-server-client.test.ts",{"duration":0,"failed":false}],[":tests/gateway-startup-mutex.test.ts",{"duration":0,"failed":true}],[":tests/auth-dashboard-edge-cases.test.ts",{"duration":0,"failed":false}],[":tests/status-reactions.test.ts",{"duration":0,"failed":false}],[":tests/auto-fallback.test.ts",{"duration":0,"failed":false}],[":tests/secret-detect.test.ts",{"duration":0,"failed":false}],[":tests/foreman-write-ops.test.ts",{"duration":0,"failed":false}],[":tests/boot-card-render.test.ts",{"duration":0,"failed":false}],[":tests/handoff-continuity.test.ts",{"duration":0,"failed":false}],[":tests/false-restart-banner.test.ts",{"duration":0,"failed":false}],[":tests/answer-stream-silent-markers.test.ts",{"duration":0,"failed":false}],[":tests/ipc-validator.test.ts",{"duration":0,"failed":false}],[":tests/stream-controller.test.ts",{"duration":0,"failed":false}],[":tests/gateway-409-retry-banner.test.ts",{"duration":0,"failed":false}],[":tests/stream-reply-error-paths.test.ts",{"duration":0,"failed":false}],[":tests/ipc-server-race.test.ts",{"duration":0,"failed":false}],[":tests/progress-update.test.ts",{"duration":0,"failed":true}],[":tests/restart-watchdog.test.ts",{"duration":0,"failed":false}],[":tests/progress-card-cross-turn.test.ts",{"duration":0,"failed":false}],[":tests/boot-card-probe-target.test.ts",{"duration":0,"failed":false}],[":tests/active-reactions.test.ts",{"duration":0,"failed":false}],[":tests/quota-cache.test.ts",{"duration":0,"failed":true}],[":tests/streaming-metrics.test.ts",{"duration":0,"failed":false}],[":tests/operator-events-session-tail.test.ts",{"duration":0,"failed":false}],[":tests/foreman-state.test.ts",{"duration":0,"failed":true}],[":tests/ipc-protocol.test.ts",{"duration":0,"failed":false}],[":tests/progress-card-pin-watchdog.test.ts",{"duration":0,"failed":false}],[":tests/telegram-button-constraints.test.ts",{"duration":0,"failed":false}],[":tests/bot-runtime.test.ts",{"duration":0,"failed":false}],[":tests/gateway-startup-network-retry.test.ts",{"duration":0,"failed":false}],[":tests/attachment-path.test.ts",{"duration":0,"failed":false}],[":tests/active-pins.test.ts",{"duration":0,"failed":false}],[":tests/subagent-tracker-hooks.test.ts",{"duration":0,"failed":true}],[":tests/fake-bot-api.test.ts",{"duration":0,"failed":false}],[":tests/quota-check.test.ts",{"duration":0,"failed":false}],[":tests/parse-mode-rotation.test.ts",{"duration":0,"failed":false}],[":tests/gateway-secret-detect.test.ts",{"duration":0,"failed":false}],[":tests/turn-flush-safety.test.ts",{"duration":0,"failed":false}],[":tests/multi-turn-continuity.test.ts",{"duration":0,"failed":false}],[":tests/status-accent.test.ts",{"duration":0,"failed":false}],[":tests/auth-code-auto-capture.test.ts",{"duration":0,"failed":false}],[":tests/auth-login-url-button.test.ts",{"duration":0,"failed":false}],[":tests/auth-dashboard-restart-flow.test.ts",{"duration":0,"failed":false}],[":tests/setup-state.test.ts",{"duration":0,"failed":true}],[":tests/progress-card-golden.test.ts",{"duration":6.658348999999987,"failed":false}],[":tests/unhandled-rejection-policy.test.ts",{"duration":0,"failed":true}],[":tests/typing-wrap.test.ts",{"duration":0,"failed":false}],[":tests/auth-account-identity-surface.test.ts",{"duration":0,"failed":false}],[":tests/secret-detect-secretlint.test.ts",{"duration":0,"failed":false}],[":tests/secret-detect-pipeline.test.ts",{"duration":0,"failed":false}],[":tests/operator-events-history.test.ts",{"duration":0,"failed":false}],[":tests/gateway-message-validator.test.ts",{"duration":0,"failed":false}],[":tests/silent-reply-guard.test.ts",{"duration":0,"failed":false}],[":tests/active-reactions-sweep.test.ts",{"duration":0,"failed":false}],[":tests/pin-event-log.test.ts",{"duration":0,"failed":false}],[":tests/secret-detect-fail-closed.test.ts",{"duration":0,"failed":false}],[":tests/ipc-server-validate-operator.test.ts",{"duration":0,"failed":false}],[":tests/idle-footer-wiring.test.ts",{"duration":0,"failed":true}],[":tests/boot-card-reason.test.ts",{"duration":0,"failed":true}],[":tests/plugin-logger.test.ts",{"duration":0,"failed":false}],[":tests/vault-grant-wizard.test.ts",{"duration":0,"failed":false}],[":tests/turn-signal-tracker.test.ts",{"duration":0,"failed":false}],[":tests/context-exhaustion.test.ts",{"duration":0,"failed":false}],[":tests/gateway-startup-reset.test.ts",{"duration":0,"failed":false}],[":tests/idle-footer.test.ts",{"duration":0,"failed":false}],[":tests/poll-health.test.ts",{"duration":0,"failed":false}],[":tests/turn-flush-prose-recovery.test.ts",{"duration":0,"failed":false}],[":tests/secret-detect-suppressor-no-silent-allow.test.ts",{"duration":0,"failed":false}],[":tests/boot-card-dedupe.test.ts",{"duration":0,"failed":true}],[":tests/protocol-fixtures.test.ts",{"duration":0,"failed":false}],[":tests/secret-detect-staging.test.ts",{"duration":0,"failed":false}],[":tests/secret-detect-audit.test.ts",{"duration":0,"failed":false}],[":tests/secret-detect-gitleaks.test.ts",{"duration":0,"failed":false}],[":tests/subagent-registry-bugs.test.ts",{"duration":0,"failed":false}],[":tests/progress-card-driver-eviction.test.ts",{"duration":22.822707999999977,"failed":false}],[":tests/progress-card-driver-fleet-shadow.test.ts",{"duration":7.463677000000018,"failed":false}],[":tests/two-zone-card-lifecycle.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-concurrent-turns-isolation.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-bg-survives-next-turn.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-card-snapshot.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-stuck-edit-throttle.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-bg-done-when-all-terminal.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-card-html-balance.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-bg-detection.test.ts",{"duration":0,"failed":false}],[":tests/progress-card-draft-flag.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-stuck-per-member.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-card-header-phases.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-stuck-recovery.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-stuck-header-escalation.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-card-fleet-row.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-card-cap.test.ts",{"duration":0,"failed":false}],[":tests/two-zone-card-sanitise.test.ts",{"duration":0,"failed":false}],[":tests/progress-card-close-paths-converge.test.ts",{"duration":10.167681000000016,"failed":false}],[":registry/subagents.test.ts",{"duration":0,"failed":true}],[":tests/issues-card.test.ts",{"duration":0,"failed":false}],[":registry/subagents-bugs.test.ts",{"duration":0,"failed":true}],[":tests/answer-stream-dedup.test.ts",{"duration":0,"failed":false}],[":tests/model-unavailable.test.ts",{"duration":0,"failed":false}],[":tests/preamble-suppressor.test.ts",{"duration":0,"failed":false}],[":tests/harness-ordering-invariants.test.ts",{"duration":0,"failed":false}],[":tests/real-gateway-spec.test.ts",{"duration":0,"failed":false}],[":tests/real-gateway-ipc-lifecycle.test.ts",{"duration":0,"failed":false}],[":tests/real-gateway-i6-turn-flush-replay-dedup.test.ts",{"duration":0,"failed":false}],[":tests/slot-banner-driver.e2e.test.ts",{"duration":0,"failed":false}],[":tests/waiting-ux.e2e.test.ts",{"duration":0,"failed":false}],[":tests/subagent-watcher-parent-marker.test.ts",{"duration":0,"failed":false}],[":tests/auth-code-redact.test.ts",{"duration":0,"failed":false}],[":tests/subagent-watcher-stall-notification.test.ts",{"duration":0,"failed":false}],[":tests/telegraph.test.ts",{"duration":0,"failed":true}],[":tests/recent-outbound-dedup.test.ts",{"duration":0,"failed":true}],[":tests/turn-flush-card-takeover.test.ts",{"duration":0,"failed":false}],[":tests/resolve-calling-subagent.test.ts",{"duration":0,"failed":true}],[":tests/secret-guard-pretool.test.ts",{"duration":0,"failed":true}],[":tests/first-paint.test.ts",{"duration":0,"failed":false}],[":tests/ask-user.test.ts",{"duration":0,"failed":true}],[":registry/api-registry.test.ts",{"duration":0,"failed":true}],[":tests/credits-watch.test.ts",{"duration":0,"failed":false}],[":admin-commands/dispatch.test.ts",{"duration":0,"failed":false}],[":tests/fleet-state.test.ts",{"duration":0,"failed":false}],[":tests/voice-transcribe.test.ts",{"duration":0,"failed":true}],[":tests/turn-active-marker.test.ts",{"duration":0,"failed":false}],[":tests/inline-keyboard-callbacks.test.ts",{"duration":0,"failed":false}],[":tests/issues-watcher.test.ts",{"duration":0,"failed":false}],[":tests/real-gateway-f1-ladder-integrity.test.ts",{"duration":0,"failed":false}],[":tests/gateway-disconnect-flush.test.ts",{"duration":0,"failed":false}],[":tests/reply-terminal-reaction.test.ts",{"duration":0,"failed":false}],[":tests/turn-flush-dedup-controller.test.ts",{"duration":0,"failed":false}],[":tests/real-gateway-f3-late-card.test.ts",{"duration":0,"failed":false}],[":tests/status-reactions-allowed-filter.test.ts",{"duration":0,"failed":false}],[":tests/pty-tail-real-fixture.test.ts",{"duration":0,"failed":false}],[":tests/real-gateway.smoke.test.ts",{"duration":0,"failed":false}],[":tests/harness-parse-mode-validation.test.ts",{"duration":0,"failed":false}],[":tests/inbound-coalesce.test.ts",{"duration":0,"failed":false}],[":tests/gateway-update-placeholder-dispatch.test.ts",{"duration":0,"failed":true}],[":tests/interrupt-marker.test.ts",{"duration":0,"failed":true}],[":tests/dm-command-gate.test.ts",{"duration":0,"failed":false}],[":tests/auto-fallback-dispatcher.e2e.test.ts",{"duration":0,"failed":false}],[":tests/sync-chat-running-subagents.test.ts",{"duration":0,"failed":false}],[":tests/subagents-schema-init-order.test.ts",{"duration":0,"failed":true}],[":tests/draft-transport.test.ts",{"duration":0,"failed":false}],[":gateway/access-validator.test.ts",{"duration":0,"failed":false}],[":registry/turns-schema.test.ts",{"duration":0,"failed":true}],[":tests/update-factory-edited-and-reactions.test.ts",{"duration":0,"failed":false}],[":tests/gateway-no-reply-single-emit.test.ts",{"duration":0,"failed":false}],[":tests/real-gateway-f2-instant-draft.test.ts",{"duration":0,"failed":false}],[":tests/gateway-boot-marker-clear.test.ts",{"duration":0,"failed":false}],[":tests/fleet-state-watcher.test.ts",{"duration":0,"failed":false}],[":tests/ipc-server-validate-update-placeholder.test.ts",{"duration":0,"failed":false}],[":tests/permission-title.test.ts",{"duration":0,"failed":false}],[":tests/slot-banner.test.ts",{"duration":0,"failed":false}],[":tests/sticker-aliases.test.ts",{"duration":0,"failed":true}],[":tests/ipc-server-anonymous-refuse.test.ts",{"duration":0,"failed":false}],[":tests/ipc-server-validate-pty-partial.test.ts",{"duration":0,"failed":false}],[":channel-envelope-safety.test.ts",{"duration":0,"failed":false}],[":tests/bridge-anonymous-refuse.test.ts",{"duration":0,"failed":false}],[":tests/send-typing-action-validation.test.ts",{"duration":0,"failed":false}],[":tests/spawn-detached-cgroup-escape.test.ts",{"duration":0,"failed":false}],[":gateway/boot-sweep-filter.test.ts",{"duration":0,"failed":false}]]}

package/telegram-plugin/operator-events-history.ts

@@ -0,0 +1,94 @@
+/**
+ * operator-events-history.ts — short-lived in-memory history of recent
+ * OperatorEvents per agent, used by /status enrichment (Phase 4b).
+ *
+ * Design:
+ *  - Keyed by agent name.
+ *  - Stores the most recent event per agent.
+ *  - Events expire after EVENT_TTL_MS (1 hour by default).
+ *  - Zero persistence — intentionally volatile; survives gateway restarts
+ *    only as long as the process lives. /status shows last event if fresh.
+ */
+
+import type { OperatorEvent } from './operator-events.js'
+
+export const EVENT_TTL_MS = 60 * 60 * 1000 // 1 hour
+
+interface StoredEvent {
+  event: OperatorEvent
+  storedAt: number
+}
+
+const store = new Map<string, StoredEvent>()
+
+/**
+ * Record the most recent OperatorEvent for an agent.
+ * Overwrites any previous event for the same agent.
+ */
+export function recordOperatorEvent(event: OperatorEvent, now: number = Date.now()): void {
+  store.set(event.agent, { event, storedAt: now })
+}
+
+/**
+ * Retrieve the most recent OperatorEvent for an agent, or null if
+ * none exists or the stored event has expired.
+ */
+export function getLastOperatorEvent(
+  agent: string,
+  now: number = Date.now(),
+  ttlMs: number = EVENT_TTL_MS,
+): OperatorEvent | null {
+  const stored = store.get(agent)
+  if (!stored) return null
+  if (now - stored.storedAt > ttlMs) {
+    store.delete(agent)
+    return null
+  }
+  return stored.event
+}
+
+/**
+ * Clear all stored events (for testing).
+ */
+export function clearOperatorEventHistory(): void {
+  store.clear()
+}
+
+/**
+ * Format a last-event line for /status display.
+ * Returns an HTML string like "  last: 🔑 credentials-expired (2m ago)"
+ * or null if no fresh event exists.
+ */
+export function formatLastEventLine(
+  agent: string,
+  now: number = Date.now(),
+  ttlMs: number = EVENT_TTL_MS,
+): string | null {
+  const ev = getLastOperatorEvent(agent, now, ttlMs)
+  if (!ev) return null
+
+  const ageSec = Math.floor((now - ev.firstSeenAt.getTime()) / 1000)
+  let age: string
+  if (ageSec < 60) {
+    age = `${ageSec}s ago`
+  } else if (ageSec < 3600) {
+    age = `${Math.floor(ageSec / 60)}m ago`
+  } else {
+    age = `${Math.floor(ageSec / 3600)}h ago`
+  }
+
+  const kindIcon = EVENT_KIND_ICON[ev.kind] ?? '⚪'
+  return `  <i>last: ${kindIcon} ${ev.kind} (${age})</i>`
+}
+
+const EVENT_KIND_ICON: Record<string, string> = {
+  'credentials-expired': '🔑',
+  'credentials-invalid': '🔑',
+  'credit-exhausted': '💳',
+  'quota-exhausted': '⚠️',
+  'rate-limited': '🚦',
+  'agent-crashed': '💥',
+  'agent-restarted-unexpectedly': '🔄',
+  'unknown-4xx': '⚠️',
+  'unknown-5xx': '🔥',
+}

package/telegram-plugin/operator-events.fixtures.json

@@ -0,0 +1,161 @@
+{
+  "_comment": "Captured error shapes per OperatorEventKind. Real API keys/IDs have been scrubbed.",
+
+  "credentials-expired": [
+    {
+      "_source": "Anthropic API — 401 with authentication_error + expired hint",
+      "status": 401,
+      "error": {
+        "type": "authentication_error",
+        "message": "Your API key has expired. Please refresh your credentials."
+      }
+    },
+    {
+      "_source": "Anthropic API — authentication_error at top level with expired token message",
+      "type": "authentication_error",
+      "message": "OAuth token expired, please re-authenticate to continue"
+    }
+  ],
+
+  "credentials-invalid": [
+    {
+      "_source": "Anthropic API — 401 with invalid_api_key",
+      "status": 401,
+      "error": {
+        "type": "invalid_api_key",
+        "message": "Invalid API key provided. Check your API key and try again."
+      }
+    },
+    {
+      "_source": "Anthropic API — authentication_error without expired hint",
+      "status": 401,
+      "error": {
+        "type": "authentication_error",
+        "message": "Your API key is not valid. Please check your API key."
+      }
+    },
+    {
+      "_source": "Top-level invalid_api_key",
+      "type": "invalid_api_key",
+      "message": "Invalid API key"
+    }
+  ],
+
+  "credit-exhausted": [
+    {
+      "_source": "Anthropic API — 402 credit_balance_too_low",
+      "status": 402,
+      "error": {
+        "type": "credit_balance_too_low",
+        "message": "Your credit balance is too low to complete this request. Please add credits at console.anthropic.com."
+      }
+    },
+    {
+      "_source": "Top-level credit_balance_too_low",
+      "type": "credit_balance_too_low",
+      "message": "credit balance insufficient"
+    }
+  ],
+
+  "quota-exhausted": [
+    {
+      "_source": "Anthropic API — 529 overloaded_error (Claude Code converts to quota-exhausted)",
+      "status": 529,
+      "error": {
+        "type": "overloaded_error",
+        "message": "Overloaded"
+      }
+    },
+    {
+      "_source": "Synthetic — set by session-tail after repeated 429 + slot exhaustion",
+      "type": "overloaded_error",
+      "message": "Service overloaded, usage limits reached"
+    }
+  ],
+
+  "rate-limited": [
+    {
+      "_source": "Anthropic API — 429 rate_limit_error",
+      "status": 429,
+      "error": {
+        "type": "rate_limit_error",
+        "message": "Rate limit exceeded. Please retry after 60 seconds."
+      }
+    },
+    {
+      "_source": "Top-level rate_limit_error",
+      "type": "rate_limit_error",
+      "message": "rate limit exceeded"
+    }
+  ],
+
+  "agent-crashed": [
+    {
+      "_source": "Synthetic — emitted by IPC bridge when Claude child exits nonzero",
+      "type": "agent-crashed",
+      "message": "Claude process exited with code 1"
+    },
+    {
+      "_source": "Synthetic — emitted by session-tail on socket disconnect",
+      "code": "agent-crashed",
+      "message": "IPC socket disconnected unexpectedly"
+    }
+  ],
+
+  "agent-restarted-unexpectedly": [
+    {
+      "_source": "Synthetic — emitted by gateway boot-banner diff when uptime drops unexpectedly",
+      "type": "agent-restarted-unexpectedly",
+      "message": "Agent restarted while user session was active"
+    },
+    {
+      "_source": "Synthetic — systemd watchdog detects unexpected restart",
+      "code": "agent-restarted-unexpectedly",
+      "message": "systemd unit restarted outside of operator request"
+    }
+  ],
+
+  "unknown-4xx": [
+    {
+      "_source": "Novel 4xx not matching any known Anthropic error type",
+      "status": 422,
+      "error": {
+        "type": "invalid_request_error",
+        "message": "Unrecognised field in request body"
+      }
+    },
+    {
+      "_source": "Bare 403 with no recognised type",
+      "status": 403,
+      "message": "Forbidden"
+    },
+    {
+      "_source": "null input — always unknown-4xx",
+      "_value": null
+    },
+    {
+      "_source": "Completely empty object",
+      "_value": {}
+    },
+    {
+      "_source": "Non-object string",
+      "_value": "something went wrong"
+    }
+  ],
+
+  "unknown-5xx": [
+    {
+      "_source": "500 with no recognised type",
+      "status": 500,
+      "message": "Internal Server Error"
+    },
+    {
+      "_source": "503 service unavailable",
+      "status": 503,
+      "error": {
+        "type": "service_unavailable",
+        "message": "Service temporarily unavailable"
+      }
+    }
+  ]
+}