switchroom 0.19.26 → 0.19.28

package/telegram-plugin/gateway/status-pin-store.ts

@@ -70,6 +70,27 @@ export interface PersistedStatusPin {
    *  unpin fails (flood-wait exhausted / transient 5xx); the row is retained
    *  for retry until BOOT_UNPIN_MAX_ATTEMPTS, then forfeited. Absent = 0. */
   attempts?: number
+  /**
+   * Wall-clock ms the pin was FIRST claimed for this key+message (#3810).
+   *
+   * Why the store needs it: the mid-session `wk:` reaper folds in rows that
+   * exist on disk but have NO in-memory claim (`storeOnlyWorkerPinCandidates`).
+   * The store used to persist no timestamp, so those candidates were stamped
+   * `pinnedAt = now` on every pass — deliberately, to avoid a spurious unpin of
+   * a pin whose real age was unknown, but with the side effect that the TTL
+   * gate could NEVER fire for a store orphan. Combined with a `wk:<agentId>`
+   * whose turnsDb row is gone (verdict `'unknown'`, never `'terminal'`), such a
+   * row was never mid-session reaped AT ALL — it waited for the next boot, the
+   * exact "stale pin glued to the top of the chat" failure the reaper exists to
+   * prevent.
+   *
+   * Recording the real claim time removes the guess: a store orphan now ages
+   * honestly and the ordinary TTL applies. Optional so a v1/v2 snapshot still
+   * loads; a row without it keeps the old conservative `now` stamp (terminal-
+   * only reaping), which self-clears within one restart since every write from
+   * this version onward carries the field.
+   */
+  pinnedAt?: number
 }
 
 /** How many boots may retry a failing boot-cleanup unpin before the row is
@@ -80,13 +101,18 @@ export interface PersistedStatusPin {
 export const BOOT_UNPIN_MAX_ATTEMPTS = 5
 
 /** Envelope version. v1 had no `pending` field; a v1 row loads as a confirmed
- *  pin (pending undefined). v2 adds the optional `pending` flag. Both load
- *  fail-open — an unknown/newer version yields []. */
+ *  pin (pending undefined). v2 adds the optional `pending` flag. v3 adds the
+ *  optional `pinnedAt` claim timestamp (#3810). All load fail-open — an
+ *  unknown/newer version yields []. Every field added since v1 is optional, so
+ *  the versions are mutually readable and a downgrade degrades rather than
+ *  breaks. */
 interface SnapshotEnvelope {
-  v: 1 | 2
+  v: 1 | 2 | 3
   pins: PersistedStatusPin[]
 }
 
+const SNAPSHOT_VERSIONS = new Set([1, 2, 3])
+
 function isPinRow(x: unknown): x is PersistedStatusPin {
   if (x == null || typeof x !== 'object') return false
   const o = x as Record<string, unknown>
@@ -98,7 +124,8 @@ function isPinRow(x: unknown): x is PersistedStatusPin {
     typeof o.messageId === 'number' &&
     (o.pending === undefined || typeof o.pending === 'boolean') &&
     (o.expiresAt === undefined || typeof o.expiresAt === 'number') &&
-    (o.attempts === undefined || typeof o.attempts === 'number')
+    (o.attempts === undefined || typeof o.attempts === 'number') &&
+    (o.pinnedAt === undefined || typeof o.pinnedAt === 'number')
   )
 }
 
@@ -127,7 +154,7 @@ export function loadStatusPins(
   }
   if (parsed == null || typeof parsed !== 'object') return []
   const env = parsed as Record<string, unknown>
-  if ((env.v !== 1 && env.v !== 2) || !Array.isArray(env.pins)) return []
+  if (typeof env.v !== 'number' || !SNAPSHOT_VERSIONS.has(env.v) || !Array.isArray(env.pins)) return []
   return env.pins.filter(isPinRow)
 }
 
@@ -143,7 +170,7 @@ export function persistStatusPins(
   snapshot: readonly PersistedStatusPin[],
   log: (line: string) => void = (l) => process.stderr.write(l),
 ): void {
-  const env: SnapshotEnvelope = { v: 2, pins: [...snapshot] }
+  const env: SnapshotEnvelope = { v: 3, pins: [...snapshot] }
   const tmp = path + '.tmp'
   try {
     fs.writeFileSync(tmp, JSON.stringify(env))
@@ -427,16 +454,34 @@ export function reconcileAndPersistStatusPin(args: {
   /** Execute the real pin/unpin; returns the confirmed message id (pin) or
    *  null (cleared). Must never throw — API errors are swallowed inside. */
   applyPin: () => Promise<{ messageId: number } | null>
+  /** Clock for the `pinnedAt` stamp (#3810). Defaults to wall clock. */
+  now?: number
   log?: (line: string) => void
 }): Promise<{ messageId: number } | null> {
   const { path, fs, pinKey, chatId, op } = args
   const log = args.log ?? ((l: string) => process.stderr.write(l))
+  const now = args.now ?? Date.now()
+
+  /**
+   * The claim age to persist for `messageId`. Carried forward from the row
+   * already on disk when it names the SAME message (a steady-state re-write
+   * must not reset the age and hand a stale pin a fresh TTL lease); stamped
+   * `now` for a genuinely new pin. Mirrors the in-memory `pinnedAt` rule in
+   * `status-pin-retarget.ts` so disk and memory can't disagree about age.
+   */
+  const claimAge = (messageId: number): number => {
+    const existing = loadStatusPins(path, fs).find((p) => p.pinKey === pinKey)
+    return existing?.messageId === messageId && existing.pinnedAt != null
+      ? existing.pinnedAt
+      : now
+  }
 
   // Hold the per-path lock across the WHOLE op so no other writer (a banner
   // persist, or another key's reconcile) can rebuild/overwrite the file during
   // the applyPin await window and drop this key's pending/confirmed row.
   return withStoreLock(path, async () => {
     if (op.kind === 'pin') {
+      const pinnedAt = claimAge(op.messageId)
       // Persist INTENT first, marked pending — BEFORE the pin API call. If we
       // crash after the pin lands but before the confirm rewrite, this pending
       // record is what boot cleanup uses to unpin the orphan.
@@ -444,7 +489,7 @@ export function reconcileAndPersistStatusPin(args: {
         path,
         fs,
         pinKey,
-        { pinKey, chatId, messageId: op.messageId, pending: true },
+        { pinKey, chatId, messageId: op.messageId, pending: true, pinnedAt },
         log,
       )
       const next = await args.applyPin()
@@ -459,7 +504,7 @@ export function reconcileAndPersistStatusPin(args: {
         path,
         fs,
         pinKey,
-        { pinKey, chatId, messageId: next.messageId },
+        { pinKey, chatId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
         log,
       )
       return next
@@ -488,11 +533,15 @@ export function reconcileAndPersistStatusPin(args: {
     if (next == null) {
       applyStatusPinRow(path, fs, pinKey, null, log)
     } else {
+      // Steady-state noop-clear (the worker feed calls syncPin on every edit):
+      // the pin is still up, so the row is preserved AND so is its original
+      // claim age — re-stamping it here would hand a genuinely old pin a fresh
+      // TTL lease on every feed edit and make it immortal (#3810).
       applyStatusPinRow(
         path,
         fs,
         pinKey,
-        { pinKey, chatId, messageId: next.messageId },
+        { pinKey, chatId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
         log,
       )
     }

package/telegram-plugin/gateway/worker-pin-reaper.ts

@@ -99,6 +99,47 @@ export interface StoreOrphanRow {
   pending?: boolean
   /** Time-scoped `tool:` pin. Never a worker orphan; excluded. */
   expiresAt?: number
+  /** Wall-clock ms the claim was first taken (#3810). Absent on a pre-v3 row. */
+  pinnedAt?: number
+}
+
+/**
+ * The live worker-activity feed's liveness probe, as the reaper needs it.
+ * Nullable ON PURPOSE — see `groupPinStatus`.
+ */
+export interface WorkerFeedLivenessProbe {
+  hasRunningInFeed(feedKey: string): boolean
+}
+
+/**
+ * Verdict for a GROUP-level pin key (`wk:group:<feedKey>`, the shape the
+ * coalesced feed produces since #3207). Vouched off the live feed rather than
+ * the sub-agent registry: `group:<feedKey>` is not a jsonl agent id.
+ *
+ * #3811 — the gateway used to write this as
+ * `feed?.hasRunningInFeed(key) ? 'running' : 'terminal'`, which collapses two
+ * very different facts into one verdict:
+ *
+ *   - "a live feed exists and affirmatively says this group is done"  → terminal
+ *   - "there is NO feed object to ask"                                → nothing known
+ *
+ * A `'terminal'` verdict reaps at ANY age — it bypasses the TTL exemption
+ * entirely — so the second case could unpin a GENUINELY LIVE group pin out from
+ * under running workers during a window where `workerActivityFeed` is null
+ * (teardown/rebuild on a bridge flap, and every tick before the first bridge
+ * connect). That is the inverse of the stale-pin bug and much harder to notice:
+ * the user loses the pin for work that is still in flight.
+ *
+ * "No feed to ask" is `'unknown'`: the TTL still applies, so a truly stale pin
+ * is still bounded, but nothing is reaped on the strength of a question we
+ * never got to ask.
+ */
+export function groupPinStatus(
+  feed: WorkerFeedLivenessProbe | null | undefined,
+  feedKey: string,
+): WorkerRegistryStatus {
+  if (feed == null) return 'unknown'
+  return feed.hasRunningInFeed(feedKey) ? 'running' : 'terminal'
 }
 
 /**
@@ -110,13 +151,21 @@ export interface StoreOrphanRow {
  * it too, group-safely (per-message unpin of a bot-tracked row — never an
  * unpin-all, never a human pin).
  *
- * The store persists NO timestamp, so a store-only candidate carries
- * `pinnedAt = now`: only a TERMINAL registry verdict can reap it. It is never
- * TTL-reaped (no trustworthy age to age it out), never touched while its worker
- * is `running`, and it is always a `wk:` row (the documented leak class).
+ * AGE (#3810). A v3+ row carries the real `pinnedAt` the claim was taken at, so
+ * a store orphan ages honestly and the ordinary TTL gate applies to it exactly
+ * as it does to an in-memory claim. A pre-v3 row has no timestamp and keeps the
+ * old conservative stamp (`pinnedAt = now`, i.e. TERMINAL-verdict reaping only)
+ * rather than guessing an age — but that case now self-clears within one
+ * restart, because every write from v3 onward records the field.
+ *
+ * Why the honest age matters: `storeOnlyWorkerPinCandidates` was the ONLY net
+ * for a `wk:` row with no in-memory claim, and with the TTL permanently
+ * disabled a `wk:<agentId>` row whose turnsDb row had been pruned (verdict
+ * `'unknown'`, never `'terminal'`) was never mid-session reaped at all — its
+ * stale `🛠 Worker` pin sat at the top of the chat until the next gateway boot.
+ *
  * `pending` rows (pin API in-flight) and time-scoped `tool:` rows are excluded,
- * as are rows already tracked in memory (the in-memory reaper owns those, with
- * a real `pinnedAt` that can drive the TTL).
+ * as are rows already tracked in memory (the in-memory reaper owns those).
  */
 export function storeOnlyWorkerPinCandidates(args: {
   rows: Iterable<StoreOrphanRow>
@@ -133,7 +182,7 @@ export function storeOnlyWorkerPinCandidates(args: {
     out.push({
       pinKey: r.pinKey,
       chatId: r.chatId,
-      pinnedAt: args.now,
+      pinnedAt: r.pinnedAt ?? args.now,
       messageId: r.messageId,
     })
   }

package/telegram-plugin/inline-keyboard-callbacks.ts

@@ -34,6 +34,7 @@ import {
   type AnyButton,
   type ButtonValidationError,
 } from './telegram-button-constraints.js'
+import type { RetryCallOpts } from './retry-api-call.js'
 
 /** Prefix used to namespace agent-emitted callback_data on the wire. */
 export const AGENT_CALLBACK_PREFIX = 'agent:'
@@ -496,6 +497,11 @@ export function validateAndWrapAgentKeyboard(
  * Minimal callback-context shape the helper needs. Real grammy
  * `Context` satisfies this; tests can implement a lightweight fake
  * without dragging the grammy types in.
+ *
+ * `editMessageReplyMarkup` / `reply` are REQUIRED (not optional) because
+ * they are the two rungs of the repaint-failure ladder (#3891). A context
+ * that cannot disarm its own keyboard is exactly the context that leaves
+ * the operator tapping a corpse, so the type refuses to construct one.
  */
 export interface FinalizeCallbackContext {
   answerCallbackQuery: (
@@ -505,8 +511,68 @@ export interface FinalizeCallbackContext {
     text: string | { markdown: string },
     opts?: Record<string, unknown>,
   ) => Promise<unknown>
+  /** Rung 2 of the ladder — strip `reply_markup` without touching the body. */
+  editMessageReplyMarkup: (
+    opts?: Record<string, unknown>,
+  ) => Promise<unknown>
+  /** Rung 3 of the ladder — a fresh in-channel message when the card is unfixable. */
+  reply: (
+    text: string,
+    opts?: Record<string, unknown>,
+  ) => Promise<unknown>
+  /**
+   * The raw `callback_query` update, typed `unknown` on purpose: this
+   * helper only reads `message.chat.id` off it (defensively, at runtime)
+   * to scope the retry policy's flood window, and a structural type here
+   * would have to track grammy's `MaybeInaccessibleMessage` union for no
+   * benefit. See {@link extractCallbackChatId}.
+   */
+  callbackQuery?: unknown
+}
+
+/**
+ * Best-effort `chat_id` for the retry policy's scope-precise flood window.
+ *
+ * Returns `undefined` rather than throwing on ANY unexpected shape — a
+ * missing scope degrades the 429 to a `global` window (still recorded, still
+ * respected), whereas a throw here would break the tap path outright.
+ */
+export function extractCallbackChatId(callbackQuery: unknown): string | undefined {
+  const msg = (callbackQuery as { message?: unknown } | null | undefined)?.message
+  const chat = (msg as { chat?: unknown } | null | undefined)?.chat
+  const id = (chat as { id?: unknown } | null | undefined)?.id
+  if (typeof id === 'number' && Number.isFinite(id)) return String(id)
+  if (typeof id === 'string' && id !== '') return id
+  return undefined
 }
 
+/**
+ * The retry/flood seam every leg of {@link finalizeCallback} transits.
+ *
+ * Structurally the gateway's `robustApiCall` (chat-lock → send gate →
+ * `createRetryApiCall`). Declared here rather than imported as
+ * `typeof robustApiCall` so this module keeps its zero-dependency-on-gateway
+ * shape and tests can hand in a counting fake.
+ */
+export type FinalizeApiCall = <T>(
+  fn: () => Promise<T>,
+  opts?: RetryCallOpts,
+) => Promise<T>
+
+/**
+ * Default notice posted when BOTH the body repaint and the keyboard strip
+ * fail — the card is still standing with live-looking buttons and the only
+ * honest thing left is to say so out loud, in the same chat.
+ *
+ * Deliberately a LITERAL plain string (no markdown, no entities): the most
+ * common cause of a repaint failure is Telegram rejecting the body's
+ * entities, so a rich fallback would be likely to fail the same way.
+ */
+export const DEAD_CARD_NOTICE =
+  '⚠️ Your tap was applied, but this card could not be updated. ' +
+  'The buttons on it are STALE — tapping them again will not change anything. ' +
+  'Scroll down for the outcome, or ask the agent to re-send the card.'
+
 export interface FinalizeCallbackOptions {
   /**
    * Toast text shown to the operator via `answerCallbackQuery`. Telegram
@@ -549,52 +615,167 @@ export interface FinalizeCallbackOptions {
    * actions that shell to the host CLI, operator-event dismiss, etc).
    */
   synthInbound?: () => void | Promise<void>
+  /**
+   * The retry/flood policy every Telegram call in this helper transits
+   * (#3891). REQUIRED — not optional-with-a-passthrough-default, because a
+   * passthrough default is exactly the bug: the tap path silently opted out
+   * of the one policy that records 429s, and nothing failed. Making it a
+   * mandatory field turns "did this call site wire the policy?" into a
+   * `tsc` error at all 8 call sites instead of a review checklist item.
+   *
+   * Pass the gateway's `robustApiCall`.
+   */
+  apiCall: FinalizeApiCall
   /** Logger seam for tests. Defaults to stderr. */
   log?: (line: string) => void
 }
 
+/**
+ * Rungs 2 and 3 of the repaint-failure ladder (#3891).
+ *
+ * Precondition: the body repaint already failed AFTER the retry policy had
+ * its go, so the card is standing with a live `reply_markup` over a decision
+ * that is already resolved. Getting the keyboard OFF is what matters here —
+ * the stale body text is cosmetic by comparison, an un-disarmed keyboard is
+ * an invitation to re-tap.
+ *
+ * Never throws. A throw would land back in `finalizeCallback` and could skip
+ * invariant 3 (the model wake-up), trading a confusing card for a wedged turn.
+ */
+async function disarmDeadCard(
+  ctx: FinalizeCallbackContext,
+  apiCall: FinalizeApiCall,
+  scope: RetryCallOpts,
+  log: (line: string) => void,
+): Promise<void> {
+  try {
+    await apiCall(
+      () => ctx.editMessageReplyMarkup({ reply_markup: { inline_keyboard: [] } }),
+      { ...scope, verb: 'editMessageReplyMarkup' },
+    )
+    log(
+      'finalizeCallback: repaint failed but the keyboard was stripped — ' +
+        'the card shows stale text and is no longer tappable\n',
+    )
+    return
+  } catch (err) {
+    log(`finalizeCallback: editMessageReplyMarkup fallback failed: ${(err as Error).message}\n`)
+  }
+  // Rung 3 — the card cannot be changed at all. Say so out loud rather than
+  // leave a live-looking keyboard with no explanation behind it.
+  try {
+    await apiCall(
+      () => ctx.reply(DEAD_CARD_NOTICE, { link_preview_options: { is_disabled: true } }),
+      { ...scope, verb: 'sendMessage' },
+    )
+  } catch (err) {
+    log(`finalizeCallback: dead-card notice failed: ${(err as Error).message}\n`)
+  }
+}
+
 /**
  * Apply the three-invariant finalize pattern. See module docstring
  * above for design rationale.
  *
- * Order: ack → edit → synth. The ack is fired-and-forgotten (so a slow
- * Telegram API doesn't delay the visible state change), but the edit
- * is awaited so `synthInbound` doesn't race ahead of the operator's
- * visual confirmation. Each step's error is logged + swallowed —
- * partial success is preferred to "tap looked dead AND the model
- * stayed stuck" full failure.
+ * Order: ack → edit (→ disarm ladder on failure) → synth. The ack is
+ * fired-and-forgotten (so a slow Telegram API doesn't delay the visible
+ * state change), but the edit is awaited so `synthInbound` doesn't race
+ * ahead of the operator's visual confirmation. Each step's error is
+ * logged + swallowed — partial success is preferred to "tap looked dead
+ * AND the model stayed stuck" full failure.
+ *
+ * ── #3891: every leg goes through `opts.apiCall` ──────────────────────
+ * Both legs used to call grammy's CONTEXT methods raw. That bypassed
+ * `robustApiCall` / `createRetryApiCall` entirely, with two consequences
+ * observed together in one live incident:
+ *
+ *   1. A 429 earned on the tap path never reached `onFloodWait`, so it was
+ *      never folded into `429-ledger.json` and never opened a window in
+ *      `flood-windows.json`. Every consumer of that state — the send gate,
+ *      `switchroom doctor`'s flood-pressure classifier, the wedge-watchdog's
+ *      Esc suppression — was reasoning about the bot's 429 pressure from a
+ *      picture with the operator's own taps cut out of it.
+ *   2. With no retry, ONE transient failure at the edit left the card
+ *      holding its `reply_markup`: still live, still tappable, decision
+ *      already resolved. The operator taps a corpse and gets a bare
+ *      "doesn't want to proceed" with no explanation.
+ *
+ * Both legs are tagged `priorityClass: 'critical'` and carry no
+ * `messageId`/`editPayload`. That is deliberate: `critical` is the one
+ * class the send gate never SHEDS, and omitting the edit-coalescing keys
+ * keeps the gate from treating a finalize repaint as a droppable
+ * last-write-wins card edit. A finalize repaint is the operator's only
+ * visual confirmation that a human-in-the-loop decision resolved — shedding
+ * or coalescing it away IS the incident this fix exists to stop.
+ *
+ * ── The repaint-failure ladder ────────────────────────────────────────
+ * `robustApiCall` already swallows the two BENIGN edit failures
+ * (MESSAGE_NOT_MODIFIED, MESSAGE_TO_EDIT_NOT_FOUND → `undefined`) and
+ * retries transient ones. So anything reaching the catch below is a real,
+ * post-retry failure — and the card is still standing with live buttons.
+ * Three rungs, cheapest and most-likely-to-work first:
+ *
+ *   1. body repaint (`editMessageText`)      — the good outcome.
+ *   2. keyboard strip (`editMessageReplyMarkup`) — the body is what usually
+ *      breaks (entity/markdown parse, length); an empty-keyboard edit sends
+ *      no body at all, so it survives the failure mode that killed rung 1.
+ *      Card keeps stale TEXT but is no longer tappable.
+ *   3. plain-text notice (`reply`)           — the card is unfixable, so
+ *      say so in-channel rather than let it keep presenting as live.
+ *
+ * Every rung is best-effort and logged; none of them may block invariant 3.
+ * NOTE the ladder changes no approval semantics — it only repaints and
+ * narrates. The decision was resolved by the caller before we were called.
  */
 export async function finalizeCallback(
   ctx: FinalizeCallbackContext,
   opts: FinalizeCallbackOptions,
 ): Promise<void> {
   const log = opts.log ?? ((line: string) => process.stderr.write(line))
+  const apiCall = opts.apiCall
+  const chatId = extractCallbackChatId(ctx.callbackQuery)
+  const scope: RetryCallOpts = {
+    ...(chatId != null ? { chat_id: chatId } : {}),
+    priorityClass: 'critical',
+  }
   // Invariant 1 — toast. Fire-and-forget; we don't want a slow
   // answerCallbackQuery round-trip to delay the message edit.
-  void ctx.answerCallbackQuery({
-    text: opts.ackText,
-    ...(opts.alert ? { show_alert: true } : {}),
-  }).catch((err: unknown) => {
+  void apiCall(
+    () =>
+      ctx.answerCallbackQuery({
+        text: opts.ackText,
+        ...(opts.alert ? { show_alert: true } : {}),
+      }),
+    { ...scope, verb: 'answerCallbackQuery' },
+  ).catch((err: unknown) => {
     log(`finalizeCallback: answerCallbackQuery failed: ${(err as Error).message}\n`)
   })
   // Invariant 2 — strip keyboard + append status line, atomic edit.
+  let repainted = true
   try {
-    await ctx.editMessageText(
-      opts.literalText ? opts.newText : { markdown: opts.newText },
-      {
-        reply_markup: { inline_keyboard: [] },
-        // Default link_preview_options off — most finalized cards don't
-        // benefit from preview cards, and a stale preview survives the
-        // edit otherwise.
-        link_preview_options: { is_disabled: true },
-      },
+    await apiCall(
+      () =>
+        ctx.editMessageText(
+          opts.literalText ? opts.newText : { markdown: opts.newText },
+          {
+            reply_markup: { inline_keyboard: [] },
+            // Default link_preview_options off — most finalized cards don't
+            // benefit from preview cards, and a stale preview survives the
+            // edit otherwise.
+            link_preview_options: { is_disabled: true },
+          },
+        ),
+      { ...scope, verb: 'editMessageText' },
     )
   } catch (err) {
     // MESSAGE_NOT_MODIFIED (text didn't change) and MESSAGE_TO_EDIT_NOT_FOUND
-    // (operator already deleted the card) are both benign. Other failures
-    // log + continue — we still want synthInbound to run.
+    // (operator already deleted the card) are both benign AND are already
+    // swallowed inside the retry policy, so they never land here. Anything
+    // that does is a real failure that left the keyboard standing.
+    repainted = false
     log(`finalizeCallback: editMessageText failed: ${(err as Error).message}\n`)
   }
+  if (!repainted) await disarmDeadCard(ctx, apiCall, scope, log)
   // Invariant 3 — model wake-up (when applicable).
   if (opts.synthInbound != null) {
     try {

package/telegram-plugin/llm-error-present.ts

@@ -34,6 +34,12 @@ import {
   parseResetTime,
 } from './model-unavailable.js'
 import { classify429Detail } from './throttle-tier.js'
+import {
+  PROVIDER_CREDIT_REGISTRY,
+  attributeProvider,
+  describeProviderCreditRemedy,
+  type ProviderCreditEntry,
+} from './provider-credit.js'
 import { classifyClaudeError } from './operator-events.js'
 import { stripRawErrorBytes, extractRequestId } from './raw-error-scrub.js'
 import { fmtLocalClock, tzAbbrev } from './shared/local-time.js'
@@ -48,10 +54,24 @@ export type LlmErrorKind =
   | 'quota_wall'
   | 'auth'
   | 'infra_misconfig'
+  /**
+   * A THIRD-PARTY model provider (OpenRouter / OpenAI / Perplexity) is out of
+   * credit. OPERATOR-actionable, never user-actionable — its `coreText` is a
+   * deliberately diagnosis-free one-liner and its recommendation names the
+   * vendor console, never `/auth`. Kept distinct from `quota_wall` (the
+   * Anthropic subscription window, which resets on a clock and is addressed by
+   * switching account slots); a vendor balance resets only when someone pays.
+   */
+  | 'provider_credit'
   | 'transient'
   | 'unknown'
 
-export type LlmErrorSource = 'anthropic' | 'litellm-local' | 'network'
+export type LlmErrorSource =
+  | 'anthropic'
+  | 'litellm-local'
+  /** A paid third-party model provider reached THROUGH the proxy (OpenRouter, OpenAI). */
+  | 'external-provider'
+  | 'network'
 
 export interface ParsedLlmError {
   kind: LlmErrorKind
@@ -65,6 +85,14 @@ export interface ParsedLlmError {
   model?: string
   /** Anthropic `request_id`, when present — the strongest dedup key. */
   requestId?: string
+  /**
+   * Registry id of the third-party provider the error was attributed to
+   * (`openrouter` / `openai` / `perplexity`), when `kind === 'provider_credit'`
+   * and the raw text named one. An ID only — never raw error text, never a
+   * credential — so the recommendation line can point at the right console
+   * without this struct ever carrying the raw string it was parsed from.
+   */
+  providerId?: string
   source: LlmErrorSource
   /** True when the harness is still retrying this error internally (mid-retry). */
   autoRetrying: boolean
@@ -96,7 +124,10 @@ const TRANSIENT_KINDS: ReadonlySet<LlmErrorKind> = new Set<LlmErrorKind>([
 
 /** The always-actionable kinds — NEVER silenced, even inside a collapse window. */
 export function isActionableKind(kind: LlmErrorKind): boolean {
-  return kind === 'auth' || kind === 'quota_wall'
+  // `provider_credit` joins the never-silenced set: an unpaid vendor balance
+  // does not clear on its own, so collapsing it inside a dedup window would
+  // hide the one signal that needs a human.
+  return kind === 'auth' || kind === 'quota_wall' || kind === 'provider_credit'
 }
 
 /**
@@ -118,6 +149,8 @@ export function parseLlmError(
     resetAt != null ? Math.max(0, resetAt.getTime() - Date.now()) : undefined
 
   const { kind, source } = classifyKindAndSource(text)
+  const providerId =
+    kind === 'provider_credit' ? (attributeProvider(text)?.id ?? undefined) : undefined
 
   // Retry / terminal semantics. Only the transient family can be mid-retry;
   // auth / quota_wall / model_unavailable are terminal by construction.
@@ -143,6 +176,7 @@ export function parseLlmError(
     ...(retryAfterMs != null ? { retryAfterMs } : {}),
     ...(model != null ? { model } : {}),
     ...(requestId != null ? { requestId } : {}),
+    ...(providerId != null ? { providerId } : {}),
     source,
     autoRetrying,
     terminal,
@@ -165,6 +199,15 @@ function classifyKindAndSource(text: string): { kind: LlmErrorKind; source: LlmE
 
   // 1. Auth — always terminal, always actionable.
   const claudeKind = classifyClaudeError({ message: text, type: text })
+  // 1b. THIRD-PARTY provider credit wall. Placed immediately after the auth
+  //     resolution and BEFORE isLitellmProxyLocal429 / detectModelUnavailable:
+  //     OpenAI reports an exhausted balance as HTTP 429 `insufficient_quota`,
+  //     and the 429 matchers below would otherwise swallow it as a transient
+  //     "retrying automatically" — the single most misleading thing to tell
+  //     someone whose provider has no money left.
+  if (claudeKind === 'provider-credit-exhausted') {
+    return { kind: 'provider_credit', source: 'external-provider' }
+  }
   if (claudeKind === 'proxy-misconfig') {
     return { kind: 'infra_misconfig', source: 'litellm-local' }
   }
@@ -216,6 +259,12 @@ function classifyKindAndSource(text: string): { kind: LlmErrorKind; source: LlmE
   return { kind: 'unknown', source: 'anthropic' }
 }
 
+/** Resolve a registry entry from a persisted `providerId`, or null. */
+function providerById(id: string | undefined): ProviderCreditEntry | null {
+  if (id == null) return null
+  return PROVIDER_CREDIT_REGISTRY.find(p => p.id === id) ?? null
+}
+
 function buildCoreText(kind: LlmErrorKind, source: LlmErrorSource): string {
   switch (kind) {
     case 'rate_limit':
@@ -230,6 +279,10 @@ function buildCoreText(kind: LlmErrorKind, source: LlmErrorSource): string {
       return 'Claude login needs re-authentication.'
     case 'infra_misconfig':
       return 'Local model-gateway auth misconfig (proxy fallback dropped the OAuth header).'
+    case 'provider_credit':
+      // Diagnosis-free by design — this string is the ceiling on what a
+      // non-operator may ever be told about a vendor billing wall.
+      return 'An upstream model provider is out of credit — the operator has been notified.'
     case 'transient':
       return source === 'network'
         ? "Couldn't reach Anthropic (network) — retrying automatically."
@@ -302,6 +355,10 @@ function buildRecommendation(parsed: ParsedLlmError, tz: string): string | undef
       // Operator-facing — NO re-auth wording (the login is fine). Points at the
       // real remedy: the local LiteLLM proxy fallback config.
       return '→ Fix the LiteLLM proxy fallback config (deployment missing OAuth passthrough).'
+    case 'provider_credit':
+      // Operator-facing. NO `/auth` wording: the credential is valid, the
+      // balance is empty, and rotating an Anthropic slot changes nothing.
+      return `→ ${describeProviderCreditRemedy(providerById(parsed.providerId))}`
     case 'quota_wall': {
       const reset = formatResetClock(parsed.resetAt, tz)
       return reset
@@ -377,6 +434,8 @@ function kindEmoji(kind: LlmErrorKind): string {
       return '🔑'
     case 'infra_misconfig':
       return '🛠️'
+    case 'provider_credit':
+      return '💳'
     case 'transient':
       return '🌐'
     case 'unknown':