switchroom 0.19.26 → 0.19.27

package/telegram-plugin/gateway/worker-pin-reaper.ts

@@ -99,6 +99,47 @@ export interface StoreOrphanRow {
   pending?: boolean
   /** Time-scoped `tool:` pin. Never a worker orphan; excluded. */
   expiresAt?: number
+  /** Wall-clock ms the claim was first taken (#3810). Absent on a pre-v3 row. */
+  pinnedAt?: number
+}
+
+/**
+ * The live worker-activity feed's liveness probe, as the reaper needs it.
+ * Nullable ON PURPOSE — see `groupPinStatus`.
+ */
+export interface WorkerFeedLivenessProbe {
+  hasRunningInFeed(feedKey: string): boolean
+}
+
+/**
+ * Verdict for a GROUP-level pin key (`wk:group:<feedKey>`, the shape the
+ * coalesced feed produces since #3207). Vouched off the live feed rather than
+ * the sub-agent registry: `group:<feedKey>` is not a jsonl agent id.
+ *
+ * #3811 — the gateway used to write this as
+ * `feed?.hasRunningInFeed(key) ? 'running' : 'terminal'`, which collapses two
+ * very different facts into one verdict:
+ *
+ *   - "a live feed exists and affirmatively says this group is done"  → terminal
+ *   - "there is NO feed object to ask"                                → nothing known
+ *
+ * A `'terminal'` verdict reaps at ANY age — it bypasses the TTL exemption
+ * entirely — so the second case could unpin a GENUINELY LIVE group pin out from
+ * under running workers during a window where `workerActivityFeed` is null
+ * (teardown/rebuild on a bridge flap, and every tick before the first bridge
+ * connect). That is the inverse of the stale-pin bug and much harder to notice:
+ * the user loses the pin for work that is still in flight.
+ *
+ * "No feed to ask" is `'unknown'`: the TTL still applies, so a truly stale pin
+ * is still bounded, but nothing is reaped on the strength of a question we
+ * never got to ask.
+ */
+export function groupPinStatus(
+  feed: WorkerFeedLivenessProbe | null | undefined,
+  feedKey: string,
+): WorkerRegistryStatus {
+  if (feed == null) return 'unknown'
+  return feed.hasRunningInFeed(feedKey) ? 'running' : 'terminal'
 }
 
 /**
@@ -110,13 +151,21 @@ export interface StoreOrphanRow {
  * it too, group-safely (per-message unpin of a bot-tracked row — never an
  * unpin-all, never a human pin).
  *
- * The store persists NO timestamp, so a store-only candidate carries
- * `pinnedAt = now`: only a TERMINAL registry verdict can reap it. It is never
- * TTL-reaped (no trustworthy age to age it out), never touched while its worker
- * is `running`, and it is always a `wk:` row (the documented leak class).
+ * AGE (#3810). A v3+ row carries the real `pinnedAt` the claim was taken at, so
+ * a store orphan ages honestly and the ordinary TTL gate applies to it exactly
+ * as it does to an in-memory claim. A pre-v3 row has no timestamp and keeps the
+ * old conservative stamp (`pinnedAt = now`, i.e. TERMINAL-verdict reaping only)
+ * rather than guessing an age — but that case now self-clears within one
+ * restart, because every write from v3 onward records the field.
+ *
+ * Why the honest age matters: `storeOnlyWorkerPinCandidates` was the ONLY net
+ * for a `wk:` row with no in-memory claim, and with the TTL permanently
+ * disabled a `wk:<agentId>` row whose turnsDb row had been pruned (verdict
+ * `'unknown'`, never `'terminal'`) was never mid-session reaped at all — its
+ * stale `🛠 Worker` pin sat at the top of the chat until the next gateway boot.
+ *
  * `pending` rows (pin API in-flight) and time-scoped `tool:` rows are excluded,
- * as are rows already tracked in memory (the in-memory reaper owns those, with
- * a real `pinnedAt` that can drive the TTL).
+ * as are rows already tracked in memory (the in-memory reaper owns those).
  */
 export function storeOnlyWorkerPinCandidates(args: {
   rows: Iterable<StoreOrphanRow>
@@ -133,7 +182,7 @@ export function storeOnlyWorkerPinCandidates(args: {
     out.push({
       pinKey: r.pinKey,
       chatId: r.chatId,
-      pinnedAt: args.now,
+      pinnedAt: r.pinnedAt ?? args.now,
       messageId: r.messageId,
     })
   }

package/telegram-plugin/llm-error-present.ts

@@ -34,6 +34,12 @@ import {
   parseResetTime,
 } from './model-unavailable.js'
 import { classify429Detail } from './throttle-tier.js'
+import {
+  PROVIDER_CREDIT_REGISTRY,
+  attributeProvider,
+  describeProviderCreditRemedy,
+  type ProviderCreditEntry,
+} from './provider-credit.js'
 import { classifyClaudeError } from './operator-events.js'
 import { stripRawErrorBytes, extractRequestId } from './raw-error-scrub.js'
 import { fmtLocalClock, tzAbbrev } from './shared/local-time.js'
@@ -48,10 +54,24 @@ export type LlmErrorKind =
   | 'quota_wall'
   | 'auth'
   | 'infra_misconfig'
+  /**
+   * A THIRD-PARTY model provider (OpenRouter / OpenAI / Perplexity) is out of
+   * credit. OPERATOR-actionable, never user-actionable — its `coreText` is a
+   * deliberately diagnosis-free one-liner and its recommendation names the
+   * vendor console, never `/auth`. Kept distinct from `quota_wall` (the
+   * Anthropic subscription window, which resets on a clock and is addressed by
+   * switching account slots); a vendor balance resets only when someone pays.
+   */
+  | 'provider_credit'
   | 'transient'
   | 'unknown'
 
-export type LlmErrorSource = 'anthropic' | 'litellm-local' | 'network'
+export type LlmErrorSource =
+  | 'anthropic'
+  | 'litellm-local'
+  /** A paid third-party model provider reached THROUGH the proxy (OpenRouter, OpenAI). */
+  | 'external-provider'
+  | 'network'
 
 export interface ParsedLlmError {
   kind: LlmErrorKind
@@ -65,6 +85,14 @@ export interface ParsedLlmError {
   model?: string
   /** Anthropic `request_id`, when present — the strongest dedup key. */
   requestId?: string
+  /**
+   * Registry id of the third-party provider the error was attributed to
+   * (`openrouter` / `openai` / `perplexity`), when `kind === 'provider_credit'`
+   * and the raw text named one. An ID only — never raw error text, never a
+   * credential — so the recommendation line can point at the right console
+   * without this struct ever carrying the raw string it was parsed from.
+   */
+  providerId?: string
   source: LlmErrorSource
   /** True when the harness is still retrying this error internally (mid-retry). */
   autoRetrying: boolean
@@ -96,7 +124,10 @@ const TRANSIENT_KINDS: ReadonlySet<LlmErrorKind> = new Set<LlmErrorKind>([
 
 /** The always-actionable kinds — NEVER silenced, even inside a collapse window. */
 export function isActionableKind(kind: LlmErrorKind): boolean {
-  return kind === 'auth' || kind === 'quota_wall'
+  // `provider_credit` joins the never-silenced set: an unpaid vendor balance
+  // does not clear on its own, so collapsing it inside a dedup window would
+  // hide the one signal that needs a human.
+  return kind === 'auth' || kind === 'quota_wall' || kind === 'provider_credit'
 }
 
 /**
@@ -118,6 +149,8 @@ export function parseLlmError(
     resetAt != null ? Math.max(0, resetAt.getTime() - Date.now()) : undefined
 
   const { kind, source } = classifyKindAndSource(text)
+  const providerId =
+    kind === 'provider_credit' ? (attributeProvider(text)?.id ?? undefined) : undefined
 
   // Retry / terminal semantics. Only the transient family can be mid-retry;
   // auth / quota_wall / model_unavailable are terminal by construction.
@@ -143,6 +176,7 @@ export function parseLlmError(
     ...(retryAfterMs != null ? { retryAfterMs } : {}),
     ...(model != null ? { model } : {}),
     ...(requestId != null ? { requestId } : {}),
+    ...(providerId != null ? { providerId } : {}),
     source,
     autoRetrying,
     terminal,
@@ -165,6 +199,15 @@ function classifyKindAndSource(text: string): { kind: LlmErrorKind; source: LlmE
 
   // 1. Auth — always terminal, always actionable.
   const claudeKind = classifyClaudeError({ message: text, type: text })
+  // 1b. THIRD-PARTY provider credit wall. Placed immediately after the auth
+  //     resolution and BEFORE isLitellmProxyLocal429 / detectModelUnavailable:
+  //     OpenAI reports an exhausted balance as HTTP 429 `insufficient_quota`,
+  //     and the 429 matchers below would otherwise swallow it as a transient
+  //     "retrying automatically" — the single most misleading thing to tell
+  //     someone whose provider has no money left.
+  if (claudeKind === 'provider-credit-exhausted') {
+    return { kind: 'provider_credit', source: 'external-provider' }
+  }
   if (claudeKind === 'proxy-misconfig') {
     return { kind: 'infra_misconfig', source: 'litellm-local' }
   }
@@ -216,6 +259,12 @@ function classifyKindAndSource(text: string): { kind: LlmErrorKind; source: LlmE
   return { kind: 'unknown', source: 'anthropic' }
 }
 
+/** Resolve a registry entry from a persisted `providerId`, or null. */
+function providerById(id: string | undefined): ProviderCreditEntry | null {
+  if (id == null) return null
+  return PROVIDER_CREDIT_REGISTRY.find(p => p.id === id) ?? null
+}
+
 function buildCoreText(kind: LlmErrorKind, source: LlmErrorSource): string {
   switch (kind) {
     case 'rate_limit':
@@ -230,6 +279,10 @@ function buildCoreText(kind: LlmErrorKind, source: LlmErrorSource): string {
       return 'Claude login needs re-authentication.'
     case 'infra_misconfig':
       return 'Local model-gateway auth misconfig (proxy fallback dropped the OAuth header).'
+    case 'provider_credit':
+      // Diagnosis-free by design — this string is the ceiling on what a
+      // non-operator may ever be told about a vendor billing wall.
+      return 'An upstream model provider is out of credit — the operator has been notified.'
     case 'transient':
       return source === 'network'
         ? "Couldn't reach Anthropic (network) — retrying automatically."
@@ -302,6 +355,10 @@ function buildRecommendation(parsed: ParsedLlmError, tz: string): string | undef
       // Operator-facing — NO re-auth wording (the login is fine). Points at the
       // real remedy: the local LiteLLM proxy fallback config.
       return '→ Fix the LiteLLM proxy fallback config (deployment missing OAuth passthrough).'
+    case 'provider_credit':
+      // Operator-facing. NO `/auth` wording: the credential is valid, the
+      // balance is empty, and rotating an Anthropic slot changes nothing.
+      return `→ ${describeProviderCreditRemedy(providerById(parsed.providerId))}`
     case 'quota_wall': {
       const reset = formatResetClock(parsed.resetAt, tz)
       return reset
@@ -377,6 +434,8 @@ function kindEmoji(kind: LlmErrorKind): string {
       return '🔑'
     case 'infra_misconfig':
       return '🛠️'
+    case 'provider_credit':
+      return '💳'
     case 'transient':
       return '🌐'
     case 'unknown':

package/telegram-plugin/model-unavailable.ts

@@ -790,6 +790,14 @@ export function resolveModelUnavailableFromOperatorEvent(
   ev: OperatorEventLike,
 ): ModelUnavailableDetection | null {
   const detail = typeof ev.detail === 'string' ? ev.detail : ''
+  // A THIRD-PARTY provider credit wall is never "model unavailable" in the
+  // Anthropic sense. Returning a detection here would route the event into the
+  // `modelUnavailable` branch of emitGatewayOperatorEvent, which renders the
+  // "⚠️ Model unavailable" card AND — for a `quota_exhausted` detection —
+  // fires `fireFleetAutoFallback`, benching a perfectly healthy Anthropic
+  // account slot because OpenRouter ran out of money. Return null so the event
+  // takes the plain `renderOperatorEvent` path with its provider-specific card.
+  if (ev.kind === 'provider-credit-exhausted') return null
   if (ev.kind === 'quota-exhausted') {
     return detectModelUnavailable(detail) ?? { kind: 'quota_exhausted', raw: detail }
   }

package/telegram-plugin/operator-events.ts

@@ -15,6 +15,11 @@
 import { escapeMarkdown } from './format.js'
 import { stripRawErrorBytes } from './raw-error-scrub.js'
 import { isLitellmProxyAuthMisconfig } from './model-unavailable.js'
+import {
+  attributeProvider,
+  describeProviderCreditRemedy,
+  detectProviderCreditExhaustion,
+} from './provider-credit.js'
 
 // ─── Taxonomy ────────────────────────────────────────────────────────────────
 
@@ -23,6 +28,16 @@ export type OperatorEventKind =
   | 'credentials-invalid'
   | 'proxy-misconfig'
   | 'credit-exhausted'
+  /**
+   * A THIRD-PARTY model provider (OpenRouter / OpenAI / Perplexity) reports no
+   * credit remaining — HTTP 402 `payment_required`, "insufficient credits",
+   * OpenAI's `insufficient_quota`, etc. Distinct from `credit-exhausted`,
+   * which is the ANTHROPIC balance and whose remedy is `/auth use <slot>`:
+   * rotating an Anthropic account slot buys zero OpenRouter tokens, so the two
+   * must not share a card. Operator-actionable (top up in the vendor console),
+   * never user-actionable — see {@link OPERATOR_ACTIONABLE_KINDS}.
+   */
+  | 'provider-credit-exhausted'
   | 'quota-exhausted'
   | 'rate-limited'
   | 'agent-crashed'
@@ -134,16 +149,38 @@ function classifyInner(raw: unknown): OperatorEventKind {
     return 'credentials-invalid'
   }
 
+  // The full scan surface for the provider-credit decision — same
+  // type/code/message union isLitellmProxyAuthMisconfig uses, because LiteLLM
+  // stamps the upstream vendor's wording in whichever field it has room for.
+  const creditScanText = `${errorType}\n${errorCode}\n${sdkCode}\n${message}`
+
+  // ANTHROPIC credit wall. Guarded with `attributeProvider(...) == null` so an
+  // OpenRouter/OpenAI/Perplexity error that happens to say "credit balance"
+  // does NOT get the Anthropic card, whose remedy (`/auth use <slot>`) is
+  // useless against a third-party balance. Unattributed credit-balance wording
+  // keeps its historical Anthropic reading — that is where it has always come
+  // from, and the alternative (guessing a vendor) sends the operator to the
+  // wrong console.
   if (
-    errorType === 'credit_balance_too_low' ||
-    errorCode === 'credit_balance_too_low' ||
-    sdkCode === 'credit_balance_too_low' ||
-    message.toLowerCase().includes('credit_balance_too_low') ||
-    message.toLowerCase().includes('credit balance')
+    (errorType === 'credit_balance_too_low' ||
+      errorCode === 'credit_balance_too_low' ||
+      sdkCode === 'credit_balance_too_low' ||
+      message.toLowerCase().includes('credit_balance_too_low') ||
+      message.toLowerCase().includes('credit balance')) &&
+    attributeProvider(creditScanText) == null
   ) {
     return 'credit-exhausted'
   }
 
+  // THIRD-PARTY provider credit wall — the OpenRouter 402 leak (#external-credit).
+  // Runs BEFORE the rate-limit branch on purpose: OpenAI signals an exhausted
+  // balance as HTTP *429* `insufficient_quota`, so a rate-limit-first ordering
+  // would classify a billing wall as a transient throttle and tell the operator
+  // to wait for a reset that will never come.
+  if (detectProviderCreditExhaustion(creditScanText, status) != null) {
+    return 'provider-credit-exhausted'
+  }
+
   if (
     errorType === 'rate_limit_error' ||
     errorCode === 'rate_limit_error' ||
@@ -316,6 +353,31 @@ export function renderOperatorEvent(ev: OperatorEvent): RenderResult {
         },
       }
 
+    // A third-party provider's balance, NOT Anthropic's. Deliberately carries
+    // NO `/auth use` / `/auth add` language (those rotate Anthropic account
+    // slots and buy zero OpenRouter tokens) and NO "🔐 Reauth" button (the key
+    // is valid; it is out of money). The remedy names the provider, the VAULT
+    // KEY NAME — never a value — and the console to act in. Dismiss-only.
+    case 'provider-credit-exhausted': {
+      const provider = attributeProvider(ev.detail)
+      const who = provider != null ? escapeMarkdown(provider.label) : 'An upstream model provider'
+      return {
+        text: [
+          `💳 **${provider != null ? `${who} credit exhausted` : 'Upstream provider credit exhausted'}** — hit by **${agent}**.`,
+          detail ? `_${detail}_` : '',
+          describeProviderCreditRemedy(provider),
+          `Anthropic account slots are unaffected — \`/auth use\` will NOT fix this.`,
+        ]
+          .filter(Boolean)
+          .join('\n'),
+        keyboard: {
+          inline_keyboard: [
+            [{ text: '❌ Dismiss', callback_data: `op:dismiss:${encodeURIComponent(ev.agent)}` }],
+          ],
+        },
+      }
+    }
+
     case 'quota-exhausted':
       // Canonical quota-exhausted text (migrated from auto-fallback.ts).
       // auto-fallback.ts's buildSwitchedMessage / buildAllExhaustedMessage
@@ -544,6 +606,11 @@ export const OPERATOR_ACTIONABLE_KINDS: ReadonlySet<OperatorEventKind> = new Set
   'credentials-expired',
   'credentials-invalid',
   'credit-exhausted',
+  // The OpenRouter/OpenAI/Perplexity 402 class. Only the operator can top up a
+  // vendor balance; an end user shown "insufficient credits" can do nothing but
+  // lose confidence. This membership is what routes it to the operator card and
+  // hands the user the brief plain-language notice instead.
+  'provider-credit-exhausted',
   'proxy-misconfig',
 ])
 

package/telegram-plugin/outbound-class.ts

@@ -0,0 +1,81 @@
+/**
+ * Outbound priority class, propagated to the grammY transformer stack.
+ *
+ * ── Why this module exists ────────────────────────────────────────────────
+ * The send gate (`send-gate.ts`) knows every outbound call's `priorityClass`
+ * — `cosmetic` (a repaint of a progress card), `useful`, or `critical` (the
+ * operator's actual answer). The edit-flood fuse (`edit-flood-fuse.ts`) is a
+ * grammY API transformer, and a transformer sees only `(method, payload)`.
+ * It therefore could NOT tell a liveness repaint apart from a reply, and had
+ * to treat every edit identically.
+ *
+ * That blindness is the core defect behind the 2026-07-27 incident: agent
+ * `overlord` sustained ~17 activity-card edits/min for hours, earned a
+ * 15908-second (4.4h) per-chat flood ban, and the ban took out ALL replies.
+ * The fuse's per-chat edit ceiling at the time was 30/60s, so it never bound
+ * — a purely cosmetic surface was allowed to consume the whole chat's real
+ * Telegram budget, and the answer path paid for it.
+ *
+ * A class-aware fuse can do the two things a class-blind one cannot:
+ *   1. hold COSMETIC traffic to a rate far below Telegram's per-chat ceiling,
+ *      without throttling approval cards or answers; and
+ *   2. RESERVE part of the per-chat budget so a reply is never starved by
+ *      repaints.
+ *
+ * ── Why AsyncLocalStorage ─────────────────────────────────────────────────
+ * The class must travel from the send gate (well above grammY) down to the
+ * transformer, through an arbitrary async chain, WITHOUT smuggling a
+ * non-Bot-API field into the outbound payload. `AsyncLocalStorage` is the
+ * Node-native mechanism for exactly that, and the plugin already uses the
+ * same pattern for `tg-post` log tags (`shared/bot-runtime.ts`).
+ *
+ * This module deliberately has NO imports beyond `async_hooks`: both
+ * `send-gate.ts` (the writer) and `edit-flood-fuse.ts` (the reader) import
+ * it, so anything heavier would create an import cycle.
+ */
+
+import { AsyncLocalStorage } from 'async_hooks'
+
+/**
+ * Mirrors `send-gate.ts`'s `PriorityClass`. Declared here rather than
+ * imported so this module stays dependency-free (see the docblock); the
+ * send gate's `PriorityClass` is assignable to it and a compile-time
+ * assertion in `send-gate.ts` keeps the two in lockstep.
+ */
+export type OutboundClass = 'critical' | 'useful' | 'cosmetic'
+
+const store = new AsyncLocalStorage<OutboundClass>()
+
+/**
+ * Run `fn` with `cls` visible to every Telegram API call it makes, including
+ * across awaits. The send gate wraps each admitted call in this.
+ */
+export function withOutboundClass<T>(cls: OutboundClass, fn: () => T): T {
+  return store.run(cls, fn)
+}
+
+/** The class of the call currently in flight, or undefined when untagged. */
+export function currentOutboundClass(): OutboundClass | undefined {
+  return store.getStore()
+}
+
+/**
+ * The class the fuse assigns to a call that arrived with no tag.
+ *
+ * Asymmetric ON PURPOSE:
+ *
+ *   - an untagged EDIT is treated as `cosmetic`. An edit that never went
+ *     through the send gate is, by construction, exactly the shape of the
+ *     incident (the activity card edited one message id with no key and no
+ *     class). Defaulting it to cosmetic means a NEW unkeyed edit call site
+ *     is rate-limited by default instead of being invisible until it earns
+ *     a ban. Dropping a repaint is free — the next render carries full
+ *     state.
+ *   - an untagged SEND is treated as `critical`, matching the send gate's
+ *     own `UNTAGGED_SEND_CLASS`. Sends create user-visible output and are
+ *     never dropped by the fuse; the conservative default here is the one
+ *     that cannot lose an answer.
+ */
+export function defaultOutboundClass(isEdit: boolean): OutboundClass {
+  return isEdit ? 'cosmetic' : 'critical'
+}