switchroom 0.19.25 → 0.19.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/bin/git-agent-attribution-hook.sh +144 -0
  2. package/dist/agent-scheduler/index.js +61 -2
  3. package/dist/auth-broker/index.js +125 -8
  4. package/dist/cli/notion-write-pretool.mjs +61 -2
  5. package/dist/cli/switchroom.js +2347 -1104
  6. package/dist/host-control/main.js +126 -9
  7. package/dist/vault/approvals/kernel-server.js +124 -8
  8. package/dist/vault/broker/server.js +124 -8
  9. package/package.json +6 -2
  10. package/profiles/_base/cron-session.sh.hbs +14 -0
  11. package/profiles/_base/start.sh.hbs +145 -4
  12. package/telegram-plugin/card-layout.ts +328 -0
  13. package/telegram-plugin/dist/bridge/bridge.js +93 -1
  14. package/telegram-plugin/dist/gateway/gateway.js +2213 -1204
  15. package/telegram-plugin/dist/server.js +96 -1
  16. package/telegram-plugin/edit-flood-fuse.ts +637 -56
  17. package/telegram-plugin/flood-429-ledger.ts +526 -0
  18. package/telegram-plugin/flood-circuit-breaker.ts +18 -0
  19. package/telegram-plugin/gateway/flood-reply-queue.ts +168 -0
  20. package/telegram-plugin/gateway/gateway.ts +103 -112
  21. package/telegram-plugin/gateway/narrative-lane.ts +14 -0
  22. package/telegram-plugin/gateway/outbound-send-path.ts +36 -0
  23. package/telegram-plugin/gateway/outbox-sweep.ts +183 -6
  24. package/telegram-plugin/gateway/periodic-sweep-guard.ts +86 -0
  25. package/telegram-plugin/gateway/pinned-message-handler.ts +12 -16
  26. package/telegram-plugin/gateway/status-pin-retarget.ts +180 -0
  27. package/telegram-plugin/gateway/status-pin-store.ts +58 -9
  28. package/telegram-plugin/gateway/worker-pin-reaper.ts +56 -7
  29. package/telegram-plugin/llm-error-present.ts +61 -2
  30. package/telegram-plugin/model-unavailable.ts +8 -0
  31. package/telegram-plugin/operator-events.ts +72 -5
  32. package/telegram-plugin/outbound-class.ts +81 -0
  33. package/telegram-plugin/provider-credit.ts +237 -0
  34. package/telegram-plugin/scripts/bun-test-ci.sh +36 -6
  35. package/telegram-plugin/send-gate.ts +24 -2
  36. package/telegram-plugin/status-no-truncate.ts +11 -0
  37. package/telegram-plugin/status-pin-driver.ts +33 -17
  38. package/telegram-plugin/status-pin.ts +51 -5
  39. package/telegram-plugin/tests/card-golden.test.ts +69 -0
  40. package/telegram-plugin/tests/card-lifecycle-render.test.ts +362 -0
  41. package/telegram-plugin/tests/card-type-distinguishability.test.ts +291 -0
  42. package/telegram-plugin/tests/card-variants.golden.txt +211 -0
  43. package/telegram-plugin/tests/card-variants.ts +366 -0
  44. package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +316 -0
  45. package/telegram-plugin/tests/edit-flood-fuse-default-deny.test.ts +319 -0
  46. package/telegram-plugin/tests/edit-flood-fuse.test.ts +11 -2
  47. package/telegram-plugin/tests/feed-edit-rate-ceiling.test.ts +462 -0
  48. package/telegram-plugin/tests/fixtures/real-429-stream.ts +220 -0
  49. package/telegram-plugin/tests/flood-429-ledger.test.ts +278 -0
  50. package/telegram-plugin/tests/flood-429-recorder-wiring.test.ts +128 -0
  51. package/telegram-plugin/tests/flood-reply-queue.test.ts +418 -0
  52. package/telegram-plugin/tests/outbox-sweep-flood-breaker.test.ts +221 -0
  53. package/telegram-plugin/tests/periodic-sweep-guard.test.ts +151 -0
  54. package/telegram-plugin/tests/pinned-card-collapse.test.ts +24 -18
  55. package/telegram-plugin/tests/pinned-message-handler.test.ts +15 -15
  56. package/telegram-plugin/tests/provider-credit-402.test.ts +243 -0
  57. package/telegram-plugin/tests/status-pin-api.test.ts +11 -11
  58. package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +36 -37
  59. package/telegram-plugin/tests/status-pin-lifecycle.test.ts +602 -0
  60. package/telegram-plugin/tests/status-pin-retarget.test.ts +244 -0
  61. package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +7 -3
  62. package/telegram-plugin/tests/status-pin-shutdown-wiring.test.ts +94 -0
  63. package/telegram-plugin/tests/status-pin-store.test.ts +179 -64
  64. package/telegram-plugin/tests/status-pin.test.ts +184 -7
  65. package/telegram-plugin/tests/test-runner-coverage.test.ts +133 -0
  66. package/telegram-plugin/tests/worker-activity-feed.test.ts +12 -10
  67. package/telegram-plugin/tests/worker-feed-coalesce.test.ts +29 -19
  68. package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +56 -59
  69. package/telegram-plugin/tests/worker-feed-terminal-edit-class.test.ts +335 -0
  70. package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +1 -1
  71. package/telegram-plugin/tier-downgrade.ts +3 -2
  72. package/telegram-plugin/tool-activity-summary.ts +239 -322
  73. package/telegram-plugin/uat/assertions.ts +33 -3
  74. package/telegram-plugin/uat/feed-matcher.test.ts +36 -0
  75. package/telegram-plugin/uat/scenarios/jtbd-liveness-narration-channel.test.ts +9 -2
  76. package/telegram-plugin/uat/scenarios/jtbd-liveness-narration-dm.test.ts +9 -2
  77. package/telegram-plugin/worker-activity-feed.ts +109 -30
  78. package/vendor/hindsight-memory/CLAUDE.md +45 -0
  79. package/vendor/hindsight-memory/scripts/lib/config.py +33 -0
  80. package/vendor/hindsight-memory/scripts/recall.py +176 -7
  81. package/vendor/hindsight-memory/scripts/tests/test_config_recall_passthrough_env.py +170 -0
  82. package/vendor/hindsight-memory/scripts/tests/test_recall_min_score.py +464 -0
  83. package/vendor/hindsight-memory/scripts/tests/test_recall_request_timeout.py +241 -0
  84. package/vendor/hindsight-memory/settings.json +1 -1
@@ -70,6 +70,27 @@ export interface PersistedStatusPin {
70
70
  * unpin fails (flood-wait exhausted / transient 5xx); the row is retained
71
71
  * for retry until BOOT_UNPIN_MAX_ATTEMPTS, then forfeited. Absent = 0. */
72
72
  attempts?: number
73
+ /**
74
+ * Wall-clock ms the pin was FIRST claimed for this key+message (#3810).
75
+ *
76
+ * Why the store needs it: the mid-session `wk:` reaper folds in rows that
77
+ * exist on disk but have NO in-memory claim (`storeOnlyWorkerPinCandidates`).
78
+ * The store used to persist no timestamp, so those candidates were stamped
79
+ * `pinnedAt = now` on every pass — deliberately, to avoid a spurious unpin of
80
+ * a pin whose real age was unknown, but with the side effect that the TTL
81
+ * gate could NEVER fire for a store orphan. Combined with a `wk:<agentId>`
82
+ * whose turnsDb row is gone (verdict `'unknown'`, never `'terminal'`), such a
83
+ * row was never mid-session reaped AT ALL — it waited for the next boot, the
84
+ * exact "stale pin glued to the top of the chat" failure the reaper exists to
85
+ * prevent.
86
+ *
87
+ * Recording the real claim time removes the guess: a store orphan now ages
88
+ * honestly and the ordinary TTL applies. Optional so a v1/v2 snapshot still
89
+ * loads; a row without it keeps the old conservative `now` stamp (terminal-
90
+ * only reaping), which self-clears within one restart since every write from
91
+ * this version onward carries the field.
92
+ */
93
+ pinnedAt?: number
73
94
  }
74
95
 
75
96
  /** How many boots may retry a failing boot-cleanup unpin before the row is
@@ -80,13 +101,18 @@ export interface PersistedStatusPin {
80
101
  export const BOOT_UNPIN_MAX_ATTEMPTS = 5
81
102
 
82
103
  /** Envelope version. v1 had no `pending` field; a v1 row loads as a confirmed
83
- * pin (pending undefined). v2 adds the optional `pending` flag. Both load
84
- * fail-open — an unknown/newer version yields []. */
104
+ * pin (pending undefined). v2 adds the optional `pending` flag. v3 adds the
105
+ * optional `pinnedAt` claim timestamp (#3810). All load fail-open — an
106
+ * unknown/newer version yields []. Every field added since v1 is optional, so
107
+ * the versions are mutually readable and a downgrade degrades rather than
108
+ * breaks. */
85
109
  interface SnapshotEnvelope {
86
- v: 1 | 2
110
+ v: 1 | 2 | 3
87
111
  pins: PersistedStatusPin[]
88
112
  }
89
113
 
114
+ const SNAPSHOT_VERSIONS = new Set([1, 2, 3])
115
+
90
116
  function isPinRow(x: unknown): x is PersistedStatusPin {
91
117
  if (x == null || typeof x !== 'object') return false
92
118
  const o = x as Record<string, unknown>
@@ -98,7 +124,8 @@ function isPinRow(x: unknown): x is PersistedStatusPin {
98
124
  typeof o.messageId === 'number' &&
99
125
  (o.pending === undefined || typeof o.pending === 'boolean') &&
100
126
  (o.expiresAt === undefined || typeof o.expiresAt === 'number') &&
101
- (o.attempts === undefined || typeof o.attempts === 'number')
127
+ (o.attempts === undefined || typeof o.attempts === 'number') &&
128
+ (o.pinnedAt === undefined || typeof o.pinnedAt === 'number')
102
129
  )
103
130
  }
104
131
 
@@ -127,7 +154,7 @@ export function loadStatusPins(
127
154
  }
128
155
  if (parsed == null || typeof parsed !== 'object') return []
129
156
  const env = parsed as Record<string, unknown>
130
- if ((env.v !== 1 && env.v !== 2) || !Array.isArray(env.pins)) return []
157
+ if (typeof env.v !== 'number' || !SNAPSHOT_VERSIONS.has(env.v) || !Array.isArray(env.pins)) return []
131
158
  return env.pins.filter(isPinRow)
132
159
  }
133
160
 
@@ -143,7 +170,7 @@ export function persistStatusPins(
143
170
  snapshot: readonly PersistedStatusPin[],
144
171
  log: (line: string) => void = (l) => process.stderr.write(l),
145
172
  ): void {
146
- const env: SnapshotEnvelope = { v: 2, pins: [...snapshot] }
173
+ const env: SnapshotEnvelope = { v: 3, pins: [...snapshot] }
147
174
  const tmp = path + '.tmp'
148
175
  try {
149
176
  fs.writeFileSync(tmp, JSON.stringify(env))
@@ -427,16 +454,34 @@ export function reconcileAndPersistStatusPin(args: {
427
454
  /** Execute the real pin/unpin; returns the confirmed message id (pin) or
428
455
  * null (cleared). Must never throw — API errors are swallowed inside. */
429
456
  applyPin: () => Promise<{ messageId: number } | null>
457
+ /** Clock for the `pinnedAt` stamp (#3810). Defaults to wall clock. */
458
+ now?: number
430
459
  log?: (line: string) => void
431
460
  }): Promise<{ messageId: number } | null> {
432
461
  const { path, fs, pinKey, chatId, op } = args
433
462
  const log = args.log ?? ((l: string) => process.stderr.write(l))
463
+ const now = args.now ?? Date.now()
464
+
465
+ /**
466
+ * The claim age to persist for `messageId`. Carried forward from the row
467
+ * already on disk when it names the SAME message (a steady-state re-write
468
+ * must not reset the age and hand a stale pin a fresh TTL lease); stamped
469
+ * `now` for a genuinely new pin. Mirrors the in-memory `pinnedAt` rule in
470
+ * `status-pin-retarget.ts` so disk and memory can't disagree about age.
471
+ */
472
+ const claimAge = (messageId: number): number => {
473
+ const existing = loadStatusPins(path, fs).find((p) => p.pinKey === pinKey)
474
+ return existing?.messageId === messageId && existing.pinnedAt != null
475
+ ? existing.pinnedAt
476
+ : now
477
+ }
434
478
 
435
479
  // Hold the per-path lock across the WHOLE op so no other writer (a banner
436
480
  // persist, or another key's reconcile) can rebuild/overwrite the file during
437
481
  // the applyPin await window and drop this key's pending/confirmed row.
438
482
  return withStoreLock(path, async () => {
439
483
  if (op.kind === 'pin') {
484
+ const pinnedAt = claimAge(op.messageId)
440
485
  // Persist INTENT first, marked pending — BEFORE the pin API call. If we
441
486
  // crash after the pin lands but before the confirm rewrite, this pending
442
487
  // record is what boot cleanup uses to unpin the orphan.
@@ -444,7 +489,7 @@ export function reconcileAndPersistStatusPin(args: {
444
489
  path,
445
490
  fs,
446
491
  pinKey,
447
- { pinKey, chatId, messageId: op.messageId, pending: true },
492
+ { pinKey, chatId, messageId: op.messageId, pending: true, pinnedAt },
448
493
  log,
449
494
  )
450
495
  const next = await args.applyPin()
@@ -459,7 +504,7 @@ export function reconcileAndPersistStatusPin(args: {
459
504
  path,
460
505
  fs,
461
506
  pinKey,
462
- { pinKey, chatId, messageId: next.messageId },
507
+ { pinKey, chatId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
463
508
  log,
464
509
  )
465
510
  return next
@@ -488,11 +533,15 @@ export function reconcileAndPersistStatusPin(args: {
488
533
  if (next == null) {
489
534
  applyStatusPinRow(path, fs, pinKey, null, log)
490
535
  } else {
536
+ // Steady-state noop-clear (the worker feed calls syncPin on every edit):
537
+ // the pin is still up, so the row is preserved AND so is its original
538
+ // claim age — re-stamping it here would hand a genuinely old pin a fresh
539
+ // TTL lease on every feed edit and make it immortal (#3810).
491
540
  applyStatusPinRow(
492
541
  path,
493
542
  fs,
494
543
  pinKey,
495
- { pinKey, chatId, messageId: next.messageId },
544
+ { pinKey, chatId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
496
545
  log,
497
546
  )
498
547
  }
@@ -99,6 +99,47 @@ export interface StoreOrphanRow {
99
99
  pending?: boolean
100
100
  /** Time-scoped `tool:` pin. Never a worker orphan; excluded. */
101
101
  expiresAt?: number
102
+ /** Wall-clock ms the claim was first taken (#3810). Absent on a pre-v3 row. */
103
+ pinnedAt?: number
104
+ }
105
+
106
+ /**
107
+ * The live worker-activity feed's liveness probe, as the reaper needs it.
108
+ * Nullable ON PURPOSE — see `groupPinStatus`.
109
+ */
110
+ export interface WorkerFeedLivenessProbe {
111
+ hasRunningInFeed(feedKey: string): boolean
112
+ }
113
+
114
+ /**
115
+ * Verdict for a GROUP-level pin key (`wk:group:<feedKey>`, the shape the
116
+ * coalesced feed produces since #3207). Vouched off the live feed rather than
117
+ * the sub-agent registry: `group:<feedKey>` is not a jsonl agent id.
118
+ *
119
+ * #3811 — the gateway used to write this as
120
+ * `feed?.hasRunningInFeed(key) ? 'running' : 'terminal'`, which collapses two
121
+ * very different facts into one verdict:
122
+ *
123
+ * - "a live feed exists and affirmatively says this group is done" → terminal
124
+ * - "there is NO feed object to ask" → nothing known
125
+ *
126
+ * A `'terminal'` verdict reaps at ANY age — it bypasses the TTL exemption
127
+ * entirely — so the second case could unpin a GENUINELY LIVE group pin out from
128
+ * under running workers during a window where `workerActivityFeed` is null
129
+ * (teardown/rebuild on a bridge flap, and every tick before the first bridge
130
+ * connect). That is the inverse of the stale-pin bug and much harder to notice:
131
+ * the user loses the pin for work that is still in flight.
132
+ *
133
+ * "No feed to ask" is `'unknown'`: the TTL still applies, so a truly stale pin
134
+ * is still bounded, but nothing is reaped on the strength of a question we
135
+ * never got to ask.
136
+ */
137
+ export function groupPinStatus(
138
+ feed: WorkerFeedLivenessProbe | null | undefined,
139
+ feedKey: string,
140
+ ): WorkerRegistryStatus {
141
+ if (feed == null) return 'unknown'
142
+ return feed.hasRunningInFeed(feedKey) ? 'running' : 'terminal'
102
143
  }
103
144
 
104
145
  /**
@@ -110,13 +151,21 @@ export interface StoreOrphanRow {
110
151
  * it too, group-safely (per-message unpin of a bot-tracked row — never an
111
152
  * unpin-all, never a human pin).
112
153
  *
113
- * The store persists NO timestamp, so a store-only candidate carries
114
- * `pinnedAt = now`: only a TERMINAL registry verdict can reap it. It is never
115
- * TTL-reaped (no trustworthy age to age it out), never touched while its worker
116
- * is `running`, and it is always a `wk:` row (the documented leak class).
154
+ * AGE (#3810). A v3+ row carries the real `pinnedAt` the claim was taken at, so
155
+ * a store orphan ages honestly and the ordinary TTL gate applies to it exactly
156
+ * as it does to an in-memory claim. A pre-v3 row has no timestamp and keeps the
157
+ * old conservative stamp (`pinnedAt = now`, i.e. TERMINAL-verdict reaping only)
158
+ * rather than guessing an age — but that case now self-clears within one
159
+ * restart, because every write from v3 onward records the field.
160
+ *
161
+ * Why the honest age matters: `storeOnlyWorkerPinCandidates` was the ONLY net
162
+ * for a `wk:` row with no in-memory claim, and with the TTL permanently
163
+ * disabled a `wk:<agentId>` row whose turnsDb row had been pruned (verdict
164
+ * `'unknown'`, never `'terminal'`) was never mid-session reaped at all — its
165
+ * stale `🛠 Worker` pin sat at the top of the chat until the next gateway boot.
166
+ *
117
167
  * `pending` rows (pin API in-flight) and time-scoped `tool:` rows are excluded,
118
- * as are rows already tracked in memory (the in-memory reaper owns those, with
119
- * a real `pinnedAt` that can drive the TTL).
168
+ * as are rows already tracked in memory (the in-memory reaper owns those).
120
169
  */
121
170
  export function storeOnlyWorkerPinCandidates(args: {
122
171
  rows: Iterable<StoreOrphanRow>
@@ -133,7 +182,7 @@ export function storeOnlyWorkerPinCandidates(args: {
133
182
  out.push({
134
183
  pinKey: r.pinKey,
135
184
  chatId: r.chatId,
136
- pinnedAt: args.now,
185
+ pinnedAt: r.pinnedAt ?? args.now,
137
186
  messageId: r.messageId,
138
187
  })
139
188
  }
@@ -34,6 +34,12 @@ import {
34
34
  parseResetTime,
35
35
  } from './model-unavailable.js'
36
36
  import { classify429Detail } from './throttle-tier.js'
37
+ import {
38
+ PROVIDER_CREDIT_REGISTRY,
39
+ attributeProvider,
40
+ describeProviderCreditRemedy,
41
+ type ProviderCreditEntry,
42
+ } from './provider-credit.js'
37
43
  import { classifyClaudeError } from './operator-events.js'
38
44
  import { stripRawErrorBytes, extractRequestId } from './raw-error-scrub.js'
39
45
  import { fmtLocalClock, tzAbbrev } from './shared/local-time.js'
@@ -48,10 +54,24 @@ export type LlmErrorKind =
48
54
  | 'quota_wall'
49
55
  | 'auth'
50
56
  | 'infra_misconfig'
57
+ /**
58
+ * A THIRD-PARTY model provider (OpenRouter / OpenAI / Perplexity) is out of
59
+ * credit. OPERATOR-actionable, never user-actionable — its `coreText` is a
60
+ * deliberately diagnosis-free one-liner and its recommendation names the
61
+ * vendor console, never `/auth`. Kept distinct from `quota_wall` (the
62
+ * Anthropic subscription window, which resets on a clock and is addressed by
63
+ * switching account slots); a vendor balance resets only when someone pays.
64
+ */
65
+ | 'provider_credit'
51
66
  | 'transient'
52
67
  | 'unknown'
53
68
 
54
- export type LlmErrorSource = 'anthropic' | 'litellm-local' | 'network'
69
+ export type LlmErrorSource =
70
+ | 'anthropic'
71
+ | 'litellm-local'
72
+ /** A paid third-party model provider reached THROUGH the proxy (OpenRouter, OpenAI). */
73
+ | 'external-provider'
74
+ | 'network'
55
75
 
56
76
  export interface ParsedLlmError {
57
77
  kind: LlmErrorKind
@@ -65,6 +85,14 @@ export interface ParsedLlmError {
65
85
  model?: string
66
86
  /** Anthropic `request_id`, when present — the strongest dedup key. */
67
87
  requestId?: string
88
+ /**
89
+ * Registry id of the third-party provider the error was attributed to
90
+ * (`openrouter` / `openai` / `perplexity`), when `kind === 'provider_credit'`
91
+ * and the raw text named one. An ID only — never raw error text, never a
92
+ * credential — so the recommendation line can point at the right console
93
+ * without this struct ever carrying the raw string it was parsed from.
94
+ */
95
+ providerId?: string
68
96
  source: LlmErrorSource
69
97
  /** True when the harness is still retrying this error internally (mid-retry). */
70
98
  autoRetrying: boolean
@@ -96,7 +124,10 @@ const TRANSIENT_KINDS: ReadonlySet<LlmErrorKind> = new Set<LlmErrorKind>([
96
124
 
97
125
  /** The always-actionable kinds — NEVER silenced, even inside a collapse window. */
98
126
  export function isActionableKind(kind: LlmErrorKind): boolean {
99
- return kind === 'auth' || kind === 'quota_wall'
127
+ // `provider_credit` joins the never-silenced set: an unpaid vendor balance
128
+ // does not clear on its own, so collapsing it inside a dedup window would
129
+ // hide the one signal that needs a human.
130
+ return kind === 'auth' || kind === 'quota_wall' || kind === 'provider_credit'
100
131
  }
101
132
 
102
133
  /**
@@ -118,6 +149,8 @@ export function parseLlmError(
118
149
  resetAt != null ? Math.max(0, resetAt.getTime() - Date.now()) : undefined
119
150
 
120
151
  const { kind, source } = classifyKindAndSource(text)
152
+ const providerId =
153
+ kind === 'provider_credit' ? (attributeProvider(text)?.id ?? undefined) : undefined
121
154
 
122
155
  // Retry / terminal semantics. Only the transient family can be mid-retry;
123
156
  // auth / quota_wall / model_unavailable are terminal by construction.
@@ -143,6 +176,7 @@ export function parseLlmError(
143
176
  ...(retryAfterMs != null ? { retryAfterMs } : {}),
144
177
  ...(model != null ? { model } : {}),
145
178
  ...(requestId != null ? { requestId } : {}),
179
+ ...(providerId != null ? { providerId } : {}),
146
180
  source,
147
181
  autoRetrying,
148
182
  terminal,
@@ -165,6 +199,15 @@ function classifyKindAndSource(text: string): { kind: LlmErrorKind; source: LlmE
165
199
 
166
200
  // 1. Auth — always terminal, always actionable.
167
201
  const claudeKind = classifyClaudeError({ message: text, type: text })
202
+ // 1b. THIRD-PARTY provider credit wall. Placed immediately after the auth
203
+ // resolution and BEFORE isLitellmProxyLocal429 / detectModelUnavailable:
204
+ // OpenAI reports an exhausted balance as HTTP 429 `insufficient_quota`,
205
+ // and the 429 matchers below would otherwise swallow it as a transient
206
+ // "retrying automatically" — the single most misleading thing to tell
207
+ // someone whose provider has no money left.
208
+ if (claudeKind === 'provider-credit-exhausted') {
209
+ return { kind: 'provider_credit', source: 'external-provider' }
210
+ }
168
211
  if (claudeKind === 'proxy-misconfig') {
169
212
  return { kind: 'infra_misconfig', source: 'litellm-local' }
170
213
  }
@@ -216,6 +259,12 @@ function classifyKindAndSource(text: string): { kind: LlmErrorKind; source: LlmE
216
259
  return { kind: 'unknown', source: 'anthropic' }
217
260
  }
218
261
 
262
+ /** Resolve a registry entry from a persisted `providerId`, or null. */
263
+ function providerById(id: string | undefined): ProviderCreditEntry | null {
264
+ if (id == null) return null
265
+ return PROVIDER_CREDIT_REGISTRY.find(p => p.id === id) ?? null
266
+ }
267
+
219
268
  function buildCoreText(kind: LlmErrorKind, source: LlmErrorSource): string {
220
269
  switch (kind) {
221
270
  case 'rate_limit':
@@ -230,6 +279,10 @@ function buildCoreText(kind: LlmErrorKind, source: LlmErrorSource): string {
230
279
  return 'Claude login needs re-authentication.'
231
280
  case 'infra_misconfig':
232
281
  return 'Local model-gateway auth misconfig (proxy fallback dropped the OAuth header).'
282
+ case 'provider_credit':
283
+ // Diagnosis-free by design — this string is the ceiling on what a
284
+ // non-operator may ever be told about a vendor billing wall.
285
+ return 'An upstream model provider is out of credit — the operator has been notified.'
233
286
  case 'transient':
234
287
  return source === 'network'
235
288
  ? "Couldn't reach Anthropic (network) — retrying automatically."
@@ -302,6 +355,10 @@ function buildRecommendation(parsed: ParsedLlmError, tz: string): string | undef
302
355
  // Operator-facing — NO re-auth wording (the login is fine). Points at the
303
356
  // real remedy: the local LiteLLM proxy fallback config.
304
357
  return '→ Fix the LiteLLM proxy fallback config (deployment missing OAuth passthrough).'
358
+ case 'provider_credit':
359
+ // Operator-facing. NO `/auth` wording: the credential is valid, the
360
+ // balance is empty, and rotating an Anthropic slot changes nothing.
361
+ return `→ ${describeProviderCreditRemedy(providerById(parsed.providerId))}`
305
362
  case 'quota_wall': {
306
363
  const reset = formatResetClock(parsed.resetAt, tz)
307
364
  return reset
@@ -377,6 +434,8 @@ function kindEmoji(kind: LlmErrorKind): string {
377
434
  return '🔑'
378
435
  case 'infra_misconfig':
379
436
  return '🛠️'
437
+ case 'provider_credit':
438
+ return '💳'
380
439
  case 'transient':
381
440
  return '🌐'
382
441
  case 'unknown':
@@ -790,6 +790,14 @@ export function resolveModelUnavailableFromOperatorEvent(
790
790
  ev: OperatorEventLike,
791
791
  ): ModelUnavailableDetection | null {
792
792
  const detail = typeof ev.detail === 'string' ? ev.detail : ''
793
+ // A THIRD-PARTY provider credit wall is never "model unavailable" in the
794
+ // Anthropic sense. Returning a detection here would route the event into the
795
+ // `modelUnavailable` branch of emitGatewayOperatorEvent, which renders the
796
+ // "⚠️ Model unavailable" card AND — for a `quota_exhausted` detection —
797
+ // fires `fireFleetAutoFallback`, benching a perfectly healthy Anthropic
798
+ // account slot because OpenRouter ran out of money. Return null so the event
799
+ // takes the plain `renderOperatorEvent` path with its provider-specific card.
800
+ if (ev.kind === 'provider-credit-exhausted') return null
793
801
  if (ev.kind === 'quota-exhausted') {
794
802
  return detectModelUnavailable(detail) ?? { kind: 'quota_exhausted', raw: detail }
795
803
  }
@@ -15,6 +15,11 @@
15
15
  import { escapeMarkdown } from './format.js'
16
16
  import { stripRawErrorBytes } from './raw-error-scrub.js'
17
17
  import { isLitellmProxyAuthMisconfig } from './model-unavailable.js'
18
+ import {
19
+ attributeProvider,
20
+ describeProviderCreditRemedy,
21
+ detectProviderCreditExhaustion,
22
+ } from './provider-credit.js'
18
23
 
19
24
  // ─── Taxonomy ────────────────────────────────────────────────────────────────
20
25
 
@@ -23,6 +28,16 @@ export type OperatorEventKind =
23
28
  | 'credentials-invalid'
24
29
  | 'proxy-misconfig'
25
30
  | 'credit-exhausted'
31
+ /**
32
+ * A THIRD-PARTY model provider (OpenRouter / OpenAI / Perplexity) reports no
33
+ * credit remaining — HTTP 402 `payment_required`, "insufficient credits",
34
+ * OpenAI's `insufficient_quota`, etc. Distinct from `credit-exhausted`,
35
+ * which is the ANTHROPIC balance and whose remedy is `/auth use <slot>`:
36
+ * rotating an Anthropic account slot buys zero OpenRouter tokens, so the two
37
+ * must not share a card. Operator-actionable (top up in the vendor console),
38
+ * never user-actionable — see {@link OPERATOR_ACTIONABLE_KINDS}.
39
+ */
40
+ | 'provider-credit-exhausted'
26
41
  | 'quota-exhausted'
27
42
  | 'rate-limited'
28
43
  | 'agent-crashed'
@@ -134,16 +149,38 @@ function classifyInner(raw: unknown): OperatorEventKind {
134
149
  return 'credentials-invalid'
135
150
  }
136
151
 
152
+ // The full scan surface for the provider-credit decision — same
153
+ // type/code/message union isLitellmProxyAuthMisconfig uses, because LiteLLM
154
+ // stamps the upstream vendor's wording in whichever field it has room for.
155
+ const creditScanText = `${errorType}\n${errorCode}\n${sdkCode}\n${message}`
156
+
157
+ // ANTHROPIC credit wall. Guarded with `attributeProvider(...) == null` so an
158
+ // OpenRouter/OpenAI/Perplexity error that happens to say "credit balance"
159
+ // does NOT get the Anthropic card, whose remedy (`/auth use <slot>`) is
160
+ // useless against a third-party balance. Unattributed credit-balance wording
161
+ // keeps its historical Anthropic reading — that is where it has always come
162
+ // from, and the alternative (guessing a vendor) sends the operator to the
163
+ // wrong console.
137
164
  if (
138
- errorType === 'credit_balance_too_low' ||
139
- errorCode === 'credit_balance_too_low' ||
140
- sdkCode === 'credit_balance_too_low' ||
141
- message.toLowerCase().includes('credit_balance_too_low') ||
142
- message.toLowerCase().includes('credit balance')
165
+ (errorType === 'credit_balance_too_low' ||
166
+ errorCode === 'credit_balance_too_low' ||
167
+ sdkCode === 'credit_balance_too_low' ||
168
+ message.toLowerCase().includes('credit_balance_too_low') ||
169
+ message.toLowerCase().includes('credit balance')) &&
170
+ attributeProvider(creditScanText) == null
143
171
  ) {
144
172
  return 'credit-exhausted'
145
173
  }
146
174
 
175
+ // THIRD-PARTY provider credit wall — the OpenRouter 402 leak (#external-credit).
176
+ // Runs BEFORE the rate-limit branch on purpose: OpenAI signals an exhausted
177
+ // balance as HTTP *429* `insufficient_quota`, so a rate-limit-first ordering
178
+ // would classify a billing wall as a transient throttle and tell the operator
179
+ // to wait for a reset that will never come.
180
+ if (detectProviderCreditExhaustion(creditScanText, status) != null) {
181
+ return 'provider-credit-exhausted'
182
+ }
183
+
147
184
  if (
148
185
  errorType === 'rate_limit_error' ||
149
186
  errorCode === 'rate_limit_error' ||
@@ -316,6 +353,31 @@ export function renderOperatorEvent(ev: OperatorEvent): RenderResult {
316
353
  },
317
354
  }
318
355
 
356
+ // A third-party provider's balance, NOT Anthropic's. Deliberately carries
357
+ // NO `/auth use` / `/auth add` language (those rotate Anthropic account
358
+ // slots and buy zero OpenRouter tokens) and NO "🔐 Reauth" button (the key
359
+ // is valid; it is out of money). The remedy names the provider, the VAULT
360
+ // KEY NAME — never a value — and the console to act in. Dismiss-only.
361
+ case 'provider-credit-exhausted': {
362
+ const provider = attributeProvider(ev.detail)
363
+ const who = provider != null ? escapeMarkdown(provider.label) : 'An upstream model provider'
364
+ return {
365
+ text: [
366
+ `💳 **${provider != null ? `${who} credit exhausted` : 'Upstream provider credit exhausted'}** — hit by **${agent}**.`,
367
+ detail ? `_${detail}_` : '',
368
+ describeProviderCreditRemedy(provider),
369
+ `Anthropic account slots are unaffected — \`/auth use\` will NOT fix this.`,
370
+ ]
371
+ .filter(Boolean)
372
+ .join('\n'),
373
+ keyboard: {
374
+ inline_keyboard: [
375
+ [{ text: '❌ Dismiss', callback_data: `op:dismiss:${encodeURIComponent(ev.agent)}` }],
376
+ ],
377
+ },
378
+ }
379
+ }
380
+
319
381
  case 'quota-exhausted':
320
382
  // Canonical quota-exhausted text (migrated from auto-fallback.ts).
321
383
  // auto-fallback.ts's buildSwitchedMessage / buildAllExhaustedMessage
@@ -544,6 +606,11 @@ export const OPERATOR_ACTIONABLE_KINDS: ReadonlySet<OperatorEventKind> = new Set
544
606
  'credentials-expired',
545
607
  'credentials-invalid',
546
608
  'credit-exhausted',
609
+ // The OpenRouter/OpenAI/Perplexity 402 class. Only the operator can top up a
610
+ // vendor balance; an end user shown "insufficient credits" can do nothing but
611
+ // lose confidence. This membership is what routes it to the operator card and
612
+ // hands the user the brief plain-language notice instead.
613
+ 'provider-credit-exhausted',
547
614
  'proxy-misconfig',
548
615
  ])
549
616
 
@@ -0,0 +1,81 @@
1
+ /**
2
+ * Outbound priority class, propagated to the grammY transformer stack.
3
+ *
4
+ * ── Why this module exists ────────────────────────────────────────────────
5
+ * The send gate (`send-gate.ts`) knows every outbound call's `priorityClass`
6
+ * — `cosmetic` (a repaint of a progress card), `useful`, or `critical` (the
7
+ * operator's actual answer). The edit-flood fuse (`edit-flood-fuse.ts`) is a
8
+ * grammY API transformer, and a transformer sees only `(method, payload)`.
9
+ * It therefore could NOT tell a liveness repaint apart from a reply, and had
10
+ * to treat every edit identically.
11
+ *
12
+ * That blindness is the core defect behind the 2026-07-27 incident: agent
13
+ * `overlord` sustained ~17 activity-card edits/min for hours, earned a
14
+ * 15908-second (4.4h) per-chat flood ban, and the ban took out ALL replies.
15
+ * The fuse's per-chat edit ceiling at the time was 30/60s, so it never bound
16
+ * — a purely cosmetic surface was allowed to consume the whole chat's real
17
+ * Telegram budget, and the answer path paid for it.
18
+ *
19
+ * A class-aware fuse can do the two things a class-blind one cannot:
20
+ * 1. hold COSMETIC traffic to a rate far below Telegram's per-chat ceiling,
21
+ * without throttling approval cards or answers; and
22
+ * 2. RESERVE part of the per-chat budget so a reply is never starved by
23
+ * repaints.
24
+ *
25
+ * ── Why AsyncLocalStorage ─────────────────────────────────────────────────
26
+ * The class must travel from the send gate (well above grammY) down to the
27
+ * transformer, through an arbitrary async chain, WITHOUT smuggling a
28
+ * non-Bot-API field into the outbound payload. `AsyncLocalStorage` is the
29
+ * Node-native mechanism for exactly that, and the plugin already uses the
30
+ * same pattern for `tg-post` log tags (`shared/bot-runtime.ts`).
31
+ *
32
+ * This module deliberately has NO imports beyond `async_hooks`: both
33
+ * `send-gate.ts` (the writer) and `edit-flood-fuse.ts` (the reader) import
34
+ * it, so anything heavier would create an import cycle.
35
+ */
36
+
37
+ import { AsyncLocalStorage } from 'async_hooks'
38
+
39
+ /**
40
+ * Mirrors `send-gate.ts`'s `PriorityClass`. Declared here rather than
41
+ * imported so this module stays dependency-free (see the docblock); the
42
+ * send gate's `PriorityClass` is assignable to it and a compile-time
43
+ * assertion in `send-gate.ts` keeps the two in lockstep.
44
+ */
45
+ export type OutboundClass = 'critical' | 'useful' | 'cosmetic'
46
+
47
+ const store = new AsyncLocalStorage<OutboundClass>()
48
+
49
+ /**
50
+ * Run `fn` with `cls` visible to every Telegram API call it makes, including
51
+ * across awaits. The send gate wraps each admitted call in this.
52
+ */
53
+ export function withOutboundClass<T>(cls: OutboundClass, fn: () => T): T {
54
+ return store.run(cls, fn)
55
+ }
56
+
57
+ /** The class of the call currently in flight, or undefined when untagged. */
58
+ export function currentOutboundClass(): OutboundClass | undefined {
59
+ return store.getStore()
60
+ }
61
+
62
+ /**
63
+ * The class the fuse assigns to a call that arrived with no tag.
64
+ *
65
+ * Asymmetric ON PURPOSE:
66
+ *
67
+ * - an untagged EDIT is treated as `cosmetic`. An edit that never went
68
+ * through the send gate is, by construction, exactly the shape of the
69
+ * incident (the activity card edited one message id with no key and no
70
+ * class). Defaulting it to cosmetic means a NEW unkeyed edit call site
71
+ * is rate-limited by default instead of being invisible until it earns
72
+ * a ban. Dropping a repaint is free — the next render carries full
73
+ * state.
74
+ * - an untagged SEND is treated as `critical`, matching the send gate's
75
+ * own `UNTAGGED_SEND_CLASS`. Sends create user-visible output and are
76
+ * never dropped by the fuse; the conservative default here is the one
77
+ * that cannot lose an answer.
78
+ */
79
+ export function defaultOutboundClass(isEdit: boolean): OutboundClass {
80
+ return isEdit ? 'cosmetic' : 'critical'
81
+ }