switchroom 0.19.27 → 0.19.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/agent-scheduler/index.js +5 -2
  2. package/dist/auth-broker/index.js +129 -8
  3. package/dist/cli/autoaccept-poll.js +225 -17
  4. package/dist/cli/notion-write-pretool.mjs +5 -2
  5. package/dist/cli/switchroom.js +796 -35
  6. package/dist/host-control/main.js +130 -9
  7. package/dist/vault/approvals/kernel-server.js +129 -8
  8. package/dist/vault/broker/server.js +129 -8
  9. package/package.json +3 -2
  10. package/profiles/_base/start.sh.hbs +70 -15
  11. package/telegram-plugin/dist/bridge/bridge.js +1 -0
  12. package/telegram-plugin/dist/gateway/gateway.js +568 -49
  13. package/telegram-plugin/dist/server.js +1 -0
  14. package/telegram-plugin/edit-flood-fuse.ts +230 -27
  15. package/telegram-plugin/gateway/callback-query-handlers.ts +6 -0
  16. package/telegram-plugin/gateway/gateway.ts +9 -2
  17. package/telegram-plugin/gateway/mcp-failure-hook.ts +74 -0
  18. package/telegram-plugin/inline-keyboard-callbacks.ts +202 -21
  19. package/telegram-plugin/mcp-credential-failure.ts +459 -0
  20. package/telegram-plugin/operator-events.ts +38 -0
  21. package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +58 -1
  22. package/telegram-plugin/tests/edit-flood-fuse-reply-reserve.test.ts +340 -0
  23. package/telegram-plugin/tests/finalize-callback-flood-policy.test.ts +298 -0
  24. package/telegram-plugin/tests/finalize-callback.test.ts +41 -8
  25. package/telegram-plugin/tests/mcp-credential-failure.test.ts +310 -0
  26. package/vendor/hindsight-memory/scripts/drain_pending.py +433 -11
  27. package/vendor/hindsight-memory/scripts/lib/pending.py +193 -28
  28. package/vendor/hindsight-memory/scripts/tests/test_drain_circuit_breaker.py +401 -0
  29. package/vendor/hindsight-memory/scripts/tests/test_drain_serialisation.py +286 -0
  30. package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py +817 -8
  31. package/vendor/hindsight-memory/settings.json +1 -1
  32. package/vendor/hindsight-memory/tests/test_hooks.py +11 -2
@@ -0,0 +1,459 @@
1
+ /**
2
+ * mcp-credential-failure.ts — "a paid MCP dependency is blocked" detection.
3
+ *
4
+ * WHY THIS EXISTS
5
+ * ---------------
6
+ * Ken: "If perplexity fails for any agent we should have hard warning via
7
+ * message here per other failures."
8
+ *
9
+ * Perplexity (and Eraser, Brevo, Meta Ads, Postiz, Google Ads …) reach
10
+ * switchroom over the MCP TOOL surface, not through LiteLLM. So the operator-
11
+ * event path that PR A fixed for an OpenRouter 402 never sees them: an expired
12
+ * Perplexity key surfaces as an ordinary `is_error` tool_result, gets rendered
13
+ * as a transient step in the live feed, and dies there. Nobody is told the
14
+ * fleet just lost a paid capability.
15
+ *
16
+ * This module is the classifier + ledger for that. It is deliberately GENERIC:
17
+ * Perplexity is one row in `PROVIDER_CREDIT_REGISTRY` / `MCP_SERVER_VAULT_KEYS`,
18
+ * not a special case, so every current and future MCP server gets the same
19
+ * treatment by adding DATA, never a new conditional at a call site.
20
+ *
21
+ * THE CLASSIFICATION RULE (stated explicitly, because it decides whether Ken's
22
+ * phone buzzes — and an alert he learns to ignore is worse than no alert):
23
+ *
24
+ * ALERT — the key itself is the problem, and only Ken can fix it:
25
+ * • credit — out of money (402, "insufficient credits", "payment_required")
26
+ * • credential— key rejected (401/403, "invalid api key", "unauthorized",
27
+ * "key has been disabled/revoked/blocked")
28
+ * • quota — hard usage wall("quota exceeded", "usage limit reached",
29
+ * "monthly limit", "plan limit", "over
30
+ * quota") — i.e. spent for the period
31
+ *
32
+ * SILENT — ordinary operational failure, retrying or rewording fixes it:
33
+ * • a bare rate limit / 429 with a retry-after (throttle, not a wall)
34
+ * • 404, 400, validation errors, a bad query, no results
35
+ * • timeouts, ECONNRESET, 5xx, transport errors
36
+ *
37
+ * The 429 split is the subtle one and is intentional: a throttle self-heals in
38
+ * seconds and must NOT page the operator; a `quota exceeded`/`usage limit`
39
+ * wording on the same status means the period's allowance is gone and only a
40
+ * plan change or a top-up clears it. Status alone is never sufficient — see the
41
+ * OpenAI case in `provider-credit.ts` (a spent balance arrives as 429).
42
+ *
43
+ * DEDUPLICATION — and its HONEST LIMIT
44
+ * ------------------------------------
45
+ * The ledger coalesces per `(server, class)` and re-notifies on the house
46
+ * cadence (`RENOTIFY_MS`, 6 h — the same interval `src/hindsight-watch` uses,
47
+ * so operator alerts across switchroom share one rhythm). Repeated failures
48
+ * inside the window are ACCUMULATED, not dropped: the next alert names every
49
+ * agent seen since the last one.
50
+ *
51
+ * The limit: this ledger is IN-PROCESS and therefore AGENT-LOCAL. Each agent
52
+ * runs its own `switchroom-<agent>` container with its own gateway process and
53
+ * no shared writable mount (`~/.switchroom/fleet` is read-only; per-agent state
54
+ * dirs are 0700 under distinct UIDs). So if six agents hit a blocked Perplexity
55
+ * key in the same minute, Ken gets up to six alerts — one per agent, each
56
+ * naming its agent — not one coalesced alert, and not thirty-six. Honouring
57
+ * "one alert naming the affected agents" fleet-wide needs a shared ledger the
58
+ * gateway does not have; that is filed as a follow-up rather than faked here.
59
+ *
60
+ * Pure module: no IPC, no bot, no FS, no network, no ambient clock (`now` is a
61
+ * parameter). Trivially unit-testable.
62
+ */
63
+
64
+ import {
65
+ PROVIDER_CREDIT_REGISTRY,
66
+ hasCreditExhaustionWording,
67
+ isProviderCreditStatus,
68
+ type ProviderCreditEntry,
69
+ } from './provider-credit.js'
70
+
71
+ // ─── Server identity ─────────────────────────────────────────────────────────
72
+
73
+ /**
74
+ * Vault key names for paid MCP servers that are NOT in the credit registry
75
+ * (the registry only covers providers that also serve inference). A NAME only —
76
+ * this module never reads or renders a secret VALUE.
77
+ *
78
+ * Adding a paid dependency is a DATA edit here.
79
+ */
80
+ export const MCP_SERVER_VAULT_KEYS: Readonly<Record<string, { label: string; vaultKey: string; consoleUrl: string }>> = {
81
+ eraser: { label: 'Eraser', vaultKey: 'eraser/api-key', consoleUrl: 'https://app.eraser.io/settings/api' },
82
+ brevo: { label: 'Brevo', vaultKey: 'brevo/api-key', consoleUrl: 'https://app.brevo.com/settings/keys/api' },
83
+ postiz: { label: 'Postiz', vaultKey: 'postiz/api-key', consoleUrl: 'https://platform.postiz.com/settings' },
84
+ 'meta-ads': { label: 'Meta Ads', vaultKey: 'meta-ads/access-token', consoleUrl: 'https://business.facebook.com/settings' },
85
+ 'google-ads': { label: 'Google Ads', vaultKey: 'google-ads/developer-token', consoleUrl: 'https://ads.google.com/aw/apicenter' },
86
+ cloudflare: { label: 'Cloudflare', vaultKey: 'cloudflare/api-token', consoleUrl: 'https://dash.cloudflare.com/profile/api-tokens' },
87
+ context7: { label: 'Context7', vaultKey: 'context7/api-key', consoleUrl: 'https://context7.com/dashboard' },
88
+ }
89
+
90
+ /**
91
+ * `mcp__perplexity__perplexity_search` → `perplexity`. Returns `null` for a
92
+ * non-MCP tool (Bash, Read, …) — those are never a credential problem, so the
93
+ * caller short-circuits on `null` and the hot path costs one `startsWith`.
94
+ */
95
+ export function parseMcpServerFromToolName(toolName: unknown): string | null {
96
+ if (typeof toolName !== 'string') return null
97
+ if (!toolName.startsWith('mcp__')) return null
98
+ const rest = toolName.slice('mcp__'.length)
99
+ const end = rest.indexOf('__')
100
+ const server = end === -1 ? rest : rest.slice(0, end)
101
+ return server.length > 0 ? server : null
102
+ }
103
+
104
+ /** Remedy metadata for a server, from either data source. `null` when unknown. */
105
+ export function describeMcpServer(
106
+ server: string,
107
+ ): { label: string; vaultKey: string; consoleUrl: string } | null {
108
+ const direct = MCP_SERVER_VAULT_KEYS[server]
109
+ if (direct != null) return direct
110
+ const entry: ProviderCreditEntry | undefined = PROVIDER_CREDIT_REGISTRY.find(e => e.id === server)
111
+ if (entry != null) {
112
+ return { label: entry.label, vaultKey: entry.vaultKey, consoleUrl: entry.consoleUrl }
113
+ }
114
+ return null
115
+ }
116
+
117
+ // ─── Classification ──────────────────────────────────────────────────────────
118
+
119
+ /** `ordinary` is the SILENT class — everything else raises an operator alert. */
120
+ export type McpFailureClass = 'credit' | 'credential' | 'quota' | 'ordinary'
121
+
122
+ /** True for the classes that page the operator. */
123
+ export function isAlertingClass(c: McpFailureClass): boolean {
124
+ return c !== 'ordinary'
125
+ }
126
+
127
+ interface ClassRule {
128
+ cls: Exclude<McpFailureClass, 'ordinary'>
129
+ /** Structured HTTP statuses that imply this class on their own. */
130
+ statuses: readonly number[]
131
+ /** Lowercase wordings that imply this class. */
132
+ signals: readonly string[]
133
+ }
134
+
135
+ /**
136
+ * Evaluated IN ORDER — first match wins. Credit precedes credential because a
137
+ * spent Perplexity balance answers 401 with "insufficient credits", and the
138
+ * remedy is "top up", not "re-issue the key".
139
+ */
140
+ const CLASS_RULES: readonly ClassRule[] = [
141
+ // `credit` is handled ahead of this table via the shared registry predicates.
142
+ {
143
+ cls: 'credential',
144
+ statuses: [401, 403],
145
+ signals: [
146
+ 'invalid api key',
147
+ 'invalid_api_key',
148
+ 'incorrect api key',
149
+ 'invalid authentication',
150
+ 'authentication_error',
151
+ 'authentication failed',
152
+ 'unauthorized',
153
+ 'unauthorised',
154
+ 'forbidden',
155
+ 'api key not found',
156
+ 'no api key provided',
157
+ 'missing api key',
158
+ 'api key expired',
159
+ 'key has been disabled',
160
+ 'key has been revoked',
161
+ 'key is disabled',
162
+ 'account has been blocked',
163
+ 'account is blocked',
164
+ 'account suspended',
165
+ 'permission_denied',
166
+ 'token expired',
167
+ 'invalid token',
168
+ ],
169
+ },
170
+ {
171
+ cls: 'quota',
172
+ // NOTE: 429 is deliberately NOT listed. A bare 429 is a throttle and must
173
+ // stay silent; only the hard-wall WORDINGS below promote it to an alert.
174
+ statuses: [],
175
+ signals: [
176
+ 'quota exceeded',
177
+ 'quota_exceeded',
178
+ 'over quota',
179
+ 'usage limit reached',
180
+ 'usage limit exceeded',
181
+ 'monthly limit',
182
+ 'monthly quota',
183
+ 'plan limit',
184
+ 'plan_limit',
185
+ 'limit reached for your plan',
186
+ 'upgrade your plan',
187
+ 'spending limit',
188
+ // NOTE: "exceeded your current quota" is deliberately NOT here. It is
189
+ // OpenAI's BILLING wording and already lives in `CREDIT_EXHAUSTION_SIGNALS`,
190
+ // where it is matched first — duplicating it would be dead data that
191
+ // silently disagrees with the shared registry.
192
+ ],
193
+ },
194
+ ]
195
+
196
+ /**
197
+ * Wordings that mean "transient throttle / operational hiccup" and must keep a
198
+ * failure SILENT even if a broader signal would otherwise match. Checked before
199
+ * the alerting rules so `429 rate limit, retry after 2s` never pages anyone.
200
+ */
201
+ const TRANSIENT_SIGNALS: readonly string[] = [
202
+ 'rate limit',
203
+ 'rate_limit',
204
+ 'too many requests',
205
+ 'retry after',
206
+ 'retry-after',
207
+ 'slow down',
208
+ 'timeout',
209
+ 'timed out',
210
+ 'etimedout',
211
+ 'econnreset',
212
+ 'econnrefused',
213
+ 'socket hang up',
214
+ 'network error',
215
+ 'service unavailable',
216
+ 'bad gateway',
217
+ 'temporarily unavailable',
218
+ ]
219
+
220
+ const MAX_SCAN_CHARS = 16_384
221
+
222
+ function sample(text: unknown): string {
223
+ if (typeof text !== 'string' || text.length === 0) return ''
224
+ return (text.length > MAX_SCAN_CHARS ? text.slice(0, MAX_SCAN_CHARS) : text).toLowerCase()
225
+ }
226
+
227
+ /**
228
+ * Pull a structured HTTP status out of free-form error text. Conservative on
229
+ * purpose: only reads a 3-digit code that is LABELLED as a status/code, never a
230
+ * bare number (ids, token counts and durations are full of them).
231
+ */
232
+ export function extractHttpStatus(text: unknown): number | null {
233
+ const lower = sample(text)
234
+ if (lower.length === 0) return null
235
+ const m =
236
+ /\b(?:http[ _-]?status|status[ _-]?code|statuscode|status|http|code|error)\b\D{0,4}\b([1-5]\d\d)\b/.exec(lower) ??
237
+ /\b([1-5]\d\d)\s+(?:unauthorized|unauthorised|forbidden|payment required|too many requests)\b/.exec(lower)
238
+ if (m == null) return null
239
+ const n = Number(m[1])
240
+ return Number.isFinite(n) ? n : null
241
+ }
242
+
243
+ /**
244
+ * Classify one failed MCP tool result. `status` may be supplied by a caller
245
+ * that has a structured one; otherwise it is recovered from the text.
246
+ *
247
+ * Returns `'ordinary'` — the SILENT class — for anything not proven to be a
248
+ * key/billing problem. That default is deliberate: a false alert costs Ken's
249
+ * trust in the channel, a missed one costs a retry.
250
+ */
251
+ export function classifyMcpFailure(text: unknown, status?: unknown): McpFailureClass {
252
+ const lower = sample(text)
253
+ const httpStatus = typeof status === 'number' ? status : extractHttpStatus(text)
254
+
255
+ // 1. Credit wall — the shared registry decides, so the MCP path and the
256
+ // LiteLLM path can never disagree about what "out of money" looks like.
257
+ if (isProviderCreditStatus(httpStatus) || hasCreditExhaustionWording(lower)) return 'credit'
258
+
259
+ // 2. An EXPLICIT alerting wording beats everything below it. Deliberately
260
+ // ahead of the transient check: providers routinely wrap a hard wall in
261
+ // throttle language ("Rate limit exceeded: monthly quota exhausted"), and
262
+ // the wall is the real news.
263
+ for (const rule of CLASS_RULES) {
264
+ if (rule.signals.some(s => lower.includes(s))) return rule.cls
265
+ }
266
+
267
+ // 3. Transient beats a STATUS-ONLY inference: a throttle is not a wall, and
268
+ // a bare status with retry/timeout language is an operational hiccup.
269
+ if (TRANSIENT_SIGNALS.some(s => lower.includes(s))) return 'ordinary'
270
+
271
+ // 4. Status-only inference, last and weakest.
272
+ for (const rule of CLASS_RULES) {
273
+ if (httpStatus != null && rule.statuses.includes(httpStatus)) return rule.cls
274
+ }
275
+
276
+ return 'ordinary'
277
+ }
278
+
279
+ // ─── Ledger ──────────────────────────────────────────────────────────────────
280
+
281
+ /**
282
+ * House re-notify cadence, matched to `src/hindsight-watch/thresholds.ts`
283
+ * (`RENOTIFY_MS`) so every standing operator alert in switchroom repeats on the
284
+ * same 6-hour rhythm.
285
+ */
286
+ export const RENOTIFY_MS = 6 * 60 * 60 * 1000
287
+
288
+ export interface McpFailureAlert {
289
+ server: string
290
+ label: string
291
+ vaultKey: string
292
+ consoleUrl: string
293
+ cls: Exclude<McpFailureClass, 'ordinary'>
294
+ /** Every agent seen failing this way since the previous alert. */
295
+ agents: string[]
296
+ /** Total failures counted since the previous alert. */
297
+ occurrences: number
298
+ /** True when this is a 6-hourly repeat rather than a first firing. */
299
+ renotify: boolean
300
+ }
301
+
302
+ interface LedgerRow {
303
+ agents: Set<string>
304
+ occurrences: number
305
+ lastAlertAt: number
306
+ }
307
+
308
+ /**
309
+ * Coalescing ledger. One instance per gateway process; `now` is injected so
310
+ * tests are deterministic and no ambient clock leaks in.
311
+ */
312
+ export class McpFailureLedger {
313
+ private readonly rows = new Map<string, LedgerRow>()
314
+
315
+ constructor(private readonly renotifyMs: number = RENOTIFY_MS) {}
316
+
317
+ /**
318
+ * Record one classified failure. Returns the alert to send, or `null` when
319
+ * this occurrence is inside an already-alerted window (deduplicated).
320
+ */
321
+ note(input: {
322
+ server: string
323
+ agent: string
324
+ cls: McpFailureClass
325
+ now: number
326
+ }): McpFailureAlert | null {
327
+ if (!isAlertingClass(input.cls)) return null
328
+ const cls = input.cls as Exclude<McpFailureClass, 'ordinary'>
329
+ const key = `${input.server}::${cls}`
330
+ const row = this.rows.get(key) ?? { agents: new Set<string>(), occurrences: 0, lastAlertAt: -Infinity }
331
+ row.agents.add(input.agent)
332
+ row.occurrences += 1
333
+ this.rows.set(key, row)
334
+
335
+ const due = input.now - row.lastAlertAt >= this.renotifyMs
336
+ if (!due) return null
337
+
338
+ const renotify = Number.isFinite(row.lastAlertAt)
339
+ const meta = describeMcpServer(input.server)
340
+ const alert: McpFailureAlert = {
341
+ server: input.server,
342
+ label: meta?.label ?? input.server,
343
+ vaultKey: meta?.vaultKey ?? `${input.server}/api-key`,
344
+ consoleUrl: meta?.consoleUrl ?? '',
345
+ cls,
346
+ agents: [...row.agents].sort(),
347
+ occurrences: row.occurrences,
348
+ renotify,
349
+ }
350
+ row.lastAlertAt = input.now
351
+ row.agents = new Set<string>()
352
+ row.occurrences = 0
353
+ return alert
354
+ }
355
+
356
+ /** Test/diagnostic helper — number of tracked (server, class) pairs. */
357
+ size(): number {
358
+ return this.rows.size
359
+ }
360
+ }
361
+
362
+ // ─── The seam watcher ────────────────────────────────────────────────────────
363
+
364
+ /** Bound on the pending tool_use→name map so a leaked id can't grow it forever. */
365
+ const PENDING_TOOL_NAMES_MAX = 512
366
+
367
+ /**
368
+ * The whole seam in one object: pair `tool_use` → `tool_result` by
369
+ * `toolUseId` (the session stream reports `toolName: null` on results, so the
370
+ * name MUST be carried across), classify, coalesce, and hand back the alert to
371
+ * emit — or `null` for the overwhelmingly common case of "nothing to say".
372
+ *
373
+ * One instance per gateway process. `now` is injected; nothing here touches a
374
+ * clock, the network, or the FS.
375
+ */
376
+ export class McpFailureWatcher {
377
+ private readonly pending = new Map<string, string>()
378
+ private readonly ledger: McpFailureLedger
379
+
380
+ constructor(renotifyMs: number = RENOTIFY_MS) {
381
+ this.ledger = new McpFailureLedger(renotifyMs)
382
+ }
383
+
384
+ /** Remember the tool name for a call in flight. Non-MCP tools are ignored. */
385
+ onToolUse(toolUseId: unknown, toolName: unknown): void {
386
+ if (typeof toolUseId !== 'string' || toolUseId.length === 0) return
387
+ if (parseMcpServerFromToolName(toolName) == null) return
388
+ if (this.pending.size >= PENDING_TOOL_NAMES_MAX) {
389
+ // Evict oldest — Map preserves insertion order.
390
+ const oldest = this.pending.keys().next()
391
+ if (!oldest.done) this.pending.delete(oldest.value)
392
+ }
393
+ this.pending.set(toolUseId, toolName as string)
394
+ }
395
+
396
+ /**
397
+ * Consume a tool result. Returns an alert only when the failure is proven to
398
+ * be a key/billing problem AND the (server, class) is outside its re-notify
399
+ * window. Everything else — a success, a non-MCP tool, an ordinary error, a
400
+ * deduplicated repeat — returns `null`.
401
+ */
402
+ onToolResult(input: {
403
+ toolUseId: unknown
404
+ isError?: boolean
405
+ errorText?: unknown
406
+ agent: string
407
+ now: number
408
+ }): McpFailureAlert | null {
409
+ const id = typeof input.toolUseId === 'string' ? input.toolUseId : ''
410
+ const toolName = id.length > 0 ? this.pending.get(id) : undefined
411
+ if (id.length > 0) this.pending.delete(id)
412
+ if (input.isError !== true) return null
413
+ if (toolName == null) return null
414
+ const server = parseMcpServerFromToolName(toolName)
415
+ if (server == null) return null
416
+ const cls = classifyMcpFailure(input.errorText)
417
+ return this.ledger.note({ server, agent: input.agent, cls, now: input.now })
418
+ }
419
+ }
420
+
421
+ // ─── Rendering ───────────────────────────────────────────────────────────────
422
+
423
+ const CLASS_HEADLINE: Record<Exclude<McpFailureClass, 'ordinary'>, string> = {
424
+ credit: 'is out of credit',
425
+ credential: 'key is being rejected',
426
+ quota: 'has hit its usage quota',
427
+ }
428
+
429
+ const CLASS_ACTION: Record<Exclude<McpFailureClass, 'ordinary'>, string> = {
430
+ credit: 'Top up the balance',
431
+ credential: 'Re-issue the key and update the vault entry',
432
+ quota: 'Raise the plan limit or wait for the quota to reset',
433
+ }
434
+
435
+ /**
436
+ * The operator-card DETAIL for an MCP credential alert. Names the provider, the
437
+ * VAULT KEY NAME (never a value), the affected agents and the action.
438
+ *
439
+ * Never includes the raw provider error: the same standing rule as the
440
+ * OpenRouter 402 work — the end user sees nothing, and the operator sees a
441
+ * remedy, not a stack trace.
442
+ *
443
+ * The output is composed ENTIRELY of registry constants, counts and sanitised
444
+ * agent slugs — no provider text, no user text — which is why the card may
445
+ * render it as Markdown without escaping (see `renderOperatorEvent`).
446
+ */
447
+ export function renderMcpFailureDetail(alert: McpFailureAlert): string {
448
+ const agents =
449
+ alert.agents.length > 0
450
+ ? alert.agents.map(a => a.replace(/[^A-Za-z0-9._-]/g, '')).filter(Boolean).join(', ')
451
+ : 'unknown agent'
452
+ const console_ = alert.consoleUrl.length > 0 ? ` → ${alert.consoleUrl}` : ''
453
+ const repeat = alert.renotify ? ' (still failing)' : ''
454
+ const times = alert.occurrences === 1 ? '1 failure' : `${alert.occurrences} failures`
455
+ return (
456
+ `${alert.label} ${CLASS_HEADLINE[alert.cls]}${repeat} — ${times}, affecting: ${agents}. ` +
457
+ `${CLASS_ACTION[alert.cls]}. Vault key: \`${alert.vaultKey}\`${console_}`
458
+ )
459
+ }
@@ -38,6 +38,18 @@ export type OperatorEventKind =
38
38
  * never user-actionable — see {@link OPERATOR_ACTIONABLE_KINDS}.
39
39
  */
40
40
  | 'provider-credit-exhausted'
41
+ /**
42
+ * A paid MCP dependency (Perplexity, Eraser, Brevo, Postiz, Meta/Google Ads,
43
+ * Cloudflare …) is refusing work because its KEY is the problem — out of
44
+ * credit, rejected, or past a hard usage wall. Raised from the tool_result
45
+ * seam, not the LLM-error seam, because these providers never touch LiteLLM.
46
+ *
47
+ * Deliberately NOT raised for an ordinary tool failure (bad query, 404,
48
+ * timeout, transient 429) — see `classifyMcpFailure` in
49
+ * `mcp-credential-failure.ts`. Operator-actionable: only the operator can top
50
+ * up or re-issue a key.
51
+ */
52
+ | 'mcp-dependency-blocked'
41
53
  | 'quota-exhausted'
42
54
  | 'rate-limited'
43
55
  | 'agent-crashed'
@@ -378,6 +390,28 @@ export function renderOperatorEvent(ev: OperatorEvent): RenderResult {
378
390
  }
379
391
  }
380
392
 
393
+ // A paid MCP dependency's KEY is blocked. `ev.detail` for this kind is
394
+ // built by `renderMcpFailureDetail` from registry constants, counts and
395
+ // sanitised agent slugs ONLY — no provider text and no user text ever
396
+ // reaches it — so it is rendered as Markdown VERBATIM (escaping it would
397
+ // mangle the `vault key` code span and the console URL). It names the
398
+ // provider, the vault key NAME (never a value), the affected agents and
399
+ // the action. Dismiss-only: `/auth` cannot fix a third-party MCP key.
400
+ case 'mcp-dependency-blocked':
401
+ return {
402
+ text: [
403
+ `🔌 **Paid dependency blocked**`,
404
+ stripRawErrorBytes(ev.detail),
405
+ ]
406
+ .filter(Boolean)
407
+ .join('\n'),
408
+ keyboard: {
409
+ inline_keyboard: [
410
+ [{ text: '❌ Dismiss', callback_data: `op:dismiss:${encodeURIComponent(ev.agent)}` }],
411
+ ],
412
+ },
413
+ }
414
+
381
415
  case 'quota-exhausted':
382
416
  // Canonical quota-exhausted text (migrated from auto-fallback.ts).
383
417
  // auto-fallback.ts's buildSwitchedMessage / buildAllExhaustedMessage
@@ -611,6 +645,10 @@ export const OPERATOR_ACTIONABLE_KINDS: ReadonlySet<OperatorEventKind> = new Set
611
645
  // lose confidence. This membership is what routes it to the operator card and
612
646
  // hands the user the brief plain-language notice instead.
613
647
  'provider-credit-exhausted',
648
+ // A paid MCP dependency's key is blocked (Perplexity out of credit, a revoked
649
+ // Eraser key, …). Only the operator holds the vendor console and the vault —
650
+ // an end user shown "401 invalid api key" can do nothing with it.
651
+ 'mcp-dependency-blocked',
614
652
  'proxy-misconfig',
615
653
  ])
616
654
 
@@ -106,7 +106,7 @@ describe('restart durability — the persisted window outranks process memory',
106
106
 
107
107
  it('untightens once the window on disk closes — it is a floor, not a latch', async () => {
108
108
  const clock = new FakeClock()
109
- let remaining = 60_000
109
+ let remaining = 12_247_000
110
110
  const fuse = createEditFloodFuse({
111
111
  clock, floodWaitRemainingMs: () => remaining,
112
112
  floodProbeIntervalMs: 0, // re-read every call, for determinism here
@@ -120,6 +120,63 @@ describe('restart durability — the persisted window outranks process memory',
120
120
  expect(fuse.stats().perMessageCeiling).toBe(20)
121
121
  })
122
122
 
123
+ it('grades the persisted window by what REMAINS, not straight to maximum (#3885)', () => {
124
+ // `makeFloodWaitRecorder` writes `flood-wait.json` for EVERY 429, including
125
+ // a routine 3-second nudge. Pinning at `maxTightenLevel` for any open
126
+ // window therefore reintroduced on the persisted path the exact defect
127
+ // #3856 fixed on the in-memory one: a nudge and a 4.4h ban produced the
128
+ // same maximal response, collapsing a whole chat to 1 call/min over a
129
+ // transient blip. The severity ladder is the same one `noteFlood` uses.
130
+ const cases: [number, number][] = [
131
+ [3_000, 1], // routine burst nudge
132
+ [45_000, 2], // sustained overrate
133
+ [300_000, 3], // a real ban
134
+ [12_247_000, 4], // the 2026-07-27 magnitude — still straight to maximum
135
+ ]
136
+ for (const [remainingMs, expected] of cases) {
137
+ const fuse = createEditFloodFuse({
138
+ clock: new FakeClock(), floodWaitRemainingMs: () => remainingMs,
139
+ floodProbeIntervalMs: 0, maxTightenLevel: 4, tightenFactor: 0.5,
140
+ })
141
+ expect(fuse.stats().persistedFloodOpen).toBe(true)
142
+ expect(fuse.stats().tightenLevel).toBe(expected)
143
+ }
144
+ })
145
+
146
+ it('a single 3s 429 does not collapse the chat to 1 call/min (#3885)', async () => {
147
+ // The production regression, end to end: one transient 429 during a fleet
148
+ // restart both bumped the in-memory level AND wrote a 3-second marker that
149
+ // pinned the fuse at maximum, taking `perChatTotalMaxPerWindow: 20` down to
150
+ // an effective 1 for the whole chat.
151
+ const clock = new FakeClock()
152
+ const fuse = createEditFloodFuse({
153
+ clock, floodProbeIntervalMs: 0,
154
+ // The marker as the real recorder would write it for a 3s penalty.
155
+ floodWaitRemainingMs: () => Math.max(0, 3_000 - clock.now()),
156
+ perChatTotalMaxPerWindow: 20, perChatReplyReserve: 8, perChatSendMaxPerWindow: 25,
157
+ perTokenMaxPerWindow: 1_000, maxTightenLevel: 4, tightenFactor: 0.5,
158
+ })
159
+ await expect(
160
+ fuse.apply('sendMessage', { chat_id: CHAT, text: 'x' }, async () => { throw flood(3) }),
161
+ ).rejects.toThrow()
162
+
163
+ // One nudge is worth one level, on both paths — so the chat keeps half its
164
+ // budget, not one-twentieth of it.
165
+ expect(fuse.stats().tightenLevel).toBe(1)
166
+ expect(fuse.stats().criticalPerChatTotalCeiling).toBe(10)
167
+
168
+ // And it BINDS at that rate: 20 replies offered inside one window, ~half
169
+ // land promptly rather than one. 9 rather than 10 because the 429'd call
170
+ // above took a slot of the same window before it failed.
171
+ let landed = 0
172
+ const calls = Array.from({ length: 20 }, () =>
173
+ fuse.apply('sendMessage', { chat_id: CHAT, text: 'answer' }, async () => { landed++; return true }))
174
+ await clock.advance(1_000)
175
+ expect(landed).toBe(9)
176
+ await clock.advance(60_000)
177
+ await Promise.all(calls)
178
+ })
179
+
123
180
  it('FAILS OPEN: an unreadable marker must never gag the bot', async () => {
124
181
  const clock = new FakeClock()
125
182
  const fuse = createEditFloodFuse({