switchroom 0.19.27 → 0.19.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +5 -2
- package/dist/auth-broker/index.js +129 -8
- package/dist/cli/autoaccept-poll.js +225 -17
- package/dist/cli/notion-write-pretool.mjs +5 -2
- package/dist/cli/switchroom.js +796 -35
- package/dist/host-control/main.js +130 -9
- package/dist/vault/approvals/kernel-server.js +129 -8
- package/dist/vault/broker/server.js +129 -8
- package/package.json +3 -2
- package/profiles/_base/start.sh.hbs +70 -15
- package/telegram-plugin/dist/bridge/bridge.js +1 -0
- package/telegram-plugin/dist/gateway/gateway.js +568 -49
- package/telegram-plugin/dist/server.js +1 -0
- package/telegram-plugin/edit-flood-fuse.ts +230 -27
- package/telegram-plugin/gateway/callback-query-handlers.ts +6 -0
- package/telegram-plugin/gateway/gateway.ts +9 -2
- package/telegram-plugin/gateway/mcp-failure-hook.ts +74 -0
- package/telegram-plugin/inline-keyboard-callbacks.ts +202 -21
- package/telegram-plugin/mcp-credential-failure.ts +459 -0
- package/telegram-plugin/operator-events.ts +38 -0
- package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +58 -1
- package/telegram-plugin/tests/edit-flood-fuse-reply-reserve.test.ts +340 -0
- package/telegram-plugin/tests/finalize-callback-flood-policy.test.ts +298 -0
- package/telegram-plugin/tests/finalize-callback.test.ts +41 -8
- package/telegram-plugin/tests/mcp-credential-failure.test.ts +310 -0
- package/vendor/hindsight-memory/scripts/drain_pending.py +433 -11
- package/vendor/hindsight-memory/scripts/lib/pending.py +193 -28
- package/vendor/hindsight-memory/scripts/tests/test_drain_circuit_breaker.py +401 -0
- package/vendor/hindsight-memory/scripts/tests/test_drain_serialisation.py +286 -0
- package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py +817 -8
- package/vendor/hindsight-memory/settings.json +1 -1
- package/vendor/hindsight-memory/tests/test_hooks.py +11 -2
|
@@ -0,0 +1,459 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* mcp-credential-failure.ts — "a paid MCP dependency is blocked" detection.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS
|
|
5
|
+
* ---------------
|
|
6
|
+
* Ken: "If perplexity fails for any agent we should have hard warning via
|
|
7
|
+
* message here per other failures."
|
|
8
|
+
*
|
|
9
|
+
* Perplexity (and Eraser, Brevo, Meta Ads, Postiz, Google Ads …) reach
|
|
10
|
+
* switchroom over the MCP TOOL surface, not through LiteLLM. So the operator-
|
|
11
|
+
* event path that PR A fixed for an OpenRouter 402 never sees them: an expired
|
|
12
|
+
* Perplexity key surfaces as an ordinary `is_error` tool_result, gets rendered
|
|
13
|
+
* as a transient step in the live feed, and dies there. Nobody is told the
|
|
14
|
+
* fleet just lost a paid capability.
|
|
15
|
+
*
|
|
16
|
+
* This module is the classifier + ledger for that. It is deliberately GENERIC:
|
|
17
|
+
* Perplexity is one row in `PROVIDER_CREDIT_REGISTRY` / `MCP_SERVER_VAULT_KEYS`,
|
|
18
|
+
* not a special case, so every current and future MCP server gets the same
|
|
19
|
+
* treatment by adding DATA, never a new conditional at a call site.
|
|
20
|
+
*
|
|
21
|
+
* THE CLASSIFICATION RULE (stated explicitly, because it decides whether Ken's
|
|
22
|
+
* phone buzzes — and an alert he learns to ignore is worse than no alert):
|
|
23
|
+
*
|
|
24
|
+
* ALERT — the key itself is the problem, and only Ken can fix it:
|
|
25
|
+
* • credit — out of money (402, "insufficient credits", "payment_required")
|
|
26
|
+
* • credential— key rejected (401/403, "invalid api key", "unauthorized",
|
|
27
|
+
* "key has been disabled/revoked/blocked")
|
|
28
|
+
* • quota — hard usage wall("quota exceeded", "usage limit reached",
|
|
29
|
+
* "monthly limit", "plan limit", "over
|
|
30
|
+
* quota") — i.e. spent for the period
|
|
31
|
+
*
|
|
32
|
+
* SILENT — ordinary operational failure, retrying or rewording fixes it:
|
|
33
|
+
* • a bare rate limit / 429 with a retry-after (throttle, not a wall)
|
|
34
|
+
* • 404, 400, validation errors, a bad query, no results
|
|
35
|
+
* • timeouts, ECONNRESET, 5xx, transport errors
|
|
36
|
+
*
|
|
37
|
+
* The 429 split is the subtle one and is intentional: a throttle self-heals in
|
|
38
|
+
* seconds and must NOT page the operator; a `quota exceeded`/`usage limit`
|
|
39
|
+
* wording on the same status means the period's allowance is gone and only a
|
|
40
|
+
* plan change or a top-up clears it. Status alone is never sufficient — see the
|
|
41
|
+
* OpenAI case in `provider-credit.ts` (a spent balance arrives as 429).
|
|
42
|
+
*
|
|
43
|
+
* DEDUPLICATION — and its HONEST LIMIT
|
|
44
|
+
* ------------------------------------
|
|
45
|
+
* The ledger coalesces per `(server, class)` and re-notifies on the house
|
|
46
|
+
* cadence (`RENOTIFY_MS`, 6 h — the same interval `src/hindsight-watch` uses,
|
|
47
|
+
* so operator alerts across switchroom share one rhythm). Repeated failures
|
|
48
|
+
* inside the window are ACCUMULATED, not dropped: the next alert names every
|
|
49
|
+
* agent seen since the last one.
|
|
50
|
+
*
|
|
51
|
+
* The limit: this ledger is IN-PROCESS and therefore AGENT-LOCAL. Each agent
|
|
52
|
+
* runs its own `switchroom-<agent>` container with its own gateway process and
|
|
53
|
+
* no shared writable mount (`~/.switchroom/fleet` is read-only; per-agent state
|
|
54
|
+
* dirs are 0700 under distinct UIDs). So if six agents hit a blocked Perplexity
|
|
55
|
+
* key in the same minute, Ken gets up to six alerts — one per agent, each
|
|
56
|
+
* naming its agent — not one coalesced alert, and not thirty-six. Honouring
|
|
57
|
+
* "one alert naming the affected agents" fleet-wide needs a shared ledger the
|
|
58
|
+
* gateway does not have; that is filed as a follow-up rather than faked here.
|
|
59
|
+
*
|
|
60
|
+
* Pure module: no IPC, no bot, no FS, no network, no ambient clock (`now` is a
|
|
61
|
+
* parameter). Trivially unit-testable.
|
|
62
|
+
*/
|
|
63
|
+
|
|
64
|
+
import {
|
|
65
|
+
PROVIDER_CREDIT_REGISTRY,
|
|
66
|
+
hasCreditExhaustionWording,
|
|
67
|
+
isProviderCreditStatus,
|
|
68
|
+
type ProviderCreditEntry,
|
|
69
|
+
} from './provider-credit.js'
|
|
70
|
+
|
|
71
|
+
// ─── Server identity ─────────────────────────────────────────────────────────
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Vault key names for paid MCP servers that are NOT in the credit registry
|
|
75
|
+
* (the registry only covers providers that also serve inference). A NAME only —
|
|
76
|
+
* this module never reads or renders a secret VALUE.
|
|
77
|
+
*
|
|
78
|
+
* Adding a paid dependency is a DATA edit here.
|
|
79
|
+
*/
|
|
80
|
+
export const MCP_SERVER_VAULT_KEYS: Readonly<Record<string, { label: string; vaultKey: string; consoleUrl: string }>> = {
|
|
81
|
+
eraser: { label: 'Eraser', vaultKey: 'eraser/api-key', consoleUrl: 'https://app.eraser.io/settings/api' },
|
|
82
|
+
brevo: { label: 'Brevo', vaultKey: 'brevo/api-key', consoleUrl: 'https://app.brevo.com/settings/keys/api' },
|
|
83
|
+
postiz: { label: 'Postiz', vaultKey: 'postiz/api-key', consoleUrl: 'https://platform.postiz.com/settings' },
|
|
84
|
+
'meta-ads': { label: 'Meta Ads', vaultKey: 'meta-ads/access-token', consoleUrl: 'https://business.facebook.com/settings' },
|
|
85
|
+
'google-ads': { label: 'Google Ads', vaultKey: 'google-ads/developer-token', consoleUrl: 'https://ads.google.com/aw/apicenter' },
|
|
86
|
+
cloudflare: { label: 'Cloudflare', vaultKey: 'cloudflare/api-token', consoleUrl: 'https://dash.cloudflare.com/profile/api-tokens' },
|
|
87
|
+
context7: { label: 'Context7', vaultKey: 'context7/api-key', consoleUrl: 'https://context7.com/dashboard' },
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* `mcp__perplexity__perplexity_search` → `perplexity`. Returns `null` for a
|
|
92
|
+
* non-MCP tool (Bash, Read, …) — those are never a credential problem, so the
|
|
93
|
+
* caller short-circuits on `null` and the hot path costs one `startsWith`.
|
|
94
|
+
*/
|
|
95
|
+
export function parseMcpServerFromToolName(toolName: unknown): string | null {
|
|
96
|
+
if (typeof toolName !== 'string') return null
|
|
97
|
+
if (!toolName.startsWith('mcp__')) return null
|
|
98
|
+
const rest = toolName.slice('mcp__'.length)
|
|
99
|
+
const end = rest.indexOf('__')
|
|
100
|
+
const server = end === -1 ? rest : rest.slice(0, end)
|
|
101
|
+
return server.length > 0 ? server : null
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Remedy metadata for a server, from either data source. `null` when unknown. */
|
|
105
|
+
export function describeMcpServer(
|
|
106
|
+
server: string,
|
|
107
|
+
): { label: string; vaultKey: string; consoleUrl: string } | null {
|
|
108
|
+
const direct = MCP_SERVER_VAULT_KEYS[server]
|
|
109
|
+
if (direct != null) return direct
|
|
110
|
+
const entry: ProviderCreditEntry | undefined = PROVIDER_CREDIT_REGISTRY.find(e => e.id === server)
|
|
111
|
+
if (entry != null) {
|
|
112
|
+
return { label: entry.label, vaultKey: entry.vaultKey, consoleUrl: entry.consoleUrl }
|
|
113
|
+
}
|
|
114
|
+
return null
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// ─── Classification ──────────────────────────────────────────────────────────
|
|
118
|
+
|
|
119
|
+
/** `ordinary` is the SILENT class — everything else raises an operator alert. */
|
|
120
|
+
export type McpFailureClass = 'credit' | 'credential' | 'quota' | 'ordinary'
|
|
121
|
+
|
|
122
|
+
/** True for the classes that page the operator. */
|
|
123
|
+
export function isAlertingClass(c: McpFailureClass): boolean {
|
|
124
|
+
return c !== 'ordinary'
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
interface ClassRule {
|
|
128
|
+
cls: Exclude<McpFailureClass, 'ordinary'>
|
|
129
|
+
/** Structured HTTP statuses that imply this class on their own. */
|
|
130
|
+
statuses: readonly number[]
|
|
131
|
+
/** Lowercase wordings that imply this class. */
|
|
132
|
+
signals: readonly string[]
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Evaluated IN ORDER — first match wins. Credit precedes credential because a
|
|
137
|
+
* spent Perplexity balance answers 401 with "insufficient credits", and the
|
|
138
|
+
* remedy is "top up", not "re-issue the key".
|
|
139
|
+
*/
|
|
140
|
+
const CLASS_RULES: readonly ClassRule[] = [
|
|
141
|
+
// `credit` is handled ahead of this table via the shared registry predicates.
|
|
142
|
+
{
|
|
143
|
+
cls: 'credential',
|
|
144
|
+
statuses: [401, 403],
|
|
145
|
+
signals: [
|
|
146
|
+
'invalid api key',
|
|
147
|
+
'invalid_api_key',
|
|
148
|
+
'incorrect api key',
|
|
149
|
+
'invalid authentication',
|
|
150
|
+
'authentication_error',
|
|
151
|
+
'authentication failed',
|
|
152
|
+
'unauthorized',
|
|
153
|
+
'unauthorised',
|
|
154
|
+
'forbidden',
|
|
155
|
+
'api key not found',
|
|
156
|
+
'no api key provided',
|
|
157
|
+
'missing api key',
|
|
158
|
+
'api key expired',
|
|
159
|
+
'key has been disabled',
|
|
160
|
+
'key has been revoked',
|
|
161
|
+
'key is disabled',
|
|
162
|
+
'account has been blocked',
|
|
163
|
+
'account is blocked',
|
|
164
|
+
'account suspended',
|
|
165
|
+
'permission_denied',
|
|
166
|
+
'token expired',
|
|
167
|
+
'invalid token',
|
|
168
|
+
],
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
cls: 'quota',
|
|
172
|
+
// NOTE: 429 is deliberately NOT listed. A bare 429 is a throttle and must
|
|
173
|
+
// stay silent; only the hard-wall WORDINGS below promote it to an alert.
|
|
174
|
+
statuses: [],
|
|
175
|
+
signals: [
|
|
176
|
+
'quota exceeded',
|
|
177
|
+
'quota_exceeded',
|
|
178
|
+
'over quota',
|
|
179
|
+
'usage limit reached',
|
|
180
|
+
'usage limit exceeded',
|
|
181
|
+
'monthly limit',
|
|
182
|
+
'monthly quota',
|
|
183
|
+
'plan limit',
|
|
184
|
+
'plan_limit',
|
|
185
|
+
'limit reached for your plan',
|
|
186
|
+
'upgrade your plan',
|
|
187
|
+
'spending limit',
|
|
188
|
+
// NOTE: "exceeded your current quota" is deliberately NOT here. It is
|
|
189
|
+
// OpenAI's BILLING wording and already lives in `CREDIT_EXHAUSTION_SIGNALS`,
|
|
190
|
+
// where it is matched first — duplicating it would be dead data that
|
|
191
|
+
// silently disagrees with the shared registry.
|
|
192
|
+
],
|
|
193
|
+
},
|
|
194
|
+
]
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Wordings that mean "transient throttle / operational hiccup" and must keep a
|
|
198
|
+
* failure SILENT even if a broader signal would otherwise match. Checked before
|
|
199
|
+
* the alerting rules so `429 rate limit, retry after 2s` never pages anyone.
|
|
200
|
+
*/
|
|
201
|
+
const TRANSIENT_SIGNALS: readonly string[] = [
|
|
202
|
+
'rate limit',
|
|
203
|
+
'rate_limit',
|
|
204
|
+
'too many requests',
|
|
205
|
+
'retry after',
|
|
206
|
+
'retry-after',
|
|
207
|
+
'slow down',
|
|
208
|
+
'timeout',
|
|
209
|
+
'timed out',
|
|
210
|
+
'etimedout',
|
|
211
|
+
'econnreset',
|
|
212
|
+
'econnrefused',
|
|
213
|
+
'socket hang up',
|
|
214
|
+
'network error',
|
|
215
|
+
'service unavailable',
|
|
216
|
+
'bad gateway',
|
|
217
|
+
'temporarily unavailable',
|
|
218
|
+
]
|
|
219
|
+
|
|
220
|
+
const MAX_SCAN_CHARS = 16_384
|
|
221
|
+
|
|
222
|
+
function sample(text: unknown): string {
|
|
223
|
+
if (typeof text !== 'string' || text.length === 0) return ''
|
|
224
|
+
return (text.length > MAX_SCAN_CHARS ? text.slice(0, MAX_SCAN_CHARS) : text).toLowerCase()
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Pull a structured HTTP status out of free-form error text. Conservative on
|
|
229
|
+
* purpose: only reads a 3-digit code that is LABELLED as a status/code, never a
|
|
230
|
+
* bare number (ids, token counts and durations are full of them).
|
|
231
|
+
*/
|
|
232
|
+
export function extractHttpStatus(text: unknown): number | null {
|
|
233
|
+
const lower = sample(text)
|
|
234
|
+
if (lower.length === 0) return null
|
|
235
|
+
const m =
|
|
236
|
+
/\b(?:http[ _-]?status|status[ _-]?code|statuscode|status|http|code|error)\b\D{0,4}\b([1-5]\d\d)\b/.exec(lower) ??
|
|
237
|
+
/\b([1-5]\d\d)\s+(?:unauthorized|unauthorised|forbidden|payment required|too many requests)\b/.exec(lower)
|
|
238
|
+
if (m == null) return null
|
|
239
|
+
const n = Number(m[1])
|
|
240
|
+
return Number.isFinite(n) ? n : null
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Classify one failed MCP tool result. `status` may be supplied by a caller
|
|
245
|
+
* that has a structured one; otherwise it is recovered from the text.
|
|
246
|
+
*
|
|
247
|
+
* Returns `'ordinary'` — the SILENT class — for anything not proven to be a
|
|
248
|
+
* key/billing problem. That default is deliberate: a false alert costs Ken's
|
|
249
|
+
* trust in the channel, a missed one costs a retry.
|
|
250
|
+
*/
|
|
251
|
+
export function classifyMcpFailure(text: unknown, status?: unknown): McpFailureClass {
|
|
252
|
+
const lower = sample(text)
|
|
253
|
+
const httpStatus = typeof status === 'number' ? status : extractHttpStatus(text)
|
|
254
|
+
|
|
255
|
+
// 1. Credit wall — the shared registry decides, so the MCP path and the
|
|
256
|
+
// LiteLLM path can never disagree about what "out of money" looks like.
|
|
257
|
+
if (isProviderCreditStatus(httpStatus) || hasCreditExhaustionWording(lower)) return 'credit'
|
|
258
|
+
|
|
259
|
+
// 2. An EXPLICIT alerting wording beats everything below it. Deliberately
|
|
260
|
+
// ahead of the transient check: providers routinely wrap a hard wall in
|
|
261
|
+
// throttle language ("Rate limit exceeded: monthly quota exhausted"), and
|
|
262
|
+
// the wall is the real news.
|
|
263
|
+
for (const rule of CLASS_RULES) {
|
|
264
|
+
if (rule.signals.some(s => lower.includes(s))) return rule.cls
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// 3. Transient beats a STATUS-ONLY inference: a throttle is not a wall, and
|
|
268
|
+
// a bare status with retry/timeout language is an operational hiccup.
|
|
269
|
+
if (TRANSIENT_SIGNALS.some(s => lower.includes(s))) return 'ordinary'
|
|
270
|
+
|
|
271
|
+
// 4. Status-only inference, last and weakest.
|
|
272
|
+
for (const rule of CLASS_RULES) {
|
|
273
|
+
if (httpStatus != null && rule.statuses.includes(httpStatus)) return rule.cls
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
return 'ordinary'
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// ─── Ledger ──────────────────────────────────────────────────────────────────
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* House re-notify cadence, matched to `src/hindsight-watch/thresholds.ts`
|
|
283
|
+
* (`RENOTIFY_MS`) so every standing operator alert in switchroom repeats on the
|
|
284
|
+
* same 6-hour rhythm.
|
|
285
|
+
*/
|
|
286
|
+
export const RENOTIFY_MS = 6 * 60 * 60 * 1000
|
|
287
|
+
|
|
288
|
+
export interface McpFailureAlert {
|
|
289
|
+
server: string
|
|
290
|
+
label: string
|
|
291
|
+
vaultKey: string
|
|
292
|
+
consoleUrl: string
|
|
293
|
+
cls: Exclude<McpFailureClass, 'ordinary'>
|
|
294
|
+
/** Every agent seen failing this way since the previous alert. */
|
|
295
|
+
agents: string[]
|
|
296
|
+
/** Total failures counted since the previous alert. */
|
|
297
|
+
occurrences: number
|
|
298
|
+
/** True when this is a 6-hourly repeat rather than a first firing. */
|
|
299
|
+
renotify: boolean
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
interface LedgerRow {
|
|
303
|
+
agents: Set<string>
|
|
304
|
+
occurrences: number
|
|
305
|
+
lastAlertAt: number
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* Coalescing ledger. One instance per gateway process; `now` is injected so
|
|
310
|
+
* tests are deterministic and no ambient clock leaks in.
|
|
311
|
+
*/
|
|
312
|
+
export class McpFailureLedger {
|
|
313
|
+
private readonly rows = new Map<string, LedgerRow>()
|
|
314
|
+
|
|
315
|
+
constructor(private readonly renotifyMs: number = RENOTIFY_MS) {}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Record one classified failure. Returns the alert to send, or `null` when
|
|
319
|
+
* this occurrence is inside an already-alerted window (deduplicated).
|
|
320
|
+
*/
|
|
321
|
+
note(input: {
|
|
322
|
+
server: string
|
|
323
|
+
agent: string
|
|
324
|
+
cls: McpFailureClass
|
|
325
|
+
now: number
|
|
326
|
+
}): McpFailureAlert | null {
|
|
327
|
+
if (!isAlertingClass(input.cls)) return null
|
|
328
|
+
const cls = input.cls as Exclude<McpFailureClass, 'ordinary'>
|
|
329
|
+
const key = `${input.server}::${cls}`
|
|
330
|
+
const row = this.rows.get(key) ?? { agents: new Set<string>(), occurrences: 0, lastAlertAt: -Infinity }
|
|
331
|
+
row.agents.add(input.agent)
|
|
332
|
+
row.occurrences += 1
|
|
333
|
+
this.rows.set(key, row)
|
|
334
|
+
|
|
335
|
+
const due = input.now - row.lastAlertAt >= this.renotifyMs
|
|
336
|
+
if (!due) return null
|
|
337
|
+
|
|
338
|
+
const renotify = Number.isFinite(row.lastAlertAt)
|
|
339
|
+
const meta = describeMcpServer(input.server)
|
|
340
|
+
const alert: McpFailureAlert = {
|
|
341
|
+
server: input.server,
|
|
342
|
+
label: meta?.label ?? input.server,
|
|
343
|
+
vaultKey: meta?.vaultKey ?? `${input.server}/api-key`,
|
|
344
|
+
consoleUrl: meta?.consoleUrl ?? '',
|
|
345
|
+
cls,
|
|
346
|
+
agents: [...row.agents].sort(),
|
|
347
|
+
occurrences: row.occurrences,
|
|
348
|
+
renotify,
|
|
349
|
+
}
|
|
350
|
+
row.lastAlertAt = input.now
|
|
351
|
+
row.agents = new Set<string>()
|
|
352
|
+
row.occurrences = 0
|
|
353
|
+
return alert
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/** Test/diagnostic helper — number of tracked (server, class) pairs. */
|
|
357
|
+
size(): number {
|
|
358
|
+
return this.rows.size
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// ─── The seam watcher ────────────────────────────────────────────────────────
|
|
363
|
+
|
|
364
|
+
/** Bound on the pending tool_use→name map so a leaked id can't grow it forever. */
|
|
365
|
+
const PENDING_TOOL_NAMES_MAX = 512
|
|
366
|
+
|
|
367
|
+
/**
|
|
368
|
+
* The whole seam in one object: pair `tool_use` → `tool_result` by
|
|
369
|
+
* `toolUseId` (the session stream reports `toolName: null` on results, so the
|
|
370
|
+
* name MUST be carried across), classify, coalesce, and hand back the alert to
|
|
371
|
+
* emit — or `null` for the overwhelmingly common case of "nothing to say".
|
|
372
|
+
*
|
|
373
|
+
* One instance per gateway process. `now` is injected; nothing here touches a
|
|
374
|
+
* clock, the network, or the FS.
|
|
375
|
+
*/
|
|
376
|
+
export class McpFailureWatcher {
|
|
377
|
+
private readonly pending = new Map<string, string>()
|
|
378
|
+
private readonly ledger: McpFailureLedger
|
|
379
|
+
|
|
380
|
+
constructor(renotifyMs: number = RENOTIFY_MS) {
|
|
381
|
+
this.ledger = new McpFailureLedger(renotifyMs)
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/** Remember the tool name for a call in flight. Non-MCP tools are ignored. */
|
|
385
|
+
onToolUse(toolUseId: unknown, toolName: unknown): void {
|
|
386
|
+
if (typeof toolUseId !== 'string' || toolUseId.length === 0) return
|
|
387
|
+
if (parseMcpServerFromToolName(toolName) == null) return
|
|
388
|
+
if (this.pending.size >= PENDING_TOOL_NAMES_MAX) {
|
|
389
|
+
// Evict oldest — Map preserves insertion order.
|
|
390
|
+
const oldest = this.pending.keys().next()
|
|
391
|
+
if (!oldest.done) this.pending.delete(oldest.value)
|
|
392
|
+
}
|
|
393
|
+
this.pending.set(toolUseId, toolName as string)
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* Consume a tool result. Returns an alert only when the failure is proven to
|
|
398
|
+
* be a key/billing problem AND the (server, class) is outside its re-notify
|
|
399
|
+
* window. Everything else — a success, a non-MCP tool, an ordinary error, a
|
|
400
|
+
* deduplicated repeat — returns `null`.
|
|
401
|
+
*/
|
|
402
|
+
onToolResult(input: {
|
|
403
|
+
toolUseId: unknown
|
|
404
|
+
isError?: boolean
|
|
405
|
+
errorText?: unknown
|
|
406
|
+
agent: string
|
|
407
|
+
now: number
|
|
408
|
+
}): McpFailureAlert | null {
|
|
409
|
+
const id = typeof input.toolUseId === 'string' ? input.toolUseId : ''
|
|
410
|
+
const toolName = id.length > 0 ? this.pending.get(id) : undefined
|
|
411
|
+
if (id.length > 0) this.pending.delete(id)
|
|
412
|
+
if (input.isError !== true) return null
|
|
413
|
+
if (toolName == null) return null
|
|
414
|
+
const server = parseMcpServerFromToolName(toolName)
|
|
415
|
+
if (server == null) return null
|
|
416
|
+
const cls = classifyMcpFailure(input.errorText)
|
|
417
|
+
return this.ledger.note({ server, agent: input.agent, cls, now: input.now })
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// ─── Rendering ───────────────────────────────────────────────────────────────
|
|
422
|
+
|
|
423
|
+
const CLASS_HEADLINE: Record<Exclude<McpFailureClass, 'ordinary'>, string> = {
|
|
424
|
+
credit: 'is out of credit',
|
|
425
|
+
credential: 'key is being rejected',
|
|
426
|
+
quota: 'has hit its usage quota',
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
const CLASS_ACTION: Record<Exclude<McpFailureClass, 'ordinary'>, string> = {
|
|
430
|
+
credit: 'Top up the balance',
|
|
431
|
+
credential: 'Re-issue the key and update the vault entry',
|
|
432
|
+
quota: 'Raise the plan limit or wait for the quota to reset',
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/**
|
|
436
|
+
* The operator-card DETAIL for an MCP credential alert. Names the provider, the
|
|
437
|
+
* VAULT KEY NAME (never a value), the affected agents and the action.
|
|
438
|
+
*
|
|
439
|
+
* Never includes the raw provider error: the same standing rule as the
|
|
440
|
+
* OpenRouter 402 work — the end user sees nothing, and the operator sees a
|
|
441
|
+
* remedy, not a stack trace.
|
|
442
|
+
*
|
|
443
|
+
* The output is composed ENTIRELY of registry constants, counts and sanitised
|
|
444
|
+
* agent slugs — no provider text, no user text — which is why the card may
|
|
445
|
+
* render it as Markdown without escaping (see `renderOperatorEvent`).
|
|
446
|
+
*/
|
|
447
|
+
export function renderMcpFailureDetail(alert: McpFailureAlert): string {
|
|
448
|
+
const agents =
|
|
449
|
+
alert.agents.length > 0
|
|
450
|
+
? alert.agents.map(a => a.replace(/[^A-Za-z0-9._-]/g, '')).filter(Boolean).join(', ')
|
|
451
|
+
: 'unknown agent'
|
|
452
|
+
const console_ = alert.consoleUrl.length > 0 ? ` → ${alert.consoleUrl}` : ''
|
|
453
|
+
const repeat = alert.renotify ? ' (still failing)' : ''
|
|
454
|
+
const times = alert.occurrences === 1 ? '1 failure' : `${alert.occurrences} failures`
|
|
455
|
+
return (
|
|
456
|
+
`${alert.label} ${CLASS_HEADLINE[alert.cls]}${repeat} — ${times}, affecting: ${agents}. ` +
|
|
457
|
+
`${CLASS_ACTION[alert.cls]}. Vault key: \`${alert.vaultKey}\`${console_}`
|
|
458
|
+
)
|
|
459
|
+
}
|
|
@@ -38,6 +38,18 @@ export type OperatorEventKind =
|
|
|
38
38
|
* never user-actionable — see {@link OPERATOR_ACTIONABLE_KINDS}.
|
|
39
39
|
*/
|
|
40
40
|
| 'provider-credit-exhausted'
|
|
41
|
+
/**
|
|
42
|
+
* A paid MCP dependency (Perplexity, Eraser, Brevo, Postiz, Meta/Google Ads,
|
|
43
|
+
* Cloudflare …) is refusing work because its KEY is the problem — out of
|
|
44
|
+
* credit, rejected, or past a hard usage wall. Raised from the tool_result
|
|
45
|
+
* seam, not the LLM-error seam, because these providers never touch LiteLLM.
|
|
46
|
+
*
|
|
47
|
+
* Deliberately NOT raised for an ordinary tool failure (bad query, 404,
|
|
48
|
+
* timeout, transient 429) — see `classifyMcpFailure` in
|
|
49
|
+
* `mcp-credential-failure.ts`. Operator-actionable: only the operator can top
|
|
50
|
+
* up or re-issue a key.
|
|
51
|
+
*/
|
|
52
|
+
| 'mcp-dependency-blocked'
|
|
41
53
|
| 'quota-exhausted'
|
|
42
54
|
| 'rate-limited'
|
|
43
55
|
| 'agent-crashed'
|
|
@@ -378,6 +390,28 @@ export function renderOperatorEvent(ev: OperatorEvent): RenderResult {
|
|
|
378
390
|
}
|
|
379
391
|
}
|
|
380
392
|
|
|
393
|
+
// A paid MCP dependency's KEY is blocked. `ev.detail` for this kind is
|
|
394
|
+
// built by `renderMcpFailureDetail` from registry constants, counts and
|
|
395
|
+
// sanitised agent slugs ONLY — no provider text and no user text ever
|
|
396
|
+
// reaches it — so it is rendered as Markdown VERBATIM (escaping it would
|
|
397
|
+
// mangle the `vault key` code span and the console URL). It names the
|
|
398
|
+
// provider, the vault key NAME (never a value), the affected agents and
|
|
399
|
+
// the action. Dismiss-only: `/auth` cannot fix a third-party MCP key.
|
|
400
|
+
case 'mcp-dependency-blocked':
|
|
401
|
+
return {
|
|
402
|
+
text: [
|
|
403
|
+
`🔌 **Paid dependency blocked**`,
|
|
404
|
+
stripRawErrorBytes(ev.detail),
|
|
405
|
+
]
|
|
406
|
+
.filter(Boolean)
|
|
407
|
+
.join('\n'),
|
|
408
|
+
keyboard: {
|
|
409
|
+
inline_keyboard: [
|
|
410
|
+
[{ text: '❌ Dismiss', callback_data: `op:dismiss:${encodeURIComponent(ev.agent)}` }],
|
|
411
|
+
],
|
|
412
|
+
},
|
|
413
|
+
}
|
|
414
|
+
|
|
381
415
|
case 'quota-exhausted':
|
|
382
416
|
// Canonical quota-exhausted text (migrated from auto-fallback.ts).
|
|
383
417
|
// auto-fallback.ts's buildSwitchedMessage / buildAllExhaustedMessage
|
|
@@ -611,6 +645,10 @@ export const OPERATOR_ACTIONABLE_KINDS: ReadonlySet<OperatorEventKind> = new Set
|
|
|
611
645
|
// lose confidence. This membership is what routes it to the operator card and
|
|
612
646
|
// hands the user the brief plain-language notice instead.
|
|
613
647
|
'provider-credit-exhausted',
|
|
648
|
+
// A paid MCP dependency's key is blocked (Perplexity out of credit, a revoked
|
|
649
|
+
// Eraser key, …). Only the operator holds the vendor console and the vault —
|
|
650
|
+
// an end user shown "401 invalid api key" can do nothing with it.
|
|
651
|
+
'mcp-dependency-blocked',
|
|
614
652
|
'proxy-misconfig',
|
|
615
653
|
])
|
|
616
654
|
|
|
@@ -106,7 +106,7 @@ describe('restart durability — the persisted window outranks process memory',
|
|
|
106
106
|
|
|
107
107
|
it('untightens once the window on disk closes — it is a floor, not a latch', async () => {
|
|
108
108
|
const clock = new FakeClock()
|
|
109
|
-
let remaining =
|
|
109
|
+
let remaining = 12_247_000
|
|
110
110
|
const fuse = createEditFloodFuse({
|
|
111
111
|
clock, floodWaitRemainingMs: () => remaining,
|
|
112
112
|
floodProbeIntervalMs: 0, // re-read every call, for determinism here
|
|
@@ -120,6 +120,63 @@ describe('restart durability — the persisted window outranks process memory',
|
|
|
120
120
|
expect(fuse.stats().perMessageCeiling).toBe(20)
|
|
121
121
|
})
|
|
122
122
|
|
|
123
|
+
it('grades the persisted window by what REMAINS, not straight to maximum (#3885)', () => {
|
|
124
|
+
// `makeFloodWaitRecorder` writes `flood-wait.json` for EVERY 429, including
|
|
125
|
+
// a routine 3-second nudge. Pinning at `maxTightenLevel` for any open
|
|
126
|
+
// window therefore reintroduced on the persisted path the exact defect
|
|
127
|
+
// #3856 fixed on the in-memory one: a nudge and a 4.4h ban produced the
|
|
128
|
+
// same maximal response, collapsing a whole chat to 1 call/min over a
|
|
129
|
+
// transient blip. The severity ladder is the same one `noteFlood` uses.
|
|
130
|
+
const cases: [number, number][] = [
|
|
131
|
+
[3_000, 1], // routine burst nudge
|
|
132
|
+
[45_000, 2], // sustained overrate
|
|
133
|
+
[300_000, 3], // a real ban
|
|
134
|
+
[12_247_000, 4], // the 2026-07-27 magnitude — still straight to maximum
|
|
135
|
+
]
|
|
136
|
+
for (const [remainingMs, expected] of cases) {
|
|
137
|
+
const fuse = createEditFloodFuse({
|
|
138
|
+
clock: new FakeClock(), floodWaitRemainingMs: () => remainingMs,
|
|
139
|
+
floodProbeIntervalMs: 0, maxTightenLevel: 4, tightenFactor: 0.5,
|
|
140
|
+
})
|
|
141
|
+
expect(fuse.stats().persistedFloodOpen).toBe(true)
|
|
142
|
+
expect(fuse.stats().tightenLevel).toBe(expected)
|
|
143
|
+
}
|
|
144
|
+
})
|
|
145
|
+
|
|
146
|
+
it('a single 3s 429 does not collapse the chat to 1 call/min (#3885)', async () => {
|
|
147
|
+
// The production regression, end to end: one transient 429 during a fleet
|
|
148
|
+
// restart both bumped the in-memory level AND wrote a 3-second marker that
|
|
149
|
+
// pinned the fuse at maximum, taking `perChatTotalMaxPerWindow: 20` down to
|
|
150
|
+
// an effective 1 for the whole chat.
|
|
151
|
+
const clock = new FakeClock()
|
|
152
|
+
const fuse = createEditFloodFuse({
|
|
153
|
+
clock, floodProbeIntervalMs: 0,
|
|
154
|
+
// The marker as the real recorder would write it for a 3s penalty.
|
|
155
|
+
floodWaitRemainingMs: () => Math.max(0, 3_000 - clock.now()),
|
|
156
|
+
perChatTotalMaxPerWindow: 20, perChatReplyReserve: 8, perChatSendMaxPerWindow: 25,
|
|
157
|
+
perTokenMaxPerWindow: 1_000, maxTightenLevel: 4, tightenFactor: 0.5,
|
|
158
|
+
})
|
|
159
|
+
await expect(
|
|
160
|
+
fuse.apply('sendMessage', { chat_id: CHAT, text: 'x' }, async () => { throw flood(3) }),
|
|
161
|
+
).rejects.toThrow()
|
|
162
|
+
|
|
163
|
+
// One nudge is worth one level, on both paths — so the chat keeps half its
|
|
164
|
+
// budget, not one-twentieth of it.
|
|
165
|
+
expect(fuse.stats().tightenLevel).toBe(1)
|
|
166
|
+
expect(fuse.stats().criticalPerChatTotalCeiling).toBe(10)
|
|
167
|
+
|
|
168
|
+
// And it BINDS at that rate: 20 replies offered inside one window, ~half
|
|
169
|
+
// land promptly rather than one. 9 rather than 10 because the 429'd call
|
|
170
|
+
// above took a slot of the same window before it failed.
|
|
171
|
+
let landed = 0
|
|
172
|
+
const calls = Array.from({ length: 20 }, () =>
|
|
173
|
+
fuse.apply('sendMessage', { chat_id: CHAT, text: 'answer' }, async () => { landed++; return true }))
|
|
174
|
+
await clock.advance(1_000)
|
|
175
|
+
expect(landed).toBe(9)
|
|
176
|
+
await clock.advance(60_000)
|
|
177
|
+
await Promise.all(calls)
|
|
178
|
+
})
|
|
179
|
+
|
|
123
180
|
it('FAILS OPEN: an unreadable marker must never gag the bot', async () => {
|
|
124
181
|
const clock = new FakeClock()
|
|
125
182
|
const fuse = createEditFloodFuse({
|