switchroom 0.19.26 → 0.19.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/git-agent-attribution-hook.sh +144 -0
- package/dist/agent-scheduler/index.js +60 -2
- package/dist/auth-broker/index.js +244 -13
- package/dist/cli/autoaccept-poll.js +225 -17
- package/dist/cli/notion-write-pretool.mjs +60 -2
- package/dist/cli/switchroom.js +2843 -1220
- package/dist/host-control/main.js +245 -14
- package/dist/vault/approvals/kernel-server.js +242 -13
- package/dist/vault/broker/server.js +242 -13
- package/package.json +7 -2
- package/profiles/_base/cron-session.sh.hbs +8 -0
- package/profiles/_base/start.sh.hbs +175 -15
- package/telegram-plugin/card-layout.ts +328 -0
- package/telegram-plugin/dist/bridge/bridge.js +94 -1
- package/telegram-plugin/dist/gateway/gateway.js +2544 -1182
- package/telegram-plugin/dist/server.js +97 -1
- package/telegram-plugin/edit-flood-fuse.ts +841 -57
- package/telegram-plugin/flood-429-ledger.ts +526 -0
- package/telegram-plugin/flood-circuit-breaker.ts +18 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +6 -0
- package/telegram-plugin/gateway/flood-reply-queue.ts +168 -0
- package/telegram-plugin/gateway/gateway.ts +67 -70
- package/telegram-plugin/gateway/mcp-failure-hook.ts +74 -0
- package/telegram-plugin/gateway/narrative-lane.ts +14 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +36 -0
- package/telegram-plugin/gateway/outbox-sweep.ts +183 -6
- package/telegram-plugin/gateway/pinned-message-handler.ts +12 -16
- package/telegram-plugin/gateway/status-pin-retarget.ts +72 -36
- package/telegram-plugin/gateway/status-pin-store.ts +58 -9
- package/telegram-plugin/gateway/worker-pin-reaper.ts +56 -7
- package/telegram-plugin/inline-keyboard-callbacks.ts +202 -21
- package/telegram-plugin/llm-error-present.ts +61 -2
- package/telegram-plugin/mcp-credential-failure.ts +459 -0
- package/telegram-plugin/model-unavailable.ts +8 -0
- package/telegram-plugin/operator-events.ts +110 -5
- package/telegram-plugin/outbound-class.ts +81 -0
- package/telegram-plugin/provider-credit.ts +237 -0
- package/telegram-plugin/scripts/bun-test-ci.sh +36 -6
- package/telegram-plugin/send-gate.ts +24 -2
- package/telegram-plugin/status-no-truncate.ts +10 -48
- package/telegram-plugin/status-pin-driver.ts +33 -45
- package/telegram-plugin/status-pin.ts +18 -1
- package/telegram-plugin/tests/card-golden.test.ts +69 -0
- package/telegram-plugin/tests/card-lifecycle-render.test.ts +362 -0
- package/telegram-plugin/tests/card-type-distinguishability.test.ts +187 -164
- package/telegram-plugin/tests/card-variants.golden.txt +211 -0
- package/telegram-plugin/tests/card-variants.ts +366 -0
- package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +373 -0
- package/telegram-plugin/tests/edit-flood-fuse-default-deny.test.ts +319 -0
- package/telegram-plugin/tests/edit-flood-fuse-reply-reserve.test.ts +340 -0
- package/telegram-plugin/tests/edit-flood-fuse.test.ts +11 -2
- package/telegram-plugin/tests/feed-edit-rate-ceiling.test.ts +462 -0
- package/telegram-plugin/tests/finalize-callback-flood-policy.test.ts +298 -0
- package/telegram-plugin/tests/finalize-callback.test.ts +41 -8
- package/telegram-plugin/tests/fixtures/real-429-stream.ts +220 -0
- package/telegram-plugin/tests/flood-429-ledger.test.ts +278 -0
- package/telegram-plugin/tests/flood-429-recorder-wiring.test.ts +128 -0
- package/telegram-plugin/tests/flood-reply-queue.test.ts +418 -0
- package/telegram-plugin/tests/mcp-credential-failure.test.ts +310 -0
- package/telegram-plugin/tests/outbox-sweep-flood-breaker.test.ts +221 -0
- package/telegram-plugin/tests/pinned-card-collapse.test.ts +19 -24
- package/telegram-plugin/tests/pinned-message-handler.test.ts +15 -15
- package/telegram-plugin/tests/provider-credit-402.test.ts +243 -0
- package/telegram-plugin/tests/status-pin-api.test.ts +11 -11
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +36 -37
- package/telegram-plugin/tests/status-pin-lifecycle.test.ts +602 -0
- package/telegram-plugin/tests/status-pin-retarget.test.ts +90 -62
- package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +7 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +109 -60
- package/telegram-plugin/tests/status-pin.test.ts +56 -5
- package/telegram-plugin/tests/test-runner-coverage.test.ts +133 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +12 -10
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +23 -29
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +56 -59
- package/telegram-plugin/tests/worker-feed-terminal-edit-class.test.ts +335 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +1 -1
- package/telegram-plugin/tool-activity-summary.ts +239 -365
- package/telegram-plugin/uat/assertions.ts +22 -11
- package/telegram-plugin/uat/feed-matcher.test.ts +24 -17
- package/telegram-plugin/worker-activity-feed.ts +105 -47
- package/vendor/hindsight-memory/CLAUDE.md +45 -0
- package/vendor/hindsight-memory/scripts/drain_pending.py +433 -11
- package/vendor/hindsight-memory/scripts/lib/config.py +33 -0
- package/vendor/hindsight-memory/scripts/lib/pending.py +193 -28
- package/vendor/hindsight-memory/scripts/recall.py +176 -7
- package/vendor/hindsight-memory/scripts/tests/test_config_recall_passthrough_env.py +170 -0
- package/vendor/hindsight-memory/scripts/tests/test_drain_circuit_breaker.py +401 -0
- package/vendor/hindsight-memory/scripts/tests/test_drain_serialisation.py +286 -0
- package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py +817 -8
- package/vendor/hindsight-memory/scripts/tests/test_recall_min_score.py +464 -0
- package/vendor/hindsight-memory/settings.json +1 -1
- package/vendor/hindsight-memory/tests/test_hooks.py +11 -2
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* mcp-credential-failure.test.ts — outcome tests for "a paid MCP dependency's
|
|
3
|
+
* key is blocked → Ken gets ONE hard warning".
|
|
4
|
+
*
|
|
5
|
+
* THE GAP. Perplexity (and Eraser, Brevo, Postiz, Meta/Google Ads, Cloudflare)
|
|
6
|
+
* reach switchroom over the MCP TOOL surface, never through LiteLLM. So the
|
|
7
|
+
* operator-event path that PR A fixed for an OpenRouter 402 never sees them: an
|
|
8
|
+
* expired Perplexity key surfaced as an ordinary red step in the live feed and
|
|
9
|
+
* died there. Nobody was told the fleet had lost a paid capability.
|
|
10
|
+
*
|
|
11
|
+
* These assert the OBSERVABLE RESULT at the gateway seam, composing the SAME
|
|
12
|
+
* production functions in the SAME order `handleSessionEvent` →
|
|
13
|
+
* `noteMcpDependencyFailure` → `emitGatewayOperatorEvent` calls them:
|
|
14
|
+
* McpFailureWatcher.onToolUse/onToolResult → renderMcpFailureDetail →
|
|
15
|
+
* renderOperatorEvent → decideOperatorEventAudience →
|
|
16
|
+
* renderUserFacingFailureNotice.
|
|
17
|
+
*
|
|
18
|
+
* The three guarantees the brief names, each asserted end-to-end:
|
|
19
|
+
* 1. a credential-class MCP failure produces EXACTLY ONE operator alert with
|
|
20
|
+
* the provider, the vault key NAME, the agents and the action;
|
|
21
|
+
* 2. a second agent failing the same way inside the window produces NO
|
|
22
|
+
* second alert;
|
|
23
|
+
* 3. an ordinary tool error (bad query, 404, timeout, transient 429)
|
|
24
|
+
* produces NONE.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { describe, it, expect } from 'vitest'
|
|
28
|
+
import {
|
|
29
|
+
McpFailureWatcher,
|
|
30
|
+
McpFailureLedger,
|
|
31
|
+
classifyMcpFailure,
|
|
32
|
+
parseMcpServerFromToolName,
|
|
33
|
+
renderMcpFailureDetail,
|
|
34
|
+
RENOTIFY_MS,
|
|
35
|
+
type McpFailureAlert,
|
|
36
|
+
} from '../mcp-credential-failure.js'
|
|
37
|
+
import {
|
|
38
|
+
renderOperatorEvent,
|
|
39
|
+
decideOperatorEventAudience,
|
|
40
|
+
isOperatorActionableKind,
|
|
41
|
+
renderUserFacingFailureNotice,
|
|
42
|
+
type OperatorEvent,
|
|
43
|
+
} from '../operator-events.js'
|
|
44
|
+
|
|
45
|
+
// ── Verbatim vendor failure bodies ──────────────────────────────────────────
|
|
46
|
+
|
|
47
|
+
/** Perplexity, key revoked / rejected. */
|
|
48
|
+
const PPLX_401 =
|
|
49
|
+
'{"error":{"message":"Invalid API key provided.","type":"authentication_error","code":401}}'
|
|
50
|
+
|
|
51
|
+
/** Perplexity, balance spent — 401 status, but the remedy is "top up". */
|
|
52
|
+
const PPLX_NO_CREDIT =
|
|
53
|
+
'api.perplexity.ai responded with status 401: {"detail":"insufficient credits — please add funds to your account"}'
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Perplexity, hard monthly wall. Note the status is 429 — the SAME status as a
|
|
57
|
+
* transient throttle — so only the wording can tell the two apart, which is
|
|
58
|
+
* exactly why 429 is not in the quota rule's status list.
|
|
59
|
+
*/
|
|
60
|
+
const PPLX_QUOTA =
|
|
61
|
+
'{"error":{"message":"Monthly quota exceeded for your plan. Upgrade your plan to continue.","code":429}}'
|
|
62
|
+
|
|
63
|
+
// ── Ordinary failures that must stay SILENT ─────────────────────────────────
|
|
64
|
+
|
|
65
|
+
const ORDINARY_FAILURES: ReadonlyArray<[string, string]> = [
|
|
66
|
+
['a bad query', 'Error: search query must not be empty'],
|
|
67
|
+
['a 404', 'Request failed with status code 404: not found'],
|
|
68
|
+
['a timeout', 'Error: ETIMEDOUT — request to api.perplexity.ai timed out after 30000ms'],
|
|
69
|
+
['a transient throttle', '{"error":{"message":"Rate limit exceeded, retry after 2s","code":429}}'],
|
|
70
|
+
['a transport reset', 'Error: socket hang up (ECONNRESET)'],
|
|
71
|
+
['an upstream 5xx', 'Request failed with status code 503: service unavailable'],
|
|
72
|
+
['no results', 'The search returned no results for that query.'],
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
const ALLOW = ['7000000001' /* operator */, '7000000002' /* end user */]
|
|
76
|
+
const T0 = Date.UTC(2026, 6, 28, 10, 0, 0)
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* One agent's gateway, driven exactly as `handleSessionEvent` drives it. The
|
|
80
|
+
* `alerts` array is what `emitGatewayOperatorEvent` would have been called with.
|
|
81
|
+
*/
|
|
82
|
+
function makeAgent(agent: string, watcher: McpFailureWatcher) {
|
|
83
|
+
const alerts: McpFailureAlert[] = []
|
|
84
|
+
let seq = 0
|
|
85
|
+
return {
|
|
86
|
+
alerts,
|
|
87
|
+
/** Drive one full MCP tool call that fails with `errorText`. */
|
|
88
|
+
fail(toolName: string, errorText: string, now: number): void {
|
|
89
|
+
const toolUseId = `toolu_${agent}_${seq++}`
|
|
90
|
+
watcher.onToolUse(toolUseId, toolName)
|
|
91
|
+
const alert = watcher.onToolResult({ toolUseId, isError: true, errorText, agent, now })
|
|
92
|
+
if (alert != null) alerts.push(alert)
|
|
93
|
+
},
|
|
94
|
+
/** A successful call — must never alert. */
|
|
95
|
+
succeed(toolName: string, now: number): void {
|
|
96
|
+
const toolUseId = `toolu_${agent}_${seq++}`
|
|
97
|
+
watcher.onToolUse(toolUseId, toolName)
|
|
98
|
+
const alert = watcher.onToolResult({ toolUseId, isError: false, agent, now })
|
|
99
|
+
if (alert != null) alerts.push(alert)
|
|
100
|
+
},
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** What the operator and a non-operator user actually receive for one alert. */
|
|
105
|
+
function route(alert: McpFailureAlert): {
|
|
106
|
+
operatorText: string | null
|
|
107
|
+
operatorChats: string[]
|
|
108
|
+
userText: string | null
|
|
109
|
+
userChats: string[]
|
|
110
|
+
buttons: string[]
|
|
111
|
+
} {
|
|
112
|
+
const ev: OperatorEvent = {
|
|
113
|
+
kind: 'mcp-dependency-blocked',
|
|
114
|
+
agent: alert.agents[0] ?? 'agent',
|
|
115
|
+
detail: renderMcpFailureDetail(alert),
|
|
116
|
+
suggestedActions: [],
|
|
117
|
+
firstSeenAt: new Date(T0),
|
|
118
|
+
}
|
|
119
|
+
const rendered = renderOperatorEvent(ev)
|
|
120
|
+
const { operatorChats, userNoticeChats } = decideOperatorEventAudience(ev.kind, ALLOW, ALLOW[0])
|
|
121
|
+
return {
|
|
122
|
+
operatorText: operatorChats.length > 0 ? rendered.text : null,
|
|
123
|
+
operatorChats,
|
|
124
|
+
userText: userNoticeChats.length > 0 ? renderUserFacingFailureNotice() : null,
|
|
125
|
+
userChats: userNoticeChats,
|
|
126
|
+
buttons: rendered.keyboard.inline_keyboard.flat().map(b => b.text),
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
describe('a blocked Perplexity key raises exactly one operator alert', () => {
|
|
131
|
+
it('produces ONE alert naming provider, vault key NAME, agent and action', () => {
|
|
132
|
+
const w = new McpFailureWatcher()
|
|
133
|
+
const klanker = makeAgent('klanker', w)
|
|
134
|
+
|
|
135
|
+
klanker.fail('mcp__perplexity__perplexity_search', PPLX_401, T0)
|
|
136
|
+
|
|
137
|
+
expect(klanker.alerts).toHaveLength(1)
|
|
138
|
+
const alert = klanker.alerts[0]
|
|
139
|
+
expect(alert.server).toBe('perplexity')
|
|
140
|
+
expect(alert.cls).toBe('credential')
|
|
141
|
+
expect(alert.agents).toEqual(['klanker'])
|
|
142
|
+
expect(alert.renotify).toBe(false)
|
|
143
|
+
|
|
144
|
+
const r = route(alert)
|
|
145
|
+
|
|
146
|
+
// Operator-only: the kind is actionable, so the user gets the brief notice.
|
|
147
|
+
expect(isOperatorActionableKind('mcp-dependency-blocked')).toBe(true)
|
|
148
|
+
expect(r.operatorChats).toEqual(['7000000001'])
|
|
149
|
+
expect(r.userChats).toEqual(['7000000002'])
|
|
150
|
+
|
|
151
|
+
// The card says what Ken can actually DO.
|
|
152
|
+
expect(r.operatorText).toContain('Perplexity')
|
|
153
|
+
expect(r.operatorText).toContain('perplexity/api-key') // vault key NAME
|
|
154
|
+
expect(r.operatorText).toContain('https://www.perplexity.ai/settings/api')
|
|
155
|
+
expect(r.operatorText).toContain('klanker')
|
|
156
|
+
expect(r.operatorText).toContain('Re-issue the key')
|
|
157
|
+
expect(r.buttons).toEqual(['❌ Dismiss'])
|
|
158
|
+
|
|
159
|
+
// NEVER a secret value, and never the raw provider error.
|
|
160
|
+
expect(r.operatorText).not.toContain('Invalid API key provided')
|
|
161
|
+
expect(r.operatorText).not.toMatch(/pplx-[A-Za-z0-9]/)
|
|
162
|
+
|
|
163
|
+
// The end user still sees only the diagnosis-free notice.
|
|
164
|
+
expect(r.userText).toBe(renderUserFacingFailureNotice())
|
|
165
|
+
for (const fragment of ['perplexity', '401', 'api key', 'authentication_error', 'vault']) {
|
|
166
|
+
expect(r.userText!.toLowerCase()).not.toContain(fragment)
|
|
167
|
+
}
|
|
168
|
+
})
|
|
169
|
+
|
|
170
|
+
it('routes a spent BALANCE to "top up", not to "re-issue the key"', () => {
|
|
171
|
+
const w = new McpFailureWatcher()
|
|
172
|
+
const a = makeAgent('klanker', w)
|
|
173
|
+
a.fail('mcp__perplexity__perplexity_ask', PPLX_NO_CREDIT, T0)
|
|
174
|
+
expect(a.alerts).toHaveLength(1)
|
|
175
|
+
// 401 status, but the credit wording must win — the key is fine, the
|
|
176
|
+
// account is empty, and re-issuing it would waste the operator's time.
|
|
177
|
+
expect(a.alerts[0].cls).toBe('credit')
|
|
178
|
+
expect(route(a.alerts[0]).operatorText).toContain('Top up the balance')
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
it('sees a hard wall wrapped in throttle language (429 + "rate limit")', () => {
|
|
182
|
+
// Providers routinely dress a monthly wall up as a rate limit. The wall is
|
|
183
|
+
// the real news; silencing it as a throttle would lose the capability
|
|
184
|
+
// quietly, which is the exact failure this whole feature exists to stop.
|
|
185
|
+
const w = new McpFailureWatcher()
|
|
186
|
+
const a = makeAgent('klanker', w)
|
|
187
|
+
a.fail(
|
|
188
|
+
'mcp__perplexity__perplexity_search',
|
|
189
|
+
'Rate limit exceeded — monthly quota exceeded for your plan (429)',
|
|
190
|
+
T0,
|
|
191
|
+
)
|
|
192
|
+
expect(a.alerts).toHaveLength(1)
|
|
193
|
+
expect(a.alerts[0].cls).toBe('quota')
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
it('routes a hard usage wall to the quota remedy', () => {
|
|
197
|
+
const w = new McpFailureWatcher()
|
|
198
|
+
const a = makeAgent('klanker', w)
|
|
199
|
+
a.fail('mcp__perplexity__perplexity_research', PPLX_QUOTA, T0)
|
|
200
|
+
expect(a.alerts).toHaveLength(1)
|
|
201
|
+
expect(a.alerts[0].cls).toBe('quota')
|
|
202
|
+
expect(route(a.alerts[0]).operatorText).toContain('Raise the plan limit')
|
|
203
|
+
})
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
describe('deduplication — a storm of failures is one alert, not a storm of alerts', () => {
|
|
207
|
+
it('a second agent failing the same way in the window produces NO second alert', () => {
|
|
208
|
+
// One shared watcher stands in for one gateway process seeing several
|
|
209
|
+
// agents/sub-agents; the cross-CONTAINER limit is documented in the module.
|
|
210
|
+
const w = new McpFailureWatcher()
|
|
211
|
+
const klanker = makeAgent('klanker', w)
|
|
212
|
+
const scribe = makeAgent('scribe', w)
|
|
213
|
+
|
|
214
|
+
klanker.fail('mcp__perplexity__perplexity_search', PPLX_401, T0)
|
|
215
|
+
scribe.fail('mcp__perplexity__perplexity_ask', PPLX_401, T0 + 1_000)
|
|
216
|
+
klanker.fail('mcp__perplexity__perplexity_search', PPLX_401, T0 + 30_000)
|
|
217
|
+
|
|
218
|
+
expect(klanker.alerts).toHaveLength(1)
|
|
219
|
+
expect(scribe.alerts).toHaveLength(0)
|
|
220
|
+
})
|
|
221
|
+
|
|
222
|
+
it('re-notifies after the 6h house cadence, and NAMES everyone seen since', () => {
|
|
223
|
+
const w = new McpFailureWatcher()
|
|
224
|
+
const klanker = makeAgent('klanker', w)
|
|
225
|
+
const scribe = makeAgent('scribe', w)
|
|
226
|
+
|
|
227
|
+
klanker.fail('mcp__perplexity__perplexity_search', PPLX_401, T0)
|
|
228
|
+
expect(klanker.alerts).toHaveLength(1)
|
|
229
|
+
|
|
230
|
+
// Silent for the whole window, however many failures land.
|
|
231
|
+
scribe.fail('mcp__perplexity__perplexity_ask', PPLX_401, T0 + RENOTIFY_MS - 1)
|
|
232
|
+
expect(scribe.alerts).toHaveLength(0)
|
|
233
|
+
|
|
234
|
+
// At the boundary it fires again, accumulating everyone seen meanwhile.
|
|
235
|
+
scribe.fail('mcp__perplexity__perplexity_ask', PPLX_401, T0 + RENOTIFY_MS)
|
|
236
|
+
expect(scribe.alerts).toHaveLength(1)
|
|
237
|
+
const repeat = scribe.alerts[0]
|
|
238
|
+
expect(repeat.renotify).toBe(true)
|
|
239
|
+
expect(repeat.agents).toEqual(['scribe'])
|
|
240
|
+
expect(repeat.occurrences).toBe(2)
|
|
241
|
+
expect(route(repeat).operatorText).toContain('still failing')
|
|
242
|
+
})
|
|
243
|
+
|
|
244
|
+
it('matches the house re-notify cadence used by the hindsight watcher', () => {
|
|
245
|
+
expect(RENOTIFY_MS).toBe(6 * 60 * 60 * 1000)
|
|
246
|
+
})
|
|
247
|
+
|
|
248
|
+
it('tracks credit and credential walls on the same server independently', () => {
|
|
249
|
+
const led = new McpFailureLedger()
|
|
250
|
+
expect(led.note({ server: 'perplexity', agent: 'a', cls: 'credential', now: T0 })).not.toBeNull()
|
|
251
|
+
// A different failure CLASS is a different problem with a different remedy,
|
|
252
|
+
// so it must not be silenced by the first one's window.
|
|
253
|
+
expect(led.note({ server: 'perplexity', agent: 'a', cls: 'credit', now: T0 })).not.toBeNull()
|
|
254
|
+
})
|
|
255
|
+
})
|
|
256
|
+
|
|
257
|
+
describe('ordinary tool failures never page the operator', () => {
|
|
258
|
+
for (const [label, body] of ORDINARY_FAILURES) {
|
|
259
|
+
it(`stays silent for ${label}`, () => {
|
|
260
|
+
const w = new McpFailureWatcher()
|
|
261
|
+
const a = makeAgent('klanker', w)
|
|
262
|
+
a.fail('mcp__perplexity__perplexity_search', body, T0)
|
|
263
|
+
expect(a.alerts).toEqual([])
|
|
264
|
+
expect(classifyMcpFailure(body)).toBe('ordinary')
|
|
265
|
+
})
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
it('stays silent for a SUCCESSFUL MCP call', () => {
|
|
269
|
+
const w = new McpFailureWatcher()
|
|
270
|
+
const a = makeAgent('klanker', w)
|
|
271
|
+
a.succeed('mcp__perplexity__perplexity_search', T0)
|
|
272
|
+
expect(a.alerts).toEqual([])
|
|
273
|
+
})
|
|
274
|
+
|
|
275
|
+
it('stays silent for a non-MCP tool that fails with auth-shaped text', () => {
|
|
276
|
+
// A Bash step printing "401 unauthorized" from some unrelated curl is NOT
|
|
277
|
+
// a fleet capability loss and must not page anyone.
|
|
278
|
+
const w = new McpFailureWatcher()
|
|
279
|
+
const a = makeAgent('klanker', w)
|
|
280
|
+
a.fail('Bash', 'curl: server returned status 401 unauthorized', T0)
|
|
281
|
+
expect(a.alerts).toEqual([])
|
|
282
|
+
})
|
|
283
|
+
})
|
|
284
|
+
|
|
285
|
+
describe('the rule is data, and it generalises past Perplexity', () => {
|
|
286
|
+
it('derives the server from any mcp__<server>__<tool> name', () => {
|
|
287
|
+
expect(parseMcpServerFromToolName('mcp__perplexity__perplexity_search')).toBe('perplexity')
|
|
288
|
+
expect(parseMcpServerFromToolName('mcp__meta-ads__list_campaigns')).toBe('meta-ads')
|
|
289
|
+
expect(parseMcpServerFromToolName('Read')).toBeNull()
|
|
290
|
+
expect(parseMcpServerFromToolName(undefined)).toBeNull()
|
|
291
|
+
})
|
|
292
|
+
|
|
293
|
+
it('covers other paid dependencies with no new code — Eraser, Brevo', () => {
|
|
294
|
+
const w = new McpFailureWatcher()
|
|
295
|
+
const a = makeAgent('klanker', w)
|
|
296
|
+
a.fail('mcp__eraser__create_diagram', '403 Forbidden: api key has been revoked', T0)
|
|
297
|
+
a.fail('mcp__brevo__send_email', '{"code":"unauthorized","message":"Key not found"}', T0)
|
|
298
|
+
expect(a.alerts.map(x => x.server)).toEqual(['eraser', 'brevo'])
|
|
299
|
+
expect(route(a.alerts[0]).operatorText).toContain('eraser/api-key')
|
|
300
|
+
expect(route(a.alerts[1]).operatorText).toContain('brevo/api-key')
|
|
301
|
+
})
|
|
302
|
+
|
|
303
|
+
it('an unregistered MCP server still alerts, with a conventional key name', () => {
|
|
304
|
+
const w = new McpFailureWatcher()
|
|
305
|
+
const a = makeAgent('klanker', w)
|
|
306
|
+
a.fail('mcp__somenewthing__do_it', '401 unauthorized: invalid api key', T0)
|
|
307
|
+
expect(a.alerts).toHaveLength(1)
|
|
308
|
+
expect(a.alerts[0].vaultKey).toBe('somenewthing/api-key')
|
|
309
|
+
})
|
|
310
|
+
})
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The outbox sweep must not hammer a flood ban it can already see (#3853).
|
|
3
|
+
*
|
|
4
|
+
* ── The bug these tests guard ────────────────────────────────────────────
|
|
5
|
+
* `startOutboxSweep` built its retry policy as `createRetryApiCall({ log })` —
|
|
6
|
+
* the ONE wiring in the tree with neither `floodWaitRemainingMs` (read the
|
|
7
|
+
* persisted breaker) nor `onFloodWait` (write to it). Combined with a fixed
|
|
8
|
+
* 5s tick and a catch that released the claim with no backoff, the sweep
|
|
9
|
+
* re-issued a single undeliverable `sendMessage` every 5 seconds for the whole
|
|
10
|
+
* of overlord's 15908s (4.4h) ban on 2026-07-27: 228 requests into a window
|
|
11
|
+
* the breaker had recorded and could have answered in one read.
|
|
12
|
+
*
|
|
13
|
+
* The gateway log is the proof and the shape of the outcome assertion:
|
|
14
|
+
* grep -c 'flood window still open' gateway-supervisor.log → 0
|
|
15
|
+
* grep -c 'flood ban of' gateway-supervisor.log → 230
|
|
16
|
+
* grep -c 'outbox-sweep: send failed' → 228
|
|
17
|
+
*
|
|
18
|
+
* Every test below asserts an OUTCOME — whether the wire was touched and
|
|
19
|
+
* whether the record survived — not that a code path executed.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
|
23
|
+
import { mkdtempSync, rmSync } from 'node:fs'
|
|
24
|
+
import { tmpdir } from 'node:os'
|
|
25
|
+
import { join } from 'node:path'
|
|
26
|
+
|
|
27
|
+
import {
|
|
28
|
+
writeOutboxRecordAtomic,
|
|
29
|
+
listPendingRecords,
|
|
30
|
+
sha256Hex,
|
|
31
|
+
type OutboxRecord,
|
|
32
|
+
} from '../outbox.js'
|
|
33
|
+
import {
|
|
34
|
+
sweepOutbox,
|
|
35
|
+
createSweepBackoff,
|
|
36
|
+
OUTBOX_SWEEP_INTERVAL_MS,
|
|
37
|
+
OUTBOX_SWEEP_BACKOFF_MAX_MS,
|
|
38
|
+
} from '../gateway/outbox-sweep.js'
|
|
39
|
+
|
|
40
|
+
function rec(over: Partial<OutboxRecord> = {}): OutboxRecord {
|
|
41
|
+
const text = over.text ?? 'the answer the operator is waiting for'
|
|
42
|
+
return {
|
|
43
|
+
turnNonce: over.turnNonce ?? 'nonce-1',
|
|
44
|
+
chatId: over.chatId ?? '111',
|
|
45
|
+
threadId: over.threadId ?? null,
|
|
46
|
+
text,
|
|
47
|
+
textSha256: sha256Hex(text),
|
|
48
|
+
createdAt: over.createdAt ?? 0,
|
|
49
|
+
source: over.source ?? 'channel',
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
describe('outbox sweep vs an open flood window', () => {
|
|
54
|
+
let dir: string
|
|
55
|
+
beforeEach(() => {
|
|
56
|
+
dir = mkdtempSync(join(tmpdir(), 'outbox-flood-'))
|
|
57
|
+
})
|
|
58
|
+
afterEach(() => {
|
|
59
|
+
rmSync(dir, { recursive: true, force: true })
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
it('does not hit the wire while a flood window is open, and keeps the record', async () => {
|
|
63
|
+
writeOutboxRecordAtomic(rec(), dir)
|
|
64
|
+
const send = vi.fn(async () => 1)
|
|
65
|
+
const log = vi.fn()
|
|
66
|
+
|
|
67
|
+
const summary = await sweepOutbox({
|
|
68
|
+
stateDir: dir,
|
|
69
|
+
log,
|
|
70
|
+
send,
|
|
71
|
+
textAlreadyDelivered: () => false,
|
|
72
|
+
// 12247s remaining — the exact shape of the window the 2026-07-27 sweep
|
|
73
|
+
// ignored 228 times.
|
|
74
|
+
floodWaitRemainingMs: () => 12_247_000,
|
|
75
|
+
now: () => 10 * 60_000,
|
|
76
|
+
})
|
|
77
|
+
|
|
78
|
+
// THE outcome: nothing reached Telegram.
|
|
79
|
+
expect(send).not.toHaveBeenCalled()
|
|
80
|
+
expect(summary.floodDeferred).toBe(true)
|
|
81
|
+
expect(summary.floodRemainingMs).toBe(12_247_000)
|
|
82
|
+
expect(summary.delivered).toBe(0)
|
|
83
|
+
// And nothing was lost — the record is still pending for the next sweep.
|
|
84
|
+
expect(listPendingRecords(dir)).toHaveLength(1)
|
|
85
|
+
// The deferral is NOT logged per-tick — a 4.4h ban is ~3181 ticks, and one
|
|
86
|
+
// line each would relocate the flood from the wire to the disk. The tick
|
|
87
|
+
// loop reports it at most once per OUTBOX_SWEEP_DEFER_LOG_INTERVAL_MS.
|
|
88
|
+
expect(log).not.toHaveBeenCalled()
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
it('delivers the SAME record once the window closes', async () => {
|
|
92
|
+
writeOutboxRecordAtomic(rec(), dir)
|
|
93
|
+
const send = vi.fn(async () => 42)
|
|
94
|
+
let remaining = 12_247_000
|
|
95
|
+
|
|
96
|
+
const deferred = await sweepOutbox({
|
|
97
|
+
stateDir: dir,
|
|
98
|
+
send,
|
|
99
|
+
textAlreadyDelivered: () => false,
|
|
100
|
+
floodWaitRemainingMs: () => remaining,
|
|
101
|
+
now: () => 10 * 60_000,
|
|
102
|
+
})
|
|
103
|
+
expect(deferred.floodDeferred).toBe(true)
|
|
104
|
+
expect(send).not.toHaveBeenCalled()
|
|
105
|
+
|
|
106
|
+
remaining = 0
|
|
107
|
+
const after = await sweepOutbox({
|
|
108
|
+
stateDir: dir,
|
|
109
|
+
send,
|
|
110
|
+
textAlreadyDelivered: () => false,
|
|
111
|
+
floodWaitRemainingMs: () => remaining,
|
|
112
|
+
now: () => 20 * 60_000,
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
expect(send).toHaveBeenCalledTimes(1)
|
|
116
|
+
expect(send.mock.calls[0]?.[2]).toContain('the answer the operator is waiting for')
|
|
117
|
+
expect(after.delivered).toBe(1)
|
|
118
|
+
expect(listPendingRecords(dir)).toHaveLength(0)
|
|
119
|
+
})
|
|
120
|
+
|
|
121
|
+
it('sweeps normally when no window is open', async () => {
|
|
122
|
+
writeOutboxRecordAtomic(rec(), dir)
|
|
123
|
+
const send = vi.fn(async () => 7)
|
|
124
|
+
const summary = await sweepOutbox({
|
|
125
|
+
stateDir: dir,
|
|
126
|
+
send,
|
|
127
|
+
textAlreadyDelivered: () => false,
|
|
128
|
+
floodWaitRemainingMs: () => 0,
|
|
129
|
+
now: () => 10 * 60_000,
|
|
130
|
+
})
|
|
131
|
+
expect(send).toHaveBeenCalledTimes(1)
|
|
132
|
+
expect(summary.delivered).toBe(1)
|
|
133
|
+
expect(summary.floodDeferred).toBeUndefined()
|
|
134
|
+
})
|
|
135
|
+
|
|
136
|
+
it('FAILS OPEN: a throwing probe must never strand the outbox', async () => {
|
|
137
|
+
writeOutboxRecordAtomic(rec(), dir)
|
|
138
|
+
const send = vi.fn(async () => 7)
|
|
139
|
+
const summary = await sweepOutbox({
|
|
140
|
+
stateDir: dir,
|
|
141
|
+
send,
|
|
142
|
+
textAlreadyDelivered: () => false,
|
|
143
|
+
floodWaitRemainingMs: () => {
|
|
144
|
+
throw new Error('EACCES: flood-wait.json owned by another uid')
|
|
145
|
+
},
|
|
146
|
+
now: () => 10 * 60_000,
|
|
147
|
+
})
|
|
148
|
+
expect(send).toHaveBeenCalledTimes(1)
|
|
149
|
+
expect(summary.delivered).toBe(1)
|
|
150
|
+
})
|
|
151
|
+
|
|
152
|
+
it('reports send failures so the tick can back off', async () => {
|
|
153
|
+
writeOutboxRecordAtomic(rec(), dir)
|
|
154
|
+
const send = vi.fn(async () => {
|
|
155
|
+
throw new Error('FLOOD_WAIT_ACTIVE')
|
|
156
|
+
})
|
|
157
|
+
const summary = await sweepOutbox({
|
|
158
|
+
stateDir: dir,
|
|
159
|
+
send,
|
|
160
|
+
textAlreadyDelivered: () => false,
|
|
161
|
+
floodWaitRemainingMs: () => 0,
|
|
162
|
+
now: () => 10 * 60_000,
|
|
163
|
+
})
|
|
164
|
+
expect(summary.sendFailures).toBe(1)
|
|
165
|
+
expect(summary.delivered).toBe(0)
|
|
166
|
+
// Claim released — the record is still pending, never lost.
|
|
167
|
+
expect(listPendingRecords(dir)).toHaveLength(1)
|
|
168
|
+
})
|
|
169
|
+
})
|
|
170
|
+
|
|
171
|
+
describe('createSweepBackoff — the fixed-5s-forever regression', () => {
|
|
172
|
+
it('rides the normal tick on the FIRST failure, then doubles', () => {
|
|
173
|
+
const b = createSweepBackoff()
|
|
174
|
+
expect(b.ready(0)).toBe(true)
|
|
175
|
+
|
|
176
|
+
expect(b.noteFailure(0)).toBe(OUTBOX_SWEEP_INTERVAL_MS) // 5s
|
|
177
|
+
expect(b.ready(4_999)).toBe(false)
|
|
178
|
+
expect(b.ready(5_000)).toBe(true)
|
|
179
|
+
|
|
180
|
+
expect(b.noteFailure(5_000)).toBe(10_000)
|
|
181
|
+
expect(b.noteFailure(15_000)).toBe(20_000)
|
|
182
|
+
expect(b.noteFailure(35_000)).toBe(40_000)
|
|
183
|
+
})
|
|
184
|
+
|
|
185
|
+
it('caps the delay so the sweep never stops entirely', () => {
|
|
186
|
+
const b = createSweepBackoff()
|
|
187
|
+
let last = 0
|
|
188
|
+
for (let i = 0; i < 40; i++) last = b.noteFailure(i * 1_000_000)
|
|
189
|
+
expect(last).toBe(OUTBOX_SWEEP_BACKOFF_MAX_MS)
|
|
190
|
+
})
|
|
191
|
+
|
|
192
|
+
it('the 4.4h-ban scenario: ≤ 60 sweeps, not 228', () => {
|
|
193
|
+
// Replay the incident's shape — a persistently failing send across a
|
|
194
|
+
// 15908s window — and count how many sweeps the pacer would permit.
|
|
195
|
+
const b = createSweepBackoff()
|
|
196
|
+
const banMs = 15_908_000
|
|
197
|
+
let now = 0
|
|
198
|
+
let sweeps = 0
|
|
199
|
+
while (now < banMs) {
|
|
200
|
+
if (b.ready(now)) {
|
|
201
|
+
sweeps++
|
|
202
|
+
b.noteFailure(now)
|
|
203
|
+
}
|
|
204
|
+
now += OUTBOX_SWEEP_INTERVAL_MS
|
|
205
|
+
}
|
|
206
|
+
// Fixed 5s ticking would be 3181 attempts; the real sweep issued 228
|
|
207
|
+
// before the operator intervened. Exponential + cap keeps it under 60.
|
|
208
|
+
expect(sweeps).toBeLessThan(60)
|
|
209
|
+
expect(sweeps).toBeGreaterThan(0)
|
|
210
|
+
})
|
|
211
|
+
|
|
212
|
+
it('a successful sweep clears the backoff', () => {
|
|
213
|
+
const b = createSweepBackoff()
|
|
214
|
+
b.noteFailure(0)
|
|
215
|
+
b.noteFailure(5_000)
|
|
216
|
+
expect(b.ready(6_000)).toBe(false)
|
|
217
|
+
b.noteSuccess()
|
|
218
|
+
expect(b.ready(6_000)).toBe(true)
|
|
219
|
+
expect(b.failures()).toBe(0)
|
|
220
|
+
})
|
|
221
|
+
})
|
|
@@ -33,7 +33,6 @@ import {
|
|
|
33
33
|
STATUS_CARD_CHAR_BUDGET,
|
|
34
34
|
STATUS_LINE_MAX,
|
|
35
35
|
WORKER_STEP_INDENT,
|
|
36
|
-
SUBORDINATE_LINE_INDENT,
|
|
37
36
|
} from '../status-no-truncate.js'
|
|
38
37
|
|
|
39
38
|
/**
|
|
@@ -135,31 +134,28 @@ describe('combined worker card survives the pinned-bar collapse (#3666)', () =>
|
|
|
135
134
|
|
|
136
135
|
it('kills the exact artifacts from the report', () => {
|
|
137
136
|
const collapsed = collapsePreview(body)
|
|
138
|
-
// 1. the count/ordinal collision — glance line into row 1's ordinal.
|
|
139
|
-
//
|
|
140
|
-
//
|
|
141
|
-
// agent card), so this seam is separator + card indent.
|
|
137
|
+
// 1. the count/ordinal collision — glance line into row 1's ordinal. Since
|
|
138
|
+
// #3842 the row header is FLUSH (the #3820 card-level indent is gone),
|
|
139
|
+
// so the separator is the ONLY thing holding this seam apart.
|
|
142
140
|
// (The reported spelling was `3 running1.`; with the packed glance line
|
|
143
141
|
// the same seam now reads `… 512.3k tok` -> `1. Fix issue`, so assert on
|
|
144
142
|
// the CURRENT last token of line 1 — an assertion on the old spelling
|
|
145
143
|
// alone would be vacuously green.)
|
|
146
144
|
expect(collapsed).not.toContain('running1.')
|
|
147
145
|
expect(collapsed).not.toContain('tok1.')
|
|
148
|
-
expect(collapsed).toContain(`tok${NB}
|
|
149
|
-
// 2. the mid-word ✓ (model tag running into the step trail).
|
|
150
|
-
//
|
|
151
|
-
//
|
|
152
|
-
//
|
|
146
|
+
expect(collapsed).toContain(`tok${NB}1. Fix issue`)
|
|
147
|
+
// 2. the mid-word ✓ (model tag running into the step trail). A step line
|
|
148
|
+
// still leads with WORKER_STEP_INDENT (three U+2800), so this seam is
|
|
149
|
+
// separator + step indent — asserted against the constant rather than a
|
|
150
|
+
// hardcoded run.
|
|
153
151
|
expect(collapsed).not.toContain('opus 5✓')
|
|
154
|
-
expect(collapsed).toContain(`opus 5${NB}${
|
|
152
|
+
expect(collapsed).toContain(`opus 5${NB}${WORKER_STEP_INDENT}✓`)
|
|
155
153
|
// 3. the step trail running into the next step, and into the next row's
|
|
156
|
-
// header (post-#
|
|
154
|
+
// header (post-#3842 that last seam is the separator alone).
|
|
157
155
|
expect(collapsed).not.toContain('gateway.ts→')
|
|
158
|
-
expect(collapsed).toContain(
|
|
159
|
-
`gateway.ts${NB}${SUBORDINATE_LINE_INDENT}${WORKER_STEP_INDENT}→`,
|
|
160
|
-
)
|
|
156
|
+
expect(collapsed).toContain(`gateway.ts${NB}${WORKER_STEP_INDENT}→`)
|
|
161
157
|
expect(collapsed).not.toContain('search2.')
|
|
162
|
-
expect(collapsed).toContain(`search${NB}
|
|
158
|
+
expect(collapsed).toContain(`search${NB}2.`)
|
|
163
159
|
})
|
|
164
160
|
|
|
165
161
|
it('leads with a self-contained glance that ends in a unit word, not a bare number', () => {
|
|
@@ -178,13 +174,10 @@ describe('combined worker card survives the pinned-bar collapse (#3666)', () =>
|
|
|
178
174
|
// show the collapsed preview mashes again. Without this, the assertions
|
|
179
175
|
// above could all be passing for reasons unrelated to the fix.
|
|
180
176
|
//
|
|
181
|
-
// The control runs on the 🤖 AGENT card
|
|
182
|
-
//
|
|
183
|
-
//
|
|
184
|
-
//
|
|
185
|
-
// separator's contribution. The agent card is the surface where the
|
|
186
|
-
// separator is still the ONLY thing holding the seams apart, which is
|
|
187
|
-
// exactly what this control must measure.
|
|
177
|
+
// The control runs on the 🤖 AGENT card. Post-#3842 the single-worker card
|
|
178
|
+
// is flush too, so either would isolate the separator's contribution; the
|
|
179
|
+
// agent card is kept because it has no step indent on ANY line, so no
|
|
180
|
+
// future indent change can quietly make this control vacuous.
|
|
188
181
|
const agent = renderActivityFeed([
|
|
189
182
|
'Reading gateway.ts',
|
|
190
183
|
'Searching memory',
|
|
@@ -228,7 +221,9 @@ describe('single-worker / agent status card survives the collapse too (#3666)',
|
|
|
228
221
|
expectNoMashedSeams(body)
|
|
229
222
|
const collapsed = collapsePreview(body)
|
|
230
223
|
expect(collapsed).not.toContain('toolsstarting')
|
|
231
|
-
|
|
224
|
+
// #3842: the single-worker card is flush, so the separator alone holds the
|
|
225
|
+
// hand-rolled `starting…` seam apart — nothing else masks a regression.
|
|
226
|
+
expect(collapsed).toContain(`0 tools${NB}starting`)
|
|
232
227
|
})
|
|
233
228
|
|
|
234
229
|
it('the nested child block stays separated even though its indent is ASCII (#3668)', () => {
|