switchroom 0.19.26 → 0.19.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/bin/git-agent-attribution-hook.sh +144 -0
  2. package/dist/agent-scheduler/index.js +60 -2
  3. package/dist/auth-broker/index.js +244 -13
  4. package/dist/cli/autoaccept-poll.js +225 -17
  5. package/dist/cli/notion-write-pretool.mjs +60 -2
  6. package/dist/cli/switchroom.js +2843 -1220
  7. package/dist/host-control/main.js +245 -14
  8. package/dist/vault/approvals/kernel-server.js +242 -13
  9. package/dist/vault/broker/server.js +242 -13
  10. package/package.json +7 -2
  11. package/profiles/_base/cron-session.sh.hbs +8 -0
  12. package/profiles/_base/start.sh.hbs +175 -15
  13. package/telegram-plugin/card-layout.ts +328 -0
  14. package/telegram-plugin/dist/bridge/bridge.js +94 -1
  15. package/telegram-plugin/dist/gateway/gateway.js +2544 -1182
  16. package/telegram-plugin/dist/server.js +97 -1
  17. package/telegram-plugin/edit-flood-fuse.ts +841 -57
  18. package/telegram-plugin/flood-429-ledger.ts +526 -0
  19. package/telegram-plugin/flood-circuit-breaker.ts +18 -0
  20. package/telegram-plugin/gateway/callback-query-handlers.ts +6 -0
  21. package/telegram-plugin/gateway/flood-reply-queue.ts +168 -0
  22. package/telegram-plugin/gateway/gateway.ts +67 -70
  23. package/telegram-plugin/gateway/mcp-failure-hook.ts +74 -0
  24. package/telegram-plugin/gateway/narrative-lane.ts +14 -0
  25. package/telegram-plugin/gateway/outbound-send-path.ts +36 -0
  26. package/telegram-plugin/gateway/outbox-sweep.ts +183 -6
  27. package/telegram-plugin/gateway/pinned-message-handler.ts +12 -16
  28. package/telegram-plugin/gateway/status-pin-retarget.ts +72 -36
  29. package/telegram-plugin/gateway/status-pin-store.ts +58 -9
  30. package/telegram-plugin/gateway/worker-pin-reaper.ts +56 -7
  31. package/telegram-plugin/inline-keyboard-callbacks.ts +202 -21
  32. package/telegram-plugin/llm-error-present.ts +61 -2
  33. package/telegram-plugin/mcp-credential-failure.ts +459 -0
  34. package/telegram-plugin/model-unavailable.ts +8 -0
  35. package/telegram-plugin/operator-events.ts +110 -5
  36. package/telegram-plugin/outbound-class.ts +81 -0
  37. package/telegram-plugin/provider-credit.ts +237 -0
  38. package/telegram-plugin/scripts/bun-test-ci.sh +36 -6
  39. package/telegram-plugin/send-gate.ts +24 -2
  40. package/telegram-plugin/status-no-truncate.ts +10 -48
  41. package/telegram-plugin/status-pin-driver.ts +33 -45
  42. package/telegram-plugin/status-pin.ts +18 -1
  43. package/telegram-plugin/tests/card-golden.test.ts +69 -0
  44. package/telegram-plugin/tests/card-lifecycle-render.test.ts +362 -0
  45. package/telegram-plugin/tests/card-type-distinguishability.test.ts +187 -164
  46. package/telegram-plugin/tests/card-variants.golden.txt +211 -0
  47. package/telegram-plugin/tests/card-variants.ts +366 -0
  48. package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +373 -0
  49. package/telegram-plugin/tests/edit-flood-fuse-default-deny.test.ts +319 -0
  50. package/telegram-plugin/tests/edit-flood-fuse-reply-reserve.test.ts +340 -0
  51. package/telegram-plugin/tests/edit-flood-fuse.test.ts +11 -2
  52. package/telegram-plugin/tests/feed-edit-rate-ceiling.test.ts +462 -0
  53. package/telegram-plugin/tests/finalize-callback-flood-policy.test.ts +298 -0
  54. package/telegram-plugin/tests/finalize-callback.test.ts +41 -8
  55. package/telegram-plugin/tests/fixtures/real-429-stream.ts +220 -0
  56. package/telegram-plugin/tests/flood-429-ledger.test.ts +278 -0
  57. package/telegram-plugin/tests/flood-429-recorder-wiring.test.ts +128 -0
  58. package/telegram-plugin/tests/flood-reply-queue.test.ts +418 -0
  59. package/telegram-plugin/tests/mcp-credential-failure.test.ts +310 -0
  60. package/telegram-plugin/tests/outbox-sweep-flood-breaker.test.ts +221 -0
  61. package/telegram-plugin/tests/pinned-card-collapse.test.ts +19 -24
  62. package/telegram-plugin/tests/pinned-message-handler.test.ts +15 -15
  63. package/telegram-plugin/tests/provider-credit-402.test.ts +243 -0
  64. package/telegram-plugin/tests/status-pin-api.test.ts +11 -11
  65. package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +36 -37
  66. package/telegram-plugin/tests/status-pin-lifecycle.test.ts +602 -0
  67. package/telegram-plugin/tests/status-pin-retarget.test.ts +90 -62
  68. package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +7 -3
  69. package/telegram-plugin/tests/status-pin-store.test.ts +109 -60
  70. package/telegram-plugin/tests/status-pin.test.ts +56 -5
  71. package/telegram-plugin/tests/test-runner-coverage.test.ts +133 -0
  72. package/telegram-plugin/tests/worker-activity-feed.test.ts +12 -10
  73. package/telegram-plugin/tests/worker-feed-coalesce.test.ts +23 -29
  74. package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +56 -59
  75. package/telegram-plugin/tests/worker-feed-terminal-edit-class.test.ts +335 -0
  76. package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +1 -1
  77. package/telegram-plugin/tool-activity-summary.ts +239 -365
  78. package/telegram-plugin/uat/assertions.ts +22 -11
  79. package/telegram-plugin/uat/feed-matcher.test.ts +24 -17
  80. package/telegram-plugin/worker-activity-feed.ts +105 -47
  81. package/vendor/hindsight-memory/CLAUDE.md +45 -0
  82. package/vendor/hindsight-memory/scripts/drain_pending.py +433 -11
  83. package/vendor/hindsight-memory/scripts/lib/config.py +33 -0
  84. package/vendor/hindsight-memory/scripts/lib/pending.py +193 -28
  85. package/vendor/hindsight-memory/scripts/recall.py +176 -7
  86. package/vendor/hindsight-memory/scripts/tests/test_config_recall_passthrough_env.py +170 -0
  87. package/vendor/hindsight-memory/scripts/tests/test_drain_circuit_breaker.py +401 -0
  88. package/vendor/hindsight-memory/scripts/tests/test_drain_serialisation.py +286 -0
  89. package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py +817 -8
  90. package/vendor/hindsight-memory/scripts/tests/test_recall_min_score.py +464 -0
  91. package/vendor/hindsight-memory/settings.json +1 -1
  92. package/vendor/hindsight-memory/tests/test_hooks.py +11 -2
@@ -0,0 +1,310 @@
1
+ /**
2
+ * mcp-credential-failure.test.ts — outcome tests for "a paid MCP dependency's
3
+ * key is blocked → Ken gets ONE hard warning".
4
+ *
5
+ * THE GAP. Perplexity (and Eraser, Brevo, Postiz, Meta/Google Ads, Cloudflare)
6
+ * reach switchroom over the MCP TOOL surface, never through LiteLLM. So the
7
+ * operator-event path that PR A fixed for an OpenRouter 402 never sees them: an
8
+ * expired Perplexity key surfaced as an ordinary red step in the live feed and
9
+ * died there. Nobody was told the fleet had lost a paid capability.
10
+ *
11
+ * These assert the OBSERVABLE RESULT at the gateway seam, composing the SAME
12
+ * production functions in the SAME order `handleSessionEvent` →
13
+ * `noteMcpDependencyFailure` → `emitGatewayOperatorEvent` calls them:
14
+ * McpFailureWatcher.onToolUse/onToolResult → renderMcpFailureDetail →
15
+ * renderOperatorEvent → decideOperatorEventAudience →
16
+ * renderUserFacingFailureNotice.
17
+ *
18
+ * The three guarantees the brief names, each asserted end-to-end:
19
+ * 1. a credential-class MCP failure produces EXACTLY ONE operator alert with
20
+ * the provider, the vault key NAME, the agents and the action;
21
+ * 2. a second agent failing the same way inside the window produces NO
22
+ * second alert;
23
+ * 3. an ordinary tool error (bad query, 404, timeout, transient 429)
24
+ * produces NONE.
25
+ */
26
+
27
+ import { describe, it, expect } from 'vitest'
28
+ import {
29
+ McpFailureWatcher,
30
+ McpFailureLedger,
31
+ classifyMcpFailure,
32
+ parseMcpServerFromToolName,
33
+ renderMcpFailureDetail,
34
+ RENOTIFY_MS,
35
+ type McpFailureAlert,
36
+ } from '../mcp-credential-failure.js'
37
+ import {
38
+ renderOperatorEvent,
39
+ decideOperatorEventAudience,
40
+ isOperatorActionableKind,
41
+ renderUserFacingFailureNotice,
42
+ type OperatorEvent,
43
+ } from '../operator-events.js'
44
+
45
+ // ── Verbatim vendor failure bodies ──────────────────────────────────────────
46
+
47
+ /** Perplexity, key revoked / rejected. */
48
+ const PPLX_401 =
49
+ '{"error":{"message":"Invalid API key provided.","type":"authentication_error","code":401}}'
50
+
51
+ /** Perplexity, balance spent — 401 status, but the remedy is "top up". */
52
+ const PPLX_NO_CREDIT =
53
+ 'api.perplexity.ai responded with status 401: {"detail":"insufficient credits — please add funds to your account"}'
54
+
55
+ /**
56
+ * Perplexity, hard monthly wall. Note the status is 429 — the SAME status as a
57
+ * transient throttle — so only the wording can tell the two apart, which is
58
+ * exactly why 429 is not in the quota rule's status list.
59
+ */
60
+ const PPLX_QUOTA =
61
+ '{"error":{"message":"Monthly quota exceeded for your plan. Upgrade your plan to continue.","code":429}}'
62
+
63
+ // ── Ordinary failures that must stay SILENT ─────────────────────────────────
64
+
65
+ const ORDINARY_FAILURES: ReadonlyArray<[string, string]> = [
66
+ ['a bad query', 'Error: search query must not be empty'],
67
+ ['a 404', 'Request failed with status code 404: not found'],
68
+ ['a timeout', 'Error: ETIMEDOUT — request to api.perplexity.ai timed out after 30000ms'],
69
+ ['a transient throttle', '{"error":{"message":"Rate limit exceeded, retry after 2s","code":429}}'],
70
+ ['a transport reset', 'Error: socket hang up (ECONNRESET)'],
71
+ ['an upstream 5xx', 'Request failed with status code 503: service unavailable'],
72
+ ['no results', 'The search returned no results for that query.'],
73
+ ]
74
+
75
+ const ALLOW = ['7000000001' /* operator */, '7000000002' /* end user */]
76
+ const T0 = Date.UTC(2026, 6, 28, 10, 0, 0)
77
+
78
+ /**
79
+ * One agent's gateway, driven exactly as `handleSessionEvent` drives it. The
80
+ * `alerts` array is what `emitGatewayOperatorEvent` would have been called with.
81
+ */
82
+ function makeAgent(agent: string, watcher: McpFailureWatcher) {
83
+ const alerts: McpFailureAlert[] = []
84
+ let seq = 0
85
+ return {
86
+ alerts,
87
+ /** Drive one full MCP tool call that fails with `errorText`. */
88
+ fail(toolName: string, errorText: string, now: number): void {
89
+ const toolUseId = `toolu_${agent}_${seq++}`
90
+ watcher.onToolUse(toolUseId, toolName)
91
+ const alert = watcher.onToolResult({ toolUseId, isError: true, errorText, agent, now })
92
+ if (alert != null) alerts.push(alert)
93
+ },
94
+ /** A successful call — must never alert. */
95
+ succeed(toolName: string, now: number): void {
96
+ const toolUseId = `toolu_${agent}_${seq++}`
97
+ watcher.onToolUse(toolUseId, toolName)
98
+ const alert = watcher.onToolResult({ toolUseId, isError: false, agent, now })
99
+ if (alert != null) alerts.push(alert)
100
+ },
101
+ }
102
+ }
103
+
104
+ /** What the operator and a non-operator user actually receive for one alert. */
105
+ function route(alert: McpFailureAlert): {
106
+ operatorText: string | null
107
+ operatorChats: string[]
108
+ userText: string | null
109
+ userChats: string[]
110
+ buttons: string[]
111
+ } {
112
+ const ev: OperatorEvent = {
113
+ kind: 'mcp-dependency-blocked',
114
+ agent: alert.agents[0] ?? 'agent',
115
+ detail: renderMcpFailureDetail(alert),
116
+ suggestedActions: [],
117
+ firstSeenAt: new Date(T0),
118
+ }
119
+ const rendered = renderOperatorEvent(ev)
120
+ const { operatorChats, userNoticeChats } = decideOperatorEventAudience(ev.kind, ALLOW, ALLOW[0])
121
+ return {
122
+ operatorText: operatorChats.length > 0 ? rendered.text : null,
123
+ operatorChats,
124
+ userText: userNoticeChats.length > 0 ? renderUserFacingFailureNotice() : null,
125
+ userChats: userNoticeChats,
126
+ buttons: rendered.keyboard.inline_keyboard.flat().map(b => b.text),
127
+ }
128
+ }
129
+
130
+ describe('a blocked Perplexity key raises exactly one operator alert', () => {
131
+ it('produces ONE alert naming provider, vault key NAME, agent and action', () => {
132
+ const w = new McpFailureWatcher()
133
+ const klanker = makeAgent('klanker', w)
134
+
135
+ klanker.fail('mcp__perplexity__perplexity_search', PPLX_401, T0)
136
+
137
+ expect(klanker.alerts).toHaveLength(1)
138
+ const alert = klanker.alerts[0]
139
+ expect(alert.server).toBe('perplexity')
140
+ expect(alert.cls).toBe('credential')
141
+ expect(alert.agents).toEqual(['klanker'])
142
+ expect(alert.renotify).toBe(false)
143
+
144
+ const r = route(alert)
145
+
146
+ // Operator-only: the kind is actionable, so the user gets the brief notice.
147
+ expect(isOperatorActionableKind('mcp-dependency-blocked')).toBe(true)
148
+ expect(r.operatorChats).toEqual(['7000000001'])
149
+ expect(r.userChats).toEqual(['7000000002'])
150
+
151
+ // The card says what Ken can actually DO.
152
+ expect(r.operatorText).toContain('Perplexity')
153
+ expect(r.operatorText).toContain('perplexity/api-key') // vault key NAME
154
+ expect(r.operatorText).toContain('https://www.perplexity.ai/settings/api')
155
+ expect(r.operatorText).toContain('klanker')
156
+ expect(r.operatorText).toContain('Re-issue the key')
157
+ expect(r.buttons).toEqual(['❌ Dismiss'])
158
+
159
+ // NEVER a secret value, and never the raw provider error.
160
+ expect(r.operatorText).not.toContain('Invalid API key provided')
161
+ expect(r.operatorText).not.toMatch(/pplx-[A-Za-z0-9]/)
162
+
163
+ // The end user still sees only the diagnosis-free notice.
164
+ expect(r.userText).toBe(renderUserFacingFailureNotice())
165
+ for (const fragment of ['perplexity', '401', 'api key', 'authentication_error', 'vault']) {
166
+ expect(r.userText!.toLowerCase()).not.toContain(fragment)
167
+ }
168
+ })
169
+
170
+ it('routes a spent BALANCE to "top up", not to "re-issue the key"', () => {
171
+ const w = new McpFailureWatcher()
172
+ const a = makeAgent('klanker', w)
173
+ a.fail('mcp__perplexity__perplexity_ask', PPLX_NO_CREDIT, T0)
174
+ expect(a.alerts).toHaveLength(1)
175
+ // 401 status, but the credit wording must win — the key is fine, the
176
+ // account is empty, and re-issuing it would waste the operator's time.
177
+ expect(a.alerts[0].cls).toBe('credit')
178
+ expect(route(a.alerts[0]).operatorText).toContain('Top up the balance')
179
+ })
180
+
181
+ it('sees a hard wall wrapped in throttle language (429 + "rate limit")', () => {
182
+ // Providers routinely dress a monthly wall up as a rate limit. The wall is
183
+ // the real news; silencing it as a throttle would lose the capability
184
+ // quietly, which is the exact failure this whole feature exists to stop.
185
+ const w = new McpFailureWatcher()
186
+ const a = makeAgent('klanker', w)
187
+ a.fail(
188
+ 'mcp__perplexity__perplexity_search',
189
+ 'Rate limit exceeded — monthly quota exceeded for your plan (429)',
190
+ T0,
191
+ )
192
+ expect(a.alerts).toHaveLength(1)
193
+ expect(a.alerts[0].cls).toBe('quota')
194
+ })
195
+
196
+ it('routes a hard usage wall to the quota remedy', () => {
197
+ const w = new McpFailureWatcher()
198
+ const a = makeAgent('klanker', w)
199
+ a.fail('mcp__perplexity__perplexity_research', PPLX_QUOTA, T0)
200
+ expect(a.alerts).toHaveLength(1)
201
+ expect(a.alerts[0].cls).toBe('quota')
202
+ expect(route(a.alerts[0]).operatorText).toContain('Raise the plan limit')
203
+ })
204
+ })
205
+
206
+ describe('deduplication — a storm of failures is one alert, not a storm of alerts', () => {
207
+ it('a second agent failing the same way in the window produces NO second alert', () => {
208
+ // One shared watcher stands in for one gateway process seeing several
209
+ // agents/sub-agents; the cross-CONTAINER limit is documented in the module.
210
+ const w = new McpFailureWatcher()
211
+ const klanker = makeAgent('klanker', w)
212
+ const scribe = makeAgent('scribe', w)
213
+
214
+ klanker.fail('mcp__perplexity__perplexity_search', PPLX_401, T0)
215
+ scribe.fail('mcp__perplexity__perplexity_ask', PPLX_401, T0 + 1_000)
216
+ klanker.fail('mcp__perplexity__perplexity_search', PPLX_401, T0 + 30_000)
217
+
218
+ expect(klanker.alerts).toHaveLength(1)
219
+ expect(scribe.alerts).toHaveLength(0)
220
+ })
221
+
222
+ it('re-notifies after the 6h house cadence, and NAMES everyone seen since', () => {
223
+ const w = new McpFailureWatcher()
224
+ const klanker = makeAgent('klanker', w)
225
+ const scribe = makeAgent('scribe', w)
226
+
227
+ klanker.fail('mcp__perplexity__perplexity_search', PPLX_401, T0)
228
+ expect(klanker.alerts).toHaveLength(1)
229
+
230
+ // Silent for the whole window, however many failures land.
231
+ scribe.fail('mcp__perplexity__perplexity_ask', PPLX_401, T0 + RENOTIFY_MS - 1)
232
+ expect(scribe.alerts).toHaveLength(0)
233
+
234
+ // At the boundary it fires again, accumulating everyone seen meanwhile.
235
+ scribe.fail('mcp__perplexity__perplexity_ask', PPLX_401, T0 + RENOTIFY_MS)
236
+ expect(scribe.alerts).toHaveLength(1)
237
+ const repeat = scribe.alerts[0]
238
+ expect(repeat.renotify).toBe(true)
239
+ expect(repeat.agents).toEqual(['scribe'])
240
+ expect(repeat.occurrences).toBe(2)
241
+ expect(route(repeat).operatorText).toContain('still failing')
242
+ })
243
+
244
+ it('matches the house re-notify cadence used by the hindsight watcher', () => {
245
+ expect(RENOTIFY_MS).toBe(6 * 60 * 60 * 1000)
246
+ })
247
+
248
+ it('tracks credit and credential walls on the same server independently', () => {
249
+ const led = new McpFailureLedger()
250
+ expect(led.note({ server: 'perplexity', agent: 'a', cls: 'credential', now: T0 })).not.toBeNull()
251
+ // A different failure CLASS is a different problem with a different remedy,
252
+ // so it must not be silenced by the first one's window.
253
+ expect(led.note({ server: 'perplexity', agent: 'a', cls: 'credit', now: T0 })).not.toBeNull()
254
+ })
255
+ })
256
+
257
+ describe('ordinary tool failures never page the operator', () => {
258
+ for (const [label, body] of ORDINARY_FAILURES) {
259
+ it(`stays silent for ${label}`, () => {
260
+ const w = new McpFailureWatcher()
261
+ const a = makeAgent('klanker', w)
262
+ a.fail('mcp__perplexity__perplexity_search', body, T0)
263
+ expect(a.alerts).toEqual([])
264
+ expect(classifyMcpFailure(body)).toBe('ordinary')
265
+ })
266
+ }
267
+
268
+ it('stays silent for a SUCCESSFUL MCP call', () => {
269
+ const w = new McpFailureWatcher()
270
+ const a = makeAgent('klanker', w)
271
+ a.succeed('mcp__perplexity__perplexity_search', T0)
272
+ expect(a.alerts).toEqual([])
273
+ })
274
+
275
+ it('stays silent for a non-MCP tool that fails with auth-shaped text', () => {
276
+ // A Bash step printing "401 unauthorized" from some unrelated curl is NOT
277
+ // a fleet capability loss and must not page anyone.
278
+ const w = new McpFailureWatcher()
279
+ const a = makeAgent('klanker', w)
280
+ a.fail('Bash', 'curl: server returned status 401 unauthorized', T0)
281
+ expect(a.alerts).toEqual([])
282
+ })
283
+ })
284
+
285
+ describe('the rule is data, and it generalises past Perplexity', () => {
286
+ it('derives the server from any mcp__<server>__<tool> name', () => {
287
+ expect(parseMcpServerFromToolName('mcp__perplexity__perplexity_search')).toBe('perplexity')
288
+ expect(parseMcpServerFromToolName('mcp__meta-ads__list_campaigns')).toBe('meta-ads')
289
+ expect(parseMcpServerFromToolName('Read')).toBeNull()
290
+ expect(parseMcpServerFromToolName(undefined)).toBeNull()
291
+ })
292
+
293
+ it('covers other paid dependencies with no new code — Eraser, Brevo', () => {
294
+ const w = new McpFailureWatcher()
295
+ const a = makeAgent('klanker', w)
296
+ a.fail('mcp__eraser__create_diagram', '403 Forbidden: api key has been revoked', T0)
297
+ a.fail('mcp__brevo__send_email', '{"code":"unauthorized","message":"Key not found"}', T0)
298
+ expect(a.alerts.map(x => x.server)).toEqual(['eraser', 'brevo'])
299
+ expect(route(a.alerts[0]).operatorText).toContain('eraser/api-key')
300
+ expect(route(a.alerts[1]).operatorText).toContain('brevo/api-key')
301
+ })
302
+
303
+ it('an unregistered MCP server still alerts, with a conventional key name', () => {
304
+ const w = new McpFailureWatcher()
305
+ const a = makeAgent('klanker', w)
306
+ a.fail('mcp__somenewthing__do_it', '401 unauthorized: invalid api key', T0)
307
+ expect(a.alerts).toHaveLength(1)
308
+ expect(a.alerts[0].vaultKey).toBe('somenewthing/api-key')
309
+ })
310
+ })
@@ -0,0 +1,221 @@
1
+ /**
2
+ * The outbox sweep must not hammer a flood ban it can already see (#3853).
3
+ *
4
+ * ── The bug these tests guard ────────────────────────────────────────────
5
+ * `startOutboxSweep` built its retry policy as `createRetryApiCall({ log })` —
6
+ * the ONE wiring in the tree with neither `floodWaitRemainingMs` (read the
7
+ * persisted breaker) nor `onFloodWait` (write to it). Combined with a fixed
8
+ * 5s tick and a catch that released the claim with no backoff, the sweep
9
+ * re-issued a single undeliverable `sendMessage` every 5 seconds for the whole
10
+ * of overlord's 15908s (4.4h) ban on 2026-07-27: 228 requests into a window
11
+ * the breaker had recorded and could have answered in one read.
12
+ *
13
+ * The gateway log is the proof and the shape of the outcome assertion:
14
+ * grep -c 'flood window still open' gateway-supervisor.log → 0
15
+ * grep -c 'flood ban of' gateway-supervisor.log → 230
16
+ * grep -c 'outbox-sweep: send failed' → 228
17
+ *
18
+ * Every test below asserts an OUTCOME — whether the wire was touched and
19
+ * whether the record survived — not that a code path executed.
20
+ */
21
+
22
+ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
23
+ import { mkdtempSync, rmSync } from 'node:fs'
24
+ import { tmpdir } from 'node:os'
25
+ import { join } from 'node:path'
26
+
27
+ import {
28
+ writeOutboxRecordAtomic,
29
+ listPendingRecords,
30
+ sha256Hex,
31
+ type OutboxRecord,
32
+ } from '../outbox.js'
33
+ import {
34
+ sweepOutbox,
35
+ createSweepBackoff,
36
+ OUTBOX_SWEEP_INTERVAL_MS,
37
+ OUTBOX_SWEEP_BACKOFF_MAX_MS,
38
+ } from '../gateway/outbox-sweep.js'
39
+
40
+ function rec(over: Partial<OutboxRecord> = {}): OutboxRecord {
41
+ const text = over.text ?? 'the answer the operator is waiting for'
42
+ return {
43
+ turnNonce: over.turnNonce ?? 'nonce-1',
44
+ chatId: over.chatId ?? '111',
45
+ threadId: over.threadId ?? null,
46
+ text,
47
+ textSha256: sha256Hex(text),
48
+ createdAt: over.createdAt ?? 0,
49
+ source: over.source ?? 'channel',
50
+ }
51
+ }
52
+
53
+ describe('outbox sweep vs an open flood window', () => {
54
+ let dir: string
55
+ beforeEach(() => {
56
+ dir = mkdtempSync(join(tmpdir(), 'outbox-flood-'))
57
+ })
58
+ afterEach(() => {
59
+ rmSync(dir, { recursive: true, force: true })
60
+ })
61
+
62
+ it('does not hit the wire while a flood window is open, and keeps the record', async () => {
63
+ writeOutboxRecordAtomic(rec(), dir)
64
+ const send = vi.fn(async () => 1)
65
+ const log = vi.fn()
66
+
67
+ const summary = await sweepOutbox({
68
+ stateDir: dir,
69
+ log,
70
+ send,
71
+ textAlreadyDelivered: () => false,
72
+ // 12247s remaining — the exact shape of the window the 2026-07-27 sweep
73
+ // ignored 228 times.
74
+ floodWaitRemainingMs: () => 12_247_000,
75
+ now: () => 10 * 60_000,
76
+ })
77
+
78
+ // THE outcome: nothing reached Telegram.
79
+ expect(send).not.toHaveBeenCalled()
80
+ expect(summary.floodDeferred).toBe(true)
81
+ expect(summary.floodRemainingMs).toBe(12_247_000)
82
+ expect(summary.delivered).toBe(0)
83
+ // And nothing was lost — the record is still pending for the next sweep.
84
+ expect(listPendingRecords(dir)).toHaveLength(1)
85
+ // The deferral is NOT logged per-tick — a 4.4h ban is ~3181 ticks, and one
86
+ // line each would relocate the flood from the wire to the disk. The tick
87
+ // loop reports it at most once per OUTBOX_SWEEP_DEFER_LOG_INTERVAL_MS.
88
+ expect(log).not.toHaveBeenCalled()
89
+ })
90
+
91
+ it('delivers the SAME record once the window closes', async () => {
92
+ writeOutboxRecordAtomic(rec(), dir)
93
+ const send = vi.fn(async () => 42)
94
+ let remaining = 12_247_000
95
+
96
+ const deferred = await sweepOutbox({
97
+ stateDir: dir,
98
+ send,
99
+ textAlreadyDelivered: () => false,
100
+ floodWaitRemainingMs: () => remaining,
101
+ now: () => 10 * 60_000,
102
+ })
103
+ expect(deferred.floodDeferred).toBe(true)
104
+ expect(send).not.toHaveBeenCalled()
105
+
106
+ remaining = 0
107
+ const after = await sweepOutbox({
108
+ stateDir: dir,
109
+ send,
110
+ textAlreadyDelivered: () => false,
111
+ floodWaitRemainingMs: () => remaining,
112
+ now: () => 20 * 60_000,
113
+ })
114
+
115
+ expect(send).toHaveBeenCalledTimes(1)
116
+ expect(send.mock.calls[0]?.[2]).toContain('the answer the operator is waiting for')
117
+ expect(after.delivered).toBe(1)
118
+ expect(listPendingRecords(dir)).toHaveLength(0)
119
+ })
120
+
121
+ it('sweeps normally when no window is open', async () => {
122
+ writeOutboxRecordAtomic(rec(), dir)
123
+ const send = vi.fn(async () => 7)
124
+ const summary = await sweepOutbox({
125
+ stateDir: dir,
126
+ send,
127
+ textAlreadyDelivered: () => false,
128
+ floodWaitRemainingMs: () => 0,
129
+ now: () => 10 * 60_000,
130
+ })
131
+ expect(send).toHaveBeenCalledTimes(1)
132
+ expect(summary.delivered).toBe(1)
133
+ expect(summary.floodDeferred).toBeUndefined()
134
+ })
135
+
136
+ it('FAILS OPEN: a throwing probe must never strand the outbox', async () => {
137
+ writeOutboxRecordAtomic(rec(), dir)
138
+ const send = vi.fn(async () => 7)
139
+ const summary = await sweepOutbox({
140
+ stateDir: dir,
141
+ send,
142
+ textAlreadyDelivered: () => false,
143
+ floodWaitRemainingMs: () => {
144
+ throw new Error('EACCES: flood-wait.json owned by another uid')
145
+ },
146
+ now: () => 10 * 60_000,
147
+ })
148
+ expect(send).toHaveBeenCalledTimes(1)
149
+ expect(summary.delivered).toBe(1)
150
+ })
151
+
152
+ it('reports send failures so the tick can back off', async () => {
153
+ writeOutboxRecordAtomic(rec(), dir)
154
+ const send = vi.fn(async () => {
155
+ throw new Error('FLOOD_WAIT_ACTIVE')
156
+ })
157
+ const summary = await sweepOutbox({
158
+ stateDir: dir,
159
+ send,
160
+ textAlreadyDelivered: () => false,
161
+ floodWaitRemainingMs: () => 0,
162
+ now: () => 10 * 60_000,
163
+ })
164
+ expect(summary.sendFailures).toBe(1)
165
+ expect(summary.delivered).toBe(0)
166
+ // Claim released — the record is still pending, never lost.
167
+ expect(listPendingRecords(dir)).toHaveLength(1)
168
+ })
169
+ })
170
+
171
+ describe('createSweepBackoff — the fixed-5s-forever regression', () => {
172
+ it('rides the normal tick on the FIRST failure, then doubles', () => {
173
+ const b = createSweepBackoff()
174
+ expect(b.ready(0)).toBe(true)
175
+
176
+ expect(b.noteFailure(0)).toBe(OUTBOX_SWEEP_INTERVAL_MS) // 5s
177
+ expect(b.ready(4_999)).toBe(false)
178
+ expect(b.ready(5_000)).toBe(true)
179
+
180
+ expect(b.noteFailure(5_000)).toBe(10_000)
181
+ expect(b.noteFailure(15_000)).toBe(20_000)
182
+ expect(b.noteFailure(35_000)).toBe(40_000)
183
+ })
184
+
185
+ it('caps the delay so the sweep never stops entirely', () => {
186
+ const b = createSweepBackoff()
187
+ let last = 0
188
+ for (let i = 0; i < 40; i++) last = b.noteFailure(i * 1_000_000)
189
+ expect(last).toBe(OUTBOX_SWEEP_BACKOFF_MAX_MS)
190
+ })
191
+
192
+ it('the 4.4h-ban scenario: ≤ 60 sweeps, not 228', () => {
193
+ // Replay the incident's shape — a persistently failing send across a
194
+ // 15908s window — and count how many sweeps the pacer would permit.
195
+ const b = createSweepBackoff()
196
+ const banMs = 15_908_000
197
+ let now = 0
198
+ let sweeps = 0
199
+ while (now < banMs) {
200
+ if (b.ready(now)) {
201
+ sweeps++
202
+ b.noteFailure(now)
203
+ }
204
+ now += OUTBOX_SWEEP_INTERVAL_MS
205
+ }
206
+ // Fixed 5s ticking would be 3181 attempts; the real sweep issued 228
207
+ // before the operator intervened. Exponential + cap keeps it under 60.
208
+ expect(sweeps).toBeLessThan(60)
209
+ expect(sweeps).toBeGreaterThan(0)
210
+ })
211
+
212
+ it('a successful sweep clears the backoff', () => {
213
+ const b = createSweepBackoff()
214
+ b.noteFailure(0)
215
+ b.noteFailure(5_000)
216
+ expect(b.ready(6_000)).toBe(false)
217
+ b.noteSuccess()
218
+ expect(b.ready(6_000)).toBe(true)
219
+ expect(b.failures()).toBe(0)
220
+ })
221
+ })
@@ -33,7 +33,6 @@ import {
33
33
  STATUS_CARD_CHAR_BUDGET,
34
34
  STATUS_LINE_MAX,
35
35
  WORKER_STEP_INDENT,
36
- SUBORDINATE_LINE_INDENT,
37
36
  } from '../status-no-truncate.js'
38
37
 
39
38
  /**
@@ -135,31 +134,28 @@ describe('combined worker card survives the pinned-bar collapse (#3666)', () =>
135
134
 
136
135
  it('kills the exact artifacts from the report', () => {
137
136
  const collapsed = collapsePreview(body)
138
- // 1. the count/ordinal collision — glance line into row 1's ordinal.
139
- // Since #3820 a worker header also carries the card-level
140
- // SUBORDINATE_LINE_INDENT (the whole worker card nests under the 🤖
141
- // agent card), so this seam is separator + card indent.
137
+ // 1. the count/ordinal collision — glance line into row 1's ordinal. Since
138
+ // #3842 the row header is FLUSH (the #3820 card-level indent is gone),
139
+ // so the separator is the ONLY thing holding this seam apart.
142
140
  // (The reported spelling was `3 running1.`; with the packed glance line
143
141
  // the same seam now reads `… 512.3k tok` -> `1. Fix issue`, so assert on
144
142
  // the CURRENT last token of line 1 — an assertion on the old spelling
145
143
  // alone would be vacuously green.)
146
144
  expect(collapsed).not.toContain('running1.')
147
145
  expect(collapsed).not.toContain('tok1.')
148
- expect(collapsed).toContain(`tok${NB}${SUBORDINATE_LINE_INDENT}1. Fix issue`)
149
- // 2. the mid-word ✓ (model tag running into the step trail). Here the
150
- // separator and WORKER_STEP_INDENT (three U+2800 leading a step line)
151
- // stack, so the seam is separator + indent, asserted against both
152
- // constants rather than a hardcoded run of spaces.
146
+ expect(collapsed).toContain(`tok${NB}1. Fix issue`)
147
+ // 2. the mid-word ✓ (model tag running into the step trail). A step line
148
+ // still leads with WORKER_STEP_INDENT (three U+2800), so this seam is
149
+ // separator + step indent asserted against the constant rather than a
150
+ // hardcoded run.
153
151
  expect(collapsed).not.toContain('opus 5✓')
154
- expect(collapsed).toContain(`opus 5${NB}${SUBORDINATE_LINE_INDENT}${WORKER_STEP_INDENT}✓`)
152
+ expect(collapsed).toContain(`opus 5${NB}${WORKER_STEP_INDENT}✓`)
155
153
  // 3. the step trail running into the next step, and into the next row's
156
- // header (post-#3820 that last seam is separator + card indent).
154
+ // header (post-#3842 that last seam is the separator alone).
157
155
  expect(collapsed).not.toContain('gateway.ts→')
158
- expect(collapsed).toContain(
159
- `gateway.ts${NB}${SUBORDINATE_LINE_INDENT}${WORKER_STEP_INDENT}→`,
160
- )
156
+ expect(collapsed).toContain(`gateway.ts${NB}${WORKER_STEP_INDENT}→`)
161
157
  expect(collapsed).not.toContain('search2.')
162
- expect(collapsed).toContain(`search${NB}${SUBORDINATE_LINE_INDENT}2.`)
158
+ expect(collapsed).toContain(`search${NB}2.`)
163
159
  })
164
160
 
165
161
  it('leads with a self-contained glance that ends in a unit word, not a bare number', () => {
@@ -178,13 +174,10 @@ describe('combined worker card survives the pinned-bar collapse (#3666)', () =>
178
174
  // show the collapsed preview mashes again. Without this, the assertions
179
175
  // above could all be passing for reasons unrelated to the fix.
180
176
  //
181
- // The control runs on the 🤖 AGENT card, not the worker card: since #3820
182
- // every line of a worker card after line 1 carries a leading
183
- // SUBORDINATE_LINE_INDENT, which separates its seams independently of the
184
- // collapse separator so a worker-card control would no longer isolate the
185
- // separator's contribution. The agent card is the surface where the
186
- // separator is still the ONLY thing holding the seams apart, which is
187
- // exactly what this control must measure.
177
+ // The control runs on the 🤖 AGENT card. Post-#3842 the single-worker card
178
+ // is flush too, so either would isolate the separator's contribution; the
179
+ // agent card is kept because it has no step indent on ANY line, so no
180
+ // future indent change can quietly make this control vacuous.
188
181
  const agent = renderActivityFeed([
189
182
  'Reading gateway.ts',
190
183
  'Searching memory',
@@ -228,7 +221,9 @@ describe('single-worker / agent status card survives the collapse too (#3666)',
228
221
  expectNoMashedSeams(body)
229
222
  const collapsed = collapsePreview(body)
230
223
  expect(collapsed).not.toContain('toolsstarting')
231
- expect(collapsed).toContain(`0 tools${NB}${SUBORDINATE_LINE_INDENT}starting`)
224
+ // #3842: the single-worker card is flush, so the separator alone holds the
225
+ // hand-rolled `starting…` seam apart — nothing else masks a regression.
226
+ expect(collapsed).toContain(`0 tools${NB}starting`)
232
227
  })
233
228
 
234
229
  it('the nested child block stays separated even though its indent is ASCII (#3668)', () => {