switchroom 0.18.17 → 0.18.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/agent-scheduler/index.js +13 -0
  2. package/dist/auth-broker/index.js +13 -0
  3. package/dist/cli/notion-write-pretool.mjs +13 -0
  4. package/dist/cli/switchroom.js +605 -479
  5. package/dist/host-control/main.js +17 -1
  6. package/dist/vault/approvals/kernel-server.js +13 -0
  7. package/dist/vault/broker/server.js +13 -0
  8. package/package.json +1 -1
  9. package/telegram-plugin/bridge/bridge.ts +7 -1
  10. package/telegram-plugin/dist/bridge/bridge.js +26 -1
  11. package/telegram-plugin/dist/gateway/gateway.js +1544 -619
  12. package/telegram-plugin/dist/server.js +32 -1
  13. package/telegram-plugin/fleet-fallback-resume.ts +26 -3
  14. package/telegram-plugin/format.ts +137 -213
  15. package/telegram-plugin/gateway/approval-hold.ts +49 -0
  16. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
  17. package/telegram-plugin/gateway/gateway.ts +399 -85
  18. package/telegram-plugin/gateway/linear-activity.ts +20 -4
  19. package/telegram-plugin/gateway/outbound-send-path.ts +9 -7
  20. package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
  21. package/telegram-plugin/gateway/session-model-file.ts +103 -0
  22. package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
  23. package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
  24. package/telegram-plugin/llm-error-present.ts +474 -0
  25. package/telegram-plugin/operator-events.ts +7 -1
  26. package/telegram-plugin/permission-title.ts +172 -10
  27. package/telegram-plugin/premium-recovery.ts +101 -0
  28. package/telegram-plugin/raw-error-scrub.ts +73 -0
  29. package/telegram-plugin/retry-api-call.ts +8 -2
  30. package/telegram-plugin/send-gate-degraded.test.ts +152 -1
  31. package/telegram-plugin/send-gate-observability.test.ts +140 -0
  32. package/telegram-plugin/send-gate-observability.ts +65 -20
  33. package/telegram-plugin/send-gate.test.ts +143 -1
  34. package/telegram-plugin/send-gate.ts +212 -19
  35. package/telegram-plugin/session-tail.ts +16 -0
  36. package/telegram-plugin/shared/local-time.ts +69 -0
  37. package/telegram-plugin/stream-reply-handler.ts +5 -14
  38. package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
  39. package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
  40. package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
  41. package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
  42. package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
  43. package/telegram-plugin/tests/format-consistency.test.ts +68 -53
  44. package/telegram-plugin/tests/formatting-parse-regression.test.ts +5 -6
  45. package/telegram-plugin/tests/formatting-torture-set.ts +1 -1
  46. package/telegram-plugin/tests/gateway-outbound-redact.test.ts +26 -0
  47. package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
  48. package/telegram-plugin/tests/llm-error-present.test.ts +481 -0
  49. package/telegram-plugin/tests/outbound-send-path.test.ts +4 -3
  50. package/telegram-plugin/tests/paragraph-normalizer.test.ts +42 -100
  51. package/telegram-plugin/tests/permission-title.test.ts +167 -4
  52. package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
  53. package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
  54. package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
  55. package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
  56. package/telegram-plugin/tests/stream-reply-handler.test.ts +9 -12
  57. package/telegram-plugin/tests/telegram-format.test.ts +86 -31
  58. package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
  59. package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
  60. package/telegram-plugin/tests/turn-flush-safety.test.ts +17 -21
  61. package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
  62. package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
  63. package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
  64. package/telegram-plugin/tier-downgrade.ts +198 -0
  65. package/telegram-plugin/tool-activity-summary.ts +99 -0
  66. package/telegram-plugin/turn-flush-safety.ts +4 -3
  67. package/telegram-plugin/worker-activity-feed.ts +509 -409
@@ -0,0 +1,492 @@
1
+ import { describe, expect, it } from 'vitest'
2
+ import {
3
+ createWorkerActivityFeed,
4
+ type BotApiForWorkerFeed,
5
+ type WorkerActivityView,
6
+ } from '../worker-activity-feed.js'
7
+ import { renderCombinedWorkerFeed } from '../tool-activity-summary.js'
8
+ import { STATUS_CARD_CHAR_BUDGET } from '../status-no-truncate.js'
9
+ import { createSendGate, isSendGateShed, type Clock } from '../send-gate.js'
10
+
11
+ /**
12
+ * Outcome tests for the coalesced worker-activity feed (#3084 follow-up).
13
+ *
14
+ * These drive the REAL send gate (perChatPerSec=1, burst 3, editFloorMs=1500 —
15
+ * the shipped defaults) with the feed's real send/edit adapters wired exactly
16
+ * as the gateway wires them (`useful` sends, `cosmetic` edits carrying
17
+ * messageId+editPayload). The assertions prove the coalescing win that the
18
+ * per-worker-message model cannot achieve:
19
+ *
20
+ * - N=15 workers in one chat produce exactly ONE combined message (not 15).
21
+ * - Every worker's latest state refreshes TOGETHER in that one message within
22
+ * one ~1.5s edit cycle (the per-worker model refreshes each card only once
23
+ * per ~N·(1/perChatPerSec) seconds — liveness collapse).
24
+ * - The gate SHEDS ~nothing under the coalesced stream (the per-worker model
25
+ * sheds ~14/15 of every second's cosmetic edits — a real ban happened).
26
+ * - A `critical` reply mid-storm is admitted, never shed/starved.
27
+ * - Redundant identical renders don't produce redundant landed edits.
28
+ */
29
+
30
+ /**
31
+ * Deterministic fake clock shared by BOTH the send gate (Clock.now/sleep) and
32
+ * the feed (`now: () => clock.now()`). Mirrors send-gate.test.ts's FakeClock.
33
+ */
34
+ class FakeClock implements Clock {
35
+ private cur = 0
36
+ private seq = 0
37
+ private timers: { at: number; id: number; resolve: () => void }[] = []
38
+ now(): number {
39
+ return this.cur
40
+ }
41
+ sleep(ms: number): Promise<void> {
42
+ return new Promise<void>((resolve) => {
43
+ this.timers.push({ at: this.cur + ms, id: this.seq++, resolve })
44
+ })
45
+ }
46
+ async advance(ms: number): Promise<void> {
47
+ const target = this.cur + ms
48
+ for (;;) {
49
+ const due = this.timers.filter((t) => t.at <= target).sort((a, b) => a.at - b.at || a.id - b.id)
50
+ if (due.length === 0) break
51
+ const t = due[0]
52
+ this.timers = this.timers.filter((x) => x !== t)
53
+ this.cur = t.at
54
+ t.resolve()
55
+ await flush()
56
+ }
57
+ this.cur = target
58
+ }
59
+ }
60
+ function flush(): Promise<void> {
61
+ return new Promise((r) => setImmediate(r))
62
+ }
63
+ /** Flush the promise-chain a few times so feed chains + gate drivers settle. */
64
+ async function settle(clock: FakeClock, ms = 0): Promise<void> {
65
+ await clock.advance(ms)
66
+ for (let i = 0; i < 6; i++) await flush()
67
+ }
68
+
69
+ interface LandedEdit {
70
+ chatId: string
71
+ messageId: number
72
+ text: string
73
+ at: number
74
+ }
75
+
76
+ /**
77
+ * Build the feed + a bot adapter routed through a REAL send gate, exactly like
78
+ * the gateway (worker-feed sends are `useful`, edits are `cosmetic` carrying
79
+ * messageId + editPayload for the gate's floor/coalesce/no-op logic).
80
+ */
81
+ function harness(clock: FakeClock, opts: { maxRows?: number } = {}) {
82
+ const gate = createSendGate({
83
+ enabled: true,
84
+ clock,
85
+ globalPerSec: 1000, // global is not the limiter under test
86
+ globalBurst: 1000,
87
+ perChatPerSec: 1,
88
+ perChatBurst: 3,
89
+ editFloorMs: 1500,
90
+ })
91
+ const sends: { chatId: string; messageId: number; at: number }[] = []
92
+ const edits: LandedEdit[] = []
93
+ let seq = 1000
94
+ const bot: BotApiForWorkerFeed = {
95
+ sendMessage: (chatId, _text, o) =>
96
+ gate.gate(
97
+ async () => {
98
+ const messageId = seq++
99
+ sends.push({ chatId, messageId, at: clock.now() })
100
+ return { message_id: messageId }
101
+ },
102
+ { chat_id: chatId, priorityClass: 'useful', verb: 'worker-feed' },
103
+ ) as Promise<{ message_id: number }>,
104
+ editMessageText: (chatId, messageId, text, o) =>
105
+ gate.gate(
106
+ async () => {
107
+ edits.push({ chatId, messageId, text, at: clock.now() })
108
+ return true
109
+ },
110
+ {
111
+ chat_id: chatId,
112
+ priorityClass: 'cosmetic',
113
+ messageId,
114
+ editPayload: text,
115
+ verb: 'worker-feed',
116
+ },
117
+ ),
118
+ }
119
+ const feed = createWorkerActivityFeed({
120
+ bot,
121
+ now: () => clock.now(),
122
+ // Let the send gate's editFloorMs (1500) be the pacing authority: the feed's
123
+ // own proactive throttle matches it so the coalesced stream never outruns
124
+ // the per-chat bucket (which is what would shed it).
125
+ minEditIntervalMs: 1500,
126
+ heartbeatTickMs: 1500,
127
+ firstPaintMinMs: 0,
128
+ // No auto-timer: the test drives heartbeatTick() deterministically.
129
+ setInterval: () => 0,
130
+ clearInterval: () => {},
131
+ maxRows: opts.maxRows ?? 8,
132
+ })
133
+ return { gate, feed, sends, edits }
134
+ }
135
+
136
+ function view(desc: string, step: string, elapsedMs: number): WorkerActivityView {
137
+ return {
138
+ description: desc,
139
+ lastTool: null,
140
+ toolCount: 3,
141
+ latestSummary: step,
142
+ elapsedMs,
143
+ state: 'running',
144
+ }
145
+ }
146
+
147
+ describe('coalesced worker feed — one message per chat under load', () => {
148
+ it('N=15 workers over 60s produce ONE message with a bounded, non-shedding edit stream', async () => {
149
+ const clock = new FakeClock()
150
+ const CHAT = 'chat-storm'
151
+ const N = 15
152
+ const { gate, feed, sends, edits } = harness(clock, { maxRows: N })
153
+ const ids = Array.from({ length: N }, (_, i) => `w${i}`)
154
+
155
+ // Latest step each worker last emitted (unique per cycle so we can prove
156
+ // freshness in the rendered body).
157
+ const latest: Record<string, string> = {}
158
+
159
+ const CYCLE = 1500
160
+ const CYCLES = 40 // 40 * 1.5s = 60s
161
+ for (let c = 0; c < CYCLES; c++) {
162
+ await clock.advance(CYCLE)
163
+ for (const id of ids) {
164
+ latest[id] = `${id}-step-${c}`
165
+ void feed.update(id, CHAT, view(`task ${id}`, latest[id], clock.now()))
166
+ }
167
+ await settle(clock)
168
+ feed.heartbeatTick() // flush the accumulated combined render
169
+ await settle(clock)
170
+ }
171
+ await settle(clock, 3000)
172
+
173
+ // ONE combined message — not one per worker. This is the structural
174
+ // distinction from the per-worker-message model (which sends N).
175
+ expect(sends.length).toBe(1)
176
+ expect(feed.size).toBe(N)
177
+
178
+ // Bounded edit stream: at most perChatPerSec·T + burst landed edits.
179
+ const seconds = (CYCLES * CYCLE) / 1000
180
+ const bound = Math.ceil(1 * seconds + 3)
181
+ expect(edits.length).toBeLessThanOrEqual(bound)
182
+ // And it actually stayed live (edited many times, not frozen).
183
+ expect(edits.length).toBeGreaterThan(CYCLES / 3)
184
+
185
+ // Shed pin: the coalesced single stream sheds ~nothing. The per-worker model
186
+ // would shed ~ (N-1)/N of every second's cosmetic edits.
187
+ expect(gate.stats().global.shed).toBeLessThanOrEqual(3)
188
+
189
+ // Liveness: the LAST landed edit carries EVERY worker's latest step — all
190
+ // rows refreshed together in the one message within one edit cycle.
191
+ const lastBody = edits[edits.length - 1].text
192
+ for (const id of ids) {
193
+ expect(lastBody).toContain(latest[id])
194
+ }
195
+ })
196
+
197
+ it('admits a critical reply mid-storm (never shed or starved by the cosmetic feed)', async () => {
198
+ const clock = new FakeClock()
199
+ const CHAT = 'chat-crit'
200
+ const { gate, feed } = harness(clock, { maxRows: 15 })
201
+ const ids = Array.from({ length: 12 }, (_, i) => `k${i}`)
202
+
203
+ // Warm the storm.
204
+ for (let c = 0; c < 4; c++) {
205
+ await clock.advance(1500)
206
+ for (const id of ids) void feed.update(id, CHAT, view(`task ${id}`, `${id}-s${c}`, clock.now()))
207
+ await settle(clock)
208
+ feed.heartbeatTick()
209
+ await settle(clock)
210
+ }
211
+
212
+ // A critical reply to the SAME chat, mid-storm. `critical` is never shed and
213
+ // never blocks behind the cosmetic feed edits.
214
+ const CRIT = { ok: true, id: 'crit-reply' }
215
+ const p = gate.gate(async () => CRIT, {
216
+ chat_id: CHAT,
217
+ priorityClass: 'critical',
218
+ verb: 'reply',
219
+ })
220
+ await settle(clock, 2000)
221
+ const res = await p
222
+ expect(isSendGateShed(res)).toBe(false)
223
+ expect(res).toEqual(CRIT)
224
+ })
225
+
226
+ it('does not produce redundant landed edits for redundant renders (coalesce/no-op)', async () => {
227
+ const clock = new FakeClock()
228
+ const CHAT = 'chat-dedupe'
229
+ const { gate, feed, edits } = harness(clock)
230
+
231
+ // First paint + one edit with a real state change.
232
+ await clock.advance(1500)
233
+ void feed.update('w', CHAT, view('task w', 'step-A', clock.now()))
234
+ await settle(clock)
235
+ feed.heartbeatTick()
236
+ await settle(clock, 2000)
237
+ const afterFirst = edits.length
238
+
239
+ // Re-drive the SAME state many times over several cycles. No new step, no new
240
+ // state → the feed dedups (body === lastBody) and/or the gate drops the no-op;
241
+ // either way, NO new landed edit and NO shed.
242
+ const shedBefore = gate.stats().global.shed
243
+ for (let c = 0; c < 5; c++) {
244
+ await clock.advance(1500)
245
+ void feed.update('w', CHAT, view('task w', 'step-A', clock.now()))
246
+ await settle(clock)
247
+ feed.heartbeatTick()
248
+ await settle(clock, 2000)
249
+ }
250
+ // The header elapsed climbs on heartbeat, so a redundant *content* update
251
+ // must not multiply landed edits beyond the heartbeat's own climb cadence.
252
+ expect(edits.length - afterFirst).toBeLessThanOrEqual(5)
253
+ // Crucially: redundant identical narrative never sheds.
254
+ expect(gate.stats().global.shed).toBe(shedBefore)
255
+ })
256
+ })
257
+
258
+ /**
259
+ * Deterministic lifecycle transitions with a DIRECT fake bot (no gate) so we
260
+ * can assert the exact rendered body at each step.
261
+ */
262
+ function directBot() {
263
+ const sent: { chatId: string; text: string; messageId: number }[] = []
264
+ const edits: { messageId: number; text: string }[] = []
265
+ let seq = 500
266
+ const bot: BotApiForWorkerFeed = {
267
+ sendMessage: async (chatId, text) => {
268
+ const messageId = seq++
269
+ sent.push({ chatId, text, messageId })
270
+ return { message_id: messageId }
271
+ },
272
+ editMessageText: async (_chatId, messageId, text) => {
273
+ edits.push({ messageId, text })
274
+ return true
275
+ },
276
+ }
277
+ return { bot, sent, edits }
278
+ }
279
+
280
+ describe('coalesced worker feed — lifecycle transitions', () => {
281
+ it('two workers in one chat share ONE message that becomes a combined body', async () => {
282
+ let clock = 0
283
+ const { bot, sent } = directBot()
284
+ const feed = createWorkerActivityFeed({
285
+ bot,
286
+ now: () => clock,
287
+ minEditIntervalMs: 0,
288
+ firstPaintMinMs: 0,
289
+ setInterval: () => 0,
290
+ clearInterval: () => {},
291
+ })
292
+ clock = 1000
293
+ await feed.update('a', 'chat', view('task A', 'a-doing', 1000))
294
+ clock = 1100
295
+ await feed.update('b', 'chat', view('task B', 'b-doing', 1100))
296
+
297
+ // ONE message for the chat (both workers share it), and its id is the same.
298
+ expect(sent.length).toBe(1)
299
+ expect(feed.messageIdOf('a')).toBe(feed.messageIdOf('b'))
300
+ expect(feed.size).toBe(2)
301
+ })
302
+
303
+ it('finishing one of two drops its row (result NOT in the feed edit); the survivor stays live', async () => {
304
+ let clock = 0
305
+ const { bot, edits } = directBot()
306
+ const feed = createWorkerActivityFeed({
307
+ bot,
308
+ now: () => clock,
309
+ minEditIntervalMs: 0,
310
+ firstPaintMinMs: 0,
311
+ setInterval: () => 0,
312
+ clearInterval: () => {},
313
+ })
314
+ clock = 1000
315
+ await feed.update('a', 'chat', view('task A', 'a-doing', 1000))
316
+ await feed.update('b', 'chat', view('task B', 'b-doing', 1000))
317
+ clock = 2000
318
+ await feed.finish('a', {
319
+ description: 'task A',
320
+ lastTool: null,
321
+ toolCount: 3,
322
+ latestSummary: 'SECRET-RESULT-A should reach the user via handback only',
323
+ elapsedMs: 2000,
324
+ state: 'done',
325
+ })
326
+
327
+ const last = edits[edits.length - 1].text
328
+ // The finished worker's RESULT is never folded into the cosmetic feed edit.
329
+ expect(last).not.toContain('SECRET-RESULT-A')
330
+ // The survivor's live step is still shown.
331
+ expect(last).toContain('b-doing')
332
+ expect(feed.size).toBe(1)
333
+ expect(feed.has('a')).toBe(false)
334
+ expect(feed.has('b')).toBe(true)
335
+ })
336
+
337
+ it('finishing the LAST worker finalizes the shared message to its terminal recap', async () => {
338
+ let clock = 0
339
+ const { bot, edits } = directBot()
340
+ const feed = createWorkerActivityFeed({
341
+ bot,
342
+ now: () => clock,
343
+ minEditIntervalMs: 0,
344
+ firstPaintMinMs: 0,
345
+ setInterval: () => 0,
346
+ clearInterval: () => {},
347
+ })
348
+ clock = 1000
349
+ await feed.update('solo', 'chat', view('task solo', 'working', 1000))
350
+ clock = 2000
351
+ await feed.finish('solo', {
352
+ description: 'task solo',
353
+ lastTool: null,
354
+ toolCount: 5,
355
+ latestSummary: 'the final recap paragraph',
356
+ elapsedMs: 2000,
357
+ state: 'done',
358
+ })
359
+ const last = edits[edits.length - 1].text
360
+ // Terminal recap: the single 🛠 Worker card carries the done state + result.
361
+ expect(last).toContain('the final recap paragraph')
362
+ expect(feed.size).toBe(0)
363
+ // The finalized worker is gated against a late resurrecting cue.
364
+ await feed.update('solo', 'chat', view('task solo', 'late tick', 3000))
365
+ expect(feed.size).toBe(0)
366
+ })
367
+ })
368
+
369
+ describe('coalesced worker feed — GROUP-level pin lifecycle (#3207 review)', () => {
370
+ interface PinCall {
371
+ feedKey: string
372
+ chatId: string
373
+ messageId: number | null
374
+ }
375
+ function pinHarness() {
376
+ const pins: PinCall[] = []
377
+ const { bot } = directBot()
378
+ let clock = 0
379
+ const feed = createWorkerActivityFeed({
380
+ bot,
381
+ now: () => clock,
382
+ minEditIntervalMs: 0,
383
+ firstPaintMinMs: 0,
384
+ setInterval: () => 0,
385
+ clearInterval: () => {},
386
+ reconcilePin: ({ feedKey, chatId, messageId }) => pins.push({ feedKey, chatId, messageId }),
387
+ })
388
+ return { feed, pins, setClock: (t: number) => (clock = t) }
389
+ }
390
+ const done = (desc: string, elapsedMs: number): WorkerActivityView => ({
391
+ description: desc,
392
+ lastTool: null,
393
+ toolCount: 3,
394
+ latestSummary: `${desc} result`,
395
+ elapsedMs,
396
+ state: 'done',
397
+ })
398
+
399
+ it('a mid-group sibling finish does NOT unpin the shared message while a worker still runs', async () => {
400
+ const { feed, pins, setClock } = pinHarness()
401
+ setClock(1000)
402
+ await feed.update('a', 'chat', view('task A', 'a-doing', 1000))
403
+ await feed.update('b', 'chat', view('task B', 'b-doing', 1000))
404
+
405
+ // The group's shared message is pinned (messageId non-null).
406
+ const pinnedAfterPaint = pins.filter((p) => p.messageId != null)
407
+ expect(pinnedAfterPaint.length).toBeGreaterThan(0)
408
+ const sharedMsgId = pinnedAfterPaint[pinnedAfterPaint.length - 1].messageId
409
+ expect(feed.messageIdOf('a')).toBe(sharedMsgId)
410
+ expect(feed.messageIdOf('b')).toBe(sharedMsgId)
411
+
412
+ // A finishes while B still runs — the pin MUST stay (B needs the message).
413
+ setClock(2000)
414
+ await feed.finish('a', done('task A', 2000))
415
+
416
+ // Every reconcilePin emitted through A's finish keeps the message pinned —
417
+ // there is NO unpin (messageId === null) while B is live. This is the exact
418
+ // regression the review flagged: a per-worker unpin here strands B unpinned.
419
+ const last = pins[pins.length - 1]
420
+ expect(last.messageId).toBe(sharedMsgId)
421
+ expect(pins.some((p) => p.messageId === null)).toBe(false)
422
+ // The feed still vouches for the group (reaper exemption stays live).
423
+ const feedKey = last.feedKey
424
+ expect(feed.hasRunningInFeed(feedKey)).toBe(true)
425
+ })
426
+
427
+ it('unpins the shared message only when the LAST worker finishes (group empties)', async () => {
428
+ const { feed, pins, setClock } = pinHarness()
429
+ setClock(1000)
430
+ await feed.update('a', 'chat', view('task A', 'a-doing', 1000))
431
+ await feed.update('b', 'chat', view('task B', 'b-doing', 1000))
432
+ setClock(2000)
433
+ await feed.finish('a', done('task A', 2000))
434
+ expect(pins.some((p) => p.messageId === null)).toBe(false) // B still runs
435
+
436
+ // Now the last worker finishes → the group empties → unpin fires.
437
+ setClock(3000)
438
+ await feed.finish('b', done('task B', 3000))
439
+ const last = pins[pins.length - 1]
440
+ expect(last.messageId).toBeNull()
441
+ const feedKey = last.feedKey
442
+ expect(feed.hasRunningInFeed(feedKey)).toBe(false) // reaper may now reap it
443
+ })
444
+ })
445
+
446
+ describe('renderCombinedWorkerFeed (pure)', () => {
447
+ const row = (i: number, step: string) => ({
448
+ description: `task number ${i}`,
449
+ elapsedMs: 12_000 + i * 1000,
450
+ toolCount: i,
451
+ currentStep: step,
452
+ })
453
+
454
+ it('renders one row-block per worker with a running count header', () => {
455
+ const body = renderCombinedWorkerFeed([row(1, 'alpha step'), row(2, 'beta step')], { maxRows: 8 })!
456
+ expect(body).toContain('Workers')
457
+ expect(body).toContain('2 running')
458
+ expect(body).toContain('task number 1')
459
+ expect(body).toContain('task number 2')
460
+ expect(body).toContain('alpha step')
461
+ expect(body).toContain('beta step')
462
+ })
463
+
464
+ it('caps at maxRows and spills the remainder to a compact +M more working line', () => {
465
+ const rows = Array.from({ length: 15 }, (_, i) => row(i, `s${i}`))
466
+ const body = renderCombinedWorkerFeed(rows, { maxRows: 8 })!
467
+ expect(body).toContain('15 running')
468
+ expect(body).toContain('+7 more working')
469
+ // The 8th row is shown, the 9th is spilled.
470
+ expect(body).toContain('s7')
471
+ expect(body).not.toContain('s8')
472
+ })
473
+
474
+ it('stays under the rich-message wire budget even with many long rows (backstop drops rows)', () => {
475
+ // 120 rows of ~700 raw chars each would be ~84k > the 32768 rich-message
476
+ // cap; the char-budget backstop must shrink the visible set until it fits.
477
+ const rows = Array.from({ length: 120 }, () => ({
478
+ description: 'x'.repeat(300),
479
+ elapsedMs: 60_000,
480
+ toolCount: 40,
481
+ currentStep: 'y'.repeat(400),
482
+ }))
483
+ const body = renderCombinedWorkerFeed(rows, { maxRows: 120 })!
484
+ expect(body.length).toBeLessThanOrEqual(STATUS_CARD_CHAR_BUDGET)
485
+ // The backstop grew the spill line rather than overflowing the wire.
486
+ expect(body).toContain('more working')
487
+ })
488
+
489
+ it('returns null for an empty worker set', () => {
490
+ expect(renderCombinedWorkerFeed([], { maxRows: 8 })).toBeNull()
491
+ })
492
+ })
@@ -0,0 +1,198 @@
1
+ /**
2
+ * tier-downgrade.ts — MODEL-TIER downgrade failover (second recovery tier).
3
+ *
4
+ * The problem this owns:
5
+ * A user selects a premium model (e.g. `/model fable`). Mid-turn that model
6
+ * is overloaded / throttled fleet-wide (HTTP 529 / 429 / overloaded_error /
7
+ * 503). The FIRST recovery tier — account-level failover (swap the OAuth
8
+ * account, same model) — is tried first and already exists
9
+ * (`runFleetAutoFallback` → `doFireFleetAutoFallback`). When that comes back
10
+ * `all-blocked` (no account still serves the premium model), the live turn
11
+ * would otherwise stall silently. This module adds a SECOND, lower-priority
12
+ * tier: DOWNGRADE the session to the agent's configured default model (the
13
+ * fleet default is `opus`) and resume, so the turn keeps moving.
14
+ *
15
+ * Precedence (HARD): account-swap FIRST. The gateway only consults this module
16
+ * inside the `all-blocked` branch of `doFireFleetAutoFallback` — i.e. AFTER
17
+ * account-swap has been tried and found no eligible account. A `switched`
18
+ * outcome never reaches here (the existing resume-after-swap path handles it),
19
+ * so a premium model that is merely throttled on ONE account is recovered by
20
+ * account-swap, never by a downgrade.
21
+ *
22
+ * There is NO automatic return to the premium model. The `/model` override is
23
+ * SESSION-SCOPED and in-memory only (`recordTypedModelSwitch` writes NO carrier
24
+ * — reference/rfcs/session-model-stickiness.md §0.1 rev 4), so it dies on the
25
+ * downgrade SIGTERM. The downgrade writes a consume-once `.session-model`
26
+ * carrier for the CONFIGURED DEFAULT: start.sh applies+deletes it on the resume
27
+ * boot, and every subsequent restart boots the configured default too. The
28
+ * premium model is never restored on its own — the user must re-issue
29
+ * `/model <premium>` once it frees up. The user-facing notice
30
+ * (`renderTierDowngradeNotice`) says exactly that; it must NOT promise any
31
+ * revert to the premium tier.
32
+ *
33
+ * Effort is NATIVE. A live `/effort` override records in gateway memory only
34
+ * (no carrier), so the self-restart sheds it and the downgraded default boots at
35
+ * the configured `thinking_effort` (the fleet `low` pin, #1978 /
36
+ * src/config/thinking-effort-risk.ts). This module writes NO effort carrier —
37
+ * that is the whole point: the downgraded opus must resolve LOW.
38
+ *
39
+ * Loop guard — the natural on-default guard is the real bound. After the
40
+ * downgrade boot the session runs the configured default (the override is gone
41
+ * with the restart), so a second `all-blocked` finds no premium tier and
42
+ * `decide()` returns `skip` ('on-default') — the turn never re-downgrades. If
43
+ * the DEFAULT is itself walled fleet-wide on the resume boot, `decide()` still
44
+ * returns `skip` and the gateway falls through to its normal all-blocked card;
45
+ * there is no restart loop. Cross-restart pacing is additionally bounded by the
46
+ * broker-side persisted account exhaustion (see fleet-fallback-resume.ts).
47
+ *
48
+ * Concurrent-turn race (single process, pre-restart): if a first turn already
49
+ * armed a resume restart (a downgrade OR an account-swap), a second turn that
50
+ * also hits `all-blocked` must NOT emit a give-up card — the armed restart will
51
+ * replay the latest interrupted turn. `planTierDowngrade` maps a `skip-inflight`
52
+ * resume-gate verdict to `suppress` for exactly this reason (no re-downgrade, no
53
+ * contradictory "could not be recovered" message).
54
+ *
55
+ * This module is pure (`decideTierDowngrade` / `planTierDowngrade` /
56
+ * `renderTierDowngradeNotice`) so the decision, the give-up suppression, and the
57
+ * user-facing wording are all unit-testable without a process restart; the
58
+ * gateway does the FS write, the latch arm, and the restart off these verdicts.
59
+ */
60
+
61
+ export type TierDowngradeDecision =
62
+ | {
63
+ /** Fire the downgrade: write a consume-once carrier for `toModel` and
64
+ * self-restart to resume the dead turn on the configured default. */
65
+ action: 'downgrade'
66
+ toModel: string
67
+ fromModel: string
68
+ }
69
+ | {
70
+ /** No downgrade is warranted. `on-default`: the session is already on the
71
+ * configured default (nothing lower to fall to). `unresolved`: the
72
+ * configured default could not be read (never downgrade blind). */
73
+ action: 'skip'
74
+ reason: 'on-default' | 'unresolved'
75
+ }
76
+
77
+ export interface TierDowngradeInput {
78
+ /** The live session model override (`sessionModelSource.getOverride()`), or
79
+ * null when the session is running the plain configured default. Null is the
80
+ * common "on default" signal (boot seeding leaves it null when the launched
81
+ * model equals the configured default). */
82
+ sessionOverride: string | null
83
+ /** The resolved configured default model token (from
84
+ * `.configured-default-model` / `resolveMainModel`). Empty/undefined => the
85
+ * boot record was unreadable and we must not downgrade blind. */
86
+ configuredDefault: string | null | undefined
87
+ /** Canonicalizer so `sessionOverride` and `configuredDefault` compare in the
88
+ * same dialect (the gateway passes `resolveMainModel`). Keeps `opus` from
89
+ * spuriously reading as a premium tier over a `claude-opus-*`-shaped default
90
+ * and vice versa, as far as the resolver can. */
91
+ resolve: (token: string) => string
92
+ }
93
+
94
+ /**
95
+ * Decide whether — and to what — the walled premium session should downgrade.
96
+ * PURE: no FS, no clock, no restart. The gateway does the I/O and the restart
97
+ * off this verdict.
98
+ */
99
+ export function decideTierDowngrade(input: TierDowngradeInput): TierDowngradeDecision {
100
+ const configured =
101
+ typeof input.configuredDefault === 'string' ? input.configuredDefault.trim() : ''
102
+ if (configured.length === 0) {
103
+ // No configured-default record — never downgrade to an unknown model.
104
+ return { action: 'skip', reason: 'unresolved' }
105
+ }
106
+
107
+ const override = input.sessionOverride
108
+ if (override == null || override.length === 0) {
109
+ // On the configured default already — no lower tier to fall to. This is also
110
+ // the natural loop bound: after the downgrade boot the override is gone, so a
111
+ // re-entry lands here and never re-downgrades.
112
+ return { action: 'skip', reason: 'on-default' }
113
+ }
114
+
115
+ // Canonicalize both sides so an alias vs resolved-id spelling of the SAME
116
+ // model reads as "on default", never as a premium tier (which would loop).
117
+ if (input.resolve(override) === input.resolve(configured)) {
118
+ return { action: 'skip', reason: 'on-default' }
119
+ }
120
+
121
+ // A premium model is active AND walled fleet-wide → downgrade to the default.
122
+ return { action: 'downgrade', toModel: configured, fromModel: override }
123
+ }
124
+
125
+ /**
126
+ * The user-facing broadcast for a fired downgrade. PURE + deterministic so the
127
+ * wording is pinned by a test.
128
+ *
129
+ * HONESTY CONTRACT (do not regress): it states the turn resumes on the DEFAULT
130
+ * and that the user must RE-ISSUE `/model <premium>` to get the premium tier
131
+ * back. It must NOT claim any automatic revert to the premium model — there is
132
+ * none (the override is session-scoped and dies on the downgrade restart).
133
+ */
134
+ export function renderTierDowngradeNotice(
135
+ fromModel: string,
136
+ toModel: string,
137
+ agent: string,
138
+ ): string {
139
+ return (
140
+ `⤵️ **Downgrading model to keep going** on agent **${agent}**\n` +
141
+ `\`${fromModel}\` is overloaded across every account right now, so this turn is ` +
142
+ `resuming on the default \`${toModel}\` to keep working. ` +
143
+ `Re-issue \`/model ${fromModel}\` once it frees up — it won't switch back on its own.`
144
+ )
145
+ }
146
+
147
+ /** The resume-gate verdict the gateway feeds `planTierDowngrade` (mirrors
148
+ * `ResumeDecision` from fleet-fallback-resume.ts, peeked WITHOUT arming). */
149
+ export type ResumeGateVerdict = 'resume' | 'skip-inflight' | 'skip-stale'
150
+
151
+ export type TierDowngradePlan =
152
+ | {
153
+ /** Fire the downgrade: write the carrier, arm the latch, broadcast
154
+ * `notice`, and self-restart. */
155
+ kind: 'downgrade'
156
+ toModel: string
157
+ fromModel: string
158
+ notice: string
159
+ }
160
+ | {
161
+ /** A resume restart is ALREADY armed in this process (a concurrent turn's
162
+ * downgrade or account-swap) — emit NO give-up card; the armed restart
163
+ * replays the interrupted turn. */
164
+ kind: 'suppress'
165
+ }
166
+ | {
167
+ /** No downgrade; caller falls through to its normal all-blocked card. */
168
+ kind: 'skip'
169
+ }
170
+
171
+ /**
172
+ * Pure orchestration off the (pure) decision + the resume-gate verdict. Keeps
173
+ * the concurrent-turn suppression (LOW-9a) and the give-up/fall-through routing
174
+ * testable without a live gateway.
175
+ *
176
+ * - decision `skip` → `skip` (fall through to the all-blocked card)
177
+ * - decision `downgrade`, gate `skip-inflight`
178
+ * → `suppress` (a restart is already armed)
179
+ * - decision `downgrade`, gate `skip-stale`
180
+ * → `skip` (turn too old to resume; give up)
181
+ * - decision `downgrade`, gate `resume`
182
+ * → `downgrade` (carry the honest notice)
183
+ */
184
+ export function planTierDowngrade(
185
+ decision: TierDowngradeDecision,
186
+ gateVerdict: ResumeGateVerdict,
187
+ agent: string,
188
+ ): TierDowngradePlan {
189
+ if (decision.action !== 'downgrade') return { kind: 'skip' }
190
+ if (gateVerdict === 'skip-inflight') return { kind: 'suppress' }
191
+ if (gateVerdict === 'skip-stale') return { kind: 'skip' }
192
+ return {
193
+ kind: 'downgrade',
194
+ toModel: decision.toModel,
195
+ fromModel: decision.fromModel,
196
+ notice: renderTierDowngradeNotice(decision.fromModel, decision.toModel, agent),
197
+ }
198
+ }