switchroom 0.18.17 → 0.18.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +13 -0
- package/dist/auth-broker/index.js +13 -0
- package/dist/cli/notion-write-pretool.mjs +13 -0
- package/dist/cli/switchroom.js +605 -479
- package/dist/host-control/main.js +17 -1
- package/dist/vault/approvals/kernel-server.js +13 -0
- package/dist/vault/broker/server.js +13 -0
- package/package.json +1 -1
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1544 -619
- package/telegram-plugin/dist/server.js +32 -1
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/format.ts +137 -213
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
- package/telegram-plugin/gateway/gateway.ts +399 -85
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/outbound-send-path.ts +9 -7
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +103 -0
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/llm-error-present.ts +474 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/send-gate-degraded.test.ts +152 -1
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +212 -19
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/stream-reply-handler.ts +5 -14
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
- package/telegram-plugin/tests/format-consistency.test.ts +68 -53
- package/telegram-plugin/tests/formatting-parse-regression.test.ts +5 -6
- package/telegram-plugin/tests/formatting-torture-set.ts +1 -1
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +26 -0
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/llm-error-present.test.ts +481 -0
- package/telegram-plugin/tests/outbound-send-path.test.ts +4 -3
- package/telegram-plugin/tests/paragraph-normalizer.test.ts +42 -100
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +9 -12
- package/telegram-plugin/tests/telegram-format.test.ts +86 -31
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +17 -21
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/turn-flush-safety.ts +4 -3
- package/telegram-plugin/worker-activity-feed.ts +509 -409
|
@@ -0,0 +1,492 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest'
|
|
2
|
+
import {
|
|
3
|
+
createWorkerActivityFeed,
|
|
4
|
+
type BotApiForWorkerFeed,
|
|
5
|
+
type WorkerActivityView,
|
|
6
|
+
} from '../worker-activity-feed.js'
|
|
7
|
+
import { renderCombinedWorkerFeed } from '../tool-activity-summary.js'
|
|
8
|
+
import { STATUS_CARD_CHAR_BUDGET } from '../status-no-truncate.js'
|
|
9
|
+
import { createSendGate, isSendGateShed, type Clock } from '../send-gate.js'
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Outcome tests for the coalesced worker-activity feed (#3084 follow-up).
|
|
13
|
+
*
|
|
14
|
+
* These drive the REAL send gate (perChatPerSec=1, burst 3, editFloorMs=1500 —
|
|
15
|
+
* the shipped defaults) with the feed's real send/edit adapters wired exactly
|
|
16
|
+
* as the gateway wires them (`useful` sends, `cosmetic` edits carrying
|
|
17
|
+
* messageId+editPayload). The assertions prove the coalescing win that the
|
|
18
|
+
* per-worker-message model cannot achieve:
|
|
19
|
+
*
|
|
20
|
+
* - N=15 workers in one chat produce exactly ONE combined message (not 15).
|
|
21
|
+
* - Every worker's latest state refreshes TOGETHER in that one message within
|
|
22
|
+
* one ~1.5s edit cycle (the per-worker model refreshes each card only once
|
|
23
|
+
* per ~N·(1/perChatPerSec) seconds — liveness collapse).
|
|
24
|
+
* - The gate SHEDS ~nothing under the coalesced stream (the per-worker model
|
|
25
|
+
* sheds ~14/15 of every second's cosmetic edits — a real ban happened).
|
|
26
|
+
* - A `critical` reply mid-storm is admitted, never shed/starved.
|
|
27
|
+
* - Redundant identical renders don't produce redundant landed edits.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Deterministic fake clock shared by BOTH the send gate (Clock.now/sleep) and
|
|
32
|
+
* the feed (`now: () => clock.now()`). Mirrors send-gate.test.ts's FakeClock.
|
|
33
|
+
*/
|
|
34
|
+
class FakeClock implements Clock {
|
|
35
|
+
private cur = 0
|
|
36
|
+
private seq = 0
|
|
37
|
+
private timers: { at: number; id: number; resolve: () => void }[] = []
|
|
38
|
+
now(): number {
|
|
39
|
+
return this.cur
|
|
40
|
+
}
|
|
41
|
+
sleep(ms: number): Promise<void> {
|
|
42
|
+
return new Promise<void>((resolve) => {
|
|
43
|
+
this.timers.push({ at: this.cur + ms, id: this.seq++, resolve })
|
|
44
|
+
})
|
|
45
|
+
}
|
|
46
|
+
async advance(ms: number): Promise<void> {
|
|
47
|
+
const target = this.cur + ms
|
|
48
|
+
for (;;) {
|
|
49
|
+
const due = this.timers.filter((t) => t.at <= target).sort((a, b) => a.at - b.at || a.id - b.id)
|
|
50
|
+
if (due.length === 0) break
|
|
51
|
+
const t = due[0]
|
|
52
|
+
this.timers = this.timers.filter((x) => x !== t)
|
|
53
|
+
this.cur = t.at
|
|
54
|
+
t.resolve()
|
|
55
|
+
await flush()
|
|
56
|
+
}
|
|
57
|
+
this.cur = target
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
function flush(): Promise<void> {
|
|
61
|
+
return new Promise((r) => setImmediate(r))
|
|
62
|
+
}
|
|
63
|
+
/** Flush the promise-chain a few times so feed chains + gate drivers settle. */
|
|
64
|
+
async function settle(clock: FakeClock, ms = 0): Promise<void> {
|
|
65
|
+
await clock.advance(ms)
|
|
66
|
+
for (let i = 0; i < 6; i++) await flush()
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
interface LandedEdit {
|
|
70
|
+
chatId: string
|
|
71
|
+
messageId: number
|
|
72
|
+
text: string
|
|
73
|
+
at: number
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Build the feed + a bot adapter routed through a REAL send gate, exactly like
|
|
78
|
+
* the gateway (worker-feed sends are `useful`, edits are `cosmetic` carrying
|
|
79
|
+
* messageId + editPayload for the gate's floor/coalesce/no-op logic).
|
|
80
|
+
*/
|
|
81
|
+
function harness(clock: FakeClock, opts: { maxRows?: number } = {}) {
|
|
82
|
+
const gate = createSendGate({
|
|
83
|
+
enabled: true,
|
|
84
|
+
clock,
|
|
85
|
+
globalPerSec: 1000, // global is not the limiter under test
|
|
86
|
+
globalBurst: 1000,
|
|
87
|
+
perChatPerSec: 1,
|
|
88
|
+
perChatBurst: 3,
|
|
89
|
+
editFloorMs: 1500,
|
|
90
|
+
})
|
|
91
|
+
const sends: { chatId: string; messageId: number; at: number }[] = []
|
|
92
|
+
const edits: LandedEdit[] = []
|
|
93
|
+
let seq = 1000
|
|
94
|
+
const bot: BotApiForWorkerFeed = {
|
|
95
|
+
sendMessage: (chatId, _text, o) =>
|
|
96
|
+
gate.gate(
|
|
97
|
+
async () => {
|
|
98
|
+
const messageId = seq++
|
|
99
|
+
sends.push({ chatId, messageId, at: clock.now() })
|
|
100
|
+
return { message_id: messageId }
|
|
101
|
+
},
|
|
102
|
+
{ chat_id: chatId, priorityClass: 'useful', verb: 'worker-feed' },
|
|
103
|
+
) as Promise<{ message_id: number }>,
|
|
104
|
+
editMessageText: (chatId, messageId, text, o) =>
|
|
105
|
+
gate.gate(
|
|
106
|
+
async () => {
|
|
107
|
+
edits.push({ chatId, messageId, text, at: clock.now() })
|
|
108
|
+
return true
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
chat_id: chatId,
|
|
112
|
+
priorityClass: 'cosmetic',
|
|
113
|
+
messageId,
|
|
114
|
+
editPayload: text,
|
|
115
|
+
verb: 'worker-feed',
|
|
116
|
+
},
|
|
117
|
+
),
|
|
118
|
+
}
|
|
119
|
+
const feed = createWorkerActivityFeed({
|
|
120
|
+
bot,
|
|
121
|
+
now: () => clock.now(),
|
|
122
|
+
// Let the send gate's editFloorMs (1500) be the pacing authority: the feed's
|
|
123
|
+
// own proactive throttle matches it so the coalesced stream never outruns
|
|
124
|
+
// the per-chat bucket (which is what would shed it).
|
|
125
|
+
minEditIntervalMs: 1500,
|
|
126
|
+
heartbeatTickMs: 1500,
|
|
127
|
+
firstPaintMinMs: 0,
|
|
128
|
+
// No auto-timer: the test drives heartbeatTick() deterministically.
|
|
129
|
+
setInterval: () => 0,
|
|
130
|
+
clearInterval: () => {},
|
|
131
|
+
maxRows: opts.maxRows ?? 8,
|
|
132
|
+
})
|
|
133
|
+
return { gate, feed, sends, edits }
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function view(desc: string, step: string, elapsedMs: number): WorkerActivityView {
|
|
137
|
+
return {
|
|
138
|
+
description: desc,
|
|
139
|
+
lastTool: null,
|
|
140
|
+
toolCount: 3,
|
|
141
|
+
latestSummary: step,
|
|
142
|
+
elapsedMs,
|
|
143
|
+
state: 'running',
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
describe('coalesced worker feed — one message per chat under load', () => {
|
|
148
|
+
it('N=15 workers over 60s produce ONE message with a bounded, non-shedding edit stream', async () => {
|
|
149
|
+
const clock = new FakeClock()
|
|
150
|
+
const CHAT = 'chat-storm'
|
|
151
|
+
const N = 15
|
|
152
|
+
const { gate, feed, sends, edits } = harness(clock, { maxRows: N })
|
|
153
|
+
const ids = Array.from({ length: N }, (_, i) => `w${i}`)
|
|
154
|
+
|
|
155
|
+
// Latest step each worker last emitted (unique per cycle so we can prove
|
|
156
|
+
// freshness in the rendered body).
|
|
157
|
+
const latest: Record<string, string> = {}
|
|
158
|
+
|
|
159
|
+
const CYCLE = 1500
|
|
160
|
+
const CYCLES = 40 // 40 * 1.5s = 60s
|
|
161
|
+
for (let c = 0; c < CYCLES; c++) {
|
|
162
|
+
await clock.advance(CYCLE)
|
|
163
|
+
for (const id of ids) {
|
|
164
|
+
latest[id] = `${id}-step-${c}`
|
|
165
|
+
void feed.update(id, CHAT, view(`task ${id}`, latest[id], clock.now()))
|
|
166
|
+
}
|
|
167
|
+
await settle(clock)
|
|
168
|
+
feed.heartbeatTick() // flush the accumulated combined render
|
|
169
|
+
await settle(clock)
|
|
170
|
+
}
|
|
171
|
+
await settle(clock, 3000)
|
|
172
|
+
|
|
173
|
+
// ONE combined message — not one per worker. This is the structural
|
|
174
|
+
// distinction from the per-worker-message model (which sends N).
|
|
175
|
+
expect(sends.length).toBe(1)
|
|
176
|
+
expect(feed.size).toBe(N)
|
|
177
|
+
|
|
178
|
+
// Bounded edit stream: at most perChatPerSec·T + burst landed edits.
|
|
179
|
+
const seconds = (CYCLES * CYCLE) / 1000
|
|
180
|
+
const bound = Math.ceil(1 * seconds + 3)
|
|
181
|
+
expect(edits.length).toBeLessThanOrEqual(bound)
|
|
182
|
+
// And it actually stayed live (edited many times, not frozen).
|
|
183
|
+
expect(edits.length).toBeGreaterThan(CYCLES / 3)
|
|
184
|
+
|
|
185
|
+
// Shed pin: the coalesced single stream sheds ~nothing. The per-worker model
|
|
186
|
+
// would shed ~ (N-1)/N of every second's cosmetic edits.
|
|
187
|
+
expect(gate.stats().global.shed).toBeLessThanOrEqual(3)
|
|
188
|
+
|
|
189
|
+
// Liveness: the LAST landed edit carries EVERY worker's latest step — all
|
|
190
|
+
// rows refreshed together in the one message within one edit cycle.
|
|
191
|
+
const lastBody = edits[edits.length - 1].text
|
|
192
|
+
for (const id of ids) {
|
|
193
|
+
expect(lastBody).toContain(latest[id])
|
|
194
|
+
}
|
|
195
|
+
})
|
|
196
|
+
|
|
197
|
+
it('admits a critical reply mid-storm (never shed or starved by the cosmetic feed)', async () => {
|
|
198
|
+
const clock = new FakeClock()
|
|
199
|
+
const CHAT = 'chat-crit'
|
|
200
|
+
const { gate, feed } = harness(clock, { maxRows: 15 })
|
|
201
|
+
const ids = Array.from({ length: 12 }, (_, i) => `k${i}`)
|
|
202
|
+
|
|
203
|
+
// Warm the storm.
|
|
204
|
+
for (let c = 0; c < 4; c++) {
|
|
205
|
+
await clock.advance(1500)
|
|
206
|
+
for (const id of ids) void feed.update(id, CHAT, view(`task ${id}`, `${id}-s${c}`, clock.now()))
|
|
207
|
+
await settle(clock)
|
|
208
|
+
feed.heartbeatTick()
|
|
209
|
+
await settle(clock)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// A critical reply to the SAME chat, mid-storm. `critical` is never shed and
|
|
213
|
+
// never blocks behind the cosmetic feed edits.
|
|
214
|
+
const CRIT = { ok: true, id: 'crit-reply' }
|
|
215
|
+
const p = gate.gate(async () => CRIT, {
|
|
216
|
+
chat_id: CHAT,
|
|
217
|
+
priorityClass: 'critical',
|
|
218
|
+
verb: 'reply',
|
|
219
|
+
})
|
|
220
|
+
await settle(clock, 2000)
|
|
221
|
+
const res = await p
|
|
222
|
+
expect(isSendGateShed(res)).toBe(false)
|
|
223
|
+
expect(res).toEqual(CRIT)
|
|
224
|
+
})
|
|
225
|
+
|
|
226
|
+
it('does not produce redundant landed edits for redundant renders (coalesce/no-op)', async () => {
|
|
227
|
+
const clock = new FakeClock()
|
|
228
|
+
const CHAT = 'chat-dedupe'
|
|
229
|
+
const { gate, feed, edits } = harness(clock)
|
|
230
|
+
|
|
231
|
+
// First paint + one edit with a real state change.
|
|
232
|
+
await clock.advance(1500)
|
|
233
|
+
void feed.update('w', CHAT, view('task w', 'step-A', clock.now()))
|
|
234
|
+
await settle(clock)
|
|
235
|
+
feed.heartbeatTick()
|
|
236
|
+
await settle(clock, 2000)
|
|
237
|
+
const afterFirst = edits.length
|
|
238
|
+
|
|
239
|
+
// Re-drive the SAME state many times over several cycles. No new step, no new
|
|
240
|
+
// state → the feed dedups (body === lastBody) and/or the gate drops the no-op;
|
|
241
|
+
// either way, NO new landed edit and NO shed.
|
|
242
|
+
const shedBefore = gate.stats().global.shed
|
|
243
|
+
for (let c = 0; c < 5; c++) {
|
|
244
|
+
await clock.advance(1500)
|
|
245
|
+
void feed.update('w', CHAT, view('task w', 'step-A', clock.now()))
|
|
246
|
+
await settle(clock)
|
|
247
|
+
feed.heartbeatTick()
|
|
248
|
+
await settle(clock, 2000)
|
|
249
|
+
}
|
|
250
|
+
// The header elapsed climbs on heartbeat, so a redundant *content* update
|
|
251
|
+
// must not multiply landed edits beyond the heartbeat's own climb cadence.
|
|
252
|
+
expect(edits.length - afterFirst).toBeLessThanOrEqual(5)
|
|
253
|
+
// Crucially: redundant identical narrative never sheds.
|
|
254
|
+
expect(gate.stats().global.shed).toBe(shedBefore)
|
|
255
|
+
})
|
|
256
|
+
})
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Deterministic lifecycle transitions with a DIRECT fake bot (no gate) so we
|
|
260
|
+
* can assert the exact rendered body at each step.
|
|
261
|
+
*/
|
|
262
|
+
function directBot() {
|
|
263
|
+
const sent: { chatId: string; text: string; messageId: number }[] = []
|
|
264
|
+
const edits: { messageId: number; text: string }[] = []
|
|
265
|
+
let seq = 500
|
|
266
|
+
const bot: BotApiForWorkerFeed = {
|
|
267
|
+
sendMessage: async (chatId, text) => {
|
|
268
|
+
const messageId = seq++
|
|
269
|
+
sent.push({ chatId, text, messageId })
|
|
270
|
+
return { message_id: messageId }
|
|
271
|
+
},
|
|
272
|
+
editMessageText: async (_chatId, messageId, text) => {
|
|
273
|
+
edits.push({ messageId, text })
|
|
274
|
+
return true
|
|
275
|
+
},
|
|
276
|
+
}
|
|
277
|
+
return { bot, sent, edits }
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
describe('coalesced worker feed — lifecycle transitions', () => {
|
|
281
|
+
it('two workers in one chat share ONE message that becomes a combined body', async () => {
|
|
282
|
+
let clock = 0
|
|
283
|
+
const { bot, sent } = directBot()
|
|
284
|
+
const feed = createWorkerActivityFeed({
|
|
285
|
+
bot,
|
|
286
|
+
now: () => clock,
|
|
287
|
+
minEditIntervalMs: 0,
|
|
288
|
+
firstPaintMinMs: 0,
|
|
289
|
+
setInterval: () => 0,
|
|
290
|
+
clearInterval: () => {},
|
|
291
|
+
})
|
|
292
|
+
clock = 1000
|
|
293
|
+
await feed.update('a', 'chat', view('task A', 'a-doing', 1000))
|
|
294
|
+
clock = 1100
|
|
295
|
+
await feed.update('b', 'chat', view('task B', 'b-doing', 1100))
|
|
296
|
+
|
|
297
|
+
// ONE message for the chat (both workers share it), and its id is the same.
|
|
298
|
+
expect(sent.length).toBe(1)
|
|
299
|
+
expect(feed.messageIdOf('a')).toBe(feed.messageIdOf('b'))
|
|
300
|
+
expect(feed.size).toBe(2)
|
|
301
|
+
})
|
|
302
|
+
|
|
303
|
+
it('finishing one of two drops its row (result NOT in the feed edit); the survivor stays live', async () => {
|
|
304
|
+
let clock = 0
|
|
305
|
+
const { bot, edits } = directBot()
|
|
306
|
+
const feed = createWorkerActivityFeed({
|
|
307
|
+
bot,
|
|
308
|
+
now: () => clock,
|
|
309
|
+
minEditIntervalMs: 0,
|
|
310
|
+
firstPaintMinMs: 0,
|
|
311
|
+
setInterval: () => 0,
|
|
312
|
+
clearInterval: () => {},
|
|
313
|
+
})
|
|
314
|
+
clock = 1000
|
|
315
|
+
await feed.update('a', 'chat', view('task A', 'a-doing', 1000))
|
|
316
|
+
await feed.update('b', 'chat', view('task B', 'b-doing', 1000))
|
|
317
|
+
clock = 2000
|
|
318
|
+
await feed.finish('a', {
|
|
319
|
+
description: 'task A',
|
|
320
|
+
lastTool: null,
|
|
321
|
+
toolCount: 3,
|
|
322
|
+
latestSummary: 'SECRET-RESULT-A should reach the user via handback only',
|
|
323
|
+
elapsedMs: 2000,
|
|
324
|
+
state: 'done',
|
|
325
|
+
})
|
|
326
|
+
|
|
327
|
+
const last = edits[edits.length - 1].text
|
|
328
|
+
// The finished worker's RESULT is never folded into the cosmetic feed edit.
|
|
329
|
+
expect(last).not.toContain('SECRET-RESULT-A')
|
|
330
|
+
// The survivor's live step is still shown.
|
|
331
|
+
expect(last).toContain('b-doing')
|
|
332
|
+
expect(feed.size).toBe(1)
|
|
333
|
+
expect(feed.has('a')).toBe(false)
|
|
334
|
+
expect(feed.has('b')).toBe(true)
|
|
335
|
+
})
|
|
336
|
+
|
|
337
|
+
it('finishing the LAST worker finalizes the shared message to its terminal recap', async () => {
|
|
338
|
+
let clock = 0
|
|
339
|
+
const { bot, edits } = directBot()
|
|
340
|
+
const feed = createWorkerActivityFeed({
|
|
341
|
+
bot,
|
|
342
|
+
now: () => clock,
|
|
343
|
+
minEditIntervalMs: 0,
|
|
344
|
+
firstPaintMinMs: 0,
|
|
345
|
+
setInterval: () => 0,
|
|
346
|
+
clearInterval: () => {},
|
|
347
|
+
})
|
|
348
|
+
clock = 1000
|
|
349
|
+
await feed.update('solo', 'chat', view('task solo', 'working', 1000))
|
|
350
|
+
clock = 2000
|
|
351
|
+
await feed.finish('solo', {
|
|
352
|
+
description: 'task solo',
|
|
353
|
+
lastTool: null,
|
|
354
|
+
toolCount: 5,
|
|
355
|
+
latestSummary: 'the final recap paragraph',
|
|
356
|
+
elapsedMs: 2000,
|
|
357
|
+
state: 'done',
|
|
358
|
+
})
|
|
359
|
+
const last = edits[edits.length - 1].text
|
|
360
|
+
// Terminal recap: the single 🛠 Worker card carries the done state + result.
|
|
361
|
+
expect(last).toContain('the final recap paragraph')
|
|
362
|
+
expect(feed.size).toBe(0)
|
|
363
|
+
// The finalized worker is gated against a late resurrecting cue.
|
|
364
|
+
await feed.update('solo', 'chat', view('task solo', 'late tick', 3000))
|
|
365
|
+
expect(feed.size).toBe(0)
|
|
366
|
+
})
|
|
367
|
+
})
|
|
368
|
+
|
|
369
|
+
describe('coalesced worker feed — GROUP-level pin lifecycle (#3207 review)', () => {
|
|
370
|
+
interface PinCall {
|
|
371
|
+
feedKey: string
|
|
372
|
+
chatId: string
|
|
373
|
+
messageId: number | null
|
|
374
|
+
}
|
|
375
|
+
function pinHarness() {
|
|
376
|
+
const pins: PinCall[] = []
|
|
377
|
+
const { bot } = directBot()
|
|
378
|
+
let clock = 0
|
|
379
|
+
const feed = createWorkerActivityFeed({
|
|
380
|
+
bot,
|
|
381
|
+
now: () => clock,
|
|
382
|
+
minEditIntervalMs: 0,
|
|
383
|
+
firstPaintMinMs: 0,
|
|
384
|
+
setInterval: () => 0,
|
|
385
|
+
clearInterval: () => {},
|
|
386
|
+
reconcilePin: ({ feedKey, chatId, messageId }) => pins.push({ feedKey, chatId, messageId }),
|
|
387
|
+
})
|
|
388
|
+
return { feed, pins, setClock: (t: number) => (clock = t) }
|
|
389
|
+
}
|
|
390
|
+
const done = (desc: string, elapsedMs: number): WorkerActivityView => ({
|
|
391
|
+
description: desc,
|
|
392
|
+
lastTool: null,
|
|
393
|
+
toolCount: 3,
|
|
394
|
+
latestSummary: `${desc} result`,
|
|
395
|
+
elapsedMs,
|
|
396
|
+
state: 'done',
|
|
397
|
+
})
|
|
398
|
+
|
|
399
|
+
it('a mid-group sibling finish does NOT unpin the shared message while a worker still runs', async () => {
|
|
400
|
+
const { feed, pins, setClock } = pinHarness()
|
|
401
|
+
setClock(1000)
|
|
402
|
+
await feed.update('a', 'chat', view('task A', 'a-doing', 1000))
|
|
403
|
+
await feed.update('b', 'chat', view('task B', 'b-doing', 1000))
|
|
404
|
+
|
|
405
|
+
// The group's shared message is pinned (messageId non-null).
|
|
406
|
+
const pinnedAfterPaint = pins.filter((p) => p.messageId != null)
|
|
407
|
+
expect(pinnedAfterPaint.length).toBeGreaterThan(0)
|
|
408
|
+
const sharedMsgId = pinnedAfterPaint[pinnedAfterPaint.length - 1].messageId
|
|
409
|
+
expect(feed.messageIdOf('a')).toBe(sharedMsgId)
|
|
410
|
+
expect(feed.messageIdOf('b')).toBe(sharedMsgId)
|
|
411
|
+
|
|
412
|
+
// A finishes while B still runs — the pin MUST stay (B needs the message).
|
|
413
|
+
setClock(2000)
|
|
414
|
+
await feed.finish('a', done('task A', 2000))
|
|
415
|
+
|
|
416
|
+
// Every reconcilePin emitted through A's finish keeps the message pinned —
|
|
417
|
+
// there is NO unpin (messageId === null) while B is live. This is the exact
|
|
418
|
+
// regression the review flagged: a per-worker unpin here strands B unpinned.
|
|
419
|
+
const last = pins[pins.length - 1]
|
|
420
|
+
expect(last.messageId).toBe(sharedMsgId)
|
|
421
|
+
expect(pins.some((p) => p.messageId === null)).toBe(false)
|
|
422
|
+
// The feed still vouches for the group (reaper exemption stays live).
|
|
423
|
+
const feedKey = last.feedKey
|
|
424
|
+
expect(feed.hasRunningInFeed(feedKey)).toBe(true)
|
|
425
|
+
})
|
|
426
|
+
|
|
427
|
+
it('unpins the shared message only when the LAST worker finishes (group empties)', async () => {
|
|
428
|
+
const { feed, pins, setClock } = pinHarness()
|
|
429
|
+
setClock(1000)
|
|
430
|
+
await feed.update('a', 'chat', view('task A', 'a-doing', 1000))
|
|
431
|
+
await feed.update('b', 'chat', view('task B', 'b-doing', 1000))
|
|
432
|
+
setClock(2000)
|
|
433
|
+
await feed.finish('a', done('task A', 2000))
|
|
434
|
+
expect(pins.some((p) => p.messageId === null)).toBe(false) // B still runs
|
|
435
|
+
|
|
436
|
+
// Now the last worker finishes → the group empties → unpin fires.
|
|
437
|
+
setClock(3000)
|
|
438
|
+
await feed.finish('b', done('task B', 3000))
|
|
439
|
+
const last = pins[pins.length - 1]
|
|
440
|
+
expect(last.messageId).toBeNull()
|
|
441
|
+
const feedKey = last.feedKey
|
|
442
|
+
expect(feed.hasRunningInFeed(feedKey)).toBe(false) // reaper may now reap it
|
|
443
|
+
})
|
|
444
|
+
})
|
|
445
|
+
|
|
446
|
+
describe('renderCombinedWorkerFeed (pure)', () => {
|
|
447
|
+
const row = (i: number, step: string) => ({
|
|
448
|
+
description: `task number ${i}`,
|
|
449
|
+
elapsedMs: 12_000 + i * 1000,
|
|
450
|
+
toolCount: i,
|
|
451
|
+
currentStep: step,
|
|
452
|
+
})
|
|
453
|
+
|
|
454
|
+
it('renders one row-block per worker with a running count header', () => {
|
|
455
|
+
const body = renderCombinedWorkerFeed([row(1, 'alpha step'), row(2, 'beta step')], { maxRows: 8 })!
|
|
456
|
+
expect(body).toContain('Workers')
|
|
457
|
+
expect(body).toContain('2 running')
|
|
458
|
+
expect(body).toContain('task number 1')
|
|
459
|
+
expect(body).toContain('task number 2')
|
|
460
|
+
expect(body).toContain('alpha step')
|
|
461
|
+
expect(body).toContain('beta step')
|
|
462
|
+
})
|
|
463
|
+
|
|
464
|
+
it('caps at maxRows and spills the remainder to a compact +M more working line', () => {
|
|
465
|
+
const rows = Array.from({ length: 15 }, (_, i) => row(i, `s${i}`))
|
|
466
|
+
const body = renderCombinedWorkerFeed(rows, { maxRows: 8 })!
|
|
467
|
+
expect(body).toContain('15 running')
|
|
468
|
+
expect(body).toContain('+7 more working')
|
|
469
|
+
// The 8th row is shown, the 9th is spilled.
|
|
470
|
+
expect(body).toContain('s7')
|
|
471
|
+
expect(body).not.toContain('s8')
|
|
472
|
+
})
|
|
473
|
+
|
|
474
|
+
it('stays under the rich-message wire budget even with many long rows (backstop drops rows)', () => {
|
|
475
|
+
// 120 rows of ~700 raw chars each would be ~84k > the 32768 rich-message
|
|
476
|
+
// cap; the char-budget backstop must shrink the visible set until it fits.
|
|
477
|
+
const rows = Array.from({ length: 120 }, () => ({
|
|
478
|
+
description: 'x'.repeat(300),
|
|
479
|
+
elapsedMs: 60_000,
|
|
480
|
+
toolCount: 40,
|
|
481
|
+
currentStep: 'y'.repeat(400),
|
|
482
|
+
}))
|
|
483
|
+
const body = renderCombinedWorkerFeed(rows, { maxRows: 120 })!
|
|
484
|
+
expect(body.length).toBeLessThanOrEqual(STATUS_CARD_CHAR_BUDGET)
|
|
485
|
+
// The backstop grew the spill line rather than overflowing the wire.
|
|
486
|
+
expect(body).toContain('more working')
|
|
487
|
+
})
|
|
488
|
+
|
|
489
|
+
it('returns null for an empty worker set', () => {
|
|
490
|
+
expect(renderCombinedWorkerFeed([], { maxRows: 8 })).toBeNull()
|
|
491
|
+
})
|
|
492
|
+
})
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tier-downgrade.ts — MODEL-TIER downgrade failover (second recovery tier).
|
|
3
|
+
*
|
|
4
|
+
* The problem this owns:
|
|
5
|
+
* A user selects a premium model (e.g. `/model fable`). Mid-turn that model
|
|
6
|
+
* is overloaded / throttled fleet-wide (HTTP 529 / 429 / overloaded_error /
|
|
7
|
+
* 503). The FIRST recovery tier — account-level failover (swap the OAuth
|
|
8
|
+
* account, same model) — is tried first and already exists
|
|
9
|
+
* (`runFleetAutoFallback` → `doFireFleetAutoFallback`). When that comes back
|
|
10
|
+
* `all-blocked` (no account still serves the premium model), the live turn
|
|
11
|
+
* would otherwise stall silently. This module adds a SECOND, lower-priority
|
|
12
|
+
* tier: DOWNGRADE the session to the agent's configured default model (the
|
|
13
|
+
* fleet default is `opus`) and resume, so the turn keeps moving.
|
|
14
|
+
*
|
|
15
|
+
* Precedence (HARD): account-swap FIRST. The gateway only consults this module
|
|
16
|
+
* inside the `all-blocked` branch of `doFireFleetAutoFallback` — i.e. AFTER
|
|
17
|
+
* account-swap has been tried and found no eligible account. A `switched`
|
|
18
|
+
* outcome never reaches here (the existing resume-after-swap path handles it),
|
|
19
|
+
* so a premium model that is merely throttled on ONE account is recovered by
|
|
20
|
+
* account-swap, never by a downgrade.
|
|
21
|
+
*
|
|
22
|
+
* There is NO automatic return to the premium model. The `/model` override is
|
|
23
|
+
* SESSION-SCOPED and in-memory only (`recordTypedModelSwitch` writes NO carrier
|
|
24
|
+
* — reference/rfcs/session-model-stickiness.md §0.1 rev 4), so it dies on the
|
|
25
|
+
* downgrade SIGTERM. The downgrade writes a consume-once `.session-model`
|
|
26
|
+
* carrier for the CONFIGURED DEFAULT: start.sh applies+deletes it on the resume
|
|
27
|
+
* boot, and every subsequent restart boots the configured default too. The
|
|
28
|
+
* premium model is never restored on its own — the user must re-issue
|
|
29
|
+
* `/model <premium>` once it frees up. The user-facing notice
|
|
30
|
+
* (`renderTierDowngradeNotice`) says exactly that; it must NOT promise any
|
|
31
|
+
* revert to the premium tier.
|
|
32
|
+
*
|
|
33
|
+
* Effort is NATIVE. A live `/effort` override records in gateway memory only
|
|
34
|
+
* (no carrier), so the self-restart sheds it and the downgraded default boots at
|
|
35
|
+
* the configured `thinking_effort` (the fleet `low` pin, #1978 /
|
|
36
|
+
* src/config/thinking-effort-risk.ts). This module writes NO effort carrier —
|
|
37
|
+
* that is the whole point: the downgraded opus must resolve LOW.
|
|
38
|
+
*
|
|
39
|
+
* Loop guard — the natural on-default guard is the real bound. After the
|
|
40
|
+
* downgrade boot the session runs the configured default (the override is gone
|
|
41
|
+
* with the restart), so a second `all-blocked` finds no premium tier and
|
|
42
|
+
* `decide()` returns `skip` ('on-default') — the turn never re-downgrades. If
|
|
43
|
+
* the DEFAULT is itself walled fleet-wide on the resume boot, `decide()` still
|
|
44
|
+
* returns `skip` and the gateway falls through to its normal all-blocked card;
|
|
45
|
+
* there is no restart loop. Cross-restart pacing is additionally bounded by the
|
|
46
|
+
* broker-side persisted account exhaustion (see fleet-fallback-resume.ts).
|
|
47
|
+
*
|
|
48
|
+
* Concurrent-turn race (single process, pre-restart): if a first turn already
|
|
49
|
+
* armed a resume restart (a downgrade OR an account-swap), a second turn that
|
|
50
|
+
* also hits `all-blocked` must NOT emit a give-up card — the armed restart will
|
|
51
|
+
* replay the latest interrupted turn. `planTierDowngrade` maps a `skip-inflight`
|
|
52
|
+
* resume-gate verdict to `suppress` for exactly this reason (no re-downgrade, no
|
|
53
|
+
* contradictory "could not be recovered" message).
|
|
54
|
+
*
|
|
55
|
+
* This module is pure (`decideTierDowngrade` / `planTierDowngrade` /
|
|
56
|
+
* `renderTierDowngradeNotice`) so the decision, the give-up suppression, and the
|
|
57
|
+
* user-facing wording are all unit-testable without a process restart; the
|
|
58
|
+
* gateway does the FS write, the latch arm, and the restart off these verdicts.
|
|
59
|
+
*/
|
|
60
|
+
|
|
61
|
+
export type TierDowngradeDecision =
|
|
62
|
+
| {
|
|
63
|
+
/** Fire the downgrade: write a consume-once carrier for `toModel` and
|
|
64
|
+
* self-restart to resume the dead turn on the configured default. */
|
|
65
|
+
action: 'downgrade'
|
|
66
|
+
toModel: string
|
|
67
|
+
fromModel: string
|
|
68
|
+
}
|
|
69
|
+
| {
|
|
70
|
+
/** No downgrade is warranted. `on-default`: the session is already on the
|
|
71
|
+
* configured default (nothing lower to fall to). `unresolved`: the
|
|
72
|
+
* configured default could not be read (never downgrade blind). */
|
|
73
|
+
action: 'skip'
|
|
74
|
+
reason: 'on-default' | 'unresolved'
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export interface TierDowngradeInput {
|
|
78
|
+
/** The live session model override (`sessionModelSource.getOverride()`), or
|
|
79
|
+
* null when the session is running the plain configured default. Null is the
|
|
80
|
+
* common "on default" signal (boot seeding leaves it null when the launched
|
|
81
|
+
* model equals the configured default). */
|
|
82
|
+
sessionOverride: string | null
|
|
83
|
+
/** The resolved configured default model token (from
|
|
84
|
+
* `.configured-default-model` / `resolveMainModel`). Empty/undefined => the
|
|
85
|
+
* boot record was unreadable and we must not downgrade blind. */
|
|
86
|
+
configuredDefault: string | null | undefined
|
|
87
|
+
/** Canonicalizer so `sessionOverride` and `configuredDefault` compare in the
|
|
88
|
+
* same dialect (the gateway passes `resolveMainModel`). Keeps `opus` from
|
|
89
|
+
* spuriously reading as a premium tier over a `claude-opus-*`-shaped default
|
|
90
|
+
* and vice versa, as far as the resolver can. */
|
|
91
|
+
resolve: (token: string) => string
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Decide whether — and to what — the walled premium session should downgrade.
|
|
96
|
+
* PURE: no FS, no clock, no restart. The gateway does the I/O and the restart
|
|
97
|
+
* off this verdict.
|
|
98
|
+
*/
|
|
99
|
+
export function decideTierDowngrade(input: TierDowngradeInput): TierDowngradeDecision {
|
|
100
|
+
const configured =
|
|
101
|
+
typeof input.configuredDefault === 'string' ? input.configuredDefault.trim() : ''
|
|
102
|
+
if (configured.length === 0) {
|
|
103
|
+
// No configured-default record — never downgrade to an unknown model.
|
|
104
|
+
return { action: 'skip', reason: 'unresolved' }
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const override = input.sessionOverride
|
|
108
|
+
if (override == null || override.length === 0) {
|
|
109
|
+
// On the configured default already — no lower tier to fall to. This is also
|
|
110
|
+
// the natural loop bound: after the downgrade boot the override is gone, so a
|
|
111
|
+
// re-entry lands here and never re-downgrades.
|
|
112
|
+
return { action: 'skip', reason: 'on-default' }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Canonicalize both sides so an alias vs resolved-id spelling of the SAME
|
|
116
|
+
// model reads as "on default", never as a premium tier (which would loop).
|
|
117
|
+
if (input.resolve(override) === input.resolve(configured)) {
|
|
118
|
+
return { action: 'skip', reason: 'on-default' }
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// A premium model is active AND walled fleet-wide → downgrade to the default.
|
|
122
|
+
return { action: 'downgrade', toModel: configured, fromModel: override }
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* The user-facing broadcast for a fired downgrade. PURE + deterministic so the
|
|
127
|
+
* wording is pinned by a test.
|
|
128
|
+
*
|
|
129
|
+
* HONESTY CONTRACT (do not regress): it states the turn resumes on the DEFAULT
|
|
130
|
+
* and that the user must RE-ISSUE `/model <premium>` to get the premium tier
|
|
131
|
+
* back. It must NOT claim any automatic revert to the premium model — there is
|
|
132
|
+
* none (the override is session-scoped and dies on the downgrade restart).
|
|
133
|
+
*/
|
|
134
|
+
export function renderTierDowngradeNotice(
|
|
135
|
+
fromModel: string,
|
|
136
|
+
toModel: string,
|
|
137
|
+
agent: string,
|
|
138
|
+
): string {
|
|
139
|
+
return (
|
|
140
|
+
`⤵️ **Downgrading model to keep going** on agent **${agent}**\n` +
|
|
141
|
+
`\`${fromModel}\` is overloaded across every account right now, so this turn is ` +
|
|
142
|
+
`resuming on the default \`${toModel}\` to keep working. ` +
|
|
143
|
+
`Re-issue \`/model ${fromModel}\` once it frees up — it won't switch back on its own.`
|
|
144
|
+
)
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** The resume-gate verdict the gateway feeds `planTierDowngrade` (mirrors
|
|
148
|
+
* `ResumeDecision` from fleet-fallback-resume.ts, peeked WITHOUT arming). */
|
|
149
|
+
export type ResumeGateVerdict = 'resume' | 'skip-inflight' | 'skip-stale'
|
|
150
|
+
|
|
151
|
+
export type TierDowngradePlan =
|
|
152
|
+
| {
|
|
153
|
+
/** Fire the downgrade: write the carrier, arm the latch, broadcast
|
|
154
|
+
* `notice`, and self-restart. */
|
|
155
|
+
kind: 'downgrade'
|
|
156
|
+
toModel: string
|
|
157
|
+
fromModel: string
|
|
158
|
+
notice: string
|
|
159
|
+
}
|
|
160
|
+
| {
|
|
161
|
+
/** A resume restart is ALREADY armed in this process (a concurrent turn's
|
|
162
|
+
* downgrade or account-swap) — emit NO give-up card; the armed restart
|
|
163
|
+
* replays the interrupted turn. */
|
|
164
|
+
kind: 'suppress'
|
|
165
|
+
}
|
|
166
|
+
| {
|
|
167
|
+
/** No downgrade; caller falls through to its normal all-blocked card. */
|
|
168
|
+
kind: 'skip'
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Pure orchestration off the (pure) decision + the resume-gate verdict. Keeps
|
|
173
|
+
* the concurrent-turn suppression (LOW-9a) and the give-up/fall-through routing
|
|
174
|
+
* testable without a live gateway.
|
|
175
|
+
*
|
|
176
|
+
* - decision `skip` → `skip` (fall through to the all-blocked card)
|
|
177
|
+
* - decision `downgrade`, gate `skip-inflight`
|
|
178
|
+
* → `suppress` (a restart is already armed)
|
|
179
|
+
* - decision `downgrade`, gate `skip-stale`
|
|
180
|
+
* → `skip` (turn too old to resume; give up)
|
|
181
|
+
* - decision `downgrade`, gate `resume`
|
|
182
|
+
* → `downgrade` (carry the honest notice)
|
|
183
|
+
*/
|
|
184
|
+
export function planTierDowngrade(
|
|
185
|
+
decision: TierDowngradeDecision,
|
|
186
|
+
gateVerdict: ResumeGateVerdict,
|
|
187
|
+
agent: string,
|
|
188
|
+
): TierDowngradePlan {
|
|
189
|
+
if (decision.action !== 'downgrade') return { kind: 'skip' }
|
|
190
|
+
if (gateVerdict === 'skip-inflight') return { kind: 'suppress' }
|
|
191
|
+
if (gateVerdict === 'skip-stale') return { kind: 'skip' }
|
|
192
|
+
return {
|
|
193
|
+
kind: 'downgrade',
|
|
194
|
+
toModel: decision.toModel,
|
|
195
|
+
fromModel: decision.fromModel,
|
|
196
|
+
notice: renderTierDowngradeNotice(decision.fromModel, decision.toModel, agent),
|
|
197
|
+
}
|
|
198
|
+
}
|