switchroom 0.18.13 → 0.18.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +49 -9
- package/dist/auth-broker/index.js +111 -7
- package/dist/cli/autoaccept-poll.js +23 -0
- package/dist/cli/drive-write-pretool.mjs +24 -1
- package/dist/cli/foreground-hog-pretool.mjs +264 -0
- package/dist/cli/notion-write-pretool.mjs +0 -1
- package/dist/cli/switchroom.js +35 -6
- package/dist/host-control/main.js +1 -2
- package/dist/vault/approvals/kernel-server.js +0 -1
- package/dist/vault/broker/server.js +0 -1
- package/package.json +1 -1
- package/profiles/coding/CLAUDE.md.hbs +2 -0
- package/profiles/default/CLAUDE.md.hbs +2 -0
- package/skills/switchroom-architecture/telegram.md +0 -1
- package/telegram-plugin/auth-snapshot-format.ts +37 -5
- package/telegram-plugin/auto-fallback-fleet.ts +29 -1
- package/telegram-plugin/bridge/bridge.ts +2 -0
- package/telegram-plugin/dist/bridge/bridge.js +2 -0
- package/telegram-plugin/dist/gateway/gateway.js +620 -67
- package/telegram-plugin/dist/server.js +2 -0
- package/telegram-plugin/gateway/auth-broker-client.ts +1 -0
- package/telegram-plugin/gateway/auth-command.ts +14 -0
- package/telegram-plugin/gateway/forward-origin.ts +235 -0
- package/telegram-plugin/gateway/gateway.ts +224 -10
- package/telegram-plugin/gateway/throttle-tier-wiring.ts +268 -0
- package/telegram-plugin/history.ts +55 -6
- package/telegram-plugin/model-unavailable.ts +20 -2
- package/telegram-plugin/render/rich-render.ts +40 -32
- package/telegram-plugin/stream-controller.ts +3 -2
- package/telegram-plugin/tests/auto-fallback-fleet.test.ts +72 -0
- package/telegram-plugin/tests/forward-origin.test.ts +309 -0
- package/telegram-plugin/tests/history.test.ts +157 -0
- package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +6 -4
- package/telegram-plugin/tests/render/rich-render.test.ts +41 -22
- package/telegram-plugin/tests/single-mode-stream-reply.test.ts +5 -3
- package/telegram-plugin/tests/status-accent.test.ts +5 -3
- package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +20 -20
- package/telegram-plugin/tests/stream-reply-handler.test.ts +5 -2
- package/telegram-plugin/tests/throttle-tier-wiring.test.ts +290 -0
- package/telegram-plugin/tests/throttle-tier.test.ts +278 -0
- package/telegram-plugin/throttle-tier.ts +226 -0
- package/telegram-plugin/uat/scenarios/jtbd-rich-formatting-render-dm.test.ts +8 -7
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* Wire-level regression test for the chunk-boundary cap bug
|
|
3
3
|
* (fix/rich-render-chunk-boundary-cap).
|
|
4
4
|
*
|
|
5
|
-
* With
|
|
5
|
+
* With the rich renderer enabled (the default — escape hatch, not opt-in), a
|
|
6
|
+
* near-cap body full of escapable chars
|
|
6
7
|
* (`_ * |`) makes `renderSafe` degrade the whole document to plain (its escaped
|
|
7
8
|
* rich form exceeds RICH_MESSAGE_MAX_CHARS). BEFORE the fix, the stream
|
|
8
9
|
* controller shipped that ~32k plain body through the plain `sendMessage`
|
|
@@ -13,7 +14,7 @@
|
|
|
13
14
|
* emitted send fits its own wire cap: rich pieces <= 32768, plain pieces
|
|
14
15
|
* <= 4096, and no fenced block is bisected.
|
|
15
16
|
*/
|
|
16
|
-
import { describe, it, expect
|
|
17
|
+
import { describe, it, expect } from "vitest";
|
|
17
18
|
import { createStreamController } from "../stream-controller.js";
|
|
18
19
|
import { createFakeBotApi } from "./fake-bot-api.js";
|
|
19
20
|
import { RICH_MESSAGE_MAX_CHARS } from "../format.js";
|
|
@@ -25,12 +26,7 @@ function fenceCount(s: string): number {
|
|
|
25
26
|
}
|
|
26
27
|
|
|
27
28
|
describe("stream-controller enforces the wire cap on the post-escape body", () => {
|
|
28
|
-
afterEach(() => {
|
|
29
|
-
delete process.env.SWITCHROOM_RICH_RENDER;
|
|
30
|
-
});
|
|
31
|
-
|
|
32
29
|
it("REGRESSION: near-cap escapable first send never exceeds the plain wire cap", async () => {
|
|
33
|
-
process.env.SWITCHROOM_RICH_RENDER = "1";
|
|
34
30
|
const bot = createFakeBotApi({ startMessageId: 1000 });
|
|
35
31
|
const unit = "a_b*c|d ";
|
|
36
32
|
const body = unit.repeat(Math.floor((RICH_MESSAGE_MAX_CHARS - 20) / unit.length));
|
|
@@ -62,7 +58,6 @@ describe("stream-controller enforces the wire cap on the post-escape body", () =
|
|
|
62
58
|
// callback. The pre-fix code re-sent all (N-1) tails as brand-new messages
|
|
63
59
|
// on each edit tick, so `sent.length` grew by (N-1) every update. After the
|
|
64
60
|
// fix, tails are parked once and edited in place — `sent.length` is flat.
|
|
65
|
-
process.env.SWITCHROOM_RICH_RENDER = "1";
|
|
66
61
|
const bot = createFakeBotApi({ startMessageId: 3000 });
|
|
67
62
|
const unit = "a_b*c|d ";
|
|
68
63
|
// Near-cap body that degrades to plain and splits into several pieces.
|
|
@@ -106,17 +101,22 @@ describe("stream-controller enforces the wire cap on the post-escape body", () =
|
|
|
106
101
|
}
|
|
107
102
|
}, 30000);
|
|
108
103
|
|
|
109
|
-
it("
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
104
|
+
it("kill-switch (=0) leaves the single-send path byte-for-byte untouched", async () => {
|
|
105
|
+
process.env.SWITCHROOM_RICH_RENDER = "0";
|
|
106
|
+
try {
|
|
107
|
+
const bot = createFakeBotApi({ startMessageId: 2000 });
|
|
108
|
+
const stream = createStreamController({
|
|
109
|
+
bot: bot as unknown as Parameters<typeof createStreamController>[0]["bot"],
|
|
110
|
+
chatId: "c1",
|
|
111
|
+
throttleMs: 0,
|
|
112
|
+
});
|
|
113
|
+
await stream.update("**hi** _there_");
|
|
114
|
+
await stream.finalize();
|
|
115
|
+
expect(bot.state.sent).toHaveLength(1);
|
|
116
|
+
expect(bot.state.sent[0].rich).toBe(true);
|
|
117
|
+
expect(bot.state.sent[0].text).toBe("**hi** _there_");
|
|
118
|
+
} finally {
|
|
119
|
+
delete process.env.SWITCHROOM_RICH_RENDER;
|
|
120
|
+
}
|
|
121
121
|
});
|
|
122
122
|
});
|
|
@@ -81,7 +81,7 @@ describe('handleStreamReply', () => {
|
|
|
81
81
|
expect(state.activeDraftStreams.size).toBe(1)
|
|
82
82
|
})
|
|
83
83
|
|
|
84
|
-
it('a non-text format ships
|
|
84
|
+
it('a non-text format ships GFM markdown unescaped (no parse_mode)', async () => {
|
|
85
85
|
const state = makeState()
|
|
86
86
|
const deps = makeDeps(bot)
|
|
87
87
|
|
|
@@ -94,7 +94,10 @@ describe('handleStreamReply', () => {
|
|
|
94
94
|
await pending
|
|
95
95
|
|
|
96
96
|
expect(bot.api.sendRichMessage).toHaveBeenCalledTimes(1)
|
|
97
|
-
|
|
97
|
+
// The default-on rich renderer (parse -> renderSafe) normalises the
|
|
98
|
+
// italic marker (`_there_` -> `*there*`, same wire entity) but the
|
|
99
|
+
// payload stays unescaped GFM markdown — no HTML, no MarkdownV2 escaping.
|
|
100
|
+
expect(richSendMarkdown(bot)).toBe('**hi** *there*')
|
|
98
101
|
// No parse_mode on rich opts.
|
|
99
102
|
expect(bot.api.sendRichMessage.mock.calls[0][2]?.parse_mode).toBeUndefined()
|
|
100
103
|
})
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for gateway/throttle-tier-wiring.ts — the 429 throttle tier's
|
|
3
|
+
* side-effect runner, driven with fully injected deps (no gateway import).
|
|
4
|
+
*
|
|
5
|
+
* Pins the OUTCOMES the wiring promises:
|
|
6
|
+
* - EVERY fire reaches the broker's mark-throttled (the escalation counter),
|
|
7
|
+
* even when the user-facing notice is cooldown-suppressed;
|
|
8
|
+
* - ONE notice per account per window, deduped locally AND fleet-wide via
|
|
9
|
+
* the broker claim verb (denied claim → no send; claim error → fail open);
|
|
10
|
+
* - the retry nudge is armed at throttled_until + slack + jitter, replaced
|
|
11
|
+
* (not stacked) by a newer throttle, and at FIRE time:
|
|
12
|
+
* - restarts when idle and the resume gate says 'resume';
|
|
13
|
+
* - DEFERS to the turn-complete drain when the in-flight gate is held
|
|
14
|
+
* by the dead throttled turn (never SIGTERM-now under a live turn);
|
|
15
|
+
* - SKIPS entirely when a NEWER turn (started after the throttle was
|
|
16
|
+
* armed) is live — restarting would kill live work and boot-resume
|
|
17
|
+
* would replay the WRONG turn;
|
|
18
|
+
* - the escalated outcome posts the corroborated-wall announcement (fleet-
|
|
19
|
+
* deduped) and nudges the resume immediately through the same guards;
|
|
20
|
+
* - a broker-unreachable fire degrades to notice-only without throwing.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { describe, it, expect } from 'vitest'
|
|
24
|
+
import {
|
|
25
|
+
createThrottleTierRunner,
|
|
26
|
+
THROTTLE_RETRY_NUDGE_SLACK_MS,
|
|
27
|
+
type ThrottleBrokerClient,
|
|
28
|
+
type ThrottleTierRunnerDeps,
|
|
29
|
+
} from '../gateway/throttle-tier-wiring.js'
|
|
30
|
+
|
|
31
|
+
const NOW = Date.UTC(2026, 6, 12, 8, 0, 0)
|
|
32
|
+
|
|
33
|
+
interface Harness {
|
|
34
|
+
deps: ThrottleTierRunnerDeps
|
|
35
|
+
calls: {
|
|
36
|
+
markThrottled: number[]
|
|
37
|
+
claims: string[]
|
|
38
|
+
notices: Array<{ chatId: string | number; markdown: string }>
|
|
39
|
+
deferrals: string[]
|
|
40
|
+
restarts: string[]
|
|
41
|
+
logs: string[]
|
|
42
|
+
timers: Array<{ ms: number; fn: () => void; cancelled: boolean }>
|
|
43
|
+
}
|
|
44
|
+
clock: { set(ms: number): void }
|
|
45
|
+
fireTimer(idx: number): void
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function makeHarness(opts: {
|
|
49
|
+
markThrottledResult?:
|
|
50
|
+
| { account: string; throttled_until: number; escalated: boolean; rolledTo?: string | null }
|
|
51
|
+
| 'unreachable'
|
|
52
|
+
| 'throw'
|
|
53
|
+
claimGranted?: (key: string) => boolean | 'throw'
|
|
54
|
+
resumeVerdict?: 'resume' | 'skip-inflight' | 'skip-stale'
|
|
55
|
+
turnInFlight?: () => boolean
|
|
56
|
+
newestTurnStartedAt?: () => number | null
|
|
57
|
+
jitterMs?: number
|
|
58
|
+
} = {}): Harness {
|
|
59
|
+
let nowMs = NOW
|
|
60
|
+
const calls: Harness['calls'] = {
|
|
61
|
+
markThrottled: [],
|
|
62
|
+
claims: [],
|
|
63
|
+
notices: [],
|
|
64
|
+
deferrals: [],
|
|
65
|
+
restarts: [],
|
|
66
|
+
logs: [],
|
|
67
|
+
timers: [],
|
|
68
|
+
}
|
|
69
|
+
const client: ThrottleBrokerClient = {
|
|
70
|
+
async markThrottled(until: number) {
|
|
71
|
+
calls.markThrottled.push(until)
|
|
72
|
+
if (opts.markThrottledResult === 'throw') throw new Error('boom')
|
|
73
|
+
const r = opts.markThrottledResult
|
|
74
|
+
if (r && r !== 'unreachable') return r
|
|
75
|
+
return { account: 'alice', throttled_until: until, escalated: false, rolledTo: null }
|
|
76
|
+
},
|
|
77
|
+
async claimNotification(key: string) {
|
|
78
|
+
calls.claims.push(key)
|
|
79
|
+
const g = opts.claimGranted?.(key) ?? true
|
|
80
|
+
if (g === 'throw') throw new Error('claim boom')
|
|
81
|
+
return { granted: g }
|
|
82
|
+
},
|
|
83
|
+
}
|
|
84
|
+
const deps: ThrottleTierRunnerDeps = {
|
|
85
|
+
agentName: 'carrie',
|
|
86
|
+
getBrokerClient: async () =>
|
|
87
|
+
opts.markThrottledResult === 'unreachable' ? null : client,
|
|
88
|
+
listNoticeChats: () => ['111', '222'],
|
|
89
|
+
sendNotice: (chatId, markdown) => calls.notices.push({ chatId, markdown }),
|
|
90
|
+
resumeDecide: () => opts.resumeVerdict ?? 'resume',
|
|
91
|
+
newestActiveTurnStartedAtMs: opts.newestTurnStartedAt ?? (() => null),
|
|
92
|
+
turnInFlight: opts.turnInFlight ?? (() => false),
|
|
93
|
+
deferRestartToTurnComplete: (_agent, reason) => calls.deferrals.push(reason),
|
|
94
|
+
restartNow: (_agent, reason) => calls.restarts.push(reason),
|
|
95
|
+
log: (m) => calls.logs.push(m),
|
|
96
|
+
now: () => nowMs,
|
|
97
|
+
schedule: (fn, ms) => {
|
|
98
|
+
const t = { ms, fn, cancelled: false }
|
|
99
|
+
calls.timers.push(t)
|
|
100
|
+
return { cancel: () => { t.cancelled = true } }
|
|
101
|
+
},
|
|
102
|
+
jitterMs: () => opts.jitterMs ?? 0,
|
|
103
|
+
}
|
|
104
|
+
return {
|
|
105
|
+
deps,
|
|
106
|
+
calls,
|
|
107
|
+
clock: { set: (ms) => { nowMs = ms } },
|
|
108
|
+
fireTimer(idx: number) {
|
|
109
|
+
const t = calls.timers[idx]
|
|
110
|
+
if (!t || t.cancelled) throw new Error('no live timer at idx ' + idx)
|
|
111
|
+
t.fn()
|
|
112
|
+
},
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
describe('throttle-tier runner — broker mark + notice dedup', () => {
|
|
117
|
+
it('every fire reaches markThrottled; the notice is sent ONCE per cooldown window', async () => {
|
|
118
|
+
const h = makeHarness()
|
|
119
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
120
|
+
await runner.fire('carrie', NOW + 60_000, true)
|
|
121
|
+
h.clock.set(NOW + 30_000) // same window
|
|
122
|
+
await runner.fire('carrie', NOW + 90_000, true)
|
|
123
|
+
|
|
124
|
+
// Both hits reached the broker (the escalation counter needs them)…
|
|
125
|
+
expect(h.calls.markThrottled).toEqual([NOW + 60_000, NOW + 90_000])
|
|
126
|
+
// …but only ONE notice per chat went out.
|
|
127
|
+
expect(h.calls.notices).toHaveLength(2) // 2 chats × 1 notice
|
|
128
|
+
expect(h.calls.notices.map((n) => n.chatId)).toEqual(['111', '222'])
|
|
129
|
+
expect(h.calls.notices[0].markdown).toContain('alice')
|
|
130
|
+
expect(h.calls.notices[0].markdown).toContain('Rate-limited, staying put')
|
|
131
|
+
})
|
|
132
|
+
|
|
133
|
+
it('fleet-wide claim dedup: a denied claim suppresses that chat, an error fails open', async () => {
|
|
134
|
+
const h = makeHarness({
|
|
135
|
+
claimGranted: (key) => (key.endsWith(':111') ? false : 'throw'),
|
|
136
|
+
})
|
|
137
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
138
|
+
await runner.fire('carrie', NOW + 60_000, true)
|
|
139
|
+
|
|
140
|
+
// chat 111 denied (another gateway won the claim), chat 222 errored → open.
|
|
141
|
+
expect(h.calls.claims).toEqual([
|
|
142
|
+
`throttle-notice:alice:111`,
|
|
143
|
+
`throttle-notice:alice:222`,
|
|
144
|
+
])
|
|
145
|
+
expect(h.calls.notices.map((n) => n.chatId)).toEqual(['222'])
|
|
146
|
+
})
|
|
147
|
+
|
|
148
|
+
it('degrades to notice-only when the broker is unreachable (no throw, no claim)', async () => {
|
|
149
|
+
const h = makeHarness({ markThrottledResult: 'unreachable' })
|
|
150
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
151
|
+
await runner.fire('carrie', NOW + 60_000, false)
|
|
152
|
+
expect(h.calls.markThrottled).toHaveLength(0)
|
|
153
|
+
expect(h.calls.claims).toHaveLength(0) // no account → no claim key
|
|
154
|
+
expect(h.calls.notices).toHaveLength(2)
|
|
155
|
+
expect(h.calls.notices[0].markdown).toContain('the active account')
|
|
156
|
+
})
|
|
157
|
+
|
|
158
|
+
it('a markThrottled throw is swallowed and the notice still goes out', async () => {
|
|
159
|
+
const h = makeHarness({ markThrottledResult: 'throw' })
|
|
160
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
161
|
+
await expect(runner.fire('carrie', NOW + 60_000, true)).resolves.toBeUndefined()
|
|
162
|
+
expect(h.calls.notices).toHaveLength(2)
|
|
163
|
+
})
|
|
164
|
+
})
|
|
165
|
+
|
|
166
|
+
describe('throttle-tier runner — retry nudge scheduling + turn safety', () => {
|
|
167
|
+
it('arms the nudge at reset + slack + jitter and restarts when idle', async () => {
|
|
168
|
+
const h = makeHarness({ jitterMs: 7_000 })
|
|
169
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
170
|
+
await runner.fire('carrie', NOW + 60_000, true)
|
|
171
|
+
|
|
172
|
+
expect(h.calls.timers).toHaveLength(1)
|
|
173
|
+
expect(h.calls.timers[0].ms).toBe(60_000 + THROTTLE_RETRY_NUDGE_SLACK_MS + 7_000)
|
|
174
|
+
expect(runner.inspect().nudgePending).toBe(true)
|
|
175
|
+
|
|
176
|
+
h.fireTimer(0)
|
|
177
|
+
expect(h.calls.restarts).toEqual(['throttle-retry-resume'])
|
|
178
|
+
expect(h.calls.deferrals).toHaveLength(0)
|
|
179
|
+
expect(runner.inspect().nudgePending).toBe(false)
|
|
180
|
+
})
|
|
181
|
+
|
|
182
|
+
it('a newer throttle REPLACES the pending nudge instead of stacking restarts', async () => {
|
|
183
|
+
const h = makeHarness()
|
|
184
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
185
|
+
await runner.fire('carrie', NOW + 60_000, true)
|
|
186
|
+
await runner.fire('carrie', NOW + 120_000, true)
|
|
187
|
+
expect(h.calls.timers).toHaveLength(2)
|
|
188
|
+
expect(h.calls.timers[0].cancelled).toBe(true)
|
|
189
|
+
expect(h.calls.timers[1].cancelled).toBe(false)
|
|
190
|
+
})
|
|
191
|
+
|
|
192
|
+
it('DEFERS to the turn-complete drain when the dead throttled turn still holds the gate', async () => {
|
|
193
|
+
const h = makeHarness({
|
|
194
|
+
turnInFlight: () => true,
|
|
195
|
+
// The in-flight turn started BEFORE the throttle was armed — it IS the
|
|
196
|
+
// dead turn (a 429-killed turn never writes its turn-end marker).
|
|
197
|
+
newestTurnStartedAt: () => NOW - 60_000,
|
|
198
|
+
})
|
|
199
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
200
|
+
await runner.fire('carrie', NOW + 60_000, true)
|
|
201
|
+
h.fireTimer(0)
|
|
202
|
+
expect(h.calls.restarts).toHaveLength(0) // never SIGTERM-now under a live gate
|
|
203
|
+
expect(h.calls.deferrals).toEqual(['throttle-retry-resume'])
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
it('SKIPS entirely when a NEWER live turn supersedes the dead one', async () => {
|
|
207
|
+
let nowAtNudge = NOW
|
|
208
|
+
const h = makeHarness({
|
|
209
|
+
turnInFlight: () => true,
|
|
210
|
+
// Started AFTER the throttle was armed — live user work.
|
|
211
|
+
newestTurnStartedAt: () => nowAtNudge + 30_000,
|
|
212
|
+
})
|
|
213
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
214
|
+
await runner.fire('carrie', NOW + 60_000, true)
|
|
215
|
+
nowAtNudge = NOW // armedAt == NOW; newest = NOW+30s > armedAt
|
|
216
|
+
h.fireTimer(0)
|
|
217
|
+
expect(h.calls.restarts).toHaveLength(0)
|
|
218
|
+
expect(h.calls.deferrals).toHaveLength(0)
|
|
219
|
+
expect(h.calls.logs.some((l) => l.includes('superseded'))).toBe(true)
|
|
220
|
+
})
|
|
221
|
+
|
|
222
|
+
it('honours the shared resume gate verdict (skip-inflight → no restart)', async () => {
|
|
223
|
+
const h = makeHarness({ resumeVerdict: 'skip-inflight' })
|
|
224
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
225
|
+
await runner.fire('carrie', NOW + 60_000, true)
|
|
226
|
+
h.fireTimer(0)
|
|
227
|
+
expect(h.calls.restarts).toHaveLength(0)
|
|
228
|
+
expect(h.calls.logs.some((l) => l.includes('skip-inflight'))).toBe(true)
|
|
229
|
+
})
|
|
230
|
+
})
|
|
231
|
+
|
|
232
|
+
describe('throttle-tier runner — escalated outcome (corroborated wall)', () => {
|
|
233
|
+
it('posts the fleet-deduped escalation announcement and resumes immediately', async () => {
|
|
234
|
+
const h = makeHarness({
|
|
235
|
+
markThrottledResult: {
|
|
236
|
+
account: 'alice',
|
|
237
|
+
throttled_until: NOW + 60_000,
|
|
238
|
+
escalated: true,
|
|
239
|
+
rolledTo: 'bob',
|
|
240
|
+
},
|
|
241
|
+
})
|
|
242
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
243
|
+
await runner.fire('carrie', NOW + 60_000, true)
|
|
244
|
+
|
|
245
|
+
// Announcement (not the staying-put notice), per chat, claim-deduped.
|
|
246
|
+
expect(h.calls.claims).toEqual([
|
|
247
|
+
`throttle-escalation:alice:111`,
|
|
248
|
+
`throttle-escalation:alice:222`,
|
|
249
|
+
])
|
|
250
|
+
expect(h.calls.notices).toHaveLength(2)
|
|
251
|
+
expect(h.calls.notices[0].markdown).toContain('actually a wall')
|
|
252
|
+
expect(h.calls.notices[0].markdown).toContain('bob')
|
|
253
|
+
|
|
254
|
+
// Immediate resume (fleet just swapped accounts) — no delayed nudge armed.
|
|
255
|
+
expect(h.calls.restarts).toEqual(['throttle-escalation-resume'])
|
|
256
|
+
expect(h.calls.timers).toHaveLength(0)
|
|
257
|
+
})
|
|
258
|
+
|
|
259
|
+
it('escalated with rolledTo=null (all blocked) announces but does NOT restart', async () => {
|
|
260
|
+
const h = makeHarness({
|
|
261
|
+
markThrottledResult: {
|
|
262
|
+
account: 'alice',
|
|
263
|
+
throttled_until: NOW + 60_000,
|
|
264
|
+
escalated: true,
|
|
265
|
+
rolledTo: null,
|
|
266
|
+
},
|
|
267
|
+
})
|
|
268
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
269
|
+
await runner.fire('carrie', NOW + 60_000, true)
|
|
270
|
+
expect(h.calls.notices[0].markdown).toContain('all blocked')
|
|
271
|
+
expect(h.calls.restarts).toHaveLength(0)
|
|
272
|
+
})
|
|
273
|
+
|
|
274
|
+
it('escalated resume respects the live-turn guards too', async () => {
|
|
275
|
+
const h = makeHarness({
|
|
276
|
+
markThrottledResult: {
|
|
277
|
+
account: 'alice',
|
|
278
|
+
throttled_until: NOW + 60_000,
|
|
279
|
+
escalated: true,
|
|
280
|
+
rolledTo: 'bob',
|
|
281
|
+
},
|
|
282
|
+
turnInFlight: () => true,
|
|
283
|
+
newestTurnStartedAt: () => NOW - 60_000, // the dead turn holds the gate
|
|
284
|
+
})
|
|
285
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
286
|
+
await runner.fire('carrie', NOW + 60_000, true)
|
|
287
|
+
expect(h.calls.restarts).toHaveLength(0)
|
|
288
|
+
expect(h.calls.deferrals).toEqual(['throttle-escalation-resume'])
|
|
289
|
+
})
|
|
290
|
+
})
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for telegram-plugin/throttle-tier.ts — the 429 throttle tier.
|
|
3
|
+
*
|
|
4
|
+
* Pins the operator-approved decision matrix ("retry in place under 5 min,
|
|
5
|
+
* else mark + failover, honest reset messaging"):
|
|
6
|
+
* - transient wording + reset ≤ threshold → throttle (stay put)
|
|
7
|
+
* - transient wording + reset > threshold → failover (escalate)
|
|
8
|
+
* - transient wording + unparseable reset → throttle, now+60s default
|
|
9
|
+
* - non-transient (wall) wording → none (caller's existing path)
|
|
10
|
+
* plus the per-account notice cooldown and the notice text itself.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { describe, it, expect } from 'vitest'
|
|
14
|
+
import {
|
|
15
|
+
decideThrottleTier,
|
|
16
|
+
evaluateThrottleNotice,
|
|
17
|
+
isAccountScopedThrottle,
|
|
18
|
+
renderThrottleEscalationNotice,
|
|
19
|
+
renderThrottleNotice,
|
|
20
|
+
throttleRetryInPlaceMaxMs,
|
|
21
|
+
THROTTLE_DEFAULT_WAIT_MS,
|
|
22
|
+
THROTTLE_NOTICE_COOLDOWN_MS,
|
|
23
|
+
THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT,
|
|
24
|
+
type ThrottleNoticeState,
|
|
25
|
+
} from '../throttle-tier.js'
|
|
26
|
+
import { formatModelUnavailableCard, parseResetTime } from '../model-unavailable.js'
|
|
27
|
+
|
|
28
|
+
const NOW = Date.UTC(2026, 6, 12, 8, 0, 0) // 2026-07-12T08:00:00Z
|
|
29
|
+
const THRESHOLD = THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT // 5 min
|
|
30
|
+
|
|
31
|
+
// The canonical transient-negation wording (Anthropic burst-429 shape).
|
|
32
|
+
const TRANSIENT = (suffix: string): string =>
|
|
33
|
+
`This request would exceed your account's rate limit (not your usage limit). ${suffix}`
|
|
34
|
+
|
|
35
|
+
describe('decideThrottleTier — decision matrix', () => {
|
|
36
|
+
it('transient wording + reset within threshold → throttle at the parsed reset', () => {
|
|
37
|
+
const d = decideThrottleTier({
|
|
38
|
+
detail: TRANSIENT('Please retry after 90 seconds.'),
|
|
39
|
+
now: NOW,
|
|
40
|
+
thresholdMs: THRESHOLD,
|
|
41
|
+
})
|
|
42
|
+
expect(d).toEqual({
|
|
43
|
+
action: 'throttle',
|
|
44
|
+
throttledUntilMs: NOW + 90_000,
|
|
45
|
+
resetParsed: true,
|
|
46
|
+
})
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
it('transient wording + "resets in 3m" → throttle (≤ 5 min)', () => {
|
|
50
|
+
const d = decideThrottleTier({
|
|
51
|
+
detail: TRANSIENT('resets in 3m'),
|
|
52
|
+
now: NOW,
|
|
53
|
+
thresholdMs: THRESHOLD,
|
|
54
|
+
})
|
|
55
|
+
expect(d).toEqual({
|
|
56
|
+
action: 'throttle',
|
|
57
|
+
throttledUntilMs: NOW + 3 * 60_000,
|
|
58
|
+
resetParsed: true,
|
|
59
|
+
})
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
it('transient wording + reset beyond threshold → failover with the parsed reset', () => {
|
|
63
|
+
const d = decideThrottleTier({
|
|
64
|
+
detail: TRANSIENT('resets in 2h 15m'),
|
|
65
|
+
now: NOW,
|
|
66
|
+
thresholdMs: THRESHOLD,
|
|
67
|
+
})
|
|
68
|
+
expect(d).toEqual({
|
|
69
|
+
action: 'failover',
|
|
70
|
+
resetAtMs: NOW + (2 * 60 + 15) * 60_000,
|
|
71
|
+
})
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
it('transient wording + NO parseable reset → throttle for the 60s default', () => {
|
|
75
|
+
const d = decideThrottleTier({
|
|
76
|
+
detail: TRANSIENT('try again later.'),
|
|
77
|
+
now: NOW,
|
|
78
|
+
thresholdMs: THRESHOLD,
|
|
79
|
+
})
|
|
80
|
+
expect(d).toEqual({
|
|
81
|
+
action: 'throttle',
|
|
82
|
+
throttledUntilMs: NOW + THROTTLE_DEFAULT_WAIT_MS,
|
|
83
|
+
resetParsed: false,
|
|
84
|
+
})
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
it('transient wording + reset in the PAST → throttle for the 60s default', () => {
|
|
88
|
+
const past = new Date(NOW - 60_000).toISOString()
|
|
89
|
+
const d = decideThrottleTier({
|
|
90
|
+
detail: TRANSIENT(`resets ${past}`),
|
|
91
|
+
now: NOW,
|
|
92
|
+
thresholdMs: THRESHOLD,
|
|
93
|
+
})
|
|
94
|
+
expect(d).toEqual({
|
|
95
|
+
action: 'throttle',
|
|
96
|
+
throttledUntilMs: NOW + THROTTLE_DEFAULT_WAIT_MS,
|
|
97
|
+
resetParsed: false,
|
|
98
|
+
})
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
it('genuine wall wording (no transient negation) → none (existing quota path owns it)', () => {
|
|
102
|
+
const d = decideThrottleTier({
|
|
103
|
+
detail: "You've hit your limit · resets 8:50am (Australia/Melbourne)",
|
|
104
|
+
now: NOW,
|
|
105
|
+
thresholdMs: THRESHOLD,
|
|
106
|
+
})
|
|
107
|
+
expect(d).toEqual({ action: 'none' })
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
it('SERVER-side transient wording (529 shape) → none — never account-throttled', () => {
|
|
111
|
+
// This carries the transient NEGATION ("not your usage limit") but does
|
|
112
|
+
// NOT affirm the account's own rate limit — it is a server-wide
|
|
113
|
+
// condition; an account-scoped throttle + restart nudge would be the
|
|
114
|
+
// wrong action. It stays on the existing calm rate-limited path.
|
|
115
|
+
const d = decideThrottleTier({
|
|
116
|
+
detail: 'Server is temporarily limiting requests (not your usage limit). retry after 60 seconds',
|
|
117
|
+
now: NOW,
|
|
118
|
+
thresholdMs: THRESHOLD,
|
|
119
|
+
})
|
|
120
|
+
expect(d).toEqual({ action: 'none' })
|
|
121
|
+
})
|
|
122
|
+
|
|
123
|
+
it('honours a custom threshold (boundary: exactly at threshold stays in place)', () => {
|
|
124
|
+
const atThreshold = decideThrottleTier({
|
|
125
|
+
detail: TRANSIENT('retry after 300 seconds'),
|
|
126
|
+
now: NOW,
|
|
127
|
+
thresholdMs: 5 * 60_000,
|
|
128
|
+
})
|
|
129
|
+
expect(atThreshold.action).toBe('throttle')
|
|
130
|
+
const beyond = decideThrottleTier({
|
|
131
|
+
detail: TRANSIENT('retry after 301 seconds'),
|
|
132
|
+
now: NOW,
|
|
133
|
+
thresholdMs: 5 * 60_000,
|
|
134
|
+
})
|
|
135
|
+
expect(beyond.action).toBe('failover')
|
|
136
|
+
})
|
|
137
|
+
})
|
|
138
|
+
|
|
139
|
+
describe('isAccountScopedThrottle — the tier gate', () => {
|
|
140
|
+
it("matches account-affirming wording (both apostrophe variants + 'not your account')", () => {
|
|
141
|
+
expect(isAccountScopedThrottle("would exceed your account's rate limit")).toBe(true)
|
|
142
|
+
expect(isAccountScopedThrottle('would exceed your account’s rate limit')).toBe(true)
|
|
143
|
+
expect(isAccountScopedThrottle("this is not your account's limit")).toBe(true)
|
|
144
|
+
})
|
|
145
|
+
|
|
146
|
+
it('rejects server-side transient / 529 wordings and walls', () => {
|
|
147
|
+
expect(isAccountScopedThrottle('Server is temporarily limiting requests (not your usage limit)')).toBe(false)
|
|
148
|
+
expect(isAccountScopedThrottle('temporarily rate limited, overloaded_error 529')).toBe(false)
|
|
149
|
+
expect(isAccountScopedThrottle("You've hit your limit · resets 8:50am")).toBe(false)
|
|
150
|
+
expect(isAccountScopedThrottle('')).toBe(false)
|
|
151
|
+
})
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
describe('throttleRetryInPlaceMaxMs — env resolution', () => {
|
|
155
|
+
it('defaults to 5 minutes', () => {
|
|
156
|
+
expect(throttleRetryInPlaceMaxMs({})).toBe(5 * 60_000)
|
|
157
|
+
})
|
|
158
|
+
|
|
159
|
+
it('reads SWITCHROOM_THROTTLE_RETRY_IN_PLACE_MAX_MS', () => {
|
|
160
|
+
expect(
|
|
161
|
+
throttleRetryInPlaceMaxMs({ SWITCHROOM_THROTTLE_RETRY_IN_PLACE_MAX_MS: '120000' }),
|
|
162
|
+
).toBe(120_000)
|
|
163
|
+
})
|
|
164
|
+
|
|
165
|
+
it('falls back to the default on junk / non-positive values', () => {
|
|
166
|
+
expect(
|
|
167
|
+
throttleRetryInPlaceMaxMs({ SWITCHROOM_THROTTLE_RETRY_IN_PLACE_MAX_MS: 'soon' }),
|
|
168
|
+
).toBe(THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT)
|
|
169
|
+
expect(
|
|
170
|
+
throttleRetryInPlaceMaxMs({ SWITCHROOM_THROTTLE_RETRY_IN_PLACE_MAX_MS: '-5' }),
|
|
171
|
+
).toBe(THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT)
|
|
172
|
+
})
|
|
173
|
+
})
|
|
174
|
+
|
|
175
|
+
describe('evaluateThrottleNotice — per-account cooldown', () => {
|
|
176
|
+
it('sends the first notice and suppresses a repeat inside the window', () => {
|
|
177
|
+
let state: ThrottleNoticeState = { lastSentAtMsByAccount: {} }
|
|
178
|
+
const first = evaluateThrottleNotice(state, 'acct-a', NOW)
|
|
179
|
+
expect(first.send).toBe(true)
|
|
180
|
+
state = first.next
|
|
181
|
+
const repeat = evaluateThrottleNotice(state, 'acct-a', NOW + 60_000)
|
|
182
|
+
expect(repeat.send).toBe(false)
|
|
183
|
+
})
|
|
184
|
+
|
|
185
|
+
it('a DIFFERENT account is not suppressed by the first account\'s window', () => {
|
|
186
|
+
const first = evaluateThrottleNotice({ lastSentAtMsByAccount: {} }, 'acct-a', NOW)
|
|
187
|
+
const other = evaluateThrottleNotice(first.next, 'acct-b', NOW + 1)
|
|
188
|
+
expect(other.send).toBe(true)
|
|
189
|
+
})
|
|
190
|
+
|
|
191
|
+
it('sends again once the cooldown window has elapsed', () => {
|
|
192
|
+
const first = evaluateThrottleNotice({ lastSentAtMsByAccount: {} }, 'acct-a', NOW)
|
|
193
|
+
const later = evaluateThrottleNotice(
|
|
194
|
+
first.next,
|
|
195
|
+
'acct-a',
|
|
196
|
+
NOW + THROTTLE_NOTICE_COOLDOWN_MS,
|
|
197
|
+
)
|
|
198
|
+
expect(later.send).toBe(true)
|
|
199
|
+
})
|
|
200
|
+
})
|
|
201
|
+
|
|
202
|
+
describe('renderThrottleNotice — honest reset messaging', () => {
|
|
203
|
+
it('names the account, the agent, the reset, and that it is NOT a quota wall', () => {
|
|
204
|
+
const text = renderThrottleNotice({
|
|
205
|
+
account: 'alice',
|
|
206
|
+
agent: 'carrie',
|
|
207
|
+
throttledUntilMs: NOW + 3 * 60_000,
|
|
208
|
+
resetParsed: true,
|
|
209
|
+
now: new Date(NOW),
|
|
210
|
+
})
|
|
211
|
+
expect(text).toContain('alice')
|
|
212
|
+
expect(text).toContain('carrie')
|
|
213
|
+
expect(text).toContain('resets in 3m')
|
|
214
|
+
expect(text).toContain('not a quota wall')
|
|
215
|
+
expect(text).toContain('Staying on')
|
|
216
|
+
expect(text).toContain('no failover')
|
|
217
|
+
})
|
|
218
|
+
|
|
219
|
+
it('degrades honestly when the account is unknown (broker unreachable)', () => {
|
|
220
|
+
const text = renderThrottleNotice({
|
|
221
|
+
account: null,
|
|
222
|
+
agent: 'carrie',
|
|
223
|
+
throttledUntilMs: NOW + THROTTLE_DEFAULT_WAIT_MS,
|
|
224
|
+
resetParsed: false,
|
|
225
|
+
now: new Date(NOW),
|
|
226
|
+
})
|
|
227
|
+
expect(text).toContain('the active account')
|
|
228
|
+
expect(text).toContain('retrying in ~60s')
|
|
229
|
+
})
|
|
230
|
+
})
|
|
231
|
+
|
|
232
|
+
describe('renderThrottleEscalationNotice — corroborated-wall announcement', () => {
|
|
233
|
+
it('names the account, the trigger agent, and the roll target', () => {
|
|
234
|
+
const text = renderThrottleEscalationNotice({
|
|
235
|
+
account: 'alice',
|
|
236
|
+
agent: 'carrie',
|
|
237
|
+
rolledTo: 'bob',
|
|
238
|
+
})
|
|
239
|
+
expect(text).toContain('alice')
|
|
240
|
+
expect(text).toContain('carrie')
|
|
241
|
+
expect(text).toContain('bob')
|
|
242
|
+
expect(text).toContain('actually a wall')
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
it('renders the all-blocked variant when no fallback had quota', () => {
|
|
246
|
+
const text = renderThrottleEscalationNotice({
|
|
247
|
+
account: 'alice',
|
|
248
|
+
agent: 'carrie',
|
|
249
|
+
rolledTo: null,
|
|
250
|
+
})
|
|
251
|
+
expect(text).toContain('all blocked')
|
|
252
|
+
expect(text).toContain('/auth add')
|
|
253
|
+
})
|
|
254
|
+
})
|
|
255
|
+
|
|
256
|
+
describe('rate_limited escalation card wording (model-unavailable integration)', () => {
|
|
257
|
+
it('formatModelUnavailableCard names the rate limit, not quota exhaustion', () => {
|
|
258
|
+
const card = formatModelUnavailableCard(
|
|
259
|
+
{ kind: 'rate_limited', resetAt: new Date(NOW + 30 * 60_000), raw: 'x' },
|
|
260
|
+
'carrie',
|
|
261
|
+
{ now: new Date(NOW), autoFallbackInFlight: true },
|
|
262
|
+
)
|
|
263
|
+
expect(card).toContain('account rate-limited')
|
|
264
|
+
expect(card).toContain('resets in 30m')
|
|
265
|
+
expect(card).not.toContain('quota exhausted')
|
|
266
|
+
})
|
|
267
|
+
})
|
|
268
|
+
|
|
269
|
+
describe('parseResetTime (exported for the throttle tier)', () => {
|
|
270
|
+
it('parses "retry after N seconds" relative to the injected clock', () => {
|
|
271
|
+
const d = parseResetTime('retry after 45 seconds', new Date(NOW))
|
|
272
|
+
expect(d?.getTime()).toBe(NOW + 45_000)
|
|
273
|
+
})
|
|
274
|
+
|
|
275
|
+
it('returns undefined for prose with no reset hint', () => {
|
|
276
|
+
expect(parseResetTime('temporarily limiting requests', new Date(NOW))).toBeUndefined()
|
|
277
|
+
})
|
|
278
|
+
})
|