switchroom 0.18.15 → 0.18.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +3 -0
- package/dist/auth-broker/index.js +432 -10
- package/dist/cli/notion-write-pretool.mjs +3 -0
- package/dist/cli/switchroom.js +50 -1
- package/dist/host-control/main.js +4 -1
- package/dist/vault/approvals/kernel-server.js +3 -0
- package/dist/vault/broker/server.js +3 -0
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +81 -139
- package/telegram-plugin/dist/gateway/gateway.js +386 -259
- package/telegram-plugin/draft-stream.ts +78 -3
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +3 -4
- package/telegram-plugin/gateway/effort-command.ts +9 -7
- package/telegram-plugin/gateway/gateway.ts +265 -220
- package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
- package/telegram-plugin/gateway/model-command.ts +96 -18
- package/telegram-plugin/gateway/pending-session-command.ts +10 -8
- package/telegram-plugin/gateway/session-model-file.ts +38 -172
- package/telegram-plugin/litellm-local-notice.ts +189 -0
- package/telegram-plugin/quota-watch.ts +16 -4
- package/telegram-plugin/runtime-metrics.ts +16 -0
- package/telegram-plugin/send-gate-degraded.test.ts +9 -7
- package/telegram-plugin/send-gate.ts +34 -4
- package/telegram-plugin/stream-controller.ts +143 -20
- package/telegram-plugin/stream-reply-handler.ts +12 -2
- package/telegram-plugin/tests/bot-api.harness.ts +7 -2
- package/telegram-plugin/tests/draft-stream.test.ts +110 -1
- package/telegram-plugin/tests/effort-command.test.ts +4 -4
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +2 -2
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
- package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
- package/telegram-plugin/tests/model-command.test.ts +84 -1
- package/telegram-plugin/tests/quota-watch.test.ts +21 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +2 -2
- package/telegram-plugin/tests/session-model-file.test.ts +7 -155
- package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +207 -0
- package/telegram-plugin/worker-activity-feed.ts +83 -8
|
@@ -0,0 +1,417 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for the litellm-local 429 notice — the debounced "fleet token
|
|
3
|
+
* limiter engaged" message (telegram-plugin/litellm-local-notice.ts) and its
|
|
4
|
+
* side-effect runner (telegram-plugin/gateway/litellm-local-notice-wiring.ts).
|
|
5
|
+
*
|
|
6
|
+
* Pins the operator spec:
|
|
7
|
+
* - first litellm-local 429 → ONE calm notice
|
|
8
|
+
* - further 429s inside the cooldown window → suppressed, counted silently
|
|
9
|
+
* - first 429 after the window → notice carrying "throttled N more times
|
|
10
|
+
* since the last notice"
|
|
11
|
+
* - a notice only ever fires on an actual throttle event (evaluate-on-event,
|
|
12
|
+
* no timers — a quiet agent posts nothing)
|
|
13
|
+
* - never fires for non-litellm-local classifications
|
|
14
|
+
* - copy names the fleet limiter (LiteLLM tpm_limit/rpm_limit), explicitly
|
|
15
|
+
* NOT an Anthropic account limit, turn retries, no action needed
|
|
16
|
+
* - window is operator-tunable with a validated 15-min default
|
|
17
|
+
* - the window is armed / the metric emitted / "posted" logged only when a
|
|
18
|
+
* send was actually ISSUED (empty allowFrom or all-throwing sends retry)
|
|
19
|
+
* - the ordering contract with the shared per-kind card cooldown
|
|
20
|
+
* (decideRateLimitedSurface at the REAL shouldEmitOperatorEvent seam):
|
|
21
|
+
* litellm-local neither arms nor is suppressed by `${agent}:rate-limited`
|
|
22
|
+
*
|
|
23
|
+
* No real Date.now in the pure module — `now` is a parameter throughout
|
|
24
|
+
* (same injected-clock convention as throttle-tier.test.ts).
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { beforeEach, describe, it, expect } from 'vitest'
|
|
28
|
+
import {
|
|
29
|
+
evaluateLitellmLocalNotice,
|
|
30
|
+
initialLitellmLocalNoticeState,
|
|
31
|
+
isLitellmLocalNoticeEligible,
|
|
32
|
+
LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT,
|
|
33
|
+
parseLitellmNoticeWindowMs,
|
|
34
|
+
renderLitellmLocalNotice,
|
|
35
|
+
type LitellmLocalNoticeState,
|
|
36
|
+
} from '../litellm-local-notice.js'
|
|
37
|
+
import {
|
|
38
|
+
createLitellmLocalNoticeRunner,
|
|
39
|
+
decideRateLimitedSurface,
|
|
40
|
+
} from '../gateway/litellm-local-notice-wiring.js'
|
|
41
|
+
import { classify429Detail } from '../throttle-tier.js'
|
|
42
|
+
import { resetAllCooldowns, shouldEmitOperatorEvent } from '../operator-events.js'
|
|
43
|
+
|
|
44
|
+
const NOW = Date.UTC(2026, 6, 12, 8, 0, 0) // 2026-07-12T08:00:00Z
|
|
45
|
+
const WINDOW = LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT
|
|
46
|
+
|
|
47
|
+
// Verbatim litellm-proxy-local fixture (same provenance as
|
|
48
|
+
// throttle-tier.test.ts / litellmProxyLocal429Signals): the v1 parallel
|
|
49
|
+
// request limiter's wrapped exception.
|
|
50
|
+
const LITELLM_LOCAL_DETAIL =
|
|
51
|
+
'litellm.RateLimitError: RateLimitError: Crossed TPM / RPM Limit. ' +
|
|
52
|
+
'current tpm: 20112, tpm limit: 20000. Limit type: tpm. ' +
|
|
53
|
+
'Rate limit exceeded for key. litellm rate limit handler'
|
|
54
|
+
|
|
55
|
+
const ACCOUNT_SCOPED_DETAIL =
|
|
56
|
+
"This request would exceed your account's rate limit. Your account will reset at 8:50am (UTC)."
|
|
57
|
+
|
|
58
|
+
// ─── Cooldown state machine ──────────────────────────────────────────────────
|
|
59
|
+
|
|
60
|
+
describe('evaluateLitellmLocalNotice — per-agent cooldown state machine', () => {
|
|
61
|
+
it('first 429 for an agent sends, with zero suppressed', () => {
|
|
62
|
+
const v = evaluateLitellmLocalNotice(initialLitellmLocalNoticeState(), 'clerk', NOW, WINDOW)
|
|
63
|
+
expect(v.send).toBe(true)
|
|
64
|
+
expect(v.suppressedSinceLastNotice).toBe(0)
|
|
65
|
+
expect(v.next.lastSentAtMsByAgent['clerk']).toBe(NOW)
|
|
66
|
+
expect(v.next.suppressedCountByAgent['clerk']).toBe(0)
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
it('second 429 inside the window is suppressed and counted', () => {
|
|
70
|
+
const first = evaluateLitellmLocalNotice(initialLitellmLocalNoticeState(), 'clerk', NOW, WINDOW)
|
|
71
|
+
const second = evaluateLitellmLocalNotice(first.next, 'clerk', NOW + 60_000, WINDOW)
|
|
72
|
+
expect(second.send).toBe(false)
|
|
73
|
+
expect(second.next.suppressedCountByAgent['clerk']).toBe(1)
|
|
74
|
+
const third = evaluateLitellmLocalNotice(second.next, 'clerk', NOW + 120_000, WINDOW)
|
|
75
|
+
expect(third.send).toBe(false)
|
|
76
|
+
expect(third.next.suppressedCountByAgent['clerk']).toBe(2)
|
|
77
|
+
// The last-sent stamp is NOT refreshed by suppressed events — the window
|
|
78
|
+
// anchors on the last NOTICE, so a sustained burst can't starve it.
|
|
79
|
+
expect(third.next.lastSentAtMsByAgent['clerk']).toBe(NOW)
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
it('first 429 after the window sends and carries the suppressed count, then resets', () => {
|
|
83
|
+
let state = evaluateLitellmLocalNotice(initialLitellmLocalNoticeState(), 'clerk', NOW, WINDOW).next
|
|
84
|
+
for (let i = 1; i <= 3; i++) {
|
|
85
|
+
state = evaluateLitellmLocalNotice(state, 'clerk', NOW + i * 60_000, WINDOW).next
|
|
86
|
+
}
|
|
87
|
+
const after = evaluateLitellmLocalNotice(state, 'clerk', NOW + WINDOW + 1, WINDOW)
|
|
88
|
+
expect(after.send).toBe(true)
|
|
89
|
+
expect(after.suppressedSinceLastNotice).toBe(3)
|
|
90
|
+
expect(after.next.suppressedCountByAgent['clerk']).toBe(0)
|
|
91
|
+
expect(after.next.lastSentAtMsByAgent['clerk']).toBe(NOW + WINDOW + 1)
|
|
92
|
+
})
|
|
93
|
+
|
|
94
|
+
it('window boundary: elapsed === windowMs sends; one ms earlier suppresses', () => {
|
|
95
|
+
const armed = evaluateLitellmLocalNotice(initialLitellmLocalNoticeState(), 'clerk', NOW, WINDOW).next
|
|
96
|
+
expect(evaluateLitellmLocalNotice(armed, 'clerk', NOW + WINDOW - 1, WINDOW).send).toBe(false)
|
|
97
|
+
expect(evaluateLitellmLocalNotice(armed, 'clerk', NOW + WINDOW, WINDOW).send).toBe(true)
|
|
98
|
+
})
|
|
99
|
+
|
|
100
|
+
it('agents debounce independently (per-agent window)', () => {
|
|
101
|
+
const a = evaluateLitellmLocalNotice(initialLitellmLocalNoticeState(), 'clerk', NOW, WINDOW)
|
|
102
|
+
const b = evaluateLitellmLocalNotice(a.next, 'gymbro', NOW + 1_000, WINDOW)
|
|
103
|
+
expect(b.send).toBe(true)
|
|
104
|
+
// clerk's window is untouched by gymbro's notice.
|
|
105
|
+
const c = evaluateLitellmLocalNotice(b.next, 'clerk', NOW + 2_000, WINDOW)
|
|
106
|
+
expect(c.send).toBe(false)
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
it('never mutates the previous state (pure)', () => {
|
|
110
|
+
const prev: LitellmLocalNoticeState = {
|
|
111
|
+
lastSentAtMsByAgent: { clerk: NOW },
|
|
112
|
+
suppressedCountByAgent: { clerk: 1 },
|
|
113
|
+
}
|
|
114
|
+
evaluateLitellmLocalNotice(prev, 'clerk', NOW + 1, WINDOW)
|
|
115
|
+
expect(prev.suppressedCountByAgent['clerk']).toBe(1)
|
|
116
|
+
evaluateLitellmLocalNotice(prev, 'clerk', NOW + WINDOW, WINDOW)
|
|
117
|
+
expect(prev.lastSentAtMsByAgent['clerk']).toBe(NOW)
|
|
118
|
+
expect(prev.suppressedCountByAgent['clerk']).toBe(1)
|
|
119
|
+
})
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
// ─── Config parsing ──────────────────────────────────────────────────────────
|
|
123
|
+
|
|
124
|
+
describe('parseLitellmNoticeWindowMs — tunable window with validated default', () => {
|
|
125
|
+
it('defaults to 15 minutes when unset', () => {
|
|
126
|
+
expect(LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT).toBe(15 * 60_000)
|
|
127
|
+
expect(parseLitellmNoticeWindowMs(undefined)).toBe(LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT)
|
|
128
|
+
expect(parseLitellmNoticeWindowMs(null)).toBe(LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT)
|
|
129
|
+
})
|
|
130
|
+
|
|
131
|
+
it('accepts a custom positive number', () => {
|
|
132
|
+
expect(parseLitellmNoticeWindowMs(5 * 60_000)).toBe(5 * 60_000)
|
|
133
|
+
expect(parseLitellmNoticeWindowMs(1)).toBe(1)
|
|
134
|
+
})
|
|
135
|
+
|
|
136
|
+
it('rejects invalid values back to the default (zero, negative, NaN, Infinity, non-number)', () => {
|
|
137
|
+
for (const bad of [0, -1, NaN, Infinity, -Infinity, '600000', {}, [], true]) {
|
|
138
|
+
expect(parseLitellmNoticeWindowMs(bad)).toBe(LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT)
|
|
139
|
+
}
|
|
140
|
+
})
|
|
141
|
+
})
|
|
142
|
+
|
|
143
|
+
// ─── Notice text ─────────────────────────────────────────────────────────────
|
|
144
|
+
|
|
145
|
+
describe('renderLitellmLocalNotice — copy contract', () => {
|
|
146
|
+
it('names the fleet limiter, negates the account reading, promises the retry', () => {
|
|
147
|
+
const text = renderLitellmLocalNotice({ agent: 'clerk', suppressedSinceLastNotice: 0 })
|
|
148
|
+
expect(text).toContain('Fleet token limiter')
|
|
149
|
+
expect(text).toContain('tpm_limit')
|
|
150
|
+
expect(text).toContain('rpm_limit')
|
|
151
|
+
expect(text).toContain('LiteLLM')
|
|
152
|
+
expect(text).toContain('not an Anthropic account limit')
|
|
153
|
+
expect(text).toContain('retries automatically')
|
|
154
|
+
expect(text).toContain('no action needed')
|
|
155
|
+
expect(text).toContain('**clerk**')
|
|
156
|
+
})
|
|
157
|
+
|
|
158
|
+
it('omits the suppressed line on a first notice, includes it after silent throttles', () => {
|
|
159
|
+
const first = renderLitellmLocalNotice({ agent: 'clerk', suppressedSinceLastNotice: 0 })
|
|
160
|
+
expect(first).not.toContain('since the last notice')
|
|
161
|
+
const later = renderLitellmLocalNotice({ agent: 'clerk', suppressedSinceLastNotice: 4 })
|
|
162
|
+
expect(later).toContain('Throttled 4 more times since the last notice.')
|
|
163
|
+
})
|
|
164
|
+
|
|
165
|
+
it('pluralizes the suppressed line correctly for exactly one silent throttle', () => {
|
|
166
|
+
const text = renderLitellmLocalNotice({ agent: 'clerk', suppressedSinceLastNotice: 1 })
|
|
167
|
+
expect(text).toContain('Throttled 1 more time since the last notice.')
|
|
168
|
+
expect(text).not.toContain('1 more times')
|
|
169
|
+
})
|
|
170
|
+
|
|
171
|
+
it('escapes markdown metacharacters in the agent name', () => {
|
|
172
|
+
const text = renderLitellmLocalNotice({ agent: 'a_b*c', suppressedSinceLastNotice: 0 })
|
|
173
|
+
expect(text).toContain('a\\_b\\*c')
|
|
174
|
+
expect(text).not.toContain('**a_b*c**')
|
|
175
|
+
})
|
|
176
|
+
})
|
|
177
|
+
|
|
178
|
+
// ─── Classification guard ────────────────────────────────────────────────────
|
|
179
|
+
|
|
180
|
+
describe('isLitellmLocalNoticeEligible — only litellm-local qualifies', () => {
|
|
181
|
+
it('accepts litellm-local and rejects everything else', () => {
|
|
182
|
+
expect(isLitellmLocalNoticeEligible('litellm-local')).toBe(true)
|
|
183
|
+
expect(isLitellmLocalNoticeEligible('account-scoped')).toBe(false)
|
|
184
|
+
expect(isLitellmLocalNoticeEligible('generic-transient')).toBe(false)
|
|
185
|
+
expect(isLitellmLocalNoticeEligible(null)).toBe(false)
|
|
186
|
+
expect(isLitellmLocalNoticeEligible(undefined)).toBe(false)
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
it('agrees with classify429Detail on real fixtures', () => {
|
|
190
|
+
expect(isLitellmLocalNoticeEligible(classify429Detail(LITELLM_LOCAL_DETAIL))).toBe(true)
|
|
191
|
+
expect(isLitellmLocalNoticeEligible(classify429Detail(ACCOUNT_SCOPED_DETAIL))).toBe(false)
|
|
192
|
+
expect(isLitellmLocalNoticeEligible(classify429Detail('overloaded_error'))).toBe(false)
|
|
193
|
+
})
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
// ─── Surface decision — the shared card-cooldown seam ────────────────────────
|
|
197
|
+
|
|
198
|
+
describe('decideRateLimitedSurface — ordering contract with shouldEmitOperatorEvent', () => {
|
|
199
|
+
// These run against the REAL shared cooldown gate (operator-events.ts), so
|
|
200
|
+
// they pin the gateway behavior change: litellm-local resolves before the
|
|
201
|
+
// gate — it neither ARMS `${agent}:rate-limited` nor is SUPPRESSED by it.
|
|
202
|
+
beforeEach(() => {
|
|
203
|
+
resetAllCooldowns()
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
const realGate = (a: string) => shouldEmitOperatorEvent(a, 'rate-limited')
|
|
207
|
+
|
|
208
|
+
it('litellm-local does NOT arm the shared rate-limited cooldown', () => {
|
|
209
|
+
expect(
|
|
210
|
+
decideRateLimitedSurface({
|
|
211
|
+
classification: 'litellm-local',
|
|
212
|
+
agent: 'seam-a',
|
|
213
|
+
shouldEmitCard: realGate,
|
|
214
|
+
}),
|
|
215
|
+
).toBe('litellm-local-notice')
|
|
216
|
+
// If litellm-local had consulted (and armed) the gate, this immediate
|
|
217
|
+
// generic-transient would read 'cooldown-suppressed'. It must get its card.
|
|
218
|
+
expect(
|
|
219
|
+
decideRateLimitedSurface({
|
|
220
|
+
classification: 'generic-transient',
|
|
221
|
+
agent: 'seam-a',
|
|
222
|
+
shouldEmitCard: realGate,
|
|
223
|
+
}),
|
|
224
|
+
).toBe('generic-card')
|
|
225
|
+
})
|
|
226
|
+
|
|
227
|
+
it('litellm-local is NOT suppressed by a cooldown a recent generic/529 card armed', () => {
|
|
228
|
+
// A generic transient (e.g. a 529 card) arms the shared cooldown…
|
|
229
|
+
expect(shouldEmitOperatorEvent('seam-b', 'rate-limited')).toBe(true)
|
|
230
|
+
expect(shouldEmitOperatorEvent('seam-b', 'rate-limited')).toBe(false) // armed
|
|
231
|
+
// …but the litellm-local notice still resolves (its own debounce governs).
|
|
232
|
+
expect(
|
|
233
|
+
decideRateLimitedSurface({
|
|
234
|
+
classification: 'litellm-local',
|
|
235
|
+
agent: 'seam-b',
|
|
236
|
+
shouldEmitCard: realGate,
|
|
237
|
+
}),
|
|
238
|
+
).toBe('litellm-local-notice')
|
|
239
|
+
})
|
|
240
|
+
|
|
241
|
+
it('generic-transient consults (and arms) the gate exactly once per decide', () => {
|
|
242
|
+
expect(
|
|
243
|
+
decideRateLimitedSurface({
|
|
244
|
+
classification: 'generic-transient',
|
|
245
|
+
agent: 'seam-c',
|
|
246
|
+
shouldEmitCard: realGate,
|
|
247
|
+
}),
|
|
248
|
+
).toBe('generic-card')
|
|
249
|
+
// The first decide armed the gate — a burst sibling is suppressed.
|
|
250
|
+
expect(
|
|
251
|
+
decideRateLimitedSurface({
|
|
252
|
+
classification: 'generic-transient',
|
|
253
|
+
agent: 'seam-c',
|
|
254
|
+
shouldEmitCard: realGate,
|
|
255
|
+
}),
|
|
256
|
+
).toBe('cooldown-suppressed')
|
|
257
|
+
})
|
|
258
|
+
})
|
|
259
|
+
|
|
260
|
+
// ─── Wiring runner ───────────────────────────────────────────────────────────
|
|
261
|
+
|
|
262
|
+
interface Harness {
|
|
263
|
+
runner: ReturnType<typeof createLitellmLocalNoticeRunner>
|
|
264
|
+
sends: Array<{ chatId: string | number; markdown: string }>
|
|
265
|
+
metrics: unknown[]
|
|
266
|
+
logs: string[]
|
|
267
|
+
clock: { now: number }
|
|
268
|
+
window: { ms: number }
|
|
269
|
+
chats: Array<string | number>
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function makeHarness(opts?: { chats?: Array<string | number> }): Harness {
|
|
273
|
+
const sends: Array<{ chatId: string | number; markdown: string }> = []
|
|
274
|
+
const metrics: unknown[] = []
|
|
275
|
+
const logs: string[] = []
|
|
276
|
+
const clock = { now: NOW }
|
|
277
|
+
const window = { ms: WINDOW }
|
|
278
|
+
const chats: Array<string | number> = opts?.chats ?? ['111', '222']
|
|
279
|
+
const runner = createLitellmLocalNoticeRunner({
|
|
280
|
+
listNoticeChats: () => chats,
|
|
281
|
+
sendNotice: (chatId, markdown) => sends.push({ chatId, markdown }),
|
|
282
|
+
windowMs: () => window.ms,
|
|
283
|
+
emitMetric: (ev) => metrics.push(ev),
|
|
284
|
+
log: (m) => logs.push(m),
|
|
285
|
+
now: () => clock.now,
|
|
286
|
+
})
|
|
287
|
+
return { runner, sends, metrics, logs, clock, window, chats }
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
describe('createLitellmLocalNoticeRunner — side-effect sequencing', () => {
|
|
291
|
+
it('first litellm-local 429 broadcasts one notice to every authorized chat + one metric, outcome sent', () => {
|
|
292
|
+
const h = makeHarness()
|
|
293
|
+
expect(h.runner.onRateLimited('litellm-local', 'clerk')).toBe('sent')
|
|
294
|
+
expect(h.sends.map((s) => s.chatId)).toEqual(['111', '222'])
|
|
295
|
+
expect(h.sends[0].markdown).toContain('Fleet token limiter')
|
|
296
|
+
// Full metric object — also pins buildLitellmLocalNoticeMetric's shape.
|
|
297
|
+
expect(h.metrics).toEqual([
|
|
298
|
+
{ kind: 'litellm_local_429_notice', agent: 'clerk', suppressed_count: 0, window_ms: WINDOW },
|
|
299
|
+
])
|
|
300
|
+
})
|
|
301
|
+
|
|
302
|
+
it('suppresses inside the window; elapsed time alone never sends; the next event reports the count', () => {
|
|
303
|
+
const h = makeHarness()
|
|
304
|
+
expect(h.runner.onRateLimited('litellm-local', 'clerk')).toBe('sent')
|
|
305
|
+
h.clock.now = NOW + 60_000
|
|
306
|
+
expect(h.runner.onRateLimited('litellm-local', 'clerk')).toBe('suppressed')
|
|
307
|
+
h.clock.now = NOW + 120_000
|
|
308
|
+
expect(h.runner.onRateLimited('litellm-local', 'clerk')).toBe('suppressed')
|
|
309
|
+
// Two suppressed events: still just the first broadcast, one metric.
|
|
310
|
+
expect(h.sends).toHaveLength(2) // 2 chats × 1 notice
|
|
311
|
+
expect(h.metrics).toHaveLength(1)
|
|
312
|
+
expect(h.logs.some((l) => l.includes('suppressed (cooldown)'))).toBe(true)
|
|
313
|
+
// A quiet stretch — the window expires, then MULTIPLE further windows
|
|
314
|
+
// pass with zero throttle events. Notices are evaluate-on-event: nothing
|
|
315
|
+
// may send or emit on elapsed time alone.
|
|
316
|
+
h.clock.now = NOW + 10 * WINDOW
|
|
317
|
+
expect(h.sends).toHaveLength(2)
|
|
318
|
+
expect(h.metrics).toHaveLength(1)
|
|
319
|
+
// The next ACTUAL event (long past expiry) posts, carrying the count of
|
|
320
|
+
// the two silently absorbed throttles.
|
|
321
|
+
expect(h.runner.onRateLimited('litellm-local', 'clerk')).toBe('sent')
|
|
322
|
+
expect(h.sends).toHaveLength(4)
|
|
323
|
+
expect(h.sends[2].markdown).toContain('Throttled 2 more times since the last notice.')
|
|
324
|
+
expect(h.metrics[1]).toMatchObject({ suppressed_count: 2 })
|
|
325
|
+
})
|
|
326
|
+
|
|
327
|
+
it('never fires for non-litellm-local classifications (outcome skipped, state untouched)', () => {
|
|
328
|
+
const h = makeHarness()
|
|
329
|
+
expect(h.runner.onRateLimited('account-scoped', 'clerk')).toBe('skipped')
|
|
330
|
+
expect(h.runner.onRateLimited('generic-transient', 'clerk')).toBe('skipped')
|
|
331
|
+
expect(h.sends).toHaveLength(0)
|
|
332
|
+
expect(h.metrics).toHaveLength(0)
|
|
333
|
+
// Non-eligible events must not perturb the debounce state either: the
|
|
334
|
+
// next real litellm-local 429 is still a FIRST notice.
|
|
335
|
+
expect(h.runner.onRateLimited('litellm-local', 'clerk')).toBe('sent')
|
|
336
|
+
expect(h.sends).toHaveLength(2)
|
|
337
|
+
expect(h.sends[0].markdown).not.toContain('since the last notice')
|
|
338
|
+
})
|
|
339
|
+
|
|
340
|
+
it('re-reads the tunable window per event (operator retune takes effect live)', () => {
|
|
341
|
+
const h = makeHarness({ chats: ['111'] })
|
|
342
|
+
h.window.ms = 5 * 60_000
|
|
343
|
+
h.runner.onRateLimited('litellm-local', 'clerk')
|
|
344
|
+
h.clock.now = NOW + 5 * 60_000
|
|
345
|
+
h.runner.onRateLimited('litellm-local', 'clerk')
|
|
346
|
+
expect(h.sends).toHaveLength(2)
|
|
347
|
+
expect(h.metrics[1]).toMatchObject({ window_ms: 5 * 60_000 })
|
|
348
|
+
})
|
|
349
|
+
|
|
350
|
+
it('empty allowFrom: no metric, no "posted" log, window NOT armed — a later event with chats sends', () => {
|
|
351
|
+
const h = makeHarness({ chats: [] })
|
|
352
|
+
expect(h.runner.onRateLimited('litellm-local', 'clerk')).toBe('skipped')
|
|
353
|
+
expect(h.sends).toHaveLength(0)
|
|
354
|
+
expect(h.metrics).toHaveLength(0)
|
|
355
|
+
expect(h.logs.some((l) => l.includes('posted'))).toBe(false)
|
|
356
|
+
expect(h.logs.some((l) => l.includes('no sends issued'))).toBe(true)
|
|
357
|
+
// A chat becomes authorized — the very next event delivers even though
|
|
358
|
+
// it is INSIDE what would have been the window (the window was never
|
|
359
|
+
// armed for a notice nobody could hear).
|
|
360
|
+
h.chats.push('111')
|
|
361
|
+
h.clock.now = NOW + 1_000
|
|
362
|
+
expect(h.runner.onRateLimited('litellm-local', 'clerk')).toBe('sent')
|
|
363
|
+
expect(h.sends).toHaveLength(1)
|
|
364
|
+
expect(h.metrics).toHaveLength(1)
|
|
365
|
+
})
|
|
366
|
+
|
|
367
|
+
it('all-throwing sends: outcome skipped, window not armed, and the caller never sees a throw', () => {
|
|
368
|
+
const sends: string[] = []
|
|
369
|
+
const logs: string[] = []
|
|
370
|
+
const clock = { now: NOW }
|
|
371
|
+
let shouldThrow = true
|
|
372
|
+
const runner = createLitellmLocalNoticeRunner({
|
|
373
|
+
listNoticeChats: () => ['111'],
|
|
374
|
+
sendNotice: (_chatId, markdown) => {
|
|
375
|
+
if (shouldThrow) throw new Error('boom')
|
|
376
|
+
sends.push(markdown)
|
|
377
|
+
},
|
|
378
|
+
windowMs: () => WINDOW,
|
|
379
|
+
emitMetric: () => {
|
|
380
|
+
if (shouldThrow) {
|
|
381
|
+
throw new Error('metric sink must not be reached when nothing was issued')
|
|
382
|
+
}
|
|
383
|
+
},
|
|
384
|
+
log: (m) => logs.push(m),
|
|
385
|
+
now: () => clock.now,
|
|
386
|
+
})
|
|
387
|
+
expect(() => runner.onRateLimited('litellm-local', 'clerk')).not.toThrow()
|
|
388
|
+
expect(runner.inspect().state.lastSentAtMsByAgent['clerk']).toBeUndefined()
|
|
389
|
+
expect(logs.some((l) => l.includes('send failed') && l.includes('boom'))).toBe(true)
|
|
390
|
+
expect(logs.some((l) => l.includes('no sends issued'))).toBe(true)
|
|
391
|
+
// Window was NOT armed by the failed attempt: the immediate next event
|
|
392
|
+
// (well inside 15 min) retries and delivers.
|
|
393
|
+
shouldThrow = false
|
|
394
|
+
clock.now = NOW + 1_000
|
|
395
|
+
expect(runner.onRateLimited('litellm-local', 'clerk')).toBe('sent')
|
|
396
|
+
expect(sends).toHaveLength(1)
|
|
397
|
+
})
|
|
398
|
+
|
|
399
|
+
it('a throwing log sink cannot void the never-throws contract', () => {
|
|
400
|
+
const runner = createLitellmLocalNoticeRunner({
|
|
401
|
+
listNoticeChats: () => {
|
|
402
|
+
throw new Error('chats exploded')
|
|
403
|
+
},
|
|
404
|
+
sendNotice: () => {},
|
|
405
|
+
windowMs: () => WINDOW,
|
|
406
|
+
emitMetric: () => {},
|
|
407
|
+
log: () => {
|
|
408
|
+
throw new Error('stderr sink died')
|
|
409
|
+
},
|
|
410
|
+
now: () => NOW,
|
|
411
|
+
})
|
|
412
|
+
// listNoticeChats throws → outer catch → deps.log throws too → still no
|
|
413
|
+
// throw escapes to the operator-event path.
|
|
414
|
+
expect(() => runner.onRateLimited('litellm-local', 'clerk')).not.toThrow()
|
|
415
|
+
expect(runner.onRateLimited('litellm-local', 'clerk')).toBe('skipped')
|
|
416
|
+
})
|
|
417
|
+
})
|
|
@@ -16,6 +16,9 @@
|
|
|
16
16
|
import { describe, it, expect, beforeAll } from "vitest";
|
|
17
17
|
import {
|
|
18
18
|
parseModelCommand,
|
|
19
|
+
planModelCommand,
|
|
20
|
+
isModelCommandBusy,
|
|
21
|
+
modelCommandReceiptLine,
|
|
19
22
|
handleModelCommand,
|
|
20
23
|
isValidModelArg,
|
|
21
24
|
isSrModel,
|
|
@@ -203,7 +206,7 @@ describe("handleModelCommand — set", () => {
|
|
|
203
206
|
const reply = await handleModelCommand({ kind: "set", model: "opus" }, deps);
|
|
204
207
|
expect(calls).toEqual([{ agent: "klanker", command: "/model opus" }]);
|
|
205
208
|
expect(reply.text).toContain("<pre>⏺ Set model to sonnet</pre>");
|
|
206
|
-
expect(reply.text).toContain("
|
|
209
|
+
expect(reply.text).toContain("lasts until the agent’s next restart");
|
|
207
210
|
expect(reply.html).toBe(true);
|
|
208
211
|
// A verified confirmation records the live model so /status stays honest
|
|
209
212
|
// (bug 1: the typed path never recorded the switch before).
|
|
@@ -1374,3 +1377,83 @@ describe("isOfflineTrustedModelToken (#3042 blocker 2a)", () => {
|
|
|
1374
1377
|
expect(isOfflineTrustedModelToken(alias.toUpperCase())).toBe(true);
|
|
1375
1378
|
});
|
|
1376
1379
|
});
|
|
1380
|
+
|
|
1381
|
+
// ---------------------------------------------------------------------------
|
|
1382
|
+
// #3177 — a typed /model must NEVER be swallowed without trace when sent
|
|
1383
|
+
// mid-turn. The routing decision folds BOTH busy signals, and every parsed
|
|
1384
|
+
// shape maps to a visible action (menu / queue / apply — never "do nothing").
|
|
1385
|
+
// ---------------------------------------------------------------------------
|
|
1386
|
+
describe("#3177 typed /model never silently swallowed mid-turn", () => {
|
|
1387
|
+
describe("isModelCommandBusy folds both busy signals", () => {
|
|
1388
|
+
it("is busy when the turn atom is set", () => {
|
|
1389
|
+
expect(isModelCommandBusy({ currentTurnActive: true, turnInFlight: false })).toBe(true);
|
|
1390
|
+
});
|
|
1391
|
+
|
|
1392
|
+
it("is busy when only the authoritative delivery-machine/approval gate is set", () => {
|
|
1393
|
+
// THE SWALLOW REGRESSION: the pre-fix handler gated on `currentTurn !==
|
|
1394
|
+
// null` ALONE. A session busy by the delivery-machine / pending-approval
|
|
1395
|
+
// gate while the turn atom is cleared (the recovered-late / premature-
|
|
1396
|
+
// turn-end window) read as idle, so the switch injected into a busy pane
|
|
1397
|
+
// and was swallowed as literal text. Folding turnInFlight closes it.
|
|
1398
|
+
expect(isModelCommandBusy({ currentTurnActive: false, turnInFlight: true })).toBe(true);
|
|
1399
|
+
});
|
|
1400
|
+
|
|
1401
|
+
it("is idle only when BOTH signals are clear", () => {
|
|
1402
|
+
expect(isModelCommandBusy({ currentTurnActive: false, turnInFlight: false })).toBe(false);
|
|
1403
|
+
});
|
|
1404
|
+
});
|
|
1405
|
+
|
|
1406
|
+
describe("planModelCommand routes every shape to a visible action", () => {
|
|
1407
|
+
const busyByAtom = { currentTurnActive: true, turnInFlight: false, menuEnabled: true };
|
|
1408
|
+
const busyByGate = { currentTurnActive: false, turnInFlight: true, menuEnabled: true };
|
|
1409
|
+
const idle = { currentTurnActive: false, turnInFlight: false, menuEnabled: true };
|
|
1410
|
+
|
|
1411
|
+
it("QUEUES a set while busy by the turn atom (ack, not silent inject)", () => {
|
|
1412
|
+
const d = planModelCommand({ kind: "set", model: "opus" }, busyByAtom);
|
|
1413
|
+
expect(d).toEqual({ kind: "queue", target: "opus" });
|
|
1414
|
+
});
|
|
1415
|
+
|
|
1416
|
+
it("QUEUES a set while busy by the delivery-machine/approval gate ONLY — the swallow case", () => {
|
|
1417
|
+
// Sabotage-verify: revert the fix (gate on currentTurnActive alone) and
|
|
1418
|
+
// this flips to { kind: 'apply' } → the direct-inject swallow returns.
|
|
1419
|
+
const d = planModelCommand({ kind: "set", model: "opus" }, busyByGate);
|
|
1420
|
+
expect(d).toEqual({ kind: "queue", target: "opus" });
|
|
1421
|
+
});
|
|
1422
|
+
|
|
1423
|
+
it("expands sr-* aliases into the queued target token", () => {
|
|
1424
|
+
const d = planModelCommand({ kind: "set", model: "flash" }, busyByAtom);
|
|
1425
|
+
expect(d).toEqual({ kind: "queue", target: "sr-gemini-2.5-flash" });
|
|
1426
|
+
});
|
|
1427
|
+
|
|
1428
|
+
it("APPLIES a set immediately when idle by both signals", () => {
|
|
1429
|
+
const d = planModelCommand({ kind: "set", model: "opus" }, idle);
|
|
1430
|
+
expect(d).toEqual({ kind: "apply", parsed: { kind: "set", model: "opus" } });
|
|
1431
|
+
});
|
|
1432
|
+
|
|
1433
|
+
it("renders the MENU for bare /model when the picker is enabled (even mid-turn)", () => {
|
|
1434
|
+
expect(planModelCommand({ kind: "show" }, busyByGate)).toEqual({ kind: "menu" });
|
|
1435
|
+
});
|
|
1436
|
+
|
|
1437
|
+
it("APPLIES the text show path when the picker is disabled", () => {
|
|
1438
|
+
const d = planModelCommand({ kind: "show" }, { ...idle, menuEnabled: false });
|
|
1439
|
+
expect(d).toEqual({ kind: "apply", parsed: { kind: "show" } });
|
|
1440
|
+
});
|
|
1441
|
+
|
|
1442
|
+
it("APPLIES help so a bad arg still gets an explicit reply", () => {
|
|
1443
|
+
const parsed = { kind: "help", reason: "not a valid model name: !!" } as const;
|
|
1444
|
+
expect(planModelCommand(parsed, busyByGate)).toEqual({ kind: "apply", parsed });
|
|
1445
|
+
});
|
|
1446
|
+
});
|
|
1447
|
+
|
|
1448
|
+
describe("modelCommandReceiptLine — the durable, greppable entry trace", () => {
|
|
1449
|
+
it("stamps agent, kind, arg, and busy for a typed set", () => {
|
|
1450
|
+
const line = modelCommandReceiptLine("finn", { kind: "set", model: "opus" }, true);
|
|
1451
|
+
expect(line).toBe("telegram gateway: gw /model received agent=finn kind=set arg=opus busy=true");
|
|
1452
|
+
});
|
|
1453
|
+
|
|
1454
|
+
it("marks the show + help forms with placeholder args", () => {
|
|
1455
|
+
expect(modelCommandReceiptLine("finn", { kind: "show" }, false)).toContain("kind=show arg=(show)");
|
|
1456
|
+
expect(modelCommandReceiptLine("finn", { kind: "help" }, false)).toContain("kind=help arg=(help)");
|
|
1457
|
+
});
|
|
1458
|
+
});
|
|
1459
|
+
});
|
|
@@ -1170,4 +1170,25 @@ describe("buildFleetRollMessage — reason attribution (#3031 PR 2 reason field)
|
|
|
1170
1170
|
expect(msg).toContain("7-day window at 96% on");
|
|
1171
1171
|
}
|
|
1172
1172
|
});
|
|
1173
|
+
|
|
1174
|
+
it("model-tier-wall (#3176) names the flagship tier and reassures opus/haiku are unaffected — not a generic quota window", () => {
|
|
1175
|
+
// A tier wall binds on the 7d_oi bucket; window/pct are absent (5h/7d read
|
|
1176
|
+
// healthy). exhausted_until carries the tier reset.
|
|
1177
|
+
const roll: FleetRollInfo = {
|
|
1178
|
+
from: "alice@example.com",
|
|
1179
|
+
to: "bob@example.com",
|
|
1180
|
+
at: NOW - 60_000,
|
|
1181
|
+
reason: "model-tier-wall",
|
|
1182
|
+
bucket: "seven_day_overage_included",
|
|
1183
|
+
exhausted_until: NOW + 2.8 * 60 * 60 * 1000,
|
|
1184
|
+
};
|
|
1185
|
+
const msg = buildFleetRollMessage(roll, NOW);
|
|
1186
|
+
expect(msg).toContain("Flagship (premium) tier weekly limit reached");
|
|
1187
|
+
expect(msg).toContain("opus/haiku on that account are unaffected");
|
|
1188
|
+
// Must NOT mislead as a generic 5h/7d window roll.
|
|
1189
|
+
expect(msg).not.toContain("quota window");
|
|
1190
|
+
expect(msg).not.toContain("Proactive switch");
|
|
1191
|
+
// The tier reset is surfaced.
|
|
1192
|
+
expect(msg).toContain("resets");
|
|
1193
|
+
});
|
|
1173
1194
|
});
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
import { describe, it, expect, vi } from 'vitest'
|
|
30
30
|
import { readFileSync } from 'node:fs'
|
|
31
31
|
import { fileURLToPath } from 'node:url'
|
|
32
|
-
import { createSendGate, type Clock } from '../send-gate.js'
|
|
32
|
+
import { createSendGate, SEND_GATE_SHED, type Clock } from '../send-gate.js'
|
|
33
33
|
import { createRetryApiCall } from '../retry-api-call.js'
|
|
34
34
|
import { errors } from './fake-bot-api.js'
|
|
35
35
|
|
|
@@ -95,7 +95,7 @@ describe('#3155 reactions route through the send gate (cosmetic)', () => {
|
|
|
95
95
|
// resolves undefined (fire-and-forget callers .catch nothing), and the gate
|
|
96
96
|
// counts the shed.
|
|
97
97
|
expect(setMessageReaction).not.toHaveBeenCalled()
|
|
98
|
-
expect(result).
|
|
98
|
+
expect(result).toBe(SEND_GATE_SHED) // shed sentinel (#3110 F1)
|
|
99
99
|
expect(sendGate.stats().global.shed).toBe(1)
|
|
100
100
|
})
|
|
101
101
|
|