switchroom 0.18.13 → 0.18.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +49 -9
- package/dist/auth-broker/index.js +152 -46
- package/dist/cli/autoaccept-poll.js +23 -0
- package/dist/cli/drive-write-pretool.mjs +24 -1
- package/dist/cli/foreground-hog-pretool.mjs +264 -0
- package/dist/cli/notion-write-pretool.mjs +0 -1
- package/dist/cli/switchroom.js +1185 -1072
- package/dist/host-control/main.js +53 -52
- package/dist/vault/approvals/kernel-server.js +16 -13
- package/dist/vault/broker/server.js +672 -669
- package/package.json +1 -1
- package/profiles/coding/CLAUDE.md.hbs +2 -0
- package/profiles/default/CLAUDE.md.hbs +2 -0
- package/skills/switchroom-architecture/telegram.md +0 -1
- package/telegram-plugin/auth-snapshot-format.ts +37 -5
- package/telegram-plugin/auto-fallback-fleet.ts +29 -1
- package/telegram-plugin/bridge/bridge.ts +2 -0
- package/telegram-plugin/dist/bridge/bridge.js +23 -0
- package/telegram-plugin/dist/gateway/gateway.js +765 -67
- package/telegram-plugin/dist/server.js +24 -1
- package/telegram-plugin/gateway/auth-broker-client.ts +1 -0
- package/telegram-plugin/gateway/auth-command.ts +14 -0
- package/telegram-plugin/gateway/forward-origin.ts +235 -0
- package/telegram-plugin/gateway/gateway.ts +270 -10
- package/telegram-plugin/gateway/throttle-tier-wiring.ts +268 -0
- package/telegram-plugin/history.ts +55 -6
- package/telegram-plugin/model-unavailable.ts +234 -2
- package/telegram-plugin/render/rich-render.ts +40 -32
- package/telegram-plugin/runtime-metrics.ts +31 -0
- package/telegram-plugin/session-tail.ts +14 -2
- package/telegram-plugin/stream-controller.ts +3 -2
- package/telegram-plugin/tests/auto-fallback-fleet.test.ts +72 -0
- package/telegram-plugin/tests/forward-origin.test.ts +309 -0
- package/telegram-plugin/tests/history.test.ts +157 -0
- package/telegram-plugin/tests/model-unavailable.test.ts +187 -0
- package/telegram-plugin/tests/operator-events-session-tail.test.ts +55 -0
- package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +6 -4
- package/telegram-plugin/tests/render/rich-render.test.ts +41 -22
- package/telegram-plugin/tests/runtime-metrics.test.ts +24 -0
- package/telegram-plugin/tests/single-mode-stream-reply.test.ts +5 -3
- package/telegram-plugin/tests/status-accent.test.ts +5 -3
- package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +20 -20
- package/telegram-plugin/tests/stream-reply-handler.test.ts +5 -2
- package/telegram-plugin/tests/throttle-tier-wiring.test.ts +290 -0
- package/telegram-plugin/tests/throttle-tier.test.ts +454 -0
- package/telegram-plugin/throttle-tier.ts +323 -0
- package/telegram-plugin/uat/scenarios/jtbd-rich-formatting-render-dm.test.ts +8 -7
|
@@ -2,6 +2,9 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
|
2
2
|
import { mkdtempSync, statSync, rmSync, existsSync } from 'fs'
|
|
3
3
|
import { tmpdir } from 'os'
|
|
4
4
|
import { join } from 'path'
|
|
5
|
+
// bun-only (this file is vitest-excluded and runs under `bun test`): used to
|
|
6
|
+
// build an OLD-schema DB file for the additive-migration test.
|
|
7
|
+
import { Database } from 'bun:sqlite'
|
|
5
8
|
import {
|
|
6
9
|
initHistory,
|
|
7
10
|
recordInbound,
|
|
@@ -715,3 +718,157 @@ describe('secret redaction at persistence (both directions)', () => {
|
|
|
715
718
|
expect(query({ chat_id: '-100' })[0]!.text).toBe('hello, how are you?')
|
|
716
719
|
})
|
|
717
720
|
})
|
|
721
|
+
|
|
722
|
+
describe('forwarded-message origin columns', () => {
|
|
723
|
+
it('round-trips forwarded_* fields on an inbound row', () => {
|
|
724
|
+
initHistory(stateDir, 30)
|
|
725
|
+
recordInbound({
|
|
726
|
+
chat_id: '-100',
|
|
727
|
+
thread_id: null,
|
|
728
|
+
message_id: 7,
|
|
729
|
+
user: 'alice',
|
|
730
|
+
user_id: '111',
|
|
731
|
+
ts: 1000,
|
|
732
|
+
text: 'fwd: look at this',
|
|
733
|
+
forwarded_from: 'Release Notes (@relnotes)',
|
|
734
|
+
forwarded_from_type: 'channel',
|
|
735
|
+
forwarded_from_id: '-100400500',
|
|
736
|
+
forwarded_date: '2026-06-15T13:46:40.000Z',
|
|
737
|
+
forwarded_message_id: 555,
|
|
738
|
+
})
|
|
739
|
+
const row = query({ chat_id: '-100' })[0]!
|
|
740
|
+
expect(row).toMatchObject({
|
|
741
|
+
forwarded_from: 'Release Notes (@relnotes)',
|
|
742
|
+
forwarded_from_type: 'channel',
|
|
743
|
+
forwarded_from_id: '-100400500',
|
|
744
|
+
forwarded_date: '2026-06-15T13:46:40.000Z',
|
|
745
|
+
forwarded_message_id: 555,
|
|
746
|
+
})
|
|
747
|
+
})
|
|
748
|
+
|
|
749
|
+
it('non-forwarded inbound stores NULL origin fields', () => {
|
|
750
|
+
initHistory(stateDir, 30)
|
|
751
|
+
recordInbound({
|
|
752
|
+
chat_id: '-100',
|
|
753
|
+
thread_id: null,
|
|
754
|
+
message_id: 8,
|
|
755
|
+
user: 'alice',
|
|
756
|
+
user_id: '111',
|
|
757
|
+
ts: 1000,
|
|
758
|
+
text: 'plain message',
|
|
759
|
+
})
|
|
760
|
+
const row = query({ chat_id: '-100' })[0]!
|
|
761
|
+
expect(row.forwarded_from).toBeNull()
|
|
762
|
+
expect(row.forwarded_from_type).toBeNull()
|
|
763
|
+
expect(row.forwarded_from_id).toBeNull()
|
|
764
|
+
expect(row.forwarded_date).toBeNull()
|
|
765
|
+
expect(row.forwarded_message_id).toBeNull()
|
|
766
|
+
})
|
|
767
|
+
|
|
768
|
+
it('migrates additively: pre-existing DB without the columns gains them and round-trips', () => {
|
|
769
|
+
// Build an OLD-schema DB file the way a pre-forward-origin build would
|
|
770
|
+
// have left it: messages table without any forwarded_* columns, one row
|
|
771
|
+
// already stored.
|
|
772
|
+
const dbPath = join(stateDir, 'history.db')
|
|
773
|
+
const old = new Database(dbPath, { create: true })
|
|
774
|
+
old.exec(`
|
|
775
|
+
CREATE TABLE IF NOT EXISTS messages (
|
|
776
|
+
chat_id TEXT NOT NULL,
|
|
777
|
+
thread_id INTEGER,
|
|
778
|
+
message_id INTEGER NOT NULL,
|
|
779
|
+
role TEXT NOT NULL,
|
|
780
|
+
user TEXT,
|
|
781
|
+
user_id TEXT,
|
|
782
|
+
ts INTEGER NOT NULL,
|
|
783
|
+
text TEXT NOT NULL,
|
|
784
|
+
attachment_kind TEXT,
|
|
785
|
+
group_id INTEGER,
|
|
786
|
+
reply_to_message_id INTEGER,
|
|
787
|
+
reply_to_text TEXT,
|
|
788
|
+
PRIMARY KEY (chat_id, thread_id, message_id)
|
|
789
|
+
)
|
|
790
|
+
`)
|
|
791
|
+
// Recent ts — initHistory's retention sweep deletes rows older than the
|
|
792
|
+
// cutoff, and this test is about migration, not retention.
|
|
793
|
+
const now = Math.floor(Date.now() / 1000)
|
|
794
|
+
old.exec(
|
|
795
|
+
`INSERT INTO messages (chat_id, thread_id, message_id, role, user, user_id, ts, text) ` +
|
|
796
|
+
`VALUES ('-100', NULL, 1, 'user', 'alice', '111', ${now - 60}, 'pre-migration row')`,
|
|
797
|
+
)
|
|
798
|
+
old.close()
|
|
799
|
+
|
|
800
|
+
// Re-open through the real init path — the additive ALTER TABLE loop
|
|
801
|
+
// must add the forwarded_* columns without touching existing rows.
|
|
802
|
+
initHistory(stateDir, 30)
|
|
803
|
+
|
|
804
|
+
const oldRow = query({ chat_id: '-100' })[0]!
|
|
805
|
+
expect(oldRow.text).toBe('pre-migration row')
|
|
806
|
+
expect(oldRow.forwarded_from).toBeNull()
|
|
807
|
+
|
|
808
|
+
recordInbound({
|
|
809
|
+
chat_id: '-100',
|
|
810
|
+
thread_id: null,
|
|
811
|
+
message_id: 2,
|
|
812
|
+
user: 'alice',
|
|
813
|
+
user_id: '111',
|
|
814
|
+
ts: now,
|
|
815
|
+
text: 'forwarded after migration',
|
|
816
|
+
forwarded_from: 'Ada Lovelace (@adalove)',
|
|
817
|
+
forwarded_from_type: 'user',
|
|
818
|
+
forwarded_from_id: '42',
|
|
819
|
+
forwarded_date: '2026-06-15T13:46:40.000Z',
|
|
820
|
+
})
|
|
821
|
+
const rows = query({ chat_id: '-100' })
|
|
822
|
+
expect(rows).toHaveLength(2)
|
|
823
|
+
const fwd = rows.find((r) => r.message_id === 2)!
|
|
824
|
+
expect(fwd.forwarded_from).toBe('Ada Lovelace (@adalove)')
|
|
825
|
+
expect(fwd.forwarded_from_type).toBe('user')
|
|
826
|
+
expect(fwd.forwarded_from_id).toBe('42')
|
|
827
|
+
expect(fwd.forwarded_date).toBe('2026-06-15T13:46:40.000Z')
|
|
828
|
+
expect(fwd.forwarded_message_id).toBeNull()
|
|
829
|
+
})
|
|
830
|
+
|
|
831
|
+
it('hostile origin name is stored raw (XML metachars belong to the meta lane)', () => {
|
|
832
|
+
initHistory(stateDir, 30)
|
|
833
|
+
// Raw XML metacharacters are EXPECTED here — escaping belongs to the
|
|
834
|
+
// channel-meta lane, the history buffer stores what the user saw.
|
|
835
|
+
recordInbound({
|
|
836
|
+
chat_id: '-100',
|
|
837
|
+
thread_id: null,
|
|
838
|
+
message_id: 9,
|
|
839
|
+
user: 'alice',
|
|
840
|
+
user_id: '111',
|
|
841
|
+
ts: 1000,
|
|
842
|
+
text: 'fwd',
|
|
843
|
+
forwarded_from: '<b>"Bob"&\'friends\'</b>',
|
|
844
|
+
forwarded_from_type: 'user',
|
|
845
|
+
forwarded_from_id: '42',
|
|
846
|
+
})
|
|
847
|
+
expect(query({ chat_id: '-100' })[0]!.forwarded_from).toBe('<b>"Bob"&\'friends\'</b>')
|
|
848
|
+
})
|
|
849
|
+
|
|
850
|
+
it('masks a secret-shaped origin name before it is stored (redaction backstop)', () => {
|
|
851
|
+
initHistory(stateDir, 30)
|
|
852
|
+
// Built by concatenation so the source never holds a contiguous
|
|
853
|
+
// secret-shaped literal (repo Push Protection / no-pii lint). Same
|
|
854
|
+
// pattern as the text/reply_to_text redaction tests above — a display
|
|
855
|
+
// name is user-controlled text and rides the same redact() backstop.
|
|
856
|
+
const GH_PAT = `ghp_${'F6g7H8i9J0'.repeat(3)}` // ghp_<30 base62>
|
|
857
|
+
recordInbound({
|
|
858
|
+
chat_id: '-100',
|
|
859
|
+
thread_id: null,
|
|
860
|
+
message_id: 10,
|
|
861
|
+
user: 'alice',
|
|
862
|
+
user_id: '111',
|
|
863
|
+
ts: 1000,
|
|
864
|
+
text: 'fwd',
|
|
865
|
+
forwarded_from: `Bob ${GH_PAT}`,
|
|
866
|
+
forwarded_from_type: 'user',
|
|
867
|
+
forwarded_from_id: '42',
|
|
868
|
+
})
|
|
869
|
+
const stored = query({ chat_id: '-100' })[0]!.forwarded_from as string
|
|
870
|
+
expect(stored).not.toContain(GH_PAT)
|
|
871
|
+
expect(stored).toContain('[REDACTED')
|
|
872
|
+
expect(stored).toContain('Bob') // surrounding name preserved
|
|
873
|
+
})
|
|
874
|
+
})
|
|
@@ -14,10 +14,52 @@ import { describe, it, expect } from 'vitest'
|
|
|
14
14
|
import {
|
|
15
15
|
detectModelUnavailable,
|
|
16
16
|
formatModelUnavailableCard,
|
|
17
|
+
isLitellmProxyLocal429,
|
|
18
|
+
parseLitellmLimitDetail,
|
|
17
19
|
resolveModelUnavailableFromOperatorEvent,
|
|
18
20
|
type ModelUnavailableDetection,
|
|
19
21
|
} from '../model-unavailable.js'
|
|
20
22
|
|
|
23
|
+
// Real LiteLLM proxy-local 429 bodies, verbatim shapes from BerriAI/litellm
|
|
24
|
+
// source (see the provenance comment on `litellmProxyLocal429Signals`).
|
|
25
|
+
const LITELLM_DEPLOYMENT_CAP_BODY =
|
|
26
|
+
'Deployment over user-defined ratelimit. tpm limit=8000. current usage=8241. ' +
|
|
27
|
+
'id=abc123def, model_group=claude-fable-5'
|
|
28
|
+
const LITELLM_DEPLOYMENT_CAP_MESSAGE =
|
|
29
|
+
'litellm.RateLimitError: Model rate limit exceeded. TPM limit=8000, current usage=8241'
|
|
30
|
+
const LITELLM_V2_STRATEGY_BODY =
|
|
31
|
+
"Deployment over defined rpm limit=60. current usage=61. id=abc123def, " +
|
|
32
|
+
"model_group=claude-fable-5. Get the model info by calling 'router.get_model_info(id)"
|
|
33
|
+
const LITELLM_ROUTER_COOLDOWN_BODY =
|
|
34
|
+
'No deployments available for selected model, Try again in 27.5 seconds. ' +
|
|
35
|
+
"Passed model=claude-fable-5. pre-call-checks=False, cooldown_list=['abc123def']"
|
|
36
|
+
const LITELLM_V3_KEY_LIMIT_BODY =
|
|
37
|
+
'Rate limit exceeded for api_key: hashed-key-1a2b3c. Limit type: tokens. ' +
|
|
38
|
+
'Current limit: 8000, Remaining: 0. Limit resets at: 2026-07-12 08:05:00 UTC'
|
|
39
|
+
// v1 shape A — the ProxyRateLimitError detail (parallel_request_limiter.py
|
|
40
|
+
// ~line 124; the interpolated CommonProxyErrors.max_parallel_request_limit_
|
|
41
|
+
// reached.value is "Crossed TPM / RPM / Max Parallel Request Limit").
|
|
42
|
+
const LITELLM_V1_PARALLEL_BODY =
|
|
43
|
+
'LiteLLM Rate Limit Handler for rate limit type = requests. ' +
|
|
44
|
+
'Crossed TPM / RPM / Max Parallel Request Limit. ' +
|
|
45
|
+
'current rpm: 61, rpm limit: 60, current tpm: 100, tpm limit: 8000, ' +
|
|
46
|
+
'current max_parallel_requests: 1, max_parallel_requests: 10'
|
|
47
|
+
// v1 shape B — raise_rate_limit_error's zero-limit branch: the standalone
|
|
48
|
+
// "Max parallel request limit reached" prefix + additional_details
|
|
49
|
+
// (parallel_request_limiter.py ~lines 88 + 183).
|
|
50
|
+
const LITELLM_V1_ZERO_LIMIT_BODY =
|
|
51
|
+
'Max parallel request limit reached Crossed TPM / RPM / Max Parallel ' +
|
|
52
|
+
'Request Limit. Hit limit for tokens. Current limits: ' +
|
|
53
|
+
'max_parallel_requests: 10, tpm_limit: 0, rpm_limit: 60'
|
|
54
|
+
// v3 limiter with a descriptor OUTSIDE any enumerated list — covered by the
|
|
55
|
+
// litellmV3LimiterSignalPair co-occurrence rule (v3 descriptor keys also
|
|
56
|
+
// include organization / team_member / model_per_team / agent / tag_per_key
|
|
57
|
+
// / mcp_per_key and keep growing, so enumeration is a treadmill).
|
|
58
|
+
const LITELLM_V3_TEAM_MODEL_BODY =
|
|
59
|
+
'Rate limit exceeded for model_per_team: team-1a2b3c:claude-fable-5. ' +
|
|
60
|
+
'Limit type: tokens. Current limit: 50000, Remaining: 0. ' +
|
|
61
|
+
'Limit resets at: 2026-07-12 08:05:00 UTC'
|
|
62
|
+
|
|
21
63
|
// ─── detectModelUnavailable ──────────────────────────────────────────────────
|
|
22
64
|
|
|
23
65
|
describe('detectModelUnavailable — quota / billing strings', () => {
|
|
@@ -117,6 +159,151 @@ describe('detectModelUnavailable — transient upstream 429 vs account quota (#2
|
|
|
117
159
|
})
|
|
118
160
|
})
|
|
119
161
|
|
|
162
|
+
describe('detectModelUnavailable — LiteLLM-proxy-LOCAL 429s (never quota)', () => {
|
|
163
|
+
// A 429 raised by LiteLLM's OWN limiter never reached Anthropic — it must
|
|
164
|
+
// classify to the calm retryable kind, never quota_exhausted (which drives
|
|
165
|
+
// the scary card + mark-exhausted + fleet failover for an account that was
|
|
166
|
+
// never touched). Bodies are verbatim LiteLLM source shapes.
|
|
167
|
+
it.each([
|
|
168
|
+
['deployment tpm cap (model_rate_limit_check body)', LITELLM_DEPLOYMENT_CAP_BODY],
|
|
169
|
+
['deployment tpm cap (RateLimitError message)', LITELLM_DEPLOYMENT_CAP_MESSAGE],
|
|
170
|
+
['usage-based-routing v2 rpm wording', LITELLM_V2_STRATEGY_BODY],
|
|
171
|
+
['router cooldown (RouterRateLimitError)', LITELLM_ROUTER_COOLDOWN_BODY],
|
|
172
|
+
['virtual-key tpm_limit (parallel_request_limiter_v3)', LITELLM_V3_KEY_LIMIT_BODY],
|
|
173
|
+
['team model cap — non-enumerated v3 descriptor (co-occurrence rule)', LITELLM_V3_TEAM_MODEL_BODY],
|
|
174
|
+
['v1 parallel-request limiter (Rate Limit Handler detail)', LITELLM_V1_PARALLEL_BODY],
|
|
175
|
+
['v1 zero-limit branch (Max parallel request limit reached prefix)', LITELLM_V1_ZERO_LIMIT_BODY],
|
|
176
|
+
])('classifies %s as overload, NOT quota_exhausted', (_name, body) => {
|
|
177
|
+
const d = detectModelUnavailable(body)
|
|
178
|
+
expect(d?.kind).toBe('overload')
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
it('a rate-limited operator event with a LiteLLM-local body resolves to NO model-unavailable card', () => {
|
|
182
|
+
// End-to-end seam the gateway uses: resolveModelUnavailableFromOperatorEvent
|
|
183
|
+
// returning null is what keeps the calm 🚦 card AND blocks the
|
|
184
|
+
// auto-fallback branch (fallback only fires when a detection resolves).
|
|
185
|
+
for (const body of [
|
|
186
|
+
LITELLM_DEPLOYMENT_CAP_BODY,
|
|
187
|
+
LITELLM_V3_KEY_LIMIT_BODY,
|
|
188
|
+
LITELLM_ROUTER_COOLDOWN_BODY,
|
|
189
|
+
]) {
|
|
190
|
+
expect(
|
|
191
|
+
resolveModelUnavailableFromOperatorEvent({ kind: 'rate-limited', detail: body }),
|
|
192
|
+
).toBeNull()
|
|
193
|
+
}
|
|
194
|
+
})
|
|
195
|
+
})
|
|
196
|
+
|
|
197
|
+
describe('isLitellmProxyLocal429 — signal matching', () => {
|
|
198
|
+
it('matches every canonical LiteLLM limiter wording', () => {
|
|
199
|
+
for (const body of [
|
|
200
|
+
LITELLM_DEPLOYMENT_CAP_BODY,
|
|
201
|
+
LITELLM_DEPLOYMENT_CAP_MESSAGE,
|
|
202
|
+
LITELLM_V2_STRATEGY_BODY,
|
|
203
|
+
LITELLM_ROUTER_COOLDOWN_BODY,
|
|
204
|
+
LITELLM_V3_KEY_LIMIT_BODY,
|
|
205
|
+
LITELLM_V1_PARALLEL_BODY,
|
|
206
|
+
LITELLM_V1_ZERO_LIMIT_BODY,
|
|
207
|
+
]) {
|
|
208
|
+
expect(isLitellmProxyLocal429(body)).toBe(true)
|
|
209
|
+
}
|
|
210
|
+
})
|
|
211
|
+
|
|
212
|
+
it('matches ANY v3 descriptor via the co-occurrence pair, not an enumerated list', () => {
|
|
213
|
+
// model_per_team is not in litellmProxyLocal429Signals — only the
|
|
214
|
+
// "rate limit exceeded for " + "limit type:" pair catches it. Same for
|
|
215
|
+
// any descriptor litellm adds later.
|
|
216
|
+
expect(isLitellmProxyLocal429(LITELLM_V3_TEAM_MODEL_BODY)).toBe(true)
|
|
217
|
+
expect(
|
|
218
|
+
isLitellmProxyLocal429(
|
|
219
|
+
'Rate limit exceeded for organization: org-1a2b3c. Limit type: requests. ' +
|
|
220
|
+
'Current limit: 100, Remaining: 0. Limit resets at: 2026-07-12 08:05:00 UTC',
|
|
221
|
+
),
|
|
222
|
+
).toBe(true)
|
|
223
|
+
// HALF the pair is not enough — "rate limit exceeded for" prose without
|
|
224
|
+
// the v3 "Limit type:" field must not classify proxy-local.
|
|
225
|
+
expect(
|
|
226
|
+
isLitellmProxyLocal429('Rate limit exceeded for this account, try later'),
|
|
227
|
+
).toBe(false)
|
|
228
|
+
expect(isLitellmProxyLocal429('Limit type: tokens')).toBe(false)
|
|
229
|
+
})
|
|
230
|
+
|
|
231
|
+
it('does NOT match Anthropic account/wall/server wordings', () => {
|
|
232
|
+
expect(
|
|
233
|
+
isLitellmProxyLocal429(
|
|
234
|
+
"This request would exceed your account's rate limit. Please try again later.",
|
|
235
|
+
),
|
|
236
|
+
).toBe(false)
|
|
237
|
+
expect(
|
|
238
|
+
isLitellmProxyLocal429('Server is temporarily limiting requests (not your usage limit)'),
|
|
239
|
+
).toBe(false)
|
|
240
|
+
expect(isLitellmProxyLocal429("You've hit your limit · resets 8:50am")).toBe(false)
|
|
241
|
+
})
|
|
242
|
+
|
|
243
|
+
it('does NOT match the bare litellm.RateLimitError exception-mapping prefix', () => {
|
|
244
|
+
// The pass-through wraps FORWARDED upstream 429s with the same prefix —
|
|
245
|
+
// it is not evidence the limit was proxy-local.
|
|
246
|
+
expect(
|
|
247
|
+
isLitellmProxyLocal429(
|
|
248
|
+
"litellm.RateLimitError: RateLimitError: This request would exceed your account's rate limit.",
|
|
249
|
+
),
|
|
250
|
+
).toBe(false)
|
|
251
|
+
})
|
|
252
|
+
|
|
253
|
+
it('never throws on weird input', () => {
|
|
254
|
+
expect(isLitellmProxyLocal429('')).toBe(false)
|
|
255
|
+
expect(isLitellmProxyLocal429(undefined as unknown as string)).toBe(false)
|
|
256
|
+
expect(isLitellmProxyLocal429(42 as unknown as string)).toBe(false)
|
|
257
|
+
// Slice guard: signal past the 16KB sample is not scanned.
|
|
258
|
+
expect(isLitellmProxyLocal429('A'.repeat(100_000) + LITELLM_DEPLOYMENT_CAP_BODY)).toBe(false)
|
|
259
|
+
})
|
|
260
|
+
})
|
|
261
|
+
|
|
262
|
+
describe('parseLitellmLimitDetail — instrumentation extraction', () => {
|
|
263
|
+
const NOW = new Date(Date.UTC(2026, 6, 12, 8, 0, 0))
|
|
264
|
+
|
|
265
|
+
it('extracts tpm limit + current usage from the deployment-cap body', () => {
|
|
266
|
+
const d = parseLitellmLimitDetail(LITELLM_DEPLOYMENT_CAP_BODY, NOW)
|
|
267
|
+
expect(d.limitType).toBe('tpm')
|
|
268
|
+
expect(d.limit).toBe(8000)
|
|
269
|
+
expect(d.currentUsage).toBe(8241)
|
|
270
|
+
expect(d.resetAtMs).toBeNull()
|
|
271
|
+
})
|
|
272
|
+
|
|
273
|
+
it('extracts rpm limit from the v2 strategy body', () => {
|
|
274
|
+
const d = parseLitellmLimitDetail(LITELLM_V2_STRATEGY_BODY, NOW)
|
|
275
|
+
expect(d.limitType).toBe('rpm')
|
|
276
|
+
expect(d.limit).toBe(60)
|
|
277
|
+
expect(d.currentUsage).toBe(61)
|
|
278
|
+
})
|
|
279
|
+
|
|
280
|
+
it('extracts the v3 limiter limit type, limit, and "Limit resets at" UTC timestamp', () => {
|
|
281
|
+
const d = parseLitellmLimitDetail(LITELLM_V3_KEY_LIMIT_BODY, NOW)
|
|
282
|
+
expect(d.limitType).toBe('tokens')
|
|
283
|
+
expect(d.limit).toBe(8000)
|
|
284
|
+
expect(d.resetAtMs).toBe(Date.UTC(2026, 6, 12, 8, 5, 0))
|
|
285
|
+
})
|
|
286
|
+
|
|
287
|
+
it('extracts the router-cooldown "Try again in N seconds" relative reset', () => {
|
|
288
|
+
const d = parseLitellmLimitDetail(LITELLM_ROUTER_COOLDOWN_BODY, NOW)
|
|
289
|
+
expect(d.resetAtMs).toBe(NOW.getTime() + 27_500)
|
|
290
|
+
})
|
|
291
|
+
|
|
292
|
+
it('extracts the v1 limiter colon-form rpm limit', () => {
|
|
293
|
+
const d = parseLitellmLimitDetail(LITELLM_V1_PARALLEL_BODY, NOW)
|
|
294
|
+
expect(d.limitType).toBe('rpm')
|
|
295
|
+
expect(d.limit).toBe(60)
|
|
296
|
+
})
|
|
297
|
+
|
|
298
|
+
it('returns all-null on non-LiteLLM prose and never throws on weird input', () => {
|
|
299
|
+
const empty = { limitType: null, limit: null, currentUsage: null, resetAtMs: null }
|
|
300
|
+
expect(parseLitellmLimitDetail('Please try again later.', NOW)).toEqual(empty)
|
|
301
|
+
expect(parseLitellmLimitDetail('', NOW)).toEqual(empty)
|
|
302
|
+
expect(parseLitellmLimitDetail(undefined as unknown as string, NOW)).toEqual(empty)
|
|
303
|
+
expect(parseLitellmLimitDetail({} as unknown as string, NOW)).toEqual(empty)
|
|
304
|
+
})
|
|
305
|
+
})
|
|
306
|
+
|
|
120
307
|
describe('detectModelUnavailable — network failures', () => {
|
|
121
308
|
it('classifies ECONNREFUSED', () => {
|
|
122
309
|
expect(detectModelUnavailable('connect ECONNREFUSED 1.2.3.4:443')?.kind).toBe('network')
|
|
@@ -198,6 +198,61 @@ describe('detectErrorInTranscriptLine — error detection', () => {
|
|
|
198
198
|
expect(detection).toBeNull()
|
|
199
199
|
})
|
|
200
200
|
|
|
201
|
+
// A 429 whose body carries LiteLLM-proxy-LOCAL limiter wording is the
|
|
202
|
+
// proxy's own tpm_limit/rpm_limit cap tripping BEFORE the request reached
|
|
203
|
+
// Anthropic. Nothing about the account is exhausted — it must take the
|
|
204
|
+
// calm rate-limited path (no model-unavailable card, no mark-exhausted,
|
|
205
|
+
// no fleet failover). Prerequisite for enabling litellm tpm caps on the
|
|
206
|
+
// fleet: without this, every cap trip would bench a healthy account.
|
|
207
|
+
it('classifies a LiteLLM-proxy-local 429 as rate-limited, NOT quota-exhausted', () => {
|
|
208
|
+
const litellmBodies = [
|
|
209
|
+
// Deployment tpm cap (model_rate_limit_check.py body, verbatim shape).
|
|
210
|
+
'API Error: 429 Deployment over user-defined ratelimit. tpm limit=8000. ' +
|
|
211
|
+
'current usage=8241. id=abc123def, model_group=claude-fable-5',
|
|
212
|
+
// Virtual-key tpm_limit (parallel_request_limiter_v3.py).
|
|
213
|
+
'API Error: 429 Rate limit exceeded for api_key: hashed-key-1a2b3c. ' +
|
|
214
|
+
'Limit type: tokens. Current limit: 8000, Remaining: 0. ' +
|
|
215
|
+
'Limit resets at: 2026-07-12 08:05:00 UTC',
|
|
216
|
+
// Router cooldown (RouterRateLimitError).
|
|
217
|
+
'API Error: 429 No deployments available for selected model, ' +
|
|
218
|
+
'Try again in 27.5 seconds. Passed model=claude-fable-5.',
|
|
219
|
+
// Team-level model cap — a v3 descriptor OUTSIDE the enumerated signal
|
|
220
|
+
// list, covered only by the co-occurrence pair
|
|
221
|
+
// (litellmV3LimiterSignalPair). Pre-fix this missed every signal →
|
|
222
|
+
// quota-exhausted → scary card + failover.
|
|
223
|
+
'API Error: 429 Rate limit exceeded for model_per_team: ' +
|
|
224
|
+
'team-1a2b3c:claude-fable-5. Limit type: tokens. ' +
|
|
225
|
+
'Current limit: 50000, Remaining: 0. ' +
|
|
226
|
+
'Limit resets at: 2026-07-12 08:05:00 UTC',
|
|
227
|
+
]
|
|
228
|
+
for (const text of litellmBodies) {
|
|
229
|
+
const line = JSON.stringify({
|
|
230
|
+
type: 'assistant',
|
|
231
|
+
message: {
|
|
232
|
+
role: 'assistant',
|
|
233
|
+
model: '<synthetic>',
|
|
234
|
+
content: [{ type: 'text', text }],
|
|
235
|
+
},
|
|
236
|
+
error: 'rate_limit',
|
|
237
|
+
isApiErrorMessage: true,
|
|
238
|
+
apiErrorStatus: 429,
|
|
239
|
+
})
|
|
240
|
+
const result = detectErrorInTranscriptLine(line)
|
|
241
|
+
expect(result).not.toBeNull()
|
|
242
|
+
expect(result!.kind).toBe('rate-limited')
|
|
243
|
+
expect(result!.transient).toBe(true)
|
|
244
|
+
// End-to-end: the resolver must NOT produce a model-unavailable card —
|
|
245
|
+
// null is what keeps the calm 🚦 card and blocks the auto-fallback
|
|
246
|
+
// branch in emitGatewayOperatorEvent.
|
|
247
|
+
expect(
|
|
248
|
+
resolveModelUnavailableFromOperatorEvent({
|
|
249
|
+
kind: result!.kind,
|
|
250
|
+
detail: result!.detail,
|
|
251
|
+
}),
|
|
252
|
+
).toBeNull()
|
|
253
|
+
}
|
|
254
|
+
})
|
|
255
|
+
|
|
201
256
|
// Guard against over-correcting: a GENUINE quota wall (no transient marker)
|
|
202
257
|
// must STILL be quota-exhausted AND still resolve to a card.
|
|
203
258
|
it('a genuine quota-wall 429 still produces the quota-exhausted card', () => {
|
|
@@ -17,8 +17,10 @@ import { describe, it, expect } from "vitest";
|
|
|
17
17
|
import { renderOutboundChunks, PLAIN_TEXT_MAX_CHARS } from "../../render/rich-render.js";
|
|
18
18
|
import { RICH_MESSAGE_MAX_CHARS } from "../../format.js";
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
// Rendering is ON BY DEFAULT (escape hatch, not opt-in — mirrors the send
|
|
21
|
+
// gate): an empty env exercises the real default; "0" is the kill-switch.
|
|
22
|
+
const ON = {} as NodeJS.ProcessEnv;
|
|
23
|
+
const OFF = { SWITCHROOM_RICH_RENDER: "0" } as NodeJS.ProcessEnv;
|
|
22
24
|
|
|
23
25
|
/** Count fenced-code delimiter lines (```) in a body. A piece that bisects a
|
|
24
26
|
* fenced block has an ODD count. */
|
|
@@ -27,7 +29,7 @@ function fenceCount(s: string): number {
|
|
|
27
29
|
}
|
|
28
30
|
|
|
29
31
|
describe("renderOutboundChunks", () => {
|
|
30
|
-
it("
|
|
32
|
+
it("disabled (=0) is a single passthrough piece (byte-for-byte)", () => {
|
|
31
33
|
const raw = "**bold** and _italic_ | a | table |";
|
|
32
34
|
const pieces = renderOutboundChunks(raw, OFF);
|
|
33
35
|
expect(pieces).toHaveLength(1);
|
|
@@ -35,7 +37,7 @@ describe("renderOutboundChunks", () => {
|
|
|
35
37
|
expect(pieces[0].mode).toBe("markdown");
|
|
36
38
|
});
|
|
37
39
|
|
|
38
|
-
it("
|
|
40
|
+
it("default (env unset), body that fits is a single piece (common case)", () => {
|
|
39
41
|
const pieces = renderOutboundChunks("just some plain prose", ON);
|
|
40
42
|
expect(pieces).toHaveLength(1);
|
|
41
43
|
expect(pieces[0].text.length).toBeLessThanOrEqual(RICH_MESSAGE_MAX_CHARS);
|
|
@@ -7,24 +7,32 @@ import {
|
|
|
7
7
|
} from "../../render/rich-render.js";
|
|
8
8
|
|
|
9
9
|
describe("parseRichRenderEnabled", () => {
|
|
10
|
-
it("defaults
|
|
11
|
-
expect(parseRichRenderEnabled(undefined)).toBe(
|
|
10
|
+
it("defaults ON when unset (escape hatch, not opt-in)", () => {
|
|
11
|
+
expect(parseRichRenderEnabled(undefined)).toBe(true);
|
|
12
12
|
});
|
|
13
|
-
it("
|
|
13
|
+
it("disabled only by the explicit off tokens (case-insensitive, trimmed)", () => {
|
|
14
|
+
for (const v of ["0", "false", "off", "no", "FALSE", " Off ", "NO"]) {
|
|
15
|
+
expect(parseRichRenderEnabled(v)).toBe(false);
|
|
16
|
+
}
|
|
17
|
+
});
|
|
18
|
+
it("truthy tokens stay ON", () => {
|
|
14
19
|
for (const v of ["1", "true", "on", "yes", "TRUE", " On "]) {
|
|
15
20
|
expect(parseRichRenderEnabled(v)).toBe(true);
|
|
16
21
|
}
|
|
17
22
|
});
|
|
18
|
-
it("
|
|
19
|
-
for (const v of ["
|
|
20
|
-
expect(parseRichRenderEnabled(v)).toBe(
|
|
23
|
+
it("empty / unrecognised values stay ON (fail-open to the default)", () => {
|
|
24
|
+
for (const v of ["", " ", "maybe", "2", "disable"]) {
|
|
25
|
+
expect(parseRichRenderEnabled(v)).toBe(true);
|
|
21
26
|
}
|
|
22
27
|
});
|
|
23
28
|
});
|
|
24
29
|
|
|
25
30
|
describe("richRenderEnabled", () => {
|
|
26
|
-
it("reads SWITCHROOM_RICH_RENDER, default
|
|
27
|
-
expect(richRenderEnabled({} as NodeJS.ProcessEnv)).toBe(
|
|
31
|
+
it("reads SWITCHROOM_RICH_RENDER, default ON", () => {
|
|
32
|
+
expect(richRenderEnabled({} as NodeJS.ProcessEnv)).toBe(true);
|
|
33
|
+
expect(
|
|
34
|
+
richRenderEnabled({ SWITCHROOM_RICH_RENDER: "0" } as NodeJS.ProcessEnv),
|
|
35
|
+
).toBe(false);
|
|
28
36
|
expect(
|
|
29
37
|
richRenderEnabled({ SWITCHROOM_RICH_RENDER: "1" } as NodeJS.ProcessEnv),
|
|
30
38
|
).toBe(true);
|
|
@@ -32,41 +40,52 @@ describe("richRenderEnabled", () => {
|
|
|
32
40
|
});
|
|
33
41
|
|
|
34
42
|
describe("maybeRenderOutbound", () => {
|
|
35
|
-
it("
|
|
43
|
+
it("disabled (=0) is a byte-for-byte passthrough (the escape hatch)", () => {
|
|
36
44
|
const raw = "**bold** and a\n\n**> collapsible quote\n> second line";
|
|
37
|
-
const r = maybeRenderOutbound(raw, {
|
|
45
|
+
const r = maybeRenderOutbound(raw, {
|
|
46
|
+
SWITCHROOM_RICH_RENDER: "0",
|
|
47
|
+
} as NodeJS.ProcessEnv);
|
|
38
48
|
expect(r.mode).toBe("markdown");
|
|
39
49
|
expect(r.text).toBe(raw);
|
|
40
50
|
});
|
|
41
51
|
|
|
42
|
-
it("
|
|
43
|
-
const r = maybeRenderOutbound("**> collapsible", {
|
|
44
|
-
SWITCHROOM_RICH_RENDER: "1",
|
|
45
|
-
} as NodeJS.ProcessEnv);
|
|
52
|
+
it("default (env unset) routes through parse -> renderSafe", () => {
|
|
53
|
+
const r = maybeRenderOutbound("**> collapsible", {} as NodeJS.ProcessEnv);
|
|
46
54
|
expect(r.mode).toBe("markdown");
|
|
47
55
|
// The expandable blockquote round-trips back to the `**> ` marker.
|
|
48
56
|
expect(r.text).toContain("**> ");
|
|
49
57
|
});
|
|
50
58
|
|
|
51
|
-
it("
|
|
52
|
-
const r = maybeRenderOutbound(
|
|
53
|
-
|
|
54
|
-
|
|
59
|
+
it("default preserves plain prose through the round-trip", () => {
|
|
60
|
+
const r = maybeRenderOutbound(
|
|
61
|
+
"just some plain text",
|
|
62
|
+
{} as NodeJS.ProcessEnv,
|
|
63
|
+
);
|
|
55
64
|
expect(r.text).toContain("just some plain text");
|
|
56
65
|
});
|
|
57
66
|
|
|
58
|
-
it("
|
|
59
|
-
const on = {
|
|
67
|
+
it("default round-trips underline / spoiler / highlight", () => {
|
|
68
|
+
const on = {} as NodeJS.ProcessEnv;
|
|
60
69
|
expect(maybeRenderOutbound("__u__", on).text).toBe("__u__");
|
|
61
70
|
expect(maybeRenderOutbound("a ||s|| b", on).text).toBe("a ||s|| b");
|
|
62
71
|
expect(maybeRenderOutbound("a ==m== b", on).text).toBe("a ==m== b");
|
|
63
72
|
});
|
|
64
73
|
|
|
65
|
-
it("
|
|
74
|
+
it("disabled (=0) passes new constructs through untouched too", () => {
|
|
66
75
|
const raw = "__u__ and ||s|| and ==m==";
|
|
67
|
-
const r = maybeRenderOutbound(raw, {
|
|
76
|
+
const r = maybeRenderOutbound(raw, {
|
|
77
|
+
SWITCHROOM_RICH_RENDER: "0",
|
|
78
|
+
} as NodeJS.ProcessEnv);
|
|
68
79
|
expect(r.text).toBe(raw);
|
|
69
80
|
});
|
|
81
|
+
|
|
82
|
+
it("a junk env value still renders (fail-open to the default)", () => {
|
|
83
|
+
const r = maybeRenderOutbound("**> collapsible", {
|
|
84
|
+
SWITCHROOM_RICH_RENDER: "maybe",
|
|
85
|
+
} as NodeJS.ProcessEnv);
|
|
86
|
+
expect(r.mode).toBe("markdown");
|
|
87
|
+
expect(r.text).toContain("**> ");
|
|
88
|
+
});
|
|
70
89
|
});
|
|
71
90
|
|
|
72
91
|
describe("renderOutbound (flag-independent)", () => {
|
|
@@ -91,6 +91,30 @@ describe('runtime-metrics — JSONL sink', () => {
|
|
|
91
91
|
expect(typeof parsed.ts).toBe('number')
|
|
92
92
|
})
|
|
93
93
|
|
|
94
|
+
it('rate_limit_429_classified carries classification + action + limit/reset detail', () => {
|
|
95
|
+
emitRuntimeMetric({
|
|
96
|
+
kind: 'rate_limit_429_classified',
|
|
97
|
+
agent: 'carrie',
|
|
98
|
+
classification: 'litellm-local',
|
|
99
|
+
action: 'calm',
|
|
100
|
+
reset_at_ms: 1_783_850_700_000,
|
|
101
|
+
reset_in_ms: 300_000,
|
|
102
|
+
limit_type: 'tpm',
|
|
103
|
+
limit: 8000,
|
|
104
|
+
current_usage: 8241,
|
|
105
|
+
})
|
|
106
|
+
const parsed = JSON.parse(readFileSync(metricsPath, 'utf-8').trim())
|
|
107
|
+
expect(parsed.kind).toBe('rate_limit_429_classified')
|
|
108
|
+
expect(parsed.agent).toBe('carrie')
|
|
109
|
+
expect(parsed.classification).toBe('litellm-local')
|
|
110
|
+
expect(parsed.action).toBe('calm')
|
|
111
|
+
expect(parsed.reset_in_ms).toBe(300_000)
|
|
112
|
+
expect(parsed.limit_type).toBe('tpm')
|
|
113
|
+
expect(parsed.limit).toBe(8000)
|
|
114
|
+
expect(parsed.current_usage).toBe(8241)
|
|
115
|
+
expect(typeof parsed.ts).toBe('number')
|
|
116
|
+
})
|
|
117
|
+
|
|
94
118
|
it('appends — does not overwrite — across calls', () => {
|
|
95
119
|
for (let i = 0; i < 5; i++) {
|
|
96
120
|
emitRuntimeMetric({
|
|
@@ -69,7 +69,7 @@ describe('stream-reply single-mode reuse (#2669)', () => {
|
|
|
69
69
|
expect(bot.api.editMessageText).toHaveBeenCalled()
|
|
70
70
|
})
|
|
71
71
|
|
|
72
|
-
it('the rich path ships
|
|
72
|
+
it('the rich path ships GFM markdown unescaped via sendRichMessage', async () => {
|
|
73
73
|
const state = makeState()
|
|
74
74
|
const deps = makeDeps(bot)
|
|
75
75
|
|
|
@@ -77,8 +77,10 @@ describe('stream-reply single-mode reuse (#2669)', () => {
|
|
|
77
77
|
|
|
78
78
|
expect(bot.state.sent).toHaveLength(1)
|
|
79
79
|
expect(bot.state.sent[0].rich).toBe(true)
|
|
80
|
-
//
|
|
81
|
-
|
|
80
|
+
// GFM markdown is the wire payload — no HTML, no MarkdownV2 escaping.
|
|
81
|
+
// The default-on rich renderer normalises the italic marker
|
|
82
|
+
// (`_italic_` -> `*italic*`, same wire entity).
|
|
83
|
+
expect(bot.state.sent[0].text).toBe('**bold** and *italic*')
|
|
82
84
|
expect(bot.state.sent[0].parse_mode).toBeUndefined()
|
|
83
85
|
})
|
|
84
86
|
|
|
@@ -91,8 +91,10 @@ describe('handleStreamReply accent integration', () => {
|
|
|
91
91
|
await pending
|
|
92
92
|
|
|
93
93
|
const sent = sentMarkdown(bot)
|
|
94
|
-
|
|
95
|
-
|
|
94
|
+
// The default-on rich renderer normalises the italic marker
|
|
95
|
+
// (`_In progress…_` -> `*In progress…*`, same wire entity).
|
|
96
|
+
expect(sent).toMatch(/^🔵 \*In progress…\*\n\n/)
|
|
97
|
+
expect(sent).toBe('🔵 *In progress…*\n\nStill working...')
|
|
96
98
|
})
|
|
97
99
|
|
|
98
100
|
it("accent='done' prepends the checkmark markdown header before the body", async () => {
|
|
@@ -176,6 +178,6 @@ describe('handleStreamReply accent integration', () => {
|
|
|
176
178
|
await microtaskFlush()
|
|
177
179
|
await p2
|
|
178
180
|
|
|
179
|
-
expect(editedMarkdown(bot)).toBe('🔵
|
|
181
|
+
expect(editedMarkdown(bot)).toBe('🔵 *In progress…*\n\nPart one Part two')
|
|
180
182
|
})
|
|
181
183
|
})
|