switchroom 0.18.14 → 0.18.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-broker/index.js +41 -39
- package/dist/cli/switchroom.js +1151 -1067
- package/dist/host-control/main.js +53 -51
- package/dist/vault/approvals/kernel-server.js +16 -12
- package/dist/vault/broker/server.js +672 -668
- package/package.json +1 -1
- package/telegram-plugin/dist/bridge/bridge.js +21 -0
- package/telegram-plugin/dist/gateway/gateway.js +150 -5
- package/telegram-plugin/dist/server.js +22 -1
- package/telegram-plugin/gateway/gateway.ts +48 -2
- package/telegram-plugin/model-unavailable.ts +214 -0
- package/telegram-plugin/runtime-metrics.ts +31 -0
- package/telegram-plugin/session-tail.ts +14 -2
- package/telegram-plugin/tests/model-unavailable.test.ts +187 -0
- package/telegram-plugin/tests/operator-events-session-tail.test.ts +55 -0
- package/telegram-plugin/tests/runtime-metrics.test.ts +24 -0
- package/telegram-plugin/tests/throttle-tier.test.ts +176 -0
- package/telegram-plugin/throttle-tier.ts +98 -1
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "switchroom",
|
|
3
3
|
"//version": "NOT the release version — source of truth is the git tag, resolved by scripts/build.mjs:resolveVersion() (see CLAUDE.md > Standard release process). This field is stale by design and only the Layer-4 dev/non-tag fallback for build.mjs + src/cli/resolve-version.ts; do NOT bump it expecting a release to pick it up. npm-pack tarball naming needs a real version — do that as an UNCOMMITTED pack-time bump (see release step 6), never a committed one.",
|
|
4
|
-
"version": "0.18.
|
|
4
|
+
"version": "0.18.15",
|
|
5
5
|
"description": "Run Claude Code 24/7 on your Claude Pro/Max subscription over Telegram. Open-source alternative to OpenClaw and NanoClaw — no API keys.",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"bin": {
|
|
@@ -23061,6 +23061,26 @@ function isTransientUpstreamSignal(text) {
|
|
|
23061
23061
|
const lower = sample.toLowerCase();
|
|
23062
23062
|
return transientUpstreamSignals.some((s) => lower.includes(s));
|
|
23063
23063
|
}
|
|
23064
|
+
var litellmProxyLocal429Signals = [
|
|
23065
|
+
"deployment over user-defined ratelimit",
|
|
23066
|
+
"model rate limit exceeded. tpm limit",
|
|
23067
|
+
"model rate limit exceeded. rpm limit",
|
|
23068
|
+
"deployment over defined rpm limit",
|
|
23069
|
+
"no deployments available for selected model",
|
|
23070
|
+
"litellm rate limit handler",
|
|
23071
|
+
"crossed tpm / rpm",
|
|
23072
|
+
"max parallel request limit reached"
|
|
23073
|
+
];
|
|
23074
|
+
var litellmV3LimiterSignalPair = ["rate limit exceeded for ", "limit type:"];
|
|
23075
|
+
function isLitellmProxyLocal429(text) {
|
|
23076
|
+
if (typeof text !== "string" || text.length === 0)
|
|
23077
|
+
return false;
|
|
23078
|
+
const sample = text.length > 16384 ? text.slice(0, 16384) : text;
|
|
23079
|
+
const lower = sample.toLowerCase();
|
|
23080
|
+
if (litellmProxyLocal429Signals.some((s) => lower.includes(s)))
|
|
23081
|
+
return true;
|
|
23082
|
+
return litellmV3LimiterSignalPair.every((s) => lower.includes(s));
|
|
23083
|
+
}
|
|
23064
23084
|
|
|
23065
23085
|
// tool-label-sidecar.ts
|
|
23066
23086
|
import { existsSync as existsSync2, readFileSync, statSync as statSync2 } from "node:fs";
|
|
@@ -23481,6 +23501,7 @@ function detectErrorInTranscriptLine(line) {
|
|
|
23481
23501
|
const errStr = typeof obj.error === "string" ? obj.error : "";
|
|
23482
23502
|
const text = extractAssistantText(obj);
|
|
23483
23503
|
const kind2 = status === 429 ? isTransientUpstreamSignal(`${text}
|
|
23504
|
+
${errStr}`) || isLitellmProxyLocal429(`${text}
|
|
23484
23505
|
${errStr}`) ? "rate-limited" : "quota-exhausted" : classifyClaudeError({ type: errStr, status, message: text });
|
|
23485
23506
|
return {
|
|
23486
23507
|
kind: kind2,
|
|
@@ -64640,6 +64640,26 @@ var transientUpstreamSignals = [
|
|
|
64640
64640
|
"would exceed your account\u2019s rate limit",
|
|
64641
64641
|
"would exceed your account's rate limit"
|
|
64642
64642
|
];
|
|
64643
|
+
var litellmProxyLocal429Signals = [
|
|
64644
|
+
"deployment over user-defined ratelimit",
|
|
64645
|
+
"model rate limit exceeded. tpm limit",
|
|
64646
|
+
"model rate limit exceeded. rpm limit",
|
|
64647
|
+
"deployment over defined rpm limit",
|
|
64648
|
+
"no deployments available for selected model",
|
|
64649
|
+
"litellm rate limit handler",
|
|
64650
|
+
"crossed tpm / rpm",
|
|
64651
|
+
"max parallel request limit reached"
|
|
64652
|
+
];
|
|
64653
|
+
var litellmV3LimiterSignalPair = ["rate limit exceeded for ", "limit type:"];
|
|
64654
|
+
function isLitellmProxyLocal429(text4) {
|
|
64655
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
64656
|
+
return false;
|
|
64657
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
64658
|
+
const lower = sample.toLowerCase();
|
|
64659
|
+
if (litellmProxyLocal429Signals.some((s) => lower.includes(s)))
|
|
64660
|
+
return true;
|
|
64661
|
+
return litellmV3LimiterSignalPair.every((s) => lower.includes(s));
|
|
64662
|
+
}
|
|
64643
64663
|
function detectModelUnavailable(stderr) {
|
|
64644
64664
|
if (typeof stderr !== "string" || stderr.length === 0)
|
|
64645
64665
|
return null;
|
|
@@ -64649,6 +64669,10 @@ function detectModelUnavailable(stderr) {
|
|
|
64649
64669
|
const resetAt = parseResetTime(sample);
|
|
64650
64670
|
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
64651
64671
|
}
|
|
64672
|
+
if (isLitellmProxyLocal429(sample)) {
|
|
64673
|
+
const resetAt = parseResetTime(sample);
|
|
64674
|
+
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
64675
|
+
}
|
|
64652
64676
|
const quotaSignals = [
|
|
64653
64677
|
"out of extra usage",
|
|
64654
64678
|
"extra usage",
|
|
@@ -64898,6 +64922,76 @@ function isTransientUpstreamSignal(text4) {
|
|
|
64898
64922
|
const lower = sample.toLowerCase();
|
|
64899
64923
|
return transientUpstreamSignals2.some((s) => lower.includes(s));
|
|
64900
64924
|
}
|
|
64925
|
+
var litellmProxyLocal429Signals2 = [
|
|
64926
|
+
"deployment over user-defined ratelimit",
|
|
64927
|
+
"model rate limit exceeded. tpm limit",
|
|
64928
|
+
"model rate limit exceeded. rpm limit",
|
|
64929
|
+
"deployment over defined rpm limit",
|
|
64930
|
+
"no deployments available for selected model",
|
|
64931
|
+
"litellm rate limit handler",
|
|
64932
|
+
"crossed tpm / rpm",
|
|
64933
|
+
"max parallel request limit reached"
|
|
64934
|
+
];
|
|
64935
|
+
var litellmV3LimiterSignalPair2 = ["rate limit exceeded for ", "limit type:"];
|
|
64936
|
+
function isLitellmProxyLocal4292(text4) {
|
|
64937
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
64938
|
+
return false;
|
|
64939
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
64940
|
+
const lower = sample.toLowerCase();
|
|
64941
|
+
if (litellmProxyLocal429Signals2.some((s) => lower.includes(s)))
|
|
64942
|
+
return true;
|
|
64943
|
+
return litellmV3LimiterSignalPair2.every((s) => lower.includes(s));
|
|
64944
|
+
}
|
|
64945
|
+
function parseLitellmLimitDetail(text4, parseTimeNow = new Date) {
|
|
64946
|
+
const empty2 = { limitType: null, limit: null, currentUsage: null, resetAtMs: null };
|
|
64947
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
64948
|
+
return empty2;
|
|
64949
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
64950
|
+
const lower = sample.toLowerCase();
|
|
64951
|
+
let limitType = null;
|
|
64952
|
+
let limit = null;
|
|
64953
|
+
const eqLimit = lower.match(/\b([tr]pm)[ _]limit[=:]\s*(\d+)/);
|
|
64954
|
+
if (eqLimit) {
|
|
64955
|
+
limitType = eqLimit[1];
|
|
64956
|
+
limit = Number(eqLimit[2]);
|
|
64957
|
+
}
|
|
64958
|
+
if (limitType == null) {
|
|
64959
|
+
const v3Type = lower.match(/limit type:\s*(tokens|requests|max_parallel_requests)/);
|
|
64960
|
+
if (v3Type)
|
|
64961
|
+
limitType = v3Type[1];
|
|
64962
|
+
}
|
|
64963
|
+
if (limit == null) {
|
|
64964
|
+
const v3Limit = lower.match(/current limit:\s*(\d+)/);
|
|
64965
|
+
if (v3Limit)
|
|
64966
|
+
limit = Number(v3Limit[1]);
|
|
64967
|
+
}
|
|
64968
|
+
let currentUsage = null;
|
|
64969
|
+
const usage = lower.match(/current usage=(\d+)/);
|
|
64970
|
+
if (usage)
|
|
64971
|
+
currentUsage = Number(usage[1]);
|
|
64972
|
+
let resetAtMs = null;
|
|
64973
|
+
const resetsAt = sample.match(/limit resets at:\s*(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) UTC/i);
|
|
64974
|
+
if (resetsAt) {
|
|
64975
|
+
const d = new Date(`${resetsAt[1]}T${resetsAt[2]}Z`);
|
|
64976
|
+
if (!Number.isNaN(d.getTime()))
|
|
64977
|
+
resetAtMs = d.getTime();
|
|
64978
|
+
}
|
|
64979
|
+
if (resetAtMs == null) {
|
|
64980
|
+
const tryAgain = lower.match(/try again in\s+(\d+(?:\.\d+)?)\s*seconds/);
|
|
64981
|
+
if (tryAgain) {
|
|
64982
|
+
const secs = Number(tryAgain[1]);
|
|
64983
|
+
if (Number.isFinite(secs) && secs > 0 && secs < 7 * 24 * 3600) {
|
|
64984
|
+
resetAtMs = parseTimeNow.getTime() + Math.round(secs * 1000);
|
|
64985
|
+
}
|
|
64986
|
+
}
|
|
64987
|
+
}
|
|
64988
|
+
return {
|
|
64989
|
+
limitType,
|
|
64990
|
+
limit: limit != null && Number.isFinite(limit) ? limit : null,
|
|
64991
|
+
currentUsage: currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
|
|
64992
|
+
resetAtMs
|
|
64993
|
+
};
|
|
64994
|
+
}
|
|
64901
64995
|
function parseResetTime2(text4, parseTimeNow = new Date) {
|
|
64902
64996
|
const lower = text4.toLowerCase();
|
|
64903
64997
|
const retryAfter = lower.match(/retry[\s-]*after[:\s]+(\d+)\s*(seconds?|s\b|minutes?|m\b|hours?|h\b)?/);
|
|
@@ -65034,6 +65128,30 @@ function isAccountScopedThrottle(text4) {
|
|
|
65034
65128
|
const lower = sample.toLowerCase();
|
|
65035
65129
|
return accountScopedThrottleSignals.some((s) => lower.includes(s));
|
|
65036
65130
|
}
|
|
65131
|
+
function classify429Detail(text4) {
|
|
65132
|
+
if (isAccountScopedThrottle(text4))
|
|
65133
|
+
return "account-scoped";
|
|
65134
|
+
if (isLitellmProxyLocal4292(text4))
|
|
65135
|
+
return "litellm-local";
|
|
65136
|
+
return "generic-transient";
|
|
65137
|
+
}
|
|
65138
|
+
function build429ClassifiedMetric(opts) {
|
|
65139
|
+
const detail = typeof opts.detail === "string" ? opts.detail : "";
|
|
65140
|
+
const litellm = parseLitellmLimitDetail(detail, new Date(opts.now));
|
|
65141
|
+
const anthropicResetMs = parseResetTime2(detail, new Date(opts.now))?.getTime() ?? null;
|
|
65142
|
+
const resetAtMs = anthropicResetMs ?? litellm.resetAtMs;
|
|
65143
|
+
return {
|
|
65144
|
+
kind: "rate_limit_429_classified",
|
|
65145
|
+
agent: opts.agent,
|
|
65146
|
+
classification: opts.classification,
|
|
65147
|
+
action: opts.action,
|
|
65148
|
+
reset_at_ms: resetAtMs,
|
|
65149
|
+
reset_in_ms: resetAtMs != null ? Math.max(0, resetAtMs - opts.now) : null,
|
|
65150
|
+
limit_type: litellm.limitType,
|
|
65151
|
+
limit: litellm.limit,
|
|
65152
|
+
current_usage: litellm.currentUsage
|
|
65153
|
+
};
|
|
65154
|
+
}
|
|
65037
65155
|
var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT = 5 * 60000;
|
|
65038
65156
|
var THROTTLE_DEFAULT_WAIT_MS = 60000;
|
|
65039
65157
|
function throttleRetryInPlaceMaxMs(env = process.env) {
|
|
@@ -69426,6 +69544,11 @@ function extractConfirmation(pane) {
|
|
|
69426
69544
|
// ../src/agents/scaffold.ts
|
|
69427
69545
|
import { join as join32, resolve as resolve6 } from "node:path";
|
|
69428
69546
|
init_atomic();
|
|
69547
|
+
|
|
69548
|
+
// ../src/agents/agent-uid.ts
|
|
69549
|
+
init_peercred();
|
|
69550
|
+
|
|
69551
|
+
// ../src/agents/scaffold.ts
|
|
69429
69552
|
init_schema();
|
|
69430
69553
|
|
|
69431
69554
|
// ../src/config/users.ts
|
|
@@ -77097,6 +77220,7 @@ function detectErrorInTranscriptLine(line) {
|
|
|
77097
77220
|
const errStr = typeof obj.error === "string" ? obj.error : "";
|
|
77098
77221
|
const text4 = extractAssistantText(obj);
|
|
77099
77222
|
const kind2 = status === 429 ? isTransientUpstreamSignal(`${text4}
|
|
77223
|
+
${errStr}`) || isLitellmProxyLocal4292(`${text4}
|
|
77100
77224
|
${errStr}`) ? "rate-limited" : "quota-exhausted" : classifyClaudeError({ type: errStr, status, message: text4 });
|
|
77101
77225
|
return {
|
|
77102
77226
|
kind: kind2,
|
|
@@ -80957,10 +81081,10 @@ function readTurnActiveMarkerAgeMs(stateDir, now) {
|
|
|
80957
81081
|
}
|
|
80958
81082
|
|
|
80959
81083
|
// ../src/build-info.ts
|
|
80960
|
-
var VERSION = "0.18.
|
|
80961
|
-
var COMMIT_SHA = "
|
|
80962
|
-
var COMMIT_DATE = "2026-07-
|
|
80963
|
-
var LATEST_PR =
|
|
81084
|
+
var VERSION = "0.18.15";
|
|
81085
|
+
var COMMIT_SHA = "2fa611a1";
|
|
81086
|
+
var COMMIT_DATE = "2026-07-12T04:44:56Z";
|
|
81087
|
+
var LATEST_PR = 3171;
|
|
80964
81088
|
var COMMITS_AHEAD_OF_TAG = 0;
|
|
80965
81089
|
|
|
80966
81090
|
// gateway/boot-version.ts
|
|
@@ -85647,12 +85771,33 @@ function emitGatewayOperatorEvent(event) {
|
|
|
85647
85771
|
const { agent, kind } = event;
|
|
85648
85772
|
let throttleEscalation = null;
|
|
85649
85773
|
let escalationFired = false;
|
|
85650
|
-
|
|
85774
|
+
const rateLimit429Classification = kind === "rate-limited" ? classify429Detail(event.detail) : null;
|
|
85775
|
+
if (rateLimit429Classification != null && rateLimit429Classification !== "account-scoped") {
|
|
85776
|
+
emitRuntimeMetric(build429ClassifiedMetric({
|
|
85777
|
+
agent,
|
|
85778
|
+
detail: event.detail,
|
|
85779
|
+
classification: rateLimit429Classification,
|
|
85780
|
+
action: "calm",
|
|
85781
|
+
now: Date.now()
|
|
85782
|
+
}));
|
|
85783
|
+
if (rateLimit429Classification === "litellm-local") {
|
|
85784
|
+
process.stderr.write(`telegram gateway: 429 classified litellm-proxy-local agent=${agent} \u2014 ` + `calm path, no account attribution, no failover
|
|
85785
|
+
`);
|
|
85786
|
+
}
|
|
85787
|
+
}
|
|
85788
|
+
if (rateLimit429Classification === "account-scoped") {
|
|
85651
85789
|
const throttleDecision = decideThrottleTier({
|
|
85652
85790
|
detail: event.detail,
|
|
85653
85791
|
now: Date.now(),
|
|
85654
85792
|
thresholdMs: throttleRetryInPlaceMaxMs()
|
|
85655
85793
|
});
|
|
85794
|
+
emitRuntimeMetric(build429ClassifiedMetric({
|
|
85795
|
+
agent,
|
|
85796
|
+
detail: event.detail,
|
|
85797
|
+
classification: "account-scoped",
|
|
85798
|
+
action: throttleDecision.action === "failover" ? "failover" : "throttle",
|
|
85799
|
+
now: Date.now()
|
|
85800
|
+
}));
|
|
85656
85801
|
if (throttleDecision.action === "throttle") {
|
|
85657
85802
|
process.stderr.write(`telegram gateway: throttle-tier staying-put agent=${agent} until=${new Date(throttleDecision.throttledUntilMs).toISOString()} parsedReset=${throttleDecision.resetParsed}
|
|
85658
85803
|
`);
|
|
@@ -17088,7 +17088,16 @@ function isTransientUpstreamSignal(text) {
|
|
|
17088
17088
|
const lower = sample.toLowerCase();
|
|
17089
17089
|
return transientUpstreamSignals.some((s) => lower.includes(s));
|
|
17090
17090
|
}
|
|
17091
|
-
|
|
17091
|
+
function isLitellmProxyLocal429(text) {
|
|
17092
|
+
if (typeof text !== "string" || text.length === 0)
|
|
17093
|
+
return false;
|
|
17094
|
+
const sample = text.length > 16384 ? text.slice(0, 16384) : text;
|
|
17095
|
+
const lower = sample.toLowerCase();
|
|
17096
|
+
if (litellmProxyLocal429Signals.some((s) => lower.includes(s)))
|
|
17097
|
+
return true;
|
|
17098
|
+
return litellmV3LimiterSignalPair.every((s) => lower.includes(s));
|
|
17099
|
+
}
|
|
17100
|
+
var transientUpstreamSignals, litellmProxyLocal429Signals, litellmV3LimiterSignalPair;
|
|
17092
17101
|
var init_model_unavailable = __esm(() => {
|
|
17093
17102
|
init_quota_check();
|
|
17094
17103
|
init_card_format();
|
|
@@ -17102,6 +17111,17 @@ var init_model_unavailable = __esm(() => {
|
|
|
17102
17111
|
"would exceed your account\u2019s rate limit",
|
|
17103
17112
|
"would exceed your account's rate limit"
|
|
17104
17113
|
];
|
|
17114
|
+
litellmProxyLocal429Signals = [
|
|
17115
|
+
"deployment over user-defined ratelimit",
|
|
17116
|
+
"model rate limit exceeded. tpm limit",
|
|
17117
|
+
"model rate limit exceeded. rpm limit",
|
|
17118
|
+
"deployment over defined rpm limit",
|
|
17119
|
+
"no deployments available for selected model",
|
|
17120
|
+
"litellm rate limit handler",
|
|
17121
|
+
"crossed tpm / rpm",
|
|
17122
|
+
"max parallel request limit reached"
|
|
17123
|
+
];
|
|
17124
|
+
litellmV3LimiterSignalPair = ["rate limit exceeded for ", "limit type:"];
|
|
17105
17125
|
});
|
|
17106
17126
|
|
|
17107
17127
|
// tool-label-sidecar.ts
|
|
@@ -17534,6 +17554,7 @@ function detectErrorInTranscriptLine(line) {
|
|
|
17534
17554
|
const errStr = typeof obj.error === "string" ? obj.error : "";
|
|
17535
17555
|
const text = extractAssistantText(obj);
|
|
17536
17556
|
const kind2 = status === 429 ? isTransientUpstreamSignal(`${text}
|
|
17557
|
+
${errStr}`) || isLitellmProxyLocal429(`${text}
|
|
17537
17558
|
${errStr}`) ? "rate-limited" : "quota-exhausted" : classifyClaudeError({ type: errStr, status, message: text });
|
|
17538
17559
|
return {
|
|
17539
17560
|
kind: kind2,
|
|
@@ -320,8 +320,9 @@ import {
|
|
|
320
320
|
type ModelUnavailableDetection,
|
|
321
321
|
} from '../model-unavailable.js'
|
|
322
322
|
import {
|
|
323
|
+
build429ClassifiedMetric,
|
|
324
|
+
classify429Detail,
|
|
323
325
|
decideThrottleTier,
|
|
324
|
-
isAccountScopedThrottle,
|
|
325
326
|
throttleRetryInPlaceMaxMs,
|
|
326
327
|
} from '../throttle-tier.js'
|
|
327
328
|
import { createThrottleTierRunner } from './throttle-tier-wiring.js'
|
|
@@ -7700,14 +7701,59 @@ function emitGatewayOperatorEvent(event: OperatorEvent): void {
|
|
|
7700
7701
|
// through to the existing calm rate-limited card unchanged — an
|
|
7701
7702
|
// account-scoped throttle would be the wrong action for a server-wide
|
|
7702
7703
|
// condition.
|
|
7704
|
+
//
|
|
7705
|
+
// Classification is three-way (classify429Detail, throttle-tier.ts):
|
|
7706
|
+
// only `account-scoped` may enter the throttle tier. `litellm-local` —
|
|
7707
|
+
// the LiteLLM proxy's OWN tpm/rpm/router limiter tripping before the
|
|
7708
|
+
// request reached Anthropic — takes the calm path with NO broker mark and
|
|
7709
|
+
// NO failover: the condition is proxy-local and says nothing about the
|
|
7710
|
+
// account. Every rate-limited event ALSO emits one
|
|
7711
|
+
// `rate_limit_429_classified` runtime metric (PostHog + JSONL), fired
|
|
7712
|
+
// here — before the cooldown gate — so operators can correlate
|
|
7713
|
+
// account-scoped 429s with fleet TPM even when the card is suppressed.
|
|
7703
7714
|
let throttleEscalation: ModelUnavailableDetection | null = null
|
|
7704
7715
|
let escalationFired = false
|
|
7705
|
-
|
|
7716
|
+
const rateLimit429Classification =
|
|
7717
|
+
kind === 'rate-limited' ? classify429Detail(event.detail) : null
|
|
7718
|
+
if (rateLimit429Classification != null && rateLimit429Classification !== 'account-scoped') {
|
|
7719
|
+
// litellm-local / generic-transient: the existing calm rate-limited
|
|
7720
|
+
// path (fall through to renderOperatorEvent below). NO broker
|
|
7721
|
+
// mark-throttled, NO throttle-tier runner, NO failover — for a
|
|
7722
|
+
// proxy-local cap trip those would bench an account that was never
|
|
7723
|
+
// touched.
|
|
7724
|
+
emitRuntimeMetric(
|
|
7725
|
+
build429ClassifiedMetric({
|
|
7726
|
+
agent,
|
|
7727
|
+
detail: event.detail,
|
|
7728
|
+
classification: rateLimit429Classification,
|
|
7729
|
+
action: 'calm',
|
|
7730
|
+
now: Date.now(),
|
|
7731
|
+
}),
|
|
7732
|
+
)
|
|
7733
|
+
if (rateLimit429Classification === 'litellm-local') {
|
|
7734
|
+
process.stderr.write(
|
|
7735
|
+
`telegram gateway: 429 classified litellm-proxy-local agent=${agent} — ` +
|
|
7736
|
+
`calm path, no account attribution, no failover\n`,
|
|
7737
|
+
)
|
|
7738
|
+
}
|
|
7739
|
+
}
|
|
7740
|
+
if (rateLimit429Classification === 'account-scoped') {
|
|
7706
7741
|
const throttleDecision = decideThrottleTier({
|
|
7707
7742
|
detail: event.detail,
|
|
7708
7743
|
now: Date.now(),
|
|
7709
7744
|
thresholdMs: throttleRetryInPlaceMaxMs(),
|
|
7710
7745
|
})
|
|
7746
|
+
emitRuntimeMetric(
|
|
7747
|
+
build429ClassifiedMetric({
|
|
7748
|
+
agent,
|
|
7749
|
+
detail: event.detail,
|
|
7750
|
+
classification: 'account-scoped',
|
|
7751
|
+
// decideThrottleTier can't return 'none' for account-scoped wording;
|
|
7752
|
+
// map the two live actions onto the metric's vocabulary.
|
|
7753
|
+
action: throttleDecision.action === 'failover' ? 'failover' : 'throttle',
|
|
7754
|
+
now: Date.now(),
|
|
7755
|
+
}),
|
|
7756
|
+
)
|
|
7711
7757
|
if (throttleDecision.action === 'throttle') {
|
|
7712
7758
|
// Reset is near (≤ threshold) or unparseable (60s default): DO NOT
|
|
7713
7759
|
// fail over. Record throttled_until broker-side, post ONE lightweight
|
|
@@ -92,6 +92,203 @@ export function isTransientUpstreamSignal(text: string): boolean {
|
|
|
92
92
|
return transientUpstreamSignals.some(s => lower.includes(s))
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
+
// ─── LiteLLM-proxy-LOCAL 429 signals (canonical, single source of truth) ─────
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Explicit markers of a 429 generated by the LiteLLM proxy's OWN rate
|
|
99
|
+
* limiters — a router `tpm_limit`/`rpm_limit` deployment cap, a virtual-key /
|
|
100
|
+
* team / user cap, or an all-deployments-cooling-down router state. The
|
|
101
|
+
* request never reached Anthropic, so the condition is PROXY-LOCAL: it says
|
|
102
|
+
* nothing about the Anthropic account's quota and must never mark the account
|
|
103
|
+
* throttled/exhausted or trigger fleet failover.
|
|
104
|
+
*
|
|
105
|
+
* Each wording key is grounded in LiteLLM's source (verified against
|
|
106
|
+
* BerriAI/litellm `main`, 2026-07), matching how
|
|
107
|
+
* `accountScopedThrottleSignals` (throttle-tier.ts) documents its keys:
|
|
108
|
+
*
|
|
109
|
+
* - "deployment over user-defined ratelimit" —
|
|
110
|
+
* `RouterErrors.user_defined_ratelimit_error` (litellm/types/router.py),
|
|
111
|
+
* the body prefix of every deployment tpm/rpm cap 429 raised by
|
|
112
|
+
* litellm/router_utils/pre_call_checks/model_rate_limit_check.py:
|
|
113
|
+
* "Deployment over user-defined ratelimit. tpm limit={n}. current
|
|
114
|
+
* usage={n}. id={id}, model_group={g}".
|
|
115
|
+
* - "model rate limit exceeded. tpm/rpm limit" — the `message` field of the
|
|
116
|
+
* same enforcement RateLimitError: "Model rate limit exceeded. TPM
|
|
117
|
+
* limit={n}, current usage={n}" (model_rate_limit_check.py).
|
|
118
|
+
* - "deployment over defined rpm limit" — the usage-based-routing-v2
|
|
119
|
+
* strategy wording "Deployment over defined rpm limit={n}. current
|
|
120
|
+
* usage={n}" (litellm/router_strategy/lowest_tpm_rpm_v2.py). There is NO
|
|
121
|
+
* tpm sibling in any known release — v2 TPM exhaustion surfaces as the
|
|
122
|
+
* `no_deployments_available` wording below (the deployment is filtered
|
|
123
|
+
* from the candidate set rather than erroring in the increment path).
|
|
124
|
+
* - "no deployments available for selected model" —
|
|
125
|
+
* `RouterErrors.no_deployments_available`, the RouterRateLimitError
|
|
126
|
+
* message when every deployment for the model group is cooling down:
|
|
127
|
+
* "No deployments available for selected model, Try again in {n}
|
|
128
|
+
* seconds…" (litellm/types/router.py).
|
|
129
|
+
* - "litellm rate limit handler" — the v1 parallel-request limiter's
|
|
130
|
+
* ProxyRateLimitError detail prefix: "LiteLLM Rate Limit Handler for
|
|
131
|
+
* rate limit type = {t}. Crossed TPM / RPM / Max Parallel Request
|
|
132
|
+
* Limit. current rpm: {n}, rpm limit: {n}…"
|
|
133
|
+
* (litellm/proxy/hooks/parallel_request_limiter.py).
|
|
134
|
+
* - "crossed tpm / rpm" — `CommonProxyErrors
|
|
135
|
+
* .max_parallel_request_limit_reached.value` = "Crossed TPM / RPM / Max
|
|
136
|
+
* Parallel Request Limit" (litellm/proxy/_types.py), interpolated into
|
|
137
|
+
* BOTH v1 detail shapes, so it also covers the zero-limit branch's
|
|
138
|
+
* "Max parallel request limit reached {additional_details}" message.
|
|
139
|
+
* - "max parallel request limit reached" — the standalone prefix
|
|
140
|
+
* `raise_rate_limit_error` builds when a key's limit is set to 0
|
|
141
|
+
* (parallel_request_limiter.py `error_message`).
|
|
142
|
+
*
|
|
143
|
+
* The proxy-side per-key/team/user limiter (parallel_request_limiter_v3.py
|
|
144
|
+
* `_handle_rate_limit_error`) is matched by CO-OCCURRENCE instead of a
|
|
145
|
+
* per-descriptor entry — see `isLitellmProxyLocal429`. Its detail shape:
|
|
146
|
+
* "Rate limit exceeded for {descriptor}: {value}. Limit type: {t}. Current
|
|
147
|
+
* limit: {n}, Remaining: {n}. Limit resets at: {ts}". Descriptor keys in
|
|
148
|
+
* source are numerous and growing (api_key / user / team / team_member /
|
|
149
|
+
* organization / end_user / agent / agent_session / model_per_key /
|
|
150
|
+
* model_per_team / model_per_organization / model_per_project / tag_per_key
|
|
151
|
+
* / mcp_per_key / mcp_per_team as of 2026-07), so enumerating them is a
|
|
152
|
+
* treadmill: the pair "rate limit exceeded for " + "limit type:" appears in
|
|
153
|
+
* every v3 body and in no Anthropic error. This is the shape a `tpm_limit`
|
|
154
|
+
* on the per-agent virtual keys (src/litellm/provision.ts) trips.
|
|
155
|
+
*
|
|
156
|
+
* Deliberately EXCLUDED: the bare exception-mapping prefix
|
|
157
|
+
* "litellm.RateLimitError:". LiteLLM wraps FORWARDED upstream 429s with the
|
|
158
|
+
* same prefix on the pass-through, so its presence is NOT evidence the limit
|
|
159
|
+
* was proxy-local — a genuine Anthropic account throttle traversing the
|
|
160
|
+
* proxy must keep its account-scoped classification (see
|
|
161
|
+
* `classify429Detail` in throttle-tier.ts for the tie-break).
|
|
162
|
+
*/
|
|
163
|
+
export const litellmProxyLocal429Signals = [
|
|
164
|
+
'deployment over user-defined ratelimit',
|
|
165
|
+
'model rate limit exceeded. tpm limit',
|
|
166
|
+
'model rate limit exceeded. rpm limit',
|
|
167
|
+
'deployment over defined rpm limit',
|
|
168
|
+
'no deployments available for selected model',
|
|
169
|
+
'litellm rate limit handler',
|
|
170
|
+
'crossed tpm / rpm',
|
|
171
|
+
'max parallel request limit reached',
|
|
172
|
+
]
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* The v3 proxy-limiter co-occurrence pair (see the provenance comment on
|
|
176
|
+
* `litellmProxyLocal429Signals`): both substrings appear in every
|
|
177
|
+
* parallel_request_limiter_v3 429 body regardless of descriptor key, and
|
|
178
|
+
* never in an Anthropic error. Exported so tests can pin the rule.
|
|
179
|
+
*/
|
|
180
|
+
export const litellmV3LimiterSignalPair = ['rate limit exceeded for ', 'limit type:'] as const
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* True when `text` carries an EXPLICIT LiteLLM-proxy-local rate-limit marker:
|
|
184
|
+
* one of `litellmProxyLocal429Signals`, OR the v3 limiter co-occurrence pair
|
|
185
|
+
* (`litellmV3LimiterSignalPair` — descriptor-agnostic, so new v3 descriptor
|
|
186
|
+
* keys are covered without a list update). Never throws on weird input. Pure
|
|
187
|
+
* wording detection only — precedence against account-scoped wording is
|
|
188
|
+
* owned by `classify429Detail` (throttle-tier.ts).
|
|
189
|
+
*/
|
|
190
|
+
export function isLitellmProxyLocal429(text: string): boolean {
|
|
191
|
+
if (typeof text !== 'string' || text.length === 0) return false
|
|
192
|
+
const sample = text.length > 16_384 ? text.slice(0, 16_384) : text
|
|
193
|
+
const lower = sample.toLowerCase()
|
|
194
|
+
if (litellmProxyLocal429Signals.some(s => lower.includes(s))) return true
|
|
195
|
+
return litellmV3LimiterSignalPair.every(s => lower.includes(s))
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Best-effort extraction of the limit detail LiteLLM embeds in its
|
|
200
|
+
* proxy-local 429 bodies, for instrumentation (the `rate_limit_429_classified`
|
|
201
|
+
* runtime metric). All fields null when nothing parseable is found — never
|
|
202
|
+
* throws. Shapes covered (same provenance as `litellmProxyLocal429Signals`):
|
|
203
|
+
*
|
|
204
|
+
* - "tpm limit=8000. current usage=8241" (model_rate_limit_check body)
|
|
205
|
+
* - "TPM limit=8000, current usage=8241" (model_rate_limit_check message)
|
|
206
|
+
* - "Deployment over defined rpm limit=60. current usage=61" (v2 strategy)
|
|
207
|
+
* - "Limit type: tokens. Current limit: 8000 … Limit resets at:
|
|
208
|
+
* 2026-07-12 08:05:00 UTC" (parallel_request_limiter_v3)
|
|
209
|
+
* - "Try again in 27.5 seconds" (RouterRateLimitError cooldown)
|
|
210
|
+
*
|
|
211
|
+
* CAVEAT — the v3 "Limit resets at" timestamp is NOT reliably UTC despite
|
|
212
|
+
* the literal: litellm formats it with a naive `datetime.fromtimestamp(...)
|
|
213
|
+
* .strftime("%Y-%m-%d %H:%M:%S UTC")` (parallel_request_limiter_v3.py
|
|
214
|
+
* `_handle_rate_limit_error`), i.e. proxy-LOCAL wall-clock time with a
|
|
215
|
+
* hard-coded "UTC" suffix. We parse it as UTC (nothing better is possible
|
|
216
|
+
* from the body alone), so `resetAtMs` — and the metric fields derived from
|
|
217
|
+
* it — can be skewed by the proxy host's UTC offset when the proxy doesn't
|
|
218
|
+
* run on UTC. Treat v3-derived resets as approximate; don't chase phantom
|
|
219
|
+
* clock drift from these metrics. The rate-limit windows are ≤1 minute
|
|
220
|
+
* anyway, so the absolute timestamp is informational.
|
|
221
|
+
*/
|
|
222
|
+
export function parseLitellmLimitDetail(
|
|
223
|
+
text: string,
|
|
224
|
+
parseTimeNow: Date = new Date(),
|
|
225
|
+
): {
|
|
226
|
+
limitType: string | null
|
|
227
|
+
limit: number | null
|
|
228
|
+
currentUsage: number | null
|
|
229
|
+
resetAtMs: number | null
|
|
230
|
+
} {
|
|
231
|
+
const empty = { limitType: null, limit: null, currentUsage: null, resetAtMs: null }
|
|
232
|
+
if (typeof text !== 'string' || text.length === 0) return empty
|
|
233
|
+
const sample = text.length > 16_384 ? text.slice(0, 16_384) : text
|
|
234
|
+
const lower = sample.toLowerCase()
|
|
235
|
+
|
|
236
|
+
// "tpm limit=8000" / "rpm limit=60" (body), "TPM limit=8000" (message),
|
|
237
|
+
// and the v1 limiter's colon form "rpm limit: 60".
|
|
238
|
+
let limitType: string | null = null
|
|
239
|
+
let limit: number | null = null
|
|
240
|
+
const eqLimit = lower.match(/\b([tr]pm)[ _]limit[=:]\s*(\d+)/)
|
|
241
|
+
if (eqLimit) {
|
|
242
|
+
limitType = eqLimit[1]
|
|
243
|
+
limit = Number(eqLimit[2])
|
|
244
|
+
}
|
|
245
|
+
// v3 limiter: "Limit type: tokens. Current limit: 8000"
|
|
246
|
+
if (limitType == null) {
|
|
247
|
+
const v3Type = lower.match(/limit type:\s*(tokens|requests|max_parallel_requests)/)
|
|
248
|
+
if (v3Type) limitType = v3Type[1]
|
|
249
|
+
}
|
|
250
|
+
if (limit == null) {
|
|
251
|
+
const v3Limit = lower.match(/current limit:\s*(\d+)/)
|
|
252
|
+
if (v3Limit) limit = Number(v3Limit[1])
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// "current usage=8241" (body/message, both `=` and `= ` never occur with
|
|
256
|
+
// separators other than the literal shown in source).
|
|
257
|
+
let currentUsage: number | null = null
|
|
258
|
+
const usage = lower.match(/current usage=(\d+)/)
|
|
259
|
+
if (usage) currentUsage = Number(usage[1])
|
|
260
|
+
|
|
261
|
+
// Reset hints LiteLLM emits that the Anthropic-shaped parseResetTime does
|
|
262
|
+
// not cover: "Limit resets at: 2026-07-12 08:05:00 UTC" (v3 limiter — note
|
|
263
|
+
// the space separator, not ISO 'T') and "Try again in 27.5 seconds"
|
|
264
|
+
// (RouterRateLimitError).
|
|
265
|
+
let resetAtMs: number | null = null
|
|
266
|
+
const resetsAt = sample.match(
|
|
267
|
+
/limit resets at:\s*(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) UTC/i,
|
|
268
|
+
)
|
|
269
|
+
if (resetsAt) {
|
|
270
|
+
const d = new Date(`${resetsAt[1]}T${resetsAt[2]}Z`)
|
|
271
|
+
if (!Number.isNaN(d.getTime())) resetAtMs = d.getTime()
|
|
272
|
+
}
|
|
273
|
+
if (resetAtMs == null) {
|
|
274
|
+
const tryAgain = lower.match(/try again in\s+(\d+(?:\.\d+)?)\s*seconds/)
|
|
275
|
+
if (tryAgain) {
|
|
276
|
+
const secs = Number(tryAgain[1])
|
|
277
|
+
if (Number.isFinite(secs) && secs > 0 && secs < 7 * 24 * 3600) {
|
|
278
|
+
resetAtMs = parseTimeNow.getTime() + Math.round(secs * 1000)
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
return {
|
|
284
|
+
limitType,
|
|
285
|
+
limit: limit != null && Number.isFinite(limit) ? limit : null,
|
|
286
|
+
currentUsage:
|
|
287
|
+
currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
|
|
288
|
+
resetAtMs,
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
95
292
|
// ─── Detection ───────────────────────────────────────────────────────────────
|
|
96
293
|
|
|
97
294
|
/**
|
|
@@ -138,6 +335,23 @@ export function detectModelUnavailable(
|
|
|
138
335
|
: { kind: 'overload', raw: stderr }
|
|
139
336
|
}
|
|
140
337
|
|
|
338
|
+
// ── 0.5. LiteLLM-proxy-LOCAL 429 — never a quota signal ─────────────────
|
|
339
|
+
// A 429 raised by the LiteLLM proxy's own limiters (`tpm_limit`/`rpm_limit`
|
|
340
|
+
// caps, router cooldown) never reached Anthropic — nothing about the
|
|
341
|
+
// account is exhausted, so it must classify to the calm retryable kind
|
|
342
|
+
// BEFORE the quota substrings run (some LiteLLM bodies contain the word
|
|
343
|
+
// "limit", which a negation-blind quota match could seize on). Runs after
|
|
344
|
+
// step 0 deliberately: account-affirming wording, when present, can only
|
|
345
|
+
// have originated upstream (LiteLLM never emits it) and wins — see the
|
|
346
|
+
// tie-break note on `classify429Detail` (throttle-tier.ts). Uses the full
|
|
347
|
+
// matcher (list + v3 co-occurrence pair) so every descriptor is covered.
|
|
348
|
+
if (isLitellmProxyLocal429(sample)) {
|
|
349
|
+
const resetAt = parseResetTime(sample)
|
|
350
|
+
return resetAt !== undefined
|
|
351
|
+
? { kind: 'overload', resetAt, raw: stderr }
|
|
352
|
+
: { kind: 'overload', raw: stderr }
|
|
353
|
+
}
|
|
354
|
+
|
|
141
355
|
// ── 1. Quota / billing exhaustion ──────────────────────────────────────
|
|
142
356
|
const quotaSignals = [
|
|
143
357
|
'out of extra usage',
|
|
@@ -164,6 +164,37 @@ export type RuntimeMetricEvent =
|
|
|
164
164
|
agent: string
|
|
165
165
|
prompt_key: string
|
|
166
166
|
}
|
|
167
|
+
/**
|
|
168
|
+
* Every terminal rate-limit-family operator event (429 burst / 529
|
|
169
|
+
* overload), classified by origin BEFORE the gateway acts on it — fires
|
|
170
|
+
* even when the user-facing card is cooldown-suppressed, so the count is
|
|
171
|
+
* honest. `classification` says where the limit lives (see
|
|
172
|
+
* `RateLimit429Classification` in throttle-tier.ts): `account-scoped` =
|
|
173
|
+
* Anthropic throttled the account (throttle tier ran), `litellm-local` =
|
|
174
|
+
* the LiteLLM proxy's own tpm/rpm/router limiter tripped (calm path, no
|
|
175
|
+
* account attribution), `generic-transient` = other server-side 429/529
|
|
176
|
+
* wording. `action` is what the gateway DECIDED (throttle / failover /
|
|
177
|
+
* calm), emitted PRE-execution: the actual failover fire is dedup-gated
|
|
178
|
+
* downstream (fleetFallbackGate), so N long-reset 429s inside one dedup
|
|
179
|
+
* window emit N `action: 'failover'` metrics for ONE real roll — count
|
|
180
|
+
* decisions here, count rolls via the broker/fallback announcements.
|
|
181
|
+
* Correlating `account-scoped` fires against fleet token throughput is
|
|
182
|
+
* the operator's evidence base for setting LiteLLM `tpm_limit` caps; the
|
|
183
|
+
* `litellm-local` count then shows those caps actually absorbing load.
|
|
184
|
+
* Limit/reset fields are best-effort parses of the error body (null when
|
|
185
|
+
* absent).
|
|
186
|
+
*/
|
|
187
|
+
| {
|
|
188
|
+
kind: 'rate_limit_429_classified'
|
|
189
|
+
agent: string
|
|
190
|
+
classification: 'account-scoped' | 'litellm-local' | 'generic-transient'
|
|
191
|
+
action: 'throttle' | 'failover' | 'calm'
|
|
192
|
+
reset_at_ms: number | null
|
|
193
|
+
reset_in_ms: number | null
|
|
194
|
+
limit_type: string | null
|
|
195
|
+
limit: number | null
|
|
196
|
+
current_usage: number | null
|
|
197
|
+
}
|
|
167
198
|
|
|
168
199
|
/**
|
|
169
200
|
* The JSONL sink lives under the runtime state dir so it's per-agent
|