switchroom 0.18.14 → 0.18.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "switchroom",
3
3
  "//version": "NOT the release version — source of truth is the git tag, resolved by scripts/build.mjs:resolveVersion() (see CLAUDE.md > Standard release process). This field is stale by design and only the Layer-4 dev/non-tag fallback for build.mjs + src/cli/resolve-version.ts; do NOT bump it expecting a release to pick it up. npm-pack tarball naming needs a real version — do that as an UNCOMMITTED pack-time bump (see release step 6), never a committed one.",
4
- "version": "0.18.14",
4
+ "version": "0.18.15",
5
5
  "description": "Run Claude Code 24/7 on your Claude Pro/Max subscription over Telegram. Open-source alternative to OpenClaw and NanoClaw — no API keys.",
6
6
  "type": "module",
7
7
  "bin": {
@@ -23061,6 +23061,26 @@ function isTransientUpstreamSignal(text) {
23061
23061
  const lower = sample.toLowerCase();
23062
23062
  return transientUpstreamSignals.some((s) => lower.includes(s));
23063
23063
  }
23064
+ var litellmProxyLocal429Signals = [
23065
+ "deployment over user-defined ratelimit",
23066
+ "model rate limit exceeded. tpm limit",
23067
+ "model rate limit exceeded. rpm limit",
23068
+ "deployment over defined rpm limit",
23069
+ "no deployments available for selected model",
23070
+ "litellm rate limit handler",
23071
+ "crossed tpm / rpm",
23072
+ "max parallel request limit reached"
23073
+ ];
23074
+ var litellmV3LimiterSignalPair = ["rate limit exceeded for ", "limit type:"];
23075
+ function isLitellmProxyLocal429(text) {
23076
+ if (typeof text !== "string" || text.length === 0)
23077
+ return false;
23078
+ const sample = text.length > 16384 ? text.slice(0, 16384) : text;
23079
+ const lower = sample.toLowerCase();
23080
+ if (litellmProxyLocal429Signals.some((s) => lower.includes(s)))
23081
+ return true;
23082
+ return litellmV3LimiterSignalPair.every((s) => lower.includes(s));
23083
+ }
23064
23084
 
23065
23085
  // tool-label-sidecar.ts
23066
23086
  import { existsSync as existsSync2, readFileSync, statSync as statSync2 } from "node:fs";
@@ -23481,6 +23501,7 @@ function detectErrorInTranscriptLine(line) {
23481
23501
  const errStr = typeof obj.error === "string" ? obj.error : "";
23482
23502
  const text = extractAssistantText(obj);
23483
23503
  const kind2 = status === 429 ? isTransientUpstreamSignal(`${text}
23504
+ ${errStr}`) || isLitellmProxyLocal429(`${text}
23484
23505
  ${errStr}`) ? "rate-limited" : "quota-exhausted" : classifyClaudeError({ type: errStr, status, message: text });
23485
23506
  return {
23486
23507
  kind: kind2,
@@ -64640,6 +64640,26 @@ var transientUpstreamSignals = [
64640
64640
  "would exceed your account\u2019s rate limit",
64641
64641
  "would exceed your account's rate limit"
64642
64642
  ];
64643
+ var litellmProxyLocal429Signals = [
64644
+ "deployment over user-defined ratelimit",
64645
+ "model rate limit exceeded. tpm limit",
64646
+ "model rate limit exceeded. rpm limit",
64647
+ "deployment over defined rpm limit",
64648
+ "no deployments available for selected model",
64649
+ "litellm rate limit handler",
64650
+ "crossed tpm / rpm",
64651
+ "max parallel request limit reached"
64652
+ ];
64653
+ var litellmV3LimiterSignalPair = ["rate limit exceeded for ", "limit type:"];
64654
+ function isLitellmProxyLocal429(text4) {
64655
+ if (typeof text4 !== "string" || text4.length === 0)
64656
+ return false;
64657
+ const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
64658
+ const lower = sample.toLowerCase();
64659
+ if (litellmProxyLocal429Signals.some((s) => lower.includes(s)))
64660
+ return true;
64661
+ return litellmV3LimiterSignalPair.every((s) => lower.includes(s));
64662
+ }
64643
64663
  function detectModelUnavailable(stderr) {
64644
64664
  if (typeof stderr !== "string" || stderr.length === 0)
64645
64665
  return null;
@@ -64649,6 +64669,10 @@ function detectModelUnavailable(stderr) {
64649
64669
  const resetAt = parseResetTime(sample);
64650
64670
  return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
64651
64671
  }
64672
+ if (isLitellmProxyLocal429(sample)) {
64673
+ const resetAt = parseResetTime(sample);
64674
+ return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
64675
+ }
64652
64676
  const quotaSignals = [
64653
64677
  "out of extra usage",
64654
64678
  "extra usage",
@@ -64898,6 +64922,76 @@ function isTransientUpstreamSignal(text4) {
64898
64922
  const lower = sample.toLowerCase();
64899
64923
  return transientUpstreamSignals2.some((s) => lower.includes(s));
64900
64924
  }
64925
+ var litellmProxyLocal429Signals2 = [
64926
+ "deployment over user-defined ratelimit",
64927
+ "model rate limit exceeded. tpm limit",
64928
+ "model rate limit exceeded. rpm limit",
64929
+ "deployment over defined rpm limit",
64930
+ "no deployments available for selected model",
64931
+ "litellm rate limit handler",
64932
+ "crossed tpm / rpm",
64933
+ "max parallel request limit reached"
64934
+ ];
64935
+ var litellmV3LimiterSignalPair2 = ["rate limit exceeded for ", "limit type:"];
64936
+ function isLitellmProxyLocal4292(text4) {
64937
+ if (typeof text4 !== "string" || text4.length === 0)
64938
+ return false;
64939
+ const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
64940
+ const lower = sample.toLowerCase();
64941
+ if (litellmProxyLocal429Signals2.some((s) => lower.includes(s)))
64942
+ return true;
64943
+ return litellmV3LimiterSignalPair2.every((s) => lower.includes(s));
64944
+ }
64945
+ function parseLitellmLimitDetail(text4, parseTimeNow = new Date) {
64946
+ const empty2 = { limitType: null, limit: null, currentUsage: null, resetAtMs: null };
64947
+ if (typeof text4 !== "string" || text4.length === 0)
64948
+ return empty2;
64949
+ const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
64950
+ const lower = sample.toLowerCase();
64951
+ let limitType = null;
64952
+ let limit = null;
64953
+ const eqLimit = lower.match(/\b([tr]pm)[ _]limit[=:]\s*(\d+)/);
64954
+ if (eqLimit) {
64955
+ limitType = eqLimit[1];
64956
+ limit = Number(eqLimit[2]);
64957
+ }
64958
+ if (limitType == null) {
64959
+ const v3Type = lower.match(/limit type:\s*(tokens|requests|max_parallel_requests)/);
64960
+ if (v3Type)
64961
+ limitType = v3Type[1];
64962
+ }
64963
+ if (limit == null) {
64964
+ const v3Limit = lower.match(/current limit:\s*(\d+)/);
64965
+ if (v3Limit)
64966
+ limit = Number(v3Limit[1]);
64967
+ }
64968
+ let currentUsage = null;
64969
+ const usage = lower.match(/current usage=(\d+)/);
64970
+ if (usage)
64971
+ currentUsage = Number(usage[1]);
64972
+ let resetAtMs = null;
64973
+ const resetsAt = sample.match(/limit resets at:\s*(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) UTC/i);
64974
+ if (resetsAt) {
64975
+ const d = new Date(`${resetsAt[1]}T${resetsAt[2]}Z`);
64976
+ if (!Number.isNaN(d.getTime()))
64977
+ resetAtMs = d.getTime();
64978
+ }
64979
+ if (resetAtMs == null) {
64980
+ const tryAgain = lower.match(/try again in\s+(\d+(?:\.\d+)?)\s*seconds/);
64981
+ if (tryAgain) {
64982
+ const secs = Number(tryAgain[1]);
64983
+ if (Number.isFinite(secs) && secs > 0 && secs < 7 * 24 * 3600) {
64984
+ resetAtMs = parseTimeNow.getTime() + Math.round(secs * 1000);
64985
+ }
64986
+ }
64987
+ }
64988
+ return {
64989
+ limitType,
64990
+ limit: limit != null && Number.isFinite(limit) ? limit : null,
64991
+ currentUsage: currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
64992
+ resetAtMs
64993
+ };
64994
+ }
64901
64995
  function parseResetTime2(text4, parseTimeNow = new Date) {
64902
64996
  const lower = text4.toLowerCase();
64903
64997
  const retryAfter = lower.match(/retry[\s-]*after[:\s]+(\d+)\s*(seconds?|s\b|minutes?|m\b|hours?|h\b)?/);
@@ -65034,6 +65128,30 @@ function isAccountScopedThrottle(text4) {
65034
65128
  const lower = sample.toLowerCase();
65035
65129
  return accountScopedThrottleSignals.some((s) => lower.includes(s));
65036
65130
  }
65131
+ function classify429Detail(text4) {
65132
+ if (isAccountScopedThrottle(text4))
65133
+ return "account-scoped";
65134
+ if (isLitellmProxyLocal4292(text4))
65135
+ return "litellm-local";
65136
+ return "generic-transient";
65137
+ }
65138
+ function build429ClassifiedMetric(opts) {
65139
+ const detail = typeof opts.detail === "string" ? opts.detail : "";
65140
+ const litellm = parseLitellmLimitDetail(detail, new Date(opts.now));
65141
+ const anthropicResetMs = parseResetTime2(detail, new Date(opts.now))?.getTime() ?? null;
65142
+ const resetAtMs = anthropicResetMs ?? litellm.resetAtMs;
65143
+ return {
65144
+ kind: "rate_limit_429_classified",
65145
+ agent: opts.agent,
65146
+ classification: opts.classification,
65147
+ action: opts.action,
65148
+ reset_at_ms: resetAtMs,
65149
+ reset_in_ms: resetAtMs != null ? Math.max(0, resetAtMs - opts.now) : null,
65150
+ limit_type: litellm.limitType,
65151
+ limit: litellm.limit,
65152
+ current_usage: litellm.currentUsage
65153
+ };
65154
+ }
65037
65155
  var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT = 5 * 60000;
65038
65156
  var THROTTLE_DEFAULT_WAIT_MS = 60000;
65039
65157
  function throttleRetryInPlaceMaxMs(env = process.env) {
@@ -69426,6 +69544,11 @@ function extractConfirmation(pane) {
69426
69544
  // ../src/agents/scaffold.ts
69427
69545
  import { join as join32, resolve as resolve6 } from "node:path";
69428
69546
  init_atomic();
69547
+
69548
+ // ../src/agents/agent-uid.ts
69549
+ init_peercred();
69550
+
69551
+ // ../src/agents/scaffold.ts
69429
69552
  init_schema();
69430
69553
 
69431
69554
  // ../src/config/users.ts
@@ -77097,6 +77220,7 @@ function detectErrorInTranscriptLine(line) {
77097
77220
  const errStr = typeof obj.error === "string" ? obj.error : "";
77098
77221
  const text4 = extractAssistantText(obj);
77099
77222
  const kind2 = status === 429 ? isTransientUpstreamSignal(`${text4}
77223
+ ${errStr}`) || isLitellmProxyLocal4292(`${text4}
77100
77224
  ${errStr}`) ? "rate-limited" : "quota-exhausted" : classifyClaudeError({ type: errStr, status, message: text4 });
77101
77225
  return {
77102
77226
  kind: kind2,
@@ -80957,10 +81081,10 @@ function readTurnActiveMarkerAgeMs(stateDir, now) {
80957
81081
  }
80958
81082
 
80959
81083
  // ../src/build-info.ts
80960
- var VERSION = "0.18.14";
80961
- var COMMIT_SHA = "b0db04aa";
80962
- var COMMIT_DATE = "2026-07-12T12:40:07+10:00";
80963
- var LATEST_PR = 3165;
81084
+ var VERSION = "0.18.15";
81085
+ var COMMIT_SHA = "2fa611a1";
81086
+ var COMMIT_DATE = "2026-07-12T04:44:56Z";
81087
+ var LATEST_PR = 3171;
80964
81088
  var COMMITS_AHEAD_OF_TAG = 0;
80965
81089
 
80966
81090
  // gateway/boot-version.ts
@@ -85647,12 +85771,33 @@ function emitGatewayOperatorEvent(event) {
85647
85771
  const { agent, kind } = event;
85648
85772
  let throttleEscalation = null;
85649
85773
  let escalationFired = false;
85650
- if (kind === "rate-limited" && isAccountScopedThrottle(event.detail)) {
85774
+ const rateLimit429Classification = kind === "rate-limited" ? classify429Detail(event.detail) : null;
85775
+ if (rateLimit429Classification != null && rateLimit429Classification !== "account-scoped") {
85776
+ emitRuntimeMetric(build429ClassifiedMetric({
85777
+ agent,
85778
+ detail: event.detail,
85779
+ classification: rateLimit429Classification,
85780
+ action: "calm",
85781
+ now: Date.now()
85782
+ }));
85783
+ if (rateLimit429Classification === "litellm-local") {
85784
+ process.stderr.write(`telegram gateway: 429 classified litellm-proxy-local agent=${agent} \u2014 ` + `calm path, no account attribution, no failover
85785
+ `);
85786
+ }
85787
+ }
85788
+ if (rateLimit429Classification === "account-scoped") {
85651
85789
  const throttleDecision = decideThrottleTier({
85652
85790
  detail: event.detail,
85653
85791
  now: Date.now(),
85654
85792
  thresholdMs: throttleRetryInPlaceMaxMs()
85655
85793
  });
85794
+ emitRuntimeMetric(build429ClassifiedMetric({
85795
+ agent,
85796
+ detail: event.detail,
85797
+ classification: "account-scoped",
85798
+ action: throttleDecision.action === "failover" ? "failover" : "throttle",
85799
+ now: Date.now()
85800
+ }));
85656
85801
  if (throttleDecision.action === "throttle") {
85657
85802
  process.stderr.write(`telegram gateway: throttle-tier staying-put agent=${agent} until=${new Date(throttleDecision.throttledUntilMs).toISOString()} parsedReset=${throttleDecision.resetParsed}
85658
85803
  `);
@@ -17088,7 +17088,16 @@ function isTransientUpstreamSignal(text) {
17088
17088
  const lower = sample.toLowerCase();
17089
17089
  return transientUpstreamSignals.some((s) => lower.includes(s));
17090
17090
  }
17091
- var transientUpstreamSignals;
17091
+ function isLitellmProxyLocal429(text) {
17092
+ if (typeof text !== "string" || text.length === 0)
17093
+ return false;
17094
+ const sample = text.length > 16384 ? text.slice(0, 16384) : text;
17095
+ const lower = sample.toLowerCase();
17096
+ if (litellmProxyLocal429Signals.some((s) => lower.includes(s)))
17097
+ return true;
17098
+ return litellmV3LimiterSignalPair.every((s) => lower.includes(s));
17099
+ }
17100
+ var transientUpstreamSignals, litellmProxyLocal429Signals, litellmV3LimiterSignalPair;
17092
17101
  var init_model_unavailable = __esm(() => {
17093
17102
  init_quota_check();
17094
17103
  init_card_format();
@@ -17102,6 +17111,17 @@ var init_model_unavailable = __esm(() => {
17102
17111
  "would exceed your account\u2019s rate limit",
17103
17112
  "would exceed your account's rate limit"
17104
17113
  ];
17114
+ litellmProxyLocal429Signals = [
17115
+ "deployment over user-defined ratelimit",
17116
+ "model rate limit exceeded. tpm limit",
17117
+ "model rate limit exceeded. rpm limit",
17118
+ "deployment over defined rpm limit",
17119
+ "no deployments available for selected model",
17120
+ "litellm rate limit handler",
17121
+ "crossed tpm / rpm",
17122
+ "max parallel request limit reached"
17123
+ ];
17124
+ litellmV3LimiterSignalPair = ["rate limit exceeded for ", "limit type:"];
17105
17125
  });
17106
17126
 
17107
17127
  // tool-label-sidecar.ts
@@ -17534,6 +17554,7 @@ function detectErrorInTranscriptLine(line) {
17534
17554
  const errStr = typeof obj.error === "string" ? obj.error : "";
17535
17555
  const text = extractAssistantText(obj);
17536
17556
  const kind2 = status === 429 ? isTransientUpstreamSignal(`${text}
17557
+ ${errStr}`) || isLitellmProxyLocal429(`${text}
17537
17558
  ${errStr}`) ? "rate-limited" : "quota-exhausted" : classifyClaudeError({ type: errStr, status, message: text });
17538
17559
  return {
17539
17560
  kind: kind2,
@@ -320,8 +320,9 @@ import {
320
320
  type ModelUnavailableDetection,
321
321
  } from '../model-unavailable.js'
322
322
  import {
323
+ build429ClassifiedMetric,
324
+ classify429Detail,
323
325
  decideThrottleTier,
324
- isAccountScopedThrottle,
325
326
  throttleRetryInPlaceMaxMs,
326
327
  } from '../throttle-tier.js'
327
328
  import { createThrottleTierRunner } from './throttle-tier-wiring.js'
@@ -7700,14 +7701,59 @@ function emitGatewayOperatorEvent(event: OperatorEvent): void {
7700
7701
  // through to the existing calm rate-limited card unchanged — an
7701
7702
  // account-scoped throttle would be the wrong action for a server-wide
7702
7703
  // condition.
7704
+ //
7705
+ // Classification is three-way (classify429Detail, throttle-tier.ts):
7706
+ // only `account-scoped` may enter the throttle tier. `litellm-local` —
7707
+ // the LiteLLM proxy's OWN tpm/rpm/router limiter tripping before the
7708
+ // request reached Anthropic — takes the calm path with NO broker mark and
7709
+ // NO failover: the condition is proxy-local and says nothing about the
7710
+ // account. Every rate-limited event ALSO emits one
7711
+ // `rate_limit_429_classified` runtime metric (PostHog + JSONL), fired
7712
+ // here — before the cooldown gate — so operators can correlate
7713
+ // account-scoped 429s with fleet TPM even when the card is suppressed.
7703
7714
  let throttleEscalation: ModelUnavailableDetection | null = null
7704
7715
  let escalationFired = false
7705
- if (kind === 'rate-limited' && isAccountScopedThrottle(event.detail)) {
7716
+ const rateLimit429Classification =
7717
+ kind === 'rate-limited' ? classify429Detail(event.detail) : null
7718
+ if (rateLimit429Classification != null && rateLimit429Classification !== 'account-scoped') {
7719
+ // litellm-local / generic-transient: the existing calm rate-limited
7720
+ // path (fall through to renderOperatorEvent below). NO broker
7721
+ // mark-throttled, NO throttle-tier runner, NO failover — for a
7722
+ // proxy-local cap trip those would bench an account that was never
7723
+ // touched.
7724
+ emitRuntimeMetric(
7725
+ build429ClassifiedMetric({
7726
+ agent,
7727
+ detail: event.detail,
7728
+ classification: rateLimit429Classification,
7729
+ action: 'calm',
7730
+ now: Date.now(),
7731
+ }),
7732
+ )
7733
+ if (rateLimit429Classification === 'litellm-local') {
7734
+ process.stderr.write(
7735
+ `telegram gateway: 429 classified litellm-proxy-local agent=${agent} — ` +
7736
+ `calm path, no account attribution, no failover\n`,
7737
+ )
7738
+ }
7739
+ }
7740
+ if (rateLimit429Classification === 'account-scoped') {
7706
7741
  const throttleDecision = decideThrottleTier({
7707
7742
  detail: event.detail,
7708
7743
  now: Date.now(),
7709
7744
  thresholdMs: throttleRetryInPlaceMaxMs(),
7710
7745
  })
7746
+ emitRuntimeMetric(
7747
+ build429ClassifiedMetric({
7748
+ agent,
7749
+ detail: event.detail,
7750
+ classification: 'account-scoped',
7751
+ // decideThrottleTier can't return 'none' for account-scoped wording;
7752
+ // map the two live actions onto the metric's vocabulary.
7753
+ action: throttleDecision.action === 'failover' ? 'failover' : 'throttle',
7754
+ now: Date.now(),
7755
+ }),
7756
+ )
7711
7757
  if (throttleDecision.action === 'throttle') {
7712
7758
  // Reset is near (≤ threshold) or unparseable (60s default): DO NOT
7713
7759
  // fail over. Record throttled_until broker-side, post ONE lightweight
@@ -92,6 +92,203 @@ export function isTransientUpstreamSignal(text: string): boolean {
92
92
  return transientUpstreamSignals.some(s => lower.includes(s))
93
93
  }
94
94
 
95
+ // ─── LiteLLM-proxy-LOCAL 429 signals (canonical, single source of truth) ─────
96
+
97
+ /**
98
+ * Explicit markers of a 429 generated by the LiteLLM proxy's OWN rate
99
+ * limiters — a router `tpm_limit`/`rpm_limit` deployment cap, a virtual-key /
100
+ * team / user cap, or an all-deployments-cooling-down router state. The
101
+ * request never reached Anthropic, so the condition is PROXY-LOCAL: it says
102
+ * nothing about the Anthropic account's quota and must never mark the account
103
+ * throttled/exhausted or trigger fleet failover.
104
+ *
105
+ * Each wording key is grounded in LiteLLM's source (verified against
106
+ * BerriAI/litellm `main`, 2026-07), matching how
107
+ * `accountScopedThrottleSignals` (throttle-tier.ts) documents its keys:
108
+ *
109
+ * - "deployment over user-defined ratelimit" —
110
+ * `RouterErrors.user_defined_ratelimit_error` (litellm/types/router.py),
111
+ * the body prefix of every deployment tpm/rpm cap 429 raised by
112
+ * litellm/router_utils/pre_call_checks/model_rate_limit_check.py:
113
+ * "Deployment over user-defined ratelimit. tpm limit={n}. current
114
+ * usage={n}. id={id}, model_group={g}".
115
+ * - "model rate limit exceeded. tpm/rpm limit" — the `message` field of the
116
+ * same enforcement RateLimitError: "Model rate limit exceeded. TPM
117
+ * limit={n}, current usage={n}" (model_rate_limit_check.py).
118
+ * - "deployment over defined rpm limit" — the usage-based-routing-v2
119
+ * strategy wording "Deployment over defined rpm limit={n}. current
120
+ * usage={n}" (litellm/router_strategy/lowest_tpm_rpm_v2.py). There is NO
121
+ * tpm sibling in any known release — v2 TPM exhaustion surfaces as the
122
+ * `no_deployments_available` wording below (the deployment is filtered
123
+ * from the candidate set rather than erroring in the increment path).
124
+ * - "no deployments available for selected model" —
125
+ * `RouterErrors.no_deployments_available`, the RouterRateLimitError
126
+ * message when every deployment for the model group is cooling down:
127
+ * "No deployments available for selected model, Try again in {n}
128
+ * seconds…" (litellm/types/router.py).
129
+ * - "litellm rate limit handler" — the v1 parallel-request limiter's
130
+ * ProxyRateLimitError detail prefix: "LiteLLM Rate Limit Handler for
131
+ * rate limit type = {t}. Crossed TPM / RPM / Max Parallel Request
132
+ * Limit. current rpm: {n}, rpm limit: {n}…"
133
+ * (litellm/proxy/hooks/parallel_request_limiter.py).
134
+ * - "crossed tpm / rpm" — `CommonProxyErrors
135
+ * .max_parallel_request_limit_reached.value` = "Crossed TPM / RPM / Max
136
+ * Parallel Request Limit" (litellm/proxy/_types.py), interpolated into
137
+ * BOTH v1 detail shapes, so it also covers the zero-limit branch's
138
+ * "Max parallel request limit reached {additional_details}" message.
139
+ * - "max parallel request limit reached" — the standalone prefix
140
+ * `raise_rate_limit_error` builds when a key's limit is set to 0
141
+ * (parallel_request_limiter.py `error_message`).
142
+ *
143
+ * The proxy-side per-key/team/user limiter (parallel_request_limiter_v3.py
144
+ * `_handle_rate_limit_error`) is matched by CO-OCCURRENCE instead of a
145
+ * per-descriptor entry — see `isLitellmProxyLocal429`. Its detail shape:
146
+ * "Rate limit exceeded for {descriptor}: {value}. Limit type: {t}. Current
147
+ * limit: {n}, Remaining: {n}. Limit resets at: {ts}". Descriptor keys in
148
+ * source are numerous and growing (api_key / user / team / team_member /
149
+ * organization / end_user / agent / agent_session / model_per_key /
150
+ * model_per_team / model_per_organization / model_per_project / tag_per_key
151
+ * / mcp_per_key / mcp_per_team as of 2026-07), so enumerating them is a
152
+ * treadmill: the pair "rate limit exceeded for " + "limit type:" appears in
153
+ * every v3 body and in no Anthropic error. This is the shape a `tpm_limit`
154
+ * on the per-agent virtual keys (src/litellm/provision.ts) trips.
155
+ *
156
+ * Deliberately EXCLUDED: the bare exception-mapping prefix
157
+ * "litellm.RateLimitError:". LiteLLM wraps FORWARDED upstream 429s with the
158
+ * same prefix on the pass-through, so its presence is NOT evidence the limit
159
+ * was proxy-local — a genuine Anthropic account throttle traversing the
160
+ * proxy must keep its account-scoped classification (see
161
+ * `classify429Detail` in throttle-tier.ts for the tie-break).
162
+ */
163
+ export const litellmProxyLocal429Signals = [
164
+ 'deployment over user-defined ratelimit',
165
+ 'model rate limit exceeded. tpm limit',
166
+ 'model rate limit exceeded. rpm limit',
167
+ 'deployment over defined rpm limit',
168
+ 'no deployments available for selected model',
169
+ 'litellm rate limit handler',
170
+ 'crossed tpm / rpm',
171
+ 'max parallel request limit reached',
172
+ ]
173
+
174
+ /**
175
+ * The v3 proxy-limiter co-occurrence pair (see the provenance comment on
176
+ * `litellmProxyLocal429Signals`): both substrings appear in every
177
+ * parallel_request_limiter_v3 429 body regardless of descriptor key, and
178
+ * never in an Anthropic error. Exported so tests can pin the rule.
179
+ */
180
+ export const litellmV3LimiterSignalPair = ['rate limit exceeded for ', 'limit type:'] as const
181
+
182
+ /**
183
+ * True when `text` carries an EXPLICIT LiteLLM-proxy-local rate-limit marker:
184
+ * one of `litellmProxyLocal429Signals`, OR the v3 limiter co-occurrence pair
185
+ * (`litellmV3LimiterSignalPair` — descriptor-agnostic, so new v3 descriptor
186
+ * keys are covered without a list update). Never throws on weird input. Pure
187
+ * wording detection only — precedence against account-scoped wording is
188
+ * owned by `classify429Detail` (throttle-tier.ts).
189
+ */
190
+ export function isLitellmProxyLocal429(text: string): boolean {
191
+ if (typeof text !== 'string' || text.length === 0) return false
192
+ const sample = text.length > 16_384 ? text.slice(0, 16_384) : text
193
+ const lower = sample.toLowerCase()
194
+ if (litellmProxyLocal429Signals.some(s => lower.includes(s))) return true
195
+ return litellmV3LimiterSignalPair.every(s => lower.includes(s))
196
+ }
197
+
198
+ /**
199
+ * Best-effort extraction of the limit detail LiteLLM embeds in its
200
+ * proxy-local 429 bodies, for instrumentation (the `rate_limit_429_classified`
201
+ * runtime metric). All fields null when nothing parseable is found — never
202
+ * throws. Shapes covered (same provenance as `litellmProxyLocal429Signals`):
203
+ *
204
+ * - "tpm limit=8000. current usage=8241" (model_rate_limit_check body)
205
+ * - "TPM limit=8000, current usage=8241" (model_rate_limit_check message)
206
+ * - "Deployment over defined rpm limit=60. current usage=61" (v2 strategy)
207
+ * - "Limit type: tokens. Current limit: 8000 … Limit resets at:
208
+ * 2026-07-12 08:05:00 UTC" (parallel_request_limiter_v3)
209
+ * - "Try again in 27.5 seconds" (RouterRateLimitError cooldown)
210
+ *
211
+ * CAVEAT — the v3 "Limit resets at" timestamp is NOT reliably UTC despite
212
+ * the literal: litellm formats it with a naive `datetime.fromtimestamp(...)
213
+ * .strftime("%Y-%m-%d %H:%M:%S UTC")` (parallel_request_limiter_v3.py
214
+ * `_handle_rate_limit_error`), i.e. proxy-LOCAL wall-clock time with a
215
+ * hard-coded "UTC" suffix. We parse it as UTC (nothing better is possible
216
+ * from the body alone), so `resetAtMs` — and the metric fields derived from
217
+ * it — can be skewed by the proxy host's UTC offset when the proxy doesn't
218
+ * run on UTC. Treat v3-derived resets as approximate; don't chase phantom
219
+ * clock drift from these metrics. The rate-limit windows are ≤1 minute
220
+ * anyway, so the absolute timestamp is informational.
221
+ */
222
+ export function parseLitellmLimitDetail(
223
+ text: string,
224
+ parseTimeNow: Date = new Date(),
225
+ ): {
226
+ limitType: string | null
227
+ limit: number | null
228
+ currentUsage: number | null
229
+ resetAtMs: number | null
230
+ } {
231
+ const empty = { limitType: null, limit: null, currentUsage: null, resetAtMs: null }
232
+ if (typeof text !== 'string' || text.length === 0) return empty
233
+ const sample = text.length > 16_384 ? text.slice(0, 16_384) : text
234
+ const lower = sample.toLowerCase()
235
+
236
+ // "tpm limit=8000" / "rpm limit=60" (body), "TPM limit=8000" (message),
237
+ // and the v1 limiter's colon form "rpm limit: 60".
238
+ let limitType: string | null = null
239
+ let limit: number | null = null
240
+ const eqLimit = lower.match(/\b([tr]pm)[ _]limit[=:]\s*(\d+)/)
241
+ if (eqLimit) {
242
+ limitType = eqLimit[1]
243
+ limit = Number(eqLimit[2])
244
+ }
245
+ // v3 limiter: "Limit type: tokens. Current limit: 8000"
246
+ if (limitType == null) {
247
+ const v3Type = lower.match(/limit type:\s*(tokens|requests|max_parallel_requests)/)
248
+ if (v3Type) limitType = v3Type[1]
249
+ }
250
+ if (limit == null) {
251
+ const v3Limit = lower.match(/current limit:\s*(\d+)/)
252
+ if (v3Limit) limit = Number(v3Limit[1])
253
+ }
254
+
255
+ // "current usage=8241" (body/message, both `=` and `= ` never occur with
256
+ // separators other than the literal shown in source).
257
+ let currentUsage: number | null = null
258
+ const usage = lower.match(/current usage=(\d+)/)
259
+ if (usage) currentUsage = Number(usage[1])
260
+
261
+ // Reset hints LiteLLM emits that the Anthropic-shaped parseResetTime does
262
+ // not cover: "Limit resets at: 2026-07-12 08:05:00 UTC" (v3 limiter — note
263
+ // the space separator, not ISO 'T') and "Try again in 27.5 seconds"
264
+ // (RouterRateLimitError).
265
+ let resetAtMs: number | null = null
266
+ const resetsAt = sample.match(
267
+ /limit resets at:\s*(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) UTC/i,
268
+ )
269
+ if (resetsAt) {
270
+ const d = new Date(`${resetsAt[1]}T${resetsAt[2]}Z`)
271
+ if (!Number.isNaN(d.getTime())) resetAtMs = d.getTime()
272
+ }
273
+ if (resetAtMs == null) {
274
+ const tryAgain = lower.match(/try again in\s+(\d+(?:\.\d+)?)\s*seconds/)
275
+ if (tryAgain) {
276
+ const secs = Number(tryAgain[1])
277
+ if (Number.isFinite(secs) && secs > 0 && secs < 7 * 24 * 3600) {
278
+ resetAtMs = parseTimeNow.getTime() + Math.round(secs * 1000)
279
+ }
280
+ }
281
+ }
282
+
283
+ return {
284
+ limitType,
285
+ limit: limit != null && Number.isFinite(limit) ? limit : null,
286
+ currentUsage:
287
+ currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
288
+ resetAtMs,
289
+ }
290
+ }
291
+
95
292
  // ─── Detection ───────────────────────────────────────────────────────────────
96
293
 
97
294
  /**
@@ -138,6 +335,23 @@ export function detectModelUnavailable(
138
335
  : { kind: 'overload', raw: stderr }
139
336
  }
140
337
 
338
+ // ── 0.5. LiteLLM-proxy-LOCAL 429 — never a quota signal ─────────────────
339
+ // A 429 raised by the LiteLLM proxy's own limiters (`tpm_limit`/`rpm_limit`
340
+ // caps, router cooldown) never reached Anthropic — nothing about the
341
+ // account is exhausted, so it must classify to the calm retryable kind
342
+ // BEFORE the quota substrings run (some LiteLLM bodies contain the word
343
+ // "limit", which a negation-blind quota match could seize on). Runs after
344
+ // step 0 deliberately: account-affirming wording, when present, can only
345
+ // have originated upstream (LiteLLM never emits it) and wins — see the
346
+ // tie-break note on `classify429Detail` (throttle-tier.ts). Uses the full
347
+ // matcher (list + v3 co-occurrence pair) so every descriptor is covered.
348
+ if (isLitellmProxyLocal429(sample)) {
349
+ const resetAt = parseResetTime(sample)
350
+ return resetAt !== undefined
351
+ ? { kind: 'overload', resetAt, raw: stderr }
352
+ : { kind: 'overload', raw: stderr }
353
+ }
354
+
141
355
  // ── 1. Quota / billing exhaustion ──────────────────────────────────────
142
356
  const quotaSignals = [
143
357
  'out of extra usage',
@@ -164,6 +164,37 @@ export type RuntimeMetricEvent =
164
164
  agent: string
165
165
  prompt_key: string
166
166
  }
167
+ /**
168
+ * Every terminal rate-limit-family operator event (429 burst / 529
169
+ * overload), classified by origin BEFORE the gateway acts on it — fires
170
+ * even when the user-facing card is cooldown-suppressed, so the count is
171
+ * honest. `classification` says where the limit lives (see
172
+ * `RateLimit429Classification` in throttle-tier.ts): `account-scoped` =
173
+ * Anthropic throttled the account (throttle tier ran), `litellm-local` =
174
+ * the LiteLLM proxy's own tpm/rpm/router limiter tripped (calm path, no
175
+ * account attribution), `generic-transient` = other server-side 429/529
176
+ * wording. `action` is what the gateway DECIDED (throttle / failover /
177
+ * calm), emitted PRE-execution: the actual failover fire is dedup-gated
178
+ * downstream (fleetFallbackGate), so N long-reset 429s inside one dedup
179
+ * window emit N `action: 'failover'` metrics for ONE real roll — count
180
+ * decisions here, count rolls via the broker/fallback announcements.
181
+ * Correlating `account-scoped` fires against fleet token throughput is
182
+ * the operator's evidence base for setting LiteLLM `tpm_limit` caps; the
183
+ * `litellm-local` count then shows those caps actually absorbing load.
184
+ * Limit/reset fields are best-effort parses of the error body (null when
185
+ * absent).
186
+ */
187
+ | {
188
+ kind: 'rate_limit_429_classified'
189
+ agent: string
190
+ classification: 'account-scoped' | 'litellm-local' | 'generic-transient'
191
+ action: 'throttle' | 'failover' | 'calm'
192
+ reset_at_ms: number | null
193
+ reset_in_ms: number | null
194
+ limit_type: string | null
195
+ limit: number | null
196
+ current_usage: number | null
197
+ }
167
198
 
168
199
  /**
169
200
  * The JSONL sink lives under the runtime state dir so it's per-agent