switchroom 0.18.14 → 0.18.17

package/telegram-plugin/litellm-local-notice.ts

@@ -0,0 +1,189 @@
+/**
+ * litellm-local-notice.ts — debounced operator notice for LiteLLM-proxy-local
+ * 429s (pure module: state machine + text + config parsing, no IPC, no bot,
+ * no clock except the injected `now`).
+ *
+ * When an agent routes through the LiteLLM gateway and trips the proxy's OWN
+ * `tpm_limit`/`rpm_limit` limiter, `classify429Detail` (throttle-tier.ts)
+ * classifies the terminal 429 `litellm-local` and the gateway takes the calm
+ * path — no broker mark, no failover, no throttle tier (the request never
+ * reached Anthropic, so the condition says nothing about the account). But
+ * pre-notice the user-facing surface was the GENERIC "🚦 Rate limited" card,
+ * which reads like an Anthropic problem. This module owns the honest
+ * replacement:
+ *
+ *   - ONE calm notice per agent per cooldown window (default 15 min,
+ *     operator-tunable via channels.telegram.litellm_notice.window_ms in
+ *     switchroom.yaml, projected into access.json by scaffold).
+ *   - Further litellm-local 429s inside the window are counted SILENTLY;
+ *     the first notice after the window expires carries "throttled N more
+ *     times since the last notice".
+ *   - A notice only ever fires on an actual throttle event — a quiet agent
+ *     posts nothing (the state machine is evaluate-on-event, no timers).
+ *
+ * The copy must make clear this is the fleet token limiter (LiteLLM
+ * `tpm_limit`/`rpm_limit`), NOT an Anthropic account limit, and that the
+ * turn retries — no action needed.
+ *
+ * Side-effect sequencing lives in gateway/litellm-local-notice-wiring.ts.
+ * This module MUST NOT change classification, failover, or quota-ledger
+ * behavior — it only renders and debounces the notice.
+ */
+
+import { escapeMarkdown } from './card-format.js'
+import type { RateLimit429Classification } from './throttle-tier.js'
+import type { RuntimeMetricEvent } from './runtime-metrics.js'
+
+// ─── Cooldown window config ──────────────────────────────────────────────────
+
+/**
+ * Default per-agent notice cooldown. 15 minutes: long enough that a sustained
+ * burst produces a handful of notices per hour at most, short enough that the
+ * operator still sees the limiter working while it's working.
+ */
+export const LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT = 15 * 60_000
+
+/**
+ * Resolve the cooldown window from the raw access-file value
+ * (`access.litellmNoticeWindowMs`, projected by scaffold from
+ * channels.telegram.litellm_notice.window_ms). Any non-finite, non-positive,
+ * or non-number value falls back to the default — an operator typo must
+ * never produce a zero-width window (notice storm) or a NaN comparison
+ * (notices never fire again).
+ */
+export function parseLitellmNoticeWindowMs(raw: unknown): number {
+  if (typeof raw !== 'number') return LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT
+  if (!Number.isFinite(raw) || raw <= 0) return LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT
+  return raw
+}
+
+// ─── Per-agent cooldown state machine ────────────────────────────────────────
+
+export interface LitellmLocalNoticeState {
+  /** agent → unix ms of the last notice sent for it. */
+  lastSentAtMsByAgent: Record<string, number>
+  /** agent → litellm-local 429s counted silently since the last notice. */
+  suppressedCountByAgent: Record<string, number>
+}
+
+export function initialLitellmLocalNoticeState(): LitellmLocalNoticeState {
+  return { lastSentAtMsByAgent: {}, suppressedCountByAgent: {} }
+}
+
+export interface LitellmLocalNoticeVerdict {
+  send: boolean
+  /** Silent throttle events since the previous notice (0 on the first). Only
+   *  meaningful when `send` is true — the renderer adds the "N more times"
+   *  line when > 0. */
+  suppressedSinceLastNotice: number
+  next: LitellmLocalNoticeState
+}
+
+/**
+ * Evaluate one litellm-local 429 for `agent` at `now`.
+ *
+ *   - First event ever (or ≥ windowMs since the last notice) → send; the
+ *     verdict carries the silently-counted events since the previous notice
+ *     and the counter resets.
+ *   - Inside the window → suppress; the counter increments.
+ *
+ * Boundary is INCLUSIVE on expiry (elapsed === windowMs sends), matching
+ * `evaluateThrottleNotice` in throttle-tier.ts. Pure: returns the next
+ * state, never mutates `prev`.
+ */
+export function evaluateLitellmLocalNotice(
+  prev: LitellmLocalNoticeState,
+  agent: string,
+  now: number,
+  windowMs: number = LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT,
+): LitellmLocalNoticeVerdict {
+  const last = prev.lastSentAtMsByAgent[agent]
+  if (last == null || now - last >= windowMs) {
+    return {
+      send: true,
+      suppressedSinceLastNotice: prev.suppressedCountByAgent[agent] ?? 0,
+      next: {
+        lastSentAtMsByAgent: { ...prev.lastSentAtMsByAgent, [agent]: now },
+        suppressedCountByAgent: { ...prev.suppressedCountByAgent, [agent]: 0 },
+      },
+    }
+  }
+  return {
+    send: false,
+    suppressedSinceLastNotice: 0,
+    next: {
+      lastSentAtMsByAgent: prev.lastSentAtMsByAgent,
+      suppressedCountByAgent: {
+        ...prev.suppressedCountByAgent,
+        [agent]: (prev.suppressedCountByAgent[agent] ?? 0) + 1,
+      },
+    },
+  }
+}
+
+// ─── Notice rendering ────────────────────────────────────────────────────────
+
+/**
+ * The ONE calm notice for a litellm-local 429. Markdown (not HTML) per the
+ * card-format.ts convention. Copy contract (operator spec):
+ *   - names the fleet token limiter (LiteLLM `tpm_limit`/`rpm_limit`)
+ *   - explicitly NOT an Anthropic account limit — nothing exhausted, no
+ *     account touched
+ *   - the turn retries automatically; no action needed
+ *   - after a window with silent suppressions, says "throttled N more
+ *     time(s) since the last notice"
+ */
+export function renderLitellmLocalNotice(opts: {
+  agent: string
+  /** Silent litellm-local 429s since the previous notice (0 on the first). */
+  suppressedSinceLastNotice: number
+}): string {
+  const agent = escapeMarkdown(opts.agent)
+  const n = opts.suppressedSinceLastNotice
+  const lines = [
+    `🚦 **Fleet token limiter engaged** — **${agent}** hit the local LiteLLM proxy cap (\`tpm_limit\`/\`rpm_limit\`).`,
+    `This is switchroom's own fleet limiter smoothing a burst, not an Anthropic account limit — nothing is exhausted and no account was touched.`,
+  ]
+  if (n > 0) {
+    lines.push(`Throttled ${n} more ${n === 1 ? 'time' : 'times'} since the last notice.`)
+  }
+  lines.push(`_The turn retries automatically — no action needed._`)
+  return lines.join('\n')
+}
+
+// ─── Metric builder ──────────────────────────────────────────────────────────
+
+/**
+ * Build the `litellm_local_429_notice` runtime metric for one SENT notice.
+ * Distinct from `rate_limit_429_classified` (which fires on EVERY classified
+ * event): this one fires only when a notice actually posts, and carries how
+ * many events the window silently absorbed — the debounce's effectiveness in
+ * one number. Pure builder so the payload shape is unit-testable; the wiring
+ * emits the result via emitRuntimeMetric (PostHog + JSONL dual sink).
+ */
+export function buildLitellmLocalNoticeMetric(opts: {
+  agent: string
+  suppressedCount: number
+  windowMs: number
+}): Extract<RuntimeMetricEvent, { kind: 'litellm_local_429_notice' }> {
+  return {
+    kind: 'litellm_local_429_notice',
+    agent: opts.agent,
+    suppressed_count: opts.suppressedCount,
+    window_ms: opts.windowMs,
+  }
+}
+
+// ─── Classification guard ────────────────────────────────────────────────────
+
+/**
+ * True only for the `litellm-local` classification. The wiring runs this
+ * guard so "a notice never fires for non-litellm-local classifications" is a
+ * deterministic mechanism, not caller discipline — account-scoped 429s keep
+ * the throttle tier, generic transients keep the calm rate-limited card.
+ */
+export function isLitellmLocalNoticeEligible(
+  classification: RateLimit429Classification | null | undefined,
+): classification is 'litellm-local' {
+  return classification === 'litellm-local'
+}

package/telegram-plugin/model-unavailable.ts

@@ -92,6 +92,203 @@ export function isTransientUpstreamSignal(text: string): boolean {
   return transientUpstreamSignals.some(s => lower.includes(s))
 }
 
+// ─── LiteLLM-proxy-LOCAL 429 signals (canonical, single source of truth) ─────
+
+/**
+ * Explicit markers of a 429 generated by the LiteLLM proxy's OWN rate
+ * limiters — a router `tpm_limit`/`rpm_limit` deployment cap, a virtual-key /
+ * team / user cap, or an all-deployments-cooling-down router state. The
+ * request never reached Anthropic, so the condition is PROXY-LOCAL: it says
+ * nothing about the Anthropic account's quota and must never mark the account
+ * throttled/exhausted or trigger fleet failover.
+ *
+ * Each wording key is grounded in LiteLLM's source (verified against
+ * BerriAI/litellm `main`, 2026-07), matching how
+ * `accountScopedThrottleSignals` (throttle-tier.ts) documents its keys:
+ *
+ *  - "deployment over user-defined ratelimit" —
+ *    `RouterErrors.user_defined_ratelimit_error` (litellm/types/router.py),
+ *    the body prefix of every deployment tpm/rpm cap 429 raised by
+ *    litellm/router_utils/pre_call_checks/model_rate_limit_check.py:
+ *    "Deployment over user-defined ratelimit. tpm limit={n}. current
+ *    usage={n}. id={id}, model_group={g}".
+ *  - "model rate limit exceeded. tpm/rpm limit" — the `message` field of the
+ *    same enforcement RateLimitError: "Model rate limit exceeded. TPM
+ *    limit={n}, current usage={n}" (model_rate_limit_check.py).
+ *  - "deployment over defined rpm limit" — the usage-based-routing-v2
+ *    strategy wording "Deployment over defined rpm limit={n}. current
+ *    usage={n}" (litellm/router_strategy/lowest_tpm_rpm_v2.py). There is NO
+ *    tpm sibling in any known release — v2 TPM exhaustion surfaces as the
+ *    `no_deployments_available` wording below (the deployment is filtered
+ *    from the candidate set rather than erroring in the increment path).
+ *  - "no deployments available for selected model" —
+ *    `RouterErrors.no_deployments_available`, the RouterRateLimitError
+ *    message when every deployment for the model group is cooling down:
+ *    "No deployments available for selected model, Try again in {n}
+ *    seconds…" (litellm/types/router.py).
+ *  - "litellm rate limit handler" — the v1 parallel-request limiter's
+ *    ProxyRateLimitError detail prefix: "LiteLLM Rate Limit Handler for
+ *    rate limit type = {t}. Crossed TPM / RPM / Max Parallel Request
+ *    Limit. current rpm: {n}, rpm limit: {n}…"
+ *    (litellm/proxy/hooks/parallel_request_limiter.py).
+ *  - "crossed tpm / rpm" — `CommonProxyErrors
+ *    .max_parallel_request_limit_reached.value` = "Crossed TPM / RPM / Max
+ *    Parallel Request Limit" (litellm/proxy/_types.py), interpolated into
+ *    BOTH v1 detail shapes, so it also covers the zero-limit branch's
+ *    "Max parallel request limit reached {additional_details}" message.
+ *  - "max parallel request limit reached" — the standalone prefix
+ *    `raise_rate_limit_error` builds when a key's limit is set to 0
+ *    (parallel_request_limiter.py `error_message`).
+ *
+ * The proxy-side per-key/team/user limiter (parallel_request_limiter_v3.py
+ * `_handle_rate_limit_error`) is matched by CO-OCCURRENCE instead of a
+ * per-descriptor entry — see `isLitellmProxyLocal429`. Its detail shape:
+ * "Rate limit exceeded for {descriptor}: {value}. Limit type: {t}. Current
+ * limit: {n}, Remaining: {n}. Limit resets at: {ts}". Descriptor keys in
+ * source are numerous and growing (api_key / user / team / team_member /
+ * organization / end_user / agent / agent_session / model_per_key /
+ * model_per_team / model_per_organization / model_per_project / tag_per_key
+ * / mcp_per_key / mcp_per_team as of 2026-07), so enumerating them is a
+ * treadmill: the pair "rate limit exceeded for " + "limit type:" appears in
+ * every v3 body and in no Anthropic error. This is the shape a `tpm_limit`
+ * on the per-agent virtual keys (src/litellm/provision.ts) trips.
+ *
+ * Deliberately EXCLUDED: the bare exception-mapping prefix
+ * "litellm.RateLimitError:". LiteLLM wraps FORWARDED upstream 429s with the
+ * same prefix on the pass-through, so its presence is NOT evidence the limit
+ * was proxy-local — a genuine Anthropic account throttle traversing the
+ * proxy must keep its account-scoped classification (see
+ * `classify429Detail` in throttle-tier.ts for the tie-break).
+ */
+export const litellmProxyLocal429Signals = [
+  'deployment over user-defined ratelimit',
+  'model rate limit exceeded. tpm limit',
+  'model rate limit exceeded. rpm limit',
+  'deployment over defined rpm limit',
+  'no deployments available for selected model',
+  'litellm rate limit handler',
+  'crossed tpm / rpm',
+  'max parallel request limit reached',
+]
+
+/**
+ * The v3 proxy-limiter co-occurrence pair (see the provenance comment on
+ * `litellmProxyLocal429Signals`): both substrings appear in every
+ * parallel_request_limiter_v3 429 body regardless of descriptor key, and
+ * never in an Anthropic error. Exported so tests can pin the rule.
+ */
+export const litellmV3LimiterSignalPair = ['rate limit exceeded for ', 'limit type:'] as const
+
+/**
+ * True when `text` carries an EXPLICIT LiteLLM-proxy-local rate-limit marker:
+ * one of `litellmProxyLocal429Signals`, OR the v3 limiter co-occurrence pair
+ * (`litellmV3LimiterSignalPair` — descriptor-agnostic, so new v3 descriptor
+ * keys are covered without a list update). Never throws on weird input. Pure
+ * wording detection only — precedence against account-scoped wording is
+ * owned by `classify429Detail` (throttle-tier.ts).
+ */
+export function isLitellmProxyLocal429(text: string): boolean {
+  if (typeof text !== 'string' || text.length === 0) return false
+  const sample = text.length > 16_384 ? text.slice(0, 16_384) : text
+  const lower = sample.toLowerCase()
+  if (litellmProxyLocal429Signals.some(s => lower.includes(s))) return true
+  return litellmV3LimiterSignalPair.every(s => lower.includes(s))
+}
+
+/**
+ * Best-effort extraction of the limit detail LiteLLM embeds in its
+ * proxy-local 429 bodies, for instrumentation (the `rate_limit_429_classified`
+ * runtime metric). All fields null when nothing parseable is found — never
+ * throws. Shapes covered (same provenance as `litellmProxyLocal429Signals`):
+ *
+ *   - "tpm limit=8000. current usage=8241"    (model_rate_limit_check body)
+ *   - "TPM limit=8000, current usage=8241"    (model_rate_limit_check message)
+ *   - "Deployment over defined rpm limit=60. current usage=61"  (v2 strategy)
+ *   - "Limit type: tokens. Current limit: 8000 … Limit resets at:
+ *      2026-07-12 08:05:00 UTC"               (parallel_request_limiter_v3)
+ *   - "Try again in 27.5 seconds"             (RouterRateLimitError cooldown)
+ *
+ * CAVEAT — the v3 "Limit resets at" timestamp is NOT reliably UTC despite
+ * the literal: litellm formats it with a naive `datetime.fromtimestamp(...)
+ * .strftime("%Y-%m-%d %H:%M:%S UTC")` (parallel_request_limiter_v3.py
+ * `_handle_rate_limit_error`), i.e. proxy-LOCAL wall-clock time with a
+ * hard-coded "UTC" suffix. We parse it as UTC (nothing better is possible
+ * from the body alone), so `resetAtMs` — and the metric fields derived from
+ * it — can be skewed by the proxy host's UTC offset when the proxy doesn't
+ * run on UTC. Treat v3-derived resets as approximate; don't chase phantom
+ * clock drift from these metrics. The rate-limit windows are ≤1 minute
+ * anyway, so the absolute timestamp is informational.
+ */
+export function parseLitellmLimitDetail(
+  text: string,
+  parseTimeNow: Date = new Date(),
+): {
+  limitType: string | null
+  limit: number | null
+  currentUsage: number | null
+  resetAtMs: number | null
+} {
+  const empty = { limitType: null, limit: null, currentUsage: null, resetAtMs: null }
+  if (typeof text !== 'string' || text.length === 0) return empty
+  const sample = text.length > 16_384 ? text.slice(0, 16_384) : text
+  const lower = sample.toLowerCase()
+
+  // "tpm limit=8000" / "rpm limit=60" (body), "TPM limit=8000" (message),
+  // and the v1 limiter's colon form "rpm limit: 60".
+  let limitType: string | null = null
+  let limit: number | null = null
+  const eqLimit = lower.match(/\b([tr]pm)[ _]limit[=:]\s*(\d+)/)
+  if (eqLimit) {
+    limitType = eqLimit[1]
+    limit = Number(eqLimit[2])
+  }
+  // v3 limiter: "Limit type: tokens. Current limit: 8000"
+  if (limitType == null) {
+    const v3Type = lower.match(/limit type:\s*(tokens|requests|max_parallel_requests)/)
+    if (v3Type) limitType = v3Type[1]
+  }
+  if (limit == null) {
+    const v3Limit = lower.match(/current limit:\s*(\d+)/)
+    if (v3Limit) limit = Number(v3Limit[1])
+  }
+
+  // "current usage=8241" (body/message, both `=` and `= ` never occur with
+  // separators other than the literal shown in source).
+  let currentUsage: number | null = null
+  const usage = lower.match(/current usage=(\d+)/)
+  if (usage) currentUsage = Number(usage[1])
+
+  // Reset hints LiteLLM emits that the Anthropic-shaped parseResetTime does
+  // not cover: "Limit resets at: 2026-07-12 08:05:00 UTC" (v3 limiter — note
+  // the space separator, not ISO 'T') and "Try again in 27.5 seconds"
+  // (RouterRateLimitError).
+  let resetAtMs: number | null = null
+  const resetsAt = sample.match(
+    /limit resets at:\s*(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) UTC/i,
+  )
+  if (resetsAt) {
+    const d = new Date(`${resetsAt[1]}T${resetsAt[2]}Z`)
+    if (!Number.isNaN(d.getTime())) resetAtMs = d.getTime()
+  }
+  if (resetAtMs == null) {
+    const tryAgain = lower.match(/try again in\s+(\d+(?:\.\d+)?)\s*seconds/)
+    if (tryAgain) {
+      const secs = Number(tryAgain[1])
+      if (Number.isFinite(secs) && secs > 0 && secs < 7 * 24 * 3600) {
+        resetAtMs = parseTimeNow.getTime() + Math.round(secs * 1000)
+      }
+    }
+  }
+
+  return {
+    limitType,
+    limit: limit != null && Number.isFinite(limit) ? limit : null,
+    currentUsage:
+      currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
+    resetAtMs,
+  }
+}
+
 // ─── Detection ───────────────────────────────────────────────────────────────
 
 /**
@@ -138,6 +335,23 @@ export function detectModelUnavailable(
       : { kind: 'overload', raw: stderr }
   }
 
+  // ── 0.5. LiteLLM-proxy-LOCAL 429 — never a quota signal ─────────────────
+  // A 429 raised by the LiteLLM proxy's own limiters (`tpm_limit`/`rpm_limit`
+  // caps, router cooldown) never reached Anthropic — nothing about the
+  // account is exhausted, so it must classify to the calm retryable kind
+  // BEFORE the quota substrings run (some LiteLLM bodies contain the word
+  // "limit", which a negation-blind quota match could seize on). Runs after
+  // step 0 deliberately: account-affirming wording, when present, can only
+  // have originated upstream (LiteLLM never emits it) and wins — see the
+  // tie-break note on `classify429Detail` (throttle-tier.ts). Uses the full
+  // matcher (list + v3 co-occurrence pair) so every descriptor is covered.
+  if (isLitellmProxyLocal429(sample)) {
+    const resetAt = parseResetTime(sample)
+    return resetAt !== undefined
+      ? { kind: 'overload', resetAt, raw: stderr }
+      : { kind: 'overload', raw: stderr }
+  }
+
   // ── 1. Quota / billing exhaustion ──────────────────────────────────────
   const quotaSignals = [
     'out of extra usage',

package/telegram-plugin/quota-watch.ts

@@ -192,10 +192,14 @@ export interface FleetRollInfo {
   /**
    * Trigger attribution (#3031 PR 2): "soft-avoid" = proactive
    * serving-preference roll off an account APPROACHING its limits;
-   * "hard-exhaustion" = the probe saw a genuine quota wall. Absent on
-   * pre-PR-2 brokers — rendered as hard exhaustion.
+   * "hard-exhaustion" = the probe saw a genuine quota wall;
+   * "model-tier-wall" = the flagship-tier (7d_oi) canary saw the account
+   * walled on the premium tier (#3176). Absent on pre-PR-2 brokers —
+   * rendered as hard exhaustion.
    */
-  reason?: "soft-avoid" | "hard-exhaustion";
+  reason?: "soft-avoid" | "hard-exhaustion" | "model-tier-wall";
+  /** #3176 — the binding tier bucket, set only when reason is model-tier-wall. */
+  bucket?: string;
 }
 
 export type FleetRollAnnounceDecision =
@@ -241,9 +245,17 @@ export function buildFleetRollMessage(roll: FleetRollInfo, now: number): string
       ? ` (resets ${formatRelative(new Date(roll.exhausted_until), new Date(now))})`
       : "";
   const softAvoid = roll.reason === "soft-avoid";
+  // #3176 — a model-tier wall binds on the flagship (premium) 7d_oi bucket, not
+  // the 5h/7d windows (which read healthy — the whole point of the bug), so it
+  // has no window/pct to cite. Name the flagship tier explicitly and reassure
+  // that opus/haiku on the walled account are unaffected (the mark is
+  // tier-scoped), rather than rendering a generic, misleading "quota window".
+  const modelTierWall = roll.reason === "model-tier-wall";
   const causeLine = softAvoid
     ? `Proactive switch — \`${codeSpanSafe(roll.from)}\` is approaching its limits (${winLabel}${pctPart})${resetPart}, so the fleet moved early instead of hitting the wall.`
-    : `${winLabel}${pctPart} on \`${codeSpanSafe(roll.from)}\`${resetPart}.`;
+    : modelTierWall
+      ? `Flagship (premium) tier weekly limit reached on \`${codeSpanSafe(roll.from)}\`${resetPart} — opus/haiku on that account are unaffected.`
+      : `${winLabel}${pctPart} on \`${codeSpanSafe(roll.from)}\`${resetPart}.`;
   return [
     `🔁 **Switched fleet to \`${codeSpanSafe(roll.to)}\`**`,
     ``,

package/telegram-plugin/runtime-metrics.ts

@@ -164,6 +164,53 @@ export type RuntimeMetricEvent =
       agent: string
       prompt_key: string
     }
+  /**
+   * Every terminal rate-limit-family operator event (429 burst / 529
+   * overload), classified by origin BEFORE the gateway acts on it — fires
+   * even when the user-facing card is cooldown-suppressed, so the count is
+   * honest. `classification` says where the limit lives (see
+   * `RateLimit429Classification` in throttle-tier.ts): `account-scoped` =
+   * Anthropic throttled the account (throttle tier ran), `litellm-local` =
+   * the LiteLLM proxy's own tpm/rpm/router limiter tripped (calm path, no
+   * account attribution), `generic-transient` = other server-side 429/529
+   * wording. `action` is what the gateway DECIDED (throttle / failover /
+   * calm), emitted PRE-execution: the actual failover fire is dedup-gated
+   * downstream (fleetFallbackGate), so N long-reset 429s inside one dedup
+   * window emit N `action: 'failover'` metrics for ONE real roll — count
+   * decisions here, count rolls via the broker/fallback announcements.
+   * Correlating `account-scoped` fires against fleet token throughput is
+   * the operator's evidence base for setting LiteLLM `tpm_limit` caps; the
+   * `litellm-local` count then shows those caps actually absorbing load.
+   * Limit/reset fields are best-effort parses of the error body (null when
+   * absent).
+   */
+  | {
+      kind: 'rate_limit_429_classified'
+      agent: string
+      classification: 'account-scoped' | 'litellm-local' | 'generic-transient'
+      action: 'throttle' | 'failover' | 'calm'
+      reset_at_ms: number | null
+      reset_in_ms: number | null
+      limit_type: string | null
+      limit: number | null
+      current_usage: number | null
+    }
+  /**
+   * A litellm-local throttle NOTICE actually posted (litellm-local-notice.ts
+   * — the debounced "fleet token limiter engaged" message). Distinct from
+   * `rate_limit_429_classified`, which fires on EVERY classified 429: this
+   * fires once per sent notice, and `suppressed_count` is how many
+   * litellm-local 429s the cooldown window silently absorbed since the
+   * previous notice (0 on the first) — the debounce's effectiveness in one
+   * number. `window_ms` is the resolved cooldown window (default 15 min;
+   * channels.telegram.litellm_notice.window_ms).
+   */
+  | {
+      kind: 'litellm_local_429_notice'
+      agent: string
+      suppressed_count: number
+      window_ms: number
+    }
 
 /**
  * The JSONL sink lives under the runtime state dir so it's per-agent

package/telegram-plugin/send-gate-degraded.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest'
-import { createSendGate, type Clock } from './send-gate.js'
+import { createSendGate, SEND_GATE_SHED, type Clock } from './send-gate.js'
 import { isFloodWaitActiveError } from './retry-api-call.js'
 
 /**
@@ -73,8 +73,10 @@ describe('send-gate PR2: cosmetic shedding', () => {
 
     const res = await gate.gate(fn('cosmetic'), { priorityClass: 'cosmetic' })
 
-    // Shed resolves undefined; fn never ran; the shed counter moved.
-    expect(res).toBeUndefined()
+    // Shed resolves the distinguishable sentinel (#3110 F1 — never a bare
+    // undefined, which is reserved for benign no-op drops); fn never ran;
+    // the shed counter moved.
+    expect(res).toBe(SEND_GATE_SHED)
     expect(calls).toHaveLength(0)
     expect(gate.stats().global.shed).toBe(1)
     expect(gate.stats().global.sent).toBe(0)
@@ -95,7 +97,7 @@ describe('send-gate PR2: cosmetic shedding', () => {
     const shed = await gate.gate(fn('b'), { chat_id: '5', priorityClass: 'cosmetic' })
 
     expect(calls.map((c) => c.label)).toEqual(['a'])
-    expect(shed).toBeUndefined()
+    expect(shed).toBe(SEND_GATE_SHED)
     expect(gate.stats().global.shed).toBe(1)
   })
 
@@ -128,7 +130,7 @@ describe('send-gate PR2: cosmetic shedding', () => {
       priorityClass: 'cosmetic',
     })
 
-    expect(res).toBeUndefined()
+    expect(res).toBe(SEND_GATE_SHED)
     expect(calls).toHaveLength(0)
     expect(gate.stats().global.shed).toBe(1)
   })
@@ -366,7 +368,7 @@ describe('send-gate PR2: H1 cross-chat message_id isolation', () => {
       priorityClass: 'cosmetic',
     })
 
-    expect(rA).toBeUndefined() // A shed
+    expect(rA).toBe(SEND_GATE_SHED) // A shed
     expect(rB).toBe('B-edit') // B sent
     expect(calls.map((c) => c.label)).toEqual(['B-edit'])
     expect(gate.stats().global.shed).toBe(1)
@@ -568,7 +570,7 @@ describe('send-gate PR2: opening a window from a 429, and persistence hook', ()
     expect(scopes).toEqual(['chat:7', 'global', 'group:7', 'msg-edit:7:3'])
     // After the window opens, a later cosmetic on the same chat sheds.
     const shed = await gate.gate(async () => 'x', { chat_id: '7', priorityClass: 'cosmetic' })
-    expect(shed).toBeUndefined()
+    expect(shed).toBe(SEND_GATE_SHED)
     expect(gate.stats().global.shed).toBe(1)
   })
 })

package/telegram-plugin/send-gate.ts

@@ -118,6 +118,32 @@ export type PriorityClass = 'critical' | 'useful' | 'cosmetic'
  */
 export const UNTAGGED_SEND_CLASS: PriorityClass = 'critical'
 
+/**
+ * Distinguishable resolution value for a SHED call (#3110 review F1).
+ *
+ * `undefined` was overloaded three ways at the robustApiCall seam: a gate
+ * SHED (cosmetic call dropped under pressure — did NOT land), a gate no-op
+ * drop (identical payload already on screen — benign), and the retry
+ * policy's swallowed benign 400s ("message is not modified" — also benign).
+ * A caller that needs to know "did my edit land?" (the draft stream's
+ * shed-honesty handling in stream-controller.ts) could not tell these
+ * apart, and treating every `undefined` as a shed froze multi-piece
+ * streams on perfectly healthy chats.
+ *
+ * A shed now resolves THIS sentinel instead. `Symbol.for` keys it in the
+ * global symbol registry so duplicated module instances (server + gateway
+ * builds) agree on identity. The no-op drop and coalesced-revert drop keep
+ * resolving `undefined` — for those the payload IS on screen. Callers that
+ * ignore the result (reactions, typing, fire-and-forget card edits) are
+ * unaffected either way.
+ */
+export const SEND_GATE_SHED: unique symbol = Symbol.for('switchroom.send-gate.shed')
+
+/** True when a gate result is the {@link SEND_GATE_SHED} sentinel. */
+export function isSendGateShed(value: unknown): value is typeof SEND_GATE_SHED {
+  return value === SEND_GATE_SHED
+}
+
 /**
  * Extra metadata a call site can attach so the gate can key the right buckets.
  * All fields optional — a call with none still passes the global bucket. These
@@ -159,8 +185,9 @@ export interface BucketCounters {
   dropped: number
   /**
    * Cosmetic calls shed under pressure — no token free OR a flood window open
-   * (part3-design §2). A shed resolves as `undefined`; the next send carries
-   * full state.
+   * (part3-design §2). A shed resolves the SEND_GATE_SHED sentinel (F1 —
+   * distinguishable from a benign no-op drop's `undefined`); the next send
+   * carries full state.
    */
   shed: number
   /** Useful calls dropped because they exceeded their queue TTL (part3-design §2). */
@@ -913,7 +940,9 @@ export function createSendGate(config: SendGateConfig): SendGate {
       const msgWait = state.suppressedUntilMs > now ? state.suppressedUntilMs - now : 0
       if (wait > 0 || msgWait > 0) {
         counters.shed++
-        return Promise.resolve(undefined as unknown as T)
+        // Distinguishable from the no-op drop below (undefined): a shed did
+        // NOT land, and edit-driving callers must be able to tell (F1).
+        return Promise.resolve(SEND_GATE_SHED as unknown as T)
       }
     }
 
@@ -991,7 +1020,8 @@ export function createSendGate(config: SendGateConfig): SendGate {
     const outcome = await admitPriority(bucketsFor(opts), priority)
     if (outcome.result === 'shed') {
       counters.shed++
-      return undefined as unknown as T
+      // See SEND_GATE_SHED: distinguishable "did not land" resolution (F1).
+      return SEND_GATE_SHED as unknown as T
     }
     if (outcome.result === 'expired') {
       counters.expired++

package/telegram-plugin/session-tail.ts

@@ -41,7 +41,7 @@ function isMultiAgentEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
   return env.PROGRESS_CARD_MULTI_AGENT !== '0'
 }
 import { classifyClaudeError, type OperatorEventKind } from './operator-events.js'
-import { isTransientUpstreamSignal } from './model-unavailable.js'
+import { isLitellmProxyLocal429, isTransientUpstreamSignal } from './model-unavailable.js'
 import { createToolLabelSidecar, type ToolLabelSidecar, type SidecarOptions } from './tool-label-sidecar.js'
 import { isModelSentinel } from './model-label.js'
 
@@ -684,9 +684,21 @@ export function detectErrorInTranscriptLine(
     // model-unavailable.ts) — an ambiguous 429 that merely says "limit" stays
     // quota-exhausted, biasing toward surfacing a real wall. Other statuses fall
     // through to the shared classifier.
+    //
+    // A 429 carrying LiteLLM-proxy-LOCAL limiter wording ("Deployment over
+    // user-defined ratelimit", "Rate limit exceeded for api_key: …" — the
+    // canonical `litellmProxyLocal429Signals` list) is the proxy's own
+    // `tpm_limit`/`rpm_limit` cap tripping BEFORE the request reached
+    // Anthropic. Nothing about the account is exhausted — blanket-labeling it
+    // quota-exhausted would fire the model-unavailable card + mark-exhausted +
+    // fleet failover for a purely proxy-local condition. It is rate-limited
+    // (calm path). Precedence when BOTH wordings appear is owned by
+    // `classify429Detail` gateway-side; here both branches yield the same
+    // 'rate-limited' kind, so order is immaterial.
     const kind: OperatorEventKind =
       status === 429
-        ? isTransientUpstreamSignal(`${text}\n${errStr}`)
+        ? isTransientUpstreamSignal(`${text}\n${errStr}`) ||
+          isLitellmProxyLocal429(`${text}\n${errStr}`)
           ? 'rate-limited'
           : 'quota-exhausted'
         : classifyClaudeError({ type: errStr, status, message: text })