switchroom 0.18.13 → 0.18.15

package/telegram-plugin/gateway/gateway.ts

@@ -94,6 +94,13 @@ import {
   buildExtraAttachmentMeta,
   resolveCoalesceMaxAttachments,
 } from './coalesce-attachments.js'
+import {
+  parseForwardOrigin,
+  dedupeForwardOrigins,
+  buildForwardOriginMeta,
+  forwardOriginDateIso,
+  type ForwardOriginInfo,
+} from './forward-origin.js'
 import { StatusReactionController } from '../status-reactions.js'
 import { DeferredDoneReactions } from '../reaction-defer.js'
 import { createWorkerActivityFeed, isWorkerActivityFeedEnabled } from '../worker-activity-feed.js'
@@ -310,7 +317,15 @@ import { recordOperatorEvent } from '../operator-events-history.js'
 import {
   formatModelUnavailableCard,
   resolveModelUnavailableFromOperatorEvent,
+  type ModelUnavailableDetection,
 } from '../model-unavailable.js'
+import {
+  build429ClassifiedMetric,
+  classify429Detail,
+  decideThrottleTier,
+  throttleRetryInPlaceMaxMs,
+} from '../throttle-tier.js'
+import { createThrottleTierRunner } from './throttle-tier-wiring.js'
 import { runFleetAutoFallback, renderFallbackFailureNotice, evaluateFallbackFailureNotice, evaluateAllBlockedNotice, type FallbackFailureNoticeState, type FallbackAllBlockedNoticeState } from '../auto-fallback-fleet.js'
 import { startRestartWatchdog } from './restart-watchdog.js'
 import { validateStringArray } from './access-validator.js'
@@ -7587,6 +7602,14 @@ type CoalescePayload = {
   attachment?: AttachmentMeta
   // Set only by `merge`: the 2nd..Nth attachments folded into this turn.
   extraAttachments?: CoalesceAttachment[]
+  // Forwarded-message origin of THIS entry (parsed at enqueue time from
+  // message.forward_origin). merged.ctx is the LAST entry's ctx, so a
+  // coalesced burst would otherwise lose the earlier entries' origins —
+  // each entry carries its own.
+  forwardOrigin?: ForwardOriginInfo
+  // Set only by `merge`: the distinct origins of the whole burst, arrival
+  // order, deduped (a single-origin album collapses to one entry).
+  forwardOrigins?: ForwardOriginInfo[]
 }
 
 // Count of attachment-bearing entries currently buffered per coalesce key.
@@ -7619,6 +7642,11 @@ const inboundCoalescer = createInboundCoalescer<CoalescePayload>({
       (e) => e.downloadImage != null || e.attachment != null,
       coalesceMaxAttachments(),
     )
+    // Distinct forwarded-message origins across the burst, arrival order.
+    // A forwarded album (one origin, N parts) collapses to a single entry
+    // so the attrs are emitted once; a burst forwarded from two different
+    // senders keeps both (numbered forwarded_from_2 siblings downstream).
+    const forwardOrigins = dedupeForwardOrigins(entries.map((e) => e.forwardOrigin))
     return {
       // Drop empty texts (e.g. caption-less album parts) so the join doesn't
       // emit blank lines between attachments.
@@ -7629,6 +7657,7 @@ const inboundCoalescer = createInboundCoalescer<CoalescePayload>({
       extraAttachments: extras.length > 0
         ? extras.map((e) => ({ downloadImage: e.downloadImage, attachment: e.attachment }))
         : undefined,
+      forwardOrigins: forwardOrigins.length > 0 ? forwardOrigins : undefined,
     }
   },
   onFlush: (key, merged) => {
@@ -7639,6 +7668,7 @@ const inboundCoalescer = createInboundCoalescer<CoalescePayload>({
       merged.downloadImage,
       merged.attachment,
       merged.extraAttachments,
+      merged.forwardOrigins,
     )
   },
 })
@@ -7659,6 +7689,117 @@ const inboundCoalescer = createInboundCoalescer<CoalescePayload>({
 function emitGatewayOperatorEvent(event: OperatorEvent): void {
   const { agent, kind } = event
 
+  // ── 429 throttle tier (operator spec: "retry in place under 5 min, else
+  // mark + failover, honest reset messaging") ────────────────────────────
+  // A terminal TRANSIENT ACCOUNT-scoped 429 — kind `rate-limited` carrying
+  // wording that affirms the account's own rate limit ("would exceed your
+  // account's rate limit") — is decided BEFORE the per-kind cooldown gate:
+  // every hit must reach the broker's mark-throttled escalation counter, and
+  // the >threshold failover ACTION must fire, even when the user-facing card
+  // is cooldown-suppressed (a calm 529 card two minutes earlier arms that
+  // cooldown). Server-side transients (529 / "server is temporarily…") fall
+  // through to the existing calm rate-limited card unchanged — an
+  // account-scoped throttle would be the wrong action for a server-wide
+  // condition.
+  //
+  // Classification is three-way (classify429Detail, throttle-tier.ts):
+  // only `account-scoped` may enter the throttle tier. `litellm-local` —
+  // the LiteLLM proxy's OWN tpm/rpm/router limiter tripping before the
+  // request reached Anthropic — takes the calm path with NO broker mark and
+  // NO failover: the condition is proxy-local and says nothing about the
+  // account. Every rate-limited event ALSO emits one
+  // `rate_limit_429_classified` runtime metric (PostHog + JSONL), fired
+  // here — before the cooldown gate — so operators can correlate
+  // account-scoped 429s with fleet TPM even when the card is suppressed.
+  let throttleEscalation: ModelUnavailableDetection | null = null
+  let escalationFired = false
+  const rateLimit429Classification =
+    kind === 'rate-limited' ? classify429Detail(event.detail) : null
+  if (rateLimit429Classification != null && rateLimit429Classification !== 'account-scoped') {
+    // litellm-local / generic-transient: the existing calm rate-limited
+    // path (fall through to renderOperatorEvent below). NO broker
+    // mark-throttled, NO throttle-tier runner, NO failover — for a
+    // proxy-local cap trip those would bench an account that was never
+    // touched.
+    emitRuntimeMetric(
+      build429ClassifiedMetric({
+        agent,
+        detail: event.detail,
+        classification: rateLimit429Classification,
+        action: 'calm',
+        now: Date.now(),
+      }),
+    )
+    if (rateLimit429Classification === 'litellm-local') {
+      process.stderr.write(
+        `telegram gateway: 429 classified litellm-proxy-local agent=${agent} — ` +
+          `calm path, no account attribution, no failover\n`,
+      )
+    }
+  }
+  if (rateLimit429Classification === 'account-scoped') {
+    const throttleDecision = decideThrottleTier({
+      detail: event.detail,
+      now: Date.now(),
+      thresholdMs: throttleRetryInPlaceMaxMs(),
+    })
+    emitRuntimeMetric(
+      build429ClassifiedMetric({
+        agent,
+        detail: event.detail,
+        classification: 'account-scoped',
+        // decideThrottleTier can't return 'none' for account-scoped wording;
+        // map the two live actions onto the metric's vocabulary.
+        action: throttleDecision.action === 'failover' ? 'failover' : 'throttle',
+        now: Date.now(),
+      }),
+    )
+    if (throttleDecision.action === 'throttle') {
+      // Reset is near (≤ threshold) or unparseable (60s default): DO NOT
+      // fail over. Record throttled_until broker-side, post ONE lightweight
+      // notice (per-account + fleet-wide dedup), nudge a retry after the
+      // reset. The runner owns the sequencing (throttle-tier-wiring.ts).
+      process.stderr.write(
+        `telegram gateway: throttle-tier staying-put agent=${agent} ` +
+          `until=${new Date(throttleDecision.throttledUntilMs).toISOString()} ` +
+          `parsedReset=${throttleDecision.resetParsed}\n`,
+      )
+      try {
+        recordOperatorEvent(event)
+      } catch { /* history is best-effort */ }
+      void throttleTierRunner.fire(
+        agent,
+        throttleDecision.throttledUntilMs,
+        throttleDecision.resetParsed,
+      )
+      return
+    }
+    if (throttleDecision.action === 'failover') {
+      // Reset beyond the retry-in-place threshold: the account is benched
+      // until then anyway, so escalate to the standard mark-exhausted +
+      // fleet-failover machinery — honest 'rate_limited' wording, parsed
+      // reset as the mark expiry. The ACTION fires HERE, before the
+      // per-kind card cooldown below can swallow it (the fleetFallbackGate
+      // provides the action-level dedup); only the card stays
+      // cooldown-bounded. The 'rate-limit' trigger tells the dispatcher to
+      // trust this terminal parsed-reset signal over the utilization probe
+      // (a transient 429 negates the usage-limit reading, so utilization
+      // typically looks healthy — the idempotency guard would self-cancel
+      // the swap).
+      const resetAt = new Date(throttleDecision.resetAtMs)
+      throttleEscalation = { kind: 'rate_limited', resetAt, raw: event.detail }
+      if (wouldFireFleetAutoFallback()) {
+        escalationFired = true
+        void fireFleetAutoFallback(
+          agent,
+          resolveExhaustUntil(resetAt.getTime()),
+          resetAt,
+          'rate-limit',
+        )
+      }
+    }
+  }
+
   if (!shouldEmitOperatorEvent(agent, kind)) {
     process.stderr.write(
       `telegram gateway: operator-event suppressed (cooldown) agent=${agent} kind=${kind}\n`,
@@ -7687,7 +7828,10 @@ function emitGatewayOperatorEvent(event: OperatorEvent): void {
   //     by the upstream classifier
   //   - detail string contains one of the model-unavailable text patterns
   //     (covers raw stderr that slipped past structured classification)
-  const modelUnavailable = resolveModelUnavailableFromOperatorEvent(event)
+  // Throttle-tier escalation (long-reset transient 429) takes precedence:
+  // resolveModelUnavailableFromOperatorEvent deliberately returns null for a
+  // bare rate-limited event, but this one is auto-fallback-eligible.
+  const modelUnavailable = throttleEscalation ?? resolveModelUnavailableFromOperatorEvent(event)
   let renderedText: string
   let renderedKeyboard: ReturnType<typeof renderOperatorEvent>['keyboard'] | undefined
   // #3031 PR 3 — when the card promises an in-flight auto-failover, record
@@ -7709,8 +7853,18 @@ function emitGatewayOperatorEvent(event: OperatorEvent): void {
     // it just produces a self-cancelling "probed healthy / Stale event?"
     // loop on every 529. Overload is handled by Claude Code's own
     // internal retry, not by switching accounts.
-    const isAutoKind = modelUnavailable.kind === 'quota_exhausted'
-    const willActuallyFire = isAutoKind && wouldFireFleetAutoFallback()
+    // 'rate_limited' is the throttle-tier escalation (transient 429 whose
+    // reset exceeds the retry-in-place threshold) — same failover mechanics
+    // as a quota wall, honest wording on the card. Its fire already happened
+    // ABOVE the cooldown gate (escalationFired) so the action can't be
+    // swallowed by the per-kind card cooldown; the card here only reports it.
+    const isAutoKind =
+      modelUnavailable.kind === 'quota_exhausted' ||
+      modelUnavailable.kind === 'rate_limited'
+    const willActuallyFire =
+      throttleEscalation != null
+        ? escalationFired
+        : isAutoKind && wouldFireFleetAutoFallback()
     process.stderr.write(
       `telegram gateway: operator-event suppressing-raw-stderr-for-model-unavailable agent=${agent} kind=${kind} detected=${modelUnavailable.kind} autoKind=${isAutoKind} willFire=${willActuallyFire}\n`,
     )
@@ -7734,9 +7888,14 @@ function emitGatewayOperatorEvent(event: OperatorEvent): void {
     // Pre-fix this called fireFleetAutoFallback(agent) with no until, so a
     // weekly wall that surfaced as a 429 got markExhausted's ~5h default and
     // the broker re-mirrored the still-walled account onto the fleet after 5h.
-    if (willActuallyFire) {
+    // Throttle-tier escalations (throttleEscalation != null) already fired
+    // above the cooldown gate — don't double-fire here.
+    if (willActuallyFire && throttleEscalation == null) {
       const untilMs = resolveExhaustUntil(modelUnavailable.resetAt?.getTime())
-      void fireFleetAutoFallback(agent, untilMs)
+      // Thread the RAW parsed reset (distinct from the floor-applied untilMs)
+      // so the fallback announcement can name it even when the live probe of
+      // the old account fails — honest reset messaging on the enriched card.
+      void fireFleetAutoFallback(agent, untilMs, modelUnavailable.resetAt)
     }
   } else {
     try {
@@ -11374,7 +11533,13 @@ const ipcServer: IpcServer = createIpcServer({
         (msg.resetAt == null ? ' (reset unparsed → +7d default)' : '') +
         ' — triggering fleet auto-fallback\n',
     )
-    void fireFleetAutoFallback(msg.agentName, untilMs)
+    void fireFleetAutoFallback(
+      msg.agentName,
+      untilMs,
+      // Enriched announcement: the sidecar-parsed reset (when present) keeps
+      // the recovery line honest even when the old account's probe fails.
+      msg.resetAt != null ? new Date(msg.resetAt) : undefined,
+    )
   },
 
   // Issue #2971 — read-only wedge-watchdog probe: is there a live pending
@@ -18070,7 +18235,15 @@ async function handleInboundCoalesced(
     ctx.message?.message_thread_id,
     String(from.id),
   )
-  const result = inboundCoalescer.enqueue(key, { text, ctx, downloadImage, attachment })
+  const result = inboundCoalescer.enqueue(key, {
+    text,
+    ctx,
+    downloadImage,
+    attachment,
+    // Parsed HERE (not in merge) because merge only sees the last ctx —
+    // each buffered entry keeps its own forwarded-message origin.
+    forwardOrigin: parseForwardOrigin(ctx.message?.forward_origin),
+  })
   // Coalescing disabled (window <= 0): flush immediately, preserving any
   // media this message carried.
   if (result.bypass) return handleInbound(ctx, text, downloadImage, attachment)
@@ -18149,6 +18322,11 @@ async function handleInbound(
   // resolved (photos downloaded) and surfaced as numbered meta fields
   // (image_path_2, attachment_file_id_2, …) alongside the primary.
   extraAttachments?: CoalesceAttachment[],
+  // Forwarded-message origins for a coalesced burst (deduped, arrival
+  // order — see the coalescer's merge). Left undefined by the direct /
+  // bypass call sites; handleInbound then parses THIS ctx's
+  // forward_origin itself, so single-message forwards work on every path.
+  coalescedForwardOrigins?: ForwardOriginInfo[],
 ): Promise<void> {
   markIdleActivity() // any inbound resets the idle auto-clear timer + re-arms
   const isTopicMessage = ctx.message?.is_topic_message ?? false
@@ -19367,6 +19545,19 @@ async function handleInbound(
     : undefined
   const replyToTextEscaped = formatReplyToText(replyToTextRaw, REPLY_TO_TEXT_MAX)
 
+  // Forwarded-message origin context. `forward_origin` is stamped by
+  // Telegram's servers (Bot API 7.0+) — the forwarding user cannot forge
+  // it via the message body, so it rides the trusted attrs lane and is
+  // NEVER folded into the body text. Coalesced bursts pass their deduped
+  // per-entry origins; direct/bypass paths parse this ctx's own origin.
+  // Same raw-vs-escaped split as reply_to_text: `forwardOrigins` carries
+  // raw (truncated) names for SQLite, `buildForwardOriginMeta` escapes at
+  // the channel-meta boundary.
+  const forwardOrigins = coalescedForwardOrigins
+    ?? dedupeForwardOrigins([parseForwardOrigin(ctx.message?.forward_origin)])
+  const forwardOriginMeta = buildForwardOriginMeta(forwardOrigins)
+  const primaryForwardOrigin = forwardOrigins[0]
+
   if (HISTORY_ENABLED) {
     try {
       recordInbound({
@@ -19380,6 +19571,11 @@ async function handleInbound(
         attachment_kind: attachment?.kind,
         reply_to_message_id: replyToMessageId ?? null,
         reply_to_text: replyToText ?? null,
+        forwarded_from: primaryForwardOrigin?.name ?? null,
+        forwarded_from_type: primaryForwardOrigin?.type ?? null,
+        forwarded_from_id: primaryForwardOrigin?.id != null ? String(primaryForwardOrigin.id) : null,
+        forwarded_date: forwardOriginDateIso(primaryForwardOrigin),
+        forwarded_message_id: primaryForwardOrigin?.messageId ?? null,
       })
     } catch (err) {
       process.stderr.write(`telegram gateway: history recordInbound failed: ${err}\n`)
@@ -19485,6 +19681,11 @@ async function handleInbound(
       // Use the XML-escaped form for the meta — the raw form is in the
       // SQLite buffer for verbatim retrieval via get_recent_messages.
       ...(replyToTextEscaped != null && replyToTextEscaped.length > 0 ? { reply_to_text: replyToTextEscaped } : {}),
+      // Forwarded-message origin (server-stamped, attrs-only — see above).
+      // forwarded_from / forwarded_from_type / forwarded_from_id /
+      // forwarded_date, plus numbered _2.. siblings for a multi-origin
+      // burst. Names are XML-escaped inside buildForwardOriginMeta.
+      ...forwardOriginMeta,
       // queued="true" when mid-turn with no steer prefix (new default), or
       // with explicit /queue or /q prefix (legacy alias).
       ...((isQueuedMidTurn || isQueuedPrefix) ? { queued: 'true' } : {}),
@@ -23203,9 +23404,14 @@ function wouldFireFleetAutoFallback(): boolean {
  * so the user sees the outcome inline with the original "Model
  * unavailable" card.
  */
-async function fireFleetAutoFallback(triggerAgent: string, untilMs?: number): Promise<void> {
+async function fireFleetAutoFallback(
+  triggerAgent: string,
+  untilMs?: number,
+  parsedResetAt?: Date,
+  trigger?: 'rate-limit',
+): Promise<void> {
   return fleetFallbackGate.fire(
-    () => doFireFleetAutoFallback(triggerAgent, untilMs),
+    () => doFireFleetAutoFallback(triggerAgent, untilMs, parsedResetAt, trigger),
     (err) => {
       process.stderr.write(
         `telegram gateway: [fleet-fallback] error agent=${triggerAgent}: ${(err as Error)?.message ?? err}\n`,
@@ -23214,6 +23420,45 @@ async function fireFleetAutoFallback(triggerAgent: string, untilMs?: number): Pr
   )
 }
 
+// ─── 429 throttle tier — side-effect wiring ─────────────────────────────────
+// Decision + notice text live in throttle-tier.ts (pure); the SEQUENCING
+// (broker mark → fleet-deduped notice → jittered retry nudge with the
+// live-turn safety guards) lives in throttle-tier-wiring.ts so it is
+// unit-testable with injected deps. This block only binds the real gateway
+// dependencies.
+const throttleTierRunner = createThrottleTierRunner({
+  agentName: process.env.SWITCHROOM_AGENT_NAME ?? '',
+  getBrokerClient: () =>
+    getAuthBrokerClient(process.env.SWITCHROOM_AGENT_NAME ?? ''),
+  listNoticeChats: () => loadAccess().allowFrom,
+  sendNotice: (chat_id, markdown) => {
+    // Status notice, not the user's answer — silence the ping (same posture
+    // as the fleet-fallback announcement).
+    void swallowingApiCall(
+      // allow-raw-bot-api: wrapped in swallowingApiCall (retry policy)
+      () => bot.api.sendRichMessage(chat_id, richMessage(markdown), { disable_notification: true }),
+      { chat_id: String(chat_id), verb: 'throttle-tier:notify' },
+    )
+  },
+  resumeDecide: (ts) => fleetFallbackResumeGate.decide(ts),
+  newestActiveTurnStartedAtMs,
+  turnInFlight: () => turnInFlightForGate(),
+  deferRestartToTurnComplete: (agentName, reason) => {
+    // Same lever the schedule_restart IPC uses: the restart drains at the
+    // turn-complete gate (never SIGTERM-now under a live turn). Note the
+    // pending-restart drain cap (PENDING_RESTART_DRAIN_CAP_MS) still bounds
+    // a never-idling session — deliberate, matching scheduled restarts.
+    process.stderr.write(
+      `telegram gateway: [throttle-tier] restart deferred to turn-complete agent=${agentName} reason=${reason}\n`,
+    )
+    pendingRestarts.set(agentName, Date.now())
+  },
+  restartNow: (agentName, reason) => {
+    triggerSelfRestart(agentName, reason)
+  },
+  log: (m) => process.stderr.write(`telegram gateway: ${m}\n`),
+})
+
 /**
  * Broadcast a fleet-fallback FAILURE notice to every authorized chat.
  *
@@ -23275,7 +23520,12 @@ function broadcastFleetFallbackFailure(triggerAgent: string, reason: string): vo
  *  user-visible announcement was broadcast). False on no-op /
  *  error / idempotent-skip — caller uses this to decide whether to
  *  arm the post-fire suppression window. */
-async function doFireFleetAutoFallback(triggerAgent: string, untilMs?: number): Promise<boolean> {
+async function doFireFleetAutoFallback(
+  triggerAgent: string,
+  untilMs?: number,
+  parsedResetAt?: Date,
+  trigger?: 'rate-limit',
+): Promise<boolean> {
   try {
     const client = await getAuthBrokerClient(triggerAgent)
     if (!client) {
@@ -23324,6 +23574,16 @@ async function doFireFleetAutoFallback(triggerAgent: string, untilMs?: number):
       },
       triggerAgent,
       tz,
+      // Enriched card (429 throttle tier): the RAW reset parsed from the
+      // error prose, so the announcement names when the old account frees
+      // even when its live probe returned nothing.
+      parsedResetAt,
+      // Throttle-tier escalation: a terminal transient 429 with a long
+      // parsed reset NEGATES the usage-limit reading, so the old account's
+      // utilization probe typically classifies healthy — the idempotency
+      // guard would self-cancel the swap ("probed healthy / Stale event?").
+      // Trust the terminal parsed-reset signal over the utilization probe.
+      rateLimitTrigger: trigger === 'rate-limit',
     })
     process.stderr.write(
       `telegram gateway: [fleet-fallback] outcome=${outcome.kind} agent=${triggerAgent}` +