switchroom 0.19.27 → 0.19.29

package/telegram-plugin/dist/server.js

@@ -17267,6 +17267,7 @@ var init_operator_events = __esm(() => {
     "credentials-invalid",
     "credit-exhausted",
     "provider-credit-exhausted",
+    "mcp-dependency-blocked",
     "proxy-misconfig"
   ]);
 });

package/telegram-plugin/edit-flood-fuse.ts

@@ -165,8 +165,46 @@
  * `maxTightenLevel`) and decays one level at a time, so sustained pressure
  * ratchets the cosmetic rate down towards the floor of 1/window.
  *
+ * ── 2026-07-28: the reserve could not fire, and the reply paid (#3885) ────
+ * Points 3 and 5 above, and the escalating backoff, combined into a defect
+ * that hit the operator directly: during a 12-agent staggered restart that drew
+ * **three** genuine 429s, one chat logged 792 fuse events in an hour against a
+ * 35-50/hr baseline, shed 221 typing indicators, deferred 18 reactions, and
+ * deferred real replies long enough that the MCP `reply` tool's 60s timeout
+ * fired — so the agent retried and the operator got the same answer twice. One
+ * ~1900-character reply was lost outright.
+ *
+ * Three 3-second 429s ratchet `tightenLevel` to 3, and each one also writes
+ * `flood-wait.json`, which pinned `levelAt` at `maxTightenLevel` outright. At
+ * level 4 the shared per-chat budget is `max(1, floor(20 * 0.5^4))` = **1 call
+ * per minute for the entire chat**. The reserve that exists for precisely this
+ * case could not help, because it was a fixed count subtracted from the BASE:
+ * `max(1, 20 - 8)` = 12, tightened to 1 — the same 1 the reply had. Reply and
+ * repaint ended up with an identical budget, which is the opposite of a
+ * reservation. Three changes:
+ *
+ *   6. **The reserve is a PROPORTION, re-derived per window.** `cosmeticTotalMax`
+ *      computes it against the EFFECTIVE ceiling, so "8 of 20" means "40% of
+ *      whatever the budget currently is", and at least one slot is always
+ *      reserved. When the effective budget shrinks to the reserve, cosmetic
+ *      traffic gets 0 and every remaining slot belongs to a reply.
+ *   7. **A hard floor under `critical`.** `perChatCriticalMinPerWindow`
+ *      (default 3) is a rate no tightening may cross, on every per-chat tier.
+ *      Shedding a typing indicator under pressure is correct; holding the
+ *      answer past the caller's own timeout is not — it does not save a call,
+ *      it doubles it.
+ *   8. **A persisted flood window tightens IN PROPORTION to what remains**,
+ *      through the same severity ladder `noteFlood` uses, instead of jumping
+ *      straight to `maxTightenLevel`. `makeFloodWaitRecorder` writes the marker
+ *      for every 429 including a 3-second nudge, so the flat jump reintroduced
+ *      on the persisted path exactly the "a nudge and a 4.4h ban are the same
+ *      signal" defect that #3856 had just fixed on the in-memory path.
+ *
  * Every ceiling is operator-overridable via env — see
- * {@link editFloodFuseConfigFromEnv}. Previously none of them were.
+ * {@link editFloodFuseConfigFromEnv}. Previously none of them were, and until
+ * #3885 the tightening CURVE (`maxTightenLevel`, `tightenFactor`) still was
+ * not, which is why the only available workaround for the above was to inflate
+ * the base ceiling fleet-wide.
  */
 
 import { createHash } from 'node:crypto'
@@ -302,8 +340,29 @@ export interface EditFloodFuseConfig {
    * consume. Default 8. This is the reply-starvation guarantee: whatever the
    * repaint surfaces do, at least this many calls per window remain available
    * to an actual answer.
+   *
+   * Read as a PROPORTION of the base, not an absolute count (#3885). The
+   * reserve used to be a fixed number subtracted from the base ceiling, which
+   * made it arithmetically dead under tightening: at `maxTightenLevel` the base
+   * 20 collapses to an effective 1, and `1 - 8` clamps to the same floor of 1
+   * that cosmetic traffic already had — so reply and repaint ended up with an
+   * identical budget in exactly the situation the reserve exists for. It is now
+   * re-derived against the EFFECTIVE ceiling every window, and always leaves at
+   * least one slot reserved.
    */
   perChatReplyReserve?: number
+  /**
+   * Absolute floor, per `perChatWindowMs`, on `critical`-class traffic in one
+   * chat — the operator's actual answer, an approval card, a reaction. Default
+   * 3. No amount of tightening may take a chat below this: a reply held past
+   * the caller's own timeout is not "shedding under pressure", it is a lost
+   * answer plus a retry that sends it twice (#3885). Cosmetic traffic has no
+   * floor and is still shed to zero, which is the correct trade.
+   *
+   * Capped by the tier's own base ceiling, so an operator who deliberately
+   * configures a base below this floor still gets what they configured.
+   */
+  perChatCriticalMinPerWindow?: number
   /**
    * Ceiling on EVERY outbound call made with this bot token, across all chats
    * and including the chat-less ones. Default 25 per `perTokenWindowMs`
@@ -340,7 +399,11 @@ export interface EditFloodFuseConfig {
   tightenFactor?: number
   /** How long ONE level of tightening stays in force before decaying. Default 10 minutes. */
   tightenMs?: number
-  /** Cap on compounding 429 tightening levels. Default 4 (⇒ 0.5^4 = 1/16). */
+  /**
+   * Cap on compounding 429 tightening levels. Default 4 (⇒ 0.5^4 = 1/16).
+   * Operator-overridable via `SWITCHROOM_EDIT_FUSE_MAX_TIGHTEN_LEVEL` (#3885);
+   * `0` disables tightening entirely.
+   */
   maxTightenLevel?: number
   /**
    * Remaining ms of a KNOWN-OPEN Telegram flood window, or 0 when none — the
@@ -389,6 +452,18 @@ export interface EditFloodFuseStats {
   cosmeticPerMessageCeiling: number
   /** Live COSMETIC per-chat ceiling (post-AIMD). */
   cosmeticPerChatCeiling: number
+  /**
+   * Live shared per-chat budget available to COSMETIC traffic (post-AIMD, after
+   * the reply reserve). Goes to 0 under heavy tightening — that is the reserve
+   * working, not a fault.
+   */
+  cosmeticPerChatTotalCeiling: number
+  /**
+   * Live shared per-chat budget available to CRITICAL traffic (post-AIMD, after
+   * the critical floor). The direct measure of #3885: this must never drop
+   * below `perChatCriticalMinPerWindow`, at any tighten level.
+   */
+  criticalPerChatTotalCeiling: number
   /**
    * Chat-targeted calls charged by the DEFAULT-DENY rule — every one of these
    * was completely unmetered before #3855. A non-zero value here is the direct
@@ -430,8 +505,18 @@ export const EDIT_FLOOD_FUSE_DEFAULTS = {
    * edits together — so this is the only window that reflects the real budget.
    */
   perChatTotalMaxPerWindow: 20,
-  /** 8 of those 20 slots/min are unreachable by cosmetic traffic. */
+  /**
+   * 8 of those 20 slots/min are unreachable by cosmetic traffic — i.e. 40% of
+   * whatever the EFFECTIVE ceiling is, re-derived per window (#3885).
+   */
   perChatReplyReserve: 8,
+  /**
+   * 3/60s. The hard floor on `critical` traffic in a chat. Sized against the
+   * thing that broke: the MCP `reply` tool times out at 60s, so a chat that can
+   * pass fewer than a couple of criticals a minute turns one answer into a
+   * timeout, a retry, and a duplicate message.
+   */
+  perChatCriticalMinPerWindow: 3,
   /**
    * 25 per second across the whole bot token. Telegram enforces ~30/s at the
    * token level and the ban it issues is token-scoped; 25 leaves headroom for
@@ -460,6 +545,20 @@ function envInt(raw: string | undefined): number | undefined {
   return Number.isFinite(n) && n >= 0 ? Math.floor(n) : undefined
 }
 
+/**
+ * Parse a multiplicative-decrease factor from env: a real in (0, 1].
+ *
+ * Rejects 0 and negatives (which would zero every ceiling) and anything above
+ * 1 (which would make a 429 LOOSEN the fuse). An out-of-range value is treated
+ * as unset rather than clamped — silently reinterpreting an operator's number
+ * is how a fuse ends up at a rate nobody chose.
+ */
+function envFactor(raw: string | undefined): number | undefined {
+  if (raw == null || raw.trim() === '') return undefined
+  const n = Number(raw)
+  return Number.isFinite(n) && n > 0 && n <= 1 ? n : undefined
+}
+
 /**
  * Operator config surface. Before the 2026-07-27 incident the fuse had exactly
  * one knob — `SWITCHROOM_EDIT_FUSE=0`, which turns the whole failsafe OFF —
@@ -478,8 +577,16 @@ export function editFloodFuseConfigFromEnv(
   assign('cosmeticPerChatMaxPerWindow', envInt(env.SWITCHROOM_FEED_EDIT_MAX_PER_CHAT_PER_MIN))
   assign('perChatTotalMaxPerWindow', envInt(env.SWITCHROOM_CHAT_TOTAL_MAX_PER_MIN))
   assign('perChatReplyReserve', envInt(env.SWITCHROOM_CHAT_REPLY_RESERVE))
+  assign('perChatCriticalMinPerWindow', envInt(env.SWITCHROOM_CHAT_CRITICAL_MIN_PER_MIN))
   assign('perTokenMaxPerWindow', envInt(env.SWITCHROOM_TOKEN_MAX_PER_SEC))
   assign('maxDeferMs', envInt(env.SWITCHROOM_EDIT_FUSE_MAX_DEFER_MS))
+  // #3885 — the tightening curve itself is now operator-tunable. Before this,
+  // `SWITCHROOM_CHAT_TOTAL_MAX_PER_MIN` was the ONLY lever on it, so an
+  // operator whose chats were collapsing under `factor^maxLevel` had to inflate
+  // the BASE ceiling fleet-wide to compensate — raising the untightened rate
+  // (the one that earns bans) to fix the tightened one.
+  assign('maxTightenLevel', envInt(env.SWITCHROOM_EDIT_FUSE_MAX_TIGHTEN_LEVEL))
+  assign('tightenFactor', envFactor(env.SWITCHROOM_EDIT_FUSE_TIGHTEN_FACTOR))
   // #3856 — durable ban awareness. The fuse reads the SAME persisted marker
   // (`flood-wait.json`) that `robustApiCall` and the outbox sweep consult, so a
   // restart mid-ban comes back tightened instead of at full rate. Wired here
@@ -529,12 +636,21 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
     perChatEditMax, config.cosmeticPerChatMaxPerWindow ?? D.cosmeticPerChatMaxPerWindow)
   const perChatTotalMax = config.perChatTotalMaxPerWindow ?? D.perChatTotalMaxPerWindow
   const perChatWindowMs = config.perChatWindowMs ?? D.perChatWindowMs
-  // Clamped so a misconfigured reserve can never make the cosmetic allowance
-  // negative (deadlocking every card) or eat the whole budget.
+  // Clamped so a misconfigured reserve can never eat more than the whole
+  // budget. Note this clamp is on the BASE only — the reserve that actually
+  // binds is re-derived per window by `cosmeticTotalMax` (#3885).
   const perChatReplyReserve = Math.min(
     Math.max(0, config.perChatReplyReserve ?? D.perChatReplyReserve),
     Math.max(0, perChatTotalMax - 1),
   )
+  /**
+   * The reserve as a FRACTION of the base. This is the whole #3885 fix: the
+   * reserve has to mean "40% of the chat's budget belongs to replies", not "8
+   * calls", because 8 is meaningless once the effective budget is 1.
+   */
+  const replyReserveFraction = perChatTotalMax > 0 ? perChatReplyReserve / perChatTotalMax : 0
+  const perChatCriticalMin = Math.max(
+    0, config.perChatCriticalMinPerWindow ?? D.perChatCriticalMinPerWindow)
   const perTokenMax = Math.max(1, config.perTokenMaxPerWindow ?? D.perTokenMaxPerWindow)
   const perTokenWindowMs = Math.max(1, config.perTokenWindowMs ?? D.perTokenWindowMs)
   // Only ever the PUBLIC bot id (or an irreversible hash) reaches this key.
@@ -602,12 +718,29 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
       tightenLevel--
       tightenedUntil = tightenLevel > 0 ? tightenedUntil + tightenMs : 0
     }
-    // #3856 — a KNOWN-OPEN persisted window pins the fuse at its tightest
-    // regardless of in-memory state. This is what survives a restart: the
-    // process forgets, the marker on disk does not. It does not MUTATE
-    // `tightenLevel`, so when the window closes the fuse returns to whatever
-    // this process actually earned rather than staying pinned.
-    if (persistedFloodRemainingMs(now) > 0) return maxTightenLevel
+    // #3856 — a KNOWN-OPEN persisted window pins the fuse regardless of
+    // in-memory state. This is what survives a restart: the process forgets,
+    // the marker on disk does not. It does not MUTATE `tightenLevel`, so when
+    // the window closes the fuse returns to whatever this process actually
+    // earned rather than staying pinned.
+    //
+    // #3885 — but pinned at a level PROPORTIONAL to the window still open, not
+    // unconditionally at `maxTightenLevel`. `flood-wait.json` is written for
+    // EVERY 429 including a routine 3-second nudge (`makeFloodWaitRecorder` →
+    // `computeFloodWait`), so the flat jump meant a 3s burst nudge and a 4.4h
+    // ban produced the identical maximal response — the exact asymmetry #3856
+    // fixed for the in-memory path and left in place here. The remaining window
+    // is graded through the SAME severity ladder `noteFlood` uses, so a big ban
+    // still pins at maximum and a nudge costs one level.
+    //
+    // Clamped by `maxTightenLevel` like every other path: `tightenStepFor`
+    // returns a fixed 1..3 for the smaller bands, so an operator who sets
+    // `SWITCHROOM_EDIT_FUSE_MAX_TIGHTEN_LEVEL=0` to disable tightening entirely
+    // must not find the fuse tightening anyway the moment a marker exists.
+    const remainingMs = persistedFloodRemainingMs(now)
+    if (remainingMs > 0) {
+      return Math.max(tightenLevel, Math.min(maxTightenLevel, tightenStepFor(remainingMs / 1000)))
+    }
     return tightenLevel
   }
 
@@ -626,6 +759,45 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
     return Math.max(1, Math.floor(base * Math.pow(tightenFactor, level)))
   }
 
+  /**
+   * The ceiling a given CLASS sees on a tier whose base is `base` (#3885).
+   *
+   * `critical` traffic — the operator's answer, an approval card, a reaction —
+   * gets an absolute floor that tightening cannot cross. Everything else takes
+   * the tightened ceiling as-is, so shedding still happens; it just happens to
+   * the traffic that can afford it.
+   *
+   * The floor is capped by `base` so it can only ever RAISE a tightened ceiling
+   * back towards what the operator configured, never above it.
+   */
+  function classCeiling(base: number, cls: OutboundClass, now: number): number {
+    const eff = ceiling(base, now)
+    if (cls !== 'critical') return eff
+    return Math.max(eff, Math.min(base, perChatCriticalMin))
+  }
+
+  /**
+   * The shared per-chat budget COSMETIC traffic may reach, re-derived against
+   * the EFFECTIVE ceiling (#3885).
+   *
+   * Returns 0 when the effective budget has shrunk to the reserve — at that
+   * point every remaining slot belongs to replies, and a repaint waits for the
+   * window or is shed. That is the intended behaviour under real flood
+   * pressure: the operator loses refresh frequency, never the answer.
+   */
+  function cosmeticTotalMax(now: number): number {
+    const eff = ceiling(perChatTotalMax, now)
+    if (replyReserveFraction <= 0) return eff
+    const reserve = Math.min(eff, Math.max(1, Math.round(eff * replyReserveFraction)))
+    return Math.max(0, eff - reserve)
+  }
+
+  /** Shared per-chat budget resolver, per class, evaluated at admission time. */
+  function totalMaxFor(cls: OutboundClass): (now: number) => number {
+    if (cls === 'cosmetic') return cosmeticTotalMax
+    return (now: number) => classCeiling(perChatTotalMax, cls, now)
+  }
+
   function win(key: string): Window {
     let w = windows.get(key)
     if (w === undefined) {
@@ -668,6 +840,14 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
   /** ms until `w` has room, or 0 if it has room now. */
   function waitFor(w: Window, now: number, windowMs: number, max: number): number {
     prune(w, now, windowMs)
+    // A ceiling of 0 is reachable now that the reply reserve is derived against
+    // the EFFECTIVE budget (#3885): under heavy tightening the whole remaining
+    // budget belongs to replies, so cosmetic traffic has none. There is no
+    // "oldest slot" to wait behind in that case, so wait out the window rather
+    // than indexing an empty array (which would produce NaN and admit).
+    if (max <= 0) {
+      return w.ts.length > 0 ? Math.max(1, w.ts[0]! + windowMs - now) : Math.max(1, windowMs)
+    }
     if (w.ts.length < max) return 0
     return Math.max(1, w.ts[0]! + windowMs - now)
   }
@@ -770,7 +950,16 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
    * Returns the reserved timestamp, or null when the call must be dropped.
    */
   async function awaitRoom(
-    key: string, windowMs: number, max: number, method: string, mode: WaitMode, cls: OutboundClass,
+    key: string, windowMs: number,
+    /**
+     * The ceiling, resolved at EVERY loop iteration rather than once at entry.
+     * It has to be a function: the effective ceiling depends on the tighten
+     * level, the tighten level decays with time, and a call can sit in this
+     * loop for `maxDeferMs`. Passing a number would pin a waiter to the ceiling
+     * that was in force when it arrived.
+     */
+    maxFor: (now: number) => number,
+    method: string, mode: WaitMode, cls: OutboundClass,
     /**
      * Consulted ONLY at the defer deadline for `mode: 'drop'`. Returning false
      * converts the drop into a late release: this call is the last thing that
@@ -805,7 +994,7 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
     let counted = false
     for (;;) {
       const now = clock.now()
-      const wait = waitFor(w, now, windowMs, ceiling(max, now))
+      const wait = waitFor(w, now, windowMs, maxFor(now))
       if (wait === 0) {
         w.ts.push(now)
         return now
@@ -897,22 +1086,29 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
     // It cannot take a per-chat slot (there is no chat to charge), but it does
     // consume the bot token's global allowance, which is the thing Telegram
     // actually bans. Charge it there and pass.
+    // The token tier is per-SECOND and global; a class floor there would be
+    // meaningless (even at maximum tightening it admits 60/min), so it takes
+    // the plain tightened ceiling.
+    const tokenMaxFor = (t: number): number => ceiling(perTokenMax, t)
+
     if (chat == null) {
       counters.chatless++
-      await awaitRoom(tokenKey, perTokenWindowMs, perTokenMax, method, 'release', cls, undefined, deadline)
+      await awaitRoom(tokenKey, perTokenWindowMs, tokenMaxFor, method, 'release', cls, undefined, deadline)
       return runObserved(next)
     }
 
     const totalKey = `t:${chat}`
-    // Cosmetic traffic may only reach `perChatTotalMax - perChatReplyReserve`;
-    // the remaining slots stay available to a real reply no matter how hot the
-    // repaint surfaces are. This reservation is the reply-starvation guarantee.
-    const totalMaxFor = (c: OutboundClass): number =>
-      c === 'cosmetic' ? Math.max(1, perChatTotalMax - perChatReplyReserve) : perChatTotalMax
+    // Cosmetic traffic may only reach the effective ceiling MINUS the reply
+    // reserve; the remaining slots stay available to a real reply no matter how
+    // hot the repaint surfaces are, and `critical` additionally cannot be
+    // tightened below `perChatCriticalMinPerWindow`. Together these are the
+    // reply-starvation guarantee, and unlike the pre-#3885 shape they hold at
+    // every tighten level rather than only at level 0.
+    const chatTotalMax = totalMaxFor(cls)
 
     /** Final tier for every chat-targeted path: the token-scoped ceiling. */
     const passToken = async (): Promise<R> => {
-      await awaitRoom(tokenKey, perTokenWindowMs, perTokenMax, method, 'release', cls, undefined, deadline)
+      await awaitRoom(tokenKey, perTokenWindowMs, tokenMaxFor, method, 'release', cls, undefined, deadline)
       return runObserved(next)
     }
 
@@ -925,7 +1121,9 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
       try {
         const msgSlot = await awaitRoom(
           msgKey, perMessageWindowMs,
-          cls === 'cosmetic' ? cosmeticPerMessageMax : perMessageMax,
+          cls === 'cosmetic'
+            ? (t) => ceiling(cosmeticPerMessageMax, t)
+            : (t) => classCeiling(perMessageMax, cls, t),
           method, 'supersede', cls, undefined, deadline,
         )
         if (msgSlot === null) return DROPPED_RESULT as unknown as R
@@ -941,7 +1139,9 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
         const chatKey = `ce:${chat}`
         const chatSlot = await awaitRoom(
           chatKey, perChatWindowMs,
-          cls === 'cosmetic' ? cosmeticPerChatMax : perChatEditMax,
+          cls === 'cosmetic'
+            ? (t) => ceiling(cosmeticPerChatMax, t)
+            : (t) => classCeiling(perChatEditMax, cls, t),
           method, 'drop', cls, dropGuard, deadline, lateKey,
         )
         if (chatSlot === null) { giveBack(); return DROPPED_RESULT as unknown as R }
@@ -951,7 +1151,7 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
         // metering. A non-cosmetic edit is RELEASED late rather than dropped:
         // an approval card or answer finalisation has no newer frame coming.
         const totalSlot = await awaitRoom(
-          totalKey, perChatWindowMs, totalMaxFor(cls), method,
+          totalKey, perChatWindowMs, chatTotalMax, method,
           cls === 'cosmetic' ? 'drop' : 'release', cls, dropGuard, deadline, lateKey,
         )
         if (totalSlot === null) { giveBack(); return DROPPED_RESULT as unknown as R }
@@ -965,8 +1165,9 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
       // Sends create user-visible output and are NEVER dropped — only paced.
       // The shared budget's reserve is what guarantees they still have room
       // when the repaint surfaces are saturated.
-      await awaitRoom(`cs:${chat}`, perChatWindowMs, perChatSendMax, method, 'release', cls, undefined, deadline)
-      await awaitRoom(totalKey, perChatWindowMs, totalMaxFor(cls), method, 'release', cls, undefined, deadline)
+      await awaitRoom(`cs:${chat}`, perChatWindowMs,
+        (t) => classCeiling(perChatSendMax, cls, t), method, 'release', cls, undefined, deadline)
+      await awaitRoom(totalKey, perChatWindowMs, chatTotalMax, method, 'release', cls, undefined, deadline)
       return passToken()
     }
 
@@ -982,14 +1183,14 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
       // short deadline also keeps a typing ping from occupying a 30s hold.
       const actionDeadline = Math.min(deadline, now + chatActionMaxDeferMs)
       const slot = await awaitRoom(
-        totalKey, perChatWindowMs, totalMaxFor(cls), method, 'drop', cls, undefined, actionDeadline,
+        totalKey, perChatWindowMs, chatTotalMax, method, 'drop', cls, undefined, actionDeadline,
       )
       if (slot === null) return DROPPED_RESULT as unknown as R
       return passToken()
     }
     // Durable, user-visible effect (a deletion, a pin, a reaction): paced,
     // never dropped — the same contract sends get.
-    await awaitRoom(totalKey, perChatWindowMs, totalMaxFor(cls), method, 'release', cls, undefined, deadline)
+    await awaitRoom(totalKey, perChatWindowMs, chatTotalMax, method, 'release', cls, undefined, deadline)
     return passToken()
   }
 
@@ -1026,6 +1227,8 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
       perMessageCeiling: ceiling(perMessageMax, now),
       cosmeticPerMessageCeiling: ceiling(cosmeticPerMessageMax, now),
       cosmeticPerChatCeiling: ceiling(cosmeticPerChatMax, now),
+      cosmeticPerChatTotalCeiling: cosmeticTotalMax(now),
+      criticalPerChatTotalCeiling: classCeiling(perChatTotalMax, 'critical', now),
       meteredByDefault: counters.meteredByDefault,
       chatless: counters.chatless,
       perTokenCeiling: ceiling(perTokenMax, now),

package/telegram-plugin/gateway/callback-query-handlers.ts

@@ -766,6 +766,7 @@ async function handleVaultRecentDenialCallback(ctx: Context, data: string): Prom
     `(grant \`${id}\`). ` +
     `Re-run /vault audit to act on remaining denials.`
   await finalizeCallback(ctx, {
+    apiCall: robustApiCall,
     ackText: '✅ Grant minted',
     newText: baseText ? `${baseText}${statusLine}` : statusLine,
     // No synthInbound — operator-only flow. The granted agent picks
@@ -1909,6 +1910,7 @@ async function handleVaultRequestSaveCallback(ctx: Context, data: string): Promi
       `\n\n✏️ **Rename mode** — send the new key name as your next message. ` +
       `The current proposed key is \`${pending.key}\`.`
     await finalizeCallback(ctx, {
+      apiCall: robustApiCall,
       ackText: 'Send the new key name as your next message.',
       newText: baseText ? `${baseText}${statusLine}` : statusLine,
     })
@@ -2512,6 +2514,7 @@ async function executeGrantWizard(ctx: Context, chatId: string, state: Extract<P
   const successText = `✅ Grant \`${id}\` created. Written to \`~/.switchroom/agents/${escapeHtmlForTg(state.agent!)}/.vault-token\``
   if (msgId != null) {
     await finalizeCallback(ctx, {
+      apiCall: robustApiCall,
       ackText: '✅ Grant created',
       newText: successText,
       // No synthInbound — operator-only flow.
@@ -2933,6 +2936,7 @@ async function handleOperatorEventCallback(ctx: Context, data: string): Promise<
       // scrollback shows the dismissal.
       const status = `\n\n✗ _Dismissed by operator._`
       await finalizeCallback(ctx, {
+        apiCall: robustApiCall,
         ackText: 'Dismissed',
         newText: sourceMsgText ? `${sourceMsgText}${status}` : status,
         // No synthInbound — dismiss is operator-only, no model in loop.
@@ -2947,6 +2951,7 @@ async function handleOperatorEventCallback(ctx: Context, data: string): Promise<
         // status line is the announcement, no separate reply needed.
         const status = `\n\n🔄 _**${escapeHtmlForTg(agent)}** restart requested by operator._`
         await finalizeCallback(ctx, {
+          apiCall: robustApiCall,
           ackText: `Restarting ${agent}…`,
           newText: sourceMsgText ? `${sourceMsgText}${status}` : status,
         })
@@ -2973,6 +2978,7 @@ async function handleOperatorEventCallback(ctx: Context, data: string): Promise<
       // collapsed card.
       const status = `\n\n🔐 _Reauth started for **${escapeHtmlForTg(agent)}** — follow the login URL below._`
       await finalizeCallback(ctx, {
+        apiCall: robustApiCall,
         ackText: `Starting reauth for ${agent}…`,
         newText: sourceMsgText ? `${sourceMsgText}${status}` : status,
         synthInbound: async () => {

package/telegram-plugin/gateway/gateway.ts

@@ -404,6 +404,7 @@ import {
   type OperatorEvent,
   type OperatorEventKind,
 } from '../operator-events.js'
+import { createMcpFailureHook } from './mcp-failure-hook.js'
 import { pendingUserNoticeGate } from '../pending-user-notice.js'
 import { recordOperatorEvent } from '../operator-events-history.js'
 import {
@@ -14007,7 +14008,13 @@ function surfaceConsolidationLegibility(
   })
 }
 
+/** Blocked-paid-dependency alerting — policy in `mcp-credential-failure.ts`,
+ *  seam in `gateway/mcp-failure-hook.ts`. Tees the SAME
+ *  `emitGatewayOperatorEvent` path so there is ONE operator-alert mechanism. */
+const noteMcpDependencyFailure = createMcpFailureHook({ agent: AGENT_NAME, emit: emitGatewayOperatorEvent })
+
 function handleSessionEvent(ev: SessionEvent): void {
+  noteMcpDependencyFailure(ev)
   handleSessionEventCore(gatewayStreamRenderDeps(), ev)
 }
 
@@ -21948,7 +21955,7 @@ bot.on('callback_query:data', async ctx => {
     // ALREADY dispatched above (independently of this host round-trip)
     // so the turn never blocked — finalizeCallback here only edits the
     // card; no synthInbound (would double-fire the verdict).
-    await finalizeCallback(ctx, {
+    await finalizeCallback(ctx, { apiCall: robustApiCall,
       ackText: 'Rule applied for this session; saving durably in background…'.slice(0, 200),
       newText: baseText ? `${baseText}\n\n${interimLabel}` : interimLabel,
     })
@@ -22260,7 +22267,7 @@ bot.on('callback_query:data', async ctx => {
   // decision. Operator could re-tap and flip Deny → Allow after the
   // permission was already broadcast. Routing through finalizeCallback
   // strips the keyboard atomically with the status-line edit.
-  await finalizeCallback(ctx, {
+  await finalizeCallback(ctx, { apiCall: robustApiCall,
     ackText: ackText.slice(0, 200),
     newText: baseText ? `${baseText}\n\n${labelWithResume}` : labelWithResume,
     synthInbound: () => {

package/telegram-plugin/gateway/mcp-failure-hook.ts

@@ -0,0 +1,74 @@
+/**
+ * mcp-failure-hook.ts — the gateway-side wiring that turns a blocked PAID MCP
+ * DEPENDENCY into an operator alert.
+ *
+ * Ken: "If perplexity fails for any agent we should have hard warning via
+ * message here per other failures."
+ *
+ * MCP providers (Perplexity, Eraser, Brevo, Postiz, Meta/Google Ads,
+ * Cloudflare) never touch LiteLLM, so the operator-event path that catches an
+ * OpenRouter 402 never sees them — an expired Perplexity key would otherwise
+ * die as a transient red step in the live feed and nobody would learn the fleet
+ * had lost a paid capability.
+ *
+ * This tees the SAME `emitGatewayOperatorEvent` path (redact → per-kind
+ * cooldown → operator-only audience via `OPERATOR_ACTIONABLE_KINDS` → brief
+ * user notice), so there is ONE operator-alert mechanism in switchroom, not
+ * two. All the policy — what counts as a key problem, what stays silent, the
+ * 6-hour re-notify cadence — lives in `../mcp-credential-failure.ts`; this file
+ * is only the seam.
+ *
+ * Extracted from `gateway.ts` rather than inlined there, per the
+ * switchroom#2996 anti-inflation ratchet (`scripts/check-gateway-line-ratchet.mjs`).
+ */
+
+import type { SessionEvent } from '../session-tail.js'
+import type { OperatorEvent } from '../operator-events.js'
+import { McpFailureWatcher, renderMcpFailureDetail } from '../mcp-credential-failure.js'
+
+export interface McpFailureHookDeps {
+  agent: string
+  emit: (event: OperatorEvent) => void
+  now?: () => number
+}
+
+/**
+ * Build the `handleSessionEvent` side-hook. One watcher per gateway process,
+ * captured in the closure.
+ *
+ * Silent by default: only a credit/credential/quota class raises anything, and
+ * only once per (server, class) per re-notify window. A bad query, a 404, a
+ * timeout, a transient 429 or a successful call all return without emitting.
+ *
+ * Never throws — alerting is a side channel and must not break the live feed.
+ */
+export function createMcpFailureHook(deps: McpFailureHookDeps): (ev: SessionEvent) => void {
+  const watcher = new McpFailureWatcher()
+  const clock = deps.now ?? (() => Date.now())
+  return function noteMcpDependencyFailure(ev: SessionEvent): void {
+    try {
+      if (ev.kind === 'tool_use') {
+        watcher.onToolUse(ev.toolUseId, ev.toolName)
+        return
+      }
+      if (ev.kind !== 'tool_result') return
+      const alert = watcher.onToolResult({
+        toolUseId: ev.toolUseId,
+        isError: ev.isError,
+        errorText: ev.errorText,
+        agent: deps.agent,
+        now: clock(),
+      })
+      if (alert == null) return
+      deps.emit({
+        kind: 'mcp-dependency-blocked',
+        agent: deps.agent,
+        detail: renderMcpFailureDetail(alert),
+        suggestedActions: [],
+        firstSeenAt: new Date(clock()),
+      })
+    } catch {
+      // Side channel — never let alerting break the live feed.
+    }
+  }
+}