switchroom 0.14.93 → 0.15.1

package/telegram-plugin/gateway/gateway.ts

@@ -157,7 +157,7 @@ import {
   formatModelUnavailableCard,
   resolveModelUnavailableFromOperatorEvent,
 } from '../model-unavailable.js'
-import { runFleetAutoFallback } from '../auto-fallback-fleet.js'
+import { runFleetAutoFallback, renderFallbackFailureNotice, evaluateFallbackFailureNotice, type FallbackFailureNoticeState } from '../auto-fallback-fleet.js'
 import { startRestartWatchdog } from './restart-watchdog.js'
 import { validateStringArray } from './access-validator.js'
 
@@ -422,6 +422,9 @@ import {
   saveQuotaWatchState,
   patchQuotaWatchState,
   emptyAccountState,
+  resolveQuotaWatchTuning,
+  buildQuotaClaimKey,
+  QUOTA_WATCH_CLAIM_WINDOW_MS,
 } from '../quota-watch.js'
 import { buildSnapshotsFromState, buildSnapshotsFromCachedState } from '../auth-snapshot-format.js'
 import {
@@ -14804,6 +14807,51 @@ async function fireFleetAutoFallback(triggerAgent: string, untilMs?: number): Pr
   )
 }
 
+/**
+ * Broadcast a fleet-fallback FAILURE notice to every authorized chat.
+ *
+ * Why this exists: the model-unavailable card renders "Auto-failover in
+ * progress — see the announcement below" BEFORE the dispatcher's outcome
+ * is known. When the dispatcher errors (broker down, listState throw,
+ * markExhausted failure), the success announcement never lands and the
+ * card's promise is broken — the 2026-06-06→07 incident sent 12 such
+ * broken-promise cards while every fallback errored "set-active requires
+ * admin". The admin-gating root cause is fixed (#2206), but ANY future
+ * dispatcher error reproduces the broken promise. This notice closes the
+ * loop deterministically: card promised an announcement → an
+ * announcement ALWAYS arrives, success or failure.
+ *
+ * Disable with SWITCHROOM_FLEET_FALLBACK_FAILURE_NOTICE=0 (log-only,
+ * pre-fix behaviour).
+ */
+let fallbackFailureNoticeState: FallbackFailureNoticeState = { lastSentAtMs: 0 }
+
+function broadcastFleetFallbackFailure(triggerAgent: string, reason: string): void {
+  if (process.env.SWITCHROOM_FLEET_FALLBACK_FAILURE_NOTICE === '0') return
+  // Notice-level cooldown (30 min, per gateway). The fleetFallbackGate's
+  // dedup window only arms on SUCCESSFUL swaps, so it bounds nothing
+  // here — and the card-less quota_wall_detected trigger re-fires every
+  // ~60s during a wall. Without this, a persistent broker outage would
+  // stream failure notices for days. See evaluateFallbackFailureNotice.
+  const verdict = evaluateFallbackFailureNotice(fallbackFailureNoticeState, Date.now())
+  if (!verdict.send) {
+    process.stderr.write(
+      `telegram gateway: [fleet-fallback] failure notice suppressed (cooldown) agent=${triggerAgent}: ${reason}\n`,
+    )
+    return
+  }
+  fallbackFailureNoticeState = verdict.next
+  const access = loadAccess()
+  if (access.allowFrom.length === 0) return
+  const html = renderFallbackFailureNotice(triggerAgent, reason)
+  for (const chat_id of access.allowFrom) {
+    void swallowingApiCall(
+      () => bot.api.sendMessage(chat_id, html, { parse_mode: 'HTML' as const }),
+      { chat_id, verb: 'fleet-fallback:failure-notify' },
+    )
+  }
+}
+
 /** Returns true iff the dispatcher actually performed a swap (and the
  *  user-visible announcement was broadcast). False on no-op /
  *  error / idempotent-skip — caller uses this to decide whether to
@@ -14815,6 +14863,9 @@ async function doFireFleetAutoFallback(triggerAgent: string, untilMs?: number):
       process.stderr.write(
         `telegram gateway: [fleet-fallback] skipped agent=${triggerAgent} reason=no-broker-client\n`,
       )
+      // The model-unavailable card may have promised an announcement —
+      // keep the promise even though nothing could run.
+      broadcastFleetFallbackFailure(triggerAgent, 'auth-broker unreachable (no client).')
       return false
     }
     const state = await client.listState()
@@ -14878,6 +14929,10 @@ async function doFireFleetAutoFallback(triggerAgent: string, untilMs?: number):
     process.stderr.write(
       `telegram gateway: [fleet-fallback] error agent=${triggerAgent}: ${(err as Error)?.message ?? err}\n`,
     )
+    // Keep the card's "see the announcement below" promise on the error
+    // path — the 06-06→07 incident sent 12 cards whose promised
+    // announcement never arrived because this catch was log-only.
+    broadcastFleetFallbackFailure(triggerAgent, (err as Error)?.message ?? String(err))
     return false
   }
 }
@@ -14969,9 +15024,34 @@ async function runCreditWatch(): Promise<void> {
  * State persists across restarts via `<stateDir>/quota-watch.json`.
  * Mirrors runCreditWatch's structure and notification routing.
  */
-async function runQuotaWatch(): Promise<void> {
+
+/**
+ * Ask the broker for the fleet-wide dedup claim on one notification key.
+ * FAIL-OPEN on any error: a broker that predates the `claim-notification`
+ * op (skewed rollout) rejects at the protocol layer, and a transient IPC
+ * failure must degrade to duplicated notifications, never dropped ones.
+ */
+async function claimQuotaNotification(
+  brokerClient: NonNullable<Awaited<ReturnType<typeof getAuthBrokerClient>>>,
+  key: string,
+): Promise<boolean> {
+  try {
+    const res = await brokerClient.claimNotification(key, QUOTA_WATCH_CLAIM_WINDOW_MS)
+    return res.granted
+  } catch (err) {
+    process.stderr.write(`telegram gateway: quota-watch: claim failed (fail-open): ${err}\n`)
+    return true
+  }
+}
+
+async function runQuotaWatch(opts: { bootTick?: boolean } = {}): Promise<void> {
   const agentName = getMyAgentName()
   const stateDir = STATE_DIR
+  const bootTick = opts.bootTick ?? false
+  // Hardening knobs (2026-06-09 incident: fleet bounce released stale
+  // recovery latches on all 11 agents at once → 26 duplicate sends).
+  // See QuotaWatchTuning in quota-watch.ts for the env contract.
+  const tuning = resolveQuotaWatchTuning(process.env)
 
   // Read broker state. The listState response now includes last_quota
   // per account — the broker's in-memory cache from previous probeQuota
@@ -15019,6 +15099,20 @@ async function runQuotaWatch(): Promise<void> {
     })
     if (fleetDecision.kind === 'notify') {
       for (const chat_id of access.allowFrom) {
+        // Fleet-level dedup: all 11 gateways detect this same edge within
+        // one poll cycle — only the broker-claim winner sends per chat.
+        if (tuning.fleetDedup) {
+          const granted = await claimQuotaNotification(
+            brokerClient,
+            buildQuotaClaimKey(FLEET_ALL_EXHAUSTED_KEY, fleetDecision.transition, chat_id),
+          )
+          if (!granted) {
+            process.stderr.write(
+              `telegram gateway: quota-watch: fleet-all-exhausted claim denied chat=${chat_id} — another agent notified\n`,
+            )
+            continue
+          }
+        }
         await swallowingApiCall(
           () =>
             bot.api.sendMessage(chat_id, fleetDecision.message, {
@@ -15056,10 +15150,21 @@ async function runQuotaWatch(): Promise<void> {
     snapshots.map((s, i) => [s.label, i]),
   )
 
+  // Reconciled transitions: state advances (latch clears) but nothing is
+  // sent — boot-tick and late recoveries (see QuotaWatchDecision docs).
+  let reconciledCount = 0
+  let mutatedState = watchState
+
   for (const snap of snapshots) {
     const prev = watchState[snap.label] ?? emptyAccountState()
-    const decision = evaluateQuotaWatchAccount({ agentName, snap, prev, now })
-    if (decision.kind !== 'skip') {
+    const decision = evaluateQuotaWatchAccount({ agentName, snap, prev, now, bootTick, tuning })
+    if (decision.kind === 'reconcile') {
+      mutatedState = patchQuotaWatchState(mutatedState, decision.accountLabel, decision.newAccountState)
+      reconciledCount++
+      process.stderr.write(
+        `telegram gateway: quota-watch: reconciled ${decision.transition} for account=${decision.accountLabel} (${decision.reason}) — no notification\n`,
+      )
+    } else if (decision.kind !== 'skip') {
       pendingTransitions.push({
         accountLabel: snap.label,
         snapIndex: labelToSnapIndex.get(snap.label) ?? -1,
@@ -15069,7 +15174,16 @@ async function runQuotaWatch(): Promise<void> {
   }
 
   if (pendingTransitions.length === 0) {
-    return // Steady-state: no notifications, no probes, no state write.
+    // Steady-state: no notifications, no probes. Persist only if a
+    // reconcile advanced the latch (otherwise no state write at all).
+    if (reconciledCount > 0) {
+      try {
+        saveQuotaWatchState(stateDir, mutatedState)
+      } catch (err) {
+        process.stderr.write(`telegram gateway: quota-watch state persist failed: ${err}\n`)
+      }
+    }
+    return
   }
 
   // Transition detected: probe ONLY the crossing accounts to get fresh
@@ -15083,16 +15197,31 @@ async function runQuotaWatch(): Promise<void> {
       freshProbeMap.set(entry.label, entry.result)
     }
   } catch (err) {
-    // Probe failed — still send notifications using cached data.
-    // Don't abort: the user should know about the threshold crossing
-    // even if the message body shows slightly stale numbers.
     process.stderr.write(`telegram gateway: quota-watch: probe for crossing accounts failed: ${err}\n`)
+    if (!tuning.sendOnProbeFail) {
+      // A quota notification must never carry numbers we could not verify
+      // live. Leave the crossing accounts' state untouched — the
+      // transition re-evaluates (and re-probes) on the next 15-min tick.
+      // Persist any reconciles already applied, then bail.
+      if (reconciledCount > 0) {
+        try {
+          saveQuotaWatchState(stateDir, mutatedState)
+        } catch (saveErr) {
+          process.stderr.write(`telegram gateway: quota-watch state persist failed: ${saveErr}\n`)
+        }
+      }
+      process.stderr.write(
+        `telegram gateway: quota-watch: deferring ${pendingTransitions.length} notification(s) until probe succeeds\n`,
+      )
+      return
+    }
+    // Legacy (SWITCHROOM_QUOTA_WATCH_SEND_ON_PROBE_FAIL=1): fall through
+    // and send from cached data.
   }
 
   // Build final notifications, enriching the snapshot with fresh probe
   // data where available.
-  let mutatedState = watchState
-  const notifications: Array<{ message: string; accountLabel: string }> = []
+  const notifications: Array<{ message: string; accountLabel: string; transition: string }> = []
 
   for (const { accountLabel, snapIndex, decision } of pendingTransitions) {
     // Re-evaluate with fresh probe data to get an accurate message body.
@@ -15100,37 +15229,88 @@ async function runQuotaWatch(): Promise<void> {
     const freshResult = freshProbeMap.get(accountLabel)
     let enrichedDecision = decision
     // pendingTransitions only ever holds notify decisions (pushed under
-    // `decision.kind !== 'skip'`). Narrow explicitly so `decision.transition`
-    // type-checks below; this continue never fires at runtime.
+    // `decision.kind !== 'skip'` / `!== 'reconcile'`). Narrow explicitly so
+    // `decision.transition` type-checks below; this continue never fires
+    // at runtime.
     if (decision.kind !== 'notify') continue
     if (freshResult && freshResult.ok && snapIndex >= 0) {
-      const enrichedSnap = { ...snapshots[snapIndex]!, quota: freshResult.data }
+      // Live numbers replace the cache — and capturedAtMs is cleared so the
+      // staleness gate never misfires on data we JUST probed.
+      const enrichedSnap = { ...snapshots[snapIndex]!, quota: freshResult.data, capturedAtMs: undefined }
       const prev = watchState[accountLabel] ?? emptyAccountState()
-      const re = evaluateQuotaWatchAccount({ agentName, snap: enrichedSnap, prev, now })
+      const re = evaluateQuotaWatchAccount({ agentName, snap: enrichedSnap, prev, now, bootTick, tuning })
       // If the fresh probe still shows the same transition, use the
       // enriched message. If it no longer shows a transition (e.g. the
       // account recovered in the 100ms between listState and probe),
       // fall through to skip this notification.
       if (re.kind === 'notify' && re.transition === decision.transition) {
         enrichedDecision = re
+      } else if (re.kind === 'reconcile') {
+        // Fresh data confirms the transition but it isn't news (boot-tick /
+        // late recovery) — advance the latch silently.
+        mutatedState = patchQuotaWatchState(mutatedState, accountLabel, re.newAccountState)
+        reconciledCount++
+        process.stderr.write(
+          `telegram gateway: quota-watch: reconciled ${re.transition} for account=${accountLabel} (${re.reason}) — no notification\n`,
+        )
+        continue
       } else if (re.kind === 'skip') {
         // State normalised by the time of the probe — don't notify.
         continue
       }
+    } else if (!tuning.sendOnProbeFail) {
+      // No verified fresh data for this account (per-account probe failure
+      // or label missing from the batch result). Same rule as the batch
+      // throw above: never send unverified numbers. State untouched —
+      // re-evaluated (and re-probed) next tick.
+      process.stderr.write(
+        `telegram gateway: quota-watch: probe unavailable for account=${accountLabel} — deferring notification\n`,
+      )
+      continue
     }
 
     if (enrichedDecision.kind !== 'notify') continue
-    notifications.push({ message: enrichedDecision.message, accountLabel })
+    notifications.push({
+      message: enrichedDecision.message,
+      accountLabel,
+      transition: enrichedDecision.transition,
+    })
     mutatedState = patchQuotaWatchState(mutatedState, accountLabel, enrichedDecision.newAccountState)
   }
 
   if (notifications.length === 0) {
-    return // All transitions resolved by the time of the live probe.
+    // All transitions resolved/deferred by the time of the live probe.
+    // Reconciles may still have advanced the latch — persist those.
+    if (reconciledCount > 0) {
+      try {
+        saveQuotaWatchState(stateDir, mutatedState)
+      } catch (err) {
+        process.stderr.write(`telegram gateway: quota-watch state persist failed: ${err}\n`)
+      }
+    }
+    return
   }
 
   // Send all notifications (one message per crossing account).
-  for (const { message, accountLabel } of notifications) {
+  for (const { message, accountLabel, transition } of notifications) {
     for (const chat_id of access.allowFrom) {
+      // Fleet-level dedup: every agent gateway independently detects the
+      // same account transition within one poll cycle. The broker claim
+      // grants exactly one sender per (account, transition, chat) per
+      // window — the other ten agents advance their local state silently.
+      // Fail-open on claim error (see claimQuotaNotification).
+      if (tuning.fleetDedup) {
+        const granted = await claimQuotaNotification(
+          brokerClient,
+          buildQuotaClaimKey(accountLabel, transition, chat_id),
+        )
+        if (!granted) {
+          process.stderr.write(
+            `telegram gateway: quota-watch: claim denied account=${accountLabel} chat=${chat_id} — another agent notified\n`,
+          )
+          continue
+        }
+      }
       // Quota-watch notify — best-effort. Wrap via swallowingApiCall so
       // flood-wait / deleted-chat / not-found surface as a stderr log
       // rather than a thrown exception that aborts the loop and leaves
@@ -20453,7 +20633,12 @@ void (async () => {
           // settle after boot (avoids a probe race with the boot-card
           // quota probe that fires in the first few seconds).
           setTimeout(() => {
-            void runQuotaWatch().catch((err) => {
+            // bootTick: recovery edges observed on the FIRST post-boot tick
+            // reconcile silently — a fleet bounce synchronizes all agents'
+            // first ticks, and a just-booted gateway can't tell "just
+            // recovered" from "recovered while we were down" (the
+            // 2026-06-09 26-message flood). Warnings still notify.
+            void runQuotaWatch({ bootTick: true }).catch((err) => {
               process.stderr.write(`telegram gateway: quota-watch initial run failed: ${err}\n`)
             })
           }, 30_000)

package/telegram-plugin/quota-watch.ts

@@ -73,6 +73,76 @@ export function emptyAccountState(): QuotaWatchAccountState {
   return { lastNotifiedHealth: null, lastNotifiedAt: 0 };
 }
 
+// ─── Tuning (env knobs) ───────────────────────────────────────────────────────
+
+/**
+ * Operational tuning for the watch loop, resolved once from env by the
+ * gateway. All three hardening behaviours are individually
+ * kill-switchable (incident 2026-06-09: a fleet bounce released
+ * days-stale recovery latches on all 11 agents at once → 26 duplicate
+ * 🟢 messages in 16 minutes):
+ *
+ *   SWITCHROOM_QUOTA_WATCH_MAX_STALE_MS      0 disables the staleness gate
+ *                                            (default 60 min)
+ *   SWITCHROOM_QUOTA_WATCH_LATE_RECOVERY_MS  0 disables silent late-recovery
+ *                                            reconciliation (default 6 h)
+ *   SWITCHROOM_QUOTA_WATCH_FLEET_DEDUP       "0" disables the broker claim
+ *                                            (every agent sends, pre-incident
+ *                                            behaviour)
+ *   SWITCHROOM_QUOTA_WATCH_SEND_ON_PROBE_FAIL "1" restores sending from
+ *                                            cached data when the pre-send
+ *                                            validation probe fails
+ */
+export interface QuotaWatchTuning {
+  /** Cached snapshots older than this are treated as unknown (no opinion). 0 = off. */
+  maxStaleMs: number;
+  /** Recovery edges whose 🟡 warning is older than this reconcile silently. 0 = off. */
+  lateRecoveryMs: number;
+  /** Route sends through the broker's claim-notification dedup. */
+  fleetDedup: boolean;
+  /** Legacy: send from cached data when the validation probe fails. */
+  sendOnProbeFail: boolean;
+}
+
+export const DEFAULT_QUOTA_WATCH_MAX_STALE_MS = 60 * 60_000;
+export const DEFAULT_QUOTA_WATCH_LATE_RECOVERY_MS = 6 * 60 * 60_000;
+
+/** Broker claim window. Must exceed one full poll cycle (15 min) plus the
+ *  boot-stagger spread so every agent's observation of the SAME edge lands
+ *  inside one window; an account genuinely re-crossing the same edge later
+ *  than this re-notifies. */
+export const QUOTA_WATCH_CLAIM_WINDOW_MS = 30 * 60_000;
+
+export function resolveQuotaWatchTuning(
+  env: Record<string, string | undefined>,
+): QuotaWatchTuning {
+  const num = (raw: string | undefined, fallback: number): number => {
+    if (raw === undefined || raw === "") return fallback;
+    const n = Number(raw);
+    return Number.isFinite(n) && n >= 0 ? n : fallback;
+  };
+  return {
+    maxStaleMs: num(env.SWITCHROOM_QUOTA_WATCH_MAX_STALE_MS, DEFAULT_QUOTA_WATCH_MAX_STALE_MS),
+    lateRecoveryMs: num(env.SWITCHROOM_QUOTA_WATCH_LATE_RECOVERY_MS, DEFAULT_QUOTA_WATCH_LATE_RECOVERY_MS),
+    fleetDedup: env.SWITCHROOM_QUOTA_WATCH_FLEET_DEDUP !== "0",
+    sendOnProbeFail: env.SWITCHROOM_QUOTA_WATCH_SEND_ON_PROBE_FAIL === "1",
+  };
+}
+
+/**
+ * Broker dedup-claim key for one (account, transition, chat) cell.
+ * Per-CHAT keys keep the audience identical to pre-dedup behaviour:
+ * every chat that any agent would have notified still receives exactly
+ * one copy — from whichever agent claims it first.
+ */
+export function buildQuotaClaimKey(
+  accountLabel: string,
+  transition: string,
+  chatId: string | number,
+): string {
+  return `quota-watch:${accountLabel}:${transition}:${chatId}`;
+}
+
 // ─── Decision logic ───────────────────────────────────────────────────────────
 
 export type QuotaWatchTransition =
@@ -87,30 +157,73 @@ export type QuotaWatchDecision =
       newAccountState: QuotaWatchAccountState;
       transition: QuotaWatchTransition;
     }
+  | {
+      /**
+       * A real transition was observed, but it is no longer NEWS — persist
+       * the new state so the edge-trigger latch clears, send nothing.
+       * Two producers: boot-tick recoveries (a just-booted gateway cannot
+       * distinguish "just recovered" from "recovered while we were down",
+       * and fleet bounces synchronize all agents' first ticks → flood) and
+       * late recoveries (the matching 🟡 is hours old; an "all clear" now
+       * is state reconciliation, not information).
+       */
+      kind: "reconcile";
+      accountLabel: string;
+      newAccountState: QuotaWatchAccountState;
+      transition: QuotaWatchTransition;
+      reason: "boot-tick-recovery" | "late-recovery";
+    }
   | { kind: "skip"; accountLabel: string; reason: string };
 
 /**
  * Evaluate one account's quota state against its last-notified health.
  *
- * Transition table:
+ * Transition table (after the staleness gate — a cached snapshot older
+ * than `maxStaleMs` is no opinion at all → skip "stale-snapshot"):
  *   healthy → healthy        skip (steady-state)
- *   healthy → throttling     notify (entered-throttling)
+ *   healthy → throttling     notify (entered-throttling) — warnings are
+ *                            level-state news, valid on any tick incl. boot
  *   healthy → blocked        skip (credits-watch covers this)
- *   throttling → healthy     notify (recovered-to-healthy)
+ *   throttling → healthy     notify (recovered-to-healthy), EXCEPT:
+ *                              boot tick           → reconcile silently
+ *                              warning > lateRecoveryMs old → reconcile silently
  *   throttling → throttling  skip (already notified)
  *   throttling → blocked     skip (credits-watch covers blocked)
  *   blocked → *              skip (credits-watch domain)
  *   unknown → *              skip (no quota data — don't spam)
  *   * → unknown              skip (probe failed — transient, don't alarm)
+ *
+ * `bootTick` / `tuning` are optional: omitted (legacy callers/tests) the
+ * behaviour is exactly the pre-hardening table (no stale gate, no
+ * reconciliation).
  */
 export function evaluateQuotaWatchAccount(args: {
   agentName: string;
   snap: AccountSnapshot;
   prev: QuotaWatchAccountState;
   now: number;
+  /** True on the gateway's first watch tick after boot. */
+  bootTick?: boolean;
+  /** Staleness / late-recovery thresholds; 0 disables each. */
+  tuning?: Pick<QuotaWatchTuning, "maxStaleMs" | "lateRecoveryMs">;
 }): QuotaWatchDecision {
   const { agentName, snap, prev, now } = args;
+  const bootTick = args.bootTick ?? false;
+  const maxStaleMs = args.tuning?.maxStaleMs ?? 0;
+  const lateRecoveryMs = args.tuning?.lateRecoveryMs ?? 0;
   const label = snap.label;
+
+  // Staleness gate: a CACHED snapshot (capturedAtMs set) past its shelf
+  // life carries no opinion about the present — neither latch nor release.
+  // Live-probe snapshots (capturedAtMs undefined) are fresh by construction.
+  if (
+    maxStaleMs > 0 &&
+    snap.capturedAtMs !== undefined &&
+    now - snap.capturedAtMs > maxStaleMs
+  ) {
+    return { kind: "skip", accountLabel: label, reason: "stale-snapshot" };
+  }
+
   const currentHealth = classifyHealth(snap);
 
   // Unknown (probe failed) or blocked — skip entirely.
@@ -147,6 +260,31 @@ export function evaluateQuotaWatchAccount(args: {
       lastNotifiedHealth: "healthy",
       lastNotifiedAt: now,
     };
+    // A recovery observed on the first post-boot tick is not attributable
+    // to "just now" — the account may have recovered any time while this
+    // gateway was down, and a fleet bounce synchronizes every agent's
+    // first tick (the 2026-06-09 26-message flood). Reconcile silently.
+    if (bootTick) {
+      return {
+        kind: "reconcile",
+        accountLabel: label,
+        newAccountState: newState,
+        transition: "recovered-to-healthy",
+        reason: "boot-tick-recovery",
+      };
+    }
+    // Recovery whose matching 🟡 warning is hours old: the "all clear" is
+    // no longer actionable news (the user has long moved on; /auth shows
+    // live state on demand). Clear the latch without a message.
+    if (lateRecoveryMs > 0 && now - prev.lastNotifiedAt > lateRecoveryMs) {
+      return {
+        kind: "reconcile",
+        accountLabel: label,
+        newAccountState: newState,
+        transition: "recovered-to-healthy",
+        reason: "late-recovery",
+      };
+    }
     return {
       kind: "notify",
       accountLabel: label,

package/telegram-plugin/tests/auth-quota-util-cell.test.ts

@@ -0,0 +1,23 @@
+/**
+ * Telegram legacy accounts-table twin of the CLI honesty fix — the
+ * legacy table renders exactly when the live probe FAILED, i.e. when
+ * cached-data disclosure matters most.
+ */
+import { describe, it, expect } from "vitest";
+import { formatQuotaUtilCell } from "../gateway/auth-command.js";
+
+const NOW = 1_780_000_000_000;
+
+describe("formatQuotaUtilCell (Telegram legacy table)", () => {
+  it("no cached snapshot → 'no data'", () => {
+    expect(formatQuotaUtilCell({ last_quota: null }, NOW)).toBe("no data");
+  });
+
+  it("renders both windows with the snapshot age", () => {
+    const cell = formatQuotaUtilCell(
+      { last_quota: { fiveHourUtilizationPct: 84.6, sevenDayUtilizationPct: 12.1, capturedAt: NOW - 90_000 } },
+      NOW,
+    );
+    expect(cell).toBe("85%·12% (1m 30s ago)");
+  });
+});

package/telegram-plugin/tests/auto-fallback-fleet.test.ts

@@ -173,3 +173,74 @@ describe('runFleetAutoFallback', () => {
     }
   });
 });
+
+// ── failure notice (broken-promise fix, 2026-06-09 incident follow-up) ──────
+
+import { renderFallbackFailureNotice } from "../auto-fallback-fleet.js";
+
+describe("renderFallbackFailureNotice", () => {
+  it("names the trigger agent, the reason, and the manual recovery verbs", () => {
+    const html = renderFallbackFailureNotice("marko", "auth-broker unreachable (no client).");
+    expect(html).toContain("Auto-failover could not run");
+    expect(html).toContain("<b>marko</b>");
+    expect(html).toContain("auth-broker unreachable");
+    expect(html).toContain("/auth use");
+    expect(html).toContain("/auth</code>");
+  });
+
+  it("escapes HTML in the error reason (broker errors can contain angle brackets)", () => {
+    const html = renderFallbackFailureNotice("a<b", 'request <probe-quota> failed & "timed out"');
+    expect(html).toContain("a&lt;b");
+    expect(html).toContain("&lt;probe-quota&gt;");
+    expect(html).toContain("&amp;");
+    expect(html).not.toMatch(/<probe-quota>/);
+  });
+});
+
+// ── failure-notice cooldown (reviewer blocker: gate window never arms on
+//    failure; quota_wall_detected re-fires ~60s → unbounded notice spam) ─────
+
+import {
+  evaluateFallbackFailureNotice,
+  FALLBACK_FAILURE_NOTICE_COOLDOWN_MS,
+} from "../auto-fallback-fleet.js";
+
+describe("evaluateFallbackFailureNotice", () => {
+  const T0 = 1_780_000_000_000;
+
+  it("first failure always sends and arms the cooldown", () => {
+    const r = evaluateFallbackFailureNotice({ lastSentAtMs: 0 }, T0);
+    expect(r.send).toBe(true);
+    expect(r.next.lastSentAtMs).toBe(T0);
+  });
+
+  it("a repeat failure inside the cooldown is suppressed and does NOT extend the window", () => {
+    const armed = { lastSentAtMs: T0 };
+    const r = evaluateFallbackFailureNotice(armed, T0 + 60_000);
+    expect(r.send).toBe(false);
+    expect(r.next).toBe(armed); // unchanged — window not extended by suppressed attempts
+  });
+
+  it("sends again once the cooldown elapses", () => {
+    const r = evaluateFallbackFailureNotice(
+      { lastSentAtMs: T0 },
+      T0 + FALLBACK_FAILURE_NOTICE_COOLDOWN_MS,
+    );
+    expect(r.send).toBe(true);
+    expect(r.next.lastSentAtMs).toBe(T0 + FALLBACK_FAILURE_NOTICE_COOLDOWN_MS);
+  });
+
+  it("bounds the 60s quota_wall_detected re-fire storm to ≤2 notices/hour", () => {
+    // Simulate a wedged agent re-signalling every 60s for one hour with a
+    // dead broker — the incident shape the reviewer flagged.
+    let state = { lastSentAtMs: 0 };
+    let sent = 0;
+    for (let t = T0; t < T0 + 3_600_000; t += 60_000) {
+      const r = evaluateFallbackFailureNotice(state, t);
+      if (r.send) sent++;
+      state = r.next;
+    }
+    expect(sent).toBeLessThanOrEqual(2);
+    expect(sent).toBeGreaterThanOrEqual(1);
+  });
+});