switchroom 0.14.0 → 0.14.1

package/telegram-plugin/gateway/gateway.ts

@@ -373,6 +373,14 @@ import {
   loadCreditState,
   saveCreditState,
 } from '../credits-watch.js'
+import {
+  evaluateQuotaWatchAccount,
+  loadQuotaWatchState,
+  saveQuotaWatchState,
+  patchQuotaWatchState,
+  emptyAccountState,
+} from '../quota-watch.js'
+import { buildSnapshotsFromState, buildSnapshotsFromCachedState } from '../auth-snapshot-format.js'
 import {
   writeTurnActiveMarker,
   touchTurnActiveMarker,
@@ -3898,11 +3906,12 @@ const ipcServer: IpcServer = createIpcServer({
           const updateOutcomeLine = (() => {
             try { return maybeRenderUpdateAnnouncement() ?? undefined } catch { return undefined }
           })()
+          const resolvedAgentDirForCard = agentDir ?? (process.env.TELEGRAM_STATE_DIR ? require('path').dirname(process.env.TELEGRAM_STATE_DIR) : '/tmp')
           startBootCard(chatId, threadId, botApiForCard, {
             agentName: agentDisplayName,
             agentSlug,
             version: formatBootVersion(),
-            agentDir: agentDir ?? (process.env.TELEGRAM_STATE_DIR ? require('path').dirname(process.env.TELEGRAM_STATE_DIR) : '/tmp'),
+            agentDir: resolvedAgentDirForCard,
             gatewayInfo: { pid: process.pid, startedAtMs: GATEWAY_STARTED_AT_MS },
             restartReason: reason,
             restartAgeMs: markerAgeMs,
@@ -3911,6 +3920,7 @@ const ipcServer: IpcServer = createIpcServer({
             probeQuotaViaBroker: (t) => probeQuotaForBootCard(agentSlug, t),
             tmuxSupervisor: process.env.SWITCHROOM_TMUX_SUPERVISOR === '1',
             dockerMode: process.env.SWITCHROOM_RUNTIME === 'docker',
+            configSnapshotPath: join(resolvedAgentDirForCard, '.config-snapshot.json'),
             ...(updateOutcomeLine ? { updateOutcomeLine } : {}),
           }, ackMsgId).then(handle => {
             activeBootCard = handle
@@ -11862,6 +11872,183 @@ async function runCreditWatch(): Promise<void> {
   }
 }
 
+/**
+ * Quota threshold-tier push loop body (#E4). Reads the broker's in-memory
+ * cached utilization (populated by previous probeQuota calls from /auth,
+ * auto-fallback, and boot cards) via `listState.accounts[].last_quota`.
+ * Classifies each account via classifyHealth, and fires one Telegram message
+ * per healthy ↔ throttling transition (edge-triggered, not level-triggered).
+ * Does NOT fire on healthy → blocked or blocked → healthy — credits-watch.ts
+ * owns those.
+ *
+ * Probe discipline:
+ *   - Steady-state polls: ONE broker `listState` IPC call only (no network).
+ *   - Accounts with no cached snapshot (null last_quota): skipped silently
+ *     (classifyHealth returns 'unknown'). Cache populates from /auth, boot
+ *     card, and auto-fallback probes in normal use.
+ *   - State transition detected: ONE targeted `probeQuota` call for ONLY the
+ *     crossing account, immediately before sending the notification, to get
+ *     fresh numbers for the message body. All other steady-state accounts
+ *     are not probed.
+ *
+ * This replaces the previous implementation that called probeQuota for ALL
+ * accounts unconditionally on every 15-minute poll (~768 live Anthropic
+ * network calls/day for an 8-account fleet). The corrected version makes
+ * 0 network calls on steady-state polls and at most 1 call per crossing
+ * event (which is also when we need to notify the user anyway).
+ *
+ * State persists across restarts via `<stateDir>/quota-watch.json`.
+ * Mirrors runCreditWatch's structure and notification routing.
+ */
+async function runQuotaWatch(): Promise<void> {
+  const agentName = getMyAgentName()
+  const stateDir = STATE_DIR
+
+  // Read broker state. The listState response now includes last_quota
+  // per account — the broker's in-memory cache from previous probeQuota
+  // calls. This is a local IPC call: no network, no Anthropic contact.
+  const brokerClient = await getAuthBrokerClient(agentName)
+  if (!brokerClient) {
+    process.stderr.write('telegram gateway: quota-watch: broker client unavailable — skipping\n')
+    return
+  }
+
+  let listStateData: Awaited<ReturnType<typeof brokerClient.listState>>
+  try {
+    listStateData = await brokerClient.listState()
+  } catch (err) {
+    process.stderr.write(`telegram gateway: quota-watch: listState failed: ${err}\n`)
+    return
+  }
+
+  if (!listStateData.accounts || listStateData.accounts.length === 0) {
+    return // No accounts — nothing to watch.
+  }
+
+  // Build AccountSnapshot[] from cached broker state only — no live probe.
+  // Accounts with null last_quota produce quota=null snapshots; classifyHealth
+  // returns 'unknown'; evaluateQuotaWatchAccount skips — no false alarms.
+  const snapshots = buildSnapshotsFromCachedState(listStateData)
+
+  // Load persisted per-account state.
+  let watchState = loadQuotaWatchState(stateDir)
+  const now = Date.now()
+  const access = loadAccess()
+
+  // First pass: evaluate all accounts against cached state. Collect
+  // labels that need a live probe (i.e. accounts with a detected transition
+  // that we're about to notify about). We probe those to get fresh
+  // utilization numbers for the notification body — not for classification.
+  const pendingTransitions: Array<{
+    accountLabel: string
+    snapIndex: number
+    decision: ReturnType<typeof evaluateQuotaWatchAccount>
+  }> = []
+
+  const labelToSnapIndex = new Map<string, number>(
+    snapshots.map((s, i) => [s.label, i]),
+  )
+
+  for (const snap of snapshots) {
+    const prev = watchState[snap.label] ?? emptyAccountState()
+    const decision = evaluateQuotaWatchAccount({ agentName, snap, prev, now })
+    if (decision.kind !== 'skip') {
+      pendingTransitions.push({
+        accountLabel: snap.label,
+        snapIndex: labelToSnapIndex.get(snap.label) ?? -1,
+        decision,
+      })
+    }
+  }
+
+  if (pendingTransitions.length === 0) {
+    return // Steady-state: no notifications, no probes, no state write.
+  }
+
+  // Transition detected: probe ONLY the crossing accounts to get fresh
+  // numbers for the notification message bodies. One batched RPC for all
+  // crossing accounts (typically 1, rarely 2+).
+  const crossingLabels = pendingTransitions.map(t => t.accountLabel)
+  let freshProbeMap = new Map<string, Awaited<ReturnType<typeof brokerClient.probeQuota>>['results'][number]['result']>()
+  try {
+    const probeData = await brokerClient.probeQuota(crossingLabels, 8000)
+    for (const entry of probeData.results) {
+      freshProbeMap.set(entry.label, entry.result)
+    }
+  } catch (err) {
+    // Probe failed — still send notifications using cached data.
+    // Don't abort: the user should know about the threshold crossing
+    // even if the message body shows slightly stale numbers.
+    process.stderr.write(`telegram gateway: quota-watch: probe for crossing accounts failed: ${err}\n`)
+  }
+
+  // Build final notifications, enriching the snapshot with fresh probe
+  // data where available.
+  let mutatedState = watchState
+  const notifications: Array<{ message: string; accountLabel: string }> = []
+
+  for (const { accountLabel, snapIndex, decision } of pendingTransitions) {
+    // Re-evaluate with fresh probe data to get an accurate message body.
+    // If the fresh probe succeeded, replace the snap's quota with live data.
+    const freshResult = freshProbeMap.get(accountLabel)
+    let enrichedDecision = decision
+    // pendingTransitions only ever holds notify decisions (pushed under
+    // `decision.kind !== 'skip'`). Narrow explicitly so `decision.transition`
+    // type-checks below; this continue never fires at runtime.
+    if (decision.kind !== 'notify') continue
+    if (freshResult && freshResult.ok && snapIndex >= 0) {
+      const enrichedSnap = { ...snapshots[snapIndex]!, quota: freshResult.data }
+      const prev = watchState[accountLabel] ?? emptyAccountState()
+      const re = evaluateQuotaWatchAccount({ agentName, snap: enrichedSnap, prev, now })
+      // If the fresh probe still shows the same transition, use the
+      // enriched message. If it no longer shows a transition (e.g. the
+      // account recovered in the 100ms between listState and probe),
+      // fall through to skip this notification.
+      if (re.kind === 'notify' && re.transition === decision.transition) {
+        enrichedDecision = re
+      } else if (re.kind === 'skip') {
+        // State normalised by the time of the probe — don't notify.
+        continue
+      }
+    }
+
+    if (enrichedDecision.kind !== 'notify') continue
+    notifications.push({ message: enrichedDecision.message, accountLabel })
+    mutatedState = patchQuotaWatchState(mutatedState, accountLabel, enrichedDecision.newAccountState)
+  }
+
+  if (notifications.length === 0) {
+    return // All transitions resolved by the time of the live probe.
+  }
+
+  // Send all notifications (one message per crossing account).
+  for (const { message, accountLabel } of notifications) {
+    for (const chat_id of access.allowFrom) {
+      // Quota-watch notify — best-effort. Wrap via swallowingApiCall so
+      // flood-wait / deleted-chat / not-found surface as a stderr log
+      // rather than a thrown exception that aborts the loop and leaves
+      // half the allowFrom chats unnotified. Matches the wrapping
+      // contract enforced by scripts/check-bot-api-wrapping.sh (#1075).
+      await swallowingApiCall(
+        () =>
+          bot.api.sendMessage(chat_id, message, {
+            parse_mode: 'HTML',
+            link_preview_options: { is_disabled: true },
+          }),
+        { chat_id, verb: 'quota-watch.notify' },
+      )
+    }
+    process.stderr.write(`telegram gateway: quota-watch: notified transition for account=${accountLabel}\n`)
+  }
+
+  // Persist updated state regardless of whether sends succeeded.
+  try {
+    saveQuotaWatchState(stateDir, mutatedState)
+  } catch (err) {
+    process.stderr.write(`telegram gateway: quota-watch state persist failed: ${err}\n`)
+  }
+}
+
 bot.command("auth", async ctx => {
   // sec WS7-F2b (#1394): `/auth` drives the auth-broker credential
   // lifecycle (`/auth add` mints/attaches an Anthropic account token,
@@ -12146,8 +12333,7 @@ function resolveAgentDirForName(agent: string): string | null {
  *   restart   — systemctl --user restart switchroom-<agent>
  *   reauth    — delegate to runSwitchroomAuthCommand (same flow as /auth reauth)
  *   logs      — post last 30 lines of journalctl for the agent
- *   swap-slot, add-slot — Phase 4c will wire these; for now toast with the
- *                         equivalent CLI command for the user to run manually.
+ *   slot management buttons — removed (E5); use /auth use or /auth add instead.
  */
 /**
  * Issue #44: handle taps on the deferred-secret card's inline buttons.
@@ -13776,15 +13962,6 @@ async function handleOperatorEventCallback(ctx: Context, data: string): Promise<
       }
       return
     }
-    case 'swap-slot':
-    case 'add-slot': {
-      await ctx.answerCallbackQuery({ text: 'Phase 4c will wire this' }).catch(() => {})
-      const cmd = action === 'swap-slot' ? `auth use ${agent} <slot-name>` : `auth add ${agent}`
-      await ctx.reply(`Phase 4c will wire ${action} buttons. Until then, run in terminal: <code>switchroom ${cmd}</code>`, {
-        parse_mode: 'HTML',
-      })
-      return
-    }
     default: {
       await ctx.answerCallbackQuery({ text: `Unknown action: ${action}` }).catch(() => {})
       return
@@ -14679,7 +14856,7 @@ bot.on('callback_query:data', async ctx => {
 
   // op:<action>:<encoded-agent> callbacks from operator-events.ts
   // renderOperatorEvent(). Agent name is URL-encoded at emit (issue #24).
-  // Actions: dismiss, restart, reauth, swap-slot, add-slot, logs.
+  // Actions: dismiss, restart, reauth, logs.
   if (data.startsWith('op:')) {
     await handleOperatorEventCallback(ctx, data)
     return
@@ -16641,11 +16818,12 @@ void (async () => {
                     const updateOutcomeLine = (() => {
                       try { return maybeRenderUpdateAnnouncement() ?? undefined } catch { return undefined }
                     })()
+                    const resolvedAgentDirForBootCard = agentDir ?? join(homedir(), '.switchroom', 'agents', agentSlug)
                     const handle = await startBootCard(chatId, threadId, botApiForCard, {
                       agentName: agentDisplayName,
                       agentSlug,
                       version: formatBootVersion(),
-                      agentDir: agentDir ?? join(homedir(), '.switchroom', 'agents', agentSlug),
+                      agentDir: resolvedAgentDirForBootCard,
                       gatewayInfo: { pid: process.pid, startedAtMs: GATEWAY_STARTED_AT_MS },
                       restartReason: reason,
                       restartAgeMs: markerAgeMs,
@@ -16654,6 +16832,7 @@ void (async () => {
                       probeQuotaViaBroker: (t) => probeQuotaForBootCard(agentSlug, t),
                       tmuxSupervisor: process.env.SWITCHROOM_TMUX_SUPERVISOR === '1',
                       dockerMode: process.env.SWITCHROOM_RUNTIME === 'docker',
+                      configSnapshotPath: join(resolvedAgentDirForBootCard, '.config-snapshot.json'),
                       ...(updateOutcomeLine ? { updateOutcomeLine } : {}),
                     }, ackMsgId)
                     activeBootCard = handle
@@ -16705,6 +16884,34 @@ void (async () => {
           }, CREDIT_WATCH_POLL_MS).unref()
         }
 
+        // Proactive quota threshold-tier push (#E4). Reads broker cached
+        // quota for all accounts in the pool, classifies health via
+        // classifyHealth, and fires one Telegram message per
+        // healthy ↔ throttling transition (edge-triggered). Does NOT
+        // cover healthy → blocked or blocked → healthy — credits-watch
+        // handles those fatal-billing transitions above.
+        //
+        // Cadence: 15 min by default (same as credit-watch). Each poll
+        // calls broker listState (local IPC, cheap) + probeQuota only
+        // when a state-change is detected (to get fresh numbers for
+        // the notification body). SWITCHROOM_QUOTA_WATCH_POLL_MS=0 disables.
+        const QUOTA_WATCH_POLL_MS = Number(process.env.SWITCHROOM_QUOTA_WATCH_POLL_MS ?? 15 * 60_000)
+        if (QUOTA_WATCH_POLL_MS > 0) {
+          // Delay the initial run by 30 s to let the broker connection
+          // settle after boot (avoids a probe race with the boot-card
+          // quota probe that fires in the first few seconds).
+          setTimeout(() => {
+            void runQuotaWatch().catch((err) => {
+              process.stderr.write(`telegram gateway: quota-watch initial run failed: ${err}\n`)
+            })
+          }, 30_000)
+          setInterval(() => {
+            void runQuotaWatch().catch((err) => {
+              process.stderr.write(`telegram gateway: quota-watch scheduled run failed: ${err}\n`)
+            })
+          }, QUOTA_WATCH_POLL_MS).unref()
+        }
+
         // Restart-watchdog: poll systemd's NRestarts for the agent unit.
         // When the count ticks up without a corresponding restart-pending
         // marker (= user-initiated /restart), emit an operator event.

package/telegram-plugin/operator-events.ts

@@ -257,16 +257,12 @@ export function renderOperatorEvent(ev: OperatorEvent): RenderResult {
         text: [
           `💳 <b>Credit balance too low</b> for <b>${agent}</b>.`,
           detail ? `<i>${detail}</i>` : '',
-          `Swap to another account slot or add a new one.`,
+          `Use <code>/auth use &lt;label&gt;</code> to switch account slot or <code>/auth add</code> to add one.`,
         ]
           .filter(Boolean)
           .join('\n'),
         keyboard: {
           inline_keyboard: [
-            [
-              { text: '🔄 Swap slot', callback_data: `op:swap-slot:${encodeURIComponent(ev.agent)}` },
-              { text: '➕ Add slot', callback_data: `op:add-slot:${encodeURIComponent(ev.agent)}` },
-            ],
             [{ text: '⏳ Wait', callback_data: `op:dismiss:${encodeURIComponent(ev.agent)}` }],
           ],
         },
@@ -280,16 +276,12 @@ export function renderOperatorEvent(ev: OperatorEvent): RenderResult {
         text: [
           `⚠️ <b>Quota exhausted</b> for <b>${agent}</b>.`,
           detail ? `<i>${detail}</i>` : '',
-          `All account slots are at the usage limit. Switchroom will auto-fallback when another slot is available.`,
+          `All account slots are at the usage limit. Switchroom will auto-fallback when another slot is available. Use <code>/auth use &lt;label&gt;</code> to switch manually.`,
         ]
           .filter(Boolean)
           .join('\n'),
         keyboard: {
           inline_keyboard: [
-            [
-              { text: '🔄 Swap slot', callback_data: `op:swap-slot:${encodeURIComponent(ev.agent)}` },
-              { text: '➕ Add slot', callback_data: `op:add-slot:${encodeURIComponent(ev.agent)}` },
-            ],
             [{ text: '⏳ Wait', callback_data: `op:dismiss:${encodeURIComponent(ev.agent)}` }],
           ],
         },

package/telegram-plugin/quota-watch.ts

@@ -0,0 +1,276 @@
+/**
+ * Proactive quota threshold-tier push (#E4).
+ *
+ * Background: JTBD `track-plan-quota-live` anti-pattern: "Quota visible
+ * only in a separate dashboard or a command. If the user has to go
+ * looking, they won't, and they'll hit the wall." The existing stack
+ * covers the wall (auto-fallback at 99.5%, credits-watch on fatal billing
+ * transitions) but fires zero proactive signal at 80% — the point where
+ * the user can still act by switching accounts. This module closes that gap.
+ *
+ * It is a pure decision layer. It reads the broker's cached quota state
+ * for all accounts, classifies health via the same `classifyHealth`
+ * three-state machine used by the /auth dashboard, compares against a
+ * persisted last-notified state, and tells the gateway whether to emit
+ * a Telegram message + what to say. The gateway wires the actual
+ * `bot.api.sendMessage` call (via `swallowingApiCall`) — same as
+ * `credits-watch.ts`.
+ *
+ * Edge-trigger discipline: only fires on health *transitions*
+ * (healthy → throttling and throttling → healthy). Does NOT fire on
+ * healthy → blocked or blocked → healthy — `credits-watch.ts` already
+ * covers those via the fatal-billing path. Steady-state throttling
+ * never re-notifies.
+ *
+ * Scope: per-account across the whole pool, not just the active one.
+ * The user's natural recovery action is switching to a healthy account,
+ * so they need visibility into non-active accounts too.
+ *
+ * Source data: broker `listState` + `probeQuota`. `listState` is a local
+ * IPC call (cheap). `probeQuota` is only called on state-change (when
+ * we're going to send a message anyway) to get fresh numbers for the
+ * notification body. On no-change polls, only `listState` is called.
+ */
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from "fs";
+import { join } from "path";
+import type { AccountSnapshot } from "./auth-snapshot-format.js";
+import {
+  classifyHealth,
+  type AccountHealth,
+  THROTTLING_THRESHOLD_PCT,
+  bindingWindow,
+  formatRelative,
+  fmtPct,
+} from "./auth-snapshot-format.js";
+import type { QuotaUtilization } from "./quota-check.js";
+
+const STATE_FILE = "quota-watch.json";
+
+// ─── State types ──────────────────────────────────────────────────────────────
+
+/**
+ * Per-account last-notified health. We only care about the
+ * healthy ↔ throttling boundary — blocked is `credits-watch`'s domain.
+ * `null` means "never notified" (treat as healthy for transition logic).
+ */
+export type QuotaWatchHealth = "healthy" | "throttling" | null;
+
+export interface QuotaWatchAccountState {
+  /** Last health we sent a notification for. null = never notified. */
+  lastNotifiedHealth: QuotaWatchHealth;
+  /** Wall-clock ms of the last notification. */
+  lastNotifiedAt: number;
+}
+
+export type QuotaWatchState = Record<string, QuotaWatchAccountState>;
+
+export function emptyQuotaWatchState(): QuotaWatchState {
+  return {};
+}
+
+export function emptyAccountState(): QuotaWatchAccountState {
+  return { lastNotifiedHealth: null, lastNotifiedAt: 0 };
+}
+
+// ─── Decision logic ───────────────────────────────────────────────────────────
+
+export type QuotaWatchTransition =
+  | "entered-throttling"
+  | "recovered-to-healthy";
+
+export type QuotaWatchDecision =
+  | {
+      kind: "notify";
+      accountLabel: string;
+      message: string;
+      newAccountState: QuotaWatchAccountState;
+      transition: QuotaWatchTransition;
+    }
+  | { kind: "skip"; accountLabel: string; reason: string };
+
+/**
+ * Evaluate one account's quota state against its last-notified health.
+ *
+ * Transition table:
+ *   healthy → healthy        skip (steady-state)
+ *   healthy → throttling     notify (entered-throttling)
+ *   healthy → blocked        skip (credits-watch covers this)
+ *   throttling → healthy     notify (recovered-to-healthy)
+ *   throttling → throttling  skip (already notified)
+ *   throttling → blocked     skip (credits-watch covers blocked)
+ *   blocked → *              skip (credits-watch domain)
+ *   unknown → *              skip (no quota data — don't spam)
+ *   * → unknown              skip (probe failed — transient, don't alarm)
+ */
+export function evaluateQuotaWatchAccount(args: {
+  agentName: string;
+  snap: AccountSnapshot;
+  prev: QuotaWatchAccountState;
+  now: number;
+}): QuotaWatchDecision {
+  const { agentName, snap, prev, now } = args;
+  const label = snap.label;
+  const currentHealth = classifyHealth(snap);
+
+  // Unknown (probe failed) or blocked — skip entirely.
+  if (currentHealth === "unknown" || currentHealth === "blocked") {
+    return { kind: "skip", accountLabel: label, reason: `${currentHealth}-not-our-domain` };
+  }
+
+  // Normalise prev: null means healthy (never alerted = was healthy).
+  const prevHealth: "healthy" | "throttling" = prev.lastNotifiedHealth ?? "healthy";
+
+  // Steady-state — no change.
+  if (currentHealth === prevHealth) {
+    return { kind: "skip", accountLabel: label, reason: "steady-state" };
+  }
+
+  // healthy → throttling: proactive threshold push.
+  if (currentHealth === "throttling" && prevHealth === "healthy") {
+    const newState: QuotaWatchAccountState = {
+      lastNotifiedHealth: "throttling",
+      lastNotifiedAt: now,
+    };
+    return {
+      kind: "notify",
+      accountLabel: label,
+      message: buildThrottlingMessage(agentName, snap),
+      newAccountState: newState,
+      transition: "entered-throttling",
+    };
+  }
+
+  // throttling → healthy: recovery.
+  if (currentHealth === "healthy" && prevHealth === "throttling") {
+    const newState: QuotaWatchAccountState = {
+      lastNotifiedHealth: "healthy",
+      lastNotifiedAt: now,
+    };
+    return {
+      kind: "notify",
+      accountLabel: label,
+      message: buildRecoveryMessage(agentName, snap),
+      newAccountState: newState,
+      transition: "recovered-to-healthy",
+    };
+  }
+
+  // Any other combination (e.g. blocked → healthy, etc.) — skip.
+  return { kind: "skip", accountLabel: label, reason: "no-matching-transition" };
+}
+
+// ─── Message builders ─────────────────────────────────────────────────────────
+
+function buildThrottlingMessage(agentName: string, snap: AccountSnapshot): string {
+  const q = snap.quota!; // classifyHealth returned throttling, so quota is non-null
+  const fiveStr = fmtPct(q.fiveHourUtilizationPct);
+  const sevenStr = fmtPct(q.sevenDayUtilizationPct);
+  const max = Math.max(q.fiveHourUtilizationPct, q.sevenDayUtilizationPct);
+  const win = max === q.fiveHourUtilizationPct ? "5h" : "7d";
+  const winLabel = win === "5h" ? "5-hour" : "7-day";
+  const resetAt = win === "5h" ? q.fiveHourResetAt : q.sevenDayResetAt;
+  const resetStr = resetAt
+    ? ` · refills in ${formatRelative(resetAt, new Date())}`
+    : "";
+
+  const activeNote = snap.isActive
+    ? ""
+    : `\nThis is a non-active account. Consider <code>/auth use ${escapeHtml(snap.label)}</code> to switch, or keep it as a fallback reserve.`;
+
+  const altNote = snap.isActive
+    ? `\nConsider <code>/auth use &lt;other-account&gt;</code> if you have a healthier account, or wait for the ${winLabel} window to refill${resetStr}.`
+    : "";
+
+  return [
+    `🟡 <b>Quota approaching limit</b> — <code>${escapeHtml(snap.label)}</code>`,
+    ``,
+    `${fiveStr} of 5h  ·  ${sevenStr} of 7d`,
+    `Binding window: ${winLabel}${resetStr}`,
+    `${activeNote}${altNote}`,
+    ``,
+    `<i>Threshold: ${THROTTLING_THRESHOLD_PCT}% on either window. Source: broker quota cache.</i>`,
+    `<i>Run /auth for full fleet status or /usage for the active account.</i>`,
+  ]
+    .join("\n")
+    .replace(/\n\n\n+/g, "\n\n")
+    .trim();
+}
+
+function buildRecoveryMessage(agentName: string, snap: AccountSnapshot): string {
+  const q = snap.quota;
+  const utilLine = q
+    ? `Current: ${fmtPct(q.fiveHourUtilizationPct)} of 5h  ·  ${fmtPct(q.sevenDayUtilizationPct)} of 7d`
+    : "Current quota data unavailable.";
+
+  return [
+    `🟢 <b>Quota back in healthy range</b> — <code>${escapeHtml(snap.label)}</code>`,
+    ``,
+    utilLine,
+    ``,
+    `<i>Below ${THROTTLING_THRESHOLD_PCT}% on both windows.</i>`,
+  ].join("\n");
+}
+
+function escapeHtml(s: string): string {
+  return s
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&#39;");
+}
+
+// ─── State persistence ────────────────────────────────────────────────────────
+
+export function loadQuotaWatchState(stateDir: string): QuotaWatchState {
+  const path = join(stateDir, STATE_FILE);
+  if (!existsSync(path)) return emptyQuotaWatchState();
+  try {
+    const raw = readFileSync(path, "utf-8");
+    const parsed = JSON.parse(raw);
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      return emptyQuotaWatchState();
+    }
+    // Validate each entry — drop malformed ones rather than failing the whole file.
+    const result: QuotaWatchState = {};
+    for (const [key, val] of Object.entries(parsed)) {
+      if (
+        val &&
+        typeof val === "object" &&
+        !Array.isArray(val) &&
+        (
+          (val as Record<string, unknown>).lastNotifiedHealth === null ||
+          (val as Record<string, unknown>).lastNotifiedHealth === "healthy" ||
+          (val as Record<string, unknown>).lastNotifiedHealth === "throttling"
+        ) &&
+        typeof (val as Record<string, unknown>).lastNotifiedAt === "number" &&
+        Number.isFinite((val as Record<string, unknown>).lastNotifiedAt as number)
+      ) {
+        result[key] = val as QuotaWatchAccountState;
+      }
+    }
+    return result;
+  } catch {
+    return emptyQuotaWatchState();
+  }
+}
+
+export function saveQuotaWatchState(stateDir: string, state: QuotaWatchState): void {
+  mkdirSync(stateDir, { recursive: true });
+  const path = join(stateDir, STATE_FILE);
+  writeFileSync(path, JSON.stringify(state, null, 2) + "\n", { mode: 0o600 });
+}
+
+/**
+ * Merge one account's updated state into a full `QuotaWatchState` map.
+ * Callers use this after each `evaluateQuotaWatchAccount` that returns
+ * `kind: "notify"` to produce the new map to persist.
+ */
+export function patchQuotaWatchState(
+  current: QuotaWatchState,
+  accountLabel: string,
+  accountState: QuotaWatchAccountState,
+): QuotaWatchState {
+  return { ...current, [accountLabel]: accountState };
+}