switchroom 0.14.81 → 0.14.83

package/dist/agent-scheduler/index.js

@@ -6932,7 +6932,7 @@ var require_public_api = __commonJS((exports) => {
 });
 
 // src/agent-scheduler/index.ts
-import { resolve as resolve4, join } from "node:path";
+import { resolve as resolve4, join as join2 } from "node:path";
 
 // src/config/loader.ts
 import { readFileSync as readFileSync2, existsSync as existsSync3 } from "node:fs";
@@ -12295,6 +12295,581 @@ class JsonlAuditSink {
   close() {}
 }
 
+// src/scheduler/quota-preflight.ts
+function decideQuotaPreflight(state) {
+  const accounts = state.accounts ?? [];
+  if (accounts.length === 0) {
+    return { defer: false, reason: "no accounts in broker state" };
+  }
+  const healthy = accounts.filter((a) => !a.exhausted);
+  if (healthy.length > 0) {
+    return {
+      defer: false,
+      reason: `${healthy.length}/${accounts.length} account(s) healthy`
+    };
+  }
+  return { defer: true, reason: `all ${accounts.length} account(s) exhausted` };
+}
+
+// src/auth/broker/client.ts
+import * as net from "node:net";
+import { homedir as homedir2 } from "node:os";
+import { randomUUID } from "node:crypto";
+import { join } from "node:path";
+
+// src/auth/broker/protocol.ts
+var MAX_FRAME_BYTES = 64 * 1024;
+var PROTOCOL_VERSION = 1;
+var ProviderNameSchema = exports_external.enum(["anthropic", "google", "microsoft"]);
+var GetCredentialsRequestSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  op: exports_external.literal("get-credentials"),
+  id: exports_external.string().min(1),
+  provider: ProviderNameSchema.optional()
+});
+var ListStateRequestSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  op: exports_external.literal("list-state"),
+  id: exports_external.string().min(1)
+});
+var SetActiveRequestSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  op: exports_external.literal("set-active"),
+  id: exports_external.string().min(1),
+  account: exports_external.string().min(1),
+  provider: ProviderNameSchema.optional()
+});
+var MarkExhaustedRequestSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  op: exports_external.literal("mark-exhausted"),
+  id: exports_external.string().min(1),
+  until: exports_external.number().int().positive().optional()
+});
+var RefreshAccountRequestSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  op: exports_external.literal("refresh-account"),
+  id: exports_external.string().min(1),
+  account: exports_external.string().min(1),
+  provider: ProviderNameSchema.optional()
+});
+var AnthropicCredentialsSchema = exports_external.object({
+  claudeAiOauth: exports_external.object({
+    accessToken: exports_external.string(),
+    refreshToken: exports_external.string().optional(),
+    expiresAt: exports_external.number().optional(),
+    scopes: exports_external.array(exports_external.string()).optional(),
+    subscriptionType: exports_external.string().optional(),
+    rateLimitTier: exports_external.string().optional()
+  })
+});
+var GoogleCredentialsSchema = exports_external.object({
+  googleOauth: exports_external.object({
+    accessToken: exports_external.string(),
+    refreshToken: exports_external.string(),
+    expiresAt: exports_external.number(),
+    scope: exports_external.string(),
+    clientId: exports_external.string(),
+    accountEmail: exports_external.string(),
+    tokenType: exports_external.literal("Bearer")
+  })
+});
+var MicrosoftCredentialsSchema = exports_external.object({
+  microsoftOauth: exports_external.object({
+    accessToken: exports_external.string(),
+    refreshToken: exports_external.string(),
+    expiresAt: exports_external.number(),
+    scope: exports_external.string(),
+    clientId: exports_external.string(),
+    accountEmail: exports_external.string(),
+    tokenType: exports_external.literal("Bearer"),
+    tenantId: exports_external.string(),
+    accountType: exports_external.enum(["personal", "work"]),
+    homeAccountId: exports_external.string()
+  })
+});
+var ProviderCredentialsSchema = exports_external.union([
+  AnthropicCredentialsSchema,
+  GoogleCredentialsSchema,
+  MicrosoftCredentialsSchema
+]);
+var AddAccountRequestSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  op: exports_external.literal("add-account"),
+  id: exports_external.string().min(1),
+  label: exports_external.string().min(1),
+  provider: ProviderNameSchema.optional(),
+  credentials: ProviderCredentialsSchema,
+  replace: exports_external.boolean().optional()
+});
+var RmAccountRequestSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  op: exports_external.literal("rm-account"),
+  id: exports_external.string().min(1),
+  label: exports_external.string().min(1),
+  provider: ProviderNameSchema.optional()
+});
+var SetOverrideRequestSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  op: exports_external.literal("set-override"),
+  id: exports_external.string().min(1),
+  agent: exports_external.string().min(1),
+  account: exports_external.string().min(1).nullable()
+});
+var ListGoogleAccountsRequestSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  op: exports_external.literal("list-google-accounts"),
+  id: exports_external.string().min(1)
+});
+var ListMicrosoftAccountsRequestSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  op: exports_external.literal("list-microsoft-accounts"),
+  id: exports_external.string().min(1)
+});
+var ProbeQuotaRequestSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  op: exports_external.literal("probe-quota"),
+  id: exports_external.string().min(1),
+  accounts: exports_external.array(exports_external.string().min(1)).min(1).max(32),
+  timeoutMs: exports_external.number().int().positive().max(60000).optional()
+});
+var RequestSchema = exports_external.discriminatedUnion("op", [
+  GetCredentialsRequestSchema,
+  ListStateRequestSchema,
+  SetActiveRequestSchema,
+  MarkExhaustedRequestSchema,
+  RefreshAccountRequestSchema,
+  AddAccountRequestSchema,
+  RmAccountRequestSchema,
+  SetOverrideRequestSchema,
+  ListGoogleAccountsRequestSchema,
+  ListMicrosoftAccountsRequestSchema,
+  ProbeQuotaRequestSchema
+]);
+var GetCredentialsDataSchema = exports_external.object({
+  account: exports_external.string(),
+  credentials: exports_external.unknown(),
+  expiresAt: exports_external.number().optional()
+});
+var AccountStateSchema = exports_external.object({
+  label: exports_external.string(),
+  expiresAt: exports_external.number().optional(),
+  exhausted: exports_external.boolean(),
+  exhausted_until: exports_external.number().optional(),
+  threshold_violations: exports_external.number().int().nonnegative().optional(),
+  last_refreshed_at: exports_external.number().optional()
+});
+var AgentStateSchema = exports_external.object({
+  name: exports_external.string(),
+  account: exports_external.string(),
+  override: exports_external.string().nullable()
+});
+var ConsumerStateSchema = exports_external.object({
+  name: exports_external.string(),
+  account: exports_external.string(),
+  last_seen_at: exports_external.number().nullable()
+});
+var ListStateDataSchema = exports_external.object({
+  active: exports_external.string(),
+  fallback_order: exports_external.array(exports_external.string()),
+  accounts: exports_external.array(AccountStateSchema),
+  agents: exports_external.array(AgentStateSchema),
+  consumers: exports_external.array(ConsumerStateSchema)
+});
+var SetActiveDataSchema = exports_external.object({
+  active: exports_external.string(),
+  fanned: exports_external.array(exports_external.string())
+});
+var MarkExhaustedDataSchema = exports_external.object({
+  account: exports_external.string(),
+  rolled: exports_external.array(exports_external.string()),
+  rolledTo: exports_external.string().nullable().optional()
+});
+var RefreshAccountDataSchema = exports_external.object({
+  account: exports_external.string(),
+  expiresAt: exports_external.number().optional()
+});
+var AddAccountDataSchema = exports_external.object({
+  label: exports_external.string(),
+  expiresAt: exports_external.number().optional()
+});
+var RmAccountDataSchema = exports_external.object({
+  label: exports_external.string()
+});
+var SetOverrideDataSchema = exports_external.object({
+  agent: exports_external.string(),
+  account: exports_external.string().nullable()
+});
+var GoogleAccountStateSchema = exports_external.object({
+  account: exports_external.string(),
+  expiresAt: exports_external.number(),
+  scope: exports_external.string(),
+  clientId: exports_external.string()
+});
+var ListGoogleAccountsDataSchema = exports_external.object({
+  accounts: exports_external.array(GoogleAccountStateSchema)
+});
+var MicrosoftAccountStateSchema = exports_external.object({
+  account: exports_external.string(),
+  expiresAt: exports_external.number(),
+  scope: exports_external.string(),
+  clientId: exports_external.string(),
+  accountType: exports_external.enum(["personal", "work"])
+});
+var ListMicrosoftAccountsDataSchema = exports_external.object({
+  accounts: exports_external.array(MicrosoftAccountStateSchema)
+});
+var ErrorBodySchema = exports_external.object({
+  code: exports_external.enum([
+    "FORBIDDEN",
+    "INVALID_ARGS",
+    "UNKNOWN_VERB",
+    "VERSION_MISMATCH",
+    "ACCOUNT_NOT_FOUND",
+    "ACCOUNT_ALREADY_EXISTS",
+    "CONFIG_INVALID",
+    "DRIFT_DETECTED",
+    "REFRESH_FAILED",
+    "INTERNAL"
+  ]),
+  message: exports_external.string()
+});
+var SuccessResponseSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  id: exports_external.string(),
+  ok: exports_external.literal(true),
+  data: exports_external.unknown()
+});
+var ErrorResponseSchema = exports_external.object({
+  v: exports_external.literal(PROTOCOL_VERSION),
+  id: exports_external.string(),
+  ok: exports_external.literal(false),
+  error: ErrorBodySchema
+});
+var ResponseSchema = exports_external.discriminatedUnion("ok", [
+  SuccessResponseSchema,
+  ErrorResponseSchema
+]);
+function encodeRequest(req) {
+  const line = JSON.stringify(RequestSchema.parse(req)) + `
+`;
+  if (Buffer.byteLength(line, "utf-8") > MAX_FRAME_BYTES) {
+    throw new Error(`auth-broker request exceeds MAX_FRAME_BYTES (${MAX_FRAME_BYTES})`);
+  }
+  return line;
+}
+function decodeResponse(line) {
+  const trimmed = line.endsWith(`
+`) ? line.slice(0, -1) : line;
+  let parsed;
+  try {
+    parsed = JSON.parse(trimmed);
+  } catch {
+    throw new Error("auth-broker response is not valid JSON");
+  }
+  return ResponseSchema.parse(parsed);
+}
+
+// src/auth/broker/client.ts
+var DEFAULT_TIMEOUT_MS = 5000;
+function reviveDate(v) {
+  if (v == null)
+    return null;
+  if (v instanceof Date)
+    return Number.isNaN(v.getTime()) ? null : v;
+  const d = new Date(v);
+  return Number.isNaN(d.getTime()) ? null : d;
+}
+function operatorSocketPath(home2 = homedir2()) {
+  return join(home2, ".switchroom", "state", "auth-broker-operator", "sock");
+}
+function resolveAuthBrokerSocketPath(opts) {
+  if (opts?.socket)
+    return opts.socket;
+  const env = process.env.SWITCHROOM_AUTH_BROKER_SOCKET;
+  if (env && env.length > 0)
+    return env;
+  return operatorSocketPath(opts?.home);
+}
+
+class AuthBrokerError extends Error {
+  code;
+  constructor(code, message) {
+    super(message);
+    this.code = code;
+    this.name = "AuthBrokerError";
+  }
+}
+
+class AuthBrokerUnreachableError extends Error {
+  reason;
+  socketPath;
+  constructor(reason, socketPath) {
+    super(`auth-broker unreachable at ${socketPath}: ${reason}. ` + `The broker may be down; existing credentials remain valid until expiry.`);
+    this.reason = reason;
+    this.socketPath = socketPath;
+    this.name = "AuthBrokerUnreachableError";
+  }
+}
+
+class AuthBrokerClient {
+  socketPath;
+  timeoutMs;
+  socket = null;
+  connecting = null;
+  buffer = "";
+  pending = new Map;
+  closed = false;
+  constructor(opts = {}) {
+    this.socketPath = resolveAuthBrokerSocketPath(opts);
+    this.timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  }
+  getSocketPath() {
+    return this.socketPath;
+  }
+  async close() {
+    this.closed = true;
+    const sock = this.socket;
+    this.socket = null;
+    this.connecting = null;
+    for (const [, p] of this.pending) {
+      clearTimeout(p.timer);
+      p.reject(new Error("auth-broker client closed"));
+    }
+    this.pending.clear();
+    if (sock) {
+      sock.destroy();
+    }
+  }
+  async getCredentials(provider) {
+    const base = {
+      v: PROTOCOL_VERSION,
+      id: randomUUID(),
+      op: "get-credentials"
+    };
+    const req = provider !== undefined ? { ...base, provider } : base;
+    const data = await this.send(req);
+    return data;
+  }
+  async listState() {
+    const data = await this.send({
+      v: PROTOCOL_VERSION,
+      id: randomUUID(),
+      op: "list-state"
+    });
+    return data;
+  }
+  async listGoogleAccounts() {
+    const data = await this.send({
+      v: PROTOCOL_VERSION,
+      id: randomUUID(),
+      op: "list-google-accounts"
+    });
+    return data;
+  }
+  async listMicrosoftAccounts() {
+    const data = await this.send({
+      v: PROTOCOL_VERSION,
+      id: randomUUID(),
+      op: "list-microsoft-accounts"
+    });
+    return data;
+  }
+  async probeQuota(accounts, timeoutMs) {
+    const data = await this.send({
+      v: PROTOCOL_VERSION,
+      id: randomUUID(),
+      op: "probe-quota",
+      accounts: [...accounts],
+      ...timeoutMs !== undefined ? { timeoutMs } : {}
+    });
+    const parsed = data;
+    for (const entry of parsed.results) {
+      if (entry.result.ok) {
+        entry.result.data.fiveHourResetAt = reviveDate(entry.result.data.fiveHourResetAt);
+        entry.result.data.sevenDayResetAt = reviveDate(entry.result.data.sevenDayResetAt);
+      }
+    }
+    return parsed;
+  }
+  async setActive(account) {
+    const data = await this.send({
+      v: PROTOCOL_VERSION,
+      id: randomUUID(),
+      op: "set-active",
+      account
+    });
+    return data;
+  }
+  async markExhausted(until) {
+    const req = until !== undefined ? { v: PROTOCOL_VERSION, id: randomUUID(), op: "mark-exhausted", until } : { v: PROTOCOL_VERSION, id: randomUUID(), op: "mark-exhausted" };
+    const data = await this.send(req);
+    return data;
+  }
+  async refreshAccount(account) {
+    const data = await this.send({
+      v: PROTOCOL_VERSION,
+      id: randomUUID(),
+      op: "refresh-account",
+      account
+    });
+    return data;
+  }
+  async addAccount(label, credentials, replace, provider) {
+    const base = {
+      v: PROTOCOL_VERSION,
+      id: randomUUID(),
+      op: "add-account",
+      label,
+      credentials
+    };
+    const withReplace = replace ? { ...base, replace: true } : base;
+    const req = provider !== undefined ? { ...withReplace, provider } : withReplace;
+    const data = await this.send(req);
+    return data;
+  }
+  async rmAccount(label, provider) {
+    const base = {
+      v: PROTOCOL_VERSION,
+      id: randomUUID(),
+      op: "rm-account",
+      label
+    };
+    const req = provider !== undefined ? { ...base, provider } : base;
+    const data = await this.send(req);
+    return data;
+  }
+  async setOverride(agent, account) {
+    const data = await this.send({
+      v: PROTOCOL_VERSION,
+      id: randomUUID(),
+      op: "set-override",
+      agent,
+      account
+    });
+    return data;
+  }
+  async ensureConnected() {
+    if (this.closed) {
+      throw new Error("auth-broker client is closed");
+    }
+    if (this.socket && !this.socket.destroyed)
+      return this.socket;
+    if (this.connecting)
+      return this.connecting;
+    this.connecting = new Promise((resolve4, reject) => {
+      const sock = new net.Socket;
+      const onError = (err) => {
+        sock.removeAllListeners();
+        sock.destroy();
+        const code = err.code ?? "ERR";
+        let reason;
+        if (code === "ENOENT")
+          reason = "socket file not found";
+        else if (code === "ECONNREFUSED")
+          reason = "connection refused";
+        else if (code === "EACCES")
+          reason = "access denied";
+        else
+          reason = err.message;
+        reject(new AuthBrokerUnreachableError(reason, this.socketPath));
+      };
+      sock.once("error", onError);
+      sock.once("connect", () => {
+        sock.removeListener("error", onError);
+        sock.on("data", (chunk) => this.onData(chunk));
+        sock.on("error", (err) => this.onSocketError(err));
+        sock.on("close", () => this.onSocketClose());
+        this.socket = sock;
+        resolve4(sock);
+      });
+      sock.connect({ path: this.socketPath });
+    });
+    try {
+      return await this.connecting;
+    } finally {
+      this.connecting = null;
+    }
+  }
+  onData(chunk) {
+    this.buffer += chunk.toString("utf8");
+    let idx;
+    while ((idx = this.buffer.indexOf(`
+`)) !== -1) {
+      const line = this.buffer.slice(0, idx);
+      this.buffer = this.buffer.slice(idx + 1);
+      if (line.length === 0)
+        continue;
+      let resp;
+      try {
+        resp = decodeResponse(line);
+      } catch (err) {
+        const msg = `unparseable auth-broker response: ${err instanceof Error ? err.message : String(err)}`;
+        this.failAll(new AuthBrokerUnreachableError(msg, this.socketPath));
+        return;
+      }
+      const p = this.pending.get(resp.id);
+      if (!p) {
+        continue;
+      }
+      this.pending.delete(resp.id);
+      clearTimeout(p.timer);
+      p.resolve(resp);
+    }
+  }
+  onSocketError(err) {
+    this.failAll(new AuthBrokerUnreachableError(err.message, this.socketPath));
+    if (this.socket) {
+      this.socket.destroy();
+      this.socket = null;
+    }
+  }
+  onSocketClose() {
+    if (this.pending.size > 0) {
+      this.failAll(new AuthBrokerUnreachableError("connection closed mid-request", this.socketPath));
+    }
+    this.socket = null;
+  }
+  failAll(err) {
+    for (const [, p] of this.pending) {
+      clearTimeout(p.timer);
+      p.reject(err);
+    }
+    this.pending.clear();
+  }
+  async send(req) {
+    const sock = await this.ensureConnected();
+    const id = req.id;
+    const frame = encodeRequest(req);
+    return new Promise((resolve4, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(id);
+        reject(new AuthBrokerUnreachableError(`request ${req.op} timed out after ${this.timeoutMs}ms`, this.socketPath));
+      }, this.timeoutMs);
+      this.pending.set(id, {
+        resolve: (resp) => {
+          if (resp.ok) {
+            resolve4(resp.data);
+          } else {
+            reject(new AuthBrokerError(resp.error.code, resp.error.message));
+          }
+        },
+        reject,
+        timer
+      });
+      sock.write(frame, (err) => {
+        if (err) {
+          const p = this.pending.get(id);
+          if (p) {
+            clearTimeout(p.timer);
+            this.pending.delete(id);
+          }
+          reject(new AuthBrokerUnreachableError(`failed to send ${req.op}: ${err.message}`, this.socketPath));
+        }
+      });
+    });
+  }
+}
+
 // src/agent-scheduler/ipc-client.ts
 import { createConnection } from "node:net";
 function createInjectIpcClient(options) {
@@ -12734,12 +13309,55 @@ function readRecentFires(jsonlPath) {
 }
 
 // src/agent-scheduler/index.ts
+var DEFAULT_MAX_QUOTA_DEFER_ATTEMPTS = 3;
+function defaultQuotaDeferBackoffMs(attempt) {
+  return [60000, 180000, 300000][attempt] ?? 300000;
+}
 function registerAgentSchedule(opts) {
   const tasks = [];
   const now = opts.now ?? Date.now;
+  const maxAttempts = opts.maxQuotaDeferAttempts ?? DEFAULT_MAX_QUOTA_DEFER_ATTEMPTS;
+  const backoff = opts.quotaDeferBackoffMs ?? defaultQuotaDeferBackoffMs;
+  const scheduleRetry = opts.scheduleRetry ?? ((fn, ms) => {
+    const t = setTimeout(fn, ms);
+    if (typeof t.unref === "function")
+      t.unref();
+    return { cancel: () => clearTimeout(t) };
+  });
   for (const entry of opts.entries) {
-    const task = opts.cronLib.schedule(entry.cron, () => {
+    const pendingRetries = new Set;
+    const attemptFire = async (attempt) => {
       const startedAt = now();
+      if (opts.quotaGate) {
+        let decision;
+        try {
+          decision = await opts.quotaGate(entry.agent);
+        } catch {
+          decision = { defer: false, reason: "quota gate error (fail-open)" };
+        }
+        if (decision.defer) {
+          const more = attempt + 1 < maxAttempts;
+          const summary2 = `deferred (quota): ${decision.reason} ` + `[attempt ${attempt + 1}/${maxAttempts}]` + (more ? "" : " — giving up; will re-run on next scheduled occurrence");
+          opts.sink.recordFire({
+            agent: entry.agent,
+            scheduleIndex: entry.scheduleIndex,
+            promptKey: entry.promptKey,
+            exitCode: -2,
+            outputSummary: summary2,
+            startedAt,
+            finishedAt: now()
+          });
+          if (more) {
+            let handle;
+            handle = scheduleRetry(() => {
+              pendingRetries.delete(handle);
+              attemptFire(attempt + 1);
+            }, backoff(attempt));
+            pendingRetries.add(handle);
+          }
+          return;
+        }
+      }
       let delivered = false;
       let summary = "";
       try {
@@ -12760,8 +13378,19 @@ function registerAgentSchedule(opts) {
         startedAt,
         finishedAt
       });
+    };
+    const task = opts.cronLib.schedule(entry.cron, () => attemptFire(0));
+    tasks.push({
+      entry,
+      task: {
+        stop: () => {
+          for (const h of pendingRetries)
+            h.cancel();
+          pendingRetries.clear();
+          task.stop();
+        }
+      }
     });
-    tasks.push({ entry, task });
   }
   return tasks;
 }
@@ -12785,7 +13414,7 @@ async function main() {
   }
   const configPath = process.env.SWITCHROOM_CONFIG ?? "/state/config/switchroom.yaml";
   const stateDir = process.env.TELEGRAM_STATE_DIR ?? "/state/agent/telegram";
-  const socketPath = process.env.SWITCHROOM_GATEWAY_SOCKET ?? join(stateDir, "gateway.sock");
+  const socketPath = process.env.SWITCHROOM_GATEWAY_SOCKET ?? join2(stateDir, "gateway.sock");
   const jsonlPath = process.env.SWITCHROOM_AGENT_SCHEDULER_JSONL ?? "/state/agent/scheduler.jsonl";
   const lockPath = process.env.SWITCHROOM_AGENT_SCHEDULER_LOCK ?? "/state/agent/scheduler.lock";
   const lock = acquireLock(lockPath);
@@ -12913,12 +13542,22 @@ Briefly and plainly tell the user these scheduled runs did not ` + "happen so th
     }
   }
   const cronLib = __require("node-cron");
+  const quotaPreflightEnabled = process.env.SWITCHROOM_DISABLE_CRON_QUOTA_PREFLIGHT !== "1";
+  const quotaGate = quotaPreflightEnabled ? async () => {
+    const client = new AuthBrokerClient;
+    try {
+      return decideQuotaPreflight(await client.listState());
+    } finally {
+      await client.close().catch(() => {});
+    }
+  } : undefined;
   const tasks = registerAgentSchedule({
     entries,
     channel,
     sink,
     cronLib,
-    dispatcher
+    dispatcher,
+    ...quotaGate ? { quotaGate } : {}
   });
   process.stdout.write(`agent-scheduler: ${agentName} registered ${tasks.length} task(s); ` + `chat=${channel.chatId} thread=${channel.threadId ?? "(none)"} ` + `socket=${socketPath} jsonl=${jsonlPath}
 `);

package/dist/cli/switchroom.js

@@ -14863,6 +14863,39 @@ async function parseSseOrJson(resp) {
   const payload = dataLine ? dataLine.slice("data: ".length) : text;
   return JSON.parse(payload);
 }
+async function fetchHindsightToolsList(apiUrl, opts) {
+  const fetchImpl = opts?.fetchImpl ?? fetch;
+  const timeoutMs = opts?.timeoutMs ?? 4000;
+  const bankId = opts?.bankId ?? "__doctor_probe__";
+  const controller = new AbortController;
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const resp = await fetchImpl(`${apiUrl}`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Accept: "application/json, text/event-stream",
+        "X-Bank-Id": bankId
+      },
+      body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }),
+      signal: controller.signal
+    });
+    clearTimeout(timeout);
+    if (!resp.ok)
+      return { ok: false, reason: `HTTP ${resp.status}` };
+    const parsed = await parseSseOrJson(resp);
+    const raw = parsed.result?.tools;
+    if (!Array.isArray(raw))
+      return { ok: false, reason: "no tools in tools/list response" };
+    const tools = raw.filter((t) => typeof t?.name === "string").map((t) => ({ name: t.name, required: t.inputSchema?.required ?? [] }));
+    return { ok: true, tools };
+  } catch (err) {
+    clearTimeout(timeout);
+    if (err.name === "AbortError")
+      return { ok: false, reason: "Timeout" };
+    return { ok: false, reason: String(err.message ?? err) };
+  }
+}
 async function probeHindsight(apiUrl, opts) {
   const fetchImpl = opts?.fetchImpl ?? fetch;
   const timeoutMs = opts?.timeoutMs ?? 3000;
@@ -14991,8 +15024,7 @@ async function ensureUserProfileMentalModel(apiUrl, bankId, opts) {
           name: "create_mental_model",
           arguments: {
             name: "user-profile",
-            source_query: "What are the key facts, preferences, context, and communication style about the user I talk to? Summarize what matters for making the agent feel like it knows them.",
-            types: ["world", "experience"]
+            source_query: "What are the key facts, preferences, context, and communication style about the user I talk to? Summarize what matters for making the agent feel like it knows them."
           }
         }
       }),
@@ -15077,6 +15109,12 @@ async function createBank(apiUrl, bankId, opts) {
     if (!toolResponse.ok) {
       return { ok: false, reason: `Tool call HTTP ${toolResponse.status}` };
     }
+    try {
+      const created = await parseSseOrJson(toolResponse);
+      if (created.result?.isError === true) {
+        return { ok: false, reason: created.result.content?.[0]?.text ?? "create_bank returned isError" };
+      }
+    } catch {}
     return { ok: true };
   } catch (err) {
     if (err.name === "AbortError") {
@@ -15137,8 +15175,8 @@ async function updateBankMissions(apiUrl, bankId, missions, opts) {
           name: "update_bank",
           arguments: {
             bank_id: bankId,
-            mission: missions.bank_mission,
-            retain_mission: missions.retain_mission
+            ...missions.bank_mission != null ? { mission: missions.bank_mission } : {},
+            ...missions.retain_mission != null ? { config_updates: { retain_mission: missions.retain_mission } } : {}
           }
         }
       }),
@@ -15148,6 +15186,12 @@ async function updateBankMissions(apiUrl, bankId, missions, opts) {
     if (!toolResponse.ok) {
       return { ok: false, reason: `Tool call HTTP ${toolResponse.status}` };
     }
+    try {
+      const updated = await parseSseOrJson(toolResponse);
+      if (updated.result?.isError === true) {
+        return { ok: false, reason: updated.result.content?.[0]?.text ?? "update_bank returned isError" };
+      }
+    } catch {}
     return { ok: true };
   } catch (err) {
     if (err.name === "AbortError") {
@@ -28959,6 +29003,30 @@ var init_manifest = __esm(() => {
   ]);
 });
 
+// src/memory/hindsight-tools.ts
+var EXPECTED_HINDSIGHT_TOOLS;
+var init_hindsight_tools = __esm(() => {
+  EXPECTED_HINDSIGHT_TOOLS = {
+    recall: { required: ["query"] },
+    reflect: { required: ["query"] },
+    retain: { required: ["content"] },
+    sync_retain: { required: ["content"] },
+    delete_document: { required: ["document_id"] },
+    create_directive: { required: ["content", "name"] },
+    list_directives: { required: [] },
+    delete_directive: { required: ["directive_id"] },
+    create_bank: { required: ["bank_id"] },
+    update_bank: { required: [] },
+    list_banks: { required: [] },
+    create_mental_model: { required: ["name", "source_query"] },
+    list_mental_models: { required: [] },
+    update_mental_model: { required: ["mental_model_id"] },
+    refresh_mental_model: { required: ["mental_model_id"] },
+    list_memories: { required: [] },
+    get_memory: { required: ["memory_id"] }
+  };
+});
+
 // src/cli/doctor-memory.ts
 import { execFileSync as execFileSync17 } from "node:child_process";
 function classifyShmSize(bytes) {
@@ -29030,8 +29098,51 @@ function checkHindsightContainerHealth(opts) {
   } catch {}
   return results;
 }
+function classifyToolContract(advertised) {
+  const byName = new Map(advertised.map((t) => [t.name, t]));
+  const results = [];
+  for (const [tool, spec] of Object.entries(EXPECTED_HINDSIGHT_TOOLS)) {
+    const real = byName.get(tool);
+    if (real === undefined) {
+      results.push({
+        name: `hindsight contract: ${tool}`,
+        status: "fail",
+        detail: `switchroom calls \`${tool}\` but the server no longer advertises it ` + `(renamed/removed upstream) \u2014 every callsite silently no-ops`,
+        fix: "Upstream hindsight changed its MCP tool contract. Update the callsite " + "+ EXPECTED_HINDSIGHT_TOOLS (src/memory/hindsight-tools.ts) to the new " + "name, refresh tests/fixtures/hindsight-tools-list.snapshot.json, or pin " + "the prior hindsight image."
+      });
+      continue;
+    }
+    const missing = spec.required.filter((arg) => !real.required.includes(arg));
+    const added = real.required.filter((arg) => !spec.required.includes(arg));
+    if (added.length > 0) {
+      results.push({
+        name: `hindsight contract: ${tool}`,
+        status: "fail",
+        detail: `server now requires [${added.join(", ")}] on \`${tool}\` which ` + `switchroom does not track \u2014 calls may silently no-op`,
+        fix: "Reconcile EXPECTED_HINDSIGHT_TOOLS + the callsite args with the new " + "server schema, then refresh the snapshot fixture."
+      });
+    } else if (missing.length > 0) {
+      results.push({
+        name: `hindsight contract: ${tool}`,
+        status: "warn",
+        detail: `switchroom treats [${missing.join(", ")}] as required on \`${tool}\` ` + `but the server no longer does (loosened upstream) \u2014 harmless, but the ` + `fixture is stale`,
+        fix: "Refresh EXPECTED_HINDSIGHT_TOOLS + the snapshot fixture."
+      });
+    }
+  }
+  if (results.length === 0) {
+    const used = Object.keys(EXPECTED_HINDSIGHT_TOOLS).length;
+    results.push({
+      name: "hindsight contract",
+      status: "ok",
+      detail: `${used} used tools present, required args satisfied (${advertised.length} advertised)`
+    });
+  }
+  return results;
+}
 var MIN_HINDSIGHT_SHM_BYTES;
 var init_doctor_memory = __esm(() => {
+  init_hindsight_tools();
   MIN_HINDSIGHT_SHM_BYTES = 1024 * 1024 * 1024;
 });
 
@@ -32043,6 +32154,10 @@ async function checkHindsight(config) {
     status: "ok",
     detail: `${probe2.serverName} ${probe2.serverVersion} at ${host}:${port}`
   });
+  const toolsList = await fetchHindsightToolsList(url);
+  if (toolsList.ok) {
+    results.push(...classifyToolContract(toolsList.tools));
+  }
   results.push(checkHindsightConsumer(config));
   results.push(...checkHindsightContainerHealth());
   for (const [agentName, agentConfig] of Object.entries(config.agents)) {
@@ -49700,8 +49815,8 @@ var {
 } = import__.default;
 
 // src/build-info.ts
-var VERSION = "0.14.81";
-var COMMIT_SHA = "4ac9cc7d";
+var VERSION = "0.14.83";
+var COMMIT_SHA = "057ab099";
 
 // src/cli/agent.ts
 init_source();

package/dist/cli/ui/index.html

@@ -559,16 +559,42 @@
       }
     }
 
+    // Guards a second click from starting a second device-code flow (each
+    // start makes Microsoft send a sign-in email → the "2 emails" bug).
+    let msConnecting = false;
+
+    // The set of Microsoft account emails currently known to the broker.
+    // Used as a resilience baseline: the connect status lives only in the web
+    // process's memory, so if that process restarts mid-connect the status
+    // reads 'unknown' even when the token WAS stored. Diffing this list tells
+    // us the account really connected regardless of the lost status.
+    async function fetchMicrosoftAccountEmails() {
+      try {
+        const r = await fetch(`${API}/api/microsoft-accounts`, { headers: authHeaders() });
+        if (!r.ok) return new Set();
+        const list = await r.json();
+        return new Set((list || []).filter(a => a.brokerKnown).map(a => String(a.account).toLowerCase()));
+      } catch { return new Set(); }
+    }
+
     // Start an in-browser Microsoft connect: show the device code + link,
     // then poll until the operator completes sign-in on Microsoft's site.
     async function connectMicrosoft() {
+      if (msConnecting) return;              // double-submit guard
+      msConnecting = true;
+      const btn = document.getElementById('ms-connect-btn');
+      if (btn) btn.disabled = true;
+      const done = () => { msConnecting = false; const b = document.getElementById('ms-connect-btn'); if (b) b.disabled = false; };
       const card = document.getElementById('ms-connect-card');
       const show = (html) => { if (card) card.innerHTML = html; };
       show('<div class="loading" style="padding:.8rem">Starting…</div>');
+      // Snapshot already-connected accounts BEFORE starting, for the
+      // restart-resilient terminal check below.
+      const before = await fetchMicrosoftAccountEmails();
       try {
         const res = await fetch(`${API}/api/connections/microsoft/connect`, { method: 'POST', headers: authHeaders() });
         const data = await res.json();
-        if (!res.ok || !data.ok) { show(''); showError(data.error || `HTTP ${res.status}`); return; }
+        if (!res.ok || !data.ok) { show(''); showError(data.error || `HTTP ${res.status}`); done(); return; }
         const url = data.verificationUri, code = data.userCode;
         show(`<div class="account-card" style="border-color:var(--accent)">
             <div class="account-card-header"><div class="account-label">Connect a Microsoft account</div></div>
@@ -580,27 +606,58 @@
             <div id="ms-connect-status" style="color:var(--text-dim);margin-top:.3rem">Waiting for sign-in… (this card expires in ~15 min)</div>
           </div>`);
         const statusEl = () => document.getElementById('ms-connect-status');
-        const started = Date.now();
+        const deadline = Date.now() + ((data.expiresInSec || 900) * 1000 + 30000);
+        const showConnected = (label) => {
+          show(`<div class="loading" style="padding:.8rem;color:var(--green)">✓ Connected ${escapeHtml(label)}. Use the access toggles below to grant an agent.</div>`);
+          fetchConnections();
+        };
+        // On any non-'connected' terminal state ('failed' or 'unknown'),
+        // re-check the broker's actual account list before declaring failure:
+        // a new account appearing means the connect really succeeded (e.g. the
+        // status was lost to a web restart). Only error if nothing new landed.
+        // Limitations (acceptable — the in-memory 'connected' state is the
+        // primary signal; this is only the lost-status fallback): a concurrent
+        // connect of a DIFFERENT account in another tab/CLI could be mistaken
+        // for this one; and re-connecting an ALREADY-known account (token
+        // refresh) shows no new account so falls through to the error.
+        const settleNonConnected = async (reason) => {
+          const after = await fetchMicrosoftAccountEmails();
+          const fresh = [...after].find(a => !before.has(a));
+          if (fresh) { showConnected(fresh); } else { show(''); showError(reason || 'connect failed'); }
+          done();
+        };
         const poll = async () => {
-          const sres = await fetch(`${API}/api/connections/microsoft/connect/${encodeURIComponent(data.requestId)}`, { headers: authHeaders() });
-          const s = sres.ok ? await sres.json() : { state: 'failed', reason: `HTTP ${sres.status}` };
+          let s;
+          try {
+            const sres = await fetch(`${API}/api/connections/microsoft/connect/${encodeURIComponent(data.requestId)}`, { headers: authHeaders() });
+            s = sres.ok ? await sres.json() : { state: 'failed', reason: `HTTP ${sres.status}` };
+          } catch {
+            // Transient fetch failure — most likely the web process restarting
+            // mid-connect (the exact case this flow must survive). Don't die
+            // (that would strand msConnecting=true and lock the button): keep
+            // polling until the device-code deadline, then settle via the
+            // broker re-check (which recovers a token stored before the restart).
+            if (Date.now() > deadline) { await settleNonConnected('connection lost'); return; }
+            setTimeout(poll, 3000);
+            return;
+          }
           if (s.state === 'pending') {
-            if (Date.now() - started > ((data.expiresInSec || 900) * 1000 + 30000)) { const e = statusEl(); if (e) e.textContent = 'Expired — click Connect to try again.'; return; }
+            if (Date.now() > deadline) { const e = statusEl(); if (e) e.textContent = 'Expired — click Connect to try again.'; done(); return; }
             setTimeout(poll, 3000);
             return;
           }
           if (s.state === 'connected') {
-            show(`<div class="loading" style="padding:.8rem;color:var(--green)">✓ Connected ${escapeHtml(s.account)} (${escapeHtml(s.accountType)}). Use the access toggles below to grant an agent.</div>`);
-            fetchConnections();
+            showConnected(`${s.account} (${s.accountType})`);
+            done();
           } else {
-            show('');
-            showError(s.reason || 'connect failed');
+            await settleNonConnected(s.reason);
           }
         };
         setTimeout(poll, 3000);
       } catch (err) {
         show('');
         showError(err.message);
+        done();
       }
     }
 
@@ -1165,7 +1222,7 @@
         <div style="margin-bottom:1.5rem">
           <h3 style="margin:0 0 .6rem;font-size:.95rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:.04em">
             Microsoft 365
-            <button onclick="connectMicrosoft()" class="usage-pill primary" style="margin-left:.6rem;cursor:pointer;border:none;text-transform:none;font-weight:600">+ Connect a Microsoft account</button>
+            <button id="ms-connect-btn" onclick="connectMicrosoft()" class="usage-pill primary" style="margin-left:.6rem;cursor:pointer;border:none;text-transform:none;font-weight:600">+ Connect a Microsoft account</button>
           </h3>
           <div id="ms-connect-card"></div>
           ${msCards

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "switchroom",
-  "version": "0.14.81",
+  "version": "0.14.83",
   "description": "Run Claude Code 24/7 on your Claude Pro/Max subscription over Telegram. Open-source alternative to OpenClaw and NanoClaw — no API keys.",
   "type": "module",
   "bin": {

package/telegram-plugin/dist/gateway/gateway.js

@@ -52661,6 +52661,53 @@ function evaluateQuotaWatchAccount(args) {
   }
   return { kind: "skip", accountLabel: label, reason: "no-matching-transition" };
 }
+var FLEET_ALL_EXHAUSTED_KEY = "__fleet_all_exhausted__";
+function evaluateFleetAllExhausted(args) {
+  const { accounts, prev, now } = args;
+  const allExhausted = accounts.length > 0 && accounts.every((a) => a.exhausted);
+  const wasAlerting = prev.lastNotifiedHealth === "throttling";
+  if (allExhausted && !wasAlerting) {
+    return {
+      kind: "notify",
+      message: buildAllExhaustedMessage(accounts, now),
+      newState: { lastNotifiedHealth: "throttling", lastNotifiedAt: now },
+      transition: "entered"
+    };
+  }
+  if (!allExhausted && wasAlerting) {
+    return {
+      kind: "notify",
+      message: buildFleetRecoveredMessage(accounts),
+      newState: { lastNotifiedHealth: "healthy", lastNotifiedAt: now },
+      transition: "recovered"
+    };
+  }
+  return { kind: "skip", reason: allExhausted ? "still-all-exhausted" : "not-all-exhausted" };
+}
+function buildAllExhaustedMessage(accounts, now) {
+  const resets = accounts.map((a) => a.exhausted_until).filter((x) => typeof x === "number" && x > now);
+  const earliest = resets.length > 0 ? Math.min(...resets) : null;
+  const resetLine = earliest ? `Earliest reset: ${formatRelative(new Date(earliest), new Date(now))}.` : `Reset time unknown (no window data).`;
+  return [
+    `\uD83D\uDD34 <b>All accounts exhausted</b>`,
+    ``,
+    `Every Anthropic account (${accounts.length}) is quota-walled \u2014 there is no healthy account to fail over to.`,
+    resetLine,
+    ``,
+    `<i>This is self-healing: agents resume and deferred scheduled jobs run automatically once a window resets. Nothing is lost. Add headroom with <code>/auth add</code> if this recurs.</i>`
+  ].join(`
+`);
+}
+function buildFleetRecoveredMessage(accounts) {
+  const healthy = accounts.filter((a) => !a.exhausted).map((a) => a.label);
+  const which = healthy.length > 0 ? ` (<code>${escapeHtml10(healthy[0])}</code>)` : "";
+  return [
+    `\uD83D\uDFE2 <b>Fleet recovered</b> \u2014 at least one account is healthy again${which}.`,
+    ``,
+    `<i>Agents are back; any deferred scheduled jobs will run on their next occurrence.</i>`
+  ].join(`
+`);
+}
 function buildThrottlingMessage(agentName3, snap) {
   const q = snap.quota;
   const fiveStr = fmtPct(q.fiveHourUtilizationPct);
@@ -52810,11 +52857,11 @@ function sweepStaleTurnActiveMarker(stateDir, opts) {
 }
 
 // ../src/build-info.ts
-var VERSION = "0.14.81";
-var COMMIT_SHA = "4ac9cc7d";
-var COMMIT_DATE = "2026-06-07T10:43:55+10:00";
+var VERSION = "0.14.83";
+var COMMIT_SHA = "057ab099";
+var COMMIT_DATE = "2026-06-07T12:57:08+10:00";
 var LATEST_PR = null;
-var COMMITS_AHEAD_OF_TAG = 2;
+var COMMITS_AHEAD_OF_TAG = 4;
 
 // gateway/boot-version.ts
 function formatRelativeAgo(iso) {
@@ -61017,6 +61064,31 @@ async function runQuotaWatch() {
   let watchState = loadQuotaWatchState(stateDir);
   const now = Date.now();
   const access = loadAccess();
+  {
+    const fleetPrev = watchState[FLEET_ALL_EXHAUSTED_KEY] ?? emptyAccountState();
+    const fleetDecision = evaluateFleetAllExhausted({
+      accounts: listStateData.accounts,
+      prev: fleetPrev,
+      now
+    });
+    if (fleetDecision.kind === "notify") {
+      for (const chat_id of access.allowFrom) {
+        await swallowingApiCall(() => bot.api.sendMessage(chat_id, fleetDecision.message, {
+          parse_mode: "HTML",
+          link_preview_options: { is_disabled: true }
+        }), { chat_id, verb: "quota-watch.fleet-all-exhausted" });
+      }
+      watchState = patchQuotaWatchState(watchState, FLEET_ALL_EXHAUSTED_KEY, fleetDecision.newState);
+      try {
+        saveQuotaWatchState(stateDir, watchState);
+      } catch (err) {
+        process.stderr.write(`telegram gateway: quota-watch: fleet-state save failed: ${err}
+`);
+      }
+      process.stderr.write(`telegram gateway: quota-watch: fleet all-exhausted ${fleetDecision.transition}
+`);
+    }
+  }
   const pendingTransitions = [];
   const labelToSnapIndex = new Map(snapshots.map((s, i) => [s.label, i]));
   for (const snap of snapshots) {

package/telegram-plugin/gateway/gateway.ts

@@ -412,6 +412,8 @@ import {
 } from '../credits-watch.js'
 import {
   evaluateQuotaWatchAccount,
+  evaluateFleetAllExhausted,
+  FLEET_ALL_EXHAUSTED_KEY,
   loadQuotaWatchState,
   saveQuotaWatchState,
   patchQuotaWatchState,
@@ -14805,6 +14807,44 @@ async function runQuotaWatch(): Promise<void> {
   const now = Date.now()
   const access = loadAccess()
 
+  // Fleet-wide all-exhausted check FIRST — must run before the per-account
+  // early-return below. When every account is exhausted, the per-account loop
+  // produces only 'blocked' skips → pendingTransitions empty → early return;
+  // so this fleet-level alert (the one the trigger-based all-blocked card
+  // misses during quiet periods / for the consumer+cron paths) would never
+  // fire if placed after. Authoritative source: broker `exhausted` flags.
+  {
+    const fleetPrev = watchState[FLEET_ALL_EXHAUSTED_KEY] ?? emptyAccountState()
+    const fleetDecision = evaluateFleetAllExhausted({
+      accounts: listStateData.accounts,
+      prev: fleetPrev,
+      now,
+    })
+    if (fleetDecision.kind === 'notify') {
+      for (const chat_id of access.allowFrom) {
+        await swallowingApiCall(
+          () =>
+            bot.api.sendMessage(chat_id, fleetDecision.message, {
+              parse_mode: 'HTML',
+              link_preview_options: { is_disabled: true },
+            }),
+          { chat_id, verb: 'quota-watch.fleet-all-exhausted' },
+        )
+      }
+      // Persist immediately — the per-account early-return path below would
+      // otherwise drop this flag change (edge-trigger would re-fire next poll).
+      watchState = patchQuotaWatchState(watchState, FLEET_ALL_EXHAUSTED_KEY, fleetDecision.newState)
+      try {
+        saveQuotaWatchState(stateDir, watchState)
+      } catch (err) {
+        process.stderr.write(`telegram gateway: quota-watch: fleet-state save failed: ${err}\n`)
+      }
+      process.stderr.write(
+        `telegram gateway: quota-watch: fleet all-exhausted ${fleetDecision.transition}\n`,
+      )
+    }
+  }
+
   // First pass: evaluate all accounts against cached state. Collect
   // labels that need a live probe (i.e. accounts with a detected transition
   // that we're about to notify about). We probe those to get fresh

package/telegram-plugin/quota-watch.ts

@@ -160,6 +160,99 @@ export function evaluateQuotaWatchAccount(args: {
   return { kind: "skip", accountLabel: label, reason: "no-matching-transition" };
 }
 
+// ─── Fleet-level: all accounts exhausted ───────────────────────────────────────
+
+/**
+ * Reserved key under which the fleet-wide "all accounts exhausted" alert state
+ * is stored in the same quota-watch.json map. Not a valid account label (emails
+ * can't contain this), so it never collides with a per-account entry, and the
+ * per-account loop (which iterates account snapshots, not state-map keys) never
+ * sees it. Encoded as a QuotaWatchAccountState so the existing load validator
+ * accepts it: lastNotifiedHealth "throttling" = currently alerting all-exhausted,
+ * "healthy"/null = not. Backward-compatible — old files simply lack the key.
+ */
+export const FLEET_ALL_EXHAUSTED_KEY = "__fleet_all_exhausted__";
+
+export type FleetAllExhaustedDecision =
+  | { kind: "notify"; message: string; newState: QuotaWatchAccountState; transition: "entered" | "recovered" }
+  | { kind: "skip"; reason: string };
+
+/**
+ * Fleet-wide all-exhausted alert (edge-triggered).
+ *
+ * Fires ONCE when every account enters the broker's exhausted state (no healthy
+ * account to fail over to — agents go quiet, crons defer, consumers/hindsight
+ * silently serve an exhausted account), and ONCE on recovery. This catches the
+ * cases the trigger-based interactive all-blocked card misses: a quiet period
+ * (no agent happens to 429 into the wall) and the consumer/cron paths.
+ *
+ * Authoritative source: the broker's per-account `exhausted` flag (set by
+ * mark-exhausted via failover + the consumer sensor), NOT probe-derived health
+ * — so there is no probe-failure false-alarm. Requires at least one account;
+ * an empty fleet never alerts.
+ */
+export function evaluateFleetAllExhausted(args: {
+  accounts: Array<{ label: string; exhausted: boolean; exhausted_until?: number }>;
+  prev: QuotaWatchAccountState;
+  now: number;
+}): FleetAllExhaustedDecision {
+  const { accounts, prev, now } = args;
+  const allExhausted = accounts.length > 0 && accounts.every((a) => a.exhausted);
+  // "throttling" doubles as the "currently alerting all-exhausted" marker.
+  const wasAlerting = prev.lastNotifiedHealth === "throttling";
+
+  if (allExhausted && !wasAlerting) {
+    return {
+      kind: "notify",
+      message: buildAllExhaustedMessage(accounts, now),
+      newState: { lastNotifiedHealth: "throttling", lastNotifiedAt: now },
+      transition: "entered",
+    };
+  }
+  if (!allExhausted && wasAlerting) {
+    return {
+      kind: "notify",
+      message: buildFleetRecoveredMessage(accounts),
+      newState: { lastNotifiedHealth: "healthy", lastNotifiedAt: now },
+      transition: "recovered",
+    };
+  }
+  return { kind: "skip", reason: allExhausted ? "still-all-exhausted" : "not-all-exhausted" };
+}
+
+function buildAllExhaustedMessage(
+  accounts: Array<{ label: string; exhausted_until?: number }>,
+  now: number,
+): string {
+  const resets = accounts
+    .map((a) => a.exhausted_until)
+    .filter((x): x is number => typeof x === "number" && x > now);
+  const earliest = resets.length > 0 ? Math.min(...resets) : null;
+  const resetLine = earliest
+    ? `Earliest reset: ${formatRelative(new Date(earliest), new Date(now))}.`
+    : `Reset time unknown (no window data).`;
+  return [
+    `🔴 <b>All accounts exhausted</b>`,
+    ``,
+    `Every Anthropic account (${accounts.length}) is quota-walled — there is no healthy account to fail over to.`,
+    resetLine,
+    ``,
+    `<i>This is self-healing: agents resume and deferred scheduled jobs run automatically once a window resets. Nothing is lost. Add headroom with <code>/auth add</code> if this recurs.</i>`,
+  ].join("\n");
+}
+
+function buildFleetRecoveredMessage(
+  accounts: Array<{ label: string; exhausted: boolean }>,
+): string {
+  const healthy = accounts.filter((a) => !a.exhausted).map((a) => a.label);
+  const which = healthy.length > 0 ? ` (<code>${escapeHtml(healthy[0]!)}</code>)` : "";
+  return [
+    `🟢 <b>Fleet recovered</b> — at least one account is healthy again${which}.`,
+    ``,
+    `<i>Agents are back; any deferred scheduled jobs will run on their next occurrence.</i>`,
+  ].join("\n");
+}
+
 // ─── Message builders ─────────────────────────────────────────────────────────
 
 function buildThrottlingMessage(agentName: string, snap: AccountSnapshot): string {

package/telegram-plugin/tests/quota-watch.test.ts

@@ -12,6 +12,7 @@ import { tmpdir } from "os";
 import { join } from "path";
 import {
   evaluateQuotaWatchAccount,
+  evaluateFleetAllExhausted,
   loadQuotaWatchState,
   saveQuotaWatchState,
   patchQuotaWatchState,
@@ -364,3 +365,74 @@ describe("patchQuotaWatchState", () => {
     expect(current["bob@example.com"]).toBeUndefined();
   });
 });
+
+describe("evaluateFleetAllExhausted", () => {
+  const notAlerting = { lastNotifiedHealth: null, lastNotifiedAt: 0 };
+  const alerting = { lastNotifiedHealth: "throttling" as const, lastNotifiedAt: 1000 };
+
+  it("notifies (entered) when every account is exhausted and we weren't alerting", () => {
+    const d = evaluateFleetAllExhausted({
+      accounts: [
+        { label: "a", exhausted: true, exhausted_until: 5_000 },
+        { label: "b", exhausted: true, exhausted_until: 9_000 },
+      ],
+      prev: notAlerting,
+      now: 1_000,
+    });
+    expect(d.kind).toBe("notify");
+    if (d.kind === "notify") {
+      expect(d.transition).toBe("entered");
+      expect(d.newState.lastNotifiedHealth).toBe("throttling");
+      expect(d.message).toContain("All accounts exhausted");
+      // earliest reset is the 5_000 one
+      expect(d.message).toContain("Earliest reset");
+    }
+  });
+
+  it("skips (still) when all exhausted and already alerting — no re-spam", () => {
+    const d = evaluateFleetAllExhausted({
+      accounts: [{ label: "a", exhausted: true }, { label: "b", exhausted: true }],
+      prev: alerting,
+      now: 2_000,
+    });
+    expect(d.kind).toBe("skip");
+  });
+
+  it("notifies (recovered) when one account frees after we were alerting", () => {
+    const d = evaluateFleetAllExhausted({
+      accounts: [{ label: "a", exhausted: false }, { label: "b", exhausted: true }],
+      prev: alerting,
+      now: 3_000,
+    });
+    expect(d.kind).toBe("notify");
+    if (d.kind === "notify") {
+      expect(d.transition).toBe("recovered");
+      expect(d.newState.lastNotifiedHealth).toBe("healthy");
+      expect(d.message).toContain("Fleet recovered");
+      expect(d.message).toContain("a"); // names the healthy account
+    }
+  });
+
+  it("skips (not-all) when some account is healthy and we weren't alerting", () => {
+    const d = evaluateFleetAllExhausted({
+      accounts: [{ label: "a", exhausted: false }, { label: "b", exhausted: true }],
+      prev: notAlerting,
+      now: 4_000,
+    });
+    expect(d.kind).toBe("skip");
+  });
+
+  it("never alerts on an empty fleet", () => {
+    expect(evaluateFleetAllExhausted({ accounts: [], prev: notAlerting, now: 1 }).kind).toBe("skip");
+  });
+
+  it("shows reset-unknown when no exhausted_until is present", () => {
+    const d = evaluateFleetAllExhausted({
+      accounts: [{ label: "a", exhausted: true }],
+      prev: notAlerting,
+      now: 1_000,
+    });
+    expect(d.kind).toBe("notify");
+    if (d.kind === "notify") expect(d.message).toContain("Reset time unknown");
+  });
+});