switchroom 0.14.82 → 0.14.83

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14863,6 +14863,39 @@ async function parseSseOrJson(resp) {
14863
14863
  const payload = dataLine ? dataLine.slice("data: ".length) : text;
14864
14864
  return JSON.parse(payload);
14865
14865
  }
14866
+ async function fetchHindsightToolsList(apiUrl, opts) {
14867
+ const fetchImpl = opts?.fetchImpl ?? fetch;
14868
+ const timeoutMs = opts?.timeoutMs ?? 4000;
14869
+ const bankId = opts?.bankId ?? "__doctor_probe__";
14870
+ const controller = new AbortController;
14871
+ const timeout = setTimeout(() => controller.abort(), timeoutMs);
14872
+ try {
14873
+ const resp = await fetchImpl(`${apiUrl}`, {
14874
+ method: "POST",
14875
+ headers: {
14876
+ "Content-Type": "application/json",
14877
+ Accept: "application/json, text/event-stream",
14878
+ "X-Bank-Id": bankId
14879
+ },
14880
+ body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }),
14881
+ signal: controller.signal
14882
+ });
14883
+ clearTimeout(timeout);
14884
+ if (!resp.ok)
14885
+ return { ok: false, reason: `HTTP ${resp.status}` };
14886
+ const parsed = await parseSseOrJson(resp);
14887
+ const raw = parsed.result?.tools;
14888
+ if (!Array.isArray(raw))
14889
+ return { ok: false, reason: "no tools in tools/list response" };
14890
+ const tools = raw.filter((t) => typeof t?.name === "string").map((t) => ({ name: t.name, required: t.inputSchema?.required ?? [] }));
14891
+ return { ok: true, tools };
14892
+ } catch (err) {
14893
+ clearTimeout(timeout);
14894
+ if (err.name === "AbortError")
14895
+ return { ok: false, reason: "Timeout" };
14896
+ return { ok: false, reason: String(err.message ?? err) };
14897
+ }
14898
+ }
14866
14899
  async function probeHindsight(apiUrl, opts) {
14867
14900
  const fetchImpl = opts?.fetchImpl ?? fetch;
14868
14901
  const timeoutMs = opts?.timeoutMs ?? 3000;
@@ -14991,8 +15024,7 @@ async function ensureUserProfileMentalModel(apiUrl, bankId, opts) {
14991
15024
  name: "create_mental_model",
14992
15025
  arguments: {
14993
15026
  name: "user-profile",
14994
- source_query: "What are the key facts, preferences, context, and communication style about the user I talk to? Summarize what matters for making the agent feel like it knows them.",
14995
- types: ["world", "experience"]
15027
+ source_query: "What are the key facts, preferences, context, and communication style about the user I talk to? Summarize what matters for making the agent feel like it knows them."
14996
15028
  }
14997
15029
  }
14998
15030
  }),
@@ -15077,6 +15109,12 @@ async function createBank(apiUrl, bankId, opts) {
15077
15109
  if (!toolResponse.ok) {
15078
15110
  return { ok: false, reason: `Tool call HTTP ${toolResponse.status}` };
15079
15111
  }
15112
+ try {
15113
+ const created = await parseSseOrJson(toolResponse);
15114
+ if (created.result?.isError === true) {
15115
+ return { ok: false, reason: created.result.content?.[0]?.text ?? "create_bank returned isError" };
15116
+ }
15117
+ } catch {}
15080
15118
  return { ok: true };
15081
15119
  } catch (err) {
15082
15120
  if (err.name === "AbortError") {
@@ -15137,8 +15175,8 @@ async function updateBankMissions(apiUrl, bankId, missions, opts) {
15137
15175
  name: "update_bank",
15138
15176
  arguments: {
15139
15177
  bank_id: bankId,
15140
- mission: missions.bank_mission,
15141
- retain_mission: missions.retain_mission
15178
+ ...missions.bank_mission != null ? { mission: missions.bank_mission } : {},
15179
+ ...missions.retain_mission != null ? { config_updates: { retain_mission: missions.retain_mission } } : {}
15142
15180
  }
15143
15181
  }
15144
15182
  }),
@@ -15148,6 +15186,12 @@ async function updateBankMissions(apiUrl, bankId, missions, opts) {
15148
15186
  if (!toolResponse.ok) {
15149
15187
  return { ok: false, reason: `Tool call HTTP ${toolResponse.status}` };
15150
15188
  }
15189
+ try {
15190
+ const updated = await parseSseOrJson(toolResponse);
15191
+ if (updated.result?.isError === true) {
15192
+ return { ok: false, reason: updated.result.content?.[0]?.text ?? "update_bank returned isError" };
15193
+ }
15194
+ } catch {}
15151
15195
  return { ok: true };
15152
15196
  } catch (err) {
15153
15197
  if (err.name === "AbortError") {
@@ -28959,6 +29003,30 @@ var init_manifest = __esm(() => {
28959
29003
  ]);
28960
29004
  });
28961
29005
 
29006
+ // src/memory/hindsight-tools.ts
29007
+ var EXPECTED_HINDSIGHT_TOOLS;
29008
+ var init_hindsight_tools = __esm(() => {
29009
+ EXPECTED_HINDSIGHT_TOOLS = {
29010
+ recall: { required: ["query"] },
29011
+ reflect: { required: ["query"] },
29012
+ retain: { required: ["content"] },
29013
+ sync_retain: { required: ["content"] },
29014
+ delete_document: { required: ["document_id"] },
29015
+ create_directive: { required: ["content", "name"] },
29016
+ list_directives: { required: [] },
29017
+ delete_directive: { required: ["directive_id"] },
29018
+ create_bank: { required: ["bank_id"] },
29019
+ update_bank: { required: [] },
29020
+ list_banks: { required: [] },
29021
+ create_mental_model: { required: ["name", "source_query"] },
29022
+ list_mental_models: { required: [] },
29023
+ update_mental_model: { required: ["mental_model_id"] },
29024
+ refresh_mental_model: { required: ["mental_model_id"] },
29025
+ list_memories: { required: [] },
29026
+ get_memory: { required: ["memory_id"] }
29027
+ };
29028
+ });
29029
+
28962
29030
  // src/cli/doctor-memory.ts
28963
29031
  import { execFileSync as execFileSync17 } from "node:child_process";
28964
29032
  function classifyShmSize(bytes) {
@@ -29030,8 +29098,51 @@ function checkHindsightContainerHealth(opts) {
29030
29098
  } catch {}
29031
29099
  return results;
29032
29100
  }
29101
+ function classifyToolContract(advertised) {
29102
+ const byName = new Map(advertised.map((t) => [t.name, t]));
29103
+ const results = [];
29104
+ for (const [tool, spec] of Object.entries(EXPECTED_HINDSIGHT_TOOLS)) {
29105
+ const real = byName.get(tool);
29106
+ if (real === undefined) {
29107
+ results.push({
29108
+ name: `hindsight contract: ${tool}`,
29109
+ status: "fail",
29110
+ detail: `switchroom calls \`${tool}\` but the server no longer advertises it ` + `(renamed/removed upstream) \u2014 every callsite silently no-ops`,
29111
+ fix: "Upstream hindsight changed its MCP tool contract. Update the callsite " + "+ EXPECTED_HINDSIGHT_TOOLS (src/memory/hindsight-tools.ts) to the new " + "name, refresh tests/fixtures/hindsight-tools-list.snapshot.json, or pin " + "the prior hindsight image."
29112
+ });
29113
+ continue;
29114
+ }
29115
+ const missing = spec.required.filter((arg) => !real.required.includes(arg));
29116
+ const added = real.required.filter((arg) => !spec.required.includes(arg));
29117
+ if (added.length > 0) {
29118
+ results.push({
29119
+ name: `hindsight contract: ${tool}`,
29120
+ status: "fail",
29121
+ detail: `server now requires [${added.join(", ")}] on \`${tool}\` which ` + `switchroom does not track \u2014 calls may silently no-op`,
29122
+ fix: "Reconcile EXPECTED_HINDSIGHT_TOOLS + the callsite args with the new " + "server schema, then refresh the snapshot fixture."
29123
+ });
29124
+ } else if (missing.length > 0) {
29125
+ results.push({
29126
+ name: `hindsight contract: ${tool}`,
29127
+ status: "warn",
29128
+ detail: `switchroom treats [${missing.join(", ")}] as required on \`${tool}\` ` + `but the server no longer does (loosened upstream) \u2014 harmless, but the ` + `fixture is stale`,
29129
+ fix: "Refresh EXPECTED_HINDSIGHT_TOOLS + the snapshot fixture."
29130
+ });
29131
+ }
29132
+ }
29133
+ if (results.length === 0) {
29134
+ const used = Object.keys(EXPECTED_HINDSIGHT_TOOLS).length;
29135
+ results.push({
29136
+ name: "hindsight contract",
29137
+ status: "ok",
29138
+ detail: `${used} used tools present, required args satisfied (${advertised.length} advertised)`
29139
+ });
29140
+ }
29141
+ return results;
29142
+ }
29033
29143
  var MIN_HINDSIGHT_SHM_BYTES;
29034
29144
  var init_doctor_memory = __esm(() => {
29145
+ init_hindsight_tools();
29035
29146
  MIN_HINDSIGHT_SHM_BYTES = 1024 * 1024 * 1024;
29036
29147
  });
29037
29148
 
@@ -32043,6 +32154,10 @@ async function checkHindsight(config) {
32043
32154
  status: "ok",
32044
32155
  detail: `${probe2.serverName} ${probe2.serverVersion} at ${host}:${port}`
32045
32156
  });
32157
+ const toolsList = await fetchHindsightToolsList(url);
32158
+ if (toolsList.ok) {
32159
+ results.push(...classifyToolContract(toolsList.tools));
32160
+ }
32046
32161
  results.push(checkHindsightConsumer(config));
32047
32162
  results.push(...checkHindsightContainerHealth());
32048
32163
  for (const [agentName, agentConfig] of Object.entries(config.agents)) {
@@ -49700,8 +49815,8 @@ var {
49700
49815
  } = import__.default;
49701
49816
 
49702
49817
  // src/build-info.ts
49703
- var VERSION = "0.14.82";
49704
- var COMMIT_SHA = "91bc41d1";
49818
+ var VERSION = "0.14.83";
49819
+ var COMMIT_SHA = "057ab099";
49705
49820
 
49706
49821
  // src/cli/agent.ts
49707
49822
  init_source();
@@ -559,16 +559,42 @@
559
559
  }
560
560
  }
561
561
 
562
+ // Guards a second click from starting a second device-code flow (each
563
+ // start makes Microsoft send a sign-in email → the "2 emails" bug).
564
+ let msConnecting = false;
565
+
566
+ // The set of Microsoft account emails currently known to the broker.
567
+ // Used as a resilience baseline: the connect status lives only in the web
568
+ // process's memory, so if that process restarts mid-connect the status
569
+ // reads 'unknown' even when the token WAS stored. Diffing this list tells
570
+ // us the account really connected regardless of the lost status.
571
+ async function fetchMicrosoftAccountEmails() {
572
+ try {
573
+ const r = await fetch(`${API}/api/microsoft-accounts`, { headers: authHeaders() });
574
+ if (!r.ok) return new Set();
575
+ const list = await r.json();
576
+ return new Set((list || []).filter(a => a.brokerKnown).map(a => String(a.account).toLowerCase()));
577
+ } catch { return new Set(); }
578
+ }
579
+
562
580
  // Start an in-browser Microsoft connect: show the device code + link,
563
581
  // then poll until the operator completes sign-in on Microsoft's site.
564
582
  async function connectMicrosoft() {
583
+ if (msConnecting) return; // double-submit guard
584
+ msConnecting = true;
585
+ const btn = document.getElementById('ms-connect-btn');
586
+ if (btn) btn.disabled = true;
587
+ const done = () => { msConnecting = false; const b = document.getElementById('ms-connect-btn'); if (b) b.disabled = false; };
565
588
  const card = document.getElementById('ms-connect-card');
566
589
  const show = (html) => { if (card) card.innerHTML = html; };
567
590
  show('<div class="loading" style="padding:.8rem">Starting…</div>');
591
+ // Snapshot already-connected accounts BEFORE starting, for the
592
+ // restart-resilient terminal check below.
593
+ const before = await fetchMicrosoftAccountEmails();
568
594
  try {
569
595
  const res = await fetch(`${API}/api/connections/microsoft/connect`, { method: 'POST', headers: authHeaders() });
570
596
  const data = await res.json();
571
- if (!res.ok || !data.ok) { show(''); showError(data.error || `HTTP ${res.status}`); return; }
597
+ if (!res.ok || !data.ok) { show(''); showError(data.error || `HTTP ${res.status}`); done(); return; }
572
598
  const url = data.verificationUri, code = data.userCode;
573
599
  show(`<div class="account-card" style="border-color:var(--accent)">
574
600
  <div class="account-card-header"><div class="account-label">Connect a Microsoft account</div></div>
@@ -580,27 +606,58 @@
580
606
  <div id="ms-connect-status" style="color:var(--text-dim);margin-top:.3rem">Waiting for sign-in… (this card expires in ~15 min)</div>
581
607
  </div>`);
582
608
  const statusEl = () => document.getElementById('ms-connect-status');
583
- const started = Date.now();
609
+ const deadline = Date.now() + ((data.expiresInSec || 900) * 1000 + 30000);
610
+ const showConnected = (label) => {
611
+ show(`<div class="loading" style="padding:.8rem;color:var(--green)">✓ Connected ${escapeHtml(label)}. Use the access toggles below to grant an agent.</div>`);
612
+ fetchConnections();
613
+ };
614
+ // On any non-'connected' terminal state ('failed' or 'unknown'),
615
+ // re-check the broker's actual account list before declaring failure:
616
+ // a new account appearing means the connect really succeeded (e.g. the
617
+ // status was lost to a web restart). Only error if nothing new landed.
618
+ // Limitations (acceptable — the in-memory 'connected' state is the
619
+ // primary signal; this is only the lost-status fallback): a concurrent
620
+ // connect of a DIFFERENT account in another tab/CLI could be mistaken
621
+ // for this one; and re-connecting an ALREADY-known account (token
622
+ // refresh) shows no new account so falls through to the error.
623
+ const settleNonConnected = async (reason) => {
624
+ const after = await fetchMicrosoftAccountEmails();
625
+ const fresh = [...after].find(a => !before.has(a));
626
+ if (fresh) { showConnected(fresh); } else { show(''); showError(reason || 'connect failed'); }
627
+ done();
628
+ };
584
629
  const poll = async () => {
585
- const sres = await fetch(`${API}/api/connections/microsoft/connect/${encodeURIComponent(data.requestId)}`, { headers: authHeaders() });
586
- const s = sres.ok ? await sres.json() : { state: 'failed', reason: `HTTP ${sres.status}` };
630
+ let s;
631
+ try {
632
+ const sres = await fetch(`${API}/api/connections/microsoft/connect/${encodeURIComponent(data.requestId)}`, { headers: authHeaders() });
633
+ s = sres.ok ? await sres.json() : { state: 'failed', reason: `HTTP ${sres.status}` };
634
+ } catch {
635
+ // Transient fetch failure — most likely the web process restarting
636
+ // mid-connect (the exact case this flow must survive). Don't die
637
+ // (that would strand msConnecting=true and lock the button): keep
638
+ // polling until the device-code deadline, then settle via the
639
+ // broker re-check (which recovers a token stored before the restart).
640
+ if (Date.now() > deadline) { await settleNonConnected('connection lost'); return; }
641
+ setTimeout(poll, 3000);
642
+ return;
643
+ }
587
644
  if (s.state === 'pending') {
588
- if (Date.now() - started > ((data.expiresInSec || 900) * 1000 + 30000)) { const e = statusEl(); if (e) e.textContent = 'Expired — click Connect to try again.'; return; }
645
+ if (Date.now() > deadline) { const e = statusEl(); if (e) e.textContent = 'Expired — click Connect to try again.'; done(); return; }
589
646
  setTimeout(poll, 3000);
590
647
  return;
591
648
  }
592
649
  if (s.state === 'connected') {
593
- show(`<div class="loading" style="padding:.8rem;color:var(--green)">✓ Connected ${escapeHtml(s.account)} (${escapeHtml(s.accountType)}). Use the access toggles below to grant an agent.</div>`);
594
- fetchConnections();
650
+ showConnected(`${s.account} (${s.accountType})`);
651
+ done();
595
652
  } else {
596
- show('');
597
- showError(s.reason || 'connect failed');
653
+ await settleNonConnected(s.reason);
598
654
  }
599
655
  };
600
656
  setTimeout(poll, 3000);
601
657
  } catch (err) {
602
658
  show('');
603
659
  showError(err.message);
660
+ done();
604
661
  }
605
662
  }
606
663
 
@@ -1165,7 +1222,7 @@
1165
1222
  <div style="margin-bottom:1.5rem">
1166
1223
  <h3 style="margin:0 0 .6rem;font-size:.95rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:.04em">
1167
1224
  Microsoft 365
1168
- <button onclick="connectMicrosoft()" class="usage-pill primary" style="margin-left:.6rem;cursor:pointer;border:none;text-transform:none;font-weight:600">+ Connect a Microsoft account</button>
1225
+ <button id="ms-connect-btn" onclick="connectMicrosoft()" class="usage-pill primary" style="margin-left:.6rem;cursor:pointer;border:none;text-transform:none;font-weight:600">+ Connect a Microsoft account</button>
1169
1226
  </h3>
1170
1227
  <div id="ms-connect-card"></div>
1171
1228
  ${msCards
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "switchroom",
3
- "version": "0.14.82",
3
+ "version": "0.14.83",
4
4
  "description": "Run Claude Code 24/7 on your Claude Pro/Max subscription over Telegram. Open-source alternative to OpenClaw and NanoClaw — no API keys.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -52661,6 +52661,53 @@ function evaluateQuotaWatchAccount(args) {
52661
52661
  }
52662
52662
  return { kind: "skip", accountLabel: label, reason: "no-matching-transition" };
52663
52663
  }
52664
+ var FLEET_ALL_EXHAUSTED_KEY = "__fleet_all_exhausted__";
52665
+ function evaluateFleetAllExhausted(args) {
52666
+ const { accounts, prev, now } = args;
52667
+ const allExhausted = accounts.length > 0 && accounts.every((a) => a.exhausted);
52668
+ const wasAlerting = prev.lastNotifiedHealth === "throttling";
52669
+ if (allExhausted && !wasAlerting) {
52670
+ return {
52671
+ kind: "notify",
52672
+ message: buildAllExhaustedMessage(accounts, now),
52673
+ newState: { lastNotifiedHealth: "throttling", lastNotifiedAt: now },
52674
+ transition: "entered"
52675
+ };
52676
+ }
52677
+ if (!allExhausted && wasAlerting) {
52678
+ return {
52679
+ kind: "notify",
52680
+ message: buildFleetRecoveredMessage(accounts),
52681
+ newState: { lastNotifiedHealth: "healthy", lastNotifiedAt: now },
52682
+ transition: "recovered"
52683
+ };
52684
+ }
52685
+ return { kind: "skip", reason: allExhausted ? "still-all-exhausted" : "not-all-exhausted" };
52686
+ }
52687
+ function buildAllExhaustedMessage(accounts, now) {
52688
+ const resets = accounts.map((a) => a.exhausted_until).filter((x) => typeof x === "number" && x > now);
52689
+ const earliest = resets.length > 0 ? Math.min(...resets) : null;
52690
+ const resetLine = earliest ? `Earliest reset: ${formatRelative(new Date(earliest), new Date(now))}.` : `Reset time unknown (no window data).`;
52691
+ return [
52692
+ `\uD83D\uDD34 <b>All accounts exhausted</b>`,
52693
+ ``,
52694
+ `Every Anthropic account (${accounts.length}) is quota-walled \u2014 there is no healthy account to fail over to.`,
52695
+ resetLine,
52696
+ ``,
52697
+ `<i>This is self-healing: agents resume and deferred scheduled jobs run automatically once a window resets. Nothing is lost. Add headroom with <code>/auth add</code> if this recurs.</i>`
52698
+ ].join(`
52699
+ `);
52700
+ }
52701
+ function buildFleetRecoveredMessage(accounts) {
52702
+ const healthy = accounts.filter((a) => !a.exhausted).map((a) => a.label);
52703
+ const which = healthy.length > 0 ? ` (<code>${escapeHtml10(healthy[0])}</code>)` : "";
52704
+ return [
52705
+ `\uD83D\uDFE2 <b>Fleet recovered</b> \u2014 at least one account is healthy again${which}.`,
52706
+ ``,
52707
+ `<i>Agents are back; any deferred scheduled jobs will run on their next occurrence.</i>`
52708
+ ].join(`
52709
+ `);
52710
+ }
52664
52711
  function buildThrottlingMessage(agentName3, snap) {
52665
52712
  const q = snap.quota;
52666
52713
  const fiveStr = fmtPct(q.fiveHourUtilizationPct);
@@ -52810,11 +52857,11 @@ function sweepStaleTurnActiveMarker(stateDir, opts) {
52810
52857
  }
52811
52858
 
52812
52859
  // ../src/build-info.ts
52813
- var VERSION = "0.14.82";
52814
- var COMMIT_SHA = "91bc41d1";
52815
- var COMMIT_DATE = "2026-06-07T12:22:49+10:00";
52860
+ var VERSION = "0.14.83";
52861
+ var COMMIT_SHA = "057ab099";
52862
+ var COMMIT_DATE = "2026-06-07T12:57:08+10:00";
52816
52863
  var LATEST_PR = null;
52817
- var COMMITS_AHEAD_OF_TAG = 2;
52864
+ var COMMITS_AHEAD_OF_TAG = 4;
52818
52865
 
52819
52866
  // gateway/boot-version.ts
52820
52867
  function formatRelativeAgo(iso) {
@@ -61017,6 +61064,31 @@ async function runQuotaWatch() {
61017
61064
  let watchState = loadQuotaWatchState(stateDir);
61018
61065
  const now = Date.now();
61019
61066
  const access = loadAccess();
61067
+ {
61068
+ const fleetPrev = watchState[FLEET_ALL_EXHAUSTED_KEY] ?? emptyAccountState();
61069
+ const fleetDecision = evaluateFleetAllExhausted({
61070
+ accounts: listStateData.accounts,
61071
+ prev: fleetPrev,
61072
+ now
61073
+ });
61074
+ if (fleetDecision.kind === "notify") {
61075
+ for (const chat_id of access.allowFrom) {
61076
+ await swallowingApiCall(() => bot.api.sendMessage(chat_id, fleetDecision.message, {
61077
+ parse_mode: "HTML",
61078
+ link_preview_options: { is_disabled: true }
61079
+ }), { chat_id, verb: "quota-watch.fleet-all-exhausted" });
61080
+ }
61081
+ watchState = patchQuotaWatchState(watchState, FLEET_ALL_EXHAUSTED_KEY, fleetDecision.newState);
61082
+ try {
61083
+ saveQuotaWatchState(stateDir, watchState);
61084
+ } catch (err) {
61085
+ process.stderr.write(`telegram gateway: quota-watch: fleet-state save failed: ${err}
61086
+ `);
61087
+ }
61088
+ process.stderr.write(`telegram gateway: quota-watch: fleet all-exhausted ${fleetDecision.transition}
61089
+ `);
61090
+ }
61091
+ }
61020
61092
  const pendingTransitions = [];
61021
61093
  const labelToSnapIndex = new Map(snapshots.map((s, i) => [s.label, i]));
61022
61094
  for (const snap of snapshots) {
@@ -412,6 +412,8 @@ import {
412
412
  } from '../credits-watch.js'
413
413
  import {
414
414
  evaluateQuotaWatchAccount,
415
+ evaluateFleetAllExhausted,
416
+ FLEET_ALL_EXHAUSTED_KEY,
415
417
  loadQuotaWatchState,
416
418
  saveQuotaWatchState,
417
419
  patchQuotaWatchState,
@@ -14805,6 +14807,44 @@ async function runQuotaWatch(): Promise<void> {
14805
14807
  const now = Date.now()
14806
14808
  const access = loadAccess()
14807
14809
 
14810
+ // Fleet-wide all-exhausted check FIRST — must run before the per-account
14811
+ // early-return below. When every account is exhausted, the per-account loop
14812
+ // produces only 'blocked' skips → pendingTransitions empty → early return;
14813
+ // so this fleet-level alert (the one the trigger-based all-blocked card
14814
+ // misses during quiet periods / for the consumer+cron paths) would never
14815
+ // fire if placed after. Authoritative source: broker `exhausted` flags.
14816
+ {
14817
+ const fleetPrev = watchState[FLEET_ALL_EXHAUSTED_KEY] ?? emptyAccountState()
14818
+ const fleetDecision = evaluateFleetAllExhausted({
14819
+ accounts: listStateData.accounts,
14820
+ prev: fleetPrev,
14821
+ now,
14822
+ })
14823
+ if (fleetDecision.kind === 'notify') {
14824
+ for (const chat_id of access.allowFrom) {
14825
+ await swallowingApiCall(
14826
+ () =>
14827
+ bot.api.sendMessage(chat_id, fleetDecision.message, {
14828
+ parse_mode: 'HTML',
14829
+ link_preview_options: { is_disabled: true },
14830
+ }),
14831
+ { chat_id, verb: 'quota-watch.fleet-all-exhausted' },
14832
+ )
14833
+ }
14834
+ // Persist immediately — the per-account early-return path below would
14835
+ // otherwise drop this flag change (edge-trigger would re-fire next poll).
14836
+ watchState = patchQuotaWatchState(watchState, FLEET_ALL_EXHAUSTED_KEY, fleetDecision.newState)
14837
+ try {
14838
+ saveQuotaWatchState(stateDir, watchState)
14839
+ } catch (err) {
14840
+ process.stderr.write(`telegram gateway: quota-watch: fleet-state save failed: ${err}\n`)
14841
+ }
14842
+ process.stderr.write(
14843
+ `telegram gateway: quota-watch: fleet all-exhausted ${fleetDecision.transition}\n`,
14844
+ )
14845
+ }
14846
+ }
14847
+
14808
14848
  // First pass: evaluate all accounts against cached state. Collect
14809
14849
  // labels that need a live probe (i.e. accounts with a detected transition
14810
14850
  // that we're about to notify about). We probe those to get fresh
@@ -160,6 +160,99 @@ export function evaluateQuotaWatchAccount(args: {
160
160
  return { kind: "skip", accountLabel: label, reason: "no-matching-transition" };
161
161
  }
162
162
 
163
+ // ─── Fleet-level: all accounts exhausted ───────────────────────────────────────
164
+
165
+ /**
166
+ * Reserved key under which the fleet-wide "all accounts exhausted" alert state
167
+ * is stored in the same quota-watch.json map. Not a valid account label (emails
168
+ * can't contain this), so it never collides with a per-account entry, and the
169
+ * per-account loop (which iterates account snapshots, not state-map keys) never
170
+ * sees it. Encoded as a QuotaWatchAccountState so the existing load validator
171
+ * accepts it: lastNotifiedHealth "throttling" = currently alerting all-exhausted,
172
+ * "healthy"/null = not. Backward-compatible — old files simply lack the key.
173
+ */
174
+ export const FLEET_ALL_EXHAUSTED_KEY = "__fleet_all_exhausted__";
175
+
176
+ export type FleetAllExhaustedDecision =
177
+ | { kind: "notify"; message: string; newState: QuotaWatchAccountState; transition: "entered" | "recovered" }
178
+ | { kind: "skip"; reason: string };
179
+
180
+ /**
181
+ * Fleet-wide all-exhausted alert (edge-triggered).
182
+ *
183
+ * Fires ONCE when every account enters the broker's exhausted state (no healthy
184
+ * account to fail over to — agents go quiet, crons defer, consumers/hindsight
185
+ * silently serve an exhausted account), and ONCE on recovery. This catches the
186
+ * cases the trigger-based interactive all-blocked card misses: a quiet period
187
+ * (no agent happens to 429 into the wall) and the consumer/cron paths.
188
+ *
189
+ * Authoritative source: the broker's per-account `exhausted` flag (set by
190
+ * mark-exhausted via failover + the consumer sensor), NOT probe-derived health
191
+ * — so there is no probe-failure false-alarm. Requires at least one account;
192
+ * an empty fleet never alerts.
193
+ */
194
+ export function evaluateFleetAllExhausted(args: {
195
+ accounts: Array<{ label: string; exhausted: boolean; exhausted_until?: number }>;
196
+ prev: QuotaWatchAccountState;
197
+ now: number;
198
+ }): FleetAllExhaustedDecision {
199
+ const { accounts, prev, now } = args;
200
+ const allExhausted = accounts.length > 0 && accounts.every((a) => a.exhausted);
201
+ // "throttling" doubles as the "currently alerting all-exhausted" marker.
202
+ const wasAlerting = prev.lastNotifiedHealth === "throttling";
203
+
204
+ if (allExhausted && !wasAlerting) {
205
+ return {
206
+ kind: "notify",
207
+ message: buildAllExhaustedMessage(accounts, now),
208
+ newState: { lastNotifiedHealth: "throttling", lastNotifiedAt: now },
209
+ transition: "entered",
210
+ };
211
+ }
212
+ if (!allExhausted && wasAlerting) {
213
+ return {
214
+ kind: "notify",
215
+ message: buildFleetRecoveredMessage(accounts),
216
+ newState: { lastNotifiedHealth: "healthy", lastNotifiedAt: now },
217
+ transition: "recovered",
218
+ };
219
+ }
220
+ return { kind: "skip", reason: allExhausted ? "still-all-exhausted" : "not-all-exhausted" };
221
+ }
222
+
223
+ function buildAllExhaustedMessage(
224
+ accounts: Array<{ label: string; exhausted_until?: number }>,
225
+ now: number,
226
+ ): string {
227
+ const resets = accounts
228
+ .map((a) => a.exhausted_until)
229
+ .filter((x): x is number => typeof x === "number" && x > now);
230
+ const earliest = resets.length > 0 ? Math.min(...resets) : null;
231
+ const resetLine = earliest
232
+ ? `Earliest reset: ${formatRelative(new Date(earliest), new Date(now))}.`
233
+ : `Reset time unknown (no window data).`;
234
+ return [
235
+ `🔴 <b>All accounts exhausted</b>`,
236
+ ``,
237
+ `Every Anthropic account (${accounts.length}) is quota-walled — there is no healthy account to fail over to.`,
238
+ resetLine,
239
+ ``,
240
+ `<i>This is self-healing: agents resume and deferred scheduled jobs run automatically once a window resets. Nothing is lost. Add headroom with <code>/auth add</code> if this recurs.</i>`,
241
+ ].join("\n");
242
+ }
243
+
244
+ function buildFleetRecoveredMessage(
245
+ accounts: Array<{ label: string; exhausted: boolean }>,
246
+ ): string {
247
+ const healthy = accounts.filter((a) => !a.exhausted).map((a) => a.label);
248
+ const which = healthy.length > 0 ? ` (<code>${escapeHtml(healthy[0]!)}</code>)` : "";
249
+ return [
250
+ `🟢 <b>Fleet recovered</b> — at least one account is healthy again${which}.`,
251
+ ``,
252
+ `<i>Agents are back; any deferred scheduled jobs will run on their next occurrence.</i>`,
253
+ ].join("\n");
254
+ }
255
+
163
256
  // ─── Message builders ─────────────────────────────────────────────────────────
164
257
 
165
258
  function buildThrottlingMessage(agentName: string, snap: AccountSnapshot): string {
@@ -12,6 +12,7 @@ import { tmpdir } from "os";
12
12
  import { join } from "path";
13
13
  import {
14
14
  evaluateQuotaWatchAccount,
15
+ evaluateFleetAllExhausted,
15
16
  loadQuotaWatchState,
16
17
  saveQuotaWatchState,
17
18
  patchQuotaWatchState,
@@ -364,3 +365,74 @@ describe("patchQuotaWatchState", () => {
364
365
  expect(current["bob@example.com"]).toBeUndefined();
365
366
  });
366
367
  });
368
+
369
+ describe("evaluateFleetAllExhausted", () => {
370
+ const notAlerting = { lastNotifiedHealth: null, lastNotifiedAt: 0 };
371
+ const alerting = { lastNotifiedHealth: "throttling" as const, lastNotifiedAt: 1000 };
372
+
373
+ it("notifies (entered) when every account is exhausted and we weren't alerting", () => {
374
+ const d = evaluateFleetAllExhausted({
375
+ accounts: [
376
+ { label: "a", exhausted: true, exhausted_until: 5_000 },
377
+ { label: "b", exhausted: true, exhausted_until: 9_000 },
378
+ ],
379
+ prev: notAlerting,
380
+ now: 1_000,
381
+ });
382
+ expect(d.kind).toBe("notify");
383
+ if (d.kind === "notify") {
384
+ expect(d.transition).toBe("entered");
385
+ expect(d.newState.lastNotifiedHealth).toBe("throttling");
386
+ expect(d.message).toContain("All accounts exhausted");
387
+ // earliest reset is the 5_000 one
388
+ expect(d.message).toContain("Earliest reset");
389
+ }
390
+ });
391
+
392
+ it("skips (still) when all exhausted and already alerting — no re-spam", () => {
393
+ const d = evaluateFleetAllExhausted({
394
+ accounts: [{ label: "a", exhausted: true }, { label: "b", exhausted: true }],
395
+ prev: alerting,
396
+ now: 2_000,
397
+ });
398
+ expect(d.kind).toBe("skip");
399
+ });
400
+
401
+ it("notifies (recovered) when one account frees after we were alerting", () => {
402
+ const d = evaluateFleetAllExhausted({
403
+ accounts: [{ label: "a", exhausted: false }, { label: "b", exhausted: true }],
404
+ prev: alerting,
405
+ now: 3_000,
406
+ });
407
+ expect(d.kind).toBe("notify");
408
+ if (d.kind === "notify") {
409
+ expect(d.transition).toBe("recovered");
410
+ expect(d.newState.lastNotifiedHealth).toBe("healthy");
411
+ expect(d.message).toContain("Fleet recovered");
412
+ expect(d.message).toContain("a"); // names the healthy account
413
+ }
414
+ });
415
+
416
+ it("skips (not-all) when some account is healthy and we weren't alerting", () => {
417
+ const d = evaluateFleetAllExhausted({
418
+ accounts: [{ label: "a", exhausted: false }, { label: "b", exhausted: true }],
419
+ prev: notAlerting,
420
+ now: 4_000,
421
+ });
422
+ expect(d.kind).toBe("skip");
423
+ });
424
+
425
+ it("never alerts on an empty fleet", () => {
426
+ expect(evaluateFleetAllExhausted({ accounts: [], prev: notAlerting, now: 1 }).kind).toBe("skip");
427
+ });
428
+
429
+ it("shows reset-unknown when no exhausted_until is present", () => {
430
+ const d = evaluateFleetAllExhausted({
431
+ accounts: [{ label: "a", exhausted: true }],
432
+ prev: notAlerting,
433
+ now: 1_000,
434
+ });
435
+ expect(d.kind).toBe("notify");
436
+ if (d.kind === "notify") expect(d.message).toContain("Reset time unknown");
437
+ });
438
+ });