switchroom 0.10.0 → 0.11.1

package/telegram-plugin/auto-fallback-fleet.ts

@@ -0,0 +1,215 @@
+/**
+ * Fleet-wide auto-fallback (RFC H — successor to the per-agent
+ * `performAutoFallback` in `auto-fallback.ts`).
+ *
+ * Why this exists alongside the legacy per-agent path:
+ *
+ *   The pre-#XYZ auto-fallback called `fallbackToNextSlot(agentDir)`,
+ *   which writes the new active slot to ONE agent's local
+ *   `.claude/credentials.json`. That left the rest of the fleet still
+ *   pointing at the just-exhausted account — which would then hit the
+ *   wall on its own next call, surfacing N separate "Model unavailable"
+ *   cards for the same root cause.
+ *
+ *   Manual `/auth use <label>` already takes the fleet-wide path
+ *   (broker.setActive → fan-out to all per-agent credential mirrors).
+ *   Auto-fallback now uses the same path so scope is consistent and
+ *   one quota event resolves the whole fleet in one swap.
+ *
+ * What this module does:
+ *
+ *   1. Probe live quota for every account in parallel via the
+ *      broker (`client.probeQuota(...)`, #1336) so we pick the best
+ *      target with current data, not stale broker disk-cache.
+ *   2. Skip blocked accounts entirely; pick the lowest-utilization
+ *      healthy candidate (or, if none, the lowest throttling one).
+ *   3. Call `client.setActive(target)` — same broker verb /auth use
+ *      uses. Broker re-mirrors creds to all agents.
+ *   4. Render the causal-shape announcement
+ *      (`renderFallbackAnnouncement`) with the OLD account's binding
+ *      window in the headline (5-hour vs 7-day) and the new
+ *      account's headroom in the body.
+ *
+ * Pure-data return shape — caller does the actual Telegram send +
+ * lockout-record bookkeeping, mirroring the legacy module's contract.
+ */
+
+import type { QuotaResult, QuotaUtilization } from './quota-check.js';
+import type { ListStateData } from '../src/auth/broker/client.js';
+import {
+  renderFallbackAnnouncement,
+  classifyHealth,
+  buildSnapshotsFromState,
+  type AccountSnapshot,
+} from './auth-snapshot-format.js';
+
+export type FleetFallbackOutcome =
+  | {
+      kind: 'switched';
+      oldLabel: string;
+      newLabel: string;
+      announcement: string;
+      /** Quota for the OLD account at the moment of failure — caller
+       *  may persist this as the broker's `quota.json` so the next
+       *  /auth render reflects the freshly-known exhaustion without
+       *  another probe. */
+      oldQuota: QuotaUtilization;
+      /** Quota for the new active account, useful for caller logging. */
+      newQuota: QuotaUtilization;
+    }
+  | {
+      kind: 'all-blocked';
+      oldLabel: string;
+      announcement: string;
+      oldQuota: QuotaUtilization | null;
+    }
+  | {
+      kind: 'no-old-active';
+      announcement: string;
+    }
+  | {
+      kind: 'no-eligible-target';
+      oldLabel: string;
+      announcement: string;
+      oldQuota: QuotaUtilization | null;
+    };
+
+export interface FleetFallbackDeps {
+  /** Live broker state. Caller passes pre-fetched data so this module
+   *  is testable without spinning up a UDS. */
+  state: ListStateData;
+  /** Parallel array of live quota probes, same order as `state.accounts`.
+   *  Get via `client.probeQuota(state.accounts.map(a => a.label))`
+   *  and map the response back to per-account results (#1336). */
+  quotas: QuotaResult[];
+  /** Broker `setActive` invoker. Returns the result for logging. */
+  setActive: (label: string) => Promise<{ active: string; fanned: string[] }>;
+  /** Agent that triggered this fallback (for the announcement byline). */
+  triggerAgent: string;
+  /** Operator timezone for absolute reset times in the announcement. */
+  tz?: string;
+  now?: Date;
+}
+
+/**
+ * Plan + execute the fleet-wide swap. Returns a structured outcome the
+ * caller can both log and notify on.
+ *
+ * Idempotency: when the active account is already healthy (a stale
+ * model-unavailable event arrives after the quota window already
+ * rolled over, for example), we DO NOT swap. Returns
+ * `'no-eligible-target'` so the caller silently no-ops the
+ * announcement.
+ */
+export async function runFleetAutoFallback(
+  deps: FleetFallbackDeps,
+): Promise<FleetFallbackOutcome> {
+  const now = deps.now ?? new Date();
+  const tz = deps.tz ?? 'UTC';
+  const snapshots = buildSnapshotsFromState(deps.state, deps.quotas);
+
+  const oldSnap = snapshots.find((s) => s.isActive);
+  if (!oldSnap) {
+    return {
+      kind: 'no-old-active',
+      announcement: '<i>Auto-fallback skipped: no active account in broker state.</i>',
+    };
+  }
+
+  // Idempotency guard: don't swap a healthy active account, even if
+  // the trigger event said quota_exhausted. The event may be stale
+  // (event posted, window rolled over, gateway picked it up late).
+  const oldHealth = classifyHealth(oldSnap);
+  if (oldHealth === 'healthy') {
+    return {
+      kind: 'no-eligible-target',
+      oldLabel: oldSnap.label,
+      oldQuota: oldSnap.quota,
+      announcement:
+        `<i>Auto-fallback skipped: ${oldSnap.label} probed healthy ` +
+        `(${pctSummary(oldSnap.quota)}). Stale event?</i>`,
+    };
+  }
+
+  const target = pickFallbackTarget(snapshots);
+  if (!target) {
+    // All-blocked path: no eligible target. Still notify the user with
+    // earliest-reset info via the announcement formatter.
+    return {
+      kind: 'all-blocked',
+      oldLabel: oldSnap.label,
+      oldQuota: oldSnap.quota,
+      announcement: renderFallbackAnnouncement({
+        oldLabel: oldSnap.label,
+        oldQuota: oldSnap.quota,
+        newLabel: null,
+        newQuota: null,
+        triggerAgent: deps.triggerAgent,
+        tz,
+        now,
+      }),
+    };
+  }
+
+  // Execute the broker swap. Caller catches and surfaces the failure
+  // — we don't double-wrap.
+  await deps.setActive(target.label);
+
+  return {
+    kind: 'switched',
+    oldLabel: oldSnap.label,
+    newLabel: target.label,
+    oldQuota: oldSnap.quota!, // non-null: only `unknown` health gets here through
+    // the no-target branch, never the switched one
+    newQuota: target.quota!,
+    announcement: renderFallbackAnnouncement({
+      oldLabel: oldSnap.label,
+      oldQuota: oldSnap.quota,
+      newLabel: target.label,
+      newQuota: target.quota,
+      triggerAgent: deps.triggerAgent,
+      tz,
+      now,
+    }),
+  };
+}
+
+/**
+ * Pick the best non-active fallback target. Selection order:
+ *   1. Healthy accounts, sorted by lowest 5h utilization (most
+ *      runway).
+ *   2. If no healthy alternative, throttling accounts sorted by
+ *      lowest binding-window utilization (least worst).
+ *   3. Skip blocked + unknown entirely — never recommend a switch
+ *      into a wall, never bet on creds we couldn't probe.
+ *
+ * Returns null when no eligible target exists.
+ */
+export function pickFallbackTarget(
+  snapshots: AccountSnapshot[],
+): AccountSnapshot | null {
+  const candidates = snapshots
+    .filter((s) => !s.isActive && s.quota != null)
+    .map((s) => ({ snap: s, health: classifyHealth(s) }));
+
+  const healthy = candidates
+    .filter((c) => c.health === 'healthy')
+    .sort((a, b) => a.snap.quota!.fiveHourUtilizationPct - b.snap.quota!.fiveHourUtilizationPct);
+  if (healthy.length > 0) return healthy[0]!.snap;
+
+  const throttling = candidates
+    .filter((c) => c.health === 'throttling')
+    .sort((a, b) => maxWindow(a.snap.quota!) - maxWindow(b.snap.quota!));
+  if (throttling.length > 0) return throttling[0]!.snap;
+
+  return null;
+}
+
+function maxWindow(q: QuotaUtilization): number {
+  return Math.max(q.fiveHourUtilizationPct, q.sevenDayUtilizationPct);
+}
+
+function pctSummary(q: QuotaUtilization | null): string {
+  if (!q) return 'no probe';
+  return `${Math.round(q.fiveHourUtilizationPct)}% / ${Math.round(q.sevenDayUtilizationPct)}%`;
+}

package/telegram-plugin/auto-fallback.ts

@@ -1,252 +1,51 @@
 /**
- * Auto-fallback on quota exhaustion — pure decision logic + side-effect
- * plan builder, separate from gateway.ts so it can be unit-tested without
- * spinning up the bot or the filesystem.
+ * Read-only persistence for the legacy per-agent auto-fallback lockout
+ * file. The lockout writer + decision logic + plan executor were retired
+ * in PR #1329 (fleet-wide auto-fallback path supersedes the per-agent
+ * one); this module's only remaining job is to support
+ * `isAutoFallbackCooldownActive` in gateway.ts, which reads the existing
+ * on-disk lockout to defer pending-restart drains while a recent
+ * rotation is still settling.
  *
- * Runtime flow (assembled by the caller):
- *   1. Poll quota via `fetchQuota` from quota-check.ts
- *   2. Pass the result into `evaluateFallbackTrigger` to decide if we
- *      should act, together with an in-memory lockout record that
- *      prevents rapid re-fire.
- *   3. If the trigger says "fallback", call `performAutoFallback`
- *      which returns a plan + side-effect descriptor the caller
- *      executes (mark exhausted, swap slot, restart agent, notify).
+ * Existing on-disk lockouts (written by pre-#1329 gateways) age out via
+ * `DEFAULT_FALLBACK_COOLDOWN_MS`; new lockouts are never written. Once
+ * every operator has run `switchroom update` post-#1329, the file goes
+ * cold and `isAutoFallbackCooldownActive` always returns false. This
+ * module + the drain-cap consumer can then be retired together in a
+ * follow-up.
  */
 
-import type { QuotaResult, QuotaUtilization } from './quota-check.js';
-import { renderOperatorEvent } from './operator-events.js';
-
-/** Threshold over which we treat the active slot as functionally out
- *  of quota. 99.5% leaves a tiny head-room for clock skew between the
- *  Anthropic rate-limit window and wall clock, matching the dashboard's
- *  own rounding behaviour. Tune with care. */
-export const DEFAULT_TRIGGER_UTILIZATION_PCT = 99.5;
-
 /** Minimum time between two consecutive fallback attempts for the same
- *  slot name, in milliseconds. Guards against a poll-storm firing the
- *  restart-notify pipeline repeatedly before the quota meta file has
- *  a chance to flush to disk. */
+ *  slot — guard against poll-storm fallback loops. Read-only since
+ *  PR #1329; only consumed by `isAutoFallbackCooldownActive` to bound
+ *  the drain-cap defer. */
 export const DEFAULT_FALLBACK_COOLDOWN_MS = 2 * 60_000;
 
 export type LockoutRecord = {
-  /** Slot name most recently marked exhausted by this process. */
+  /** Slot name most recently marked exhausted by the legacy writer. */
   lastTransitionedFrom: string | null;
-  /** Wall-clock ms timestamp of the last transition. */
+  /** Wall-clock ms timestamp of that transition. */
   lastTransitionAt: number;
 };
 
-export type FallbackDecision =
-  | { action: 'noop'; reason: string }
-  | {
-      action: 'fallback';
-      triggerReason: 'utilization-over-threshold' | '429-response' | 'explicit';
-      resetAtMs: number | null;
-      utilizationPct: number | null;
-    };
-
-export type EvaluateArgs = {
-  quota: QuotaResult;
-  activeSlot: string | null;
-  now: number;
-  lockout: LockoutRecord;
-  thresholdPct?: number;
-  cooldownMs?: number;
-  /** Set to true when the caller already saw a 429 response body;
-   *  this short-circuits past utilization-based decisions. */
-  saw429?: boolean;
-};
-
-/** Pure decision function — takes a quota result + lockout state and
- *  returns whether the caller should trigger auto-fallback.
- *  No side effects. Throws only on programmer error. */
-export function evaluateFallbackTrigger(args: EvaluateArgs): FallbackDecision {
-  const threshold = args.thresholdPct ?? DEFAULT_TRIGGER_UTILIZATION_PCT;
-  const cooldown = args.cooldownMs ?? DEFAULT_FALLBACK_COOLDOWN_MS;
-
-  if (!args.activeSlot) {
-    return { action: 'noop', reason: 'no active slot (nothing to fall back from)' };
-  }
-
-  // Cooldown guard: if we already transitioned out of this slot
-  // recently, don't flap. The caller can safely re-poll without
-  // creating noise.
-  if (
-    args.lockout.lastTransitionedFrom === args.activeSlot &&
-    args.now - args.lockout.lastTransitionAt < cooldown
-  ) {
-    return { action: 'noop', reason: 'recent transition, within cooldown' };
-  }
-
-  if (args.saw429) {
-    return {
-      action: 'fallback',
-      triggerReason: '429-response',
-      resetAtMs: extractNearestResetMs(args.quota),
-      utilizationPct: extractHighestUtilization(args.quota),
-    };
-  }
-
-  if (!args.quota.ok) {
-    return { action: 'noop', reason: `quota check failed: ${args.quota.reason}` };
-  }
-
-  const highest = extractHighestUtilization(args.quota);
-  if (highest == null) {
-    return { action: 'noop', reason: 'no utilization headers' };
-  }
-
-  if (highest >= threshold) {
-    return {
-      action: 'fallback',
-      triggerReason: 'utilization-over-threshold',
-      resetAtMs: extractNearestResetMs(args.quota),
-      utilizationPct: highest,
-    };
-  }
-
-  return { action: 'noop', reason: `utilization ${highest.toFixed(1)}% below ${threshold}%` };
-}
-
-function extractHighestUtilization(q: QuotaResult): number | null {
-  if (!q.ok) return null;
-  const u: QuotaUtilization = q.data;
-  const five = u.fiveHourUtilizationPct ?? null;
-  const seven = u.sevenDayUtilizationPct ?? null;
-  if (five == null && seven == null) return null;
-  if (five == null) return seven;
-  if (seven == null) return five;
-  return Math.max(five, seven);
-}
-
-function extractNearestResetMs(q: QuotaResult): number | null {
-  if (!q.ok) return null;
-  const candidates: number[] = [];
-  if (q.data.fiveHourResetAt) candidates.push(q.data.fiveHourResetAt.getTime());
-  if (q.data.sevenDayResetAt) candidates.push(q.data.sevenDayResetAt.getTime());
-  if (candidates.length === 0) return null;
-  return Math.min(...candidates);
-}
-
-/** The full plan built by the orchestrator — mirrored by the
- *  executor in gateway.ts. Pure data so tests can assert on it. */
-export type FallbackPlan =
-  | {
-      kind: 'executed';
-      previousSlot: string;
-      newSlot: string;
-      resetAtMs: number | null;
-      notificationHtml: string;
-      agentName: string;
-      /** Carried through from the FallbackDecision so the executor can
-       *  decide whether to do a hard or graceful restart. Reactive
-       *  (`429-response`) failover wants a hard restart — the request
-       *  the user just made already failed, so there's no in-flight
-       *  turn worth preserving. Preemptive (`utilization-over-threshold`
-       *  / `explicit`) failover wants a graceful one. See #420. */
-      triggerReason: 'utilization-over-threshold' | '429-response' | 'explicit';
-    }
-  | {
-      kind: 'exhausted-all';
-      activeSlot: string;
-      resetAtMs: number | null;
-      notificationHtml: string;
-      agentName: string;
-    };
-
-export type PerformArgs = {
-  agentDir: string;
-  agentName: string;
-  decision: Extract<FallbackDecision, { action: 'fallback' }>;
-  deps: {
-    /** Current active slot; null means caller has already detached. */
-    currentActiveSlot: (agentDir: string) => string | null;
-    markSlotQuotaExhausted: (agentDir: string, slot: string, resetAtMs?: number, reason?: string) => void;
-    fallbackToNextSlot: (name: string, agentDir: string) => { newActive: string | null; previous: string | null };
-  };
-};
-
-/** Run the side-effects for a fallback decision and return a plan
- *  describing what happened. Caller is responsible for:
- *   - Executing the agent restart CLI (via runSwitchroomCommand)
- *   - Sending the notification via Telegram
- *   - Updating the in-memory lockout record (see `nextLockout`)
- */
-export function performAutoFallback(args: PerformArgs): FallbackPlan {
-  const active = args.deps.currentActiveSlot(args.agentDir);
-  if (!active) {
-    return {
-      kind: 'exhausted-all',
-      activeSlot: 'unknown',
-      resetAtMs: args.decision.resetAtMs,
-      notificationHtml: buildAllExhaustedMessage('unknown', args.agentName, args.decision.resetAtMs),
-      agentName: args.agentName,
-    };
-  }
-
-  args.deps.markSlotQuotaExhausted(
-    args.agentDir,
-    active,
-    args.decision.resetAtMs ?? undefined,
-    args.decision.triggerReason,
-  );
-
-  const { newActive, previous } = args.deps.fallbackToNextSlot(args.agentName, args.agentDir);
-  const prev = previous ?? active;
-
-  if (!newActive || newActive === prev) {
-    return {
-      kind: 'exhausted-all',
-      activeSlot: prev,
-      resetAtMs: args.decision.resetAtMs,
-      notificationHtml: buildAllExhaustedMessage(prev, args.agentName, args.decision.resetAtMs),
-      agentName: args.agentName,
-    };
-  }
-
-  return {
-    kind: 'executed',
-    previousSlot: prev,
-    newSlot: newActive,
-    resetAtMs: args.decision.resetAtMs,
-    notificationHtml: buildSwitchedMessage(prev, newActive, args.agentName, args.decision.resetAtMs),
-    agentName: args.agentName,
-    triggerReason: args.decision.triggerReason,
-  };
-}
-
-/** Compute the next lockout record after a successful fallback. */
-export function nextLockout(previousSlot: string, now: number): LockoutRecord {
-  return { lastTransitionedFrom: previousSlot, lastTransitionAt: now };
-}
-
-export function emptyLockout(): LockoutRecord {
-  return { lastTransitionedFrom: null, lastTransitionAt: 0 };
-}
-
-/**
- * Disk-persistence helpers for the lockout record. The cooldown guard
- * lives entirely in process memory pre-fix, so a gateway restart inside
- * the cooldown window resets the timer to zero — and a quota-flap on
- * the now-recovering slot can re-trigger fallback the moment the
- * gateway comes back. See #417.
- *
- * Storage path: \`<agentDir>/.claude/auto-fallback-lockout.json\`. We
- * tolerate any read/parse error by returning emptyLockout (the same
- * outcome as a fresh process), since the cooldown is a noise filter,
- * not a security boundary.
- */
-const LOCKOUT_FILE = "auto-fallback-lockout.json";
-
 export interface LockoutPersistOps {
   readFileSync: (path: string, encoding: BufferEncoding) => string;
+  // writeFileSync + mkdirSync stay in the interface so the gateway's
+  // existing lockoutOps bundle still type-checks. They're never called
+  // by this module any more (the writer was retired).
   writeFileSync: (path: string, data: string, opts: { mode?: number }) => void;
   existsSync: (path: string) => boolean;
   mkdirSync: (path: string, opts: { recursive: true }) => void;
   joinPath: (...parts: string[]) => string;
-  now?: () => number;
 }
 
-export function lockoutPath(agentDir: string, joinPath: LockoutPersistOps['joinPath']): string {
+const LOCKOUT_FILE = "auto-fallback-lockout.json";
+
+function emptyLockout(): LockoutRecord {
+  return { lastTransitionedFrom: null, lastTransitionAt: 0 };
+}
+
+function lockoutPath(agentDir: string, joinPath: LockoutPersistOps['joinPath']): string {
   return joinPath(agentDir, '.claude', LOCKOUT_FILE);
 }
 
@@ -274,75 +73,3 @@ export function loadLockout(agentDir: string, ops: LockoutPersistOps): LockoutRe
   }
   return emptyLockout();
 }
-
-export function saveLockout(
-  agentDir: string,
-  record: LockoutRecord,
-  ops: LockoutPersistOps,
-): void {
-  const path = lockoutPath(agentDir, ops.joinPath);
-  // Best-effort: ensure the .claude directory exists, then write. Any
-  // failure is swallowed by the caller's try/catch — losing the lockout
-  // file just degrades to in-memory-only behaviour, not a hard failure.
-  ops.mkdirSync(ops.joinPath(agentDir, '.claude'), { recursive: true });
-  ops.writeFileSync(
-    path,
-    JSON.stringify(record, null, 2) + '\n',
-    { mode: 0o600 },
-  );
-}
-
-/**
- * Build the notification HTML for a successful slot switch.
- * Delegates to renderOperatorEvent for quota-exhausted; appends
- * slot-transition detail as structured context.
- */
-function buildSwitchedMessage(
-  prev: string,
-  next: string,
-  agent: string,
-  resetAtMs: number | null,
-): string {
-  const reset = resetAtMs ? formatResetAt(resetAtMs) : 'unknown';
-  const detail = [
-    `Switched from slot ${prev} to ${next}. Restarting agent.`,
-    `Reset at: ${reset}.`,
-  ].join(' ');
-  return renderOperatorEvent({
-    kind: 'quota-exhausted',
-    agent,
-    detail,
-    suggestedActions: [],
-    firstSeenAt: new Date(),
-  }).text;
-}
-
-/**
- * Build the notification HTML when all slots are exhausted.
- * Delegates to renderOperatorEvent for quota-exhausted; appends
- * all-exhausted detail.
- */
-function buildAllExhaustedMessage(
-  active: string,
-  agent: string,
-  resetAtMs: number | null,
-): string {
-  const reset = resetAtMs ? formatResetAt(resetAtMs) : 'unknown';
-  const detail = [
-    `All account slots exhausted. Active slot: ${active}.`,
-    `Earliest reset at: ${reset}.`,
-    `Run /auth add ${agent} to attach another subscription.`,
-  ].join(' ');
-  return renderOperatorEvent({
-    kind: 'quota-exhausted',
-    agent,
-    detail,
-    suggestedActions: [],
-    firstSeenAt: new Date(),
-  }).text;
-}
-
-function formatResetAt(ms: number): string {
-  // ISO with seconds trimmed — Telegram doesn't need millisecond precision.
-  return new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
-}