talon-agent 5.24.0 → 5.24.1

package/src/core/daemon/discovery.ts

@@ -17,6 +17,7 @@
  */
 
 import { readPidRecord, isProcessAlive } from "./pidfile.js";
+import { log, logWarn } from "../../util/log.js";
 import {
   gatewayAuthHeaders,
   readGatewayToken,
@@ -174,3 +175,162 @@ export async function findRunningInstance(
 
   return scanForDaemon();
 }
+
+// ── Single-instance guard ───────────────────────────────────────────────────
+
+/**
+ * Single-instance guard — a daemon refuses to boot while another one runs.
+ *
+ * `talon start` already checked for a running instance, but only the CLI
+ * did. The daemon entry wrote its pidfile unconditionally. Anything that
+ * started the entry directly (systemd, a stray `bun src/index.ts`, the
+ * handoff watcher's fallback racing a slow successor) got a second daemon.
+ * On 2026-09-27 that ran for 13 minutes:
+ *   - the newcomer overwrote the pidfile;
+ *   - its gateway and bridge fell back to 19877/19881;
+ *   - both polled Telegram, and every getUpdates answered 409;
+ *   - the newcomer's trigger resume killed the running daemon's watchers
+ *     as "orphans".
+ *
+ * This guard runs first thing in `app.ts`, before any side effect. It
+ * refuses when an identity-checked `/health` answers as a daemon (or a
+ * live pidfile pid turns into one while we wait for it to finish booting).
+ *
+ * The one daemon allowed to overlap is our own predecessor. A `/restart`
+ * successor is spawned in the last moments of the old process's graceful
+ * shutdown, so the guard waits for that pid to exit and then carries on.
+ */
+
+/**
+ * Set by `spawnSuccessor()` on the child it spawns: the pid of the daemon
+ * handing over. The guard waits that process out instead of refusing.
+ */
+export const PREDECESSOR_PID_ENV = "TALON_PREDECESSOR_PID";
+
+/** How long a successor waits for its predecessor to exit. */
+const PREDECESSOR_WAIT_MS = 20_000;
+/** How long a live-but-silent pidfile pid gets to answer /health. */
+const UNVERIFIED_WAIT_MS = 15_000;
+const GUARD_POLL_MS = 250;
+
+export type InstanceCheck =
+  { ok: true } | { ok: false; instance: RunningInstance };
+
+export interface InstanceCheckDeps {
+  find?: (pidfilePath?: string) => Promise<RunningInstance | null>;
+  isAlive?: (pid: number) => boolean;
+  sleep?: (ms: number) => Promise<void>;
+  now?: () => number;
+  env?: NodeJS.ProcessEnv;
+  pidfilePath?: string;
+  predecessorWaitMs?: number;
+  unverifiedWaitMs?: number;
+}
+
+const defaultSleep = (ms: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, ms));
+
+function predecessorPid(env: NodeJS.ProcessEnv): number | undefined {
+  const pid = Number(env[PREDECESSOR_PID_ENV]);
+  return Number.isInteger(pid) && pid > 0 && pid !== process.pid
+    ? pid
+    : undefined;
+}
+
+/** Poll until `done()` or the deadline; resolves whether `done()` held. */
+async function waitFor(
+  done: () => boolean | Promise<boolean>,
+  ms: number,
+  sleep: (ms: number) => Promise<void>,
+  now: () => number,
+): Promise<boolean> {
+  const deadline = now() + ms;
+  for (;;) {
+    if (await done()) return true;
+    if (now() >= deadline) return false;
+    await sleep(GUARD_POLL_MS);
+  }
+}
+
+/**
+ * Decide whether this process may boot as the daemon. Never throws: a
+ * discovery failure answers "ok" (a guard that keeps the only daemon down
+ * is worse than the duplicate it prevents).
+ */
+export async function checkSingleInstance(
+  deps: InstanceCheckDeps = {},
+): Promise<InstanceCheck> {
+  const find = deps.find ?? findRunningInstance;
+  const isAlive = deps.isAlive ?? isProcessAlive;
+  const sleep = deps.sleep ?? defaultSleep;
+  const now = deps.now ?? Date.now;
+  const env = deps.env ?? process.env;
+
+  try {
+    const predecessor = predecessorPid(env);
+    // Consumed: children of this daemon must not inherit it.
+    delete env[PREDECESSOR_PID_ENV];
+    if (predecessor !== undefined && isAlive(predecessor)) {
+      log("bot", `Waiting for predecessor daemon ${predecessor} to exit`);
+      const gone = await waitFor(
+        () => !isAlive(predecessor),
+        deps.predecessorWaitMs ?? PREDECESSOR_WAIT_MS,
+        sleep,
+        now,
+      );
+      if (!gone) {
+        // It is past its own 15s force-exit timer and has already let go of
+        // the frontends. Staying down would leave nothing running.
+        logWarn(
+          "bot",
+          `Predecessor daemon ${predecessor} is still alive — booting anyway`,
+        );
+      }
+    }
+
+    let instance = await find(deps.pidfilePath);
+    if (instance?.source === "pidfile-unverified") {
+      // A live pid with no /health: a daemon still booting, or a recycled
+      // pid. Give it time to answer before deciding.
+      const pid = instance.pid;
+      await waitFor(
+        async () => {
+          if (!isAlive(pid)) return true;
+          instance = await find(deps.pidfilePath);
+          return instance?.source !== "pidfile-unverified";
+        },
+        deps.unverifiedWaitMs ?? UNVERIFIED_WAIT_MS,
+        sleep,
+        now,
+      );
+      if (instance?.source === "pidfile-unverified") {
+        logWarn(
+          "bot",
+          `pidfile names live pid ${instance.pid} but no daemon answers — ` +
+            `treating it as a recycled pid`,
+        );
+        return { ok: true };
+      }
+    }
+    if (!instance) return { ok: true };
+    if (instance.pid === process.pid || instance.pid === predecessor) {
+      return { ok: true };
+    }
+    return { ok: false, instance };
+  } catch (err) {
+    logWarn(
+      "bot",
+      `single-instance check failed (${err instanceof Error ? err.message : String(err)}) — booting`,
+    );
+    return { ok: true };
+  }
+}
+
+/** One line for the log and stderr when the guard refuses. */
+export function describeRefusal(instance: RunningInstance): string {
+  const where = instance.port ? ` on :${instance.port}` : "";
+  return (
+    `another Talon daemon is already running (pid ${instance.pid}${where}). ` +
+    "Refusing to start a second one — use `talon restart` to replace it."
+  );
+}

package/src/core/daemon/pidfile.ts

@@ -96,3 +96,110 @@ export function isProcessAlive(pid: number): boolean {
     return (err as NodeJS.ErrnoException).code === "EPERM";
   }
 }
+
+// ── Child ownership ─────────────────────────────────────────────────────────
+
+/**
+ * Daemon ownership of child processes.
+ *
+ * Every process the daemon spawns (backend CLIs, MCP children, trigger
+ * scripts) inherits its environment. At boot the daemon stamps two
+ * variables into that environment: its own pid and its /proc start time.
+ * Any orphan sweep can then tell a child whose daemon is gone (safe to
+ * reap) from a child of a daemon that is still running.
+ *
+ * The distinction matters because "orphan" used to mean "alive and
+ * tagged with our chat or trigger id". On 2026-09-27 two daemons ran at
+ * once for 13 minutes. Each one's sweeps saw the other's live children as
+ * leftovers from a previous run, and the newcomer's trigger resume
+ * SIGKILLed the running daemon's watchers.
+ *
+ * Linux-only in practice: the reads go through /proc. Where /proc is
+ * absent, {@link childBelongsToLiveDaemon} answers false and the sweeps
+ * behave exactly as before.
+ */
+
+/** Pid of the daemon that spawned this process (inherited env). */
+export const DAEMON_PID_ENV = "TALON_DAEMON_PID";
+/** That daemon's /proc start time, so a recycled pid cannot pass for it. */
+export const DAEMON_STARTTIME_ENV = "TALON_DAEMON_STARTTIME";
+
+/**
+ * Field 22 of /proc/<pid>/stat: start time in jiffies since boot.
+ * Monotonic per boot and unchanged by exec(), so it pins a pid to one
+ * process. `undefined` without /proc or when the read fails.
+ *
+ * The `comm` field (2nd) is wrapped in parens and may itself contain ')'
+ * — split after the LAST ')'; index 19 of the rest is field 22.
+ */
+export function readPidStarttimeSync(pid: number): number | undefined {
+  try {
+    const stat = readFileSync(`/proc/${pid}/stat`, "utf-8");
+    const lastParen = stat.lastIndexOf(")");
+    if (lastParen < 0) return undefined;
+    const tail = stat.slice(lastParen + 2).split(" ");
+    const starttime = Number(tail[19]);
+    return Number.isFinite(starttime) ? starttime : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Stamp this process as the daemon in `env` (default: our own), so every
+ * child spawned from here on names us. Overwrites any inherited stamp: a
+ * `/restart` successor inherits its predecessor's environment.
+ */
+export function stampDaemonOwner(env: NodeJS.ProcessEnv = process.env): void {
+  env[DAEMON_PID_ENV] = String(process.pid);
+  const starttime = readPidStarttimeSync(process.pid);
+  if (starttime !== undefined) env[DAEMON_STARTTIME_ENV] = String(starttime);
+  else delete env[DAEMON_STARTTIME_ENV];
+}
+
+export interface DaemonOwner {
+  pid: number;
+  starttime?: number;
+}
+
+/** The owner stamp carried in a NUL-split `/proc/<pid>/environ`. */
+export function ownerFromEnviron(entries: string[]): DaemonOwner | undefined {
+  const value = (key: string): string | undefined => {
+    const prefix = `${key}=`;
+    return entries.find((e) => e.startsWith(prefix))?.slice(prefix.length);
+  };
+  const pid = Number(value(DAEMON_PID_ENV));
+  if (!Number.isInteger(pid) || pid <= 0) return undefined;
+  const starttime = Number(value(DAEMON_STARTTIME_ENV));
+  return Number.isFinite(starttime) && value(DAEMON_STARTTIME_ENV)
+    ? { pid, starttime }
+    : { pid };
+}
+
+/**
+ * Whether `owner` is a daemon other than this one that is still running.
+ * A matching pid with a different start time is a recycled pid, so the
+ * owner is dead.
+ */
+export function isOtherLiveDaemon(owner: DaemonOwner | undefined): boolean {
+  if (!owner || owner.pid === process.pid) return false;
+  if (!isProcessAlive(owner.pid)) return false;
+  if (owner.starttime === undefined) return true;
+  const current = readPidStarttimeSync(owner.pid);
+  return current === undefined || current === owner.starttime;
+}
+
+/**
+ * Whether process `pid` was spawned by a daemon other than this one that
+ * is still alive. Orphan sweeps must leave such a process alone. Reads
+ * `/proc/<pid>/environ`; false when it can't be read.
+ */
+export function childBelongsToLiveDaemon(pid: number): boolean {
+  let raw: string;
+  try {
+    raw = readFileSync(`/proc/${pid}/environ`, "utf-8");
+  } catch {
+    return false;
+  }
+  return isOtherLiveDaemon(ownerFromEnviron(raw.split("\0")));
+}

package/src/core/daemon/respawn.ts

@@ -55,6 +55,7 @@
 import { spawn } from "node:child_process";
 import { log, logError, openRespawnLog } from "../../util/log.js";
 import { HANDOFF_WATCH_SUBCOMMAND } from "./handoff.js";
+import { PREDECESSOR_PID_ENV } from "./discovery.js";
 
 let pendingReason: string | null = null;
 let shutdown: ((reason: string) => void) | null = null;
@@ -187,7 +188,9 @@ export function spawnSuccessor(spawnFn: SpawnFn = spawn): void {
       cwd: process.cwd(),
       detached: true,
       stdio: ["ignore", fd ?? "ignore", fd ?? "ignore"],
-      env: { ...process.env },
+      // Tells the successor's single-instance guard that the daemon it
+      // can still see is us, on our way out — wait, don't refuse.
+      env: { ...process.env, [PREDECESSOR_PID_ENV]: String(process.pid) },
     });
     child.once("error", (err) => {
       logError("shutdown", `Respawn failed (${reason}) — see respawn.log`, err);

package/src/core/engine/backend-router/breaker.ts

@@ -0,0 +1,158 @@
+/**
+ * Backend circuit breaker — "is this backend actually working right now?"
+ *
+ * Headroom answers "how much plan is left". It says nothing about whether
+ * runs on the backend succeed. In Sep 2026 Codex's login expired and its
+ * default model was retired, so every run on it failed. Codex still had no
+ * usage signal and read as 100% headroom, and the router kept sending
+ * background work there for 36 hours. This module holds the missing
+ * signal, fed by the outcome of every routed run:
+ *
+ *   - an auth failure (401, expired login, "run `… login`") opens the
+ *     breaker at once: the next run is not going to fix a credential;
+ *   - {@link BREAKER_FAILURE_THRESHOLD} consecutive failures of any other
+ *     kind open it too;
+ *   - a success closes it and clears the count.
+ *
+ * An open breaker reads as zero headroom (see `headroom.ts`) until its
+ * cool-off ends. After that the backend is half-open: it can win again,
+ * and one more failure without a success in between re-opens it straight
+ * away with twice the cool-off, capped at {@link BREAKER_MAX_COOLOFF_MS}.
+ *
+ * State lives in memory only. A restart gives every backend a fresh
+ * chance, which is what an operator who just ran `codex login` and
+ * restarted expects.
+ */
+
+import { logWarn } from "../../../util/log.js";
+
+/** Consecutive non-auth failures that open the breaker. */
+export const BREAKER_FAILURE_THRESHOLD = 3;
+/** First cool-off. Each re-trip without a success in between doubles it. */
+export const BREAKER_BASE_COOLOFF_MS = 15 * 60_000;
+/** Ceiling on the cool-off. */
+export const BREAKER_MAX_COOLOFF_MS = 4 * 60 * 60_000;
+
+interface BreakerState {
+  /** Failures since the last success. */
+  consecutive: number;
+  /** Times the breaker has opened since the last success. */
+  trips: number;
+  /** Epoch ms the current cool-off ends; undefined when never opened. */
+  openUntil?: number;
+  /** Why it last opened. */
+  reason?: string;
+}
+
+/** An open breaker, as `headroom.ts` and the router log see it. */
+export interface OpenBreaker {
+  readonly reason: string;
+  /** Epoch ms the cool-off ends. */
+  readonly until: number;
+}
+
+const breakers = new Map<string, BreakerState>();
+
+/**
+ * Credential failures, not model or transport ones. Covers what the
+ * backends actually say: HTTP 401, Codex's "refresh token" / "login
+ * expired" wording, agy's "authentication required", and any "run
+ * `x login`" remedy.
+ */
+const AUTH_FAILURE_RE =
+  /\b401\b|unauthori[sz]ed|authentication (?:required|failed)|not (?:logged|signed) in|log(?:in|ged in) (?:has )?expired|refresh[_ ]token|invalid[_ ](?:api[_ ])?key|run [`'"]?[\w-]+ login/i;
+
+/** Whether an error message describes a credential problem. */
+export function isAuthFailureMessage(message: string): boolean {
+  return AUTH_FAILURE_RE.test(message);
+}
+
+function messageOf(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}
+
+/**
+ * A run the caller cancelled says nothing about the backend. A timeout
+ * does count: a backend that hangs is as unusable as one that errors.
+ */
+function isCallerAbort(err: unknown): boolean {
+  if (err instanceof Error && err.name === "AbortError") return true;
+  return /aborted before the run started/i.test(messageOf(err));
+}
+
+function stateFor(id: string): BreakerState {
+  let state = breakers.get(id);
+  if (!state) {
+    state = { consecutive: 0, trips: 0 };
+    breakers.set(id, state);
+  }
+  return state;
+}
+
+function open(id: string, state: BreakerState, reason: string, now: number) {
+  state.trips += 1;
+  const cooloff = Math.min(
+    BREAKER_BASE_COOLOFF_MS * 2 ** (state.trips - 1),
+    BREAKER_MAX_COOLOFF_MS,
+  );
+  state.openUntil = now + cooloff;
+  state.reason = reason;
+  logWarn(
+    "router",
+    `breaker open for ${id} (${reason}) — no routed work for ` +
+      `${Math.round(cooloff / 60_000)}m`,
+  );
+}
+
+/** Record a failed run on a backend. */
+export function recordBackendRunFailure(
+  id: string,
+  err: unknown,
+  now = Date.now(),
+): void {
+  if (isCallerAbort(err)) return;
+  const state = stateFor(id);
+  state.consecutive += 1;
+  const message = messageOf(err).split("\n")[0]?.trim().slice(0, 160) ?? "";
+  // Already open: a failure in the cool-off (a pinned run, say) must not
+  // stretch it.
+  if (state.openUntil !== undefined && now < state.openUntil) return;
+  if (isAuthFailureMessage(message)) {
+    open(id, state, `auth failure: ${message}`, now);
+    return;
+  }
+  // Half-open (tripped before, no success since): the probe failed, so
+  // re-open at once rather than waiting for a fresh run of failures.
+  if (state.trips > 0) {
+    open(id, state, `still failing after cool-off: ${message}`, now);
+    return;
+  }
+  if (state.consecutive >= BREAKER_FAILURE_THRESHOLD) {
+    open(
+      id,
+      state,
+      `${state.consecutive} consecutive failures: ${message}`,
+      now,
+    );
+  }
+}
+
+/** Record a successful run: the backend works, forget its failures. */
+export function recordBackendRunSuccess(id: string): void {
+  breakers.delete(id);
+}
+
+/** The breaker, when it is open at `now`; undefined when the backend may run. */
+export function openBreaker(
+  id: string,
+  now = Date.now(),
+): OpenBreaker | undefined {
+  const state = breakers.get(id);
+  if (!state?.openUntil || now >= state.openUntil) return undefined;
+  return { reason: state.reason ?? "failing", until: state.openUntil };
+}
+
+/** Test seam — close every breaker. */
+export function resetBackendBreakersForTest(): void {
+  breakers.clear();
+}

package/src/core/engine/backend-router/headroom.ts

@@ -11,11 +11,21 @@
  *                the operator's soft budget (`config.backendBudgets`). The
  *                fallback for backends with no account API.
  *
- * A backend with neither reports `source: "none"` and headroom 1. That is a
- * deliberate "no evidence of pressure", not a claim of capacity — the
- * router's comparator ranks it *below* any backend with real telemetry at
- * the same headroom, so an unmeasured backend never outranks a measured one
- * it is tied with.
+ * A backend with neither reports `source: "none"` and headroom 0: nothing
+ * is known about it, so it is never *preferred*. It still clears the
+ * ceiling (there is no window to be over), so it runs work when nothing
+ * measured is available. It used to read as 1. That made a Codex install
+ * with an expired login, and so no usage signal, the router's favourite
+ * for 36 hours while every run on it failed.
+ *
+ * On top of either source sit two "this backend is not working" signals.
+ * Either one zeroes the headroom and pins the limiting window at 100%, so
+ * the ceiling drops the backend whenever anything else is left:
+ *
+ *   - the backend's own telemetry reports a rejected credential
+ *     (`UsageTelemetry.getAuthFailure`, e.g. the Codex usage endpoint's 401);
+ *   - the run breaker is open (`breaker.ts`): an auth failure or repeated
+ *     failures on runs routed there.
  *
  * Reads are cached for 60s per backend: `/usage`, the router and the
  * `plan_usage` tool all ask, and a plan lookup can be a subprocess spawn. A
@@ -31,6 +41,7 @@ import {
   listAvailableBackends,
 } from "../backend-controller/index.js";
 import { ledgerUsage } from "./ledger.js";
+import { openBreaker } from "./breaker.js";
 
 /** How long a headroom reading is reused before the source is asked again. */
 export const HEADROOM_CACHE_MS = 60_000;
@@ -63,6 +74,11 @@ export interface BackendHeadroom {
    * same (cached) fetch the router ranked on, instead of asking twice.
    */
   readonly plan?: PlanUsage;
+  /**
+   * Why the backend is treated as unusable right now (a rejected login, an
+   * open breaker). Set means headroom 0 and a 100% limiting window.
+   */
+  readonly unavailable?: string;
 }
 
 interface CacheEntry {
@@ -183,13 +199,29 @@ export function headroomFromLedger(
   };
 }
 
-/** The "nothing to measure" reading. Headroom 1, but lowest ranking source. */
+/** The "nothing to measure" reading: headroom 0, but under any ceiling. */
 function unknownHeadroom(
   id: string,
   label: string,
   now: number,
 ): BackendHeadroom {
-  return { id, label, headroom: 1, source: "none", fetchedAt: now };
+  return { id, label, headroom: 0, source: "none", fetchedAt: now };
+}
+
+/** What a backend's telemetry said: its plan windows and any auth failure. */
+interface PlanRead {
+  usage?: PlanUsage;
+  authFailure?: string;
+}
+
+function authFailureOf(
+  usage: { getAuthFailure?(): string | undefined } | undefined,
+): string | undefined {
+  try {
+    return usage?.getAuthFailure?.call(usage) || undefined;
+  } catch {
+    return undefined;
+  }
 }
 
 /**
@@ -202,22 +234,35 @@ function unknownHeadroom(
  * is *not* currently in use. The read is the backend's own cached one, so at
  * most one boot per cache window.
  */
-async function readPlanUsage(id: string): Promise<PlanUsage | undefined> {
+async function readPlanUsage(id: string): Promise<PlanRead> {
   const pooled = getPooledBackend(id);
-  if (pooled?.usage?.getPlanUsage) {
-    return pooled.usage.getPlanUsage.call(pooled.usage);
+  if (pooled) {
+    const usage = pooled.usage?.getPlanUsage
+      ? await pooled.usage.getPlanUsage.call(pooled.usage)
+      : undefined; // pooled, but reports no plan windows
+    // Read after the plan fetch: that fetch is what discovers a 401.
+    const authFailure = authFailureOf(pooled.usage);
+    return {
+      ...(usage ? { usage } : {}),
+      ...(authFailure ? { authFailure } : {}),
+    };
   }
-  if (pooled) return undefined; // pooled, but reports no plan windows
   let acquired;
   try {
     acquired = await acquireBackendInstance(id);
   } catch {
-    return undefined; // can't boot it (not configured, no auth) — stay quiet
+    return {}; // can't boot it (not configured, no auth) — stay quiet
   }
   try {
-    const read = acquired.backend.usage?.getPlanUsage;
-    if (!read || !acquired.backend.usage) return undefined;
-    return await read.call(acquired.backend.usage);
+    const telemetry = acquired.backend.usage;
+    const usage = telemetry?.getPlanUsage
+      ? await telemetry.getPlanUsage.call(telemetry)
+      : undefined;
+    const authFailure = authFailureOf(telemetry);
+    return {
+      ...(usage ? { usage } : {}),
+      ...(authFailure ? { authFailure } : {}),
+    };
   } finally {
     await acquired.release().catch(() => {});
   }
@@ -238,16 +283,18 @@ export async function getBackendHeadroom(
   const now = options?.now ?? Date.now();
   const cached = cache.get(id);
   if (!options?.force && cached && now - cached.cachedAt < HEADROOM_CACHE_MS) {
-    return cached.value;
+    return withBreaker(cached.value, now);
   }
 
   let value: BackendHeadroom;
   try {
-    const plan = headroomFromPlan(id, label, await readPlanUsage(id));
+    const read = await readPlanUsage(id);
+    const plan = headroomFromPlan(id, label, read.usage);
     value =
       plan ??
       headroomFromLedger(id, label, config, now) ??
       unknownHeadroom(id, label, now);
+    if (read.authFailure) value = unavailable(value, read.authFailure);
   } catch {
     // The source is unreachable this minute. Keeping the last good reading
     // is the conservative answer: forgetting it would read as "empty" and
@@ -257,7 +304,29 @@ export async function getBackendHeadroom(
       : unknownHeadroom(id, label, now);
   }
   cache.set(id, { value, cachedAt: now });
-  return value;
+  return withBreaker(value, now);
+}
+
+/** Mark a reading unusable: zero headroom, limiting window pinned at 100%. */
+function unavailable(value: BackendHeadroom, why: string): BackendHeadroom {
+  return {
+    ...value,
+    headroom: 0,
+    limiting: { label: why, percent: 100 },
+    unavailable: why,
+  };
+}
+
+/**
+ * Overlay the run breaker. Applied on every read rather than cached: the
+ * breaker opens and closes on run outcomes, not on the headroom clock.
+ */
+function withBreaker(value: BackendHeadroom, now: number): BackendHeadroom {
+  if (value.unavailable) return value;
+  const breaker = openBreaker(value.id, now);
+  if (!breaker) return value;
+  const mins = Math.max(1, Math.ceil((breaker.until - now) / 60_000));
+  return unavailable(value, `breaker open ${mins}m — ${breaker.reason}`);
 }
 
 /** Headroom for every backend the config exposes, in config order. */
@@ -275,11 +344,10 @@ export async function collectBackendHeadroom(
 
 /** One-line rendering shared by `/usage`, `plan_usage` and the router log. */
 export function formatHeadroom(entry: BackendHeadroom): string {
+  if (entry.unavailable) return `0% — unavailable: ${entry.unavailable}`;
+  if (entry.source === "none") return "unmeasured — no usage signal";
   const pct = `${Math.round(entry.headroom * 100)}%`;
-  const detail =
-    entry.source === "none"
-      ? "no usage signal"
-      : `${entry.limiting?.label ?? "window"} ${Math.round(entry.limiting?.percent ?? 0)}% used`;
+  const detail = `${entry.limiting?.label ?? "window"} ${Math.round(entry.limiting?.percent ?? 0)}% used`;
   const tag = entry.source === "ledger" ? " (local budget)" : "";
   const stale = entry.stale ? " (stale)" : "";
   return `${pct} — ${detail}${tag}${stale}`;

package/src/core/engine/backend-router/index.ts

@@ -20,6 +20,17 @@ export {
   LEDGER_RETENTION_MS,
   LEDGER_SHORT_WINDOW_MS,
 } from "./ledger.js";
+export {
+  isAuthFailureMessage,
+  openBreaker,
+  recordBackendRunFailure,
+  recordBackendRunSuccess,
+  resetBackendBreakersForTest,
+  BREAKER_BASE_COOLOFF_MS,
+  BREAKER_FAILURE_THRESHOLD,
+  BREAKER_MAX_COOLOFF_MS,
+  type OpenBreaker,
+} from "./breaker.js";
 export {
   collectBackendHeadroom,
   formatHeadroom,

package/src/core/engine/gateway-actions/cron.ts

@@ -133,6 +133,9 @@ export const cronHandlers: SharedActionHandlers = {
         : null,
       endAt !== undefined ? `ends: ${new Date(endAt).toISOString()}` : null,
       spec.catchup ? `catch-up: ${spec.catchup}` : null,
+      spec.timeoutMs !== undefined
+        ? `timeout: ${Math.round(spec.timeoutMs / 1000)}s`
+        : null,
     ]
       .filter(Boolean)
       .join(", ");
@@ -173,6 +176,8 @@ export const cronHandlers: SharedActionHandlers = {
       if (j.catchup && j.catchup !== "skip")
         bounds.push(`catch-up: ${j.catchup}`);
       if (j.model) bounds.push(`model: ${j.model}`);
+      if (j.timeoutMs !== undefined)
+        bounds.push(`timeout: ${Math.round(j.timeoutMs / 1000)}s`);
       return [
         `- ${j.name} (${status})`,
         `  ID: ${j.id}`,