talon-agent 5.4.0 → 5.5.0

package/src/core/daemon/respawn.ts

@@ -13,9 +13,9 @@
  * systemd, foreman, pm2, or running under a debugger. Respawning from
  * our own `process.argv` works regardless of launch method.
  *
- * Ordering matters. `respawnSelf()` only *arms* the handoff and raises
- * SIGTERM; the successor is spawned by `spawnSuccessor()` at the tail of
- * graceful shutdown, once the frontends have stopped. Spawning up-front
+ * Ordering matters. `respawnSelf()` only *arms* the handoff and enters
+ * graceful shutdown; the successor is spawned by `spawnSuccessor()` at
+ * the tail of it, once the frontends have stopped. Spawning up-front
  * (the original behaviour) left the successor long-polling `getUpdates`
  * while the outgoing process was still draining in-flight queries — up
  * to DRAIN_TIMEOUT_MS of two live pollers. Telegram answers only one of
@@ -40,6 +40,16 @@
  *     within a bounded window, and starts the daemon the way `talon
  *     start` does if the successor never comes up. Nothing in the
  *     handoff depends on a process that is about to call process.exit().
+ *
+ * And one thing 2026-09-20 added: the shutdown is entered directly,
+ * through the function app.ts registers with `setRespawnShutdown()`,
+ * not by sending ourselves a SIGTERM. The signal round trip bought
+ * nothing, and under Bun it lost the whole restart: by the time
+ * `/restart` ran, the process had silently lost its OS-level SIGTERM
+ * handler (see ./signals.ts), so the signal terminated it on the spot —
+ * no shutdown, no successor, no pidfile cleanup, and nothing in any log
+ * after "Respawn requested". Only a process that never registered — an
+ * embedder, a test — still falls back to the signal.
  */
 
 import { spawn } from "node:child_process";
@@ -47,6 +57,18 @@ import { log, logError, openRespawnLog } from "../../util/log.js";
 import { HANDOFF_WATCH_SUBCOMMAND } from "./handoff.js";
 
 let pendingReason: string | null = null;
+let shutdown: ((reason: string) => void) | null = null;
+
+/**
+ * Register the graceful-shutdown entry a respawn should run. app.ts
+ * hands over `gracefulShutdown`; `respawnSelf()` then calls it in-process
+ * instead of round-tripping a SIGTERM through the OS. `null` clears it.
+ */
+export function setRespawnShutdown(
+  fn: ((reason: string) => void) | null,
+): void {
+  shutdown = fn;
+}
 
 /**
  * Argv flag that makes the daemon entry resolve its whole import graph
@@ -84,9 +106,8 @@ function selfInvocation(extra: string[]): { cmd: string; args: string[] } {
 }
 
 /**
- * Arm a respawn and raise SIGTERM on ourselves so the existing
- * graceful-shutdown path cleanly stops the frontends, flushes state,
- * and hands off via `spawnSuccessor()`.
+ * Arm a respawn and enter graceful shutdown, which stops the frontends,
+ * flushes state, and hands off via `spawnSuccessor()`.
  *
  * `reason` is logged for operator visibility (e.g. "telegram
  * /restart"). The function returns immediately; the successor starts
@@ -95,10 +116,13 @@ function selfInvocation(extra: string[]): { cmd: string; args: string[] } {
 export function respawnSelf(reason: string): void {
   log("shutdown", `Respawn requested (${reason})`);
   pendingReason = reason;
-  // SIGTERM triggers the graceful-shutdown handler in src/app.ts,
-  // which stops the frontends, flushes state, spawns the successor,
-  // and calls process.exit(0). Don't exit here directly — that would
-  // skip the flush and leave the PID file dangling.
+  // Don't exit here directly — that would skip the flush and leave the
+  // PID file dangling. The registered shutdown is the same path a
+  // SIGTERM takes, entered without the signal.
+  if (shutdown) {
+    shutdown(reason);
+    return;
+  }
   process.kill(process.pid, "SIGTERM");
 }
 

package/src/core/daemon/signals.ts

@@ -0,0 +1,111 @@
+/**
+ * Keep OS-level signal handlers armed under Bun.
+ *
+ * Bun (observed on 1.3.9) uninstalls the process-wide handler for a
+ * signal as soon as ANY listener for it is removed — even while other
+ * listeners remain:
+ *
+ *   process.on("SIGTERM", a);
+ *   process.on("SIGTERM", b);
+ *   process.removeListener("SIGTERM", b);
+ *   // `a` is still registered in JS, yet SigCgt in /proc/<pid>/status
+ *   // no longer lists SIGTERM: the next one terminates the process
+ *   // with the default action, running no JS at all.
+ *
+ * Node keeps the handler until the last listener goes. Adding any
+ * listener makes Bun install the handler again, which is what this
+ * guard relies on.
+ *
+ * Talon hit it through `write-file-atomic`: each call loads signal-exit,
+ * which puts its own listeners on the termination signals, and unloads
+ * it afterwards, removing them. The daemon writes its pidfile that way
+ * while booting — so from that moment every SIGTERM it sent itself
+ * (`/restart` and `/update`, before respawn.ts entered the shutdown
+ * directly) or received from outside (`talon stop`'s fallback, systemd,
+ * docker) killed it outright: no graceful shutdown, no successor, no
+ * pidfile cleanup. Every `/update` between 2026-09-18 (the move to Bun)
+ * and 2026-09-20 ended this way.
+ *
+ * The guard wraps `process.removeListener` / `process.off`. After a
+ * removal that still leaves real listeners on a signal, it re-adds a
+ * no-op sentinel listener (removing any earlier copy first), so the
+ * handler is installed again before the caller gets control back. Once
+ * the last real listener is gone the sentinel goes too, and the signal
+ * falls back to its default action exactly as it would on Node — a
+ * lingering no-op listener would otherwise make the process ignore it.
+ *
+ * Installed by the entry shim (src/index.ts) so every Talon process —
+ * daemon, MCP supervisor, handoff watcher — is covered before any
+ * library gets a chance to remove a listener. A no-op on other runtimes.
+ */
+
+import { isBunRuntime } from "../../util/runtime.js";
+
+type Listener = (...args: unknown[]) => void;
+
+/** The slice of `process` the guard touches; tests pass an EventEmitter. */
+export interface SignalEmitter {
+  on(event: string | symbol, listener: Listener): unknown;
+  removeListener(event: string | symbol, listener: Listener): unknown;
+  off?(event: string | symbol, listener: Listener): unknown;
+  listeners(event: string | symbol): readonly unknown[];
+}
+
+export interface SignalGuardOptions {
+  /** Install even off Bun (tests exercise the wrapper on Node). */
+  force?: boolean;
+}
+
+const SIGNAL_EVENT = /^SIG[A-Z0-9]+$/;
+
+const INSTALLED = Symbol.for("talon.signalListenerGuard");
+
+type Guarded = SignalEmitter & { [INSTALLED]?: () => void };
+
+/** The sentinel. Its only job is to exist, so Bun keeps the handler. */
+function talonSignalRearm(): void {}
+
+/**
+ * Wrap the emitter's listener removal so a signal never silently loses
+ * its OS handler. Idempotent: a second install returns the first one's
+ * uninstall. Production never uninstalls; tests do.
+ */
+export function installSignalListenerGuard(
+  target: SignalEmitter = process,
+  opts: SignalGuardOptions = {},
+): () => void {
+  const emitter = target as Guarded;
+  if (!opts.force && !isBunRuntime()) return () => {};
+  const existing = emitter[INSTALLED];
+  if (existing) return existing;
+
+  const originalRemove = emitter.removeListener;
+  const originalOff = emitter.off;
+
+  const guardedRemove = function (
+    this: Guarded,
+    event: string | symbol,
+    listener: Listener,
+  ): unknown {
+    const result = originalRemove.call(this, event, listener);
+    if (typeof event !== "string" || !SIGNAL_EVENT.test(event)) return result;
+    // Bun has just dropped the handler. What is left decides whether the
+    // signal should still be handled at all.
+    const remaining = this.listeners(event).filter(
+      (fn) => fn !== talonSignalRearm,
+    );
+    originalRemove.call(this, event, talonSignalRearm);
+    if (remaining.length > 0) this.on(event, talonSignalRearm);
+    return result;
+  };
+
+  emitter.removeListener = guardedRemove;
+  if (originalOff) emitter.off = guardedRemove;
+  const uninstall = (): void => {
+    emitter.removeListener = originalRemove;
+    if (originalOff) emitter.off = originalOff;
+    delete emitter[INSTALLED];
+  };
+  emitter[INSTALLED] = uninstall;
+  return uninstall;
+}

package/src/core/engine/backend-controller/index.ts

@@ -38,6 +38,7 @@ export {
   listAvailableBackends,
   isBackendAvailable,
   getAvailableBackends,
+  getPoolConfig,
   getPooledBackend,
   acquireBackendInstance,
   isModelValidForBackend,

package/src/core/engine/backend-controller/pool.ts

@@ -152,6 +152,17 @@ export function getAvailableBackends(): { id: string; label: string }[] {
   return listAvailableBackends(ctx.poolConfig ?? undefined);
 }
 
+/**
+ * The config the pool was initialised with, or `null` before init.
+ *
+ * For runtime readers that need more of the config than the backend list —
+ * the plan-aware router reads `router` and `backendBudgets` from here rather
+ * than having config threaded through every background call site.
+ */
+export function getPoolConfig(): TalonConfig | null {
+  return ctx.poolConfig;
+}
+
 /**
  * The live pooled `Backend` instance for an id, or `null` if it isn't currently
  * pooled. Unlike `getBackendForRole`/`getBackendForChat` this never initialises

package/src/core/engine/backend-router/headroom.ts

@@ -0,0 +1,267 @@
+/**
+ * Headroom — "how much of this backend is left?", answered the same way for
+ * every backend regardless of what it can tell us about itself.
+ *
+ * Two sources, in precedence order:
+ *
+ *   - `plan`   — the backend's own subscription windows
+ *                (`UsageTelemetry.getPlanUsage`). Authoritative: it is the
+ *                provider's own count, including spend from outside Talon.
+ *   - `ledger` — Talon's local rolling token count (see `ledger.ts`) against
+ *                the operator's soft budget (`config.backendBudgets`). The
+ *                fallback for backends with no account API.
+ *
+ * A backend with neither reports `source: "none"` and headroom 1. That is a
+ * deliberate "no evidence of pressure", not a claim of capacity — the
+ * router's comparator ranks it *below* any backend with real telemetry at
+ * the same headroom, so an unmeasured backend never outranks a measured one
+ * it is tied with.
+ *
+ * Reads are cached for 60s per backend: `/usage`, the router and the
+ * `plan_usage` tool all ask, and a plan lookup can be a subprocess spawn. A
+ * failed refresh keeps the last good value and flags it `stale` rather than
+ * pretending the backend emptied.
+ */
+
+import type { PlanUsage } from "../../agent-runtime/capabilities.js";
+import type { TalonConfig } from "../../config/index.js";
+import {
+  getPooledBackend,
+  listAvailableBackends,
+} from "../backend-controller/index.js";
+import { ledgerUsage } from "./ledger.js";
+
+/** How long a headroom reading is reused before the source is asked again. */
+export const HEADROOM_CACHE_MS = 60_000;
+
+/** Where a headroom figure came from. Also its ranking priority. */
+export type HeadroomSource = "plan" | "ledger" | "none";
+
+/** The window that is closest to its limit — what the ceiling is judged on. */
+export interface LimitingWindow {
+  readonly label: string;
+  /** 0-100. */
+  readonly percent: number;
+  readonly resetsAt?: string;
+}
+
+export interface BackendHeadroom {
+  readonly id: string;
+  readonly label: string;
+  /** 0..1 — 1 is empty, 0 is at the limit. */
+  readonly headroom: number;
+  readonly limiting?: LimitingWindow;
+  readonly source: HeadroomSource;
+  /** Epoch ms of the underlying read. */
+  readonly fetchedAt: number;
+  /** True when the last refresh failed and this is the previous value. */
+  readonly stale?: boolean;
+  /**
+   * The raw plan reading this came from, when `source` is `"plan"`. Carried
+   * so `/usage` and the `plan_usage` tool render the real windows off the
+   * same (cached) fetch the router ranked on, instead of asking twice.
+   */
+  readonly plan?: PlanUsage;
+}
+
+interface CacheEntry {
+  value: BackendHeadroom;
+  /** Epoch ms the value was computed (not the same as a stale `fetchedAt`). */
+  cachedAt: number;
+}
+
+const cache = new Map<string, CacheEntry>();
+
+function clampPercent(percent: number): number {
+  if (!Number.isFinite(percent)) return 0;
+  return Math.max(0, Math.min(100, percent));
+}
+
+/** headroom = 1 − (tightest window) / 100. */
+function headroomFor(percent: number): number {
+  return Math.max(0, Math.min(1, 1 - clampPercent(percent) / 100));
+}
+
+/**
+ * The tightest of a plan's windows. `undefined` when the plan reports none,
+ * which is how a backend that answers but has nothing to say is told apart
+ * from one that never answered.
+ */
+export function limitingWindowOf(
+  usage: PlanUsage | undefined,
+): LimitingWindow | undefined {
+  if (!usage || usage.windows.length === 0) return undefined;
+  let worst = usage.windows[0] as NonNullable<(typeof usage.windows)[0]>;
+  for (const window of usage.windows) {
+    if (window.percent > worst.percent) worst = window;
+  }
+  return {
+    label: worst.label,
+    percent: clampPercent(worst.percent),
+    ...(worst.resetsAt ? { resetsAt: worst.resetsAt } : {}),
+  };
+}
+
+/** Headroom from a `PlanUsage`, or `undefined` when it carries no windows. */
+export function headroomFromPlan(
+  id: string,
+  label: string,
+  usage: PlanUsage | undefined,
+): BackendHeadroom | undefined {
+  const limiting = limitingWindowOf(usage);
+  if (!limiting || !usage) return undefined;
+  return {
+    id,
+    label,
+    headroom: headroomFor(limiting.percent),
+    limiting,
+    source: "plan",
+    fetchedAt: usage.fetchedAt,
+    plan: usage,
+  };
+}
+
+/** The soft budget an operator declared for a backend, if any. */
+function budgetFor(
+  config: TalonConfig | undefined,
+  id: string,
+): { tokensPer5h?: number; tokensPerDay?: number } | undefined {
+  const budget = config?.backendBudgets?.[id];
+  if (!budget) return undefined;
+  if (budget.tokensPer5h === undefined && budget.tokensPerDay === undefined) {
+    return undefined;
+  }
+  return budget;
+}
+
+/** Whether the operator gave this backend a local budget to measure against. */
+export function hasBudget(
+  config: TalonConfig | undefined,
+  id: string,
+): boolean {
+  return budgetFor(config, id) !== undefined;
+}
+
+/**
+ * Headroom from the local ledger. The tighter of the two configured windows
+ * wins, so a backend that is fine on the day but has just burned its 5h
+ * allowance still reads as full.
+ */
+export function headroomFromLedger(
+  id: string,
+  label: string,
+  config: TalonConfig | undefined,
+  now = Date.now(),
+): BackendHeadroom | undefined {
+  const budget = budgetFor(config, id);
+  if (!budget) return undefined;
+  const used = ledgerUsage(id, now);
+  const windows: LimitingWindow[] = [];
+  if (budget.tokensPer5h !== undefined) {
+    windows.push({
+      label: "5h (local budget)",
+      percent: clampPercent((used.tokens5h / budget.tokensPer5h) * 100),
+    });
+  }
+  if (budget.tokensPerDay !== undefined) {
+    windows.push({
+      label: "24h (local budget)",
+      percent: clampPercent((used.tokensDay / budget.tokensPerDay) * 100),
+    });
+  }
+  let worst = windows[0] as LimitingWindow;
+  for (const window of windows)
+    if (window.percent > worst.percent) worst = window;
+  return {
+    id,
+    label,
+    headroom: headroomFor(worst.percent),
+    limiting: worst,
+    source: "ledger",
+    fetchedAt: now,
+  };
+}
+
+/** The "nothing to measure" reading. Headroom 1, but lowest ranking source. */
+function unknownHeadroom(
+  id: string,
+  label: string,
+  now: number,
+): BackendHeadroom {
+  return { id, label, headroom: 1, source: "none", fetchedAt: now };
+}
+
+/** Ask a pooled backend for its plan windows. Rejects like the backend does. */
+async function readPlanUsage(id: string): Promise<PlanUsage | undefined> {
+  const backend = getPooledBackend(id);
+  const read = backend?.usage?.getPlanUsage;
+  if (!read || !backend?.usage) return undefined;
+  return read.call(backend.usage);
+}
+
+/**
+ * Headroom for one backend, cached for {@link HEADROOM_CACHE_MS}.
+ *
+ * `force` skips the cache — the `plan_usage` tool asks for a fresh read
+ * because the operator is looking at the number right now.
+ */
+export async function getBackendHeadroom(
+  id: string,
+  label: string,
+  config: TalonConfig | undefined,
+  options?: { force?: boolean; now?: number },
+): Promise<BackendHeadroom> {
+  const now = options?.now ?? Date.now();
+  const cached = cache.get(id);
+  if (!options?.force && cached && now - cached.cachedAt < HEADROOM_CACHE_MS) {
+    return cached.value;
+  }
+
+  let value: BackendHeadroom;
+  try {
+    const plan = headroomFromPlan(id, label, await readPlanUsage(id));
+    value =
+      plan ??
+      headroomFromLedger(id, label, config, now) ??
+      unknownHeadroom(id, label, now);
+  } catch {
+    // The source is unreachable this minute. Keeping the last good reading
+    // is the conservative answer: forgetting it would read as "empty" and
+    // send the next background run straight at a backend near its ceiling.
+    value = cached
+      ? { ...cached.value, stale: true }
+      : unknownHeadroom(id, label, now);
+  }
+  cache.set(id, { value, cachedAt: now });
+  return value;
+}
+
+/** Headroom for every backend the config exposes, in config order. */
+export async function collectBackendHeadroom(
+  config: TalonConfig | undefined,
+  options?: { force?: boolean; now?: number },
+): Promise<BackendHeadroom[]> {
+  const backends = listAvailableBackends(config);
+  return Promise.all(
+    backends.map(({ id, label }) =>
+      getBackendHeadroom(id, label, config, options),
+    ),
+  );
+}
+
+/** One-line rendering shared by `/usage`, `plan_usage` and the router log. */
+export function formatHeadroom(entry: BackendHeadroom): string {
+  const pct = `${Math.round(entry.headroom * 100)}%`;
+  const detail =
+    entry.source === "none"
+      ? "no usage signal"
+      : `${entry.limiting?.label ?? "window"} ${Math.round(entry.limiting?.percent ?? 0)}% used`;
+  const tag = entry.source === "ledger" ? " (local budget)" : "";
+  const stale = entry.stale ? " (stale)" : "";
+  return `${pct} — ${detail}${tag}${stale}`;
+}
+
+/** Test seam — drop every cached reading. */
+export function resetHeadroomCacheForTest(): void {
+  cache.clear();
+}

package/src/core/engine/backend-router/index.ts

@@ -0,0 +1,52 @@
+/**
+ * Plan-aware backend router.
+ *
+ *   - `ledger`   — the local rolling token count that gives budget-only
+ *                  backends a headroom signal.
+ *   - `headroom` — one comparable "how much is left" per backend, from the
+ *                  plan API where there is one and the ledger where there
+ *                  is not.
+ *   - `router`   — the decision: who runs this background job.
+ */
+
+export {
+  flushBackendLedger,
+  ledgerUsage,
+  loadBackendLedger,
+  recordBackendRunUsage,
+  recordBackendUsage,
+  resetBackendLedgerForTest,
+  tokensInWindow,
+  LEDGER_RETENTION_MS,
+  LEDGER_SHORT_WINDOW_MS,
+} from "./ledger.js";
+export {
+  collectBackendHeadroom,
+  formatHeadroom,
+  getBackendHeadroom,
+  hasBudget,
+  headroomFromLedger,
+  headroomFromPlan,
+  limitingWindowOf,
+  resetHeadroomCacheForTest,
+  HEADROOM_CACHE_MS,
+  type BackendHeadroom,
+  type HeadroomSource,
+  type LimitingWindow,
+} from "./headroom.js";
+export {
+  collectBackendUsage,
+  leadWith,
+  type BackendUsageSnapshot,
+} from "./usage.js";
+export {
+  chooseBackend,
+  resolveRoutedModel,
+  taskClassForEffort,
+  DEFAULT_CEILING_PERCENT,
+  type RouteDecision,
+  type RouteHints,
+  type RoutePurpose,
+  type RouteRequest,
+  type TaskClass,
+} from "./router.js";