talon-agent 5.24.0 → 5.25.0

package/src/core/engine/backend-router/headroom.ts

@@ -11,11 +11,21 @@
  *                the operator's soft budget (`config.backendBudgets`). The
  *                fallback for backends with no account API.
  *
- * A backend with neither reports `source: "none"` and headroom 1. That is a
- * deliberate "no evidence of pressure", not a claim of capacity — the
- * router's comparator ranks it *below* any backend with real telemetry at
- * the same headroom, so an unmeasured backend never outranks a measured one
- * it is tied with.
+ * A backend with neither reports `source: "none"` and headroom 0: nothing
+ * is known about it, so it is never *preferred*. It still clears the
+ * ceiling (there is no window to be over), so it runs work when nothing
+ * measured is available. It used to read as 1. That made a Codex install
+ * with an expired login, and so no usage signal, the router's favourite
+ * for 36 hours while every run on it failed.
+ *
+ * On top of either source sit two "this backend is not working" signals.
+ * Either one zeroes the headroom and pins the limiting window at 100%, so
+ * the ceiling drops the backend whenever anything else is left:
+ *
+ *   - the backend's own telemetry reports a rejected credential
+ *     (`UsageTelemetry.getAuthFailure`, e.g. the Codex usage endpoint's 401);
+ *   - the run breaker is open (`breaker.ts`): an auth failure or repeated
+ *     failures on runs routed there.
  *
  * Reads are cached for 60s per backend: `/usage`, the router and the
  * `plan_usage` tool all ask, and a plan lookup can be a subprocess spawn. A
@@ -31,6 +41,7 @@ import {
   listAvailableBackends,
 } from "../backend-controller/index.js";
 import { ledgerUsage } from "./ledger.js";
+import { openBreaker } from "./breaker.js";
 
 /** How long a headroom reading is reused before the source is asked again. */
 export const HEADROOM_CACHE_MS = 60_000;
@@ -63,6 +74,11 @@ export interface BackendHeadroom {
    * same (cached) fetch the router ranked on, instead of asking twice.
    */
   readonly plan?: PlanUsage;
+  /**
+   * Why the backend is treated as unusable right now (a rejected login, an
+   * open breaker). Set means headroom 0 and a 100% limiting window.
+   */
+  readonly unavailable?: string;
 }
 
 interface CacheEntry {
@@ -183,13 +199,29 @@ export function headroomFromLedger(
   };
 }
 
-/** The "nothing to measure" reading. Headroom 1, but lowest ranking source. */
+/** The "nothing to measure" reading: headroom 0, but under any ceiling. */
 function unknownHeadroom(
   id: string,
   label: string,
   now: number,
 ): BackendHeadroom {
-  return { id, label, headroom: 1, source: "none", fetchedAt: now };
+  return { id, label, headroom: 0, source: "none", fetchedAt: now };
+}
+
+/** What a backend's telemetry said: its plan windows and any auth failure. */
+interface PlanRead {
+  usage?: PlanUsage;
+  authFailure?: string;
+}
+
+function authFailureOf(
+  usage: { getAuthFailure?(): string | undefined } | undefined,
+): string | undefined {
+  try {
+    return usage?.getAuthFailure?.call(usage) || undefined;
+  } catch {
+    return undefined;
+  }
 }
 
 /**
@@ -202,22 +234,35 @@ function unknownHeadroom(
  * is *not* currently in use. The read is the backend's own cached one, so at
  * most one boot per cache window.
  */
-async function readPlanUsage(id: string): Promise<PlanUsage | undefined> {
+async function readPlanUsage(id: string): Promise<PlanRead> {
   const pooled = getPooledBackend(id);
-  if (pooled?.usage?.getPlanUsage) {
-    return pooled.usage.getPlanUsage.call(pooled.usage);
+  if (pooled) {
+    const usage = pooled.usage?.getPlanUsage
+      ? await pooled.usage.getPlanUsage.call(pooled.usage)
+      : undefined; // pooled, but reports no plan windows
+    // Read after the plan fetch: that fetch is what discovers a 401.
+    const authFailure = authFailureOf(pooled.usage);
+    return {
+      ...(usage ? { usage } : {}),
+      ...(authFailure ? { authFailure } : {}),
+    };
   }
-  if (pooled) return undefined; // pooled, but reports no plan windows
   let acquired;
   try {
     acquired = await acquireBackendInstance(id);
   } catch {
-    return undefined; // can't boot it (not configured, no auth) — stay quiet
+    return {}; // can't boot it (not configured, no auth) — stay quiet
   }
   try {
-    const read = acquired.backend.usage?.getPlanUsage;
-    if (!read || !acquired.backend.usage) return undefined;
-    return await read.call(acquired.backend.usage);
+    const telemetry = acquired.backend.usage;
+    const usage = telemetry?.getPlanUsage
+      ? await telemetry.getPlanUsage.call(telemetry)
+      : undefined;
+    const authFailure = authFailureOf(telemetry);
+    return {
+      ...(usage ? { usage } : {}),
+      ...(authFailure ? { authFailure } : {}),
+    };
   } finally {
     await acquired.release().catch(() => {});
   }
@@ -238,16 +283,18 @@ export async function getBackendHeadroom(
   const now = options?.now ?? Date.now();
   const cached = cache.get(id);
   if (!options?.force && cached && now - cached.cachedAt < HEADROOM_CACHE_MS) {
-    return cached.value;
+    return withBreaker(cached.value, now);
   }
 
   let value: BackendHeadroom;
   try {
-    const plan = headroomFromPlan(id, label, await readPlanUsage(id));
+    const read = await readPlanUsage(id);
+    const plan = headroomFromPlan(id, label, read.usage);
     value =
       plan ??
       headroomFromLedger(id, label, config, now) ??
       unknownHeadroom(id, label, now);
+    if (read.authFailure) value = unavailable(value, read.authFailure);
   } catch {
     // The source is unreachable this minute. Keeping the last good reading
     // is the conservative answer: forgetting it would read as "empty" and
@@ -257,7 +304,29 @@ export async function getBackendHeadroom(
       : unknownHeadroom(id, label, now);
   }
   cache.set(id, { value, cachedAt: now });
-  return value;
+  return withBreaker(value, now);
+}
+
+/** Mark a reading unusable: zero headroom, limiting window pinned at 100%. */
+function unavailable(value: BackendHeadroom, why: string): BackendHeadroom {
+  return {
+    ...value,
+    headroom: 0,
+    limiting: { label: why, percent: 100 },
+    unavailable: why,
+  };
+}
+
+/**
+ * Overlay the run breaker. Applied on every read rather than cached: the
+ * breaker opens and closes on run outcomes, not on the headroom clock.
+ */
+function withBreaker(value: BackendHeadroom, now: number): BackendHeadroom {
+  if (value.unavailable) return value;
+  const breaker = openBreaker(value.id, now);
+  if (!breaker) return value;
+  const mins = Math.max(1, Math.ceil((breaker.until - now) / 60_000));
+  return unavailable(value, `breaker open ${mins}m — ${breaker.reason}`);
 }
 
 /** Headroom for every backend the config exposes, in config order. */
@@ -275,11 +344,10 @@ export async function collectBackendHeadroom(
 
 /** One-line rendering shared by `/usage`, `plan_usage` and the router log. */
 export function formatHeadroom(entry: BackendHeadroom): string {
+  if (entry.unavailable) return `0% — unavailable: ${entry.unavailable}`;
+  if (entry.source === "none") return "unmeasured — no usage signal";
   const pct = `${Math.round(entry.headroom * 100)}%`;
-  const detail =
-    entry.source === "none"
-      ? "no usage signal"
-      : `${entry.limiting?.label ?? "window"} ${Math.round(entry.limiting?.percent ?? 0)}% used`;
+  const detail = `${entry.limiting?.label ?? "window"} ${Math.round(entry.limiting?.percent ?? 0)}% used`;
   const tag = entry.source === "ledger" ? " (local budget)" : "";
   const stale = entry.stale ? " (stale)" : "";
   return `${pct} — ${detail}${tag}${stale}`;

package/src/core/engine/backend-router/index.ts

@@ -20,6 +20,17 @@ export {
   LEDGER_RETENTION_MS,
   LEDGER_SHORT_WINDOW_MS,
 } from "./ledger.js";
+export {
+  isAuthFailureMessage,
+  openBreaker,
+  recordBackendRunFailure,
+  recordBackendRunSuccess,
+  resetBackendBreakersForTest,
+  BREAKER_BASE_COOLOFF_MS,
+  BREAKER_FAILURE_THRESHOLD,
+  BREAKER_MAX_COOLOFF_MS,
+  type OpenBreaker,
+} from "./breaker.js";
 export {
   collectBackendHeadroom,
   formatHeadroom,

package/src/core/engine/gateway-actions/agents/control.ts

@@ -20,6 +20,7 @@ import {
   getAgentCaps,
   killAgent,
   spawnAgent,
+  wantsPreflight,
   type AgentParent,
   type AgentRecord,
 } from "../../../agents/index.js";
@@ -150,6 +151,7 @@ function readSpawnBody(body: Record<string, unknown>):
       model?: string;
       reasoningEffort?: ReasoningEffortLevel;
       timeoutMs?: number;
+      preflight: boolean;
     }
   | { ok: false; error: string } {
   const brief = String(body.brief ?? "").trim();
@@ -168,10 +170,15 @@ function readSpawnBody(body: Record<string, unknown>):
   if (timeoutS !== undefined && !Number.isFinite(timeoutS)) {
     return { ok: false, error: "timeout_s must be a number of seconds" };
   }
+  const preflight = body.preflight;
+  if (preflight !== undefined && typeof preflight !== "boolean") {
+    return { ok: false, error: "preflight must be true or false" };
+  }
   return {
     ok: true,
     brief,
     label,
+    preflight: wantsPreflight(brief, preflight),
     ...(body.backend ? { backendId: String(body.backend) } : {}),
     ...(body.model ? { model: String(body.model) } : {}),
     ...(effort !== undefined
@@ -200,6 +207,7 @@ export const agentControlHandlers: SharedActionHandlers = {
       ...(parsed.timeoutMs !== undefined
         ? { timeoutMs: parsed.timeoutMs }
         : {}),
+      preflight: parsed.preflight,
     });
     if (!outcome.ok) return { ok: false, error: outcome.error };
     const timeoutS = Math.round(clampTimeout(parsed.timeoutMs) / 1000);
@@ -211,6 +219,7 @@ export const agentControlHandlers: SharedActionHandlers = {
         `Backend: ${outcome.backendId}/${outcome.model}` +
         `${outcome.routing ? ` (routed: ${outcome.routing})` : ""}\n` +
         `Timeout: ${timeoutS}s\n` +
+        (parsed.preflight ? `Pre-flight lane: on\n` : "") +
         `It runs in the background. You will be woken with its report — ` +
         `carry on with what you were doing.`,
     };

package/src/core/engine/gateway-actions/agents/index.ts

@@ -6,6 +6,8 @@
  *   - `report`  — what a sub-agent calls about itself and its siblings:
  *     report_result / message_parent / check_inbox / list_peers /
  *     message_peer.
+ *   - `preflight` — run_preflight: the pre-flight lane (light CI suite) in
+ *     the caller's checkout, so an agent pushes only on green.
  *
  * Both sets are reachable from an `agent:<id>` context: the gateway routes
  * those chat keys straight here (see `Gateway.handleAction`), because a
@@ -15,11 +17,13 @@
 
 import type { SharedActionHandlers } from "../types.js";
 import { agentControlHandlers } from "./control.js";
+import { agentPreflightHandlers } from "./preflight.js";
 import { agentReportHandlers } from "./report.js";
 
 export const agentHandlers: SharedActionHandlers = {
   ...agentControlHandlers,
   ...agentReportHandlers,
+  ...agentPreflightHandlers,
 };
 
 /**

package/src/core/engine/gateway-actions/agents/preflight.ts

@@ -0,0 +1,215 @@
+/**
+ * `run_preflight` — run a checkout's pre-flight lane and return its verdict.
+ *
+ * The pre-flight lane (`scripts/preflight.sh`, `npm run preflight`) is the
+ * light CI suite an agent runs before `git push`, so GitHub confirms a change
+ * instead of being the first compiler it meets. This action runs it in the
+ * caller's checkout on the daemon host and answers with the one-line verdict,
+ * the per-step table and the tail of every failing step's log — enough to
+ * fix the change without re-running anything by hand.
+ *
+ * Available from a chat and from inside a sub-agent (it is in
+ * `agentContextActions`), because agents are the ones opening PRs.
+ */
+
+import { spawn } from "node:child_process";
+import { readFile, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { dirs } from "../../../../util/paths.js";
+import { resolvePathParam } from "../native/params.js";
+import type { ActionResult } from "../../../types.js";
+import type { SharedActionHandlers } from "../types.js";
+
+/** Relative path of the lane inside a checkout. */
+const SCRIPT = join("scripts", "preflight.sh");
+/** Hard cap on one run. The lane targets ≤5 min; this is the backstop. */
+const PREFLIGHT_TIMEOUT_MS = 600_000;
+/** Grace between SIGTERM and SIGKILL of the run's process group. */
+const KILL_GRACE_MS = 5_000;
+/** Lines of each failing step's log to hand back. */
+const FAIL_TAIL_LINES = 30;
+
+interface PreflightStep {
+  readonly name: string;
+  readonly status: "pass" | "fail" | "skipped";
+  readonly ms: number;
+  readonly note?: string;
+}
+
+interface PreflightSummary {
+  readonly verdict: "green" | "red";
+  readonly ok: boolean;
+  readonly base: string;
+  readonly totalMs: number;
+  readonly failed: readonly string[];
+  readonly steps: readonly PreflightStep[];
+}
+
+interface RunOutcome {
+  readonly code: number | null;
+  readonly timedOut: boolean;
+  readonly output: string;
+}
+
+/** The checkout root for `dir`: its git toplevel, else `dir` itself. */
+function repoRoot(dir: string): Promise<string> {
+  return new Promise((resolveRoot) => {
+    const child = spawn("git", ["rev-parse", "--show-toplevel"], { cwd: dir });
+    let out = "";
+    child.stdout.on("data", (chunk: Buffer) => (out += chunk.toString()));
+    child.on("error", () => resolveRoot(dir));
+    child.on("close", (code) =>
+      resolveRoot(code === 0 && out.trim() ? out.trim() : dir),
+    );
+  });
+}
+
+async function isFile(path: string): Promise<boolean> {
+  try {
+    return (await stat(path)).isFile();
+  } catch {
+    return false;
+  }
+}
+
+/** Run the lane in its own process group so a timeout takes every child. */
+function runLane(root: string, timeoutMs: number): Promise<RunOutcome> {
+  return new Promise((resolveRun) => {
+    const child = spawn("bash", [SCRIPT], {
+      cwd: root,
+      detached: process.platform !== "win32",
+      env: { ...process.env, PREFLIGHT_QUIET: "1", CI: "1" },
+    });
+    let output = "";
+    const keep = (chunk: Buffer): void => {
+      output = (output + chunk.toString()).slice(-8_000);
+    };
+    child.stdout.on("data", keep);
+    child.stderr.on("data", keep);
+    let timedOut = false;
+    const killGroup = (signal: NodeJS.Signals): void => {
+      try {
+        if (child.pid && process.platform !== "win32") {
+          process.kill(-child.pid, signal);
+        } else child.kill(signal);
+      } catch {
+        // Already gone.
+      }
+    };
+    const timer = setTimeout(() => {
+      timedOut = true;
+      killGroup("SIGTERM");
+      setTimeout(() => killGroup("SIGKILL"), KILL_GRACE_MS).unref();
+    }, timeoutMs);
+    child.on("error", (err) => {
+      clearTimeout(timer);
+      resolveRun({ code: null, timedOut, output: String(err) });
+    });
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      resolveRun({ code, timedOut, output });
+    });
+  });
+}
+
+async function readSummary(root: string): Promise<PreflightSummary | null> {
+  try {
+    const raw = await readFile(join(root, ".preflight", "last.json"), "utf8");
+    return JSON.parse(raw) as PreflightSummary;
+  } catch {
+    return null;
+  }
+}
+
+async function logTail(root: string, step: string): Promise<string> {
+  try {
+    const raw = await readFile(join(root, ".preflight", `${step}.log`), "utf8");
+    return raw.trimEnd().split("\n").slice(-FAIL_TAIL_LINES).join("\n");
+  } catch {
+    return "(no log)";
+  }
+}
+
+function stepLine(step: PreflightStep): string {
+  const mark =
+    step.status === "pass" ? "✓" : step.status === "fail" ? "✗" : "·";
+  const time =
+    step.status === "skipped" ? "" : ` (${Math.round(step.ms / 1000)}s)`;
+  const note = step.status === "skipped" && step.note ? ` — ${step.note}` : "";
+  return `${mark} ${step.name}${time}${note}`;
+}
+
+/** Render the summary a model can act on: verdict, table, failing tails. */
+async function renderPreflight(
+  root: string,
+  summary: PreflightSummary,
+): Promise<string> {
+  const verdict = summary.ok
+    ? `Pre-flight GREEN in ${Math.round(summary.totalMs / 1000)}s — safe to push.`
+    : `Pre-flight RED in ${Math.round(summary.totalMs / 1000)}s — failed: ` +
+      `${summary.failed.join(", ")}. Fix before pushing, or explain in the PR body.`;
+  const parts = [
+    verdict,
+    `Base: ${summary.base}. Summary: ${join(root, ".preflight", "last.json")}`,
+    "",
+    ...summary.steps.map(stepLine),
+  ];
+  for (const name of summary.failed) {
+    parts.push("", `── ${name} (last ${FAIL_TAIL_LINES} lines) ──`);
+    parts.push(await logTail(root, name));
+  }
+  return parts.join("\n");
+}
+
+/** Run the lane in `dir`'s checkout and describe the result. */
+async function runPreflight(
+  dir: string,
+  timeoutMs: number = PREFLIGHT_TIMEOUT_MS,
+): Promise<ActionResult> {
+  try {
+    if (!(await stat(dir)).isDirectory()) {
+      return { ok: false, error: `cwd is not a directory: ${dir}` };
+    }
+  } catch {
+    return { ok: false, error: `Working directory does not exist: ${dir}` };
+  }
+  const root = await repoRoot(dir);
+  if (!(await isFile(join(root, SCRIPT)))) {
+    return {
+      ok: false,
+      error:
+        `No ${SCRIPT} in ${root}. Pass cwd = the root of a talon checkout ` +
+        `(a branch that has the pre-flight lane).`,
+    };
+  }
+  const run = await runLane(root, timeoutMs);
+  if (run.timedOut) {
+    return {
+      ok: false,
+      error:
+        `Pre-flight did not finish within ${Math.round(timeoutMs / 1000)}s ` +
+        `and was killed. Tail of its output:\n${run.output.slice(-2_000)}`,
+    };
+  }
+  const summary = await readSummary(root);
+  if (!summary) {
+    return {
+      ok: false,
+      error:
+        `Pre-flight exited ${run.code} without writing .preflight/last.json. ` +
+        `Output:\n${run.output.slice(-2_000)}`,
+    };
+  }
+  // A red lane is a successful tool call with a red verdict — the model is
+  // meant to read it and act, not treat it as the tool breaking.
+  return { ok: true, text: await renderPreflight(root, summary) };
+}
+
+export const agentPreflightHandlers: SharedActionHandlers = {
+  run_preflight: (body) => {
+    const raw =
+      typeof body.cwd === "string" && body.cwd.trim() ? body.cwd.trim() : "";
+    const dir = raw ? resolvePathParam(raw, undefined) : dirs.workspace;
+    return runPreflight(dir);
+  },
+};

package/src/core/engine/gateway-actions/cron.ts

@@ -133,6 +133,9 @@ export const cronHandlers: SharedActionHandlers = {
         : null,
       endAt !== undefined ? `ends: ${new Date(endAt).toISOString()}` : null,
       spec.catchup ? `catch-up: ${spec.catchup}` : null,
+      spec.timeoutMs !== undefined
+        ? `timeout: ${Math.round(spec.timeoutMs / 1000)}s`
+        : null,
     ]
       .filter(Boolean)
       .join(", ");
@@ -173,6 +176,8 @@ export const cronHandlers: SharedActionHandlers = {
       if (j.catchup && j.catchup !== "skip")
         bounds.push(`catch-up: ${j.catchup}`);
       if (j.model) bounds.push(`model: ${j.model}`);
+      if (j.timeoutMs !== undefined)
+        bounds.push(`timeout: ${Math.round(j.timeoutMs / 1000)}s`);
       return [
         `- ${j.name} (${status})`,
         `  ID: ${j.id}`,

package/src/core/engine/gateway-actions/fetch-url/guard.ts

@@ -7,15 +7,16 @@
  * credentials), the loopback gateway, a router admin page on the LAN.
  * So before every request — the first one and each redirect hop — the
  * host is resolved and EVERY address it resolves to must be public.
- * Redirects are followed by hand (never by fetch) so a public page
- * cannot bounce the request into a private one.
+ * Redirects are followed by hand (never by the transport) in the fetch
+ * ladder (core/fetch/ladder.ts), which calls `assertPublicUrl` on every
+ * hop, so a public page cannot bounce the request into a private one.
  *
- * Residual risk, stated plainly: fetch resolves the name again after we
- * checked it, so a DNS-rebinding server with a zero TTL can still race
- * us. Pinning the checked address would need a custom connector, which
- * Bun (one of the two runtimes) does not honour. The guard closes the
- * direct, redirect and static-DNS paths; rebinding needs a hostile DNS
- * server and a lucky race.
+ * Residual risk, stated plainly: the runtime's fetch (the "plain" rung)
+ * resolves the name again after we checked it, so a DNS-rebinding server
+ * with a zero TTL can still race it. Pinning the checked address would
+ * need a custom connector, which Bun does not honour. The curl rungs do
+ * pin (`--resolve` to the checked addresses); requests through a SOCKS
+ * exit are resolved by the exit, outside this host's network.
  *
  * The guard is opt-in: `fetch_url` applies it only when the operator sets
  * `fetchUrl.allowPrivateNetworks: false`. By default the agent can read
@@ -33,9 +34,6 @@ const defaultResolver: Resolver = async (host) =>
 
 export class BlockedUrlError extends Error {}
 
-const MAX_REDIRECTS = 5;
-const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
-
 // ── Address classification ──────────────────────────────────────────────────
 
 /** Non-public IPv4 ranges: [network, prefix length]. */
@@ -135,11 +133,13 @@ export function isBlockedAddress(ip: string): boolean {
 /**
  * Throw unless `url` is http(s) and its host resolves only to public
  * addresses. A literal IP is judged directly; a name is resolved.
+ * Returns the checked addresses so a caller that can pin the connection
+ * to them (curl `--resolve`) closes the rebinding race.
  */
 export async function assertPublicUrl(
   url: URL,
   resolve: Resolver = defaultResolver,
-): Promise<void> {
+): Promise<string[]> {
   if (url.protocol !== "http:" && url.protocol !== "https:") {
     throw new BlockedUrlError("URL must use http or https protocol");
   }
@@ -166,36 +166,5 @@ export async function assertPublicUrl(
         `Remove fetchUrl.allowPrivateNetworks: false from config.json (or set it to true) to allow local addresses.`,
     );
   }
-}
-
-export type GuardedFetchOptions = {
-  allowPrivateNetworks?: boolean;
-  resolve?: Resolver;
-  maxRedirects?: number;
-};
-
-/**
- * `fetch` with the guard applied to the first request and to every
- * redirect hop. Returns the final (non-redirect) response.
- */
-export async function guardedFetch(
-  input: string,
-  init: RequestInit,
-  options: GuardedFetchOptions = {},
-): Promise<Response> {
-  const maxRedirects = options.maxRedirects ?? MAX_REDIRECTS;
-  let url = new URL(input);
-  for (let hop = 0; ; hop++) {
-    if (!options.allowPrivateNetworks) {
-      await assertPublicUrl(url, options.resolve);
-    }
-    const resp = await fetch(url, { ...init, redirect: "manual" });
-    const location = resp.headers.get("location");
-    if (!REDIRECT_STATUSES.has(resp.status) || !location) return resp;
-    if (hop >= maxRedirects) {
-      throw new BlockedUrlError(`Too many redirects (max ${maxRedirects})`);
-    }
-    await resp.body?.cancel().catch(() => {});
-    url = new URL(location, url);
-  }
+  return addresses;
 }