talon-agent 5.24.0 → 5.25.0

package/src/core/config/index.ts

@@ -318,6 +318,80 @@ const whatsappConfigSchema = z
   })
   .strict();
 
+/** `fetch` — the fetch ladder behind `fetch_url` (docs/fetch-ladder.md). */
+export const fetchConfigSchema = z
+  .object({
+    /** Browser-TLS impersonation via curl-impersonate (default true). */
+    impersonate: z.boolean().default(true),
+    /** curl-impersonate profiles for the direct rung, tried in order. */
+    impersonateTargets: z
+      .array(
+        z
+          .string()
+          .regex(
+            /^[a-z0-9_]+$/,
+            "a curl-impersonate target name, e.g. safari184",
+          ),
+      )
+      .min(1)
+      .optional(),
+    /** Use this curl-impersonate binary instead of the first-use download. */
+    curlImpersonatePath: z.string().trim().min(1).optional(),
+    /**
+     * SOCKS exits for the impersonation rung, e.g.
+     * "socks5h://socks-nl1.nordvpn.com:1080". Credentials are never
+     * accepted here: they come from TALON_FETCH_SOCKS_USER /
+     * TALON_FETCH_SOCKS_PASS or `socksCredentialsFile` (one line,
+     * "user:pass").
+     */
+    socksExits: z
+      .array(
+        z
+          .string()
+          .trim()
+          .refine((value) => {
+            try {
+              return /^socks(4a?|5h?):$/.test(new URL(value).protocol);
+            } catch {
+              return false;
+            }
+          }, "must be a socks5h://, socks5://, socks4a:// or socks4:// URL")
+          .refine((value) => {
+            try {
+              const u = new URL(value);
+              return !u.username && !u.password;
+            } catch {
+              return true;
+            }
+          }, "must not contain credentials — set TALON_FETCH_SOCKS_USER/TALON_FETCH_SOCKS_PASS or fetch.socksCredentialsFile"),
+      )
+      .default([]),
+    socksCredentialsFile: z.string().trim().min(1).optional(),
+    /**
+     * Anti-detect browser rung: connect to the Camoufox/Playwright server
+     * at `browserEndpoint` (default: the playwright plugin's endpoint).
+     */
+    camoufox: z.boolean().default(false),
+    browserEndpoint: z.string().trim().min(1).optional(),
+    /**
+     * Last resort: run curl on this mesh device (id or name). Off unless
+     * set — the device's owner pays for the traffic and the IP.
+     */
+    egressDevice: z.string().trim().min(1).optional(),
+    /**
+     * Daily cap (bytes) on traffic through relayed rungs (SOCKS exits and
+     * the egress device). 0 disables those rungs. Direct rungs are
+     * uncapped. The counter is in-memory and resets at UTC midnight and
+     * on restart.
+     */
+    dailyByteCap: z
+      .number()
+      .int()
+      .min(0)
+      .default(100 * 1024 * 1024),
+  })
+  .strict();
+
 const playwrightConfigSchema = z.object({
   enabled: z.boolean().default(false),
   /** Browser engine: chromium (default), chrome, firefox, webkit, msedge */
@@ -690,6 +764,14 @@ const configSchema = z.object({
     .object({ allowPrivateNetworks: z.boolean().default(true) })
     .strict()
     .optional(),
+  /**
+   * The fetch ladder behind `fetch_url` (docs/fetch-ladder.md): when a page
+   * answers with a bot wall it retries through browser-TLS impersonation,
+   * SOCKS exits, a plain fetch, an anti-detect browser and — only if set —
+   * a mesh device, in that order. Everything but impersonation is off
+   * until configured.
+   */
+  fetch: fetchConfigSchema.optional(),
   /**
    * Codex-specific OpenAI API key. Prefer this, CODEX_API_KEY, or
    * TALON_CODEX_KEY when the Codex backend should use API-key billing

package/src/core/daemon/discovery.ts

@@ -17,6 +17,7 @@
  */
 
 import { readPidRecord, isProcessAlive } from "./pidfile.js";
+import { log, logWarn } from "../../util/log.js";
 import {
   gatewayAuthHeaders,
   readGatewayToken,
@@ -174,3 +175,162 @@ export async function findRunningInstance(
 
   return scanForDaemon();
 }
+
+// ── Single-instance guard ───────────────────────────────────────────────────
+
+/**
+ * Single-instance guard — a daemon refuses to boot while another one runs.
+ *
+ * `talon start` already checked for a running instance, but only the CLI
+ * did. The daemon entry wrote its pidfile unconditionally. Anything that
+ * started the entry directly (systemd, a stray `bun src/index.ts`, the
+ * handoff watcher's fallback racing a slow successor) got a second daemon.
+ * On 2026-09-27 that ran for 13 minutes:
+ *   - the newcomer overwrote the pidfile;
+ *   - its gateway and bridge fell back to 19877/19881;
+ *   - both polled Telegram, and every getUpdates answered 409;
+ *   - the newcomer's trigger resume killed the running daemon's watchers
+ *     as "orphans".
+ *
+ * This guard runs first thing in `app.ts`, before any side effect. It
+ * refuses when an identity-checked `/health` answers as a daemon (or a
+ * live pidfile pid turns into one while we wait for it to finish booting).
+ *
+ * The one daemon allowed to overlap is our own predecessor. A `/restart`
+ * successor is spawned in the last moments of the old process's graceful
+ * shutdown, so the guard waits for that pid to exit and then carries on.
+ */
+
+/**
+ * Set by `spawnSuccessor()` on the child it spawns: the pid of the daemon
+ * handing over. The guard waits that process out instead of refusing.
+ */
+export const PREDECESSOR_PID_ENV = "TALON_PREDECESSOR_PID";
+
+/** How long a successor waits for its predecessor to exit. */
+const PREDECESSOR_WAIT_MS = 20_000;
+/** How long a live-but-silent pidfile pid gets to answer /health. */
+const UNVERIFIED_WAIT_MS = 15_000;
+const GUARD_POLL_MS = 250;
+
+export type InstanceCheck =
+  { ok: true } | { ok: false; instance: RunningInstance };
+
+export interface InstanceCheckDeps {
+  find?: (pidfilePath?: string) => Promise<RunningInstance | null>;
+  isAlive?: (pid: number) => boolean;
+  sleep?: (ms: number) => Promise<void>;
+  now?: () => number;
+  env?: NodeJS.ProcessEnv;
+  pidfilePath?: string;
+  predecessorWaitMs?: number;
+  unverifiedWaitMs?: number;
+}
+
+const defaultSleep = (ms: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, ms));
+
+function predecessorPid(env: NodeJS.ProcessEnv): number | undefined {
+  const pid = Number(env[PREDECESSOR_PID_ENV]);
+  return Number.isInteger(pid) && pid > 0 && pid !== process.pid
+    ? pid
+    : undefined;
+}
+
+/** Poll until `done()` or the deadline; resolves whether `done()` held. */
+async function waitFor(
+  done: () => boolean | Promise<boolean>,
+  ms: number,
+  sleep: (ms: number) => Promise<void>,
+  now: () => number,
+): Promise<boolean> {
+  const deadline = now() + ms;
+  for (;;) {
+    if (await done()) return true;
+    if (now() >= deadline) return false;
+    await sleep(GUARD_POLL_MS);
+  }
+}
+
+/**
+ * Decide whether this process may boot as the daemon. Never throws: a
+ * discovery failure answers "ok" (a guard that keeps the only daemon down
+ * is worse than the duplicate it prevents).
+ */
+export async function checkSingleInstance(
+  deps: InstanceCheckDeps = {},
+): Promise<InstanceCheck> {
+  const find = deps.find ?? findRunningInstance;
+  const isAlive = deps.isAlive ?? isProcessAlive;
+  const sleep = deps.sleep ?? defaultSleep;
+  const now = deps.now ?? Date.now;
+  const env = deps.env ?? process.env;
+
+  try {
+    const predecessor = predecessorPid(env);
+    // Consumed: children of this daemon must not inherit it.
+    delete env[PREDECESSOR_PID_ENV];
+    if (predecessor !== undefined && isAlive(predecessor)) {
+      log("bot", `Waiting for predecessor daemon ${predecessor} to exit`);
+      const gone = await waitFor(
+        () => !isAlive(predecessor),
+        deps.predecessorWaitMs ?? PREDECESSOR_WAIT_MS,
+        sleep,
+        now,
+      );
+      if (!gone) {
+        // It is past its own 15s force-exit timer and has already let go of
+        // the frontends. Staying down would leave nothing running.
+        logWarn(
+          "bot",
+          `Predecessor daemon ${predecessor} is still alive — booting anyway`,
+        );
+      }
+    }
+
+    let instance = await find(deps.pidfilePath);
+    if (instance?.source === "pidfile-unverified") {
+      // A live pid with no /health: a daemon still booting, or a recycled
+      // pid. Give it time to answer before deciding.
+      const pid = instance.pid;
+      await waitFor(
+        async () => {
+          if (!isAlive(pid)) return true;
+          instance = await find(deps.pidfilePath);
+          return instance?.source !== "pidfile-unverified";
+        },
+        deps.unverifiedWaitMs ?? UNVERIFIED_WAIT_MS,
+        sleep,
+        now,
+      );
+      if (instance?.source === "pidfile-unverified") {
+        logWarn(
+          "bot",
+          `pidfile names live pid ${instance.pid} but no daemon answers — ` +
+            `treating it as a recycled pid`,
+        );
+        return { ok: true };
+      }
+    }
+    if (!instance) return { ok: true };
+    if (instance.pid === process.pid || instance.pid === predecessor) {
+      return { ok: true };
+    }
+    return { ok: false, instance };
+  } catch (err) {
+    logWarn(
+      "bot",
+      `single-instance check failed (${err instanceof Error ? err.message : String(err)}) — booting`,
+    );
+    return { ok: true };
+  }
+}
+
+/** One line for the log and stderr when the guard refuses. */
+export function describeRefusal(instance: RunningInstance): string {
+  const where = instance.port ? ` on :${instance.port}` : "";
+  return (
+    `another Talon daemon is already running (pid ${instance.pid}${where}). ` +
+    "Refusing to start a second one — use `talon restart` to replace it."
+  );
+}

package/src/core/daemon/pidfile.ts

@@ -96,3 +96,110 @@ export function isProcessAlive(pid: number): boolean {
     return (err as NodeJS.ErrnoException).code === "EPERM";
   }
 }
+
+// ── Child ownership ─────────────────────────────────────────────────────────
+
+/**
+ * Daemon ownership of child processes.
+ *
+ * Every process the daemon spawns (backend CLIs, MCP children, trigger
+ * scripts) inherits its environment. At boot the daemon stamps two
+ * variables into that environment: its own pid and its /proc start time.
+ * Any orphan sweep can then tell a child whose daemon is gone (safe to
+ * reap) from a child of a daemon that is still running.
+ *
+ * The distinction matters because "orphan" used to mean "alive and
+ * tagged with our chat or trigger id". On 2026-09-27 two daemons ran at
+ * once for 13 minutes. Each one's sweeps saw the other's live children as
+ * leftovers from a previous run, and the newcomer's trigger resume
+ * SIGKILLed the running daemon's watchers.
+ *
+ * Linux-only in practice: the reads go through /proc. Where /proc is
+ * absent, {@link childBelongsToLiveDaemon} answers false and the sweeps
+ * behave exactly as before.
+ */
+
+/** Pid of the daemon that spawned this process (inherited env). */
+export const DAEMON_PID_ENV = "TALON_DAEMON_PID";
+/** That daemon's /proc start time, so a recycled pid cannot pass for it. */
+export const DAEMON_STARTTIME_ENV = "TALON_DAEMON_STARTTIME";
+
+/**
+ * Field 22 of /proc/<pid>/stat: start time in jiffies since boot.
+ * Monotonic per boot and unchanged by exec(), so it pins a pid to one
+ * process. `undefined` without /proc or when the read fails.
+ *
+ * The `comm` field (2nd) is wrapped in parens and may itself contain ')'
+ * — split after the LAST ')'; index 19 of the rest is field 22.
+ */
+export function readPidStarttimeSync(pid: number): number | undefined {
+  try {
+    const stat = readFileSync(`/proc/${pid}/stat`, "utf-8");
+    const lastParen = stat.lastIndexOf(")");
+    if (lastParen < 0) return undefined;
+    const tail = stat.slice(lastParen + 2).split(" ");
+    const starttime = Number(tail[19]);
+    return Number.isFinite(starttime) ? starttime : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Stamp this process as the daemon in `env` (default: our own), so every
+ * child spawned from here on names us. Overwrites any inherited stamp: a
+ * `/restart` successor inherits its predecessor's environment.
+ */
+export function stampDaemonOwner(env: NodeJS.ProcessEnv = process.env): void {
+  env[DAEMON_PID_ENV] = String(process.pid);
+  const starttime = readPidStarttimeSync(process.pid);
+  if (starttime !== undefined) env[DAEMON_STARTTIME_ENV] = String(starttime);
+  else delete env[DAEMON_STARTTIME_ENV];
+}
+
+export interface DaemonOwner {
+  pid: number;
+  starttime?: number;
+}
+
+/** The owner stamp carried in a NUL-split `/proc/<pid>/environ`. */
+export function ownerFromEnviron(entries: string[]): DaemonOwner | undefined {
+  const value = (key: string): string | undefined => {
+    const prefix = `${key}=`;
+    return entries.find((e) => e.startsWith(prefix))?.slice(prefix.length);
+  };
+  const pid = Number(value(DAEMON_PID_ENV));
+  if (!Number.isInteger(pid) || pid <= 0) return undefined;
+  const starttime = Number(value(DAEMON_STARTTIME_ENV));
+  return Number.isFinite(starttime) && value(DAEMON_STARTTIME_ENV)
+    ? { pid, starttime }
+    : { pid };
+}
+
+/**
+ * Whether `owner` is a daemon other than this one that is still running.
+ * A matching pid with a different start time is a recycled pid, so the
+ * owner is dead.
+ */
+export function isOtherLiveDaemon(owner: DaemonOwner | undefined): boolean {
+  if (!owner || owner.pid === process.pid) return false;
+  if (!isProcessAlive(owner.pid)) return false;
+  if (owner.starttime === undefined) return true;
+  const current = readPidStarttimeSync(owner.pid);
+  return current === undefined || current === owner.starttime;
+}
+
+/**
+ * Whether process `pid` was spawned by a daemon other than this one that
+ * is still alive. Orphan sweeps must leave such a process alone. Reads
+ * `/proc/<pid>/environ`; false when it can't be read.
+ */
+export function childBelongsToLiveDaemon(pid: number): boolean {
+  let raw: string;
+  try {
+    raw = readFileSync(`/proc/${pid}/environ`, "utf-8");
+  } catch {
+    return false;
+  }
+  return isOtherLiveDaemon(ownerFromEnviron(raw.split("\0")));
+}

package/src/core/daemon/respawn.ts

@@ -55,6 +55,7 @@
 import { spawn } from "node:child_process";
 import { log, logError, openRespawnLog } from "../../util/log.js";
 import { HANDOFF_WATCH_SUBCOMMAND } from "./handoff.js";
+import { PREDECESSOR_PID_ENV } from "./discovery.js";
 
 let pendingReason: string | null = null;
 let shutdown: ((reason: string) => void) | null = null;
@@ -187,7 +188,9 @@ export function spawnSuccessor(spawnFn: SpawnFn = spawn): void {
       cwd: process.cwd(),
       detached: true,
       stdio: ["ignore", fd ?? "ignore", fd ?? "ignore"],
-      env: { ...process.env },
+      // Tells the successor's single-instance guard that the daemon it
+      // can still see is us, on our way out — wait, don't refuse.
+      env: { ...process.env, [PREDECESSOR_PID_ENV]: String(process.pid) },
     });
     child.once("error", (err) => {
       logError("shutdown", `Respawn failed (${reason}) — see respawn.log`, err);

package/src/core/engine/backend-router/breaker.ts

@@ -0,0 +1,158 @@
+/**
+ * Backend circuit breaker — "is this backend actually working right now?"
+ *
+ * Headroom answers "how much plan is left". It says nothing about whether
+ * runs on the backend succeed. In Sep 2026 Codex's login expired and its
+ * default model was retired, so every run on it failed. Codex still had no
+ * usage signal and read as 100% headroom, and the router kept sending
+ * background work there for 36 hours. This module holds the missing
+ * signal, fed by the outcome of every routed run:
+ *
+ *   - an auth failure (401, expired login, "run `… login`") opens the
+ *     breaker at once: the next run is not going to fix a credential;
+ *   - {@link BREAKER_FAILURE_THRESHOLD} consecutive failures of any other
+ *     kind open it too;
+ *   - a success closes it and clears the count.
+ *
+ * An open breaker reads as zero headroom (see `headroom.ts`) until its
+ * cool-off ends. After that the backend is half-open: it can win again,
+ * and one more failure without a success in between re-opens it straight
+ * away with twice the cool-off, capped at {@link BREAKER_MAX_COOLOFF_MS}.
+ *
+ * State lives in memory only. A restart gives every backend a fresh
+ * chance, which is what an operator who just ran `codex login` and
+ * restarted expects.
+ */
+
+import { logWarn } from "../../../util/log.js";
+
+/** Consecutive non-auth failures that open the breaker. */
+export const BREAKER_FAILURE_THRESHOLD = 3;
+/** First cool-off. Each re-trip without a success in between doubles it. */
+export const BREAKER_BASE_COOLOFF_MS = 15 * 60_000;
+/** Ceiling on the cool-off. */
+export const BREAKER_MAX_COOLOFF_MS = 4 * 60 * 60_000;
+
+interface BreakerState {
+  /** Failures since the last success. */
+  consecutive: number;
+  /** Times the breaker has opened since the last success. */
+  trips: number;
+  /** Epoch ms the current cool-off ends; undefined when never opened. */
+  openUntil?: number;
+  /** Why it last opened. */
+  reason?: string;
+}
+
+/** An open breaker, as `headroom.ts` and the router log see it. */
+export interface OpenBreaker {
+  readonly reason: string;
+  /** Epoch ms the cool-off ends. */
+  readonly until: number;
+}
+
+const breakers = new Map<string, BreakerState>();
+
+/**
+ * Credential failures, not model or transport ones. Covers what the
+ * backends actually say: HTTP 401, Codex's "refresh token" / "login
+ * expired" wording, agy's "authentication required", and any "run
+ * `x login`" remedy.
+ */
+const AUTH_FAILURE_RE =
+  /\b401\b|unauthori[sz]ed|authentication (?:required|failed)|not (?:logged|signed) in|log(?:in|ged in) (?:has )?expired|refresh[_ ]token|invalid[_ ](?:api[_ ])?key|run [`'"]?[\w-]+ login/i;
+
+/** Whether an error message describes a credential problem. */
+export function isAuthFailureMessage(message: string): boolean {
+  return AUTH_FAILURE_RE.test(message);
+}
+
+function messageOf(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}
+
+/**
+ * A run the caller cancelled says nothing about the backend. A timeout
+ * does count: a backend that hangs is as unusable as one that errors.
+ */
+function isCallerAbort(err: unknown): boolean {
+  if (err instanceof Error && err.name === "AbortError") return true;
+  return /aborted before the run started/i.test(messageOf(err));
+}
+
+function stateFor(id: string): BreakerState {
+  let state = breakers.get(id);
+  if (!state) {
+    state = { consecutive: 0, trips: 0 };
+    breakers.set(id, state);
+  }
+  return state;
+}
+
+function open(id: string, state: BreakerState, reason: string, now: number) {
+  state.trips += 1;
+  const cooloff = Math.min(
+    BREAKER_BASE_COOLOFF_MS * 2 ** (state.trips - 1),
+    BREAKER_MAX_COOLOFF_MS,
+  );
+  state.openUntil = now + cooloff;
+  state.reason = reason;
+  logWarn(
+    "router",
+    `breaker open for ${id} (${reason}) — no routed work for ` +
+      `${Math.round(cooloff / 60_000)}m`,
+  );
+}
+
+/** Record a failed run on a backend. */
+export function recordBackendRunFailure(
+  id: string,
+  err: unknown,
+  now = Date.now(),
+): void {
+  if (isCallerAbort(err)) return;
+  const state = stateFor(id);
+  state.consecutive += 1;
+  const message = messageOf(err).split("\n")[0]?.trim().slice(0, 160) ?? "";
+  // Already open: a failure in the cool-off (a pinned run, say) must not
+  // stretch it.
+  if (state.openUntil !== undefined && now < state.openUntil) return;
+  if (isAuthFailureMessage(message)) {
+    open(id, state, `auth failure: ${message}`, now);
+    return;
+  }
+  // Half-open (tripped before, no success since): the probe failed, so
+  // re-open at once rather than waiting for a fresh run of failures.
+  if (state.trips > 0) {
+    open(id, state, `still failing after cool-off: ${message}`, now);
+    return;
+  }
+  if (state.consecutive >= BREAKER_FAILURE_THRESHOLD) {
+    open(
+      id,
+      state,
+      `${state.consecutive} consecutive failures: ${message}`,
+      now,
+    );
+  }
+}
+
+/** Record a successful run: the backend works, forget its failures. */
+export function recordBackendRunSuccess(id: string): void {
+  breakers.delete(id);
+}
+
+/** The breaker, when it is open at `now`; undefined when the backend may run. */
+export function openBreaker(
+  id: string,
+  now = Date.now(),
+): OpenBreaker | undefined {
+  const state = breakers.get(id);
+  if (!state?.openUntil || now >= state.openUntil) return undefined;
+  return { reason: state.reason ?? "failing", until: state.openUntil };
+}
+
+/** Test seam — close every breaker. */
+export function resetBackendBreakersForTest(): void {
+  breakers.clear();
+}