talon-agent 5.24.0 → 5.25.0

package/src/core/fetch/rungs.ts

@@ -0,0 +1,319 @@
+/**
+ * The non-impersonation rungs: plain fetch, anti-detect browser
+ * (Camoufox over the Playwright wire protocol) and a mesh egress device.
+ * curl-impersonate lives in curl-impersonate.ts.
+ */
+
+import { execFile } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { binaryOnPath } from "../../util/binary-on-path.js";
+import {
+  readBodyLimited,
+  ResponseTooLargeError,
+} from "../../util/http-body.js";
+import { getMeshService } from "../mesh/index.js";
+import { CHALLENGE_PATTERN } from "./classify.js";
+import { fetchError } from "./errors.js";
+import type { RawResponse, Rung, RungRequest } from "./types.js";
+
+// ── plain ───────────────────────────────────────────────────────────────────
+
+/** The runtime's own fetch, honest User-Agent, one hop per call. */
+export function plainRung(): Rung {
+  return {
+    name: "plain",
+    relayed: false,
+    followsRedirects: false,
+    publicOnly: false,
+    async available() {
+      return true;
+    },
+    async request(req: RungRequest): Promise<RawResponse> {
+      const resp = await fetch(req.url, {
+        redirect: "manual",
+        signal: AbortSignal.timeout(req.timeoutMs),
+        headers: { "User-Agent": "Talon/1.0", ...req.headers },
+      });
+      const contentLength = Number(resp.headers.get("content-length") ?? "");
+      if (contentLength > req.maxBytes) {
+        await resp.body?.cancel().catch(() => {});
+        throw new ResponseTooLargeError();
+      }
+      const body =
+        resp.status >= 300 && resp.status < 400
+          ? (await resp.body?.cancel().catch(() => {}), Buffer.alloc(0))
+          : await readBodyLimited(resp, req.maxBytes);
+      return {
+        status: resp.status,
+        headers: resp.headers,
+        body,
+        url: req.url.href,
+      };
+    },
+  };
+}
+
+// ── anti-detect browser ─────────────────────────────────────────────────────
+
+/**
+ * playwright-core as bundled by @playwright/mcp — the one client version
+ * the playwright plugin already pins to match the Camoufox server (see
+ * plugins/playwright/version-coupling.ts). Undefined when not installed
+ * (e.g. a standalone binary without node_modules).
+ */
+function playwrightCorePath(): string | undefined {
+  try {
+    const here = createRequire(import.meta.url);
+    const mcpPkg = here.resolve("@playwright/mcp/package.json");
+    return createRequire(mcpPkg).resolve("playwright-core");
+  } catch {
+    return undefined;
+  }
+}
+
+/** Read the playwright plugin's endpoint the same way the plugin does. */
+export function playwrightEndpoint(pw?: {
+  endpoint?: string;
+  endpointFile?: string;
+}): string | undefined {
+  if (pw?.endpoint) return pw.endpoint;
+  if (pw?.endpointFile) {
+    try {
+      return readFileSync(pw.endpointFile, "utf-8").trim() || undefined;
+    } catch {
+      return undefined;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * The browser driver, run under Node in a child process: playwright's
+ * WebSocket client does not connect under Bun (verified 2026-09-30 —
+ * `firefox.connect` times out), which is also why the playwright plugin
+ * runs @playwright/mcp under `node`. argv: playwright-core path, endpoint,
+ * url, timeout ms, challenge regex source. Prints one JSON object.
+ */
+const BROWSER_DRIVER = `
+const [pwPath, endpoint, url, timeoutMs, challenge] = process.argv.slice(1);
+const { firefox } = require(pwPath);
+const CH = new RegExp(challenge, "i");
+(async () => {
+  const browser = await firefox.connect(endpoint, { timeout: Math.min(15000, +timeoutMs) });
+  let out;
+  try {
+    const ctx = await browser.newContext();
+    try {
+      const page = await ctx.newPage();
+      let last = null;
+      page.on("response", (r) => {
+        if (r.request().isNavigationRequest() && r.frame() === page.mainFrame()) last = r;
+      });
+      const first = await page.goto(url, { waitUntil: "domcontentloaded", timeout: +timeoutMs });
+      last = last || first;
+      let html = await page.content();
+      if (html.length < 32768 && CH.test(html)) {
+        await page.waitForLoadState("networkidle", { timeout: 12000 }).catch(() => {});
+        await page.waitForTimeout(1000);
+        html = await page.content();
+      }
+      const headers = last ? await last.allHeaders().catch(() => ({})) : {};
+      out = { status: last ? last.status() : 200, url: page.url(), headers, html };
+    } finally {
+      await ctx.close().catch(() => {});
+    }
+  } finally {
+    // For a connected browser this disconnects; the server keeps running.
+    await browser.close().catch(() => {});
+  }
+  process.stdout.write(JSON.stringify(out));
+})().catch((e) => {
+  process.stderr.write(String((e && e.message) || e).split("\\n")[0]);
+  process.exit(1);
+});
+`;
+
+type DriverOutput = {
+  status: number;
+  url: string;
+  headers: Record<string, string>;
+  html: string;
+};
+
+/** Runs the driver; resolves with its parsed stdout. Test seam. */
+type RunBrowserDriver = (
+  args: string[],
+  timeoutMs: number,
+) => Promise<DriverOutput>;
+
+const runBrowserDriver: RunBrowserDriver = (args, timeoutMs) =>
+  new Promise((resolve, reject) => {
+    const node = process.versions.bun ? "node" : process.execPath;
+    execFile(
+      node,
+      ["-e", BROWSER_DRIVER, ...args],
+      { timeout: timeoutMs + 20_000, maxBuffer: 64 * 1024 * 1024 },
+      (err, stdout, stderr) => {
+        if (err) {
+          reject(fetchError(String(stderr).trim() || err.message));
+          return;
+        }
+        try {
+          resolve(JSON.parse(stdout) as DriverOutput);
+        } catch {
+          reject(fetchError("browser driver returned no result"));
+        }
+      },
+    );
+  });
+
+export type BrowserRungOptions = {
+  endpoint: string;
+  /** fetchUrl.allowPrivateNetworks: false — see available(). */
+  guard: boolean;
+  corePath?: () => string | undefined;
+  run?: RunBrowserDriver;
+};
+
+export function browserRung(opts: BrowserRungOptions): Rung {
+  const corePath = opts.corePath ?? playwrightCorePath;
+  const run = opts.run ?? runBrowserDriver;
+  return {
+    name: "camoufox",
+    relayed: false,
+    followsRedirects: true,
+    publicOnly: true,
+    timeoutMs: 45_000,
+    async available() {
+      // A page pulls subresources and follows redirects inside the
+      // browser, out of reach of the per-hop SSRF check.
+      if (opts.guard) {
+        return "not used with fetchUrl.allowPrivateNetworks: false (the browser's own requests can't be guarded)";
+      }
+      if (!corePath())
+        return "playwright-core (via @playwright/mcp) is not installed";
+      if (process.versions.bun && !binaryOnPath("node")) {
+        return "node is not on PATH (the browser driver runs under Node)";
+      }
+      return true;
+    },
+    async request(req: RungRequest): Promise<RawResponse> {
+      const core = corePath();
+      if (!core) throw fetchError("playwright-core not installed");
+      const out = await run(
+        [
+          core,
+          opts.endpoint,
+          req.url.href,
+          String(req.timeoutMs),
+          CHALLENGE_PATTERN,
+        ],
+        req.timeoutMs,
+      );
+      const headers = new Headers();
+      for (const [k, v] of Object.entries(out.headers ?? {})) {
+        try {
+          headers.set(k, v);
+        } catch {
+          /* skip headers Headers refuses */
+        }
+      }
+      // The body is the rendered DOM, serialized as UTF-8 HTML.
+      headers.set("content-type", "text/html; charset=utf-8");
+      headers.delete("content-length");
+      headers.delete("content-encoding");
+      headers.delete("location");
+      const body = Buffer.from(out.html ?? "", "utf8");
+      if (body.length > req.maxBytes) throw new ResponseTooLargeError();
+      return {
+        status: out.status || 200,
+        headers,
+        body,
+        url: out.url || req.url.href,
+      };
+    },
+  };
+}
+
+// ── mesh egress device ──────────────────────────────────────────────────────
+
+/** Runs a shell command on a device; resolves with its stdout. */
+export type DeviceExec = (
+  device: string,
+  cmd: string,
+  timeoutMs: number,
+) => Promise<{ ok: boolean; stdout: string; error?: string }>;
+
+const meshExec: DeviceExec = async (device, cmd, timeoutMs) => {
+  const dispatched = await getMeshService().dispatchCommand(
+    device,
+    "exec",
+    { cmd, timeoutMs },
+    timeoutMs + 5_000,
+  );
+  if ("error" in dispatched)
+    return { ok: false, stdout: "", error: dispatched.error };
+  const data = dispatched.result.data ?? {};
+  return {
+    ok: dispatched.result.ok,
+    stdout: typeof data.stdout === "string" ? data.stdout : "",
+    error: dispatched.result.message,
+  };
+};
+
+/** POSIX single-quote a shell word. */
+export function shellQuote(value: string): string {
+  return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+
+const EGRESS_MARK = "__TALON_FETCH__";
+const EGRESS_UA =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.4 Safari/605.1.15";
+
+export function egressRung(device: string, exec: DeviceExec = meshExec): Rung {
+  return {
+    name: `egress:${device}`,
+    relayed: true,
+    followsRedirects: false,
+    publicOnly: true,
+    async available() {
+      return true;
+    },
+    async request(req: RungRequest): Promise<RawResponse> {
+      const secs = Math.max(1, Math.ceil(req.timeoutMs / 1000) - 2);
+      const parts = [
+        "curl -sS --proto =http,https",
+        `--max-time ${secs}`,
+        `--max-filesize ${req.maxBytes}`,
+        `-A ${shellQuote(EGRESS_UA)}`,
+        ...Object.entries(req.headers).map(
+          ([k, v]) => `-H ${shellQuote(`${k}: ${v}`)}`,
+        ),
+        "-o -",
+        `-w ${shellQuote(`\n${EGRESS_MARK} %{http_code} %{redirect_url} %{content_type}`)}`,
+        shellQuote(req.url.href),
+      ];
+      const out = await exec(device, parts.join(" "), req.timeoutMs);
+      const at = out.stdout.lastIndexOf(`\n${EGRESS_MARK} `);
+      if (at < 0) {
+        throw fetchError(
+          out.error || `egress device ${device} returned no HTTP answer`,
+        );
+      }
+      const [code = "0", redirect = "", ...ct] = out.stdout
+        .slice(at + EGRESS_MARK.length + 2)
+        .trim()
+        .split(" ");
+      const status = Number(code);
+      if (!status)
+        throw fetchError(out.error || `curl on ${device} got no HTTP answer`);
+      const headers = new Headers();
+      if (ct.length) headers.set("content-type", ct.join(" "));
+      if (redirect) headers.set("location", redirect);
+      // Device exec carries text; binary bodies do not survive the trip.
+      const body = Buffer.from(out.stdout.slice(0, at), "utf8");
+      return { status, headers, body, url: req.url.href };
+    },
+  };
+}

package/src/core/fetch/types.ts

@@ -0,0 +1,105 @@
+/**
+ * Shared shapes for the fetch ladder (see ladder.ts and docs/fetch-ladder.md).
+ */
+
+/** One HTTP request as a rung sees it. */
+export type RungRequest = {
+  url: URL;
+  /** Extra headers on top of whatever the rung's browser profile sends. */
+  headers: Record<string, string>;
+  timeoutMs: number;
+  /** Hard body cap; a rung must fail rather than buffer more. */
+  maxBytes: number;
+  /**
+   * When the SSRF guard is on: the public addresses the host was checked
+   * against. A rung that can pin the connection to them (curl `--resolve`)
+   * should, closing the DNS-rebinding race the guard otherwise leaves.
+   */
+  pinnedAddresses?: readonly string[];
+  /** Cookie jar shared across the redirect hops of one attempt (curl rungs). */
+  cookieJar?: string;
+};
+
+/** What a rung got back for one hop. */
+export type RawResponse = {
+  status: number;
+  headers: Headers;
+  body: Buffer;
+  /** Final URL (differs from the request only for rungs that follow redirects). */
+  url: string;
+};
+
+export type Rung = {
+  /** Shown to the model and in logs: "impersonate:safari184", "plain", … */
+  readonly name: string;
+  /**
+   * True for rungs whose bytes cross somebody else's link (SOCKS exits, a
+   * mesh device) — they count against `fetch.dailyByteCap`.
+   */
+  readonly relayed: boolean;
+  /**
+   * True when the rung follows redirects itself (a browser, a remote curl).
+   * Otherwise the ladder follows them hop by hop so the guard sees each one.
+   */
+  readonly followsRedirects: boolean;
+  /**
+   * True for rungs that make no sense for a private/loopback target (a
+   * home-lab page has no bot wall, and an exit can't reach it anyway).
+   */
+  readonly publicOnly: boolean;
+  /** Per-attempt timeout override (a browser needs longer than curl). */
+  readonly timeoutMs?: number;
+  /**
+   * Whether the rung can run right now (binary present, endpoint set,
+   * device online). May do one-time setup such as a binary download.
+   * Returning a string means "no" with a reason worth reporting.
+   */
+  available(): Promise<true | string>;
+  request(req: RungRequest): Promise<RawResponse>;
+};
+
+/**
+ * How a response is judged:
+ *  - ok: usable content.
+ *  - blocked: a bot wall (403/429/503/999 or a challenge page) — climb on.
+ *  - network: no HTTP answer at all (timeout, reset, DNS) — climb on.
+ *  - not-found: 404/410 — the URL is wrong, not blocked; stop climbing.
+ *  - http-error: any other definitive HTTP failure — stop climbing.
+ */
+export type Verdict = "ok" | "blocked" | "network" | "not-found" | "http-error";
+
+/** One rung's outcome, as recorded in the attempt log. */
+export type RungOutcome = {
+  via: string;
+  ok: boolean;
+  verdict: Verdict | "skipped";
+  status?: number;
+  bytes?: number;
+  /** Why the rung failed or was skipped. */
+  detail?: string;
+};
+
+type LadderSuccess = {
+  ok: true;
+  via: string;
+  status: number;
+  headers: Headers;
+  body: Buffer;
+  url: string;
+  attempts: RungOutcome[];
+  /** Set when the result deserves a caveat (see FetchLadder.fetch). */
+  note?: string;
+};
+
+type LadderFailure = {
+  ok: false;
+  /** The verdict that ended the climb (last rung's, or the definitive one). */
+  verdict: Verdict | "skipped";
+  status?: number;
+  via?: string;
+  /** Human-readable, honest summary for the tool result. */
+  error: string;
+  attempts: RungOutcome[];
+};
+
+export type LadderResult = LadderSuccess | LadderFailure;

package/src/core/frontend-runtime/admin-notify.ts

@@ -4,9 +4,11 @@
  *
  * Core cannot import frontends, so the composition root injects the
  * delivery function at boot (bootstrap.ts, next to the plan-alerts
- * wiring, which does the same thing privately). Unwired — early boot,
- * tests, terminal mode with no admin — notifications degrade to a log
- * line rather than throwing.
+ * wiring, which does the same thing privately). Alerts raised before that
+ * — early boot is exactly when restore reports and security alerts fire —
+ * are held in a small bounded queue and flushed, oldest first, the moment
+ * a notifier is wired. With nothing ever wired (tests, terminal mode with
+ * no admin) they stay a log line; nothing throws.
  *
  * First consumer: WhatsApp pairing. When WhatsApp unlinks the device,
  * recovery needs a human to type a pairing code into the phone — a code
@@ -16,30 +18,116 @@
 
 import { log, logWarn } from "../../util/log.js";
 
-let deliver: ((text: string) => Promise<void>) | null = null;
+type Deliver = (text: string) => Promise<void>;
 
-/** Wire (or clear) the delivery function. Called by the composition root. */
-export function setAdminNotifier(
-  fn: ((text: string) => Promise<void>) | null,
-): void {
+let deliver: Deliver | null = null;
+
+/** Most alerts held while no notifier is wired; the oldest are dropped past it. */
+export const ADMIN_NOTIFY_QUEUE_MAX = 20;
+
+type Pending = { text: string; at: number };
+const pending: Pending[] = [];
+let droppedWhileUnwired = 0;
+let flushing: Promise<void> | null = null;
+
+/**
+ * Wire (or clear) the delivery function. Called by the composition root.
+ * Wiring a notifier flushes anything queued before it existed, in the
+ * background (the call itself stays synchronous).
+ */
+export function setAdminNotifier(fn: Deliver | null): void {
   deliver = fn;
+  if (fn && (pending.length > 0 || droppedWhileUnwired > 0)) {
+    flushing = flushPending(fn).finally(() => {
+      flushing = null;
+    });
+  }
+}
+
+/** Resolves once any in-flight flush of queued alerts has finished. */
+export function adminNotifyFlushed(): Promise<void> {
+  return flushing ?? Promise.resolve();
+}
+
+/** Drop queued, undelivered alerts (tests; a notifier that will never come). */
+export function clearPendingAdminNotifications(): void {
+  pending.length = 0;
+  droppedWhileUnwired = 0;
+}
+
+/** How many alerts are waiting for a notifier. */
+export function pendingAdminNotificationCount(): number {
+  return pending.length;
+}
+
+async function flushPending(fn: Deliver): Promise<void> {
+  const batch = pending.splice(0, pending.length);
+  const dropped = droppedWhileUnwired;
+  droppedWhileUnwired = 0;
+  if (dropped > 0) {
+    batch.unshift({
+      text: `${dropped} earlier admin alert(s) were dropped before a notifier was wired (queue holds ${ADMIN_NOTIFY_QUEUE_MAX}); see the daemon log.`,
+      at: Date.now(),
+    });
+  }
+  log("notify", `Flushing ${batch.length} queued admin alert(s)`);
+  for (const item of batch) {
+    // The notifier was swapped or cleared mid-flush: requeue the rest for
+    // whichever notifier comes next rather than sending on a stale one.
+    if (deliver !== fn) {
+      enqueue(item);
+      continue;
+    }
+    const ageS = Math.round((Date.now() - item.at) / 1000);
+    const text = ageS >= 5 ? `(delayed ${ageS}s) ${item.text}` : item.text;
+    try {
+      await fn(text);
+      log("notify", `Admin notified (queued): ${preview(item.text)}`);
+    } catch (err) {
+      // No requeue: a notifier that just failed would fail again, and a
+      // retry loop on the alert path is worse than one lost alert.
+      logWarn(
+        "notify",
+        `Queued admin notification failed: ${err instanceof Error ? err.message : err}; dropping: ${item.text.slice(0, 120)}`,
+      );
+    }
+  }
+}
+
+function enqueue(item: Pending): void {
+  pending.push(item);
+  while (pending.length > ADMIN_NOTIFY_QUEUE_MAX) {
+    const lost = pending.shift();
+    droppedWhileUnwired++;
+    if (lost)
+      logWarn(
+        "notify",
+        `Admin alert queue full; dropping oldest: ${lost.text.slice(0, 120)}`,
+      );
+  }
+}
+
+function preview(text: string): string {
+  return text.slice(0, 80).replace(/\n/g, " ");
 }
 
 /**
- * Send `text` to the admin chat. Never throws; returns whether delivery
- * was attempted (false = no notifier wired).
+ * Send `text` to the admin chat. Never throws; returns whether it was
+ * delivered now (false = failed, or queued because no notifier is wired
+ * yet — it is sent when one is).
  */
 export async function notifyAdmin(text: string): Promise<boolean> {
   if (!deliver) {
+    enqueue({ text, at: Date.now() });
     logWarn(
       "notify",
-      `No admin notifier wired; dropping: ${text.slice(0, 120)}`,
+      `No admin notifier wired yet; queued (${pending.length}/${ADMIN_NOTIFY_QUEUE_MAX}): ${text.slice(0, 120)}`,
     );
     return false;
   }
   try {
     await deliver(text);
-    log("notify", `Admin notified: ${text.slice(0, 80).replace(/\n/g, " ")}`);
+    log("notify", `Admin notified: ${preview(text)}`);
     return true;
   } catch (err) {
     log(