talon-agent 5.24.0 → 5.24.1

package/src/core/engine/gateway-actions/fetch-url/guard.ts

@@ -7,15 +7,16 @@
  * credentials), the loopback gateway, a router admin page on the LAN.
  * So before every request — the first one and each redirect hop — the
  * host is resolved and EVERY address it resolves to must be public.
- * Redirects are followed by hand (never by fetch) so a public page
- * cannot bounce the request into a private one.
+ * Redirects are followed by hand (never by the transport) in the fetch
+ * ladder (core/fetch/ladder.ts), which calls `assertPublicUrl` on every
+ * hop, so a public page cannot bounce the request into a private one.
  *
- * Residual risk, stated plainly: fetch resolves the name again after we
- * checked it, so a DNS-rebinding server with a zero TTL can still race
- * us. Pinning the checked address would need a custom connector, which
- * Bun (one of the two runtimes) does not honour. The guard closes the
- * direct, redirect and static-DNS paths; rebinding needs a hostile DNS
- * server and a lucky race.
+ * Residual risk, stated plainly: the runtime's fetch (the "plain" rung)
+ * resolves the name again after we checked it, so a DNS-rebinding server
+ * with a zero TTL can still race it. Pinning the checked address would
+ * need a custom connector, which Bun does not honour. The curl rungs do
+ * pin (`--resolve` to the checked addresses); requests through a SOCKS
+ * exit are resolved by the exit, outside this host's network.
  *
  * The guard is opt-in: `fetch_url` applies it only when the operator sets
  * `fetchUrl.allowPrivateNetworks: false`. By default the agent can read
@@ -33,9 +34,6 @@ const defaultResolver: Resolver = async (host) =>
 
 export class BlockedUrlError extends Error {}
 
-const MAX_REDIRECTS = 5;
-const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
-
 // ── Address classification ──────────────────────────────────────────────────
 
 /** Non-public IPv4 ranges: [network, prefix length]. */
@@ -135,11 +133,13 @@ export function isBlockedAddress(ip: string): boolean {
 /**
  * Throw unless `url` is http(s) and its host resolves only to public
  * addresses. A literal IP is judged directly; a name is resolved.
+ * Returns the checked addresses so a caller that can pin the connection
+ * to them (curl `--resolve`) closes the rebinding race.
  */
 export async function assertPublicUrl(
   url: URL,
   resolve: Resolver = defaultResolver,
-): Promise<void> {
+): Promise<string[]> {
   if (url.protocol !== "http:" && url.protocol !== "https:") {
     throw new BlockedUrlError("URL must use http or https protocol");
   }
@@ -166,36 +166,5 @@ export async function assertPublicUrl(
         `Remove fetchUrl.allowPrivateNetworks: false from config.json (or set it to true) to allow local addresses.`,
     );
   }
-}
-
-export type GuardedFetchOptions = {
-  allowPrivateNetworks?: boolean;
-  resolve?: Resolver;
-  maxRedirects?: number;
-};
-
-/**
- * `fetch` with the guard applied to the first request and to every
- * redirect hop. Returns the final (non-redirect) response.
- */
-export async function guardedFetch(
-  input: string,
-  init: RequestInit,
-  options: GuardedFetchOptions = {},
-): Promise<Response> {
-  const maxRedirects = options.maxRedirects ?? MAX_REDIRECTS;
-  let url = new URL(input);
-  for (let hop = 0; ; hop++) {
-    if (!options.allowPrivateNetworks) {
-      await assertPublicUrl(url, options.resolve);
-    }
-    const resp = await fetch(url, { ...init, redirect: "manual" });
-    const location = resp.headers.get("location");
-    if (!REDIRECT_STATUSES.has(resp.status) || !location) return resp;
-    if (hop >= maxRedirects) {
-      throw new BlockedUrlError(`Too many redirects (max ${maxRedirects})`);
-    }
-    await resp.body?.cancel().catch(() => {});
-    url = new URL(location, url);
-  }
+  return addresses;
 }

package/src/core/engine/gateway-actions/fetch-url/index.ts

@@ -6,10 +6,6 @@
 import { randomUUID } from "node:crypto";
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { resolve } from "node:path";
-import {
-  readBodyLimited,
-  ResponseTooLargeError,
-} from "../../../../util/http-body.js";
 import {
   advertisedBinaryKind,
   decodeText,
@@ -22,7 +18,9 @@ import {
 import { dirs } from "../../../../util/paths.js";
 import { getPoolConfig } from "../../backend-controller/index.js";
 import type { SharedActionHandlers } from "../types.js";
-import { BlockedUrlError, guardedFetch } from "./guard.js";
+import { logDebug } from "../../../../util/log.js";
+import { buildFetchLadder } from "../../../fetch/index.js";
+import { BlockedUrlError } from "./guard.js";
 
 const MAX_RESPONSE_MB = 50;
 const MAX_RESPONSE_BYTES = MAX_RESPONSE_MB * 1024 * 1024;
@@ -69,55 +67,37 @@ function textResult(
   return { ok: true, text: capText(text) };
 }
 
+/** Footer naming the rung that got the page, plus any caveat. */
+function footer(via: string, note?: string): string {
+  return `\n\n[fetched via ${via}${note ? ` — ${note}` : ""}]`;
+}
+
 export const fetchUrlHandlers: SharedActionHandlers = {
   fetch_url: async (body) => {
     const url = String(body.url ?? "");
     const invalid = urlError(url);
     if (invalid) return { ok: false, error: invalid };
     try {
+      // The ladder (core/fetch) climbs browser-TLS impersonation → SOCKS
+      // exits → plain fetch → browser → egress device until one answers
+      // with content, and stops early on a definitive answer (a 404).
       // Local addresses are reachable by default; with
       // `fetchUrl.allowPrivateNetworks: false` every hop is checked against
       // private/loopback/link-local ranges (see guard.ts).
-      const resp = await guardedFetch(
-        url,
-        {
-          signal: AbortSignal.timeout(15_000),
-          headers: { "User-Agent": "Talon/1.0" },
-        },
-        {
-          allowPrivateNetworks:
-            getPoolConfig()?.fetchUrl?.allowPrivateNetworks !== false,
-        },
-      );
-      if (!resp.ok) return { ok: false, error: `HTTP ${resp.status}` };
-      const ct = resp.headers.get("content-type") ?? "";
-
-      // Reject oversized responses before downloading the body.
-      // The Content-Length header is advisory but saves bandwidth when present.
-      const contentLength = resp.headers.get("content-length");
-      if (contentLength && Number(contentLength) > MAX_RESPONSE_BYTES) {
-        return {
-          ok: false,
-          error: `File too large (${(Number(contentLength) / 1024 / 1024).toFixed(0)}MB, max ${MAX_RESPONSE_MB}MB)`,
-        };
-      }
+      const ladder = buildFetchLadder(getPoolConfig(), {
+        maxBytes: MAX_RESPONSE_BYTES,
+      });
+      const result = await ladder.fetch(url);
+      if (!result.ok) return { ok: false, error: result.error };
+      logDebug("fetch", `fetch via ${result.via} (${new URL(url).host})`);
+      const via = footer(result.via, result.note);
+      const ct = result.headers.get("content-type") ?? "";
+      const buffer = result.body;
 
       const mimeType = ct.split(";")[0].trim().toLowerCase();
-      let buffer: Buffer;
-      try {
-        buffer = await readBodyLimited(resp, MAX_RESPONSE_BYTES);
-      } catch (err) {
-        if (err instanceof ResponseTooLargeError) {
-          return {
-            ok: false,
-            error: `Response too large (max ${MAX_RESPONSE_MB}MB)`,
-          };
-        }
-        throw err;
-      }
-
       if (isTextContent(mimeType, buffer)) {
-        return textResult(mimeType, buffer, ct);
+        const text = textResult(mimeType, buffer, ct);
+        return { ...text, text: text.text + via };
       }
 
       if (buffer.length === 0)
@@ -148,7 +128,7 @@ export const fetchUrlHandlers: SharedActionHandlers = {
         : (detected?.ext ?? ct.split("/")[1]?.split(";")[0] ?? "file");
       return {
         ok: true,
-        text: `Downloaded ${typeLabel} (${(buffer.length / 1024).toFixed(0)}KB) to: ${filePath}\nRead it with the Read tool or send it with send(type="file", file_path="${filePath}").`,
+        text: `Downloaded ${typeLabel} (${(buffer.length / 1024).toFixed(0)}KB) to: ${filePath}\nRead it with the Read tool or send it with send(type="file", file_path="${filePath}").${via}`,
       };
     } catch (err) {
       if (err instanceof BlockedUrlError) {

package/src/core/engine/gateway-actions/mesh.ts

@@ -91,6 +91,9 @@ export const meshHandlers: SharedActionHandlers = {
       body.name,
       body.bridge_url,
     ),
+  // Companion pairing: the agent-tool twin of Telegram's `/mesh pair`.
+  make_companion_pair_link: async (body) =>
+    getMeshService().makeCompanionPairLinkText(body.label, body.bridge_url),
 };
 
 /**
@@ -124,4 +127,5 @@ export const chatFreeActions: ReadonlySet<string> = new Set([
   "update_node",
   "get_node_binary",
   "make_node_install_link",
+  "make_companion_pair_link",
 ]);

package/src/core/engine/gateway-actions/models.ts

@@ -18,10 +18,21 @@ import {
   collectBackendUsage,
   formatHeadroom,
   leadWith,
+  type BackendHeadroom,
   type BackendUsageSnapshot,
 } from "../backend-router/index.js";
 import type { SharedActionHandlers } from "./types.js";
 
+/**
+ * Headroom as agents see it, to 2dp. `null` for an unmeasured backend: the
+ * router scores those 0 so they are never preferred, but telling an agent
+ * "0" would read as "spent", which is not known either.
+ */
+function headroomFigure(entry: BackendHeadroom): number | null {
+  if (entry.source === "none" && !entry.unavailable) return null;
+  return Math.round(entry.headroom * 100) / 100;
+}
+
 /** One `plan_usage` block: the headroom line, then any plan windows. */
 function usageLines(entry: BackendUsageSnapshot): string[] {
   const head = `- ${entry.label || entry.id}: ${formatHeadroom(entry.headroom)}`;
@@ -143,10 +154,13 @@ export const modelHandlers: SharedActionHandlers = {
         id: entry.id,
         label: entry.label,
         current: entry.id === currentId,
-        headroom: Math.round(entry.headroom.headroom * 100) / 100,
+        headroom: headroomFigure(entry.headroom),
         source: entry.headroom.source,
         limiting: entry.headroom.limiting ?? null,
         stale: entry.headroom.stale ?? false,
+        ...(entry.headroom.unavailable
+          ? { unavailable: entry.headroom.unavailable }
+          : {}),
         plan: entry.plan?.plan ?? null,
         windows: entry.plan?.windows ?? [],
         ...(entry.note ? { note: entry.note } : {}),
@@ -170,10 +184,11 @@ export const modelHandlers: SharedActionHandlers = {
         id: b.id,
         label: b.label,
         current: b.id === currentId,
-        headroom: entry
-          ? Math.round(entry.headroom.headroom * 100) / 100
-          : null,
+        headroom: entry ? headroomFigure(entry.headroom) : null,
         headroomSource: entry?.headroom.source ?? null,
+        ...(entry?.headroom.unavailable
+          ? { unavailable: entry.headroom.unavailable }
+          : {}),
       };
     });
     const lines = backends.map((b) => {

package/src/core/engine/gateway-routes.ts

@@ -320,16 +320,45 @@ export async function dispatchGatewayRoute(
 
 const PORT_RETRIES = 5;
 
+/**
+ * Asked before walking past a busy port. Resolves a reason to refuse the
+ * fallback (the bind then fails with it), or `undefined` to keep walking.
+ */
+export type PortInUseCheck = (busyPort: number) => Promise<string | undefined>;
+
 /**
  * Bind to 127.0.0.1, walking up from `port` on EADDRINUSE (at most
  * `PORT_RETRIES` times). Resolves with the port actually bound — `port`
  * 0 asks the OS for a free one — and leaves a persistent error handler
  * on the server, so a later server-level error is logged instead of
  * crashing the process via an unhandled 'error' event.
+ *
+ * `onPortInUse` can veto the walk: the daemon uses it to fail outright when
+ * the busy port belongs to another Talon daemon, rather than settling on
+ * the next port and running as a second instance.
  */
-export function listenWithRetry(server: Server, port: number): Promise<number> {
+export function listenWithRetry(
+  server: Server,
+  port: number,
+  onPortInUse?: PortInUseCheck,
+): Promise<number> {
   return new Promise<number>((resolve, reject) => {
     let attempt = 0;
+    const walkOn = async (
+      candidate: number,
+      err: NodeJS.ErrnoException,
+    ): Promise<void> => {
+      const refusal = onPortInUse
+        ? await onPortInUse(candidate).catch(() => undefined)
+        : undefined;
+      if (refusal) {
+        reject(
+          Object.assign(new Error(refusal), { code: err.code, cause: err }),
+        );
+        return;
+      }
+      tryPort(candidate + 1);
+    };
     const tryPort = (candidate: number): void => {
       server.once("error", (err: NodeJS.ErrnoException) => {
         if (err.code === "EADDRINUSE" && attempt < PORT_RETRIES) {
@@ -339,7 +368,7 @@ export function listenWithRetry(server: Server, port: number): Promise<number> {
           // registered; drop it or every stale callback fires when a later
           // port finally binds.
           server.removeAllListeners("listening");
-          tryPort(candidate + 1);
+          void walkOn(candidate, err);
         } else {
           reject(err);
         }

package/src/core/engine/gateway.ts

@@ -30,6 +30,7 @@ import { getHubSessionCount } from "../mcp-hub/index.js";
 import {
   dispatchGatewayRoute,
   listenWithRetry,
+  type PortInUseCheck,
   type GatewayRouteHost,
 } from "./gateway-routes.js";
 import { gatewayToken } from "./gateway-auth.js";
@@ -471,6 +472,15 @@ export class Gateway {
    * A failed bind clears the in-flight state, so a later `start()` retries;
    * so does `stop()`, so start-after-stop binds afresh.
    */
+  /**
+   * Veto for the EADDRINUSE port walk (see `listenWithRetry`). The daemon
+   * installs one that refuses when the busy port is another daemon.
+   */
+  setPortInUseCheck(check: PortInUseCheck | undefined): void {
+    this.portInUseCheck = check;
+  }
+  private portInUseCheck: PortInUseCheck | undefined;
+
   async start(port = 19876): Promise<number> {
     if (this.server) return this.port;
     if (this.starting) return this.starting;
@@ -509,7 +519,7 @@ export class Gateway {
     const httpServer = createServer(
       (req, res) => void dispatchGatewayRoute(req, res, host),
     );
-    const bound = await listenWithRetry(httpServer, port);
+    const bound = await listenWithRetry(httpServer, port, this.portInUseCheck);
     this.server = httpServer;
     this.port = bound;
     log("gateway", `Action gateway on :${this.port}`);

package/src/core/fetch/classify.ts

@@ -0,0 +1,73 @@
+/**
+ * Response classification for the fetch ladder: is this content, a bot
+ * wall worth climbing past, or a definitive answer (a wrong URL) that no
+ * amount of climbing will change?
+ *
+ * Getting "blocked" vs "not found" right is the whole point: climbing on a
+ * 404 hammers a site through every exit for nothing and then reports a
+ * block that never happened; stopping on a 403 challenge gives up on a
+ * page that the next rung would have fetched.
+ */
+
+import type { Verdict } from "./types.js";
+
+/** Statuses bot walls answer with. 999 is LinkedIn's. */
+const BLOCK_STATUSES = new Set([403, 429, 503, 999]);
+const NOT_FOUND_STATUSES = new Set([404, 410]);
+
+/**
+ * Challenge pages are small; a real article that merely mentions
+ * "captcha" is not. Only bodies under this size are scanned.
+ */
+const CHALLENGE_SCAN_BYTES = 32 * 1024;
+
+/**
+ * Markers of interstitial challenge pages (Cloudflare, DataDome, Akamai,
+ * Imperva/Incapsula, PerimeterX). Deliberately no bare "captcha" or
+ * reCAPTCHA marker: plenty of ordinary pages embed a form captcha. A 2xx
+ * page flagged here is kept as a fallback by the ladder, so a false
+ * positive costs extra rungs, never the content.
+ */
+const CHALLENGE_MARKERS: readonly RegExp[] = [
+  // Not "challenge-platform": Cloudflare injects that script path into
+  // ordinary pages too. Same for the DataDome/PerimeterX tag scripts —
+  // only their challenge iframes/widgets count.
+  /cf-chl-|cf_chl_opt|<title>Just a moment\.\.\.<\/title>/i,
+  /Attention Required! \| Cloudflare/i,
+  /captcha-delivery\.com/i,
+  /_Incapsula_Resource|Incapsula incident/i,
+  /px-captcha/i,
+  /Pardon Our Interruption/i,
+  /<title>Access Denied<\/title>[\s\S]*Reference #/i,
+  /verify (that )?you are (a )?human|are you a robot/i,
+  /challenges\.cloudflare\.com\/turnstile/i,
+];
+
+/** All markers as one regex source (for the browser driver subprocess). */
+export const CHALLENGE_PATTERN = CHALLENGE_MARKERS.map(
+  (re) => `(?:${re.source})`,
+).join("|");
+
+/** True when a small body looks like a bot-check interstitial. */
+export function looksLikeChallenge(body: Buffer | string): boolean {
+  const size = typeof body === "string" ? body.length : body.byteLength;
+  if (size === 0 || size > CHALLENGE_SCAN_BYTES) return false;
+  const text = typeof body === "string" ? body : body.toString("latin1");
+  return CHALLENGE_MARKERS.some((re) => re.test(text));
+}
+
+/** Judge one HTTP answer. Network failures never reach here. */
+export function classifyResponse(status: number, body: Buffer): Verdict {
+  if (BLOCK_STATUSES.has(status)) return "blocked";
+  if (looksLikeChallenge(body)) return "blocked";
+  if (status >= 200 && status < 300) return "ok";
+  if (NOT_FOUND_STATUSES.has(status)) return "not-found";
+  return "http-error";
+}
+
+/** Verdicts after which trying another rung is pointless. */
+export function isDefinitive(verdict: Verdict): boolean {
+  return (
+    verdict === "ok" || verdict === "not-found" || verdict === "http-error"
+  );
+}