pi-unsloth-webtools 0.7.2 → 0.7.4

package/README.md

@@ -42,7 +42,8 @@ Mirrors Unsloth Studio's `web_search` tool:
   `Snippet:` blocks separated by `---`, ending with the hint to pass `{"url": "<URL>"}` to
   read a full page.
 - Accepts an optional `url` parameter; when given, fetches that page's text instead of
-  searching (optionally truncated with `maxChars`).
+  searching (optionally truncated with `maxChars`). An HTTP 403 on that fetch falls back
+  to `web_render` when the tool is enabled.
 - Rate-limit, timeout, and empty-result messages mirror Studio's `_search_failure_message`.
 - Transient engine failures (network errors or null responses) are retried once with a short
   backoff inside the same timeout budget (a retry that cannot fit in the remaining budget is
@@ -80,6 +81,11 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
 - Transient DNS failures (EAI_AGAIN, resolver timeouts, connection refusals) are retried once
   with a short backoff inside the same deadline, so a brief resolver hiccup does not fail a
   fetch; the deadline abort cuts a retry short when no budget remains.
+- Proxy environment variables are honored when they name a SOCKS5 proxy: `HTTPS_PROXY` /
+  `HTTP_PROXY` / `ALL_PROXY` (with `NO_PROXY` exclusions) tunnel the pinned connection, so a
+  Tor-mode agent routes `web_fetch` and `web_search` url mode through its exit. `socks5h` is
+  treated like `socks5`: the host is resolved locally for the guard and the pinned IP is what the
+  proxy connects to. Other proxy schemes are ignored (direct connection).
 - GitHub repo root pages are rewritten to the unauthenticated README API
   (`Accept: application/vnd.github.raw+json`), falling back to the raw README URL
   (`raw.githubusercontent.com`, no API rate limit) and then to the HTML page on failure.
@@ -119,6 +125,9 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   `Date:` (`article:published_time` / `dc.date` / `date`) and `Site:` (`og:site_name` /
   `application-name`) lines are added when declared, so the model can judge recency and
   provenance.
+- A direct fetch refused with HTTP 403 is retried through `web_render` automatically when that
+  tool is enabled; the rendered page is prefixed with a note saying so. When `web_render` is
+  disabled or the render also fails, the original `Failed to fetch URL: HTTP 403 ...` is returned.
 
 ### web_render
 
@@ -137,6 +146,7 @@ that need JavaScript to render:
   [Companion: rotating exit IPs](#companion-rotating-exit-ips).
 - Enabled by default. Disable it with `/webtools-config` (`webRenderEnabled` in
   `~/.config/pi-unsloth-webtools/config.json`), which deactivates the tool for the session.
+- Also used automatically when a `web_fetch` or `web_search` url-mode fetch is refused with HTTP 403.
 
 ## Known differences from Studio
 
@@ -148,6 +158,8 @@ that need JavaScript to render:
 - Third-party rendering: the extra `web_render` tool asks the Jina Reader (`r.jina.ai`) to fetch
   the page, so the target URL leaves the machine. Studio has no third-party rendering path. This
   path always refuses local files and non-public addresses, regardless of the local-access settings.
+  A direct fetch refused with HTTP 403 is retried through it automatically when enabled, so those
+  targets also leave the machine in that case.
 - PDF styling: MuPDF.js exposes one font per line, so mixed-style lines style the
   whole line instead of per-span; superscript, subscript, underline, strikeout, and
   highlight markers are not emitted. Tables use a conservative text-grid detector:
@@ -167,9 +179,11 @@ that need JavaScript to render:
   generic engine failures. The timeout budget bounds the entire sweep: per-engine
   timeouts shrink as the budget is consumed, so the reported timeout matches the
   worst-case wall time.
-- Proxies: Studio routes through environment proxies; this port's direct fetch always connects
-  directly with DNS pinning (deliberately out of scope). The search and `web_render` paths use the
-  process-wide `fetch`, so an agent-level proxy dispatcher does apply to them — see
+- Proxies: Studio routes through environment proxies; this port resolves and pins the target IP and
+  tunnels that connection through `HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY` when the proxy is a
+  SOCKS5 proxy (`NO_PROXY` exclusions respected; DNS stays local for the guard). Other proxy
+  schemes fall back to a direct connection. The search and `web_render` paths use the process-wide
+  `fetch`, so an agent-level proxy dispatcher applies there too — see
   [Companion: rotating exit IPs](#companion-rotating-exit-ips).
 - Dedup and titles: the aggregator keys on canonicalized hrefs (`utm_*`/tracking parameters
   and fragments stripped, then the URL re-serialized); fetched HTML pages are prefixed with
@@ -211,8 +225,10 @@ the current Tor exit, and Jina rate-limits keyless Reader requests per outgoing
 pi install npm:pi-unsloth-webtools npm:pi-tor-proxy
 ```
 
-`web_fetch` is not routed: it connects directly through `node:http`/`node:https` with a pinned,
-validated IP and ignores the proxy variables (see the proxy note under Known differences).
+`web_fetch` and `web_search` url mode also route: they resolve and pin the target IP, then tunnel
+the connection through the SOCKS5 proxy named by `HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY` (with
+`NO_PROXY` exclusions, so localhost and local files stay direct). DNS is still resolved locally for
+the SSRF guard, and the proxy connects to that pinned IP.
 Caveats: Tor mode supports Linux and macOS only, adds latency, and many search engines and
 Cloudflare-fronted services challenge or block Tor exits, so cycling helps with per-IP limits but
 is not a guarantee.
@@ -260,14 +276,14 @@ first changed:
 
 | Key | Default | Description |
 |---|---|---|
-| `webRenderEnabled` | `true` | When `false`, the `web_render` tool is deactivated for the session; `web_search` and `web_fetch` are unaffected |
+| `webRenderEnabled` | `true` | When `false`, the `web_render` tool is deactivated for the session and the automatic HTTP 403 fallback in `web_fetch` / `web_search` url mode is disabled |
 
 On non-Windows platforms the directory honors `XDG_CONFIG_HOME` when set (falling back to
 `~/.config`); on Windows it always uses `~/.config`.
 
 Tool params always win over file defaults. Search dedup also strips default ports, so `https://example.com:443/a` and `https://example.com/a` collapse.
 
-Environment overrides: `PI_UNSLOTH_CACHE_DIR` changes the fetch cache directory, `PI_UNSLOTH_WEBTOOLS_STATS` opts into append-only sweep stats JSONL, `PI_CODING_AGENT_DIR` / `PI_AGENT_DIR` change the global settings directory, and `JINA_API_KEY` supplies the `web_render` key when no settings key is set. Cache entries live 1 hour and stale copies are served only after a network failure.
+Environment overrides: `PI_UNSLOTH_CACHE_DIR` changes the fetch cache directory, `PI_UNSLOTH_WEBTOOLS_STATS` opts into append-only sweep stats JSONL, `PI_CODING_AGENT_DIR` / `PI_AGENT_DIR` change the global settings directory, and `JINA_API_KEY` supplies the `web_render` key when no settings key is set. Cache entries live 1 hour and stale copies are served only after a network failure. SOCKS5 proxies named by `HTTPS_PROXY`, `HTTP_PROXY`, or `ALL_PROXY` are honored on every fetch (`NO_PROXY` exclusions apply).
 
 ## Troubleshooting
 
@@ -284,7 +300,8 @@ Match on the exact prefix. Do not retry blocked hosts with spelling tricks.
 | Private address blocked | `Blocked: refusing to fetch the non-public address ...` | The SSRF guard is active (`allowPrivateAddresses: false`); remove it or set `true` to reach localhost/LAN, and write the scheme explicitly (`http://localhost:3000`). |
 | Local file blocked | `Blocked: the URL has an invalid hostname or port.` for paths | Local files are disabled: remove `allowLocalFiles: false` to read `file://`, absolute, `~/`, or `./` paths. |
 | File read failed | `Failed to read file: ...` | Check the path exists and is a regular file. |
-| HTTP failure | `Failed to fetch URL: HTTP ...` | Fix the URL. A 404 automatically tries a Wayback snapshot. |
+| HTTP failure | `Failed to fetch URL: HTTP ...` | Fix the URL. A 404 automatically tries a Wayback snapshot; a 403 retries through `web_render` when enabled. |
+| Proxy failure | `Failed to fetch URL: SOCKS5 proxy ...` | The SOCKS5 proxy refused or failed (for example Tor is stopping). Check the proxy, or unset the proxy variables for a direct fetch. |
 | Non-text / binary | `(non-text content:` / `(binary content,` | Not readable as text by design. |
 | PDF without text | `(PDF contains no extractable text)` / `(PDF content could not be read as text...)` | Scanned or encrypted PDF. |
 | Download cap hit | `... (page truncated at the download limit)` | Raw fetch hit 512 KiB (10 MiB for PDFs). |

package/index.ts

@@ -6,7 +6,7 @@ import { SEARCH_TIMEOUT_MS, webSearch as defaultWebSearch } from "./web-search.t
 import { DEFAULT_FETCH_TIMEOUT_MS, fetchPageText as defaultFetchPageText } from "./web-fetch.ts";
 import { renderPageText as defaultRenderPageText } from "./web-render.ts";
 import { loadDefaultFetchSettings, loadDefaultFetchTimeoutMs, loadJinaApiKey } from "./settings.ts";
-import { readConfigWithStatus, toggleWebRender } from "./config.ts";
+import { readConfig, readConfigWithStatus, toggleWebRender } from "./config.ts";
 import { WebToolsConfigOverlay } from "./config-ui.ts";
 
 function toolCallLine(theme: Theme, name: string, detail: string) {
@@ -35,6 +35,12 @@ async function fetchDefaults(cwd: string | undefined, params: { timeoutMs?: unkn
   };
 }
 
+function fetchWasForbidden(text: string): boolean {
+  return /^Failed to fetch URL: HTTP 403\b/m.test(text);
+}
+
+const FORBIDDEN_FALLBACK_NOTE = "Direct fetch failed with HTTP 403; rendered via the Jina Reader instead.";
+
 const WebSearchParams = Type.Object({
   query: Type.Optional(
     Type.String({ description: "The search query" }),
@@ -100,19 +106,46 @@ export interface WebToolsDeps {
   fetchPageText?: typeof defaultFetchPageText;
   webSearch?: typeof defaultWebSearch;
   renderPageText?: typeof defaultRenderPageText;
+  webRenderEnabled?: () => Promise<boolean>;
 }
 
 export function createWebTools(deps: WebToolsDeps = {}) {
   const fetchPageText = deps.fetchPageText ?? defaultFetchPageText;
   const webSearch = deps.webSearch ?? defaultWebSearch;
   const renderPageText = deps.renderPageText ?? defaultRenderPageText;
+  const webRenderEnabled =
+    deps.webRenderEnabled ?? (async () => (await readConfig()).webRenderEnabled !== false);
+
+  const renderOnForbidden = async (
+    url: string,
+    failedText: string,
+    options: { timeoutMs: number; maxChars?: number; signal?: AbortSignal; cwd?: string },
+  ): Promise<string | null> => {
+    if (!fetchWasForbidden(failedText)) return null;
+    try {
+      if (!(await webRenderEnabled())) return null;
+      const apiKey = await loadJinaApiKey(options.cwd);
+      const rendered = await renderPageText(url, {
+        timeoutMs: options.timeoutMs,
+        maxChars: options.maxChars,
+        signal: options.signal,
+        apiKey,
+      });
+      if (rendered.startsWith("Failed to render URL:") || rendered.startsWith("Blocked:")) return null;
+      return FORBIDDEN_FALLBACK_NOTE + "\n\n" + rendered;
+    } catch {
+      return null;
+    }
+  };
+
   return {
     webSearchTool: defineTool({
       name: "web_search",
       label: "Web Search",
       description:
         "Search the web and fetch page content. Returns snippets for all results. " +
-        "Use the url parameter to fetch full page text from a specific URL.",
+        "Use the url parameter to fetch full page text from a specific URL. " +
+        "A direct fetch refused with HTTP 403 falls back to web_render when that tool is enabled.",
       promptSnippet: "Search the web and fetch page content",
       promptGuidelines: [
         'Use web_search with the url parameter (e.g. {"url": "<URL>"}) to read the full text of a page found in search results.',
@@ -129,21 +162,20 @@ export function createWebTools(deps: WebToolsDeps = {}) {
           onUpdate?.({ content: [{ type: "text", text: `Fetching ${url}...` }], details: {} });
           const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
           const { timeoutMs, maxChars, allowPrivateAddresses, allowLocalFiles } = await fetchDefaults(cwd, params);
-          return {
-            content: [
-              {
-                type: "text",
-                text: await fetchPageText(url, {
-                  timeoutMs,
-                  signal: signal ?? undefined,
-                  maxChars,
-                  allowPrivateAddresses,
-                  allowLocalFiles,
-                }),
-              },
-            ],
-            details: {},
-          };
+          const text = await fetchPageText(url, {
+            timeoutMs,
+            signal: signal ?? undefined,
+            maxChars,
+            allowPrivateAddresses,
+            allowLocalFiles,
+          });
+          const rendered = await renderOnForbidden(url, text, {
+            timeoutMs,
+            maxChars,
+            signal: signal ?? undefined,
+            cwd,
+          });
+          return { content: [{ type: "text", text: rendered ?? text }], details: {} };
         }
         onUpdate?.({ content: [{ type: "text", text: "Searching the web..." }], details: {} });
         const timeoutParam = positiveNumber(params.timeoutMs);
@@ -168,7 +200,11 @@ export function createWebTools(deps: WebToolsDeps = {}) {
         "README API, so the README is returned instead of the repo page's UI chrome. " +
         "Private/loopback/link-local targets and local files (file:// URLs, absolute, ~/ or ./ paths, including " +
         "PDFs) are supported by default; opt out with webFetch.allowPrivateAddresses: false or " +
-        "webFetch.allowLocalFiles: false in settings. The download size is capped.",
+        "webFetch.allowLocalFiles: false in settings. The download size is capped. " +
+        "A direct fetch refused with HTTP 403 falls back to web_render (Jina Reader) when that tool is enabled.",
+      promptGuidelines: [
+        "web_fetch automatically retries HTTP 403 responses through web_render (Jina Reader); do not call web_render again for the same URL after a 403.",
+      ],
       promptSnippet: "Fetch a web page and return readable text content",
       parameters: WebFetchParams,
       renderCall(args, theme) {
@@ -185,7 +221,13 @@ export function createWebTools(deps: WebToolsDeps = {}) {
           allowPrivateAddresses,
           allowLocalFiles,
         });
-        return { content: [{ type: "text", text }], details: {} };
+        const rendered = await renderOnForbidden(params.url, text, {
+          timeoutMs,
+          maxChars,
+          signal: signal ?? undefined,
+          cwd,
+        });
+        return { content: [{ type: "text", text: rendered ?? text }], details: {} };
       },
     }),
     webRenderTool: defineTool({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.7.2",
+  "version": "0.7.4",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools that began as a port of the Unsloth Studio codebase and now diverge from it (multi-engine search, opt-in SSRF guard, HTML-to-Markdown extraction)",
   "main": "index.ts",
@@ -33,6 +33,7 @@
     "engines.ts",
     "entities.ts",
     "pdf.ts",
+    "proxy.ts",
     "user-agents.ts",
     "cache.ts",
     "settings.ts",

package/proxy.ts

@@ -0,0 +1,413 @@
+import http from "node:http";
+import https from "node:https";
+import net from "node:net";
+import tls from "node:tls";
+import type { Duplex } from "node:stream";
+
+export interface SocksProxy {
+  host: string;
+  port: number;
+  username: string | null;
+  password: string | null;
+}
+
+export interface TunnelOptions {
+  proxy: SocksProxy;
+  ip: string;
+  family: number;
+  port: number;
+  servername: string;
+  timeoutMs: number;
+  signal?: AbortSignal;
+}
+
+interface Budget {
+  timeoutMs: number;
+  signal?: AbortSignal;
+}
+
+const HTTPS_PROXY_VARS = ["HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"];
+const HTTP_PROXY_VARS = ["HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy"];
+const NO_PROXY_VARS = ["NO_PROXY", "no_proxy"];
+const SOCKS_PROTOCOLS = new Set(["socks5:", "socks5h:"]);
+const DEFAULT_SOCKS_PORT = 1080;
+const SOCKS_VERSION = 0x05;
+const SOCKS_AUTH_NONE = 0x00;
+const SOCKS_AUTH_USER_PASSWORD = 0x02;
+const SOCKS_AUTH_REJECTED = 0xff;
+const SOCKS_COMMAND_CONNECT = 0x01;
+const SOCKS_ATYP_IPV4 = 0x01;
+const SOCKS_ATYP_DOMAIN = 0x03;
+const SOCKS_ATYP_IPV6 = 0x04;
+const SOCKS_REPLY_MESSAGES: Record<number, string> = {
+  1: "general failure",
+  2: "connection not allowed",
+  3: "network unreachable",
+  4: "host unreachable",
+  5: "connection refused",
+  6: "TTL expired",
+  7: "command not supported",
+  8: "address type not supported",
+};
+
+function firstEnv(names: readonly string[]): string | null {
+  for (const name of names) {
+    const value = process.env[name];
+    if (typeof value === "string" && value.trim()) return value.trim();
+  }
+  return null;
+}
+
+function decodeCredential(value: string): string {
+  try {
+    return decodeURIComponent(value);
+  } catch {
+    return value;
+  }
+}
+
+export function parseSocksProxy(value: string | null | undefined): SocksProxy | null {
+  if (!value) return null;
+  let parsed: URL;
+  try {
+    parsed = new URL(value);
+  } catch {
+    return null;
+  }
+  if (!SOCKS_PROTOCOLS.has(parsed.protocol)) return null;
+  if (!parsed.hostname) return null;
+  const port = parsed.port ? Number(parsed.port) : DEFAULT_SOCKS_PORT;
+  if (!Number.isInteger(port) || port < 1 || port > 65535) return null;
+  const username = parsed.username ? decodeCredential(parsed.username) : "";
+  const password = parsed.password ? decodeCredential(parsed.password) : "";
+  return { host: parsed.hostname, port, username: username || null, password: password || null };
+}
+
+function noProxyEntries(): string[] {
+  const raw = firstEnv(NO_PROXY_VARS);
+  if (!raw) return [];
+  return raw.split(/[\s,]+/).filter((entry) => entry.length > 0);
+}
+
+function splitHostPort(entry: string): { host: string; port: number | null } {
+  if (entry.startsWith("[")) {
+    const end = entry.indexOf("]");
+    if (end === -1) return { host: entry, port: null };
+    const host = entry.slice(1, end);
+    const rest = entry.slice(end + 1);
+    if (rest.startsWith(":") && /^\d+$/.test(rest.slice(1))) return { host, port: Number(rest.slice(1)) };
+    return { host, port: null };
+  }
+  const colon = entry.lastIndexOf(":");
+  if (colon > 0 && entry.indexOf(":") === colon && /^\d+$/.test(entry.slice(colon + 1))) {
+    return { host: entry.slice(0, colon), port: Number(entry.slice(colon + 1)) };
+  }
+  return { host: entry, port: null };
+}
+
+function bypassesProxy(hostname: string, port: number): boolean {
+  const host = hostname.toLowerCase();
+  for (const rawEntry of noProxyEntries()) {
+    const entry = rawEntry.toLowerCase();
+    if (entry === "*") return true;
+    const { host: rawHost, port: entryPort } = splitHostPort(entry);
+    if (entryPort !== null && entryPort !== port) continue;
+    const pattern = rawHost.replace(/^\./, "").replace(/\.$/, "");
+    if (!pattern) continue;
+    if (host === pattern) return true;
+    if (!net.isIP(pattern) && host.endsWith(`.${pattern}`)) return true;
+  }
+  return false;
+}
+
+export function socksProxyForUrl(url: URL): SocksProxy | null {
+  const proxy = parseSocksProxy(firstEnv(url.protocol === "https:" ? HTTPS_PROXY_VARS : HTTP_PROXY_VARS));
+  if (!proxy) return null;
+  const port = url.port ? Number(url.port) : url.protocol === "https:" ? 443 : 80;
+  if (bypassesProxy(url.hostname, port)) return null;
+  return proxy;
+}
+
+function readExact(socket: net.Socket, length: number, budget: Budget): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    if (budget.signal?.aborted) {
+      reject(new Error("cancelled"));
+      return;
+    }
+    let settled = false;
+    let timer: NodeJS.Timeout | undefined;
+    function cleanup(): void {
+      if (timer) clearTimeout(timer);
+      budget.signal?.removeEventListener("abort", onAbort);
+      socket.removeListener("readable", onReadable);
+      socket.removeListener("error", onError);
+      socket.removeListener("close", onClose);
+    }
+    function finish(error: Error | null, data?: Buffer): void {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      if (error) {
+        socket.destroy();
+        reject(error);
+      } else {
+        resolve(data!);
+      }
+    }
+    function onReadable(): void {
+      const chunk = socket.read(length) as Buffer | null;
+      if (chunk === null) return;
+      finish(null, chunk);
+    }
+    function onError(error: Error): void {
+      finish(error);
+    }
+    function onClose(): void {
+      finish(new Error("SOCKS5 proxy closed the connection"));
+    }
+    function onAbort(): void {
+      finish(new Error("cancelled"));
+    }
+    timer = setTimeout(() => finish(new Error("timed out")), budget.timeoutMs);
+    budget.signal?.addEventListener("abort", onAbort, { once: true });
+    socket.on("readable", onReadable);
+    socket.on("error", onError);
+    socket.on("close", onClose);
+    onReadable();
+  });
+}
+
+function writeAll(socket: net.Socket, data: Buffer, budget: Budget): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (budget.signal?.aborted) {
+      reject(new Error("cancelled"));
+      return;
+    }
+    let settled = false;
+    let timer: NodeJS.Timeout | undefined;
+    function cleanup(): void {
+      if (timer) clearTimeout(timer);
+      budget.signal?.removeEventListener("abort", onAbort);
+      socket.removeListener("error", onError);
+    }
+    function finish(error: Error | null): void {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      if (error) {
+        socket.destroy();
+        reject(error);
+      } else {
+        resolve();
+      }
+    }
+    function onError(error: Error): void {
+      finish(error);
+    }
+    function onAbort(): void {
+      finish(new Error("cancelled"));
+    }
+    timer = setTimeout(() => finish(new Error("timed out")), budget.timeoutMs);
+    budget.signal?.addEventListener("abort", onAbort, { once: true });
+    socket.once("error", onError);
+    socket.write(data, (error: Error | null | undefined) => finish(error ?? null));
+  });
+}
+
+function connectProxy(proxy: SocksProxy, budget: Budget): Promise<net.Socket> {
+  return new Promise((resolve, reject) => {
+    if (budget.signal?.aborted) {
+      reject(new Error("cancelled"));
+      return;
+    }
+    const socket = net.connect({ host: proxy.host, port: proxy.port });
+    let settled = false;
+    let timer: NodeJS.Timeout | undefined;
+    function cleanup(): void {
+      if (timer) clearTimeout(timer);
+      budget.signal?.removeEventListener("abort", onAbort);
+      socket.removeListener("connect", onConnect);
+      socket.removeListener("error", onError);
+      socket.removeListener("timeout", onTimeout);
+      socket.setTimeout(0);
+    }
+    function finish(error: Error | null): void {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      if (error) {
+        socket.destroy();
+        reject(error);
+      } else {
+        resolve(socket);
+      }
+    }
+    function onConnect(): void {
+      finish(null);
+    }
+    function onError(error: Error): void {
+      finish(error);
+    }
+    function onTimeout(): void {
+      finish(new Error("timed out"));
+    }
+    function onAbort(): void {
+      finish(new Error("cancelled"));
+    }
+    socket.once("connect", onConnect);
+    socket.once("error", onError);
+    socket.setTimeout(budget.timeoutMs, onTimeout);
+    budget.signal?.addEventListener("abort", onAbort, { once: true });
+    timer = setTimeout(() => finish(new Error("timed out")), budget.timeoutMs);
+  });
+}
+
+async function authenticate(socket: net.Socket, proxy: SocksProxy, budget: Budget): Promise<void> {
+  const username = Buffer.from(proxy.username ?? "", "utf8");
+  const password = Buffer.from(proxy.password ?? "", "utf8");
+  if (username.length > 255 || password.length > 255) throw new Error("SOCKS5 proxy credentials are too long");
+  await writeAll(
+    socket,
+    Buffer.concat([Buffer.from([0x01, username.length]), username, Buffer.from([password.length]), password]),
+    budget,
+  );
+  const reply = await readExact(socket, 2, budget);
+  if (reply[0] !== 0x01 || reply[1] !== 0x00) throw new Error("SOCKS5 proxy authentication failed");
+}
+
+function ipv4Bytes(ip: string): Buffer {
+  const parts = ip.split(".");
+  if (parts.length !== 4) throw new Error(`Invalid IPv4 address: ${ip}`);
+  const bytes = Buffer.alloc(4);
+  parts.forEach((part, index) => {
+    const value = Number(part);
+    if (!Number.isInteger(value) || value < 0 || value > 255) throw new Error(`Invalid IPv4 address: ${ip}`);
+    bytes[index] = value;
+  });
+  return bytes;
+}
+
+function ipv6Bytes(ip: string): Buffer {
+  const halves = ip.split("::");
+  if (halves.length > 2) throw new Error(`Invalid IPv6 address: ${ip}`);
+  const toGroups = (part: string) => (part ? part.split(":") : []);
+  const head = toGroups(halves[0]);
+  const tail = halves.length === 2 ? toGroups(halves[1]) : [];
+  for (const group of [...head, ...tail]) {
+    if (!/^[0-9a-fA-F]{1,4}$/.test(group)) throw new Error(`Invalid IPv6 address: ${ip}`);
+  }
+  const missing = 8 - head.length - tail.length;
+  if (halves.length === 2 ? missing < 1 : missing !== 0) throw new Error(`Invalid IPv6 address: ${ip}`);
+  const groups = [...head, ...Array.from({ length: missing }, () => "0"), ...tail];
+  const bytes = Buffer.alloc(16);
+  groups.forEach((group, index) => bytes.writeUInt16BE(Number.parseInt(group, 16), index * 2));
+  return bytes;
+}
+
+function addressBytes(ip: string, family: number): { atyp: number; bytes: Buffer } {
+  if (family === 6 || ip.includes(":")) return { atyp: SOCKS_ATYP_IPV6, bytes: ipv6Bytes(ip) };
+  return { atyp: SOCKS_ATYP_IPV4, bytes: ipv4Bytes(ip) };
+}
+
+async function sendConnect(socket: net.Socket, options: TunnelOptions, budget: Budget): Promise<void> {
+  const address = addressBytes(options.ip, options.family);
+  const port = Buffer.alloc(2);
+  port.writeUInt16BE(options.port);
+  await writeAll(
+    socket,
+    Buffer.concat([Buffer.from([SOCKS_VERSION, SOCKS_COMMAND_CONNECT, 0x00, address.atyp]), address.bytes, port]),
+    budget,
+  );
+  const head = await readExact(socket, 4, budget);
+  if (head[0] !== SOCKS_VERSION) throw new Error("SOCKS5 proxy returned an invalid version");
+  if (head[1] !== 0x00) {
+    const reason = SOCKS_REPLY_MESSAGES[head[1]] ?? `reply ${head[1]}`;
+    throw new Error(`SOCKS5 proxy connect failed: ${reason}`);
+  }
+  const atyp = head[3];
+  if (atyp === SOCKS_ATYP_IPV4) {
+    await readExact(socket, 6, budget);
+  } else if (atyp === SOCKS_ATYP_IPV6) {
+    await readExact(socket, 18, budget);
+  } else if (atyp === SOCKS_ATYP_DOMAIN) {
+    const length = (await readExact(socket, 1, budget))[0];
+    await readExact(socket, length + 2, budget);
+  } else {
+    throw new Error("SOCKS5 proxy returned an invalid address type");
+  }
+}
+
+async function socksHandshake(socket: net.Socket, options: TunnelOptions, budget: Budget): Promise<void> {
+  const methods = options.proxy.username !== null ? [SOCKS_AUTH_USER_PASSWORD] : [SOCKS_AUTH_NONE];
+  await writeAll(socket, Buffer.from([SOCKS_VERSION, methods.length, ...methods]), budget);
+  const greeting = await readExact(socket, 2, budget);
+  if (greeting[0] !== SOCKS_VERSION) throw new Error("SOCKS5 proxy returned an invalid version");
+  const method = greeting[1];
+  if (method === SOCKS_AUTH_REJECTED) throw new Error("SOCKS5 proxy rejected all authentication methods");
+  if (method === SOCKS_AUTH_USER_PASSWORD) {
+    await authenticate(socket, options.proxy, budget);
+  } else if (method !== SOCKS_AUTH_NONE) {
+    throw new Error(`SOCKS5 proxy selected an unsupported authentication method (${method})`);
+  }
+  await sendConnect(socket, options, budget);
+}
+
+export async function openTunnel(options: TunnelOptions): Promise<net.Socket> {
+  const budget: Budget = { timeoutMs: options.timeoutMs, signal: options.signal };
+  const socket = await connectProxy(options.proxy, budget);
+  try {
+    await socksHandshake(socket, options, budget);
+    return socket;
+  } catch (error) {
+    socket.destroy();
+    throw error;
+  }
+}
+
+function tlsWrap(socket: net.Socket, servername: string): tls.TLSSocket {
+  if (net.isIP(servername)) return tls.connect({ socket, host: servername });
+  return tls.connect({ socket, servername });
+}
+
+function tunnelCreateConnection(
+  open: () => Promise<Duplex>,
+  callback?: (err: Error | null, stream: Duplex) => void,
+): Duplex | null | undefined {
+  if (typeof callback !== "function") return undefined;
+  open().then(
+    (stream) => callback(null, stream),
+    (error: unknown) => callback(error instanceof Error ? error : new Error(String(error)), undefined as unknown as Duplex),
+  );
+  return undefined;
+}
+
+export function tunnelAgent(url: URL, options: TunnelOptions): http.Agent {
+  const open = (): Promise<Duplex> =>
+    openTunnel(options).then((socket) => (url.protocol === "https:" ? tlsWrap(socket, options.servername) : socket));
+  if (url.protocol === "https:") {
+    class SecureTunnelAgent extends https.Agent {
+      constructor() {
+        super({ keepAlive: false });
+      }
+      override createConnection(
+        _options: http.ClientRequestArgs,
+        callback?: (err: Error | null, stream: Duplex) => void,
+      ): Duplex | null | undefined {
+        return tunnelCreateConnection(open, callback);
+      }
+    }
+    return new SecureTunnelAgent();
+  }
+  class PlainTunnelAgent extends http.Agent {
+    constructor() {
+      super({ keepAlive: false });
+    }
+    override createConnection(
+      _options: http.ClientRequestArgs,
+      callback?: (err: Error | null, stream: Duplex) => void,
+    ): Duplex | null | undefined {
+      return tunnelCreateConnection(open, callback);
+    }
+  }
+  return new PlainTunnelAgent();
+}

package/web-fetch.ts

@@ -26,6 +26,7 @@ import type { AttrDict } from "./html-to-md.ts";
 import { INVALID_CHARREFS } from "./entities.ts";
 import { getCached, isFresh, setCached, staleNotice } from "./cache.ts";
 import { extractPdfText } from "./pdf.ts";
+import { socksProxyForUrl, tunnelAgent } from "./proxy.ts";
 import { randomUserAgent } from "./user-agents.ts";
 
 const MAX_FETCH_BYTES = 512 * 1024;
@@ -635,23 +636,37 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
   return new Promise((resolve, reject) => {
     const url = opts.url;
     const transport = url.protocol === "https:" ? https : http;
+    const port = url.port ? Number(url.port) : url.protocol === "https:" ? 443 : 80;
     const options: https.RequestOptions = {
       method: "GET",
       host: url.hostname,
-      port: url.port ? Number(url.port) : url.protocol === "https:" ? 443 : 80,
+      port,
       path: url.pathname + url.search,
       headers: opts.headers,
       timeout: opts.inactivityMs,
-      servername: url.protocol === "https:" ? url.hostname : undefined,
-      lookup: ((_hostname: string, _options: unknown, _callback: unknown) => {
+    };
+    const proxy = socksProxyForUrl(url);
+    if (proxy) {
+      options.agent = tunnelAgent(url, {
+        proxy,
+        ip: opts.pinnedIp,
+        family: opts.family,
+        port,
+        servername: url.hostname,
+        timeoutMs: Math.max(1, opts.inactivityMs),
+        signal: opts.signal,
+      });
+    } else {
+      options.servername = url.protocol === "https:" ? url.hostname : undefined;
+      options.lookup = ((_hostname: string, _options: unknown, _callback: unknown) => {
         const done = (typeof _options === "function" ? _options : _callback) as (err: unknown, address: unknown, family?: unknown) => void;
         if ((_options as { all?: boolean } | undefined)?.all) {
           done(null, [{ address: opts.pinnedIp, family: opts.family }]);
         } else {
           done(null, opts.pinnedIp, opts.family);
         }
-      }) as typeof options.lookup,
-    };
+      }) as typeof options.lookup;
+    }
     let settled = false;
     let resRef: IncomingMessage | null = null;
     let decoderRef: Transform | null = null;
@@ -898,6 +913,8 @@ export async function fetchUrlRaw(
         signal,
       });
     } catch (err) {
+      const message = err instanceof Error ? err.message : "";
+      if (message === "cancelled" || message === "timed out") return emptyResult(fetchErrorMessage(err));
       if (
         !(err instanceof FetchCancelledError) &&
         !(err instanceof FetchTimeoutError) &&