pi-unsloth-webtools 0.2.2 → 0.2.4

package/README.md

@@ -26,7 +26,7 @@ Mirrors Unsloth Studio's `web_search` tool:
 
 - Searches exactly like Studio's pinned `ddgs==9.14.4` `DDGS.text()`: the same seven engines
   (duckduckgo, brave, google, mojeek, yahoo, yandex, wikipedia; bing is disabled upstream),
-  the same provider-deduplication, href-dedupe aggregator with frequency ordering, and the
+  the same provider deduplication, href-dedupe aggregator with frequency ordering, and the
   same `SimpleFilterRanker` re-ranking. Formats results identically: `Title:` / `URL:` /
   `Snippet:` blocks separated by `---`, ending with the hint to pass `{"url": "<URL>"}` to
   read a full page.
@@ -45,7 +45,7 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   validation and fetch.
 - GitHub repo root pages are rewritten to the unauthenticated README API
   (`Accept: application/vnd.github.raw+json`), falling back to the HTML page on failure.
-- Up to 5 redirect hops, each re-validated and re-resolved against the same rules.
+- Up to 4 redirect hops, each re-validated and re-resolved against the same rules.
 - 512 KiB download cap (10 MiB for PDFs), overall deadline + per-hop socket timeouts, abort-aware
   (`signal` cancels mid-flight).
 - PDF text extraction via the official MuPDF.js engine (the same C library pymupdf wraps):
@@ -67,19 +67,19 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
 
 ## Known differences from Studio
 
-- **PDF styling**: MuPDF.js exposes one font per line, so mixed-style lines style the
-  whole line instead of per-span; superscript/subscript/underline/strikeout/highlight
-  markers are not emitted. Tables use a conservative text-grid detector — aligned text
-  tables are detected, drawn-rule-only tables are not.
-- **Search engines**: Node's `fetch` TLS fingerprint differs from ddgs's `primp`
+- PDF styling: MuPDF.js exposes one font per line, so mixed-style lines style the
+  whole line instead of per-span; superscript, subscript, underline, strikeout, and
+  highlight markers are not emitted. Tables use a conservative text-grid detector:
+  aligned text tables are detected, drawn-rule-only tables are not.
+- Search engines: Node's `fetch` TLS fingerprint differs from ddgs's `primp`
   impersonation, so Google/Brave/Yahoo/Yandex may block or serve consent pages more
   aggressively (a blocked engine simply contributes no results). User agents are a
   fixed browser set plus ddgs's Android Google UA generator, not `fake_useragent`'s
   database.
-- **Empty sweeps**: ddgs 9.14.4 raises the last engine exception; this port reports a
+- Empty sweeps: ddgs 9.14.4 raises the last engine exception; this port reports a
   timeout whenever any engine timed out, so the timeout message is not masked by later
   generic engine failures.
-- **Proxies**: Studio routes through environment proxies; this port always connects
+- Proxies: Studio routes through environment proxies; this port always connects
   directly with DNS pinning (deliberately out of scope).
 
 ## Development
@@ -99,23 +99,23 @@ and `npm run test:smoke` runs only those.
 
 The suite ports Unsloth Studio's own tests for these tools:
 
-- `test/html-to-md.test.ts` — hidden-element stripping and main-content scoping (from
+- `test/html-to-md.test.ts`: hidden-element stripping and main-content scoping (from
   `test_web_fetch_extraction.py`)
-- `test/header-strip.test.ts` — the header link-density suite plus article-vs-main selection
+- `test/header-strip.test.ts`: the header link-density suite plus article-vs-main selection
   and boilerplate cases (from `test_web_fetch_extraction.py`)
-- `test/binary-guard.test.ts` — the MIME/magic/charset/PDF matrix (from
+- `test/binary-guard.test.ts`: the MIME/magic/charset/PDF matrix (from
   `test_web_fetch_binary_guard.py`)
-- `test/web-search-policy.test.ts` — policy filtering, overfetch, and failure messages
+- `test/web-search-policy.test.ts`: policy filtering, overfetch, and failure messages
   (from `test_web_access_policy.py`)
-- `test/fetch-flow.test.ts` — GitHub README rewrite, deadline/cancellation, HTML sniffing
+- `test/fetch-flow.test.ts`: GitHub README rewrite, deadline/cancellation, HTML sniffing
   (from `test_web_fetch_extraction.py`; the fetch client is injected via seams)
-- `test/engines.test.ts` — the ddgs engine port: normalizers, the XPath subset, the
+- `test/engines.test.ts`: the ddgs engine port, normalizers, the XPath subset, the
   aggregator, the ranker, and the Wikipedia engine with a stubbed fetch
-- `test/pdf-parity.test.ts` — MuPDF engine capabilities: PDF 1.5 object streams,
+- `test/pdf-parity.test.ts`: MuPDF engine capabilities, PDF 1.5 object streams,
   ASCII85Decode, font `/Differences` encodings, pymupdf4llm-style headings/links/tables
-- `test/entities.test.ts` — `decodeHtmlEntities` parity with CPython `html.unescape`: legacy
-  refs, longest-prefix rule, Windows-1252 numeric mappings, invalid codepoints
-- `test/smoke.test.ts` — live network checks against real hosts
+- `test/entities.test.ts`: `decodeHtmlEntities` parity with CPython `html.unescape`,
+  legacy refs, longest-prefix rule, Windows-1252 numeric mappings, invalid codepoints
+- `test/smoke.test.ts`: live network checks against real hosts
 
 The seams (`seams.resolve` / `seams.request` / `rawFetch`) replace the network stack
 with fakes, mirroring how the Studio suite monkeypatches `_validate_and_resolve_host`
@@ -125,4 +125,4 @@ and `build_opener`.
 
 The ported logic derives from Unsloth Studio
 ([AGPL-3.0-only](https://github.com/unslothai/unsloth/blob/main/studio/LICENSE.AGPL-3.0)), so this
-package is released under the same **AGPL-3.0-only** license.
+package is released under the same AGPL-3.0-only license.

package/engines.ts

@@ -1,6 +1,7 @@
 import { randomBytes } from "node:crypto";
 import { decodeHtmlEntities, feedHtml } from "./html-to-md.ts";
 import type { AttrDict } from "./html-to-md.ts";
+import { randomUserAgent } from "./user-agents.ts";
 export class EmptySweepError extends Error {
   constructor() {
     super("No results found");
@@ -110,6 +111,32 @@ interface XStep {
   preds: Pred[];
   terminal?: "text" | string;
 }
+function parsePredicateBlocks(input: string, start: number): { preds: Pred[]; next: number } {
+  const preds: Pred[] = [];
+  let pos = start;
+  while (pos < input.length && input[pos] === "[") {
+    const innerStart = pos + 1;
+    let depth = 1;
+    let quote: string | null = null;
+    let j = innerStart;
+    while (j < input.length && depth) {
+      const c = input[j];
+      if (quote !== null) {
+        if (c === quote) quote = null;
+      } else if (c === "'" || c === '"') {
+        quote = c;
+      } else if (c === "[") {
+        depth++;
+      } else if (c === "]") {
+        depth--;
+      }
+      j++;
+    }
+    preds.push(parsePredExpr(input.slice(innerStart, j - 1)));
+    pos = j;
+  }
+  return { preds, next: pos };
+}
 
 function parsePredExpr(input: string): Pred {
   let pos = 0;
@@ -173,35 +200,10 @@ function parsePredExpr(input: string): Pred {
       return { op: "desc", tag: name };
     }
     const name = word();
-    const preds = parsePredBlocks();
+    const { preds, next } = parsePredicateBlocks(input, pos);
+    pos = next;
     return { op: "child", tag: name, preds };
   };
-  const parsePredBlocks = (): Pred[] => {
-    const preds: Pred[] = [];
-    while (pos < input.length && input[pos] === "[") {
-      const start = pos + 1;
-      let depth = 1;
-      let quote: string | null = null;
-      let i = start;
-      while (i < input.length && depth) {
-        const c = input[i];
-        if (quote !== null) {
-          if (c === quote) quote = null;
-        } else if (c === "'" || c === '"') {
-          quote = c;
-        } else if (c === "[") {
-          depth++;
-        } else if (c === "]") {
-          depth--;
-        }
-        i++;
-      }
-      const inner = input.slice(start, i - 1);
-      preds.push(parsePredExpr(inner));
-      pos = i;
-    }
-    return preds;
-  };
   const parseAnd = (): Pred => {
     let left = atom();
     while (true) {
@@ -274,28 +276,8 @@ function parsePath(expr: string): XStep[] {
     if (!m) break;
     const name = m[0];
     i += m[0].length;
-    const preds: Pred[] = [];
-    while (i < expr.length && expr[i] === "[") {
-      const start = i + 1;
-      let depth = 1;
-      let quote: string | null = null;
-      let j = start;
-      while (j < expr.length && depth) {
-        const c = expr[j];
-        if (quote !== null) {
-          if (c === quote) quote = null;
-        } else if (c === "'" || c === '"') {
-          quote = c;
-        } else if (c === "[") {
-          depth++;
-        } else if (c === "]") {
-          depth--;
-        }
-        j++;
-      }
-      preds.push(parsePredExpr(expr.slice(start, j - 1)));
-      i = j;
-    }
+    const { preds, next } = parsePredicateBlocks(expr, i);
+    i = next;
     steps.push({ axis, name, preds });
   }
   return steps;
@@ -409,19 +391,6 @@ export function extractResults(
   return results;
 }
 
-const USER_AGENTS = [
-  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15",
-];
-
-function randomUserAgent(): string {
-  return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
-}
-
 function googleUserAgent(): string {
   const devices: [string, string, number, number][] = [
     ["5.0", "SM-G900P Build/LRX21T", 39, 60],
@@ -503,6 +472,28 @@ async function httpPost(
   return httpFetch(url, { ...options, method: "POST", body: new URLSearchParams(data).toString() });
 }
 
+const MAX_ENGINE_RESPONSE_BYTES = 5 * 1024 * 1024;
+
+async function readBodyCapped(response: Response): Promise<string | null> {
+  const declared = Number(response.headers.get("content-length") ?? "0");
+  if (declared > MAX_ENGINE_RESPONSE_BYTES) return null;
+  if (!response.body) return "";
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    total += value.length;
+    if (total > MAX_ENGINE_RESPONSE_BYTES) {
+      await reader.cancel();
+      return null;
+    }
+    chunks.push(value);
+  }
+  return new TextDecoder("utf-8").decode(Buffer.concat(chunks));
+}
+
 async function httpFetch(
   url: string,
   options: {
@@ -536,13 +527,18 @@ async function httpFetch(
       signal: AbortSignal.any(signals),
     });
   } catch (err) {
-    if (err instanceof DOMException && err.name === "TimeoutError") {
-      throw new Error("timed out");
-    }
+    if (err instanceof DOMException && err.name === "TimeoutError") throw new SearchTimeoutError();
+    if (err instanceof DOMException && err.name === "AbortError") throw new SearchCancelled();
     throw err;
   }
   if (response.status !== 200) return null;
-  return response.text();
+  try {
+    return await readBodyCapped(response);
+  } catch (err) {
+    if (err instanceof DOMException && err.name === "TimeoutError") throw new SearchTimeoutError();
+    if (err instanceof DOMException && err.name === "AbortError") throw new SearchCancelled();
+    throw err;
+  }
 }
 
 const DUCKDUCKGO: Engine = {
@@ -823,6 +819,7 @@ export async function autoTextSearch(
   const aggregator = new ResultsAggregator();
   const ctx: EngineContext = { region: "us-en", safesearch: "moderate" };
   let timedOut = false;
+  let cancelled = false;
   const uniqueProviders = new Set(engines.map((e) => e.provider)).size;
   const maxWorkers = Math.min(uniqueProviders, Math.ceil(maxResults / 10) + 1);
   let i = 0;
@@ -835,7 +832,8 @@ export async function autoTextSearch(
         seenProviders.add(engine.provider);
       }
     } catch (e) {
-      if (e instanceof Error && e.message.includes("timed out")) timedOut = true;
+      if (e instanceof SearchCancelled) cancelled = true;
+      if (e instanceof SearchTimeoutError) timedOut = true;
     }
   };
   while (i < engines.length) {
@@ -843,11 +841,13 @@ export async function autoTextSearch(
     const engine = engines[i++];
     if (seenProviders.has(engine.provider)) continue;
     pending.push(run(engine));
-    if (pending.length >= maxWorkers || i >= maxWorkers) {
+    if (pending.length >= maxWorkers) {
       await Promise.allSettled(pending);
       pending = [];
     }
   }
+  await Promise.allSettled(pending);
+  if (cancelled) throw new SearchCancelled();
   const results = rankResults(aggregator.extractDicts(), query);
   if (results.length) return results.slice(0, maxResults);
   if (timedOut) throw new SearchTimeoutError();

package/index.ts

@@ -62,12 +62,12 @@ export default function (pi: ExtensionAPI) {
     name: "web_fetch",
     label: "Web Fetch",
     description:
-      "Fetch a URL and return readable text content. HTML responses are converted to Markdown " +
-      "with a main-content heuristic (article/main scoping, hidden-element and boilerplate " +
-      "stripping); non-HTML text responses are returned as-is. GitHub repo root pages are " +
-      "rewritten to the README API so the model reads the README instead of the repo page's UI " +
-      "chrome. Blocks private/loopback/link-local targets (SSRF protection) and caps the " +
-      "download size.",
+      "Fetch a URL and return its readable text. HTML pages are converted to Markdown using a " +
+      "main-content heuristic: article/main scoping plus hidden-element and boilerplate " +
+      "stripping. Non-HTML text is returned as-is. GitHub repo root pages are rewritten to the " +
+      "README API, so the README is returned instead of the repo page's UI chrome. " +
+      "Private/loopback/link-local targets are blocked (SSRF protection), and the download size " +
+      "is capped.",
     promptSnippet: "Fetch a web page and return readable text content",
     parameters: WebFetchParams,
     async execute(_toolCallId, params, signal, _onUpdate, _ctx) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.2.2",
+  "version": "0.2.4",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools ported from the Unsloth Studio codebase (DuckDuckGo search, SSRF-safe direct fetching, HTML-to-Markdown extraction)",
   "main": "index.ts",
@@ -30,6 +30,7 @@
     "engines.ts",
     "entities.ts",
     "pdf.ts",
+    "user-agents.ts",
     "README.md",
     "LICENSE"
   ],

package/pdf.ts

@@ -527,7 +527,7 @@ function assemblePages(
     return "";
   }
   if (pageLimitReached) {
-    text += `\n\n... (PDF extraction page processing capped at ${MAX_WEB_PDF_PAGES} pages)`;
+    text += `\n\n... (PDF extraction is capped at ${MAX_WEB_PDF_PAGES} pages)`;
   }
   return text;
 }

package/user-agents.ts

@@ -0,0 +1,12 @@
+const USER_AGENTS = [
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15",
+];
+
+export function randomUserAgent(): string {
+  return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
+}

package/web-access.ts

@@ -173,46 +173,46 @@ export function checkUrlAccess(
   policy: WebsitePolicy | null,
 ): [boolean, string, string] {
   if (typeof url !== "string" || !url.trim()) {
-    return [false, "Blocked: URL is empty.", ""];
+    return [false, "Blocked: the URL is empty.", ""];
   }
   const candidate = url.trim();
   if (
     Array.from(candidate).some((char) => /\s/.test(char) || char.charCodeAt(0) < 32) ||
     candidate.includes("\\")
   ) {
-    return [false, "Blocked: URL contains invalid characters.", ""];
+    return [false, "Blocked: the URL contains invalid characters.", ""];
   }
   let parsed: URL;
   try {
     parsed = new URL(candidate);
   } catch {
-    return [false, "Blocked: URL has an invalid hostname or port.", ""];
+    return [false, "Blocked: the URL has an invalid hostname or port.", ""];
   }
   const scheme = parsed.protocol.replace(/:$/, "").toLowerCase();
   if (scheme !== "http" && scheme !== "https") {
     return [false, "Blocked: only http/https URLs are allowed.", ""];
   }
   if (parsed.username || parsed.password || parsed.hostname.includes("%")) {
-    return [false, "Blocked: URL credentials or encoded hostnames are not allowed.", ""];
+    return [false, "Blocked: URLs with credentials or encoded hostnames are not allowed.", ""];
   }
   if (!parsed.hostname) {
-    return [false, "Blocked: URL has an invalid hostname or port.", ""];
+    return [false, "Blocked: the URL has an invalid hostname or port.", ""];
   }
   try {
     if (parsed.port && !(PORT_RE.test(parsed.port) && Number(parsed.port) >= 1 && Number(parsed.port) <= 65535)) {
-      return [false, "Blocked: URL has an invalid hostname or port.", ""];
+      return [false, "Blocked: the URL has an invalid hostname or port.", ""];
     }
   } catch {
-    return [false, "Blocked: URL has an invalid hostname or port.", ""];
+    return [false, "Blocked: the URL has an invalid hostname or port.", ""];
   }
   let hostname: string;
   try {
     hostname = normalizeDomain(parsed.hostname);
   } catch {
-    return [false, "Blocked: URL has an invalid hostname or port.", ""];
+    return [false, "Blocked: the URL has an invalid hostname or port.", ""];
   }
   if (!hostnameAllowed(hostname, policy)) {
-    return [false, `Blocked: website access policy disallows ${hostname}.`, hostname];
+    return [false, `Blocked: the website access policy disallows ${hostname}.`, hostname];
   }
   return [true, "", hostname];
 }
@@ -368,6 +368,8 @@ export function isPublicIp(ip: string): boolean {
   if (lower.startsWith("2001:db8")) return false;
   if (lower.startsWith("64:ff9b:")) return false;
   if (lower.startsWith("2001:10:")) return false;
+  if (lower.startsWith("2002:")) return false;
+  if (lower.startsWith("2001:0:") || lower.startsWith("2001::")) return false;
   const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(lower);
   if (mapped) return isPublicIp(mapped[1]);
   if (lower.startsWith("::ffff:")) return false;

package/web-fetch.ts

@@ -10,21 +10,13 @@ import {
   type WebsitePolicy,
 } from "./web-access.ts";
 import { htmlToMarkdown } from "./html-to-md.ts";
+import { INVALID_CHARREFS } from "./entities.ts";
 import { extractPdfText, PdfParseError } from "./pdf.ts";
+import { randomUserAgent } from "./user-agents.ts";
 
-const MIN_PAGE_CHARS = 2000;
 const MAX_FETCH_BYTES = 512 * 1024;
 const MAX_PDF_FETCH_BYTES = 10 * 1024 * 1024;
-const MAX_REDIRECTS = 5;
-
-const USER_AGENTS = [
-  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15",
-];
+const MAX_REQUESTS = 5;
 
 const UTF32_LE_BOM = Buffer.from([0xff, 0xfe, 0x00, 0x00]);
 const UTF32_BE_BOM = Buffer.from([0x00, 0x00, 0xfe, 0xff]);
@@ -106,14 +98,17 @@ const ASCII_TEXT_BYTES = new Set<number>([
   0x1b,
 ]);
 
-const CP1252_HIGH: Record<number, number> = {
-  0x80: 0x20ac, 0x82: 0x201a, 0x83: 0x0192, 0x84: 0x201e, 0x85: 0x2026,
-  0x86: 0x2020, 0x87: 0x2021, 0x88: 0x02c6, 0x89: 0x2030, 0x8a: 0x0160,
-  0x8b: 0x2039, 0x8c: 0x0152, 0x8e: 0x017d, 0x91: 0x2018, 0x92: 0x2019,
-  0x93: 0x201c, 0x94: 0x201d, 0x95: 0x2022, 0x96: 0x2013, 0x97: 0x2014,
-  0x98: 0x02dc, 0x99: 0x2122, 0x9a: 0x0161, 0x9b: 0x203a, 0x9c: 0x0153,
-  0x9e: 0x017e, 0x9f: 0x0178,
-};
+export class FetchCancelledError extends Error {
+  constructor() {
+    super("cancelled");
+  }
+}
+
+export class FetchTimeoutError extends Error {
+  constructor() {
+    super("timed out");
+  }
+}
 
 export interface FetchPageOptions {
   timeoutMs?: number;
@@ -154,6 +149,8 @@ export interface HopOptions {
   maxBytes: number;
   maxPdfBytes: number;
   inactivityMs: number;
+  deadlineMs?: number;
+  nowMs?: () => number;
   signal?: AbortSignal;
 }
 
@@ -348,8 +345,8 @@ function decodeSingleByte(bytes: Buffer, cp1252: boolean): string {
   for (const byte of bytes) {
     if (byte < 0x80) {
       out += String.fromCharCode(byte);
-    } else if (cp1252 && byte in CP1252_HIGH) {
-      out += String.fromCodePoint(CP1252_HIGH[byte]);
+    } else if (cp1252 && byte in INVALID_CHARREFS) {
+      out += INVALID_CHARREFS[byte];
     } else {
       out += String.fromCharCode(byte);
     }
@@ -421,7 +418,7 @@ async function resolveAndValidate(hostname: string, signal?: AbortSignal): Promi
   }
   for (const entry of addresses) {
     if (!isPublicIp(entry.address)) {
-      return { ok: false, reason: `Blocked: refusing to fetch non-public address ${entry.address}.`, ip: "", family: 0 };
+      return { ok: false, reason: `Blocked: refusing to fetch the non-public address ${entry.address}.`, ip: "", family: 0 };
     }
   }
   const first = addresses[0];
@@ -440,7 +437,7 @@ function fetchBudgetExceeded(
 }
 
 
-function requestHop(opts: HopOptions): Promise<HopResponse> {
+export function requestHop(opts: HopOptions): Promise<HopResponse> {
   return new Promise((resolve, reject) => {
     const url = opts.url;
     const transport = url.protocol === "https:" ? https : http;
@@ -455,6 +452,13 @@ function requestHop(opts: HopOptions): Promise<HopResponse> {
       lookup: (_host, _opts, callback) =>
         callback(null, [{ address: opts.pinnedIp, family: opts.family }]),
     };
+    let settled = false;
+    const settle = (action: () => void) => {
+      if (settled) return;
+      settled = true;
+      opts.signal?.removeEventListener("abort", onAbort);
+      action();
+    };
     const request = transport.request(options, (res: IncomingMessage) => {
       const chunks: Buffer[] = [];
       let total = 0;
@@ -463,17 +467,27 @@ function requestHop(opts: HopOptions): Promise<HopResponse> {
       let extendedForPdf = false;
       const finish = (err: string | null, body: Buffer) => {
         settle(() => {
-          if (err) reject(new Error(err));
-          else
+          if (err) {
+            if (err === "cancelled") reject(new FetchCancelledError());
+            else if (err === "timed out") reject(new FetchTimeoutError());
+            else reject(new Error(err));
+          } else {
             resolve({
               status: res.statusCode ?? 0,
               headers: res.headers as Record<string, string | string[] | undefined>,
               body,
             });
+          }
         });
       };
       res.on("data", (chunk: Buffer) => {
         if (settled) return;
+        const now = opts.nowMs ?? Date.now;
+        if (opts.deadlineMs !== undefined && now() >= opts.deadlineMs) {
+          res.destroy();
+          finish("timed out", Buffer.concat(chunks));
+          return;
+        }
         if (!declaredPdf && !extendedForPdf && total + chunk.length > opts.maxBytes) {
           if (hasPdfMagic(Buffer.concat(chunks))) {
             limit = opts.maxPdfBytes;
@@ -497,16 +511,9 @@ function requestHop(opts: HopOptions): Promise<HopResponse> {
       res.on("end", () => finish(null, Buffer.concat(chunks)));
       res.on("error", (err) => finish(err.message, Buffer.concat(chunks)));
     });
-    const onAbort = () => request.destroy(new Error("cancelled"));
+    const onAbort = () => request.destroy(new FetchCancelledError());
     opts.signal?.addEventListener("abort", onAbort, { once: true });
-    let settled = false;
-    const settle = (action: () => void) => {
-      if (settled) return;
-      settled = true;
-      opts.signal?.removeEventListener("abort", onAbort);
-      action();
-    };
-    request.on("timeout", () => request.destroy(new Error("timed out")));
+    request.on("timeout", () => request.destroy(new FetchTimeoutError()));
     request.on("error", (err) => settle(() => reject(err)));
     request.end();
   });
@@ -541,9 +548,9 @@ export async function fetchUrlRaw(
   let currentUrl = url;
   let pinnedIp = resolved.ip;
   let pinnedFamily = resolved.family;
-  const userAgent = USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
+  const userAgent = randomUserAgent();
 
-  for (let hop = 0; hop < MAX_REDIRECTS; hop++) {
+  for (let hop = 0; hop < MAX_REQUESTS; hop++) {
     budgetError = fetchBudgetExceeded(deadline, signal, now);
     if (budgetError !== null) return { error: budgetError, body: "", contentType: "" };
     const parsed = new URL(currentUrl);
@@ -566,9 +573,15 @@ export async function fetchUrlRaw(
         maxBytes,
         maxPdfBytes,
         inactivityMs: inactivity,
+        deadlineMs: deadline,
+        nowMs: now,
         signal,
       });
     } catch (err) {
+      if (err instanceof FetchCancelledError)
+        return { error: "Failed to fetch URL: cancelled.", body: "", contentType: "" };
+      if (err instanceof FetchTimeoutError)
+        return { error: "Failed to fetch URL: timed out.", body: "", contentType: "" };
       const message = err instanceof Error ? err.message : String(err);
       if (message === "cancelled")
         return { error: "Failed to fetch URL: cancelled.", body: "", contentType: "" };
@@ -590,12 +603,16 @@ export async function fetchUrlRaw(
       const location = Array.isArray(rawLocation) ? rawLocation[0] : rawLocation;
       if (!location) {
         return {
-          error: "Failed to fetch URL: redirect missing Location header.",
+          error: "Failed to fetch URL: the redirect is missing a Location header.",
           body: "",
           contentType: "",
         };
       }
-      currentUrl = new URL(location, currentUrl).toString();
+      try {
+        currentUrl = new URL(location, currentUrl).toString();
+      } catch {
+        return { error: "Failed to fetch URL: the redirect has an invalid Location.", body: "", contentType: "" };
+      }
       const [redirectAllowed, redirectReason, redirectHost] = checkUrlAccess(
         currentUrl,
         policy,

package/web-search.ts

@@ -82,7 +82,7 @@ export async function webSearch(
 export function searchFailureMessage(exc: unknown, timeoutMs = SEARCH_TIMEOUT_MS): string {
   if (exc instanceof SearchCancelled) return "Search cancelled.";
   if (exc instanceof SearchTimeoutError) {
-    return `Search failed: the search engines did not respond within ${Math.round(timeoutMs / 1000)}s.`;
+    return `Search failed: the search engines did not respond within ${Math.round(timeoutMs / 1000)} seconds.`;
   }
   if (exc instanceof EmptySweepError || (exc instanceof Error && exc.message.includes("No results found"))) {
     return EMPTY_SEARCH_RESULTS[0];
@@ -100,7 +100,7 @@ export function formatSearchResults(results: SearchResult[]): string {
   const text = parts.join("\n\n---\n\n");
   return (
     text +
-    "\n\n---\n\nIMPORTANT: These are only short snippets. " +
+    "\n\n---\n\nThese are only short snippets. " +
     'To get the full page content, call web_search with the url parameter (e.g. {"url": "<URL>"}).'
   );
 }