pi-unsloth-webtools 0.5.0 → 0.6.0

package/README.md

@@ -50,11 +50,19 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
 - URL validation: http/https only, no credentials or encoded hostnames, hostname/port checks (any
   port 1–65535 is permitted; SSRF protection is enforced at the resolved-IP layer, not by port
   allowlists).
-  Canonical public IPv4 literals are accepted like IPv6 literals; private literals are still
-  blocked at the resolved-IP layer (non-canonical numeric encodings like `0x7f.0.0.1` /
-  `013.0.0.1` / `2130706433` are rejected by URL validation before DNS).
+  Canonical public IPv4 literals are accepted like IPv6 literals; private literals are blocked
+  at the resolved-IP layer unless `webFetch.allowPrivateAddresses` is enabled (the default;
+  non-canonical numeric encodings like `0x7f.0.0.1` / `013.0.0.1` / `2130706433` are rejected
+  by URL validation before DNS either way).
+- Local access by default: private/loopback/link-local targets (localhost dev servers, LAN hosts)
+  and local files (`file://` URLs, absolute, `~/`, and `./` paths) are fetched directly — local
+  PDFs run the same MuPDF extraction, local HTML gets the same Markdown conversion and metadata
+  prefix, and the same byte caps and truncation notices apply. Opt out with
+  `webFetch.allowPrivateAddresses: false` to restore the resolved-IP SSRF guard, or
+  `webFetch.allowLocalFiles: false` to refuse local paths; both are settings, not tool parameters.
 - DNS resolution with SSRF protection: every resolved address is validated against
-  private/loopback/link-local/CGNAT/documentation/multicast/reserved ranges, then the validated IP
+  private/loopback/link-local/CGNAT/documentation/multicast/reserved ranges (unless
+  `webFetch.allowPrivateAddresses` is enabled, the default), then the validated IP
   is pinned for the connection (custom `lookup` + SNI `servername`), so DNS cannot rebind between
   validation and fetch; resolution shares the caller's abort signal and the overall deadline,
   so a stuck resolver cannot outlive the fetch.
@@ -174,10 +182,15 @@ Optional settings in `~/.pi/agent/settings.json` or `.pi/settings.json` (project
 |---|---|---|
 | `unslothWebTools.maxResults` | `5` | Default `maxResults` for `web_search` (clamped 1-20) |
 | `unslothWebTools.maxChars` / `webFetch.maxChars` / `smartFetchDefaultMaxChars` | tool param | Default `maxChars` for `web_fetch` and `web_search` url mode |
-| `unslothWebTools.timeoutMs` / `webFetch.timeoutMs` / `smartFetchDefaultTimeoutMs` | `60000` | Default `timeoutMs` (>=1000) |
+| `unslothWebTools.timeoutMs` / `webFetch.timeoutMs` / `smartFetchDefaultTimeoutMs` | `60000` fetch, `300000` search | Default `timeoutMs` when the tool param is absent (>=1000). Fetch and `web_search` url mode fall back to 60000; `web_search` query mode falls back to 300000 |
 | `webSearch.maxResults` / `smartWebSearch.resultsPerQuery` | same as above | Legacy aliases for `maxResults` |
+| `websitePolicy` | none | Not read from settings. Tools run unrestricted by default; `websitePolicy` is a programmatic option the host passes to `webSearch` / `fetchPageText` |
+| `unslothWebTools.allowPrivateAddresses` / `webFetch.allowPrivateAddresses` | `true` | Opt out to restore the resolved-IP SSRF guard: private/loopback/link-local hosts (localhost, LAN IPs) are refused again. Non-canonical numeric IP encodings stay blocked either way |
+| `unslothWebTools.allowLocalFiles` / `webFetch.allowLocalFiles` | `true` | Opt out to refuse local files in `web_fetch` and `web_search` url mode (`file://` URLs, absolute, `~/`, or `./` paths); when enabled, PDFs are extracted and HTML converted |
 
-Tool params always win over file defaults.
+Tool params always win over file defaults. Search dedup also strips default ports, so `https://example.com:443/a` and `https://example.com/a` collapse.
+
+Environment overrides: `PI_UNSLOTH_CACHE_DIR` changes the fetch cache directory, `PI_UNSLOTH_WEBTOOLS_STATS` opts into append-only sweep stats JSONL, `PI_CODING_AGENT_DIR` / `PI_AGENT_DIR` change the global settings directory. Cache entries live 1 hour and stale copies are served only after a network failure.
 
 ## Troubleshooting
 
@@ -190,7 +203,10 @@ Match on the exact prefix. Do not retry blocked hosts with spelling tricks.
 | Search timeout | `Search failed: the search engines (...) did not respond within N seconds.` | Retry, narrow the query, or raise `timeoutMs`. |
 | No results | `No results found.` | Rephrase the query. |
 | Policy filtered everything | `No results found within the website access limits.` | Widen `websitePolicy`, do not work around it. |
-| Blocked URL | `Blocked: ...` | Respect it. Includes non-http schemes, credentials, invalid hosts, non-public IPs, and policy denials. |
+| Blocked URL | `Blocked: ...` | Respect it. Includes non-http schemes, credentials, invalid hosts, non-canonical numeric IP encodings, and policy denials. |
+| Private address blocked | `Blocked: refusing to fetch the non-public address ...` | The SSRF guard is active (`allowPrivateAddresses: false`); remove it or set `true` to reach localhost/LAN, and write the scheme explicitly (`http://localhost:3000`). |
+| Local file blocked | `Blocked: the URL has an invalid hostname or port.` for paths | Local files are disabled: remove `allowLocalFiles: false` to read `file://`, absolute, `~/`, or `./` paths. |
+| File read failed | `Failed to read file: ...` | Check the path exists and is a regular file. |
 | HTTP failure | `Failed to fetch URL: HTTP ...` | Fix the URL. A 404 automatically tries a Wayback snapshot. |
 | Non-text / binary | `(non-text content:` / `(binary content,` | Not readable as text by design. |
 | PDF without text | `(PDF contains no extractable text)` / `(PDF content could not be read as text...)` | Scanned or encrypted PDF. |

package/cache.ts

@@ -98,9 +98,17 @@ async function prune(): Promise<void> {
   } catch {
     return;
   }
+  const now = Date.now();
+  const tmpFiles = files.filter((f) => f.includes(".tmp."));
+  for (const file of tmpFiles) {
+    try {
+      const full = join(dir, file);
+      const s = await stat(full);
+      if (now - s.mtimeMs >= CACHE_TTL_MS) await unlinkSafe(full);
+    } catch {}
+  }
   const jsonFiles = files.filter((f) => f.endsWith(".json"));
   if (!jsonFiles.length) return;
-  const now = Date.now();
   const entries: { file: string; size: number; timestamp: number }[] = [];
   let total = 0;
   for (const file of jsonFiles) {

package/engines.ts

@@ -82,6 +82,9 @@ export function canonicalizeHref(href: string): string {
       }
     }
     url.hash = "";
+    if ((url.protocol === "http:" && url.port === "80") || (url.protocol === "https:" && url.port === "443")) {
+      url.port = "";
+    }
     return url.toString();
   } catch {
     return href;

package/index.ts

@@ -1,8 +1,8 @@
 import { defineTool, type ExtensionAPI, type ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
-import { webSearch as defaultWebSearch } from "./web-search.ts";
+import { SEARCH_TIMEOUT_MS, webSearch as defaultWebSearch } from "./web-search.ts";
 import { DEFAULT_FETCH_TIMEOUT_MS, fetchPageText as defaultFetchPageText } from "./web-fetch.ts";
-import { loadDefaultFetchSettings } from "./settings.ts";
+import { loadDefaultFetchSettings, loadDefaultFetchTimeoutMs } from "./settings.ts";
 
 function positiveNumber(value: unknown): number | undefined {
   if (typeof value !== "number" || !Number.isFinite(value)) return undefined;
@@ -12,11 +12,12 @@ function positiveNumber(value: unknown): number | undefined {
 async function fetchDefaults(cwd: string | undefined, params: { timeoutMs?: unknown; maxChars?: unknown }) {
   const timeoutParam = positiveNumber(params.timeoutMs);
   const maxCharsParam = positiveNumber(params.maxChars);
-  if (timeoutParam !== undefined && maxCharsParam !== undefined) return { timeoutMs: timeoutParam, maxChars: maxCharsParam };
   const defaults = await loadDefaultFetchSettings(cwd);
   return {
     timeoutMs: timeoutParam ?? defaults.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS,
     maxChars: maxCharsParam ?? defaults.maxChars,
+    allowPrivateAddresses: defaults.allowPrivateAddresses,
+    allowLocalFiles: defaults.allowLocalFiles,
   };
 }
 
@@ -91,7 +92,7 @@ export function createWebTools(deps: WebToolsDeps = {}) {
           const url = params.url.trim();
           onUpdate?.({ content: [{ type: "text", text: `Fetching ${url}...` }], details: {} });
           const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
-          const { timeoutMs, maxChars } = await fetchDefaults(cwd, params);
+          const { timeoutMs, maxChars, allowPrivateAddresses, allowLocalFiles } = await fetchDefaults(cwd, params);
           return {
             content: [
               {
@@ -100,6 +101,8 @@ export function createWebTools(deps: WebToolsDeps = {}) {
                   timeoutMs,
                   signal: signal ?? undefined,
                   maxChars,
+                  allowPrivateAddresses,
+                  allowLocalFiles,
                 }),
               },
             ],
@@ -107,11 +110,14 @@ export function createWebTools(deps: WebToolsDeps = {}) {
           };
         }
         onUpdate?.({ content: [{ type: "text", text: "Searching the web..." }], details: {} });
+        const timeoutParam = positiveNumber(params.timeoutMs);
+        const searchCwd = (_ctx as ExtensionContext | undefined)?.cwd;
+        const searchTimeoutMs = timeoutParam ?? (await loadDefaultFetchTimeoutMs(searchCwd)) ?? SEARCH_TIMEOUT_MS;
         const text = await webSearch(params.query, {
           signal: signal ?? undefined,
-          timeoutMs: positiveNumber(params.timeoutMs),
+          timeoutMs: searchTimeoutMs,
           maxResults: positiveNumber(params.maxResults),
-          cwd: (_ctx as ExtensionContext | undefined)?.cwd,
+          cwd: searchCwd,
         });
         return { content: [{ type: "text", text }], details: {} };
       },
@@ -124,18 +130,21 @@ export function createWebTools(deps: WebToolsDeps = {}) {
         "main-content heuristic: article/main scoping plus hidden-element and boilerplate " +
         "stripping. Non-HTML text is returned as-is. GitHub repo root pages are rewritten to the " +
         "README API, so the README is returned instead of the repo page's UI chrome. " +
-        "Private/loopback/link-local targets are blocked (SSRF protection), and the download size " +
-        "is capped.",
+        "Private/loopback/link-local targets and local files (file:// URLs, absolute, ~/ or ./ paths, including " +
+        "PDFs) are supported by default; opt out with webFetch.allowPrivateAddresses: false or " +
+        "webFetch.allowLocalFiles: false in settings. The download size is capped.",
       promptSnippet: "Fetch a web page and return readable text content",
       parameters: WebFetchParams,
       async execute(_toolCallId, params, signal, onUpdate, _ctx) {
         onUpdate?.({ content: [{ type: "text", text: `Fetching ${params.url}...` }], details: {} });
         const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
-        const { timeoutMs, maxChars } = await fetchDefaults(cwd, params);
+        const { timeoutMs, maxChars, allowPrivateAddresses, allowLocalFiles } = await fetchDefaults(cwd, params);
         const text = await fetchPageText(params.url, {
           timeoutMs,
           signal: signal ?? undefined,
           maxChars,
+          allowPrivateAddresses,
+          allowLocalFiles,
         });
         return { content: [{ type: "text", text }], details: {} };
       },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools ported from the Unsloth Studio codebase (DuckDuckGo search, SSRF-safe direct fetching, HTML-to-Markdown extraction)",
   "main": "index.ts",
@@ -55,7 +55,7 @@
     "test:smoke": "vitest run test/smoke.test.ts",
     "typecheck": "tsc --noEmit",
     "check:package": "node -e \"const fs=require('fs');const pkg=require('./package.json');const listed=new Set(pkg.files);const bad=[];for(const f of pkg.files){if(!fs.existsSync(f))bad.push('missing file: '+f)}for(const f of fs.readdirSync('.').filter(f=>f.endsWith('.ts'))){if(!listed.has(f))bad.push('unlisted source: '+f)}if(bad.length){console.error(bad.join('\\n'));process.exit(1)}\"",
-    "prepublishOnly": "npm run typecheck && npm run test:unit && npm run check:package && npm run check:publint",
+    "prepublishOnly": "npm run typecheck && npm run lint && npm run test:unit && npm run check:package && npm run check:publint",
     "lint": "oxlint .",
     "check:publint": "publint",
     "check:attw": "attw --pack"

package/pdf.ts

@@ -2,6 +2,7 @@ import { createRequire } from "node:module";
 import { inflateSync } from "node:zlib";
 
 export const MAX_WEB_PDF_PAGES = 50;
+const MAX_PDF_INFLATE_BYTES = 64 * 1024 * 1024;
 
 export class PdfParseError extends Error {
   constructor() {
@@ -786,7 +787,7 @@ async function extractPdfTextFallback(data: Buffer): Promise<string> {
     let bytes = raw;
     if (isFlate) {
       try {
-        bytes = inflateSync(raw);
+        bytes = inflateSync(raw, { maxOutputLength: MAX_PDF_INFLATE_BYTES });
       } catch {
         continue;
       }

package/settings.ts

@@ -34,6 +34,21 @@ function pickNumber(data: Record<string, unknown>, paths: string[][]): number |
   return undefined;
 }
 
+function pickBoolean(data: Record<string, unknown>, paths: string[][]): boolean | undefined {
+  for (const path of paths) {
+    let cur: unknown = data;
+    for (const key of path) {
+      if (cur && typeof cur === "object" && !Array.isArray(cur)) cur = (cur as Record<string, unknown>)[key];
+      else {
+        cur = undefined;
+        break;
+      }
+    }
+    if (typeof cur === "boolean") return cur;
+  }
+  return undefined;
+}
+
 const MAX_RESULTS = 5;
 
 const MAX_RESULTS_PATHS: string[][] = [
@@ -54,6 +69,16 @@ const FETCH_TIMEOUT_PATHS: string[][] = [
   ["smartFetchDefaultTimeoutMs"],
 ];
 
+const ALLOW_PRIVATE_ADDRESSES_PATHS: string[][] = [
+  ["unslothWebTools", "allowPrivateAddresses"],
+  ["webFetch", "allowPrivateAddresses"],
+];
+
+const ALLOW_LOCAL_FILES_PATHS: string[][] = [
+  ["unslothWebTools", "allowLocalFiles"],
+  ["webFetch", "allowLocalFiles"],
+];
+
 function clampMaxResults(value: number): number {
   return Math.min(20, Math.max(1, value));
 }
@@ -101,14 +126,25 @@ export async function loadDefaultFetchTimeoutMs(cwd?: string): Promise<number |
   return loadFetchSetting(cwd, FETCH_TIMEOUT_PATHS, (n) => n >= 1000);
 }
 
-export async function loadDefaultFetchSettings(cwd?: string): Promise<{ maxChars?: number; timeoutMs?: number }> {
+export async function loadDefaultFetchSettings(cwd?: string): Promise<{
+  maxChars?: number;
+  timeoutMs?: number;
+  allowPrivateAddresses: boolean;
+  allowLocalFiles: boolean;
+}> {
   let maxChars: number | undefined;
   let timeoutMs: number | undefined;
+  let allowPrivateAddresses = true;
+  let allowLocalFiles = true;
   for (const data of await settingsEntries(cwd)) {
     const c = pickNumber(data, FETCH_MAX_CHARS_PATHS);
     if (c !== undefined && c > 0) maxChars = c;
     const t = pickNumber(data, FETCH_TIMEOUT_PATHS);
     if (t !== undefined && t >= 1000) timeoutMs = t;
+    const p = pickBoolean(data, ALLOW_PRIVATE_ADDRESSES_PATHS);
+    if (p !== undefined) allowPrivateAddresses = p;
+    const l = pickBoolean(data, ALLOW_LOCAL_FILES_PATHS);
+    if (l !== undefined) allowLocalFiles = l;
   }
-  return { maxChars, timeoutMs };
+  return { maxChars, timeoutMs, allowPrivateAddresses, allowLocalFiles };
 }

package/user-agents.ts

@@ -1,12 +1,11 @@
 const USER_AGENTS = [
-  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:145.0) Gecko/20100101 Firefox/145.0",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:145.0) Gecko/20100101 Firefox/145.0",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15",
 ];
-
 export function randomUserAgent(): string {
   return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
 }

package/web-fetch.ts

@@ -2,6 +2,11 @@ import { lookup as dnsLookup } from "node:dns/promises";
 import type { LookupAllOptions } from "node:dns";
 import http from "node:http";
 import https from "node:https";
+import { open } from "node:fs/promises";
+import type { FileHandle } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
 
 import { createBrotliDecompress, createGunzip, createInflate, createInflateRaw } from "node:zlib";
 import type { IncomingMessage } from "node:http";
@@ -171,6 +176,8 @@ export interface FetchPageOptions {
   maxChars?: number;
   maxBytes?: number;
   maxPdfBytes?: number;
+  allowPrivateAddresses?: boolean;
+  allowLocalFiles?: boolean;
   seams?: FetchSeams;
   rawFetch?: (url: string, options: RawFetchOptions) => Promise<RawFetchResult>;
 }
@@ -191,7 +198,7 @@ export interface ResolvedHost {
 }
 
 export interface FetchSeams {
-  resolve?: (hostname: string, signal?: AbortSignal) => Promise<ResolvedHost>;
+  resolve?: (hostname: string, signal?: AbortSignal, allowPrivateAddresses?: boolean) => Promise<ResolvedHost>;
   request?: (opts: HopOptions) => Promise<HopResponse>;
 }
 
@@ -217,6 +224,7 @@ export interface RawFetchOptions {
   websitePolicy?: WebsitePolicy | null;
   maxBytes?: number;
   maxPdfBytes?: number;
+  allowPrivateAddresses?: boolean;
   seams?: FetchSeams;
 }
 
@@ -549,7 +557,11 @@ function sleepAbortable(ms: number, signal?: AbortSignal): Promise<void> {
   });
 }
 
-async function resolveAndValidate(hostname: string, signal?: AbortSignal): Promise<ResolvedHost> {
+async function resolveAndValidate(
+  hostname: string,
+  signal?: AbortSignal,
+  allowPrivateAddresses = false,
+): Promise<ResolvedHost> {
   let lastError: unknown = null;
   for (let attempt = 0; attempt < MAX_DNS_ATTEMPTS; attempt++) {
     if (signal?.aborted) break;
@@ -568,7 +580,7 @@ async function resolveAndValidate(hostname: string, signal?: AbortSignal): Promi
         return { ok: false, reason: `Failed to resolve host: no addresses for '${hostname}'`, ip: "", family: 0 };
       }
       for (const entry of addresses) {
-        if (!isPublicIp(entry.address)) {
+        if (!allowPrivateAddresses && !isPublicIp(entry.address)) {
           return { ok: false, reason: `Blocked: refusing to fetch the non-public address ${entry.address}.`, ip: "", family: 0 };
         }
       }
@@ -631,8 +643,14 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
       headers: opts.headers,
       timeout: opts.inactivityMs,
       servername: url.protocol === "https:" ? url.hostname : undefined,
-      lookup: (_host, _opts, callback) =>
-        callback(null, [{ address: opts.pinnedIp, family: opts.family }]),
+      lookup: ((_hostname: string, _options: unknown, _callback: unknown) => {
+        const done = (typeof _options === "function" ? _options : _callback) as (err: unknown, address: unknown, family?: unknown) => void;
+        if ((_options as { all?: boolean } | undefined)?.all) {
+          done(null, [{ address: opts.pinnedIp, family: opts.family }]);
+        } else {
+          done(null, opts.pinnedIp, opts.family);
+        }
+      }) as typeof options.lookup,
     };
     let settled = false;
     let resRef: IncomingMessage | null = null;
@@ -798,7 +816,7 @@ export async function fetchUrlRaw(
   url: string,
   options: RawFetchOptions = {},
 ): Promise<RawFetchResult> {
-  const timeoutMs = options.timeoutMs ?? 30_000;
+  const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
   const now = options.nowMs ?? Date.now;
   const deadline = options.deadlineMs ?? now() + timeoutMs;
   const signal = options.signal;
@@ -807,6 +825,7 @@ export async function fetchUrlRaw(
   const maxPdfBytes = options.maxPdfBytes ?? MAX_PDF_FETCH_BYTES;
   const seams = options.seams ?? {};
   const resolveHost = seams.resolve ?? resolveAndValidate;
+  const allowPrivateAddresses = options.allowPrivateAddresses ?? true;
   const performRequest = seams.request ?? requestHop;
   const resolveWithBudget = async (hostname: string): Promise<ResolvedHost> => {
     const abortController = new AbortController();
@@ -818,7 +837,7 @@ export async function fetchUrlRaw(
       deadlineId = setTimeout(() => resolve({ ok: false, reason: FETCH_TIMEOUT_MESSAGE, ip: "", family: 0 }), waitMs);
     });
     try {
-      const resolved = await Promise.race([resolveHost(hostname, resolveSignal), deadlinePromise]);
+      const resolved = await Promise.race([resolveHost(hostname, resolveSignal, allowPrivateAddresses), deadlinePromise]);
       if (resolved.ok) return resolved;
       if (signal?.aborted) return { ...resolved, reason: FETCH_CANCELLED_MESSAGE };
       if (resolveSignal.aborted) return { ...resolved, reason: FETCH_TIMEOUT_MESSAGE };
@@ -847,7 +866,9 @@ export async function fetchUrlRaw(
     const budgetResult = checkBudget();
     if (budgetResult !== null) return budgetResult;
     const parsed = new URL(currentUrl);
-    const hostHeader = parsed.hostname + (parsed.port ? `:${parsed.port}` : "");
+    const hostHeader = parsed.hostname.includes(":")
+      ? `[${parsed.hostname}]${parsed.port ? `:${parsed.port}` : ""}`
+      : parsed.hostname + (parsed.port ? `:${parsed.port}` : "");
     const headers: Record<string, string> = {
       "User-Agent": userAgent,
       Host: hostHeader,
@@ -1233,6 +1254,74 @@ async function fetchWaybackSnapshot(
   if (!snapResult || snapResult.error !== null) return null;
   return { body: snapResult.body, contentType: snapResult.contentType, timestamp };
 }
+const WINDOWS_PATH_RE = /^[a-zA-Z]:[\\/]/;
+
+function parseLocalPath(url: string): string | null {
+  const trimmed = url.trim();
+  if (/^file:/i.test(trimmed)) {
+    try {
+      return fileURLToPath(trimmed);
+    } catch {
+      return "";
+    }
+  }
+  if (trimmed.startsWith("~/") || trimmed.startsWith("~\\")) return join(homedir(), trimmed.slice(2));
+  if (WINDOWS_PATH_RE.test(trimmed) || trimmed.startsWith("/") || /^\.{1,2}\//.test(trimmed)) return trimmed;
+  return null;
+}
+
+function localFileFailure(err: unknown): string {
+  return `Failed to read file: ${err instanceof Error ? err.message : String(err)}`;
+}
+
+async function readLocalFile(
+  filePath: string,
+  options: { signal?: AbortSignal; maxChars?: number; maxBytes?: number; maxPdfBytes?: number },
+): Promise<string> {
+  if (options.signal?.aborted) return "Failed to read file: cancelled.";
+  let handle: FileHandle;
+  try {
+    handle = await open(filePath, "r");
+  } catch (err) {
+    return localFileFailure(err);
+  }
+  try {
+    const stats = await handle.stat();
+    if (stats.isDirectory()) return `Failed to read file: ${filePath} is a directory.`;
+    const maxBytes = options.maxBytes ?? MAX_FETCH_BYTES;
+    const maxPdfBytes = options.maxPdfBytes ?? MAX_PDF_FETCH_BYTES;
+    const length = Math.min(stats.size, Math.max(maxBytes, maxPdfBytes));
+    const buffer = Buffer.alloc(Number(length));
+    let read = 0;
+    while (read < length) {
+      const { bytesRead } = await handle.read(buffer, read, length - read, read);
+      if (!bytesRead) break;
+      read += bytesRead;
+    }
+    if (options.signal?.aborted) return "Failed to read file: cancelled.";
+    const head = read < length ? buffer.subarray(0, read) : buffer;
+    const isPdf = hasPdfMagic(head);
+    const limit = isPdf ? maxPdfBytes : maxBytes;
+    const truncated = stats.size > limit;
+    const body = head.length > limit ? head.subarray(0, limit) : head;
+    if (isPdf) {
+      let pdfText: string;
+      try {
+        pdfText = await extractPdfText(body);
+      } catch {
+        return withTruncation("(PDF content could not be read as text)", truncated);
+      }
+      if (!pdfText) pdfText = "(PDF contains no extractable text)";
+      return truncatePageText(withTruncation(pdfText, truncated), options.maxChars);
+    }
+    const text = decodeWithCodec(body, bomCodecFor(body) ?? "utf-8");
+    if (looksBinary(text)) return `(binary content, ${body.length} bytes; not readable as text)`;
+    return truncatePageText(withTruncation(renderBody(text, ""), truncated), options.maxChars);
+  } finally {
+    await handle.close().catch(() => {});
+  }
+}
+
 export async function fetchPageText(
   url: string,
   options: FetchPageOptions = {},
@@ -1244,6 +1333,18 @@ export async function fetchPageText(
   const policy = options.websitePolicy ?? null;
   const maxChars = options.maxChars;
   const rawFetch = options.rawFetch ?? fetchUrlRaw;
+  if (options.allowLocalFiles !== false) {
+    const localPath = parseLocalPath(url);
+    if (localPath !== null) {
+      if (!localPath) return "Failed to read file: invalid file URL.";
+      return readLocalFile(localPath, {
+        signal,
+        maxChars,
+        maxBytes: options.maxBytes,
+        maxPdfBytes: options.maxPdfBytes,
+      });
+    }
+  }
   url = normalizeUrlScheme(url);
   const [allowed, reason] = checkUrlAccess(url, policy);
   if (!allowed) return reason;
@@ -1251,6 +1352,7 @@ export async function fetchPageText(
     deadlineMs,
     signal,
     websitePolicy: policy,
+    allowPrivateAddresses: options.allowPrivateAddresses,
     maxBytes: options.maxBytes,
     maxPdfBytes: options.maxPdfBytes,
     seams: options.seams,

package/web-search.ts

@@ -11,7 +11,7 @@ import {
 
 export { EmptySweepError, SearchCancelled, SearchTimeoutError } from "./engines.ts";
 
-const SEARCH_TIMEOUT_MS = 300_000;
+export const SEARCH_TIMEOUT_MS = 300_000;
 const MAX_RESULTS = 5;
 
 export const EMPTY_SEARCH_RESULTS = [
@@ -56,7 +56,8 @@ export async function webSearch(
   const saneMax = Number.isFinite(rawMaxResults) ? Math.floor(rawMaxResults as number) : MAX_RESULTS;
   const maxResults = Math.min(20, Math.max(1, saneMax));
   const rawTimeout = options.timeoutMs ?? SEARCH_TIMEOUT_MS;
-  const timeoutMs = Number.isFinite(rawTimeout) ? (rawTimeout as number) : SEARCH_TIMEOUT_MS;
+  const flooredTimeout = Number.isFinite(rawTimeout) ? Math.floor(rawTimeout as number) : SEARCH_TIMEOUT_MS;
+  const timeoutMs = flooredTimeout >= 1 ? flooredTimeout : SEARCH_TIMEOUT_MS;
   const signal = options.signal;
   const policy = options.websitePolicy ?? null;
   const client = options.client ?? ddgSearch;