pi-unsloth-webtools 0.5.1 → 0.6.0

package/README.md

@@ -50,11 +50,19 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
 - URL validation: http/https only, no credentials or encoded hostnames, hostname/port checks (any
   port 1–65535 is permitted; SSRF protection is enforced at the resolved-IP layer, not by port
   allowlists).
-  Canonical public IPv4 literals are accepted like IPv6 literals; private literals are still
-  blocked at the resolved-IP layer (non-canonical numeric encodings like `0x7f.0.0.1` /
-  `013.0.0.1` / `2130706433` are rejected by URL validation before DNS).
+  Canonical public IPv4 literals are accepted like IPv6 literals; private literals are blocked
+  at the resolved-IP layer unless `webFetch.allowPrivateAddresses` is enabled (the default;
+  non-canonical numeric encodings like `0x7f.0.0.1` / `013.0.0.1` / `2130706433` are rejected
+  by URL validation before DNS either way).
+- Local access by default: private/loopback/link-local targets (localhost dev servers, LAN hosts)
+  and local files (`file://` URLs, absolute, `~/`, and `./` paths) are fetched directly — local
+  PDFs run the same MuPDF extraction, local HTML gets the same Markdown conversion and metadata
+  prefix, and the same byte caps and truncation notices apply. Opt out with
+  `webFetch.allowPrivateAddresses: false` to restore the resolved-IP SSRF guard, or
+  `webFetch.allowLocalFiles: false` to refuse local paths; both are settings, not tool parameters.
 - DNS resolution with SSRF protection: every resolved address is validated against
-  private/loopback/link-local/CGNAT/documentation/multicast/reserved ranges, then the validated IP
+  private/loopback/link-local/CGNAT/documentation/multicast/reserved ranges (unless
+  `webFetch.allowPrivateAddresses` is enabled, the default), then the validated IP
   is pinned for the connection (custom `lookup` + SNI `servername`), so DNS cannot rebind between
   validation and fetch; resolution shares the caller's abort signal and the overall deadline,
   so a stuck resolver cannot outlive the fetch.
@@ -177,6 +185,8 @@ Optional settings in `~/.pi/agent/settings.json` or `.pi/settings.json` (project
 | `unslothWebTools.timeoutMs` / `webFetch.timeoutMs` / `smartFetchDefaultTimeoutMs` | `60000` fetch, `300000` search | Default `timeoutMs` when the tool param is absent (>=1000). Fetch and `web_search` url mode fall back to 60000; `web_search` query mode falls back to 300000 |
 | `webSearch.maxResults` / `smartWebSearch.resultsPerQuery` | same as above | Legacy aliases for `maxResults` |
 | `websitePolicy` | none | Not read from settings. Tools run unrestricted by default; `websitePolicy` is a programmatic option the host passes to `webSearch` / `fetchPageText` |
+| `unslothWebTools.allowPrivateAddresses` / `webFetch.allowPrivateAddresses` | `true` | Opt out to restore the resolved-IP SSRF guard: private/loopback/link-local hosts (localhost, LAN IPs) are refused again. Non-canonical numeric IP encodings stay blocked either way |
+| `unslothWebTools.allowLocalFiles` / `webFetch.allowLocalFiles` | `true` | Opt out to refuse local files in `web_fetch` and `web_search` url mode (`file://` URLs, absolute, `~/`, or `./` paths); when enabled, PDFs are extracted and HTML converted |
 
 Tool params always win over file defaults. Search dedup also strips default ports, so `https://example.com:443/a` and `https://example.com/a` collapse.
 
@@ -193,7 +203,10 @@ Match on the exact prefix. Do not retry blocked hosts with spelling tricks.
 | Search timeout | `Search failed: the search engines (...) did not respond within N seconds.` | Retry, narrow the query, or raise `timeoutMs`. |
 | No results | `No results found.` | Rephrase the query. |
 | Policy filtered everything | `No results found within the website access limits.` | Widen `websitePolicy`, do not work around it. |
-| Blocked URL | `Blocked: ...` | Respect it. Includes non-http schemes, credentials, invalid hosts, non-public IPs, and policy denials. |
+| Blocked URL | `Blocked: ...` | Respect it. Includes non-http schemes, credentials, invalid hosts, non-canonical numeric IP encodings, and policy denials. |
+| Private address blocked | `Blocked: refusing to fetch the non-public address ...` | The SSRF guard is active (`allowPrivateAddresses: false`); remove it or set `true` to reach localhost/LAN, and write the scheme explicitly (`http://localhost:3000`). |
+| Local file blocked | `Blocked: the URL has an invalid hostname or port.` for paths | Local files are disabled: remove `allowLocalFiles: false` to read `file://`, absolute, `~/`, or `./` paths. |
+| File read failed | `Failed to read file: ...` | Check the path exists and is a regular file. |
 | HTTP failure | `Failed to fetch URL: HTTP ...` | Fix the URL. A 404 automatically tries a Wayback snapshot. |
 | Non-text / binary | `(non-text content:` / `(binary content,` | Not readable as text by design. |
 | PDF without text | `(PDF contains no extractable text)` / `(PDF content could not be read as text...)` | Scanned or encrypted PDF. |

package/index.ts

@@ -12,11 +12,12 @@ function positiveNumber(value: unknown): number | undefined {
 async function fetchDefaults(cwd: string | undefined, params: { timeoutMs?: unknown; maxChars?: unknown }) {
   const timeoutParam = positiveNumber(params.timeoutMs);
   const maxCharsParam = positiveNumber(params.maxChars);
-  if (timeoutParam !== undefined && maxCharsParam !== undefined) return { timeoutMs: timeoutParam, maxChars: maxCharsParam };
   const defaults = await loadDefaultFetchSettings(cwd);
   return {
     timeoutMs: timeoutParam ?? defaults.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS,
     maxChars: maxCharsParam ?? defaults.maxChars,
+    allowPrivateAddresses: defaults.allowPrivateAddresses,
+    allowLocalFiles: defaults.allowLocalFiles,
   };
 }
 
@@ -91,7 +92,7 @@ export function createWebTools(deps: WebToolsDeps = {}) {
           const url = params.url.trim();
           onUpdate?.({ content: [{ type: "text", text: `Fetching ${url}...` }], details: {} });
           const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
-          const { timeoutMs, maxChars } = await fetchDefaults(cwd, params);
+          const { timeoutMs, maxChars, allowPrivateAddresses, allowLocalFiles } = await fetchDefaults(cwd, params);
           return {
             content: [
               {
@@ -100,6 +101,8 @@ export function createWebTools(deps: WebToolsDeps = {}) {
                   timeoutMs,
                   signal: signal ?? undefined,
                   maxChars,
+                  allowPrivateAddresses,
+                  allowLocalFiles,
                 }),
               },
             ],
@@ -127,18 +130,21 @@ export function createWebTools(deps: WebToolsDeps = {}) {
         "main-content heuristic: article/main scoping plus hidden-element and boilerplate " +
         "stripping. Non-HTML text is returned as-is. GitHub repo root pages are rewritten to the " +
         "README API, so the README is returned instead of the repo page's UI chrome. " +
-        "Private/loopback/link-local targets are blocked (SSRF protection), and the download size " +
-        "is capped.",
+        "Private/loopback/link-local targets and local files (file:// URLs, absolute, ~/ or ./ paths, including " +
+        "PDFs) are supported by default; opt out with webFetch.allowPrivateAddresses: false or " +
+        "webFetch.allowLocalFiles: false in settings. The download size is capped.",
       promptSnippet: "Fetch a web page and return readable text content",
       parameters: WebFetchParams,
       async execute(_toolCallId, params, signal, onUpdate, _ctx) {
         onUpdate?.({ content: [{ type: "text", text: `Fetching ${params.url}...` }], details: {} });
         const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
-        const { timeoutMs, maxChars } = await fetchDefaults(cwd, params);
+        const { timeoutMs, maxChars, allowPrivateAddresses, allowLocalFiles } = await fetchDefaults(cwd, params);
         const text = await fetchPageText(params.url, {
           timeoutMs,
           signal: signal ?? undefined,
           maxChars,
+          allowPrivateAddresses,
+          allowLocalFiles,
         });
         return { content: [{ type: "text", text }], details: {} };
       },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools ported from the Unsloth Studio codebase (DuckDuckGo search, SSRF-safe direct fetching, HTML-to-Markdown extraction)",
   "main": "index.ts",

package/settings.ts

@@ -34,6 +34,21 @@ function pickNumber(data: Record<string, unknown>, paths: string[][]): number |
   return undefined;
 }
 
+function pickBoolean(data: Record<string, unknown>, paths: string[][]): boolean | undefined {
+  for (const path of paths) {
+    let cur: unknown = data;
+    for (const key of path) {
+      if (cur && typeof cur === "object" && !Array.isArray(cur)) cur = (cur as Record<string, unknown>)[key];
+      else {
+        cur = undefined;
+        break;
+      }
+    }
+    if (typeof cur === "boolean") return cur;
+  }
+  return undefined;
+}
+
 const MAX_RESULTS = 5;
 
 const MAX_RESULTS_PATHS: string[][] = [
@@ -54,6 +69,16 @@ const FETCH_TIMEOUT_PATHS: string[][] = [
   ["smartFetchDefaultTimeoutMs"],
 ];
 
+const ALLOW_PRIVATE_ADDRESSES_PATHS: string[][] = [
+  ["unslothWebTools", "allowPrivateAddresses"],
+  ["webFetch", "allowPrivateAddresses"],
+];
+
+const ALLOW_LOCAL_FILES_PATHS: string[][] = [
+  ["unslothWebTools", "allowLocalFiles"],
+  ["webFetch", "allowLocalFiles"],
+];
+
 function clampMaxResults(value: number): number {
   return Math.min(20, Math.max(1, value));
 }
@@ -101,14 +126,25 @@ export async function loadDefaultFetchTimeoutMs(cwd?: string): Promise<number |
   return loadFetchSetting(cwd, FETCH_TIMEOUT_PATHS, (n) => n >= 1000);
 }
 
-export async function loadDefaultFetchSettings(cwd?: string): Promise<{ maxChars?: number; timeoutMs?: number }> {
+export async function loadDefaultFetchSettings(cwd?: string): Promise<{
+  maxChars?: number;
+  timeoutMs?: number;
+  allowPrivateAddresses: boolean;
+  allowLocalFiles: boolean;
+}> {
   let maxChars: number | undefined;
   let timeoutMs: number | undefined;
+  let allowPrivateAddresses = true;
+  let allowLocalFiles = true;
   for (const data of await settingsEntries(cwd)) {
     const c = pickNumber(data, FETCH_MAX_CHARS_PATHS);
     if (c !== undefined && c > 0) maxChars = c;
     const t = pickNumber(data, FETCH_TIMEOUT_PATHS);
     if (t !== undefined && t >= 1000) timeoutMs = t;
+    const p = pickBoolean(data, ALLOW_PRIVATE_ADDRESSES_PATHS);
+    if (p !== undefined) allowPrivateAddresses = p;
+    const l = pickBoolean(data, ALLOW_LOCAL_FILES_PATHS);
+    if (l !== undefined) allowLocalFiles = l;
   }
-  return { maxChars, timeoutMs };
+  return { maxChars, timeoutMs, allowPrivateAddresses, allowLocalFiles };
 }

package/web-fetch.ts

@@ -2,6 +2,11 @@ import { lookup as dnsLookup } from "node:dns/promises";
 import type { LookupAllOptions } from "node:dns";
 import http from "node:http";
 import https from "node:https";
+import { open } from "node:fs/promises";
+import type { FileHandle } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
 
 import { createBrotliDecompress, createGunzip, createInflate, createInflateRaw } from "node:zlib";
 import type { IncomingMessage } from "node:http";
@@ -171,6 +176,8 @@ export interface FetchPageOptions {
   maxChars?: number;
   maxBytes?: number;
   maxPdfBytes?: number;
+  allowPrivateAddresses?: boolean;
+  allowLocalFiles?: boolean;
   seams?: FetchSeams;
   rawFetch?: (url: string, options: RawFetchOptions) => Promise<RawFetchResult>;
 }
@@ -191,7 +198,7 @@ export interface ResolvedHost {
 }
 
 export interface FetchSeams {
-  resolve?: (hostname: string, signal?: AbortSignal) => Promise<ResolvedHost>;
+  resolve?: (hostname: string, signal?: AbortSignal, allowPrivateAddresses?: boolean) => Promise<ResolvedHost>;
   request?: (opts: HopOptions) => Promise<HopResponse>;
 }
 
@@ -217,6 +224,7 @@ export interface RawFetchOptions {
   websitePolicy?: WebsitePolicy | null;
   maxBytes?: number;
   maxPdfBytes?: number;
+  allowPrivateAddresses?: boolean;
   seams?: FetchSeams;
 }
 
@@ -549,7 +557,11 @@ function sleepAbortable(ms: number, signal?: AbortSignal): Promise<void> {
   });
 }
 
-async function resolveAndValidate(hostname: string, signal?: AbortSignal): Promise<ResolvedHost> {
+async function resolveAndValidate(
+  hostname: string,
+  signal?: AbortSignal,
+  allowPrivateAddresses = false,
+): Promise<ResolvedHost> {
   let lastError: unknown = null;
   for (let attempt = 0; attempt < MAX_DNS_ATTEMPTS; attempt++) {
     if (signal?.aborted) break;
@@ -568,7 +580,7 @@ async function resolveAndValidate(hostname: string, signal?: AbortSignal): Promi
         return { ok: false, reason: `Failed to resolve host: no addresses for '${hostname}'`, ip: "", family: 0 };
       }
       for (const entry of addresses) {
-        if (!isPublicIp(entry.address)) {
+        if (!allowPrivateAddresses && !isPublicIp(entry.address)) {
           return { ok: false, reason: `Blocked: refusing to fetch the non-public address ${entry.address}.`, ip: "", family: 0 };
         }
       }
@@ -813,6 +825,7 @@ export async function fetchUrlRaw(
   const maxPdfBytes = options.maxPdfBytes ?? MAX_PDF_FETCH_BYTES;
   const seams = options.seams ?? {};
   const resolveHost = seams.resolve ?? resolveAndValidate;
+  const allowPrivateAddresses = options.allowPrivateAddresses ?? true;
   const performRequest = seams.request ?? requestHop;
   const resolveWithBudget = async (hostname: string): Promise<ResolvedHost> => {
     const abortController = new AbortController();
@@ -824,7 +837,7 @@ export async function fetchUrlRaw(
       deadlineId = setTimeout(() => resolve({ ok: false, reason: FETCH_TIMEOUT_MESSAGE, ip: "", family: 0 }), waitMs);
     });
     try {
-      const resolved = await Promise.race([resolveHost(hostname, resolveSignal), deadlinePromise]);
+      const resolved = await Promise.race([resolveHost(hostname, resolveSignal, allowPrivateAddresses), deadlinePromise]);
       if (resolved.ok) return resolved;
       if (signal?.aborted) return { ...resolved, reason: FETCH_CANCELLED_MESSAGE };
       if (resolveSignal.aborted) return { ...resolved, reason: FETCH_TIMEOUT_MESSAGE };
@@ -853,7 +866,9 @@ export async function fetchUrlRaw(
     const budgetResult = checkBudget();
     if (budgetResult !== null) return budgetResult;
     const parsed = new URL(currentUrl);
-    const hostHeader = parsed.hostname + (parsed.port ? `:${parsed.port}` : "");
+    const hostHeader = parsed.hostname.includes(":")
+      ? `[${parsed.hostname}]${parsed.port ? `:${parsed.port}` : ""}`
+      : parsed.hostname + (parsed.port ? `:${parsed.port}` : "");
     const headers: Record<string, string> = {
       "User-Agent": userAgent,
       Host: hostHeader,
@@ -1239,6 +1254,74 @@ async function fetchWaybackSnapshot(
   if (!snapResult || snapResult.error !== null) return null;
   return { body: snapResult.body, contentType: snapResult.contentType, timestamp };
 }
+const WINDOWS_PATH_RE = /^[a-zA-Z]:[\\/]/;
+
+function parseLocalPath(url: string): string | null {
+  const trimmed = url.trim();
+  if (/^file:/i.test(trimmed)) {
+    try {
+      return fileURLToPath(trimmed);
+    } catch {
+      return "";
+    }
+  }
+  if (trimmed.startsWith("~/") || trimmed.startsWith("~\\")) return join(homedir(), trimmed.slice(2));
+  if (WINDOWS_PATH_RE.test(trimmed) || trimmed.startsWith("/") || /^\.{1,2}\//.test(trimmed)) return trimmed;
+  return null;
+}
+
+function localFileFailure(err: unknown): string {
+  return `Failed to read file: ${err instanceof Error ? err.message : String(err)}`;
+}
+
+async function readLocalFile(
+  filePath: string,
+  options: { signal?: AbortSignal; maxChars?: number; maxBytes?: number; maxPdfBytes?: number },
+): Promise<string> {
+  if (options.signal?.aborted) return "Failed to read file: cancelled.";
+  let handle: FileHandle;
+  try {
+    handle = await open(filePath, "r");
+  } catch (err) {
+    return localFileFailure(err);
+  }
+  try {
+    const stats = await handle.stat();
+    if (stats.isDirectory()) return `Failed to read file: ${filePath} is a directory.`;
+    const maxBytes = options.maxBytes ?? MAX_FETCH_BYTES;
+    const maxPdfBytes = options.maxPdfBytes ?? MAX_PDF_FETCH_BYTES;
+    const length = Math.min(stats.size, Math.max(maxBytes, maxPdfBytes));
+    const buffer = Buffer.alloc(Number(length));
+    let read = 0;
+    while (read < length) {
+      const { bytesRead } = await handle.read(buffer, read, length - read, read);
+      if (!bytesRead) break;
+      read += bytesRead;
+    }
+    if (options.signal?.aborted) return "Failed to read file: cancelled.";
+    const head = read < length ? buffer.subarray(0, read) : buffer;
+    const isPdf = hasPdfMagic(head);
+    const limit = isPdf ? maxPdfBytes : maxBytes;
+    const truncated = stats.size > limit;
+    const body = head.length > limit ? head.subarray(0, limit) : head;
+    if (isPdf) {
+      let pdfText: string;
+      try {
+        pdfText = await extractPdfText(body);
+      } catch {
+        return withTruncation("(PDF content could not be read as text)", truncated);
+      }
+      if (!pdfText) pdfText = "(PDF contains no extractable text)";
+      return truncatePageText(withTruncation(pdfText, truncated), options.maxChars);
+    }
+    const text = decodeWithCodec(body, bomCodecFor(body) ?? "utf-8");
+    if (looksBinary(text)) return `(binary content, ${body.length} bytes; not readable as text)`;
+    return truncatePageText(withTruncation(renderBody(text, ""), truncated), options.maxChars);
+  } finally {
+    await handle.close().catch(() => {});
+  }
+}
+
 export async function fetchPageText(
   url: string,
   options: FetchPageOptions = {},
@@ -1250,6 +1333,18 @@ export async function fetchPageText(
   const policy = options.websitePolicy ?? null;
   const maxChars = options.maxChars;
   const rawFetch = options.rawFetch ?? fetchUrlRaw;
+  if (options.allowLocalFiles !== false) {
+    const localPath = parseLocalPath(url);
+    if (localPath !== null) {
+      if (!localPath) return "Failed to read file: invalid file URL.";
+      return readLocalFile(localPath, {
+        signal,
+        maxChars,
+        maxBytes: options.maxBytes,
+        maxPdfBytes: options.maxPdfBytes,
+      });
+    }
+  }
   url = normalizeUrlScheme(url);
   const [allowed, reason] = checkUrlAccess(url, policy);
   if (!allowed) return reason;
@@ -1257,6 +1352,7 @@ export async function fetchPageText(
     deadlineMs,
     signal,
     websitePolicy: policy,
+    allowPrivateAddresses: options.allowPrivateAddresses,
     maxBytes: options.maxBytes,
     maxPdfBytes: options.maxPdfBytes,
     seams: options.seams,