pi-unsloth-webtools 0.4.2 → 0.5.1

package/README.md

@@ -174,10 +174,35 @@ Optional settings in `~/.pi/agent/settings.json` or `.pi/settings.json` (project
 |---|---|---|
 | `unslothWebTools.maxResults` | `5` | Default `maxResults` for `web_search` (clamped 1-20) |
 | `unslothWebTools.maxChars` / `webFetch.maxChars` / `smartFetchDefaultMaxChars` | tool param | Default `maxChars` for `web_fetch` and `web_search` url mode |
-| `unslothWebTools.timeoutMs` / `webFetch.timeoutMs` / `smartFetchDefaultTimeoutMs` | `60000` | Default `timeoutMs` (>=1000) |
+| `unslothWebTools.timeoutMs` / `webFetch.timeoutMs` / `smartFetchDefaultTimeoutMs` | `60000` fetch, `300000` search | Default `timeoutMs` when the tool param is absent (>=1000). Fetch and `web_search` url mode fall back to 60000; `web_search` query mode falls back to 300000 |
 | `webSearch.maxResults` / `smartWebSearch.resultsPerQuery` | same as above | Legacy aliases for `maxResults` |
+| `websitePolicy` | none | Not read from settings. Tools run unrestricted by default; `websitePolicy` is a programmatic option the host passes to `webSearch` / `fetchPageText` |
 
-Tool params always win over file defaults.
+Tool params always win over file defaults. Search dedup also strips default ports, so `https://example.com:443/a` and `https://example.com/a` collapse.
+
+Environment overrides: `PI_UNSLOTH_CACHE_DIR` changes the fetch cache directory, `PI_UNSLOTH_WEBTOOLS_STATS` opts into append-only sweep stats JSONL, `PI_CODING_AGENT_DIR` / `PI_AGENT_DIR` change the global settings directory. Cache entries live 1 hour and stale copies are served only after a network failure.
+
+## Troubleshooting
+
+Match on the exact prefix. Do not retry blocked hosts with spelling tricks.
+
+| Symptom | Exact string | Action |
+|---|---|---|
+| Empty query | `No query provided.` | Pass `query` or `url`. |
+| Cancelled | `Search cancelled.` / `Failed to fetch URL: cancelled.` | Retry only if the abort was not intentional. |
+| Search timeout | `Search failed: the search engines (...) did not respond within N seconds.` | Retry, narrow the query, or raise `timeoutMs`. |
+| No results | `No results found.` | Rephrase the query. |
+| Policy filtered everything | `No results found within the website access limits.` | Widen `websitePolicy`, do not work around it. |
+| Blocked URL | `Blocked: ...` | Respect it. Includes non-http schemes, credentials, invalid hosts, non-public IPs, and policy denials. |
+| HTTP failure | `Failed to fetch URL: HTTP ...` | Fix the URL. A 404 automatically tries a Wayback snapshot. |
+| Non-text / binary | `(non-text content:` / `(binary content,` | Not readable as text by design. |
+| PDF without text | `(PDF contains no extractable text)` / `(PDF content could not be read as text...)` | Scanned or encrypted PDF. |
+| Download cap hit | `... (page truncated at the download limit)` | Raw fetch hit 512 KiB (10 MiB for PDFs). |
+| maxChars cut | `... (truncated, N chars total)` | Raise `maxChars` for the full text. |
+| Empty page | `(page returned no readable text)` | Page had no extractable text; JS-rendered pages need a browser tool. |
+| GitHub rewrite | `README of ... (fetched via the GitHub README API):` | Expected repo-root rewrite, not the HTML chrome. |
+| Cache fallback | `Served from cache` / `STALE cache from YYYY-MM-DD` | Network failed; output is the cached copy with its date. |
+| Wayback fallback | `Fetched from Wayback Machine snapshot (YYYY-MM-DD) for ...` | Original 404'd; output is the archived copy with its date. |
 
 ## Development
 

package/cache.ts

@@ -98,9 +98,17 @@ async function prune(): Promise<void> {
   } catch {
     return;
   }
+  const now = Date.now();
+  const tmpFiles = files.filter((f) => f.includes(".tmp."));
+  for (const file of tmpFiles) {
+    try {
+      const full = join(dir, file);
+      const s = await stat(full);
+      if (now - s.mtimeMs >= CACHE_TTL_MS) await unlinkSafe(full);
+    } catch {}
+  }
   const jsonFiles = files.filter((f) => f.endsWith(".json"));
   if (!jsonFiles.length) return;
-  const now = Date.now();
   const entries: { file: string; size: number; timestamp: number }[] = [];
   let total = 0;
   for (const file of jsonFiles) {

package/engines.ts

@@ -82,6 +82,9 @@ export function canonicalizeHref(href: string): string {
       }
     }
     url.hash = "";
+    if ((url.protocol === "http:" && url.port === "80") || (url.protocol === "https:" && url.port === "443")) {
+      url.port = "";
+    }
     return url.toString();
   } catch {
     return href;
@@ -954,9 +957,9 @@ export async function autoTextSearch(
   const timedOutProviders = new Set<string>();
   let cancelled = false;
   const uniqueProviders = new Set(engines.map((e) => e.provider)).size;
-  const maxWorkers = Math.min(uniqueProviders, Math.ceil(maxResults / 10) + 1);
+  const maxWorkers = Math.min(uniqueProviders, Math.max(4, Math.ceil(maxResults / 5) + 1));
   let i = 0;
-  let pending: Promise<void>[] = [];
+  const pending = new Set<Promise<void>>();
   const run = async (engine: Engine) => {
     let results: SearchResult[] | null = null;
     for (let attempt = 0; attempt < 2 && results === null; attempt++) {
@@ -1000,18 +1003,35 @@ export async function autoTextSearch(
       if (aggregator.size >= maxResults) controller.abort();
     }
   };
-  while (i < engines.length) {
+  while (i < engines.length || pending.size > 0) {
     if (aggregator.size >= maxResults || cancelled) {
       controller.abort();
       break;
     }
-    const engine = engines[i++];
-    if (seenProviders.has(engine.provider)) continue;
-    pending.push(run(engine));
-    if (pending.length >= maxWorkers) {
-      await Promise.allSettled(pending);
-      pending = [];
+    while (i < engines.length && pending.size < maxWorkers) {
+      if (aggregator.size >= maxResults || cancelled) {
+        controller.abort();
+        break;
+      }
+      const engine = engines[i++];
+      if (seenProviders.has(engine.provider)) continue;
+      const task = run(engine);
+      pending.add(task);
+      void task.then(
+        () => {
+          pending.delete(task);
+        },
+        () => {
+          pending.delete(task);
+        },
+      );
+    }
+    if (pending.size === 0) break;
+    if (aggregator.size >= maxResults || cancelled) {
+      controller.abort();
+      break;
     }
+    await Promise.race(pending);
   }
   await Promise.allSettled(pending);
   if (onAbort && signal) signal.removeEventListener("abort", onAbort);

package/index.ts

@@ -1,8 +1,8 @@
 import { defineTool, type ExtensionAPI, type ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
-import { webSearch as defaultWebSearch } from "./web-search.ts";
+import { SEARCH_TIMEOUT_MS, webSearch as defaultWebSearch } from "./web-search.ts";
 import { DEFAULT_FETCH_TIMEOUT_MS, fetchPageText as defaultFetchPageText } from "./web-fetch.ts";
-import { loadDefaultFetchSettings } from "./settings.ts";
+import { loadDefaultFetchSettings, loadDefaultFetchTimeoutMs } from "./settings.ts";
 
 function positiveNumber(value: unknown): number | undefined {
   if (typeof value !== "number" || !Number.isFinite(value)) return undefined;
@@ -107,11 +107,14 @@ export function createWebTools(deps: WebToolsDeps = {}) {
           };
         }
         onUpdate?.({ content: [{ type: "text", text: "Searching the web..." }], details: {} });
+        const timeoutParam = positiveNumber(params.timeoutMs);
+        const searchCwd = (_ctx as ExtensionContext | undefined)?.cwd;
+        const searchTimeoutMs = timeoutParam ?? (await loadDefaultFetchTimeoutMs(searchCwd)) ?? SEARCH_TIMEOUT_MS;
         const text = await webSearch(params.query, {
           signal: signal ?? undefined,
-          timeoutMs: positiveNumber(params.timeoutMs),
+          timeoutMs: searchTimeoutMs,
           maxResults: positiveNumber(params.maxResults),
-          cwd: (_ctx as ExtensionContext | undefined)?.cwd,
+          cwd: searchCwd,
         });
         return { content: [{ type: "text", text }], details: {} };
       },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.4.2",
+  "version": "0.5.1",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools ported from the Unsloth Studio codebase (DuckDuckGo search, SSRF-safe direct fetching, HTML-to-Markdown extraction)",
   "main": "index.ts",
@@ -55,16 +55,22 @@
     "test:smoke": "vitest run test/smoke.test.ts",
     "typecheck": "tsc --noEmit",
     "check:package": "node -e \"const fs=require('fs');const pkg=require('./package.json');const listed=new Set(pkg.files);const bad=[];for(const f of pkg.files){if(!fs.existsSync(f))bad.push('missing file: '+f)}for(const f of fs.readdirSync('.').filter(f=>f.endsWith('.ts'))){if(!listed.has(f))bad.push('unlisted source: '+f)}if(bad.length){console.error(bad.join('\\n'));process.exit(1)}\"",
-    "prepublishOnly": "npm run typecheck && npm run test:unit && npm run check:package"
+    "prepublishOnly": "npm run typecheck && npm run lint && npm run test:unit && npm run check:package && npm run check:publint",
+    "lint": "oxlint .",
+    "check:publint": "publint",
+    "check:attw": "attw --pack"
   },
   "devDependencies": {
     "@earendil-works/pi-coding-agent": "^0.84.0",
     "@types/node": "^24.0.0",
     "typebox": "^1.3.7",
     "typescript": "~5.9.3",
-    "vitest": "^4.1.9"
+    "vitest": "^4.1.9",
+    "@arethetypeswrong/cli": "^0.18.5",
+    "oxlint": "^1.81.0",
+    "publint": "^0.3.24"
   },
-  "dependencies": {
+  "optionalDependencies": {
     "mupdf": "^1.28.0"
   }
 }

package/pdf.ts

@@ -2,6 +2,7 @@ import { createRequire } from "node:module";
 import { inflateSync } from "node:zlib";
 
 export const MAX_WEB_PDF_PAGES = 50;
+const MAX_PDF_INFLATE_BYTES = 64 * 1024 * 1024;
 
 export class PdfParseError extends Error {
   constructor() {
@@ -786,7 +787,7 @@ async function extractPdfTextFallback(data: Buffer): Promise<string> {
     let bytes = raw;
     if (isFlate) {
       try {
-        bytes = inflateSync(raw);
+        bytes = inflateSync(raw, { maxOutputLength: MAX_PDF_INFLATE_BYTES });
       } catch {
         continue;
       }

package/user-agents.ts

@@ -1,12 +1,11 @@
 const USER_AGENTS = [
-  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:145.0) Gecko/20100101 Firefox/145.0",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:145.0) Gecko/20100101 Firefox/145.0",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15",
 ];
-
 export function randomUserAgent(): string {
   return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
 }

package/web-access.ts

@@ -381,6 +381,8 @@ function ipv4Octets(ip: string): number[] | null {
 }
 
 export function isPublicIp(ip: string): boolean {
+  if (ip.includes("%")) return false;
+  if (ip.includes(".") && ip.includes(":")) return false;
   if (ip.includes(".")) {
     const o = ipv4Octets(ip);
     if (!o) return false;

package/web-fetch.ts

@@ -631,8 +631,14 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
       headers: opts.headers,
       timeout: opts.inactivityMs,
       servername: url.protocol === "https:" ? url.hostname : undefined,
-      lookup: (_host, _opts, callback) =>
-        callback(null, [{ address: opts.pinnedIp, family: opts.family }]),
+      lookup: ((_hostname: string, _options: unknown, _callback: unknown) => {
+        const done = (typeof _options === "function" ? _options : _callback) as (err: unknown, address: unknown, family?: unknown) => void;
+        if ((_options as { all?: boolean } | undefined)?.all) {
+          done(null, [{ address: opts.pinnedIp, family: opts.family }]);
+        } else {
+          done(null, opts.pinnedIp, opts.family);
+        }
+      }) as typeof options.lookup,
     };
     let settled = false;
     let resRef: IncomingMessage | null = null;
@@ -798,7 +804,7 @@ export async function fetchUrlRaw(
   url: string,
   options: RawFetchOptions = {},
 ): Promise<RawFetchResult> {
-  const timeoutMs = options.timeoutMs ?? 30_000;
+  const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
   const now = options.nowMs ?? Date.now;
   const deadline = options.deadlineMs ?? now() + timeoutMs;
   const signal = options.signal;
@@ -854,7 +860,13 @@ export async function fetchUrlRaw(
       Accept: "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.5",
       "Accept-Encoding": "identity",
     };
-    if (options.extraHeaders) Object.assign(headers, options.extraHeaders);
+    if (options.extraHeaders) {
+      for (const [key, value] of Object.entries(options.extraHeaders)) {
+        if (key.toLowerCase() === "host") continue;
+        headers[key] = value;
+      }
+    }
+    headers["Host"] = hostHeader;
     const inactivity = remainingMs(deadline, now);
     let response: HopResponse;
     try {
@@ -1212,6 +1224,11 @@ async function fetchWaybackSnapshot(
     return null;
   }
   if (!snapshotUrl) return null;
+  const snapshotAccess = checkUrlAccess(snapshotUrl, options.websitePolicy ?? null);
+  if (!snapshotAccess[0]) return null;
+  const snapshotHost = snapshotAccess[2];
+  const snapshotIsIp = snapshotHost.includes(":") || /^\d+\.\d+\.\d+\.\d+$/.test(snapshotHost);
+  if (snapshotIsIp && !isPublicIp(snapshotHost)) return null;
   if (now() >= deadlineMs) return null;
   let snapResult: RawFetchResult | null = null;
   try {

package/web-search.ts

@@ -11,7 +11,7 @@ import {
 
 export { EmptySweepError, SearchCancelled, SearchTimeoutError } from "./engines.ts";
 
-const SEARCH_TIMEOUT_MS = 300_000;
+export const SEARCH_TIMEOUT_MS = 300_000;
 const MAX_RESULTS = 5;
 
 export const EMPTY_SEARCH_RESULTS = [
@@ -56,7 +56,8 @@ export async function webSearch(
   const saneMax = Number.isFinite(rawMaxResults) ? Math.floor(rawMaxResults as number) : MAX_RESULTS;
   const maxResults = Math.min(20, Math.max(1, saneMax));
   const rawTimeout = options.timeoutMs ?? SEARCH_TIMEOUT_MS;
-  const timeoutMs = Number.isFinite(rawTimeout) ? (rawTimeout as number) : SEARCH_TIMEOUT_MS;
+  const flooredTimeout = Number.isFinite(rawTimeout) ? Math.floor(rawTimeout as number) : SEARCH_TIMEOUT_MS;
+  const timeoutMs = flooredTimeout >= 1 ? flooredTimeout : SEARCH_TIMEOUT_MS;
   const signal = options.signal;
   const policy = options.websitePolicy ?? null;
   const client = options.client ?? ddgSearch;