pi-unsloth-webtools 0.2.3 → 0.2.5

package/README.md

@@ -45,6 +45,8 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   validation and fetch.
 - GitHub repo root pages are rewritten to the unauthenticated README API
   (`Accept: application/vnd.github.raw+json`), falling back to the HTML page on failure.
+- HTTP 4xx/5xx responses are reported as errors with the status reason instead of
+  returning error-page content.
 - Up to 4 redirect hops, each re-validated and re-resolved against the same rules.
 - 512 KiB download cap (10 MiB for PDFs), overall deadline + per-hop socket timeouts, abort-aware
   (`signal` cancels mid-flight).
@@ -53,7 +55,7 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   pymupdf4llm-style markdown layer (headings, bold/italic, code fences, links, tables)
   with Studio's corrupted/incomplete fallback to plain text.
 - Content sniffing: MIME allow/deny, binary magic signatures, PDF magic detection, and charset
-  decoding (declared charset, BOM sniffing for UTF-8/16/32, `<meta charset>` sniffing for
+  decoding (BOM sniffing for UTF-8/16/32 first, then the declared charset, `<meta charset>` sniffing for
   CJK and Windows/ISO encodings, cp1252 rescue for mislabeled single-byte pages).
 - HTML → Markdown conversion ported from Studio's dependency-free `_html_to_md.py`: headings,
   links, emphasis, lists, tables, blockquotes, code fences, entity decoding; hidden-element
@@ -78,7 +80,9 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   database.
 - Empty sweeps: ddgs 9.14.4 raises the last engine exception; this port reports a
   timeout whenever any engine timed out, so the timeout message is not masked by later
-  generic engine failures.
+  generic engine failures. The timeout budget bounds the entire sweep: per-engine
+  timeouts shrink as the budget is consumed, so the reported timeout matches the
+  worst-case wall time.
 - Proxies: Studio routes through environment proxies; this port always connects
   directly with DNS pinning (deliberately out of scope).
 

package/engines.ts

@@ -472,6 +472,28 @@ async function httpPost(
   return httpFetch(url, { ...options, method: "POST", body: new URLSearchParams(data).toString() });
 }
 
+const MAX_ENGINE_RESPONSE_BYTES = 5 * 1024 * 1024;
+
+async function readBodyCapped(response: Response): Promise<string | null> {
+  const declared = Number(response.headers.get("content-length") ?? "0");
+  if (declared > MAX_ENGINE_RESPONSE_BYTES) return null;
+  if (!response.body) return "";
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    total += value.length;
+    if (total > MAX_ENGINE_RESPONSE_BYTES) {
+      await reader.cancel();
+      return null;
+    }
+    chunks.push(value);
+  }
+  return new TextDecoder("utf-8").decode(Buffer.concat(chunks));
+}
+
 async function httpFetch(
   url: string,
   options: {
@@ -488,6 +510,7 @@ async function httpFetch(
     Accept: "*/*",
     ...options.headers,
   };
+  if (options.method === "POST") headers["Content-Type"] = "application/x-www-form-urlencoded";
   const cookie = options.cookies
     ? Object.entries(options.cookies)
         .map(([key, value]) => `${key}=${value}`)
@@ -505,13 +528,18 @@ async function httpFetch(
       signal: AbortSignal.any(signals),
     });
   } catch (err) {
-    if (err instanceof DOMException && err.name === "TimeoutError") {
-      throw new Error("timed out");
-    }
+    if (err instanceof DOMException && err.name === "TimeoutError") throw new SearchTimeoutError();
+    if (err instanceof DOMException && err.name === "AbortError") throw new SearchCancelled();
     throw err;
   }
   if (response.status !== 200) return null;
-  return response.text();
+  try {
+    return await readBodyCapped(response);
+  } catch (err) {
+    if (err instanceof DOMException && err.name === "TimeoutError") throw new SearchTimeoutError();
+    if (err instanceof DOMException && err.name === "AbortError") throw new SearchCancelled();
+    throw err;
+  }
 }
 
 const DUCKDUCKGO: Engine = {
@@ -668,6 +696,7 @@ const WIKIPEDIA: Engine = {
   provider: "wikipedia",
   priority: 2,
   async search(query, ctx, timeoutMs, signal) {
+    const started = Date.now();
     const lang = ctx.region.toLowerCase().split("-")[1] ?? "en";
     const encoded = encodeURIComponent(query);
     const opensearchUrl =
@@ -688,7 +717,7 @@ const WIKIPEDIA: Engine = {
     const extractUrl =
       `https://${lang}.wikipedia.org/w/api.php?action=query&format=json&prop=extracts` +
       `&titles=${encodeURIComponent(title)}&explaintext=0&exintro=0&redirects=1`;
-    const extract = await httpGet(extractUrl, {}, { timeoutMs, signal });
+    const extract = await httpGet(extractUrl, {}, { timeoutMs: Math.max(1, timeoutMs - (Date.now() - started)), signal });
     if (extract) {
       try {
         const pageData = JSON.parse(extract) as {
@@ -788,23 +817,27 @@ export async function autoTextSearch(
   signal?: AbortSignal,
 ): Promise<SearchResult[]> {
   const engines = shuffledEngines();
+  const deadline = Date.now() + timeoutMs;
   const seenProviders = new Set<string>();
   const aggregator = new ResultsAggregator();
   const ctx: EngineContext = { region: "us-en", safesearch: "moderate" };
   let timedOut = false;
+  let cancelled = false;
   const uniqueProviders = new Set(engines.map((e) => e.provider)).size;
   const maxWorkers = Math.min(uniqueProviders, Math.ceil(maxResults / 10) + 1);
   let i = 0;
   let pending: Promise<void>[] = [];
   const run = async (engine: Engine) => {
+    const remaining = Math.max(1, deadline - Date.now());
     try {
-      const results = await engine.search(query, ctx, timeoutMs, signal);
+      const results = await engine.search(query, ctx, remaining, signal);
       if (results && results.length) {
         aggregator.extend(results);
         seenProviders.add(engine.provider);
       }
     } catch (e) {
-      if (e instanceof Error && e.message.includes("timed out")) timedOut = true;
+      if (e instanceof SearchCancelled) cancelled = true;
+      if (e instanceof SearchTimeoutError) timedOut = true;
     }
   };
   while (i < engines.length) {
@@ -818,6 +851,7 @@ export async function autoTextSearch(
     }
   }
   await Promise.allSettled(pending);
+  if (cancelled) throw new SearchCancelled();
   const results = rankResults(aggregator.extractDicts(), query);
   if (results.length) return results.slice(0, maxResults);
   if (timedOut) throw new SearchTimeoutError();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.2.3",
+  "version": "0.2.5",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools ported from the Unsloth Studio codebase (DuckDuckGo search, SSRF-safe direct fetching, HTML-to-Markdown extraction)",
   "main": "index.ts",
@@ -51,7 +51,8 @@
     "test:unit": "vitest run --exclude **/smoke.test.ts",
     "test:smoke": "vitest run test/smoke.test.ts",
     "typecheck": "tsc --noEmit",
-    "prepublishOnly": "npm run typecheck && npm run test:unit"
+    "check:package": "node -e \"const fs=require('fs');const pkg=require('./package.json');const listed=new Set(pkg.files);const bad=[];for(const f of pkg.files){if(!fs.existsSync(f))bad.push('missing file: '+f)}for(const f of fs.readdirSync('.').filter(f=>f.endsWith('.ts'))){if(!listed.has(f))bad.push('unlisted source: '+f)}if(bad.length){console.error(bad.join('\\n'));process.exit(1)}\"",
+    "prepublishOnly": "npm run typecheck && npm run test:unit && npm run check:package"
   },
   "devDependencies": {
     "@earendil-works/pi-coding-agent": "^0.84.0",

package/web-fetch.ts

@@ -10,6 +10,7 @@ import {
   type WebsitePolicy,
 } from "./web-access.ts";
 import { htmlToMarkdown } from "./html-to-md.ts";
+import { INVALID_CHARREFS } from "./entities.ts";
 import { extractPdfText, PdfParseError } from "./pdf.ts";
 import { randomUserAgent } from "./user-agents.ts";
 
@@ -97,14 +98,17 @@ const ASCII_TEXT_BYTES = new Set<number>([
   0x1b,
 ]);
 
-const CP1252_HIGH: Record<number, number> = {
-  0x80: 0x20ac, 0x82: 0x201a, 0x83: 0x0192, 0x84: 0x201e, 0x85: 0x2026,
-  0x86: 0x2020, 0x87: 0x2021, 0x88: 0x02c6, 0x89: 0x2030, 0x8a: 0x0160,
-  0x8b: 0x2039, 0x8c: 0x0152, 0x8e: 0x017d, 0x91: 0x2018, 0x92: 0x2019,
-  0x93: 0x201c, 0x94: 0x201d, 0x95: 0x2022, 0x96: 0x2013, 0x97: 0x2014,
-  0x98: 0x02dc, 0x99: 0x2122, 0x9a: 0x0161, 0x9b: 0x203a, 0x9c: 0x0153,
-  0x9e: 0x017e, 0x9f: 0x0178,
-};
+export class FetchCancelledError extends Error {
+  constructor() {
+    super("cancelled");
+  }
+}
+
+export class FetchTimeoutError extends Error {
+  constructor() {
+    super("timed out");
+  }
+}
 
 export interface FetchPageOptions {
   timeoutMs?: number;
@@ -145,6 +149,8 @@ export interface HopOptions {
   maxBytes: number;
   maxPdfBytes: number;
   inactivityMs: number;
+  deadlineMs?: number;
+  nowMs?: () => number;
   signal?: AbortSignal;
 }
 
@@ -339,8 +345,8 @@ function decodeSingleByte(bytes: Buffer, cp1252: boolean): string {
   for (const byte of bytes) {
     if (byte < 0x80) {
       out += String.fromCharCode(byte);
-    } else if (cp1252 && byte in CP1252_HIGH) {
-      out += String.fromCodePoint(CP1252_HIGH[byte]);
+    } else if (cp1252 && byte in INVALID_CHARREFS) {
+      out += INVALID_CHARREFS[byte];
     } else {
       out += String.fromCharCode(byte);
     }
@@ -431,7 +437,7 @@ function fetchBudgetExceeded(
 }
 
 
-function requestHop(opts: HopOptions): Promise<HopResponse> {
+export function requestHop(opts: HopOptions): Promise<HopResponse> {
   return new Promise((resolve, reject) => {
     const url = opts.url;
     const transport = url.protocol === "https:" ? https : http;
@@ -446,27 +452,49 @@ function requestHop(opts: HopOptions): Promise<HopResponse> {
       lookup: (_host, _opts, callback) =>
         callback(null, [{ address: opts.pinnedIp, family: opts.family }]),
     };
+    let settled = false;
+    const settle = (action: () => void) => {
+      if (settled) return;
+      settled = true;
+      opts.signal?.removeEventListener("abort", onAbort);
+      action();
+    };
     const request = transport.request(options, (res: IncomingMessage) => {
       const chunks: Buffer[] = [];
       let total = 0;
+      let head = Buffer.alloc(0);
       const declaredPdf = String(res.headers["content-type"] ?? "").toLowerCase().includes("pdf");
       let limit = declaredPdf ? opts.maxPdfBytes : opts.maxBytes;
       let extendedForPdf = false;
       const finish = (err: string | null, body: Buffer) => {
         settle(() => {
-          if (err) reject(new Error(err));
-          else
+          if (err) {
+            if (err === "cancelled") reject(new FetchCancelledError());
+            else if (err === "timed out") reject(new FetchTimeoutError());
+            else reject(new Error(err));
+          } else {
             resolve({
               status: res.statusCode ?? 0,
               headers: res.headers as Record<string, string | string[] | undefined>,
               body,
             });
+          }
         });
       };
       res.on("data", (chunk: Buffer) => {
         if (settled) return;
+        const now = opts.nowMs ?? Date.now;
+        if (opts.deadlineMs !== undefined && now() >= opts.deadlineMs) {
+          res.destroy();
+          finish("timed out", Buffer.concat(chunks));
+          return;
+        }
+        if (head.length < 1024) {
+          const need = 1024 - head.length;
+          head = Buffer.concat([head, chunk.subarray(0, Math.min(need, chunk.length))]);
+        }
         if (!declaredPdf && !extendedForPdf && total + chunk.length > opts.maxBytes) {
-          if (hasPdfMagic(Buffer.concat(chunks))) {
+          if (hasPdfMagic(head)) {
             limit = opts.maxPdfBytes;
             extendedForPdf = true;
           }
@@ -488,16 +516,9 @@ function requestHop(opts: HopOptions): Promise<HopResponse> {
       res.on("end", () => finish(null, Buffer.concat(chunks)));
       res.on("error", (err) => finish(err.message, Buffer.concat(chunks)));
     });
-    const onAbort = () => request.destroy(new Error("cancelled"));
+    const onAbort = () => request.destroy(new FetchCancelledError());
     opts.signal?.addEventListener("abort", onAbort, { once: true });
-    let settled = false;
-    const settle = (action: () => void) => {
-      if (settled) return;
-      settled = true;
-      opts.signal?.removeEventListener("abort", onAbort);
-      action();
-    };
-    request.on("timeout", () => request.destroy(new Error("timed out")));
+    request.on("timeout", () => request.destroy(new FetchTimeoutError()));
     request.on("error", (err) => settle(() => reject(err)));
     request.end();
   });
@@ -557,9 +578,15 @@ export async function fetchUrlRaw(
         maxBytes,
         maxPdfBytes,
         inactivityMs: inactivity,
+        deadlineMs: deadline,
+        nowMs: now,
         signal,
       });
     } catch (err) {
+      if (err instanceof FetchCancelledError)
+        return { error: "Failed to fetch URL: cancelled.", body: "", contentType: "" };
+      if (err instanceof FetchTimeoutError)
+        return { error: "Failed to fetch URL: timed out.", body: "", contentType: "" };
       const message = err instanceof Error ? err.message : String(err);
       if (message === "cancelled")
         return { error: "Failed to fetch URL: cancelled.", body: "", contentType: "" };
@@ -606,6 +633,15 @@ export async function fetchUrlRaw(
     budgetError = fetchBudgetExceeded(deadline, signal, now);
     if (budgetError !== null) return { error: budgetError, body: "", contentType: "" };
 
+    if (response.status >= 400) {
+      const reason = http.STATUS_CODES[response.status] ?? "";
+      return {
+        error: `Failed to fetch URL: HTTP ${response.status}${reason ? ` ${reason}` : ""}`,
+        body: "",
+        contentType: "",
+      };
+    }
+
     const contentTypeHeader = response.headers["content-type"];
     const contentType = contentTypeHeader
       ? (/^[\w.+-]+\/[\w.+-]+/.exec(String(contentTypeHeader).toLowerCase()) ?? [""])[0]
@@ -657,7 +693,7 @@ export async function fetchUrlRaw(
     const bomCodec = bomCodecFor(response.body);
     const rawHtml = decodeWithCodec(
       response.body,
-      declaredCodec ?? bomCodec ?? sniffMetaCharsetForHtml(response.body, contentType) ?? "utf-8",
+      bomCodec ?? declaredCodec ?? sniffMetaCharsetForHtml(response.body, contentType) ?? "utf-8",
     );
 
     if (looksBinary(rawHtml)) {

package/web-search.ts

@@ -93,7 +93,7 @@ export function searchFailureMessage(exc: unknown, timeoutMs = SEARCH_TIMEOUT_MS
 export function formatSearchResults(results: SearchResult[]): string {
   const parts = results.map((result) => {
     const title = String(result.title ?? "").replace(/\s+/g, " ");
-    const href = String(result.href ?? "").trim();
+    const href = String(result.href ?? "").replace(/\s+/g, " ").trim();
     const snippet = String(result.body ?? "").replace(/\s+/g, " ");
     return `Title: ${title}\nURL: ${href}\nSnippet: ${snippet}`;
   });