pi-unsloth-webtools 0.2.0 → 0.2.2

package/README.md

@@ -53,8 +53,8 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   pymupdf4llm-style markdown layer (headings, bold/italic, code fences, links, tables)
   with Studio's corrupted/incomplete fallback to plain text.
 - Content sniffing: MIME allow/deny, binary magic signatures, PDF magic detection, and charset
-  decoding (declared charset, BOM sniffing for UTF-8/16/32, cp1252 rescue for mislabeled
-  single-byte pages).
+  decoding (declared charset, BOM sniffing for UTF-8/16/32, `<meta charset>` sniffing for
+  CJK and Windows/ISO encodings, cp1252 rescue for mislabeled single-byte pages).
 - HTML → Markdown conversion ported from Studio's dependency-free `_html_to_md.py`: headings,
   links, emphasis, lists, tables, blockquotes, code fences, entity decoding; hidden-element
   stripping (`hidden`, `aria-hidden`, inline styles); `<article>`/`<main>` main-content scoping
@@ -65,8 +65,22 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
 - HTML entity decoding replicates CPython's `html.unescape` (full 2,231-entry HTML5 table,
   longest-prefix rule, Windows-1252 numeric mappings), matching Studio byte-for-byte.
 
-See `ROADMAP.md` for the remaining gaps (per-line styling, table detection, TLS
-fingerprinting, proxies).
+## Known differences from Studio
+
+- **PDF styling**: MuPDF.js exposes one font per line, so mixed-style lines style the
+  whole line instead of per-span; superscript/subscript/underline/strikeout/highlight
+  markers are not emitted. Tables use a conservative text-grid detector — aligned text
+  tables are detected, drawn-rule-only tables are not.
+- **Search engines**: Node's `fetch` TLS fingerprint differs from ddgs's `primp`
+  impersonation, so Google/Brave/Yahoo/Yandex may block or serve consent pages more
+  aggressively (a blocked engine simply contributes no results). User agents are a
+  fixed browser set plus ddgs's Android Google UA generator, not `fake_useragent`'s
+  database.
+- **Empty sweeps**: ddgs 9.14.4 raises the last engine exception; this port reports a
+  timeout whenever any engine timed out, so the timeout message is not masked by later
+  generic engine failures.
+- **Proxies**: Studio routes through environment proxies; this port always connects
+  directly with DNS pinning (deliberately out of scope).
 
 ## Development
 
@@ -74,10 +88,15 @@ fingerprinting, proxies).
 npm install
 npm run typecheck
 npm test
+npm run test:unit
+npm run test:smoke
 ```
 
 ## Tests
 
+`npm test` runs the full suite. `npm run test:unit` skips the live-network smoke tests,
+and `npm run test:smoke` runs only those.
+
 The suite ports Unsloth Studio's own tests for these tools:
 
 - `test/html-to-md.test.ts` — hidden-element stripping and main-content scoping (from
@@ -94,6 +113,8 @@ The suite ports Unsloth Studio's own tests for these tools:
   aggregator, the ranker, and the Wikipedia engine with a stubbed fetch
 - `test/pdf-parity.test.ts` — MuPDF engine capabilities: PDF 1.5 object streams,
   ASCII85Decode, font `/Differences` encodings, pymupdf4llm-style headings/links/tables
+- `test/entities.test.ts` — `decodeHtmlEntities` parity with CPython `html.unescape`: legacy
+  refs, longest-prefix rule, Windows-1252 numeric mappings, invalid codepoints
 - `test/smoke.test.ts` — live network checks against real hosts
 
 The seams (`seams.resolve` / `seams.request` / `rawFetch`) replace the network stack

package/engines.ts

@@ -822,7 +822,7 @@ export async function autoTextSearch(
   const seenProviders = new Set<string>();
   const aggregator = new ResultsAggregator();
   const ctx: EngineContext = { region: "us-en", safesearch: "moderate" };
-  let err: unknown = null;
+  let timedOut = false;
   const uniqueProviders = new Set(engines.map((e) => e.provider)).size;
   const maxWorkers = Math.min(uniqueProviders, Math.ceil(maxResults / 10) + 1);
   let i = 0;
@@ -835,7 +835,7 @@ export async function autoTextSearch(
         seenProviders.add(engine.provider);
       }
     } catch (e) {
-      err = e;
+      if (e instanceof Error && e.message.includes("timed out")) timedOut = true;
     }
   };
   while (i < engines.length) {
@@ -850,6 +850,6 @@ export async function autoTextSearch(
   }
   const results = rankResults(aggregator.extractDicts(), query);
   if (results.length) return results.slice(0, maxResults);
-  if (err instanceof Error && err.message.includes("timed out")) throw new SearchTimeoutError();
+  if (timedOut) throw new SearchTimeoutError();
   throw new EmptySweepError();
 }

package/html-to-md.ts

@@ -197,7 +197,7 @@ export function decodeHtmlEntities(text: string): string {
   return text.replace(CHARREF_RE, (whole, s: string) => {
     if (s[0] === "#") {
       const hex = s[1] === "x" || s[1] === "X";
-      const num = parseInt(s.slice(2).replace(/;+$/, ""), hex ? 16 : 10);
+      const num = parseInt(s.slice(hex ? 2 : 1).replace(/;+$/, ""), hex ? 16 : 10);
       const mapped = INVALID_CHARREFS[num];
       if (mapped !== undefined) return mapped;
       if ((num >= 0xd800 && num <= 0xdfff) || num > 0x10ffff) return "\ufffd";
@@ -227,6 +227,21 @@ interface HtmlHandlers {
 const START_TAG_NAME_RE = /^[a-zA-Z][^\s/>]*/;
 const ATTR_NAME_RE = /^[^\s=/>]+/;
 
+const RAW_TEXT_TAGS = [
+  "script",
+  "style",
+  "textarea",
+  "title",
+  "xmp",
+  "iframe",
+  "noembed",
+  "noframes",
+];
+
+const RAW_TEXT_CLOSERS: Record<string, RegExp> = Object.fromEntries(
+  RAW_TEXT_TAGS.map((name) => [name, new RegExp(`</${name}\\s*>`, "i")]),
+);
+
 function parseAttrsUntilClose(input: string, pos: number): [AttrDict, number, boolean] {
   const attrs: AttrDict = {};
   while (pos < input.length) {
@@ -346,8 +361,22 @@ export function feedHtml(input: string, handlers: HtmlHandlers): void {
       continue;
     }
     emitText(input.slice(textStart, i));
-    if (tag.kind === "start") handlers.handleStartTag(tag.name!, tag.attrs!);
-    else if (tag.kind === "startend") handlers.handleStartEndTag(tag.name!, tag.attrs!);
+    if (tag.kind === "start") {
+      const rawCloser = RAW_TEXT_CLOSERS[tag.name!];
+      if (rawCloser) {
+        const after = input.slice(tag.end);
+        const closeMatch = rawCloser.exec(after);
+        if (closeMatch) {
+          handlers.handleStartTag(tag.name!, tag.attrs!);
+          emitText(after.slice(0, closeMatch.index));
+          handlers.handleEndTag(tag.name!);
+          textStart = tag.end + closeMatch.index + closeMatch[0].length;
+          i = textStart;
+          continue;
+        }
+      }
+      handlers.handleStartTag(tag.name!, tag.attrs!);
+    } else if (tag.kind === "startend") handlers.handleStartEndTag(tag.name!, tag.attrs!);
     else if (tag.kind === "end") handlers.handleEndTag(tag.name!);
     textStart = tag.end;
     i = tag.end;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools ported from the Unsloth Studio codebase (DuckDuckGo search, SSRF-safe direct fetching, HTML-to-Markdown extraction)",
   "main": "index.ts",
@@ -31,7 +31,6 @@
     "entities.ts",
     "pdf.ts",
     "README.md",
-    "ROADMAP.md",
     "LICENSE"
   ],
   "pi": {
@@ -48,8 +47,10 @@
   },
   "scripts": {
     "test": "vitest run",
+    "test:unit": "vitest run --exclude **/smoke.test.ts",
+    "test:smoke": "vitest run test/smoke.test.ts",
     "typecheck": "tsc --noEmit",
-    "prepublishOnly": "npm run typecheck"
+    "prepublishOnly": "npm run typecheck && npm run test:unit"
   },
   "devDependencies": {
     "@earendil-works/pi-coding-agent": "^0.84.0",

package/web-fetch.ts

@@ -237,6 +237,34 @@ function hasSingleByteTextEvidence(data: Buffer): boolean {
   return ascii / data.length >= MIN_SINGLE_BYTE_ASCII_RATIO;
 }
 
+const CHARSET_ALIASES: Record<string, string> = {
+  gbk: "gbk",
+  gb2312: "gbk",
+  "gb-2312": "gbk",
+  cp936: "gbk",
+  "x-gbk": "gbk",
+  gb18030: "gb18030",
+  big5: "big5",
+  "big5-hkscs": "big5",
+  sjis: "shift_jis",
+  "x-sjis": "shift_jis",
+  cp932: "shift_jis",
+  "shift-jis": "shift_jis",
+  "euc-jp": "euc-jp",
+  "euc-kr": "euc-kr",
+  ksc5601: "euc-kr",
+  "ks_c_5601-1987": "euc-kr",
+  "ks_c_5601-1989": "euc-kr",
+  "iso-2022-jp": "iso-2022-jp",
+  "koi8-r": "koi8-r",
+  "koi8-u": "koi8-u",
+  cp866: "cp866",
+  "x-mac-cyrillic": "x-mac-cyrillic",
+  "windows-874": "windows-874",
+  cp874: "windows-874",
+  "tis-620": "tis-620",
+};
+
 function normalizeCharset(name: string): string | null {
   const n = name.trim().replace(/["']/g, "").toLowerCase();
   switch (n) {
@@ -270,8 +298,28 @@ function normalizeCharset(name: string): string | null {
     case "utf-32be":
       return "utf-32be";
     default:
-      return null;
+      break;
+  }
+  const alias = CHARSET_ALIASES[n];
+  if (alias !== undefined) return alias;
+  if (/^windows-125[0-8]$/.test(n) || /^iso-8859-(?:[2-9]|1[0-6])$/.test(n)) return n;
+  return null;
+}
+
+function sniffMetaCharset(bytes: Buffer): string | null {
+  const head = bytes.subarray(0, 1024).toString("latin1").toLowerCase();
+  const match =
+    /<meta\b[^>]*\bcharset\s*=\s*["']?\s*([a-z0-9_.\-]+)/.exec(head) ??
+    /<meta\b[^>]*\bhttp-equiv\s*=\s*["']?content-type["']?[^>]*\bcharset\s*=\s*["']?\s*([a-z0-9_.\-]+)/.exec(head);
+  return match ? normalizeCharset(match[1]) : null;
+}
+
+function sniffMetaCharsetForHtml(bytes: Buffer, contentType: string): string | null {
+  if (!contentType.includes("html")) {
+    const probe = bytes.subarray(0, 256).toString("latin1");
+    if (!looksLikeHtmlDocument(probe)) return null;
   }
+  return sniffMetaCharset(bytes);
 }
 
 function decodeUtf32(bytes: Buffer, littleEndian: boolean): string {
@@ -328,11 +376,38 @@ function decodeWithCodec(bytes: Buffer, codec: string | null): string {
     case "cp1252":
       return decodeSingleByte(bytes, true);
     case "iso8859-1":
-    default:
       return decodeSingleByte(bytes, false);
+    default:
+      return decodeWithLabel(bytes, codec);
   }
 }
 
+function decodeWithLabel(bytes: Buffer, label: string | null): string {
+  if (label === null) return decodeSingleByte(bytes, false);
+  if (label === "tis-620") return decodeTis620(bytes);
+  try {
+    return new TextDecoder(label, { fatal: false }).decode(bytes);
+  } catch {
+    return decodeSingleByte(bytes, false);
+  }
+}
+
+function decodeTis620(bytes: Buffer): string {
+  let out = "";
+  for (const byte of bytes) {
+    if (byte < 0x80) {
+      out += String.fromCharCode(byte);
+    } else if (byte >= 0xa1 && byte <= 0xfb) {
+      out += String.fromCodePoint(0x0e01 + byte - 0xa1);
+    } else if (byte === 0xa0) {
+      out += "\u00a0";
+    } else {
+      out += "\ufffd";
+    }
+  }
+  return out;
+}
+
 
 async function resolveAndValidate(hostname: string, signal?: AbortSignal): Promise<ResolvedHost> {
   let addresses: { address: string; family: number }[];
@@ -386,20 +461,19 @@ function requestHop(opts: HopOptions): Promise<HopResponse> {
       const declaredPdf = String(res.headers["content-type"] ?? "").toLowerCase().includes("pdf");
       let limit = declaredPdf ? opts.maxPdfBytes : opts.maxBytes;
       let extendedForPdf = false;
-      let done = false;
       const finish = (err: string | null, body: Buffer) => {
-        if (done) return;
-        done = true;
-        if (err) reject(new Error(err));
-        else
-          resolve({
-            status: res.statusCode ?? 0,
-            headers: res.headers as Record<string, string | string[] | undefined>,
-            body,
-          });
+        settle(() => {
+          if (err) reject(new Error(err));
+          else
+            resolve({
+              status: res.statusCode ?? 0,
+              headers: res.headers as Record<string, string | string[] | undefined>,
+              body,
+            });
+        });
       };
       res.on("data", (chunk: Buffer) => {
-        if (done) return;
+        if (settled) return;
         if (!declaredPdf && !extendedForPdf && total + chunk.length > opts.maxBytes) {
           if (hasPdfMagic(Buffer.concat(chunks))) {
             limit = opts.maxPdfBytes;
@@ -423,10 +497,17 @@ function requestHop(opts: HopOptions): Promise<HopResponse> {
       res.on("end", () => finish(null, Buffer.concat(chunks)));
       res.on("error", (err) => finish(err.message, Buffer.concat(chunks)));
     });
-    request.on("timeout", () => request.destroy(new Error("timed out")));
-    request.on("error", (err) => reject(err));
     const onAbort = () => request.destroy(new Error("cancelled"));
     opts.signal?.addEventListener("abort", onAbort, { once: true });
+    let settled = false;
+    const settle = (action: () => void) => {
+      if (settled) return;
+      settled = true;
+      opts.signal?.removeEventListener("abort", onAbort);
+      action();
+    };
+    request.on("timeout", () => request.destroy(new Error("timed out")));
+    request.on("error", (err) => settle(() => reject(err)));
     request.end();
   });
 }
@@ -471,6 +552,7 @@ export async function fetchUrlRaw(
       "User-Agent": userAgent,
       Host: hostHeader,
       Accept: "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.5",
+      "Accept-Encoding": "identity",
     };
     if (options.extraHeaders) Object.assign(headers, options.extraHeaders);
     const inactivity = Math.max(1, deadline - now());
@@ -497,8 +579,9 @@ export async function fetchUrlRaw(
 
     if (response.status >= 300 && response.status < 400) {
       if (![301, 302, 303, 307, 308].includes(response.status)) {
+        const reason = http.STATUS_CODES[response.status] ?? "";
         return {
-          error: `Failed to fetch URL: HTTP ${response.status} ${statusReason(response.status)}`,
+          error: `Failed to fetch URL: HTTP ${response.status}${reason ? ` ${reason}` : ""}`,
           body: "",
           contentType: "",
         };
@@ -577,7 +660,10 @@ export async function fetchUrlRaw(
 
     const declaredCodec = declaredCharset ? normalizeCharset(declaredCharset) : null;
     const bomCodec = bomCodecFor(response.body);
-    const rawHtml = decodeWithCodec(response.body, declaredCodec ?? bomCodec ?? "utf-8");
+    const rawHtml = decodeWithCodec(
+      response.body,
+      declaredCodec ?? bomCodec ?? sniffMetaCharsetForHtml(response.body, contentType) ?? "utf-8",
+    );
 
     if (looksBinary(rawHtml)) {
       let alt: string | null = null;
@@ -612,13 +698,6 @@ function bomCodecFor(bytes: Buffer): string | null {
   return null;
 }
 
-function statusReason(status: number): string {
-  const reasons: Record<number, string> = {
-    301: "Moved Permanently", 302: "Found", 303: "See Other", 307: "Temporary Redirect", 308: "Permanent Redirect",
-  };
-  return reasons[status] ?? "";
-}
-
 export function truncatePageText(text: string, maxChars?: number): string {
   if (!text) return "(page returned no readable text)";
   if (typeof maxChars === "number" && maxChars > 0 && text.length > maxChars) {

package/web-search.ts

@@ -64,23 +64,17 @@ export async function webSearch(
     const results = await client(effectiveQuery, wanted, signal);
     if (signal?.aborted) return "Search cancelled.";
     if (!results.length) return EMPTY_SEARCH_RESULTS[0];
-    const parts: string[] = [];
+    const allowed: SearchResult[] = [];
     for (const result of results) {
-      if (parts.length >= maxResults) break;
+      if (allowed.length >= maxResults) break;
       const href = String(result.href ?? "").trim();
       if (href && !checkUrlAccess(href, policy)[0]) continue;
-      const title = String(result.title ?? "").replace(/\s+/g, " ");
-      const snippet = String(result.body ?? "").replace(/\s+/g, " ");
-      parts.push(`Title: ${title}\nURL: ${href}\nSnippet: ${snippet}`);
+      allowed.push(result);
     }
-    if (!parts.length) return EMPTY_SEARCH_RESULTS[1];
-    const text = parts.join("\n\n---\n\n");
-    return (
-      text +
-      "\n\n---\n\nIMPORTANT: These are only short snippets. " +
-      'To get the full page content, call web_search with the url parameter (e.g. {"url": "<URL>"}).'
-    );
+    if (!allowed.length) return EMPTY_SEARCH_RESULTS[1];
+    return formatSearchResults(allowed);
   } catch (err) {
+    if (signal?.aborted) return "Search cancelled.";
     return searchFailureMessage(err, timeoutMs);
   }
 }
@@ -98,9 +92,9 @@ export function searchFailureMessage(exc: unknown, timeoutMs = SEARCH_TIMEOUT_MS
 
 export function formatSearchResults(results: SearchResult[]): string {
   const parts = results.map((result) => {
-    const title = result.title.replace(/\s+/g, " ");
-    const href = result.href.trim();
-    const snippet = result.body.replace(/\s+/g, " ");
+    const title = String(result.title ?? "").replace(/\s+/g, " ");
+    const href = String(result.href ?? "").trim();
+    const snippet = String(result.body ?? "").replace(/\s+/g, " ");
     return `Title: ${title}\nURL: ${href}\nSnippet: ${snippet}`;
   });
   const text = parts.join("\n\n---\n\n");

package/ROADMAP.md

@@ -1,77 +0,0 @@
-# Roadmap
-
-Tracking the remaining gaps between this port and Unsloth Studio's web tools, and the
-planned work to close them.
-
-## Implemented
-
-### PDF text extraction (MuPDF engine)
-
-`pdf.ts` uses the official **MuPDF.js** (`mupdf` npm package) — the same C engine that
-PyMuPDF wraps — replacing the earlier minimal built-in extractor:
-
-- Full xref handling: tables, cross-reference streams, and PDF 1.5+ object streams
-- All standard stream filters (FlateDecode, ASCII85Decode, LZW, RunLength, DCT, JPX, ...)
-- Font encodings and ToUnicode mapping (non-Latin text extracts correctly)
-- Encryption detection via `needsPassword()` (reported as unreadable, matching pymupdf's
-  no-password behavior)
-- The markdown layer replicates pymupdf4llm's algorithm: `IdentifyHeaders` font-size
-  heading detection, `get_raw_lines` line reconstruction (tolerance 3, 10% span-join
-  delta), `write_text` styling (bold/italic/mono, code fences, bullets, link
-  resolution with `%0x`-escaped URIs), and Studio's corrupted/incomplete fallback to
-  plain MuPDF text with the exact thresholds from `backend/core/rag/parsers.py`
-- Table detection and pipe-markdown rendering matching pymupdf's `Table.to_markdown`
-  shape (`|header|`, `|---|`, detail rows, `Col{i}` fill for empty headers)
-
-Known deltas vs pymupdf4llm:
-
-- Span-level styling: MuPDF.js's structured-text JSON exposes one font per line, so
-  mixed-style lines (one bold word inside a body line) style the whole line instead of
-  per-span. Line-level styling matches for homogeneous lines.
-- Superscript/subscript/underline/strikeout/highlight markers are not emitted (the
-  JSON does not expose char-level flags).
-- Table detection is a conservative text-grid detector (column-start clustering with
-  a 5 pt tolerance, contiguous multi-row bands) instead of PyMuPDF's
-  `find_tables()` vector-graphics analysis. Aligned text tables are detected; tables
-  defined only by drawn rules without aligned text are not.
-
-The minimal extractor remains as an automatic fallback when the `mupdf` package cannot
-be loaded (for example a stripped install).
-
-## Not planned (explicit decisions)
-
-### Proxy support
-
-Studio routes requests through urllib's environment proxies and honors
-`UNSLOTH_STUDIO_DISABLE_DNS_PINNING` for enterprise proxies. This port always connects
-directly with DNS pinning. Deliberately out of scope.
-
-### Page size budgets
-
-Studio's `_page_char_budget()` sizes fetched pages to the serving model's context window.
-This port deliberately has **no character budget at all**: fetched pages and PDFs are
-returned in full (the PDF page-count cap of 50 pages remains, matching Studio). The raw
-download caps (512 KiB text / 10 MiB PDF) still bound what is fetched. A `maxChars`
-parameter remains available on the tool for callers that want to truncate.
-
-## Known behavioral differences
-
-### Search engine set
-
-The port implements ddgs 9.14.4's `DDGS.text()` exactly: the same seven engines
-(duckduckgo, brave, google, mojeek, yahoo, yandex, wikipedia — bing is `disabled` in
-ddgs upstream), the same provider-deduplication, href-dedupe aggregator with
-frequency ordering, and the same `SimpleFilterRanker` re-ranking. Remaining deltas:
-
-- **TLS fingerprinting**: ddgs uses `primp` with browser TLS impersonation. Node's
-  `fetch` has a different fingerprint, so Google/Brave/Yahoo/Yandex may block or serve
-  consent pages more aggressively. When an engine is blocked it simply contributes no
-  results, exactly as when ddgs is blocked.
-- **User agents**: ddgs uses `fake_useragent`'s database; the port uses a fixed set of
-  browser UAs plus ddgs's own Android Google UA generator.
-
-### Entity decoding
-
-`decodeHtmlEntities` replicates CPython's `html.unescape` exactly (full HTML5 table,
-longest-prefix rule, Windows-1252 numeric mappings, invalid-codepoint handling), so the
-HTML-to-Markdown converter and search-result normalization match Studio's output.