pi-unsloth-webtools 0.1.0 → 0.2.1

package/README.md

@@ -4,6 +4,20 @@ A [pi](https://github.com/earendil-works/pi-coding-agent) extension providing `w
 `web_fetch` tools, ported from the Unsloth Studio codebase
 ([`unslothai/unsloth`](https://github.com/unslothai/unsloth), `studio/backend/core/inference/`).
 
+## Install
+
+```sh
+pi install npm:pi-unsloth-webtools
+```
+
+or add `npm:pi-unsloth-webtools` to the `packages` array in `~/.pi/agent/settings.json`.
+
+To install from source instead:
+
+```sh
+pi install /path/to/pi-unsloth-webtools
+```
+
 ## What it does
 
 ### web_search
@@ -38,9 +52,9 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   object streams, all filters, ToUnicode fonts, encryption detection, and a
   pymupdf4llm-style markdown layer (headings, bold/italic, code fences, links, tables)
   with Studio's corrupted/incomplete fallback to plain text.
-- Content sniffing: MIME allow/deny, binary magic signatures, PDF magic + text extraction
-  (built-in zlib-based extractor, not pymupdf-grade), charset decoding (declared charset, BOM
-  sniffing for UTF-8/16/32, cp1252 rescue for mislabeled single-byte pages).
+- Content sniffing: MIME allow/deny, binary magic signatures, PDF magic detection, and charset
+  decoding (declared charset, BOM sniffing for UTF-8/16/32, cp1252 rescue for mislabeled
+  single-byte pages).
 - HTML → Markdown conversion ported from Studio's dependency-free `_html_to_md.py`: headings,
   links, emphasis, lists, tables, blockquotes, code fences, entity decoding; hidden-element
   stripping (`hidden`, `aria-hidden`, inline styles); `<article>`/`<main>` main-content scoping
@@ -51,16 +65,8 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
 - HTML entity decoding replicates CPython's `html.unescape` (full 2,231-entry HTML5 table,
   longest-prefix rule, Windows-1252 numeric mappings), matching Studio byte-for-byte.
 
-See `ROADMAP.md` for the remaining gaps (PDF extraction, TLS fingerprinting, proxies).
-
-## Install
-
-```sh
-pi install /path/to/pi-unsloth-webtools
-```
-
-or add `npm:pi-unsloth-webtools` (once published) to the `packages` array in
-`~/.pi/agent/settings.json`.
+See `ROADMAP.md` for the remaining gaps (per-line styling, table detection, TLS
+fingerprinting, proxies).
 
 ## Development
 

package/html-to-md.ts

@@ -227,6 +227,21 @@ interface HtmlHandlers {
 const START_TAG_NAME_RE = /^[a-zA-Z][^\s/>]*/;
 const ATTR_NAME_RE = /^[^\s=/>]+/;
 
+const RAW_TEXT_TAGS = [
+  "script",
+  "style",
+  "textarea",
+  "title",
+  "xmp",
+  "iframe",
+  "noembed",
+  "noframes",
+];
+
+const RAW_TEXT_CLOSERS: Record<string, RegExp> = Object.fromEntries(
+  RAW_TEXT_TAGS.map((name) => [name, new RegExp(`</${name}\\s*>`, "i")]),
+);
+
 function parseAttrsUntilClose(input: string, pos: number): [AttrDict, number, boolean] {
   const attrs: AttrDict = {};
   while (pos < input.length) {
@@ -346,8 +361,22 @@ export function feedHtml(input: string, handlers: HtmlHandlers): void {
       continue;
     }
     emitText(input.slice(textStart, i));
-    if (tag.kind === "start") handlers.handleStartTag(tag.name!, tag.attrs!);
-    else if (tag.kind === "startend") handlers.handleStartEndTag(tag.name!, tag.attrs!);
+    if (tag.kind === "start") {
+      const rawCloser = RAW_TEXT_CLOSERS[tag.name!];
+      if (rawCloser) {
+        const after = input.slice(tag.end);
+        const closeMatch = rawCloser.exec(after);
+        if (closeMatch) {
+          handlers.handleStartTag(tag.name!, tag.attrs!);
+          emitText(after.slice(0, closeMatch.index));
+          handlers.handleEndTag(tag.name!);
+          textStart = tag.end + closeMatch.index + closeMatch[0].length;
+          i = textStart;
+          continue;
+        }
+      }
+      handlers.handleStartTag(tag.name!, tag.attrs!);
+    } else if (tag.kind === "startend") handlers.handleStartEndTag(tag.name!, tag.attrs!);
     else if (tag.kind === "end") handlers.handleEndTag(tag.name!);
     textStart = tag.end;
     i = tag.end;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools ported from the Unsloth Studio codebase (DuckDuckGo search, SSRF-safe direct fetching, HTML-to-Markdown extraction)",
   "main": "index.ts",