pi-unsloth-webtools 0.7.6 → 0.9.0

package/README.md

@@ -1,12 +1,12 @@
 # pi-unsloth-webtools
 
-A [pi](https://github.com/earendil-works/pi-coding-agent) extension providing `web_search`,
-`web_fetch`, and `web_render` tools. It began as a port of the Unsloth Studio codebase
+A [pi](https://github.com/earendil-works/pi-coding-agent) extension providing `web_search` and
+`web_fetch` tools. It began as a port of the Unsloth Studio codebase
 ([`unslothai/unsloth`](https://github.com/unslothai/unsloth), `studio/backend/core/inference/`);
 the engine, extraction, and PDF layers are still derived from it, but the package is no longer
 behavior-identical to Studio — it enables local file and private-address fetching by default
-and adds a fetch cache, Wayback fallbacks, page metadata, a third-party rendering tool
-(`web_render`), and other behavior Studio does not have. See
+and adds a fetch cache, Wayback fallbacks, page metadata, a browser-fingerprint retry, local
+local Lightpanda rendering, and other behavior Studio does not have.
 [Known differences from Studio](#known-differences-from-studio). The `unsloth` in the name marks
 provenance, not affiliation.
 
@@ -24,9 +24,15 @@ To install from source instead:
 pi install /path/to/pi-unsloth-webtools
 ```
 
+Installing pulls one runtime dependency, `wreq-js`, whose prebuilt native binding gives `web_fetch`
+its browser-shaped TLS fingerprint. Bindings exist for Linux (x64/arm64), macOS and Windows; because
+the binding is what `--omit=optional` skips, that install gets the shim without the engine and
+`web_fetch` quietly uses the plain Node transport instead. `mupdf` (PDF text extraction) stays an
+optional dependency, and the package works without it.
+
 ## What it does
 
-All three tools display their target in the TUI tool row: `web_search "query"`, `web_search <url>` in url mode, `web_fetch <url>`, and `web_render <url>`.
+Both tools display their target in the TUI tool row: `web_search "query"` and `web_fetch <url>`.
 
 ### web_search
 
@@ -41,9 +47,6 @@ Mirrors Unsloth Studio's `web_search` tool:
   re-ranking. Formats results identically: `Title:` / `URL:` /
   `Snippet:` blocks separated by `---`, ending with the hint to call `web_fetch` to
   read a full page.
-- Accepts an optional `url` parameter; when given, fetches that page's text instead of
-  searching (optionally truncated with `maxChars`). An HTTP 403 on that fetch falls back
-  to `web_render` when the tool is enabled.
 - Rate-limit, timeout, and empty-result messages mirror Studio's `_search_failure_message`.
 - Transient engine failures (network errors or null responses) are retried once with a short
   backoff inside the same timeout budget (a retry that cannot fit in the remaining budget is
@@ -83,7 +86,7 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   fetch; the deadline abort cuts a retry short when no budget remains.
 - Proxy environment variables are honored when they name a SOCKS5 proxy: `HTTPS_PROXY` /
   `HTTP_PROXY` / `ALL_PROXY` (with `NO_PROXY` exclusions) tunnel the pinned connection, so a
-  Tor-mode agent routes `web_fetch` and `web_search` url mode through its exit. `socks5h` is
+  Tor-mode agent routes `web_fetch` through its exit. `socks5h` is
   treated like `socks5`: the host is resolved locally for the guard and the pinned IP is what the
   proxy connects to. Other proxy schemes are ignored (direct connection).
 - GitHub repo root pages are rewritten to the unauthenticated README API
@@ -116,7 +119,7 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   with link-density header stripping; boilerplate-line removal.
 - No page-size budget: fetched pages and PDFs are returned in full (Studio's window-aware
   cap is deliberately dropped; the optional `maxChars` parameter still truncates when given,
-  on `web_fetch` and on `web_search`'s url mode).
+  on `web_fetch`).
   The 512 KiB / 10 MiB download caps still bound the raw fetch.
 - HTML entity decoding replicates CPython's `html.unescape` (full 2,231-entry HTML5 table,
   longest-prefix rule, Windows-1252 numeric mappings), matching Studio byte-for-byte.
@@ -125,28 +128,67 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   `Date:` (`article:published_time` / `dc.date` / `date`) and `Site:` (`og:site_name` /
   `application-name`) lines are added when declared, so the model can judge recency and
   provenance.
-- A direct fetch refused with HTTP 403 is retried through `web_render` automatically when that
-  tool is enabled; the rendered page is prefixed with a note saying so. When `web_render` is
-  disabled or the render also fails, the original `Failed to fetch URL: HTTP 403 ...` is returned.
-
-### web_render
-
-Renders a public page to Markdown through the third-party Jina Reader (`r.jina.ai`) for pages
-that need JavaScript to render:
-
-- Every target is validated and resolved locally first: http/https only, and any private,
-  loopback, link-local, or otherwise non-public address is refused. Local files are refused on
-  this path regardless of `webFetch.allowPrivateAddresses` / `webFetch.allowLocalFiles`, because
-  the URL is sent to Jina.
-- `unslothWebTools.jinaApiKey` (or `webRender.jinaApiKey`, or the `JINA_API_KEY` environment
-  variable) raises the Reader's rate limits; without a key it still works at Jina's free limits.
-- Output is Markdown prefixed with `Title:` / `URL:` lines and a `Rendered via the Jina Reader`
-  provenance line. An optional `maxChars` truncates, like `web_fetch`.
-- Keyless Reader requests are rate-limited per outgoing IP; see
-  [Companion: rotating exit IPs](#companion-rotating-exit-ips).
-- Enabled by default. Disable it with `/webtools-config` (`webRenderEnabled` in
-  `~/.config/pi-unsloth-webtools/config.json`), which deactivates the tool for the session.
-- Also used automatically when a `web_fetch` or `web_search` url-mode fetch is refused with HTTP 403.
+- A direct fetch refused with HTTP 403 is retried through a browser-fingerprint transport
+  (`wreq-js`, Chrome TLS/HTTP2 shape) that keeps the pinned DNS and the website policy checks.
+  This is the default transport and clears many bot walls without a browser and without a third
+  party; opt out with `webFetch.transport: "direct-first"` or `"off"`.
+- A page that is still refused, or that looks JavaScript-rendered, is rendered locally with
+  Lightpanda when the binary is installed. There is no third-party rendering service: nothing is
+  sent anywhere except to the target itself. Bot-challenge pages are rejected, so an interstitial
+  never replaces a real render. When nothing renders, the original error or a
+  `*(JavaScript-rendered page; content may be incomplete)*` note is returned.
+
+### Rendering and anti-bot tiers
+
+`web_fetch` escalates through two tiers, cheapest first:
+
+1. **Browser-fingerprint transport** (`wreq-js`, Chrome TLS/HTTP2 shape) is the default, and it is
+   a required dependency: it is a small native Rust module with prebuilt bindings for Linux
+   (x64/arm64, glibc and musl), macOS (x64/arm64) and Windows (x64/arm64). Requests are pinned to
+   the resolved, validated IP, browser-emulation headers are left intact so the fingerprint stays
+   coherent, and redirects are handed back to the main hop loop, so every hop is re-validated
+   against the website policy. The plain Node transport takes over when no prebuilt binding exists
+   for the platform, when a SOCKS5 proxy is configured (traffic must keep using the tunnelling
+   path), or when the request fails at the connection level; a 403 from either transport triggers
+   one retry through the other. `webFetch.transport` selects `tls-first` (default), `direct-first`,
+   or `off`.
+2. **Local rendering** with a locally installed
+   [Lightpanda](https://github.com/lightpanda-io/browser) binary when a page is still refused or
+   looks JavaScript-rendered.
+
+There is no third tier. If a page needs a browser that the local renderer cannot provide, the
+original error (or the incomplete-content note) is returned rather than shipping the URL to a
+third-party rendering service.
+
+#### Local rendering (Lightpanda)
+
+- Runs `lightpanda fetch --dump html --wait-until networkidle --block-private-networks` and puts
+  the dump through the same Markdown pipeline as any other fetch, so titles, metadata,
+  main-content scoping and boilerplate removal match the direct path.
+- `--block-private-networks` is always passed in addition to the pre-flight resolve check, so a
+  redirect or subresource inside the browser cannot reach a private address.
+- A render is only accepted when it contains real prose, or beats the fetched text by 1.5x. A
+  non-zero exit, a timeout, or a bot-challenge page counts as a failed tier, so the next tier
+  still gets its chance.
+- Lightpanda identifies itself honestly and refuses to impersonate a browser user agent, so hard
+  anti-bot walls are returned as failures (the direct error, or the incomplete-content note).
+- Binary resolution: `lightpanda` on `PATH`, then `PI_LIGHTPANDA_BIN`, then
+  `webRender.lightpandaPath`. Prebuilt binaries exist for Linux (glibc; musl needs a source
+  build) and macOS, plus Docker images; Windows needs WSL2.
+- Version matters: 1.0.0 renders JavaScript-heavy pages that the 0.2.x line cannot — measured on
+  the same machine, IMDb went from a 76-byte empty document to 21k characters, dribbble from 32
+  characters to 18k, and a Medium article from a challenge page to real text. Linux builds from
+  0.3 on require glibc 2.38, so older distributions are stuck on 0.2.x without the next bullet.
+- Windows has no native Lightpanda build: install it inside WSL2 and point `webRender.lightpandaCommand`
+  at `["wsl.exe", "-e", "<path inside WSL>"]`, which the renderer then drives like any other binary.
+  The same setting works for a container (`["docker", "run", "--rm", "lightpanda/browser:nightly"]`)
+  or any wrapper. `scripts/install-lightpanda.sh` prints that recipe when run outside WSL.
+- `bash scripts/install-lightpanda.sh` installs the newest stable release. When the system glibc
+  predates what the binary needs, it downloads Debian's `libc6` for the current stable suite,
+  extracts it into the install directory, and writes a launcher shim — so 1.0.0 runs on a
+  glibc 2.36 host without touching the system libraries. The script prints the path to use.
+- Disable the tier with `webRender.lightpandaEnabled: false`; `web_fetch` then stops after the
+  network attempts and reports the original error.
 
 ## Known differences from Studio
 
@@ -155,11 +197,15 @@ that need JavaScript to render:
   (`file://` URLs and absolute, `~/`, `./` paths) by default; opt out with
   `webFetch.allowPrivateAddresses: false` and `webFetch.allowLocalFiles: false` to restore
   Studio's behavior.
-- Third-party rendering: the extra `web_render` tool asks the Jina Reader (`r.jina.ai`) to fetch
-  the page, so the target URL leaves the machine. Studio has no third-party rendering path. This
-  path always refuses local files and non-public addresses, regardless of the local-access settings.
-  A direct fetch refused with HTTP 403 is retried through it automatically when enabled, so those
-  targets also leave the machine in that case.
+- Browser-fingerprint retry: a 403 is retried through `wreq-js` (Chrome TLS/HTTP2 shape) before any
+  rendering, keeping the pinned IP, the website policy, and the local-file refusal rules. Studio has
+  no such path.
+- Local rendering: Lightpanda renders the page in a local browser and the dump runs through the same
+  extraction pipeline; nothing leaves the machine, and `--block-private-networks` is always passed.
+  Studio has no local rendering path.
+- No third-party rendering: Studio has no rendering path at all; this port renders refused or
+  JavaScript-heavy pages locally with Lightpanda and never sends the URL to a rendering service.
+  Local files and non-public addresses stay refused for the browser tier.
 - PDF styling: MuPDF.js exposes one font per line, so mixed-style lines style the
   whole line instead of per-span; superscript, subscript, underline, strikeout, and
   highlight markers are not emitted. Tables use a conservative text-grid detector:
@@ -182,8 +228,8 @@ that need JavaScript to render:
 - Proxies: Studio routes through environment proxies; this port resolves and pins the target IP and
   tunnels that connection through `HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY` when the proxy is a
   SOCKS5 proxy (`NO_PROXY` exclusions respected; DNS stays local for the guard). Other proxy
-  schemes fall back to a direct connection. The search and `web_render` paths use the process-wide
-  `fetch`, so an agent-level proxy dispatcher applies there too — see
+  schemes fall back to a direct connection. The search path uses the process-wide `fetch`, so an
+  agent-level proxy dispatcher applies there too — see
   [Companion: rotating exit IPs](#companion-rotating-exit-ips).
 - Dedup and titles: the aggregator keys on canonicalized hrefs (`utm_*`/tracking parameters
   and fragments stripped, then the URL re-serialized); fetched HTML pages are prefixed with
@@ -195,16 +241,17 @@ that need JavaScript to render:
 
 ## When to use alternatives
 
-This package keeps Studio's deterministic, zero-dependency pipeline and test parity with
-`unsloth/studio`, then layers local access, third-party rendering, and other Studio-independent
-behavior on top. The SSRF guard is real and thoroughly tested, but it is opt-in:
+This package keeps Studio's deterministic extraction and search pipeline — and its test parity with
+`unsloth/studio` — then layers browser-fingerprint fetching, local rendering, local access, and other
+Studio-independent behavior on top. Its only runtime dependency is `wreq-js` (the default transport);
+`mupdf` stays optional. The SSRF guard is real and thoroughly tested, but it is opt-in:
 `allowPrivateAddresses` defaults to `true`, and local files are readable unless `allowLocalFiles`
 is `false`. For other tradeoffs, prefer:
 
 | Need | Use |
 |---|---|
-| Browser-like TLS/HTTP fingerprinting to unblock bot-defended pages | `pi-smart-fetch` (`wreq-js` `chrome_145`) |
-| Headless Chrome for JS-rendered SPAs/YouTube/Reddit threads | Built-in `web_render` (Jina Reader) first; `georgebashi/pi-web-fetch` (puppeteer + trafilatura) when the Reader falls short |
+| Browser-like TLS/HTTP fingerprinting to unblock bot-defended pages | Built in: `webFetch.transport` defaults to `tls-first`, which speaks Chrome's TLS/HTTP2 shape through `wreq-js`; `pi-smart-fetch` (`wreq-js` `chrome_145`) if you want a separate tool |
+| Headless Chrome for JS-rendered SPAs/YouTube/Reddit threads | Built-in local Lightpanda rendering; `georgebashi/pi-web-fetch` (puppeteer + trafilatura) or a Patchright-based tier when it falls short |
 | Hosted search with semantic ranking and no scraping | `Brave Search API` / `Tavily` / `Exa` via `pi-ollama-web-search` |
 | Prompt-focused page distillation to save context | `pi-web-fetch` `prompt` -> sub-agent or Claude Code `WebFetch(url,prompt)` |
 | Batch fetching many URLs concurrently | `pi-smart-fetch` `batch_web_fetch` or call `web_fetch` in parallel |
@@ -216,8 +263,8 @@ choose the best tool per URL. No need to fork this package to add those features
 
 [`pi-tor-proxy`](https://github.com/YuGiMob/pi-tor-proxy) routes pi's in-process `fetch` traffic
 through Tor (it downloads and manages its own Tor binary) and gives each pi instance its own
-circuit and exit IP. The search sweep and `web_render` both use `fetch`, so they leave through
-the current Tor exit, and Jina rate-limits keyless Reader requests per outgoing IP —
+circuit and exit IP. The search sweep uses the process-wide `fetch`, so it leaves through the
+current Tor exit, and many search engines rate-limit or challenge per outgoing IP —
 `/tor-cycle` swaps the exit those limits are counted against, while `/tor-country` and
 `/tor-exclude` constrain which exits are used.
 
@@ -225,7 +272,7 @@ the current Tor exit, and Jina rate-limits keyless Reader requests per outgoing
 pi install npm:pi-unsloth-webtools npm:pi-tor-proxy
 ```
 
-`web_fetch` and `web_search` url mode also route: they resolve and pin the target IP, then tunnel
+`web_fetch` also routes: it resolves and pins the target IP, then tunnels
 the connection through the SOCKS5 proxy named by `HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY` (with
 `NO_PROXY` exclusions, so localhost and local files stay direct). DNS is still resolved locally for
 the SSRF guard, and the proxy connects to that pinned IP.
@@ -254,36 +301,18 @@ Optional settings in `~/.pi/agent/settings.json` or `.pi/settings.json` (project
 | Key | Default | Description |
 |---|---|---|
 | `unslothWebTools.maxResults` | `5` | Default `maxResults` for `web_search` (clamped 1-20) |
-| `unslothWebTools.maxChars` / `webFetch.maxChars` / `smartFetchDefaultMaxChars` | tool param | Default `maxChars` for `web_fetch` and `web_search` url mode |
-| `unslothWebTools.timeoutMs` / `webFetch.timeoutMs` / `smartFetchDefaultTimeoutMs` | `60000` fetch, `300000` search | Default `timeoutMs` when the tool param is absent (>=1000). Fetch and `web_search` url mode fall back to 60000; `web_search` query mode falls back to 300000 |
+| `unslothWebTools.maxChars` / `webFetch.maxChars` / `smartFetchDefaultMaxChars` | tool param | Default `maxChars` for `web_fetch` |
+| `unslothWebTools.timeoutMs` / `webFetch.timeoutMs` / `smartFetchDefaultTimeoutMs` | `60000` fetch, `300000` search | Default `timeoutMs` when the tool param is absent (>=1000). Fetch falls back to 60000; `web_search` query mode falls back to 300000 |
 | `webSearch.maxResults` / `smartWebSearch.resultsPerQuery` | same as above | Legacy aliases for `maxResults` |
 | `websitePolicy` | none | Not read from settings. Tools run unrestricted by default; `websitePolicy` is a programmatic option the host passes to `webSearch` / `fetchPageText` |
 | `unslothWebTools.allowPrivateAddresses` / `webFetch.allowPrivateAddresses` | `true` | Opt out to restore the resolved-IP SSRF guard: private/loopback/link-local hosts (localhost, LAN IPs) are refused again. Non-canonical numeric IP encodings stay blocked either way |
-| `unslothWebTools.allowLocalFiles` / `webFetch.allowLocalFiles` | `true` | Opt out to refuse local files in `web_fetch` and `web_search` url mode (`file://` URLs, absolute, `~/`, or `./` paths); when enabled, PDFs are extracted and HTML converted |
-| `unslothWebTools.jinaApiKey` / `webRender.jinaApiKey` | none (`JINA_API_KEY` fallback) | API key for `web_render`'s Jina Reader; raises its rate limits. Settings keys win over the environment variable |
-
-### Settings window
-
-`/webtools-config` opens an interactive settings window (↑↓ navigate, space toggle, q close). Settings
-persist across sessions in `~/.config/pi-unsloth-webtools/config.json`, created when a setting is
-first changed:
+| `unslothWebTools.allowLocalFiles` / `webFetch.allowLocalFiles` | `true` | Opt out to refuse local files in `web_fetch` (`file://` URLs, absolute, `~/`, or `./` paths); when enabled, PDFs are extracted and HTML converted |
+| `webFetch.transport` / `unslothWebTools.transport` | `tls-first` | Fetch transport order: `tls-first` (default), `direct-first`, or `off` to disable the browser-fingerprint transport entirely |
+| `webRender.lightpandaEnabled` / `unslothWebTools.lightpandaEnabled` | `true` | Opt out to disable local Lightpanda rendering |
+| `webRender.lightpandaPath` / `unslothWebTools.lightpandaPath` | `lightpanda` on `PATH` (`PI_LIGHTPANDA_BIN` fallback) | Path to the Lightpanda binary used for local rendering |
+| `webRender.lightpandaCommand` / `unslothWebTools.lightpandaCommand` | none | Command prefix that launches the renderer, for WSL (`["wsl.exe","-e","<path>"]`) or containers; overrides `lightpandaPath`. The fetch flags are appended to it |
 
-```json
-{
-  "webRenderEnabled": true
-}
-```
-
-| Key | Default | Description |
-|---|---|---|
-| `webRenderEnabled` | `true` | When `false`, the `web_render` tool is deactivated for the session and the automatic HTTP 403 fallback in `web_fetch` / `web_search` url mode is disabled |
-
-On non-Windows platforms the directory honors `XDG_CONFIG_HOME` when set (falling back to
-`~/.config`); on Windows it always uses `~/.config`.
-
-Tool params always win over file defaults. Search dedup also strips default ports, so `https://example.com:443/a` and `https://example.com/a` collapse.
-
-Environment overrides: `PI_UNSLOTH_CACHE_DIR` changes the fetch cache directory, `PI_UNSLOTH_WEBTOOLS_STATS` opts into append-only sweep stats JSONL, `PI_CODING_AGENT_DIR` / `PI_AGENT_DIR` change the global settings directory, and `JINA_API_KEY` supplies the `web_render` key when no settings key is set. Cache entries live 1 hour and stale copies are served only after a network failure. SOCKS5 proxies named by `HTTPS_PROXY`, `HTTP_PROXY`, or `ALL_PROXY` are honored on every fetch (`NO_PROXY` exclusions apply).
+Environment overrides: `PI_UNSLOTH_CACHE_DIR` changes the fetch cache directory, `PI_UNSLOTH_WEBTOOLS_STATS` opts into append-only sweep stats JSONL, `PI_CODING_AGENT_DIR` / `PI_AGENT_DIR` change the global settings directory, and `PI_LIGHTPANDA_BIN` points at the local renderer binary. Cache entries live 1 hour and stale copies are served only after a network failure. SOCKS5 proxies named by `HTTPS_PROXY`, `HTTP_PROXY`, or `ALL_PROXY` are honored on every fetch (`NO_PROXY` exclusions apply).
 
 ## Troubleshooting
 
@@ -300,20 +329,22 @@ Match on the exact prefix. Do not retry blocked hosts with spelling tricks.
 | Private address blocked | `Blocked: refusing to fetch the non-public address ...` | The SSRF guard is active (`allowPrivateAddresses: false`); remove it or set `true` to reach localhost/LAN, and write the scheme explicitly (`http://localhost:3000`). |
 | Local file blocked | `Blocked: the URL has an invalid hostname or port.` for paths | Local files are disabled: remove `allowLocalFiles: false` to read `file://`, absolute, `~/`, or `./` paths. |
 | File read failed | `Failed to read file: ...` | Check the path exists and is a regular file. |
-| HTTP failure | `Failed to fetch URL: HTTP ...` | Fix the URL. A 404 automatically tries a Wayback snapshot; a 403 retries through `web_render` when enabled. |
+| HTTP failure | `Failed to fetch URL: HTTP ...` | Fix the URL. A 404 automatically tries a Wayback snapshot; a 403 retries through the browser-fingerprint transport first. |
 | Proxy failure | `Failed to fetch URL: SOCKS5 proxy ...` | The SOCKS5 proxy refused or failed (for example Tor is stopping). Check the proxy, or unset the proxy variables for a direct fetch. |
 | Non-text / binary | `(non-text content:` / `(binary content,` | Not readable as text by design. |
 | PDF without text | `(PDF contains no extractable text)` / `(PDF content could not be read as text...)` | Scanned or encrypted PDF. |
 | Download cap hit | `... (page truncated at the download limit)` | Raw fetch hit 512 KiB (10 MiB for PDFs). |
 | maxChars cut | `... (truncated, N chars total)` | Raise `maxChars` for the full text. |
-| Empty page | `(page returned no readable text)` | Page had no extractable text; try `web_render`, which renders JavaScript pages. |
-| Render blocked (local file) | `Blocked: web_render cannot fetch local files.` | Use `web_fetch`, which reads local files by default. |
-| Render blocked (private host) | `Blocked: refusing to fetch the non-public address ...` | `web_render` only reaches public hosts; use `web_fetch` for localhost/LAN. |
-| Render failure | `Failed to render URL: ...` | Jina rejected or failed the request (rate limit, bad key, unreachable page). Retry, or check `unslothWebTools.jinaApiKey` / `JINA_API_KEY`. |
+| Empty page | `(page returned no readable text)` | Page had no extractable text; `web_fetch` then tries local Lightpanda if it is installed. |
+| JavaScript-rendered page | `*(JavaScript-rendered page; content may be incomplete)*` | Rendering was disabled or produced no more text; the page likely needs a browser. |
 | GitHub rewrite | `README of ... (fetched via the GitHub README API):` | Expected repo-root rewrite, not the HTML chrome. |
 | Cache fallback | `Served from cache` / `STALE cache from YYYY-MM-DD` | Network failed; output is the cached copy with its date. |
 | Wayback fallback | `Fetched from Wayback Machine snapshot (YYYY-MM-DD) for ...` | Original 404'd; output is the archived copy with its date. |
 
+| No browser-fingerprint binding | fetch behaves as if `webFetch.transport` were `direct-first` | `wreq-js` ships prebuilt bindings for Linux, macOS and Windows only; on other platforms the tier is skipped automatically |
+| No renderer available | `*(JavaScript-rendered page; content may be incomplete)*` with no local note | Install Lightpanda, or set `webRender.lightpandaPath` / `PI_LIGHTPANDA_BIN`, to render JavaScript-heavy pages locally |
+| Local renderer failed | `Failed to render URL: Lightpanda exited with code N.` | The failed tier falls through to the next one; check the binary by hand with `lightpanda fetch --dump markdown <url>` |
+| Local renderer blocked a target | `Blocked: the local renderer cannot fetch local files.` | The local browser refuses local paths by design; fetch them with `web_fetch` directly instead |
 ## Development
 
 ```sh
@@ -322,12 +353,43 @@ npm run typecheck
 npm test
 npm run test:unit
 npm run test:smoke
+bash scripts/install-lightpanda.sh
+npm run compare:fetch
+npm run compare:browsers
+npm run camoufox:warmup
+npm run stealth:matrix
+/tmp/pyenv/bin/python scripts/stealth-python.py
 ```
 
+`npm run compare:fetch` runs a live head-to-head of the direct fetch, the TLS-impersonation retry,
+local Lightpanda rendering over a target list. It accepts URLs as arguments and `--no-lightpanda`
+to drop the render tier. `npm run compare:browsers` adds a Camoufox column;
+install it separately (`npm i camoufox-js playwright-core && npx camoufox-js fetch`, plus GTK3
+libraries on Linux) and use `--seconds=N` to bound how long it waits out a JS challenge.
+`npm run stealth:matrix` compares stealth-browser options against one walled page (plus
+`bot.sannysoft.com` detection rows and a plain-page sanity check): raw CDP to a system Chromium
+(`PI_CHROMIUM_BIN` to point at it), Playwright with its bundled Chromium, Patchright, and Camoufox.
+Every browser dependency is loaded through a guarded dynamic import, so nothing is added to
+`package.json`; install whichever rows you want to measure. `--attempts=N` and `--seconds=N` bound
+the walled-page attempts, and passing row names runs a subset (`raw-cdp`, `playwright`, `patchright`,
+`camoufox`).
+
+`scripts/stealth-python.py` is the same idea for the Python-side options (nodriver, CloakBrowser,
+DrissionPage, cloudscraper, curl_cffi) against the same walled page; it needs a venv with those
+packages installed and is not wired into any npm script. Measured findings are in its module docstring.
+
+`npm run camoufox:warmup` measures what a warm Camoufox costs and buys: launch time, idle CPU and
+RSS, per-fetch latency with the browser already running, and whether a persistent profile
+(`user_data_dir`, pinned fingerprint) lets a Cloudflare clearance survive a restart. Flags:
+`--virtual` for a virtual display, `--pin=0` to let Camoufox rotate fingerprints, `--seconds=N`,
+`--idle=N`.
+
 ## Tests
 
 `npm test` runs the full suite. `npm run test:unit` skips the live-network smoke tests,
-and `npm run test:smoke` runs only those.
+`npm run test:smoke` runs only those, and `npm run test:lightpanda` drives a real Lightpanda binary
+against live pages — it is skipped unless `PI_LIGHTPANDA_E2E=1` is set (`PI_LIGHTPANDA_E2E_BIN` points
+at the binary), and CI runs it in a dedicated job after `scripts/install-lightpanda.sh`.
 
 The suite ports Unsloth Studio's own tests for these tools:
 
@@ -350,12 +412,17 @@ The suite ports Unsloth Studio's own tests for these tools:
 - `test/smoke.test.ts`: live network checks against real hosts, including a per-engine
   result-health sweep (at least two engines must return well-formed results; engines
   that block or reset connections from datacenter IPs count as unhealthy, not failures)
-- `test/web-render.test.ts`: Jina Reader rendering with a stubbed fetch and DNS, the public-only
-  guard, policy enforcement, error mapping, truncation, cancellation, and API key precedence
 
-The seams (`seams.resolve` / `seams.request` / `rawFetch`) replace the network stack
-with fakes, mirroring how the Studio suite monkeypatches `_validate_and_resolve_host`
-and `build_opener`.
+- `test/tls-fetch.test.ts`: the browser-fingerprint transport with a stubbed native module, including
+  transport reuse, body caps, redirect passthrough, and abort/timeout mapping
+- `test/lightpanda.test.ts`: the local renderer's command line, guards, failure modes, and a real
+  child-process run against a scripted binary
+- `test/impersonation.test.ts`: the HTTP 403 retry inside the fetch pipeline, including redirect
+  re-validation, the disabled path, and the stubbed-network contract
+The seams (`seams.resolve` / `seams.request` / `seams.impersonate` / `rawFetch`) replace the network
+stack with fakes, mirroring how the Studio suite monkeypatches `_validate_and_resolve_host` and
+`build_opener`. Supplying a `request` seam also disables the browser-fingerprint retry, so a stubbed
+transport never escapes to the real network.
 
 ## License
 

package/entities.ts

@@ -1,4 +1,4 @@
-export const NAMED_ENTITIES: Record<string, string> = {
+export const NAMED_ENTITIES: Record<string, string> = Object.assign(Object.create(null), {
   "AElig": "\u00c6",
   "AElig;": "\u00c6",
   "AMP": "&",
@@ -2230,7 +2230,7 @@ export const NAMED_ENTITIES: Record<string, string> = {
   "zscr;": "\ud835\udccf",
   "zwj;": "\u200d",
   "zwnj;": "\u200c",
-};
+});
 
 export const INVALID_CHARREFS: Record<number, string> = {
   0: "\ufffd",

package/html-to-md.ts

@@ -66,7 +66,7 @@ const P_CLOSING_TAGS = new Set([
   "ul",
 ]);
 
-const IMPLICIT_CLOSERS: Record<string, Set<string>> = {
+const IMPLICIT_CLOSERS: Record<string, Set<string>> = Object.assign(Object.create(null), {
   p: P_CLOSING_TAGS,
   li: new Set(["li"]),
   dt: new Set(["dt", "dd"]),
@@ -76,9 +76,9 @@ const IMPLICIT_CLOSERS: Record<string, Set<string>> = {
   th: new Set(["td", "th", "tr"]),
   option: new Set(["option", "optgroup"]),
   optgroup: new Set(["optgroup"]),
-};
+});
 
-const CLOSE_BARRIERS: Record<string, Set<string>> = {
+const CLOSE_BARRIERS: Record<string, Set<string>> = Object.assign(Object.create(null), {
   li: new Set(["ul", "ol", "menu"]),
   dt: new Set(["dl"]),
   dd: new Set(["dl"]),
@@ -87,7 +87,7 @@ const CLOSE_BARRIERS: Record<string, Set<string>> = {
   th: new Set(["table"]),
   option: new Set(["select", "datalist"]),
   optgroup: new Set(["select", "datalist"]),
-};
+});
 
 const BLOCK_TAGS = new Set([
   "p",
@@ -106,7 +106,7 @@ const BLOCK_TAGS = new Set([
 ]);
 
 const HEADING_TAGS = new Set(["h1", "h2", "h3", "h4", "h5", "h6"]);
-const INLINE_EMPHASIS: Record<string, string> = { strong: "**", b: "**", em: "*", i: "*" };
+const INLINE_EMPHASIS: Record<string, string> = Object.assign(Object.create(null), { strong: "**", b: "**", em: "*", i: "*" });
 
 const HEADER_LINK_DENSITY = 0.93;
 const HEADER_MIN_CHARS = 150;
@@ -114,6 +114,8 @@ const HEADER_MAX_RENDERED_CHARS = 800;
 const MAX_HEADER_NESTING = 8;
 
 const MIN_MAIN_CONTENT_CHARS = 200;
+const LISTING_MIN_SEGMENTS = 3;
+const LISTING_DOMINANCE_RATIO = 3;
 
 export type AttrDict = Record<string, string | null>;
 
@@ -1052,23 +1054,36 @@ function render(sourceHtml: string, scopeTags: Set<string> | null, stripHeader =
   return cleanup(newRenderer(sourceHtml, scopeTags, stripHeader).out.join(""));
 }
 
-function selectMainScopeRender(sourceHtml: string, tag: string): [number, string] {
+function selectMainScopeRender(sourceHtml: string, tag: string): [number, string, boolean] {
   const renderer = newRenderer(sourceHtml, new Set([tag]), true);
   const dropped = renderer.scopeDropped;
   const headingProse = renderer.scopeHeadingProse;
   let bestLen = 0;
   let bestRender = "";
+  let bestProse = 0;
+  let secondProse = 0;
+  let qualifiers = 0;
   for (let i = 0; i < renderer.scopeSegments.length; i++) {
     const rendered = stripBoilerplateLines(cleanup(renderer.scopeSegments[i]));
     const prose = visibleChars(rendered) - headingProse[i];
     if (prose < MIN_MAIN_CONTENT_CHARS) continue;
+    qualifiers++;
+    if (prose > bestProse) {
+      secondProse = bestProse;
+      bestProse = prose;
+    } else if (prose > secondProse) {
+      secondProse = prose;
+    }
     const size = rendered.length + Math.min(dropped[i], rendered.length);
     if (size > bestLen) {
       bestLen = size;
       bestRender = rendered;
     }
   }
-  return [bestLen, bestRender];
+  const listing =
+    qualifiers >= LISTING_MIN_SEGMENTS &&
+    bestProse < LISTING_DOMINANCE_RATIO * Math.max(1, secondProse);
+  return [bestLen, bestRender, listing];
 }
 
 export function visibleChars(text: string): number {
@@ -1141,8 +1156,8 @@ export function htmlToMarkdown(sourceHtml: string, mainContent = false): string
   sourceHtml = sourceHtml.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
   if (mainContent) {
     for (const scopeTag of ["article", "main"]) {
-      const [length, rendered] = selectMainScopeRender(sourceHtml, scopeTag);
-      if (length >= MIN_MAIN_CONTENT_CHARS) return rendered;
+      const [length, rendered, listing] = selectMainScopeRender(sourceHtml, scopeTag);
+      if (length >= MIN_MAIN_CONTENT_CHARS && !listing) return rendered;
     }
     return stripBoilerplateLines(render(sourceHtml, null, true));
   }