pi-unsloth-webtools 0.7.5 → 0.8.0

package/README.md

@@ -1,12 +1,12 @@
 # pi-unsloth-webtools
 
-A [pi](https://github.com/earendil-works/pi-coding-agent) extension providing `web_search`,
-`web_fetch`, and `web_render` tools. It began as a port of the Unsloth Studio codebase
+A [pi](https://github.com/earendil-works/pi-coding-agent) extension providing `web_search` and
+`web_fetch` tools. It began as a port of the Unsloth Studio codebase
 ([`unslothai/unsloth`](https://github.com/unslothai/unsloth), `studio/backend/core/inference/`);
 the engine, extraction, and PDF layers are still derived from it, but the package is no longer
 behavior-identical to Studio — it enables local file and private-address fetching by default
-and adds a fetch cache, Wayback fallbacks, page metadata, a third-party rendering tool
-(`web_render`), and other behavior Studio does not have. See
+and adds a fetch cache, Wayback fallbacks, page metadata, automatic Jina Reader rendering, and
+other behavior Studio does not have. See
 [Known differences from Studio](#known-differences-from-studio). The `unsloth` in the name marks
 provenance, not affiliation.
 
@@ -26,7 +26,7 @@ pi install /path/to/pi-unsloth-webtools
 
 ## What it does
 
-All three tools display their target in the TUI tool row: `web_search "query"`, `web_search <url>` in url mode, `web_fetch <url>`, and `web_render <url>`.
+Both tools display their target in the TUI tool row: `web_search "query"` and `web_fetch <url>`.
 
 ### web_search
 
@@ -39,11 +39,8 @@ Mirrors Unsloth Studio's `web_search` tool:
   re-serialized, collapsing host-case, default-port, and trailing-slash variants — so the
   same page found via different tracking links collapses), and the same `SimpleFilterRanker`
   re-ranking. Formats results identically: `Title:` / `URL:` /
-  `Snippet:` blocks separated by `---`, ending with the hint to pass `{"url": "<URL>"}` to
+  `Snippet:` blocks separated by `---`, ending with the hint to call `web_fetch` to
   read a full page.
-- Accepts an optional `url` parameter; when given, fetches that page's text instead of
-  searching (optionally truncated with `maxChars`). An HTTP 403 on that fetch falls back
-  to `web_render` when the tool is enabled.
 - Rate-limit, timeout, and empty-result messages mirror Studio's `_search_failure_message`.
 - Transient engine failures (network errors or null responses) are retried once with a short
   backoff inside the same timeout budget (a retry that cannot fit in the remaining budget is
@@ -83,7 +80,7 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   fetch; the deadline abort cuts a retry short when no budget remains.
 - Proxy environment variables are honored when they name a SOCKS5 proxy: `HTTPS_PROXY` /
   `HTTP_PROXY` / `ALL_PROXY` (with `NO_PROXY` exclusions) tunnel the pinned connection, so a
-  Tor-mode agent routes `web_fetch` and `web_search` url mode through its exit. `socks5h` is
+  Tor-mode agent routes `web_fetch` through its exit. `socks5h` is
   treated like `socks5`: the host is resolved locally for the guard and the pinned IP is what the
   proxy connects to. Other proxy schemes are ignored (direct connection).
 - GitHub repo root pages are rewritten to the unauthenticated README API
@@ -116,7 +113,7 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   with link-density header stripping; boilerplate-line removal.
 - No page-size budget: fetched pages and PDFs are returned in full (Studio's window-aware
   cap is deliberately dropped; the optional `maxChars` parameter still truncates when given,
-  on `web_fetch` and on `web_search`'s url mode).
+  on `web_fetch`).
   The 512 KiB / 10 MiB download caps still bound the raw fetch.
 - HTML entity decoding replicates CPython's `html.unescape` (full 2,231-entry HTML5 table,
   longest-prefix rule, Windows-1252 numeric mappings), matching Studio byte-for-byte.
@@ -125,28 +122,35 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   `Date:` (`article:published_time` / `dc.date` / `date`) and `Site:` (`og:site_name` /
   `application-name`) lines are added when declared, so the model can judge recency and
   provenance.
-- A direct fetch refused with HTTP 403 is retried through `web_render` automatically when that
-  tool is enabled; the rendered page is prefixed with a note saying so. When `web_render` is
+- A direct fetch refused with HTTP 403 is retried through the Jina Reader automatically when
+  rendering is enabled; the rendered page is prefixed with a note saying so. When rendering is
   disabled or the render also fails, the original `Failed to fetch URL: HTTP 403 ...` is returned.
+- Pages that look JavaScript-rendered are re-fetched through the Jina Reader the same way. When
+  rendering is disabled or does not produce more text, the page gets a
+  `*(JavaScript-rendered page; content may be incomplete)*` note, so a shell is not mistaken for
+  the whole page.
 
-### web_render
+### JavaScript rendering
 
-Renders a public page to Markdown through the third-party Jina Reader (`r.jina.ai`) for pages
-that need JavaScript to render:
+`web_fetch` retries through the third-party Jina Reader (`r.jina.ai`) when
+a direct fetch is refused with HTTP 403 or returns a page that looks JavaScript-rendered (thin
+converted text plus SPA markers, script-heavy markup, a noscript body, or a description meta tag):
 
 - Every target is validated and resolved locally first: http/https only, and any private,
-  loopback, link-local, or otherwise non-public address is refused. Local files are refused on
-  this path regardless of `webFetch.allowPrivateAddresses` / `webFetch.allowLocalFiles`, because
-  the URL is sent to Jina.
+  loopback, link-local, or otherwise non-public address is refused. Local files are never sent,
+  regardless of `webFetch.allowPrivateAddresses` / `webFetch.allowLocalFiles`.
 - `unslothWebTools.jinaApiKey` (or `webRender.jinaApiKey`, or the `JINA_API_KEY` environment
   variable) raises the Reader's rate limits; without a key it still works at Jina's free limits.
-- Output is Markdown prefixed with `Title:` / `URL:` lines and a `Rendered via the Jina Reader`
-  provenance line. An optional `maxChars` truncates, like `web_fetch`.
+- Rendered output is Markdown prefixed with `Title:` / `URL:` lines and a `Rendered via the Jina
+  Reader` provenance line; the 403 fallback prefixes a note instead. An optional `maxChars`
+  truncates, like any fetch.
+- When rendering is disabled or does not produce more text, a page that looks JavaScript-rendered
+  is returned with a `*(JavaScript-rendered page; content may be incomplete)*` note, so a shell is
+  not mistaken for the whole page.
+- Enabled by default. Disable it with `/webtools-config` (`webRenderEnabled` in
+  `~/.config/pi-unsloth-webtools/config.json`), which turns off both escalation paths.
 - Keyless Reader requests are rate-limited per outgoing IP; see
   [Companion: rotating exit IPs](#companion-rotating-exit-ips).
-- Enabled by default. Disable it with `/webtools-config` (`webRenderEnabled` in
-  `~/.config/pi-unsloth-webtools/config.json`), which deactivates the tool for the session.
-- Also used automatically when a `web_fetch` or `web_search` url-mode fetch is refused with HTTP 403.
 
 ## Known differences from Studio
 
@@ -155,11 +159,10 @@ that need JavaScript to render:
   (`file://` URLs and absolute, `~/`, `./` paths) by default; opt out with
   `webFetch.allowPrivateAddresses: false` and `webFetch.allowLocalFiles: false` to restore
   Studio's behavior.
-- Third-party rendering: the extra `web_render` tool asks the Jina Reader (`r.jina.ai`) to fetch
-  the page, so the target URL leaves the machine. Studio has no third-party rendering path. This
-  path always refuses local files and non-public addresses, regardless of the local-access settings.
-  A direct fetch refused with HTTP 403 is retried through it automatically when enabled, so those
-  targets also leave the machine in that case.
+- Third-party rendering: when rendering is enabled, a fetch refused with HTTP 403 or returning a
+  page that looks JavaScript-rendered is retried through the Jina Reader (`r.jina.ai`), so the
+  target URL leaves the machine. Studio has no third-party rendering path. This path always refuses
+  local files and non-public addresses, regardless of the local-access settings.
 - PDF styling: MuPDF.js exposes one font per line, so mixed-style lines style the
   whole line instead of per-span; superscript, subscript, underline, strikeout, and
   highlight markers are not emitted. Tables use a conservative text-grid detector:
@@ -182,7 +185,7 @@ that need JavaScript to render:
 - Proxies: Studio routes through environment proxies; this port resolves and pins the target IP and
   tunnels that connection through `HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY` when the proxy is a
   SOCKS5 proxy (`NO_PROXY` exclusions respected; DNS stays local for the guard). Other proxy
-  schemes fall back to a direct connection. The search and `web_render` paths use the process-wide
+  schemes fall back to a direct connection. The search and Jina rendering paths use the process-wide
   `fetch`, so an agent-level proxy dispatcher applies there too — see
   [Companion: rotating exit IPs](#companion-rotating-exit-ips).
 - Dedup and titles: the aggregator keys on canonicalized hrefs (`utm_*`/tracking parameters
@@ -204,7 +207,7 @@ is `false`. For other tradeoffs, prefer:
 | Need | Use |
 |---|---|
 | Browser-like TLS/HTTP fingerprinting to unblock bot-defended pages | `pi-smart-fetch` (`wreq-js` `chrome_145`) |
-| Headless Chrome for JS-rendered SPAs/YouTube/Reddit threads | Built-in `web_render` (Jina Reader) first; `georgebashi/pi-web-fetch` (puppeteer + trafilatura) when the Reader falls short |
+| Headless Chrome for JS-rendered SPAs/YouTube/Reddit threads | Built-in automatic Jina Reader rendering in `web_fetch` first; `georgebashi/pi-web-fetch` (puppeteer + trafilatura) when the Reader falls short |
 | Hosted search with semantic ranking and no scraping | `Brave Search API` / `Tavily` / `Exa` via `pi-ollama-web-search` |
 | Prompt-focused page distillation to save context | `pi-web-fetch` `prompt` -> sub-agent or Claude Code `WebFetch(url,prompt)` |
 | Batch fetching many URLs concurrently | `pi-smart-fetch` `batch_web_fetch` or call `web_fetch` in parallel |
@@ -216,7 +219,7 @@ choose the best tool per URL. No need to fork this package to add those features
 
 [`pi-tor-proxy`](https://github.com/YuGiMob/pi-tor-proxy) routes pi's in-process `fetch` traffic
 through Tor (it downloads and manages its own Tor binary) and gives each pi instance its own
-circuit and exit IP. The search sweep and `web_render` both use `fetch`, so they leave through
+circuit and exit IP. The search sweep and Jina rendering both use `fetch`, so they leave through
 the current Tor exit, and Jina rate-limits keyless Reader requests per outgoing IP —
 `/tor-cycle` swaps the exit those limits are counted against, while `/tor-country` and
 `/tor-exclude` constrain which exits are used.
@@ -225,7 +228,7 @@ the current Tor exit, and Jina rate-limits keyless Reader requests per outgoing
 pi install npm:pi-unsloth-webtools npm:pi-tor-proxy
 ```
 
-`web_fetch` and `web_search` url mode also route: they resolve and pin the target IP, then tunnel
+`web_fetch` also routes: it resolves and pins the target IP, then tunnels
 the connection through the SOCKS5 proxy named by `HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY` (with
 `NO_PROXY` exclusions, so localhost and local files stay direct). DNS is still resolved locally for
 the SSRF guard, and the proxy connects to that pinned IP.
@@ -254,13 +257,13 @@ Optional settings in `~/.pi/agent/settings.json` or `.pi/settings.json` (project
 | Key | Default | Description |
 |---|---|---|
 | `unslothWebTools.maxResults` | `5` | Default `maxResults` for `web_search` (clamped 1-20) |
-| `unslothWebTools.maxChars` / `webFetch.maxChars` / `smartFetchDefaultMaxChars` | tool param | Default `maxChars` for `web_fetch` and `web_search` url mode |
-| `unslothWebTools.timeoutMs` / `webFetch.timeoutMs` / `smartFetchDefaultTimeoutMs` | `60000` fetch, `300000` search | Default `timeoutMs` when the tool param is absent (>=1000). Fetch and `web_search` url mode fall back to 60000; `web_search` query mode falls back to 300000 |
+| `unslothWebTools.maxChars` / `webFetch.maxChars` / `smartFetchDefaultMaxChars` | tool param | Default `maxChars` for `web_fetch` |
+| `unslothWebTools.timeoutMs` / `webFetch.timeoutMs` / `smartFetchDefaultTimeoutMs` | `60000` fetch, `300000` search | Default `timeoutMs` when the tool param is absent (>=1000). Fetch falls back to 60000; `web_search` query mode falls back to 300000 |
 | `webSearch.maxResults` / `smartWebSearch.resultsPerQuery` | same as above | Legacy aliases for `maxResults` |
 | `websitePolicy` | none | Not read from settings. Tools run unrestricted by default; `websitePolicy` is a programmatic option the host passes to `webSearch` / `fetchPageText` |
 | `unslothWebTools.allowPrivateAddresses` / `webFetch.allowPrivateAddresses` | `true` | Opt out to restore the resolved-IP SSRF guard: private/loopback/link-local hosts (localhost, LAN IPs) are refused again. Non-canonical numeric IP encodings stay blocked either way |
-| `unslothWebTools.allowLocalFiles` / `webFetch.allowLocalFiles` | `true` | Opt out to refuse local files in `web_fetch` and `web_search` url mode (`file://` URLs, absolute, `~/`, or `./` paths); when enabled, PDFs are extracted and HTML converted |
-| `unslothWebTools.jinaApiKey` / `webRender.jinaApiKey` | none (`JINA_API_KEY` fallback) | API key for `web_render`'s Jina Reader; raises its rate limits. Settings keys win over the environment variable |
+| `unslothWebTools.allowLocalFiles` / `webFetch.allowLocalFiles` | `true` | Opt out to refuse local files in `web_fetch` (`file://` URLs, absolute, `~/`, or `./` paths); when enabled, PDFs are extracted and HTML converted |
+| `unslothWebTools.jinaApiKey` / `webRender.jinaApiKey` | none (`JINA_API_KEY` fallback) | API key for automatic Jina Reader rendering; raises its rate limits. Settings keys win over the environment variable |
 
 ### Settings window
 
@@ -276,14 +279,14 @@ first changed:
 
 | Key | Default | Description |
 |---|---|---|
-| `webRenderEnabled` | `true` | When `false`, the `web_render` tool is deactivated for the session and the automatic HTTP 403 fallback in `web_fetch` / `web_search` url mode is disabled |
+| `webRenderEnabled` | `true` | When `false`, automatic Jina Reader rendering is disabled: HTTP 403 and JavaScript-page escalation in `web_fetch` no longer run |
 
 On non-Windows platforms the directory honors `XDG_CONFIG_HOME` when set (falling back to
 `~/.config`); on Windows it always uses `~/.config`.
 
 Tool params always win over file defaults. Search dedup also strips default ports, so `https://example.com:443/a` and `https://example.com/a` collapse.
 
-Environment overrides: `PI_UNSLOTH_CACHE_DIR` changes the fetch cache directory, `PI_UNSLOTH_WEBTOOLS_STATS` opts into append-only sweep stats JSONL, `PI_CODING_AGENT_DIR` / `PI_AGENT_DIR` change the global settings directory, and `JINA_API_KEY` supplies the `web_render` key when no settings key is set. Cache entries live 1 hour and stale copies are served only after a network failure. SOCKS5 proxies named by `HTTPS_PROXY`, `HTTP_PROXY`, or `ALL_PROXY` are honored on every fetch (`NO_PROXY` exclusions apply).
+Environment overrides: `PI_UNSLOTH_CACHE_DIR` changes the fetch cache directory, `PI_UNSLOTH_WEBTOOLS_STATS` opts into append-only sweep stats JSONL, `PI_CODING_AGENT_DIR` / `PI_AGENT_DIR` change the global settings directory, and `JINA_API_KEY` supplies the Jina Reader key when no settings key is set. Cache entries live 1 hour and stale copies are served only after a network failure. SOCKS5 proxies named by `HTTPS_PROXY`, `HTTP_PROXY`, or `ALL_PROXY` are honored on every fetch (`NO_PROXY` exclusions apply).
 
 ## Troubleshooting
 
@@ -300,16 +303,14 @@ Match on the exact prefix. Do not retry blocked hosts with spelling tricks.
 | Private address blocked | `Blocked: refusing to fetch the non-public address ...` | The SSRF guard is active (`allowPrivateAddresses: false`); remove it or set `true` to reach localhost/LAN, and write the scheme explicitly (`http://localhost:3000`). |
 | Local file blocked | `Blocked: the URL has an invalid hostname or port.` for paths | Local files are disabled: remove `allowLocalFiles: false` to read `file://`, absolute, `~/`, or `./` paths. |
 | File read failed | `Failed to read file: ...` | Check the path exists and is a regular file. |
-| HTTP failure | `Failed to fetch URL: HTTP ...` | Fix the URL. A 404 automatically tries a Wayback snapshot; a 403 retries through `web_render` when enabled. |
+| HTTP failure | `Failed to fetch URL: HTTP ...` | Fix the URL. A 404 automatically tries a Wayback snapshot; a 403 retries through the Jina Reader when rendering is enabled. |
 | Proxy failure | `Failed to fetch URL: SOCKS5 proxy ...` | The SOCKS5 proxy refused or failed (for example Tor is stopping). Check the proxy, or unset the proxy variables for a direct fetch. |
 | Non-text / binary | `(non-text content:` / `(binary content,` | Not readable as text by design. |
 | PDF without text | `(PDF contains no extractable text)` / `(PDF content could not be read as text...)` | Scanned or encrypted PDF. |
 | Download cap hit | `... (page truncated at the download limit)` | Raw fetch hit 512 KiB (10 MiB for PDFs). |
 | maxChars cut | `... (truncated, N chars total)` | Raise `maxChars` for the full text. |
-| Empty page | `(page returned no readable text)` | Page had no extractable text; try `web_render`, which renders JavaScript pages. |
-| Render blocked (local file) | `Blocked: web_render cannot fetch local files.` | Use `web_fetch`, which reads local files by default. |
-| Render blocked (private host) | `Blocked: refusing to fetch the non-public address ...` | `web_render` only reaches public hosts; use `web_fetch` for localhost/LAN. |
-| Render failure | `Failed to render URL: ...` | Jina rejected or failed the request (rate limit, bad key, unreachable page). Retry, or check `unslothWebTools.jinaApiKey` / `JINA_API_KEY`. |
+| Empty page | `(page returned no readable text)` | Page had no extractable text; `web_fetch` retries through the Jina Reader automatically when rendering is enabled. |
+| JavaScript-rendered page | `*(JavaScript-rendered page; content may be incomplete)*` | Rendering was disabled or produced no more text; the page likely needs a browser. |
 | GitHub rewrite | `README of ... (fetched via the GitHub README API):` | Expected repo-root rewrite, not the HTML chrome. |
 | Cache fallback | `Served from cache` / `STALE cache from YYYY-MM-DD` | Network failed; output is the cached copy with its date. |
 | Wayback fallback | `Fetched from Wayback Machine snapshot (YYYY-MM-DD) for ...` | Original 404'd; output is the archived copy with its date. |

package/config-ui.ts

@@ -15,8 +15,8 @@ export function configRows(config: Config): ConfigRow[] {
   return [
     {
       key: "webRenderEnabled",
-      label: "Web render",
-      hint: "web_render tool (Jina Reader)",
+      label: "JavaScript rendering",
+      hint: "Jina Reader fallback",
       enabled: config.webRenderEnabled !== false,
     },
   ];

package/entities.ts

@@ -1,4 +1,4 @@
-export const NAMED_ENTITIES: Record<string, string> = {
+export const NAMED_ENTITIES: Record<string, string> = Object.assign(Object.create(null), {
   "AElig": "\u00c6",
   "AElig;": "\u00c6",
   "AMP": "&",
@@ -2230,7 +2230,7 @@ export const NAMED_ENTITIES: Record<string, string> = {
   "zscr;": "\ud835\udccf",
   "zwj;": "\u200d",
   "zwnj;": "\u200c",
-};
+});
 
 export const INVALID_CHARREFS: Record<number, string> = {
   0: "\ufffd",

package/html-to-md.ts

@@ -66,7 +66,7 @@ const P_CLOSING_TAGS = new Set([
   "ul",
 ]);
 
-const IMPLICIT_CLOSERS: Record<string, Set<string>> = {
+const IMPLICIT_CLOSERS: Record<string, Set<string>> = Object.assign(Object.create(null), {
   p: P_CLOSING_TAGS,
   li: new Set(["li"]),
   dt: new Set(["dt", "dd"]),
@@ -76,9 +76,9 @@ const IMPLICIT_CLOSERS: Record<string, Set<string>> = {
   th: new Set(["td", "th", "tr"]),
   option: new Set(["option", "optgroup"]),
   optgroup: new Set(["optgroup"]),
-};
+});
 
-const CLOSE_BARRIERS: Record<string, Set<string>> = {
+const CLOSE_BARRIERS: Record<string, Set<string>> = Object.assign(Object.create(null), {
   li: new Set(["ul", "ol", "menu"]),
   dt: new Set(["dl"]),
   dd: new Set(["dl"]),
@@ -87,7 +87,7 @@ const CLOSE_BARRIERS: Record<string, Set<string>> = {
   th: new Set(["table"]),
   option: new Set(["select", "datalist"]),
   optgroup: new Set(["select", "datalist"]),
-};
+});
 
 const BLOCK_TAGS = new Set([
   "p",
@@ -106,7 +106,7 @@ const BLOCK_TAGS = new Set([
 ]);
 
 const HEADING_TAGS = new Set(["h1", "h2", "h3", "h4", "h5", "h6"]);
-const INLINE_EMPHASIS: Record<string, string> = { strong: "**", b: "**", em: "*", i: "*" };
+const INLINE_EMPHASIS: Record<string, string> = Object.assign(Object.create(null), { strong: "**", b: "**", em: "*", i: "*" });
 
 const HEADER_LINK_DENSITY = 0.93;
 const HEADER_MIN_CHARS = 150;

package/index.ts

@@ -1,9 +1,14 @@
 import { defineTool, type ExtensionAPI, type ExtensionContext, type Theme } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
 import { truncateToWidth } from "@earendil-works/pi-tui";
-import { collapseWhitespace } from "./html-to-md.ts";
+import { collapseWhitespace, visibleChars } from "./html-to-md.ts";
 import { SEARCH_TIMEOUT_MS, webSearch as defaultWebSearch } from "./web-search.ts";
-import { DEFAULT_FETCH_TIMEOUT_MS, fetchPageText as defaultFetchPageText } from "./web-fetch.ts";
+import {
+  DEFAULT_FETCH_TIMEOUT_MS,
+  fetchPageOutcome as defaultFetchPageOutcome,
+  fetchPageText as defaultFetchPageText,
+  type FetchPageOutcome,
+} from "./web-fetch.ts";
 import { renderPageText as defaultRenderPageText } from "./web-render.ts";
 import { loadDefaultFetchSettings, loadDefaultFetchTimeoutMs, loadJinaApiKey } from "./settings.ts";
 import { readConfig, readConfigWithStatus, toggleWebRender } from "./config.ts";
@@ -39,24 +44,26 @@ function fetchWasForbidden(text: string): boolean {
   return /^Failed to fetch URL: HTTP 403\b/m.test(text);
 }
 
+function renderFailed(rendered: string): boolean {
+  return (
+    rendered.startsWith("Failed to render URL:") ||
+    rendered.startsWith("Blocked:") ||
+    rendered.startsWith("(page returned no readable text)")
+  );
+}
+
+function renderImprovesPage(rendered: string, fetched: string): boolean {
+  return visibleChars(rendered) > visibleChars(fetched) * 1.5;
+}
+
 const FORBIDDEN_FALLBACK_NOTE = "Direct fetch failed with HTTP 403; rendered via the Jina Reader instead.";
+const THIN_FALLBACK_NOTE = "Direct fetch returned little content; rendered via the Jina Reader instead.";
+const INCOMPLETE_NOTE = "\n\n*(JavaScript-rendered page; content may be incomplete)*";
 
 const WebSearchParams = Type.Object({
   query: Type.Optional(
     Type.String({ description: "The search query" }),
   ),
-  url: Type.Optional(
-    Type.String({
-      description:
-        "A URL to fetch full page content from (instead of searching). Use this to read a page found in search results.",
-    }),
-  ),
-  maxChars: Type.Optional(
-    Type.Number({
-      description:
-        "Truncate the fetched page to this many characters (only used with the url parameter)",
-    }),
-  ),
   maxResults: Type.Optional(
     Type.Number({
       minimum: 1,
@@ -87,43 +94,37 @@ const WebFetchParams = Type.Object({
   ),
 });
 
-const WebRenderParams = Type.Object({
-  url: Type.String({ description: "Public URL of the page to render" }),
-  maxChars: Type.Optional(
-    Type.Number({
-      description: "Truncate the returned content to this many characters (default: no limit)",
-    }),
-  ),
-  timeoutMs: Type.Optional(
-    Type.Number({
-      minimum: 1000,
-      description: "Overall timeout in milliseconds (default: 60000)",
-    }),
-  ),
-});
-
 export interface WebToolsDeps {
   fetchPageText?: typeof defaultFetchPageText;
+  fetchPageOutcome?: typeof defaultFetchPageOutcome;
   webSearch?: typeof defaultWebSearch;
   renderPageText?: typeof defaultRenderPageText;
   webRenderEnabled?: () => Promise<boolean>;
 }
 
 export function createWebTools(deps: WebToolsDeps = {}) {
-  const fetchPageText = deps.fetchPageText ?? defaultFetchPageText;
   const webSearch = deps.webSearch ?? defaultWebSearch;
   const renderPageText = deps.renderPageText ?? defaultRenderPageText;
+  const legacyFetchPageText = deps.fetchPageText;
+  const fetchOutcome: typeof defaultFetchPageOutcome =
+    deps.fetchPageOutcome ??
+    (legacyFetchPageText
+      ? async (url, options) => ({ text: await legacyFetchPageText(url, options), hint: null })
+      : defaultFetchPageOutcome);
   const webRenderEnabled =
     deps.webRenderEnabled ?? (async () => (await readConfig()).webRenderEnabled !== false);
 
-  const renderOnForbidden = async (
+  const renderFallback = async (
     url: string,
-    failedText: string,
+    outcome: FetchPageOutcome,
     options: { timeoutMs: number; maxChars?: number; signal?: AbortSignal; cwd?: string },
   ): Promise<string | null> => {
-    if (!fetchWasForbidden(failedText)) return null;
+    const forbidden = fetchWasForbidden(outcome.text);
+    const thin = outcome.hint !== null && !forbidden;
+    if (!forbidden && !thin) return null;
+    const incomplete = outcome.text + INCOMPLETE_NOTE;
     try {
-      if (!(await webRenderEnabled())) return null;
+      if (!(await webRenderEnabled())) return thin ? incomplete : null;
       const apiKey = await loadJinaApiKey(options.cwd);
       const rendered = await renderPageText(url, {
         timeoutMs: options.timeoutMs,
@@ -131,10 +132,11 @@ export function createWebTools(deps: WebToolsDeps = {}) {
         signal: options.signal,
         apiKey,
       });
-      if (rendered.startsWith("Failed to render URL:") || rendered.startsWith("Blocked:")) return null;
-      return FORBIDDEN_FALLBACK_NOTE + "\n\n" + rendered;
+      if (renderFailed(rendered)) return thin ? incomplete : null;
+      if (thin && !renderImprovesPage(rendered, outcome.text)) return incomplete;
+      return (forbidden ? FORBIDDEN_FALLBACK_NOTE : THIN_FALLBACK_NOTE) + "\n\n" + rendered;
     } catch {
-      return null;
+      return thin ? incomplete : null;
     }
   };
 
@@ -143,40 +145,17 @@ export function createWebTools(deps: WebToolsDeps = {}) {
       name: "web_search",
       label: "Web Search",
       description:
-        "Search the web and fetch page content. Returns snippets for all results. " +
-        "Use the url parameter to fetch full page text from a specific URL. " +
-        "A direct fetch refused with HTTP 403 falls back to web_render when that tool is enabled.",
-      promptSnippet: "Search the web and fetch page content",
+        "Search the web and return snippets for the top results. Use web_fetch to read a page found in the results.",
+      promptSnippet: "Search the web and return snippets",
       promptGuidelines: [
-        'Use web_search with the url parameter (e.g. {"url": "<URL>"}) to read the full text of a page found in search results.',
+        "Web tool order: web_search to discover, then web_fetch to read a page.",
       ],
       parameters: WebSearchParams,
       renderCall(args, theme) {
-        const url = collapsedArg(args.url);
         const query = collapsedArg(args.query);
-        return toolCallLine(theme, "web_search", url || (query ? `"${query}"` : ""));
+        return toolCallLine(theme, "web_search", query ? `"${query}"` : "");
       },
       async execute(_toolCallId, params, signal, onUpdate, _ctx) {
-        if (params.url?.trim()) {
-          const url = params.url.trim();
-          onUpdate?.({ content: [{ type: "text", text: `Fetching ${url}...` }], details: {} });
-          const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
-          const { timeoutMs, maxChars, allowPrivateAddresses, allowLocalFiles } = await fetchDefaults(cwd, params);
-          const text = await fetchPageText(url, {
-            timeoutMs,
-            signal: signal ?? undefined,
-            maxChars,
-            allowPrivateAddresses,
-            allowLocalFiles,
-          });
-          const rendered = await renderOnForbidden(url, text, {
-            timeoutMs,
-            maxChars,
-            signal: signal ?? undefined,
-            cwd,
-          });
-          return { content: [{ type: "text", text: rendered ?? text }], details: {} };
-        }
         onUpdate?.({ content: [{ type: "text", text: "Searching the web..." }], details: {} });
         const timeoutParam = positiveNumber(params.timeoutMs);
         const searchCwd = (_ctx as ExtensionContext | undefined)?.cwd;
@@ -196,14 +175,15 @@ export function createWebTools(deps: WebToolsDeps = {}) {
       description:
         "Fetch a URL and return its readable text. HTML pages are converted to Markdown using a " +
         "main-content heuristic: article/main scoping plus hidden-element and boilerplate " +
-        "stripping. Non-HTML text is returned as-is. GitHub repo root pages are rewritten to the " +
+        "stripping. Pages that look JavaScript-rendered are retried through the Jina Reader " +
+        "automatically when rendering is enabled, as are HTTP 403 responses. " +
+        "Non-HTML text is returned as-is. GitHub repo root pages are rewritten to the " +
         "README API, so the README is returned instead of the repo page's UI chrome. " +
         "Private/loopback/link-local targets and local files (file:// URLs, absolute, ~/ or ./ paths, including " +
         "PDFs) are supported by default; opt out with webFetch.allowPrivateAddresses: false or " +
-        "webFetch.allowLocalFiles: false in settings. The download size is capped. " +
-        "A direct fetch refused with HTTP 403 falls back to web_render (Jina Reader) when that tool is enabled.",
+        "webFetch.allowLocalFiles: false in settings. The download size is capped.",
       promptGuidelines: [
-        "web_fetch automatically retries HTTP 403 responses through web_render (Jina Reader); do not call web_render again for the same URL after a 403.",
+        "web_fetch escalates JavaScript-rendered pages and HTTP 403 responses to the Jina Reader automatically when rendering is enabled.",
       ],
       promptSnippet: "Fetch a web page and return readable text content",
       parameters: WebFetchParams,
@@ -214,77 +194,45 @@ export function createWebTools(deps: WebToolsDeps = {}) {
         onUpdate?.({ content: [{ type: "text", text: `Fetching ${params.url}...` }], details: {} });
         const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
         const { timeoutMs, maxChars, allowPrivateAddresses, allowLocalFiles } = await fetchDefaults(cwd, params);
-        const text = await fetchPageText(params.url, {
+        const deadlineMs = Date.now() + timeoutMs;
+        const outcome = await fetchOutcome(params.url, {
           timeoutMs,
+          deadlineMs,
           signal: signal ?? undefined,
           maxChars,
           allowPrivateAddresses,
           allowLocalFiles,
         });
-        const rendered = await renderOnForbidden(params.url, text, {
-          timeoutMs,
+        const rendered = await renderFallback(params.url, outcome, {
+          timeoutMs: Math.max(1, deadlineMs - Date.now()),
           maxChars,
           signal: signal ?? undefined,
           cwd,
         });
-        return { content: [{ type: "text", text: rendered ?? text }], details: {} };
-      },
-    }),
-    webRenderTool: defineTool({
-      name: "web_render",
-      label: "Web Render",
-      description:
-        "Render a public web page to Markdown through the third-party Jina Reader (r.jina.ai). " +
-        "Use it for JavaScript-rendered pages that web_fetch cannot read. " +
-        "The target URL is sent to Jina; local files and non-public addresses are refused. " +
-        "An optional JINA_API_KEY or unslothWebTools.jinaApiKey raises the Reader's rate limits.",
-      promptSnippet: "Render a JavaScript-rendered page to Markdown via the Jina Reader",
-      promptGuidelines: [
-        'Use web_render when web_fetch reports "(page returned no readable text)" or the page only fills in through JavaScript.',
-      ],
-      parameters: WebRenderParams,
-      renderCall(args, theme) {
-        return toolCallLine(theme, "web_render", collapsedArg(args.url));
-      },
-      async execute(_toolCallId, params, signal, onUpdate, _ctx) {
-        onUpdate?.({ content: [{ type: "text", text: `Rendering ${params.url}...` }], details: {} });
-        const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
-        const { timeoutMs, maxChars } = await fetchDefaults(cwd, params);
-        const apiKey = await loadJinaApiKey(cwd);
-        const text = await renderPageText(params.url, {
-          timeoutMs,
-          maxChars,
-          signal: signal ?? undefined,
-          apiKey,
-        });
-        return { content: [{ type: "text", text }], details: {} };
+        return { content: [{ type: "text", text: rendered ?? outcome.text }], details: {} };
       },
     }),
   };
 }
 
 export default function (pi: ExtensionAPI) {
-  const { webSearchTool, webFetchTool, webRenderTool } = createWebTools();
+  const { webSearchTool, webFetchTool } = createWebTools();
   pi.registerTool(webSearchTool);
   pi.registerTool(webFetchTool);
-  pi.registerTool(webRenderTool);
 
   pi.on("session_start", async (_event, ctx) => {
     try {
-      const { config, corrupted } = await readConfigWithStatus();
+      const { corrupted } = await readConfigWithStatus();
       if (corrupted && ctx.hasUI) {
         ctx.ui.notify("Web tools config was corrupt and was reset to defaults", "warning");
       }
-      if (!config.webRenderEnabled) {
-        pi.setActiveTools(pi.getActiveTools().filter((name) => name !== "web_render"));
-      }
     } catch (error) {
       console.error("Failed to load web tools config:", error);
     }
   });
 
   pi.registerCommand("webtools-config", {
-    description: "Open the web tools settings window (web_render on/off)",
+    description: "Open the web tools settings window (JavaScript rendering on/off)",
     handler: async (_args, ctx) => {
       if (!ctx.hasUI) {
         ctx.ui.notify("/webtools-config requires interactive mode", "error");
@@ -298,13 +246,7 @@ export default function (pi: ExtensionAPI) {
             done,
             onToggle: async (key) => {
               if (key !== "webRenderEnabled") return;
-              const enabled = await toggleWebRender();
-              const active = pi.getActiveTools();
-              pi.setActiveTools(
-                enabled
-                  ? [...new Set([...active, "web_render"])]
-                  : active.filter((name) => name !== "web_render"),
-              );
+              await toggleWebRender();
             },
           });
           await overlay.load();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.7.5",
+  "version": "0.8.0",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools that began as a port of the Unsloth Studio codebase and now diverge from it (multi-engine search, opt-in SSRF guard, HTML-to-Markdown extraction)",
   "main": "index.ts",

package/proxy.ts

@@ -3,6 +3,7 @@ import https from "node:https";
 import net from "node:net";
 import tls from "node:tls";
 import type { Duplex } from "node:stream";
+import { stripIpv6Brackets } from "./web-access.ts";
 
 export interface SocksProxy {
   host: string;
@@ -124,7 +125,7 @@ export function socksProxyForUrl(url: URL): SocksProxy | null {
   const proxy = parseSocksProxy(firstEnv(url.protocol === "https:" ? HTTPS_PROXY_VARS : HTTP_PROXY_VARS));
   if (!proxy) return null;
   const port = url.port ? Number(url.port) : url.protocol === "https:" ? 443 : 80;
-  if (bypassesProxy(url.hostname, port)) return null;
+  if (bypassesProxy(stripIpv6Brackets(url.hostname), port)) return null;
   return proxy;
 }
 

package/web-access.ts

@@ -70,6 +70,10 @@ export function normalizeDomain(value: unknown): string {
   return asciiDomain;
 }
 
+export function stripIpv6Brackets(hostname: string): string {
+  return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
+}
+
 function compressIpv6(ip: string): string {
   const segments = ip.toLowerCase().split("::");
   if (segments.length > 2) throw new Error("invalid ipv6");

package/web-fetch.ts

@@ -2,6 +2,7 @@ import { lookup as dnsLookup } from "node:dns/promises";
 import type { LookupAllOptions } from "node:dns";
 import http from "node:http";
 import https from "node:https";
+import { isIP } from "node:net";
 import { open } from "node:fs/promises";
 import type { FileHandle } from "node:fs/promises";
 import { homedir } from "node:os";
@@ -19,9 +20,10 @@ import {
   isPublicIp,
   MAX_SIGNAL_TIMEOUT_MS,
   normalizeUrlScheme,
+  stripIpv6Brackets,
   type WebsitePolicy,
 } from "./web-access.ts";
-import { collapseWhitespace, decodeHtmlEntities, feedHtml, htmlToMarkdown } from "./html-to-md.ts";
+import { collapseWhitespace, decodeHtmlEntities, feedHtml, htmlToMarkdown, visibleChars } from "./html-to-md.ts";
 import type { AttrDict } from "./html-to-md.ts";
 import { INVALID_CHARREFS } from "./entities.ts";
 import { getCached, isFresh, setCached, staleNotice } from "./cache.ts";
@@ -125,6 +127,14 @@ const HTML_LEADING_RE = new RegExp(
 );
 const HTML_DOCUMENT_RE = /^<(?:!doctype\s+html\b|\/?(?:html|head|body)\b)/;
 
+const RENDER_THIN_PROSE_CHARS = 400;
+const RENDER_SCRIPT_SHARE = 0.5;
+const RENDER_SCRIPT_MIN_HTML_BYTES = 8192;
+const RENDER_MIN_SCRIPTS = 3;
+const RENDER_NOSCRIPT_MIN_CHARS = 40;
+const RENDER_SPA_MARKERS =
+  /(?:id\s*=\s*["']?(?:root|app|__next|__nuxt)["']?|data-reactroot|ng-version|__NEXT_DATA__|__NUXT__|__PRELOADED_STATE__)/i;
+
 const MIN_SINGLE_BYTE_ASCII_RATIO = 3 / 4;
 const ASCII_TEXT_BYTES = new Set<number>([
   ...Array.from({ length: 0x7f - 0x20 }, (_, i) => i + 0x20),
@@ -235,6 +245,16 @@ export interface RawFetchResult {
   contentType: string;
 }
 
+export interface RenderHint {
+  reason: "empty" | "js-shell";
+  evidence: string[];
+}
+
+export interface FetchPageOutcome {
+  text: string;
+  hint: RenderHint | null;
+}
+
 function htmlProbe(body: string, re: RegExp): boolean {
   let i = 0;
   const n = body.length;
@@ -265,6 +285,46 @@ export function looksLikeHtmlDocument(body: string): boolean {
   return htmlProbe(body, HTML_DOCUMENT_RE);
 }
 
+function noscriptText(html: string): string {
+  const parts: string[] = [];
+  for (const match of html.match(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi) ?? []) {
+    parts.push(match.replace(/<[^>]*>/g, " "));
+  }
+  return collapseWhitespace(decodeHtmlEntities(parts.join(" ")));
+}
+
+export function needsRenderHint(html: string, converted: string): RenderHint | null {
+  const prose = visibleChars(converted);
+  if (prose >= RENDER_THIN_PROSE_CHARS) return null;
+  if (prose === 0) return { reason: "empty", evidence: [] };
+  const evidence: string[] = [];
+  let score = 0;
+  if (RENDER_SPA_MARKERS.test(html)) {
+    evidence.push("spa-markers");
+    score += 2;
+  }
+  const scriptCount = (html.match(/<script\b/gi) ?? []).length;
+  let scriptBytes = 0;
+  for (const script of html.match(/<script\b[^>]*>[\s\S]*?<\/script>/gi) ?? []) scriptBytes += script.length;
+  if (html.length >= RENDER_SCRIPT_MIN_HTML_BYTES && scriptBytes >= html.length * RENDER_SCRIPT_SHARE) {
+    evidence.push("script-heavy");
+    score += 2;
+  }
+  if (scriptCount >= RENDER_MIN_SCRIPTS) {
+    evidence.push("scripts");
+    score += 1;
+  }
+  if (noscriptText(html).length >= RENDER_NOSCRIPT_MIN_CHARS) {
+    evidence.push("noscript");
+    score += 1;
+  }
+  if (extractPageMeta(html).description) {
+    evidence.push("description");
+    score += 1;
+  }
+  return score >= 2 ? { reason: "js-shell", evidence } : null;
+}
+
 function parseContentType(value: string | null | undefined): string {
   return /^[\w.+-]+\/[\w.+-]+/.exec(value ?? "")?.[0] ?? "";
 }
@@ -320,7 +380,7 @@ function hasSingleByteTextEvidence(data: Buffer): boolean {
   return ascii / data.length >= MIN_SINGLE_BYTE_ASCII_RATIO;
 }
 
-const CHARSET_ALIASES: Record<string, string> = {
+const CHARSET_ALIASES: Record<string, string> = Object.assign(Object.create(null), {
   gbk: "gbk",
   gb2312: "gbk",
   "gb-2312": "gbk",
@@ -346,7 +406,7 @@ const CHARSET_ALIASES: Record<string, string> = {
   "windows-874": "windows-874",
   cp874: "windows-874",
   "tis-620": "tis-620",
-};
+});
 
 function normalizeCharset(name: string): string | null {
   const n = name.trim().replace(/["']/g, "").toLowerCase();
@@ -637,9 +697,10 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
     const url = opts.url;
     const transport = url.protocol === "https:" ? https : http;
     const port = url.port ? Number(url.port) : url.protocol === "https:" ? 443 : 80;
+    const hostname = stripIpv6Brackets(url.hostname);
     const options: https.RequestOptions = {
       method: "GET",
-      host: url.hostname,
+      host: hostname,
       port,
       path: url.pathname + url.search,
       headers: opts.headers,
@@ -652,12 +713,12 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
         ip: opts.pinnedIp,
         family: opts.family,
         port,
-        servername: url.hostname,
+        servername: hostname,
         timeoutMs: Math.max(1, opts.inactivityMs),
         signal: opts.signal,
       });
     } else {
-      options.servername = url.protocol === "https:" ? url.hostname : undefined;
+      options.servername = url.protocol === "https:" && !isIP(hostname) ? hostname : undefined;
       options.lookup = ((_hostname: string, _options: unknown, _callback: unknown) => {
         const done = (typeof _options === "function" ? _options : _callback) as (err: unknown, address: unknown, family?: unknown) => void;
         if ((_options as { all?: boolean } | undefined)?.all) {
@@ -881,9 +942,10 @@ export async function fetchUrlRaw(
     const budgetResult = checkBudget();
     if (budgetResult !== null) return budgetResult;
     const parsed = new URL(currentUrl);
-    const hostHeader = parsed.hostname.includes(":")
-      ? `[${parsed.hostname}]${parsed.port ? `:${parsed.port}` : ""}`
-      : parsed.hostname + (parsed.port ? `:${parsed.port}` : "");
+    const parsedHostname = stripIpv6Brackets(parsed.hostname);
+    const hostHeader = parsedHostname.includes(":")
+      ? `[${parsedHostname}]${parsed.port ? `:${parsed.port}` : ""}`
+      : parsedHostname + (parsed.port ? `:${parsed.port}` : "");
     const headers: Record<string, string> = {
       "User-Agent": userAgent,
       Host: hostHeader,
@@ -1133,9 +1195,10 @@ interface PageMeta {
   author: string;
   date: string;
   site: string;
+  description: string;
 }
 
-const META_KEYS: Record<string, keyof PageMeta> = {
+const META_KEYS: Record<string, keyof PageMeta> = Object.assign(Object.create(null), {
   author: "author",
   "article:author": "author",
   "dc.creator": "author",
@@ -1145,7 +1208,9 @@ const META_KEYS: Record<string, keyof PageMeta> = {
   datepublished: "date",
   "og:site_name": "site",
   "application-name": "site",
-};
+  description: "description",
+  "og:description": "description",
+});
 
 function cutAtCharBoundary(text: string, maxChars: number): string {
   const sliced = text.slice(0, maxChars);
@@ -1165,7 +1230,7 @@ function capMetaValue(value: string): string {
 }
 
 function extractPageMeta(html: string): PageMeta {
-  const meta: PageMeta = { title: extractPageTitle(html), author: "", date: "", site: "" };
+  const meta: PageMeta = { title: extractPageTitle(html), author: "", date: "", site: "", description: "" };
   const seen = new Set<string>();
   const record = (name: string, attrs: AttrDict) => {
     if (name !== "meta") return;
@@ -1291,6 +1356,13 @@ function localFileFailure(err: unknown): string {
   return `Failed to read file: ${err instanceof Error ? err.message : String(err)}`;
 }
 
+const HTML_FILE_EXTENSIONS = new Set([".htm", ".html", ".xht", ".xhtml"]);
+
+function localFileContentType(filePath: string): string {
+  const dot = filePath.lastIndexOf(".");
+  return dot !== -1 && HTML_FILE_EXTENSIONS.has(filePath.slice(dot).toLowerCase()) ? "text/html" : "";
+}
+
 async function readLocalFile(
   filePath: string,
   options: { signal?: AbortSignal; maxChars?: number; maxBytes?: number; maxPdfBytes?: number },
@@ -1333,16 +1405,16 @@ async function readLocalFile(
     }
     const text = decodeWithCodec(body, bomCodecFor(body) ?? "utf-8");
     if (looksBinary(text)) return `(binary content, ${body.length} bytes; not readable as text)`;
-    return truncatePageText(withTruncation(renderBody(text, ""), truncated), options.maxChars);
+    return truncatePageText(withTruncation(renderBody(text, localFileContentType(filePath)), truncated), options.maxChars);
   } finally {
     await handle.close().catch(() => {});
   }
 }
 
-export async function fetchPageText(
+export async function fetchPageOutcome(
   url: string,
   options: FetchPageOptions = {},
-): Promise<string> {
+): Promise<FetchPageOutcome> {
   const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
   const now = options.nowMs ?? Date.now;
   const deadlineMs = options.deadlineMs ?? now() + timeoutMs;
@@ -1353,18 +1425,19 @@ export async function fetchPageText(
   if (options.allowLocalFiles !== false) {
     const localPath = parseLocalPath(url);
     if (localPath !== null) {
-      if (!localPath) return "Failed to read file: invalid file URL.";
-      return readLocalFile(localPath, {
+      if (!localPath) return { text: "Failed to read file: invalid file URL.", hint: null };
+      const localText = await readLocalFile(localPath, {
         signal,
         maxChars,
         maxBytes: options.maxBytes,
         maxPdfBytes: options.maxPdfBytes,
       });
+      return { text: localText, hint: null };
     }
   }
   url = normalizeUrlScheme(url);
   const [allowed, reason] = checkUrlAccess(url, policy);
-  if (!allowed) return reason;
+  if (!allowed) return { text: reason, hint: null };
   const rawFetchOptions = {
     deadlineMs,
     signal,
@@ -1381,7 +1454,7 @@ export async function fetchPageText(
     if (rawResult.error === null) {
       const out = renderBody(rawResult.body, rawResult.contentType);
       await persistCache(url, rawResult.body, rawResult.contentType, useCache);
-      return truncatePageText(out, maxChars);
+      return { text: truncatePageText(out, maxChars), hint: null };
     }
   }
   const readmeApiUrl = githubRepoReadmeApiUrl(url);
@@ -1397,7 +1470,7 @@ export async function fetchPageText(
     if (apiBody.trim()) {
       const rendered = `README of ${url} (fetched via the GitHub README API):\n\n` + apiBody;
       await persistCache(url, rendered, "text/markdown", useCache);
-      return truncatePageText(rendered, maxChars);
+      return { text: truncatePageText(rendered, maxChars), hint: null };
     }
     const rawReadmeUrl = githubRepoRawReadmeUrl(url);
     if (rawReadmeUrl) {
@@ -1406,7 +1479,7 @@ export async function fetchPageText(
       if (rawBody.trim()) {
         const rendered = `README of ${url} (fetched via the GitHub raw README URL):\n\n` + rawBody;
         await persistCache(url, rendered, "text/markdown", useCache);
-        return truncatePageText(rendered, maxChars);
+        return { text: truncatePageText(rendered, maxChars), hint: null };
       }
     }
   }
@@ -1421,7 +1494,7 @@ export async function fetchPageText(
             let cachedOut = renderBody(cached.body, cached.contentType);
             if (!isFresh(cached, now())) cachedOut += staleNotice(cached);
             else cachedOut += "\n\n*Served from cache — network fetch failed*";
-            return truncatePageText(cachedOut, maxChars);
+            return { text: truncatePageText(cachedOut, maxChars), hint: null };
           }
         } catch {}
       }
@@ -1433,14 +1506,19 @@ export async function fetchPageText(
             const ts = wb.timestamp ? `${wb.timestamp.slice(0, 4)}-${wb.timestamp.slice(4, 6)}-${wb.timestamp.slice(6, 8)}` : "unknown date";
             out = `*Fetched from Wayback Machine snapshot (${ts}) for ${originalUrl}:*\n\n` + out;
             await persistCache(originalUrl, wb.body, wb.contentType, useCache);
-            return truncatePageText(out, maxChars);
+            return { text: truncatePageText(out, maxChars), hint: null };
           }
         } catch {}
       }
     }
-    return result.error;
+    return { text: result.error, hint: null };
   }
   const finalOut = renderBody(result.body, result.contentType);
+  const hint = isHtmlContent(result.body, result.contentType) ? needsRenderHint(result.body, finalOut) : null;
   await persistCache(originalUrl, result.body, result.contentType, useCache);
-  return truncatePageText(finalOut, maxChars);
+  return { text: truncatePageText(finalOut, maxChars), hint };
+}
+
+export async function fetchPageText(url: string, options: FetchPageOptions = {}): Promise<string> {
+  return (await fetchPageOutcome(url, options)).text;
 }

package/web-render.ts

@@ -6,7 +6,7 @@ const DEFAULT_RENDER_TIMEOUT_MS = 60_000;
 const JINA_READER_URL = "https://r.jina.ai/";
 const CANCELLED_MESSAGE = "Failed to render URL: cancelled.";
 const TIMED_OUT_MESSAGE = "Failed to render URL: timed out.";
-const LOCAL_FILE_MESSAGE = "Blocked: web_render cannot fetch local files.";
+const LOCAL_FILE_MESSAGE = "Blocked: the Jina Reader cannot fetch local files.";
 const EMPTY_READER_MESSAGE = "Failed to render URL: the Jina Reader returned no content.";
 
 export interface RenderPageOptions {
@@ -74,7 +74,7 @@ export async function renderPageText(url: string, options: RenderPageOptions = {
   const resolved = await resolveAndValidateHost(hostname, signal, false);
   const resolutionBudget = budgetMessage(options.signal, signal);
   if (resolutionBudget !== null) return resolutionBudget;
-  if (!resolved.ok) return resolved.reason;
+  if (!resolved.ok) return resolved.reason.startsWith("Blocked:") ? resolved.reason : `Failed to render URL: ${resolved.reason}`;
   const headers: Record<string, string> = { Accept: "application/json" };
   const apiKey = options.apiKey?.trim();
   if (apiKey) headers["Authorization"] = `Bearer ${apiKey}`;

package/web-search.ts

@@ -114,6 +114,6 @@ export function formatSearchResults(results: SearchResult[]): string {
   return (
     text +
     "\n\n---\n\nThese are only short snippets. " +
-    'To get the full page content, call web_search with the url parameter (e.g. {"url": "<URL>"}).'
+    "To read a page, call web_fetch with its URL."
   );
 }