pi-unsloth-webtools 0.2.4 → 0.3.0

package/README.md

@@ -26,44 +26,72 @@ Mirrors Unsloth Studio's `web_search` tool:
 
 - Searches exactly like Studio's pinned `ddgs==9.14.4` `DDGS.text()`: the same seven engines
   (duckduckgo, brave, google, mojeek, yahoo, yandex, wikipedia; bing is disabled upstream),
-  the same provider deduplication, href-dedupe aggregator with frequency ordering, and the
-  same `SimpleFilterRanker` re-ranking. Formats results identically: `Title:` / `URL:` /
+  the same provider deduplication, href-dedupe aggregator with frequency ordering (hrefs are
+  canonicalized first — `utm_*`/tracking parameters and fragments are dropped and the URL is
+  re-serialized, collapsing host-case, default-port, and trailing-slash variants — so the
+  same page found via different tracking links collapses), and the same `SimpleFilterRanker`
+  re-ranking. Formats results identically: `Title:` / `URL:` /
   `Snippet:` blocks separated by `---`, ending with the hint to pass `{"url": "<URL>"}` to
   read a full page.
-- Accepts an optional `url` parameter; when given, fetches that page's text instead of searching.
+- Accepts an optional `url` parameter; when given, fetches that page's text instead of
+  searching (optionally truncated with `maxChars`).
 - Rate-limit, timeout, and empty-result messages mirror Studio's `_search_failure_message`.
+- Transient engine failures (network errors or null responses) are retried once with a short
+  backoff inside the same timeout budget (a retry that cannot fit in the remaining budget is
+  skipped); timeouts and cancellations are never retried.
 
 ### web_fetch
 
 Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
 
 - URL scheme normalization (bare hosts like `google.com` become `https://google.com`).
-- URL validation: http/https only, no credentials or encoded hostnames, hostname/port checks.
+- URL validation: http/https only, no credentials or encoded hostnames, hostname/port checks (any
+  port 1–65535 is permitted; SSRF protection is enforced at the resolved-IP layer, not by port
+  allowlists).
 - DNS resolution with SSRF protection: every resolved address is validated against
   private/loopback/link-local/CGNAT/documentation/multicast/reserved ranges, then the validated IP
   is pinned for the connection (custom `lookup` + SNI `servername`), so DNS cannot rebind between
-  validation and fetch.
+  validation and fetch; resolution shares the caller's abort signal and the overall deadline,
+  so a stuck resolver cannot outlive the fetch.
 - GitHub repo root pages are rewritten to the unauthenticated README API
-  (`Accept: application/vnd.github.raw+json`), falling back to the HTML page on failure.
+  (`Accept: application/vnd.github.raw+json`), falling back to the raw README URL
+  (`raw.githubusercontent.com`, no API rate limit) and then to the HTML page on failure.
+  returning error-page content.
 - Up to 4 redirect hops, each re-validated and re-resolved against the same rules.
 - 512 KiB download cap (10 MiB for PDFs), overall deadline + per-hop socket timeouts, abort-aware
-  (`signal` cancels mid-flight).
+  (`signal` cancels mid-flight). Fetches cut off by a download cap are marked with a trailing
+  truncation notice, so a partial page is not mistaken for a complete one.
+- Responses sent with a `Content-Encoding` of gzip, deflate, or brotli (servers that ignore the
+  `Accept-Encoding: identity` request) are decompressed while streaming, so the download caps
+  bound the decoded page text (a gzip'd PDF still gets the 10 MiB PDF budget via magic sniffing on
+  the decoded head) and a stream cut by the cap or a mid-stream decode failure returns the readable
+  decoded prefix with the truncation notice instead of a binary-content error. A 64 MiB
+  decoded-output cap bounds a compressed bomb.
+- Long fetches and searches report a short progress note to the session before they start, so
+  slow tool calls are not silent.
 - PDF text extraction via the official MuPDF.js engine (the same C library pymupdf wraps):
   object streams, all filters, ToUnicode fonts, encryption detection, and a
-  pymupdf4llm-style markdown layer (headings, bold/italic, code fences, links, tables)
+  pymupdf4llm-style markdown layer (headings, bold/italic, code fences, links, tables),
+  running header/footer and page-number stripping,
   with Studio's corrupted/incomplete fallback to plain text.
 - Content sniffing: MIME allow/deny, binary magic signatures, PDF magic detection, and charset
-  decoding (declared charset, BOM sniffing for UTF-8/16/32, `<meta charset>` sniffing for
+  decoding (BOM sniffing for UTF-8/16/32 first, then the declared charset, `<meta charset>` sniffing for
   CJK and Windows/ISO encodings, cp1252 rescue for mislabeled single-byte pages).
 - HTML → Markdown conversion ported from Studio's dependency-free `_html_to_md.py`: headings,
   links, emphasis, lists, tables, blockquotes, code fences, entity decoding; hidden-element
   stripping (`hidden`, `aria-hidden`, inline styles); `<article>`/`<main>` main-content scoping
   with link-density header stripping; boilerplate-line removal.
 - No page-size budget: fetched pages and PDFs are returned in full (Studio's window-aware
-  cap is deliberately dropped; the optional `maxChars` parameter still truncates when given).
+  cap is deliberately dropped; the optional `maxChars` parameter still truncates when given,
+  on `web_fetch` and on `web_search`'s url mode).
   The 512 KiB / 10 MiB download caps still bound the raw fetch.
 - HTML entity decoding replicates CPython's `html.unescape` (full 2,231-entry HTML5 table,
   longest-prefix rule, Windows-1252 numeric mappings), matching Studio byte-for-byte.
+- Fetched HTML pages are prefixed with the decoded document `<title>`, so the model can
+  see which page it is reading. `Author:` (`meta name=author` / `article:author` / `dc.creator`),
+  `Date:` (`article:published_time` / `dc.date` / `date`) and `Site:` (`og:site_name` /
+  `application-name`) lines are added when declared, so the model can judge recency and
+  provenance.
 
 ## Known differences from Studio
 
@@ -71,6 +99,11 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   whole line instead of per-span; superscript, subscript, underline, strikeout, and
   highlight markers are not emitted. Tables use a conservative text-grid detector:
   aligned text tables are detected, drawn-rule-only tables are not.
+- Running headers and footers: lines repeated at the same page-edge position on at
+  least half the pages (two pages minimum) are dropped from the markdown layer, as is
+  any numeric-only line at a fixed edge position where page numbers appear on at least
+  half the pages (so a one-off number sharing that position is dropped too, while fused
+  labels like `Page 3 of 12` survive). Studio and pymupdf4llm return them verbatim.
 - Search engines: Node's `fetch` TLS fingerprint differs from ddgs's `primp`
   impersonation, so Google/Brave/Yahoo/Yandex may block or serve consent pages more
   aggressively (a blocked engine simply contributes no results). User agents are a
@@ -78,9 +111,18 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   database.
 - Empty sweeps: ddgs 9.14.4 raises the last engine exception; this port reports a
   timeout whenever any engine timed out, so the timeout message is not masked by later
-  generic engine failures.
+  generic engine failures. The timeout budget bounds the entire sweep: per-engine
+  timeouts shrink as the budget is consumed, so the reported timeout matches the
+  worst-case wall time.
 - Proxies: Studio routes through environment proxies; this port always connects
   directly with DNS pinning (deliberately out of scope).
+- Dedup and titles: the aggregator keys on canonicalized hrefs (`utm_*`/tracking parameters
+  and fragments stripped, then the URL re-serialized); fetched HTML pages are prefixed with
+  the document `<title>`. Studio keys on raw hrefs and returns the converted body alone.
+- Upstream drift: current ddgs ships ten backends (adding bing, startpage, grokipedia),
+  requires a `vqd` token for DuckDuckGo, and exposes an `extract()` mode. This port
+  deliberately pins the Studio snapshot — seven engines, bing disabled upstream, no vqd,
+  no pagination — so engine behavior matches Studio rather than ddgs head.
 
 ## Development
 
@@ -115,7 +157,8 @@ The suite ports Unsloth Studio's own tests for these tools:
   ASCII85Decode, font `/Differences` encodings, pymupdf4llm-style headings/links/tables
 - `test/entities.test.ts`: `decodeHtmlEntities` parity with CPython `html.unescape`,
   legacy refs, longest-prefix rule, Windows-1252 numeric mappings, invalid codepoints
-- `test/smoke.test.ts`: live network checks against real hosts
+- `test/smoke.test.ts`: live network checks against real hosts, including a per-engine
+  result-health sweep (at least three engines must return well-formed results)
 
 The seams (`seams.resolve` / `seams.request` / `rawFetch`) replace the network stack
 with fakes, mirroring how the Studio suite monkeypatches `_validate_and_resolve_host`

package/engines.ts

@@ -1,5 +1,5 @@
 import { randomBytes } from "node:crypto";
-import { decodeHtmlEntities, feedHtml } from "./html-to-md.ts";
+import { collapseWhitespace, decodeHtmlEntities, feedHtml } from "./html-to-md.ts";
 import type { AttrDict } from "./html-to-md.ts";
 import { randomUserAgent } from "./user-agents.ts";
 export class EmptySweepError extends Error {
@@ -35,7 +35,7 @@ export function normalizeText(raw: string): string {
   text = decodeHtmlEntities(text);
   text = text.normalize("NFC");
   text = text.replace(/[\p{Cc}\p{Cf}\p{Co}\p{Cs}\p{Cn}]/gu, "");
-  return text.trim().split(/\s+/).join(" ");
+  return collapseWhitespace(text);
 }
 
 export function normalizeUrl(url: string): string {
@@ -47,6 +47,41 @@ export function normalizeUrl(url: string): string {
   }
 }
 
+const TRACKING_PARAM_NAMES = new Set([
+  "_hsenc",
+  "_hsmi",
+  "dclid",
+  "fbclid",
+  "gbraid",
+  "gclid",
+  "gclsrc",
+  "igshid",
+  "mc_cid",
+  "mc_eid",
+  "msclkid",
+  "srsltid",
+  "twclid",
+  "wbraid",
+  "yclid",
+]);
+
+export function canonicalizeHref(href: string): string {
+  if (!href) return "";
+  try {
+    const url = new URL(href);
+    for (const key of [...url.searchParams.keys()]) {
+      const lower = key.toLowerCase();
+      if (lower.startsWith("utm_") || TRACKING_PARAM_NAMES.has(lower)) {
+        url.searchParams.delete(key);
+      }
+    }
+    url.hash = "";
+    return url.toString();
+  } catch {
+    return href;
+  }
+}
+
 export interface DomNode {
   tag: string;
   attrs: Record<string, string>;
@@ -382,7 +417,7 @@ export function extractResults(
       ["body", elementsXpath.body],
     ] as const;
     for (const [key, value] of entries) {
-      const data = xpathText(value, item).join("").trim().split(/\s+/).join(" ");
+      const data = collapseWhitespace(xpathText(value, item).join(""));
       if (!data) continue;
       result[key] = key === "href" ? normalizeUrl(data) : normalizeText(data);
     }
@@ -444,15 +479,22 @@ export interface Engine {
   ): Promise<SearchResult[] | null>;
 }
 
+interface HttpRequestOptions {
+  headers?: Record<string, string>;
+  cookies?: Record<string, string>;
+  timeoutMs: number;
+  signal?: AbortSignal;
+}
+
+interface HttpOptions extends HttpRequestOptions {
+  method?: string;
+  body?: string;
+}
+
 async function httpGet(
   url: string,
   params: Record<string, string>,
-  options: {
-    headers?: Record<string, string>;
-    cookies?: Record<string, string>;
-    timeoutMs: number;
-    signal?: AbortSignal;
-  },
+  options: HttpRequestOptions,
 ): Promise<string | null> {
   const target = new URL(url);
   for (const [key, value] of Object.entries(params)) target.searchParams.set(key, value);
@@ -462,17 +504,15 @@ async function httpGet(
 async function httpPost(
   url: string,
   data: Record<string, string>,
-  options: {
-    headers?: Record<string, string>;
-    cookies?: Record<string, string>;
-    timeoutMs: number;
-    signal?: AbortSignal;
-  },
+  options: HttpRequestOptions,
 ): Promise<string | null> {
   return httpFetch(url, { ...options, method: "POST", body: new URLSearchParams(data).toString() });
 }
 
 const MAX_ENGINE_RESPONSE_BYTES = 5 * 1024 * 1024;
+const ENGINE_RETRY_BACKOFF_MS = 250;
+
+const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
 
 async function readBodyCapped(response: Response): Promise<string | null> {
   const declared = Number(response.headers.get("content-length") ?? "0");
@@ -494,22 +534,22 @@ async function readBodyCapped(response: Response): Promise<string | null> {
   return new TextDecoder("utf-8").decode(Buffer.concat(chunks));
 }
 
+function mapFetchError(err: unknown): never {
+  if (err instanceof DOMException && err.name === "TimeoutError") throw new SearchTimeoutError();
+  if (err instanceof DOMException && err.name === "AbortError") throw new SearchCancelled();
+  throw err;
+}
+
 async function httpFetch(
   url: string,
-  options: {
-    method?: string;
-    body?: string;
-    headers?: Record<string, string>;
-    cookies?: Record<string, string>;
-    timeoutMs: number;
-    signal?: AbortSignal;
-  },
+  options: HttpOptions,
 ): Promise<string | null> {
   const headers: Record<string, string> = {
     "User-Agent": options.headers?.["User-Agent"] ?? randomUserAgent(),
     Accept: "*/*",
     ...options.headers,
   };
+  if (options.method === "POST") headers["Content-Type"] = "application/x-www-form-urlencoded";
   const cookie = options.cookies
     ? Object.entries(options.cookies)
         .map(([key, value]) => `${key}=${value}`)
@@ -527,17 +567,20 @@ async function httpFetch(
       signal: AbortSignal.any(signals),
     });
   } catch (err) {
-    if (err instanceof DOMException && err.name === "TimeoutError") throw new SearchTimeoutError();
-    if (err instanceof DOMException && err.name === "AbortError") throw new SearchCancelled();
-    throw err;
+    throw mapFetchError(err);
+  }
+  if (response.status !== 200) {
+    try {
+      await response.body?.cancel();
+    } catch {
+      return null;
+    }
+    return null;
   }
-  if (response.status !== 200) return null;
   try {
     return await readBodyCapped(response);
   } catch (err) {
-    if (err instanceof DOMException && err.name === "TimeoutError") throw new SearchTimeoutError();
-    if (err instanceof DOMException && err.name === "AbortError") throw new SearchCancelled();
-    throw err;
+    throw mapFetchError(err);
   }
 }
 
@@ -648,7 +691,7 @@ const MOJEEK: Engine = {
 const YAHOO: Engine = {
   name: "yahoo",
   provider: "bing",
-  async search(query, ctx, timeoutMs, signal) {
+  async search(query, _ctx, timeoutMs, signal) {
     const ylt = tokenUrlSafe(18);
     const ylu = tokenUrlSafe(35);
     const html = await httpGet(
@@ -674,7 +717,7 @@ const YAHOO: Engine = {
 const YANDEX: Engine = {
   name: "yandex",
   provider: "yandex",
-  async search(query, ctx, timeoutMs, signal) {
+  async search(query, _ctx, timeoutMs, signal) {
     const searchid = 1000000 + Math.floor(Math.random() * 9000000);
     const html = await httpGet(
       "https://yandex.com/search/site/",
@@ -695,6 +738,7 @@ const WIKIPEDIA: Engine = {
   provider: "wikipedia",
   priority: 2,
   async search(query, ctx, timeoutMs, signal) {
+    const started = Date.now();
     const lang = ctx.region.toLowerCase().split("-")[1] ?? "en";
     const encoded = encodeURIComponent(query);
     const opensearchUrl =
@@ -715,7 +759,7 @@ const WIKIPEDIA: Engine = {
     const extractUrl =
       `https://${lang}.wikipedia.org/w/api.php?action=query&format=json&prop=extracts` +
       `&titles=${encodeURIComponent(title)}&explaintext=0&exintro=0&redirects=1`;
-    const extract = await httpGet(extractUrl, {}, { timeoutMs, signal });
+    const extract = await httpGet(extractUrl, {}, { timeoutMs: Math.max(1, timeoutMs - (Date.now() - started)), signal });
     if (extract) {
       try {
         const pageData = JSON.parse(extract) as {
@@ -732,7 +776,7 @@ const WIKIPEDIA: Engine = {
   },
 };
 
-const TEXT_ENGINES: Engine[] = [DUCKDUCKGO, BRAVE, GOOGLE, MOJEEK, YAHOO, YANDEX, WIKIPEDIA];
+export const TEXT_ENGINES: Engine[] = [DUCKDUCKGO, BRAVE, GOOGLE, MOJEEK, YAHOO, YANDEX, WIKIPEDIA];
 
 export class ResultsAggregator {
   private cache = new Map<string, SearchResult>();
@@ -743,10 +787,10 @@ export class ResultsAggregator {
   }
 
   append(item: SearchResult): void {
-    const key = item.href;
+    const key = canonicalizeHref(item.href);
     const existing = this.cache.get(key);
     if (!existing || item.body.length > existing.body.length) {
-      this.cache.set(key, item);
+      this.cache.set(key, { ...item, href: key });
     }
     this.counter.set(key, (this.counter.get(key) ?? 0) + 1);
   }
@@ -815,6 +859,7 @@ export async function autoTextSearch(
   signal?: AbortSignal,
 ): Promise<SearchResult[]> {
   const engines = shuffledEngines();
+  const deadline = Date.now() + timeoutMs;
   const seenProviders = new Set<string>();
   const aggregator = new ResultsAggregator();
   const ctx: EngineContext = { region: "us-en", safesearch: "moderate" };
@@ -825,19 +870,40 @@ export async function autoTextSearch(
   let i = 0;
   let pending: Promise<void>[] = [];
   const run = async (engine: Engine) => {
-    try {
-      const results = await engine.search(query, ctx, timeoutMs, signal);
-      if (results && results.length) {
-        aggregator.extend(results);
-        seenProviders.add(engine.provider);
+    let results: SearchResult[] | null = null;
+    for (let attempt = 0; attempt < 2 && results === null; attempt++) {
+      const budgetLeft = deadline - Date.now();
+      if (budgetLeft <= 0) return;
+      if (attempt > 0 && budgetLeft < ENGINE_RETRY_BACKOFF_MS) return;
+      const remaining = Math.max(1, budgetLeft);
+      try {
+        results = await engine.search(query, ctx, remaining, signal);
+      } catch (e) {
+        if (e instanceof SearchCancelled) {
+          cancelled = true;
+          return;
+        }
+        if (e instanceof SearchTimeoutError) {
+          timedOut = true;
+          return;
+        }
       }
-    } catch (e) {
-      if (e instanceof SearchCancelled) cancelled = true;
-      if (e instanceof SearchTimeoutError) timedOut = true;
+      if (results === null && attempt === 0) {
+        const backoff = Math.min(ENGINE_RETRY_BACKOFF_MS, Math.max(0, deadline - Date.now()));
+        if (backoff > 0) await sleep(backoff);
+        if (signal?.aborted) {
+          cancelled = true;
+          return;
+        }
+      }
+    }
+    if (results && results.length) {
+      aggregator.extend(results);
+      seenProviders.add(engine.provider);
     }
   };
   while (i < engines.length) {
-    if (aggregator.size >= maxResults) break;
+    if (aggregator.size >= maxResults || cancelled) break;
     const engine = engines[i++];
     if (seenProviders.has(engine.provider)) continue;
     pending.push(run(engine));

package/html-to-md.ts

@@ -194,7 +194,7 @@ class HeaderFrame {
 const CHARREF_RE = /&(#[0-9]+;?|#[xX][0-9a-fA-F]+;?|[^\t\n\f <&#;]{1,32};?)/g;
 
 export function decodeHtmlEntities(text: string): string {
-  return text.replace(CHARREF_RE, (whole, s: string) => {
+  return text.replace(CHARREF_RE, (_whole, s: string) => {
     if (s[0] === "#") {
       const hex = s[1] === "x" || s[1] === "X";
       const num = parseInt(s.slice(hex ? 2 : 1).replace(/;+$/, ""), hex ? 16 : 10);
@@ -214,6 +214,10 @@ export function decodeHtmlEntities(text: string): string {
   });
 }
 
+export function collapseWhitespace(text: string): string {
+  return text.replace(/\s+/g, " ").trim();
+}
+
 
 interface HtmlHandlers {
   handleStartTag(name: string, attrs: AttrDict): void;
@@ -331,9 +335,9 @@ export function feedHtml(input: string, handlers: HtmlHandlers): void {
         pos = amp + named[0].length;
         continue;
       }
-      const numeric = /^&#(?:[xX]([0-9a-fA-F]+)|([0-9]+));/.exec(text.slice(amp));
+      const numeric = /^&#([xX][0-9a-fA-F]+|[0-9]+);?/.exec(text.slice(amp));
       if (numeric) {
-        handlers.handleCharRef(numeric[1] ?? numeric[2]);
+        handlers.handleCharRef(numeric[1]);
         pos = amp + numeric[0].length;
         continue;
       }
@@ -384,6 +388,10 @@ export function feedHtml(input: string, handlers: HtmlHandlers): void {
   emitText(input.slice(textStart));
 }
 
+function popMarksAbove(marks: number[], index: number): void {
+  while (marks.length && marks[marks.length - 1] >= index) marks.pop();
+}
+
 class MarkdownRenderer {
   out: string[] = [];
   private skipDepth = 0;
@@ -524,8 +532,8 @@ class MarkdownRenderer {
   }
 
   private finishLink(): void {
-    const text = this.linkTextParts.join("").replace(/\s+/g, " ").trim();
-    const headingText = this.linkHeadingParts.join("").replace(/\s+/g, " ").trim();
+    const text = collapseWhitespace(this.linkTextParts.join(""));
+    const headingText = collapseWhitespace(this.linkHeadingParts.join(""));
     const href = this.linkHref ?? "";
     this.inLink = false;
     this.linkTextParts = [];
@@ -571,12 +579,8 @@ class MarkdownRenderer {
       }
       if (closeAt === null) break;
       this.truncateOpenTags(closeAt);
-      while (this.hiddenMarks.length && this.hiddenMarks[this.hiddenMarks.length - 1] >= closeAt) {
-        this.hiddenMarks.pop();
-      }
-      while (this.headingMarks.length && this.headingMarks[this.headingMarks.length - 1] >= closeAt) {
-        this.headingMarks.pop();
-      }
+      popMarksAbove(this.hiddenMarks, closeAt);
+      popMarksAbove(this.headingMarks, closeAt);
       this.closeHeaderFrames(closeAt);
     }
   }
@@ -692,12 +696,8 @@ class MarkdownRenderer {
       for (let i = this.openTags.length - 1; i >= 0; i--) {
         if (this.openTags[i] === tag) {
           this.truncateOpenTags(i);
-          while (this.hiddenMarks.length && this.hiddenMarks[this.hiddenMarks.length - 1] >= i) {
-            this.hiddenMarks.pop();
-          }
-          while (this.headingMarks.length && this.headingMarks[this.headingMarks.length - 1] >= i) {
-            this.headingMarks.pop();
-          }
+          popMarksAbove(this.hiddenMarks, i);
+          popMarksAbove(this.headingMarks, i);
           this.closeHeaderFrames(i, tag === "header");
           break;
         }
@@ -715,7 +715,11 @@ class MarkdownRenderer {
     return !suppressed;
   }
 
-  handleStartEndTag(_name: string, _attrs: AttrDict): void {}
+  handleStartEndTag(name: string, attrs: AttrDict): void {
+    if (!VOID_TAGS.has(name)) return;
+    this.handleStartTag(name, attrs);
+    this.handleEndTag(name);
+  }
   handleStartTag(tag: string, attrs: AttrDict): void {
     if (this.skipDepth) {
       if (SKIP_TAGS.has(tag)) this.skipDepth++;

package/index.ts

@@ -1,9 +1,11 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
 import { webSearch } from "./web-search.ts";
-import { fetchPageText } from "./web-fetch.ts";
+import { DEFAULT_FETCH_TIMEOUT_MS, fetchPageText } from "./web-fetch.ts";
 
-const FETCH_TIMEOUT_MS = 60_000;
+function positiveMaxChars(value: unknown): number | undefined {
+  return typeof value === "number" && value > 0 ? value : undefined;
+}
 
 const WebSearchParams = Type.Object({
   query: Type.Optional(
@@ -15,6 +17,12 @@ const WebSearchParams = Type.Object({
         "A URL to fetch full page content from (instead of searching). Use this to read a page found in search results.",
     }),
   ),
+  maxChars: Type.Optional(
+    Type.Number({
+      description:
+        "Truncate the fetched page to this many characters (only used with the url parameter)",
+    }),
+  ),
 });
 
 const WebFetchParams = Type.Object({
@@ -38,21 +46,25 @@ export default function (pi: ExtensionAPI) {
       "Use web_search with the url parameter (e.g. {\"url\": \"<URL>\"}) to read the full text of a page found in search results.",
     ],
     parameters: WebSearchParams,
-    async execute(_toolCallId, params, signal, _onUpdate, _ctx) {
+    async execute(_toolCallId, params, signal, onUpdate, _ctx) {
       if (params.url?.trim()) {
+        const url = params.url.trim();
+        onUpdate?.({ content: [{ type: "text", text: `Fetching ${url}...` }], details: {} });
         return {
           content: [
             {
               type: "text",
-              text: await fetchPageText(params.url.trim(), {
-                timeoutMs: FETCH_TIMEOUT_MS,
+              text: await fetchPageText(url, {
+                timeoutMs: DEFAULT_FETCH_TIMEOUT_MS,
                 signal: signal ?? undefined,
+                maxChars: positiveMaxChars(params.maxChars),
               }),
             },
           ],
           details: {},
         };
       }
+      onUpdate?.({ content: [{ type: "text", text: "Searching the web..." }], details: {} });
       const text = await webSearch(params.query, { signal: signal ?? undefined });
       return { content: [{ type: "text", text }], details: {} };
     },
@@ -70,13 +82,11 @@ export default function (pi: ExtensionAPI) {
       "is capped.",
     promptSnippet: "Fetch a web page and return readable text content",
     parameters: WebFetchParams,
-    async execute(_toolCallId, params, signal, _onUpdate, _ctx) {
-      const maxChars =
-        typeof params.maxChars === "number" && params.maxChars > 0
-          ? params.maxChars
-          : undefined;
+    async execute(_toolCallId, params, signal, onUpdate, _ctx) {
+      const maxChars = positiveMaxChars(params.maxChars);
+      onUpdate?.({ content: [{ type: "text", text: `Fetching ${params.url}...` }], details: {} });
       const text = await fetchPageText(params.url, {
-        timeoutMs: FETCH_TIMEOUT_MS,
+        timeoutMs: DEFAULT_FETCH_TIMEOUT_MS,
         signal: signal ?? undefined,
         maxChars,
       });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.2.4",
+  "version": "0.3.0",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools ported from the Unsloth Studio codebase (DuckDuckGo search, SSRF-safe direct fetching, HTML-to-Markdown extraction)",
   "main": "index.ts",
@@ -51,7 +51,8 @@
     "test:unit": "vitest run --exclude **/smoke.test.ts",
     "test:smoke": "vitest run test/smoke.test.ts",
     "typecheck": "tsc --noEmit",
-    "prepublishOnly": "npm run typecheck && npm run test:unit"
+    "check:package": "node -e \"const fs=require('fs');const pkg=require('./package.json');const listed=new Set(pkg.files);const bad=[];for(const f of pkg.files){if(!fs.existsSync(f))bad.push('missing file: '+f)}for(const f of fs.readdirSync('.').filter(f=>f.endsWith('.ts'))){if(!listed.has(f))bad.push('unlisted source: '+f)}if(bad.length){console.error(bad.join('\\n'));process.exit(1)}\"",
+    "prepublishOnly": "npm run typecheck && npm run test:unit && npm run check:package"
   },
   "devDependencies": {
     "@earendil-works/pi-coding-agent": "^0.84.0",