pi-web-toolkit 0.1.2 → 0.2.1

package/extensions/web_fetch.ts

@@ -13,7 +13,6 @@
 import {
   defineTool,
   type ExtensionAPI,
-  truncateHead,
   formatSize,
   DEFAULT_MAX_BYTES,
   DEFAULT_MAX_LINES,
@@ -24,6 +23,9 @@ import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import { runScraplingWithFallback } from "./utils/scrapling";
+import { extractPreview } from "./utils/content-preview";
+import { writeWithFallback } from "./utils/output-sink";
+import { abbreviateUrl, getDomain, getErrorText, normalizeWhitespace, formatExtraction } from "./utils/render-helpers";
 
 export const WebFetchParamsSchema = Type.Object({
   url: Type.String({ description: "Full URL to fetch (e.g. https://example.com/article)" }),
@@ -45,9 +47,11 @@ const webFetchTool = defineTool({
   ].join(" "),
   promptSnippet: "Fetch full page content from a URL as markdown",
   promptGuidelines: [
-    "Use web_fetch after web_search to read full articles, docs, or pages found in search results.",
-    "Always pass the full URL including https://.",
+    "Use web_fetch to read a single static page (article, doc, or blog) when given a specific URL.",
+    "For a single URL, always use web_fetch instead of web_batch_fetch.",
     "If the page is dynamic/JavaScript-heavy, the tool automatically uses browser automation.",
+    "When reading multiple (2–5) pages at once (e.g., after web_search), prefer web_batch_fetch over repeated web_fetch calls.",
+    "Always pass the full URL including https://.",
   ],
   parameters: WebFetchParamsSchema,
 
@@ -71,23 +75,23 @@ const webFetchTool = defineTool({
       const content = await fs.promises.readFile(tmpFile, "utf-8");
       const stats = await fs.promises.stat(tmpFile);
 
+      const preview = extractPreview(content, 500);
       const rawText = `Fetched: ${params.url}\nSize: ${stats.size} bytes\n\n---\n\n${content}`;
-      const truncation = truncateHead(rawText, {
-        maxLines: DEFAULT_MAX_LINES,
-        maxBytes: DEFAULT_MAX_BYTES,
+      const sink = await writeWithFallback(rawText, {
+        tmpPrefix: "pi-web-fetch-full-",
       });
-
-      let finalText = truncation.content;
-      if (truncation.truncated) {
-        const tmpFullDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "pi-web-fetch-full-"));
-        tmpFull = path.join(tmpFullDir, "output.txt");
-        await fs.promises.writeFile(tmpFull, rawText, "utf-8");
-        finalText += `\n\n[Output truncated: ${truncation.outputLines} of ${truncation.totalLines} lines (${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)}). Full output saved to: ${tmpFull}]`;
-      }
+      tmpFull = sink.fullOutputPath;
 
       return {
-        content: [{ type: "text", text: finalText }],
-        details: { url: params.url, bytes: stats.size, fullOutputPath: tmpFull },
+        content: [{ type: "text", text: sink.text }],
+        details: {
+          url: params.url,
+          bytes: stats.size,
+          fullOutputPath: tmpFull,
+          preview,
+          selector: params.selector,
+          stealthy: params.stealthy,
+        },
       };
     } catch (err: any) {
       throw new Error(`Error fetching ${params.url}: ${err.message ?? err}`);
@@ -99,27 +103,73 @@ const webFetchTool = defineTool({
   renderCall(args, theme) {
     let text = theme.fg("toolTitle", theme.bold("web_fetch "));
     text += theme.fg("muted", args.url);
+    if (args.stealthy) {
+      text += theme.fg("dim", " [stealthy]");
+    }
     if (args.selector) {
-      text += theme.fg("dim", ` selector=${args.selector}`);
+      text += theme.fg("dim", ` [selector=${args.selector}]`);
     }
     return new Text(text, 0, 0);
   },
 
-  renderResult(result, { expanded, isPartial }, theme) {
+  renderResult(result, { expanded, isPartial }, theme, context) {
+    const isError = context?.isError ?? false;
+
     if (isPartial) {
-      return new Text(theme.fg("warning", "Fetching..."), 0, 0);
+      const url = (result.details as any)?.url as string | undefined;
+      const domain = url ? getDomain(url) : "";
+      const label = domain ? `Fetching ${domain}...` : "Fetching...";
+      return new Text(theme.fg("warning", label), 0, 0);
+    }
+    const details = result.details as {
+      url?: string;
+      bytes?: number;
+      fullOutputPath?: string;
+      preview?: string;
+      selector?: string;
+      stealthy?: boolean;
+    } | undefined;
+
+    if (isError) {
+      const errText = getErrorText(result);
+      let text = theme.fg("error", "✗ Fetch failed");
+      if (details?.url) text += `  ${theme.fg("dim", abbreviateUrl(details.url))}`;
+      text += `\n\n  ${theme.fg("toolOutput", errText)}`;
+      return new Text(text, 0, 0);
     }
-    const details = result.details as { url?: string; bytes?: number; fullOutputPath?: string } | undefined;
+
     let text = theme.fg("success", "✓ Fetched");
-    if (details?.bytes) {
-      text += theme.fg("muted", ` (${formatSize(details.bytes)})`);
+    if (details?.url) {
+      text += `  ${theme.fg("dim", abbreviateUrl(details.url))}`;
     }
+    if (details?.bytes && details?.preview) {
+      text += `  ${theme.fg("muted", formatExtraction(details.bytes, details.preview.length))}`;
+    }
+
+    if (details?.selector) {
+      text += `\n  ${theme.fg("dim", `[selector=${details.selector}]`)}`;
+    }
+    if (details?.stealthy) {
+      text += `${details?.selector ? "" : "\n  "}${theme.fg("dim", "[stealthy]")}`;
+    }
+
+    if (!expanded && details?.preview) {
+      const snippet = normalizeWhitespace(details.preview);
+      const short = snippet.length > 160
+        ? snippet.slice(0, 160).replace(/\s+\S*$/, "") + "..."
+        : snippet;
+      text += `\n\n  ${theme.fg("muted", short)}`;
+    }
+
     if (expanded) {
-      text += `\n${theme.fg("dim", details?.url ?? "")}`;
+      if (details?.preview) {
+        text += `\n\n  ${theme.fg("muted", normalizeWhitespace(details.preview))}`;
+      }
       if (details?.fullOutputPath) {
-        text += `\n${theme.fg("dim", `Full output: ${details.fullOutputPath}`)}`;
+        text += `\n\n${theme.fg("accent", `Full output: ${details.fullOutputPath}`)}`;
       }
     }
+
     return new Text(text, 0, 0);
   },
 });

package/extensions/web_search.ts

@@ -13,16 +13,14 @@
 import {
   defineTool,
   type ExtensionAPI,
-  truncateHead,
   formatSize,
   DEFAULT_MAX_BYTES,
   DEFAULT_MAX_LINES,
 } from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
 import { Type, type Static } from "typebox";
-import { mkdtemp, writeFile } from "node:fs/promises";
-import * as os from "node:os";
-import * as path from "node:path";
+import { writeWithFallback } from "./utils/output-sink";
+import { abbreviateUrl, getDomain, getErrorText, normalizeWhitespace } from "./utils/render-helpers";
 
 
 
@@ -42,8 +40,8 @@ interface SearxResponse {
 
 export const WebSearchParamsSchema = Type.Object({
   query: Type.String({ description: "Search query" }),
-  language: Type.Optional(Type.String({ description: "Language code (e.g. en, en-US, de). Default: auto", default: "auto" })),
-  results: Type.Optional(Type.Integer({ description: "Max number of results to return (1-50). Default: 10", minimum: 1, maximum: 50, default: 10 })),
+  language: Type.Optional(Type.String({ description: "Language code (e.g. en, en-US, de). Omit to use SearXNG default.", default: "" })),
+  results: Type.Optional(Type.Integer({ description: "Max number of results to return (1-60). Default: 20 (one page). Automatically pages through SearXNG (up to 3 pages) if needed.", minimum: 1, maximum: 60, default: 20 })),
 });
 
 export type WebSearchInput = Static<typeof WebSearchParamsSchema>;
@@ -54,6 +52,7 @@ const webSearchTool = defineTool({
   description: [
     "Search the web using a SearXNG instance.",
     "Returns a list of results with title, URL, and snippet.",
+    "Automatically aggregates up to 3 pages of SearXNG results when more than ~20 are needed.",
     "Use web_search when the user asks about current events, facts, or anything",
     "that requires up-to-date information beyond the model's training data.",
     `Output is truncated to ${DEFAULT_MAX_LINES} lines or ${formatSize(DEFAULT_MAX_BYTES)}; if truncated, full output is saved to a temp file.`,
@@ -62,54 +61,84 @@ const webSearchTool = defineTool({
   promptGuidelines: [
     "Use web_search when the user asks about recent events, current data, or external facts.",
     "Use web_search to verify claims, find documentation, or discover resources online.",
+    "If web_search returns no results but includes suggestions, consider using a suggested query to refine your search.",
+    "If web_search returns multiple (2–5) relevant results that all need to be read, prefer web_batch_fetch to fetch them in parallel instead of calling web_fetch repeatedly.",
   ],
   parameters: WebSearchParamsSchema,
 
   async execute(_toolCallId, params, signal) {
     const searxngUrl = (process.env.SEARXNG_URL || "http://localhost:8080").replace(/\/$/, "");
-    const maxResults = Math.floor(Math.min(50, Math.max(1, params.results ?? 10)));
-    const searchParams = new URLSearchParams({
-      q: params.query,
-      format: "json",
-      language: params.language ?? "auto",
-    });
-
-    const url = `${searxngUrl}/search?${searchParams.toString()}`;
+    const maxResults = Math.floor(Math.min(60, Math.max(1, params.results ?? 20)));
+    const language = params.language ?? "";
 
+    const allResults: SearxResult[] = [];
+    const seenUrls = new Set<string>();
+    let suggestions: string[] | undefined;
+    let finalQuery = params.query;
     let fullOutputPath: string | undefined;
+    const MAX_PAGES = 3;
 
     try {
-      const response = await fetch(url, {
-        method: "GET",
-        headers: { Accept: "application/json" },
-        signal,
-      });
+      for (let page = 1; page <= MAX_PAGES; page++) {
+        const searchParams = new URLSearchParams({
+          q: params.query,
+          format: "json",
+          pageno: String(page),
+        });
+        if (language) searchParams.set("language", language);
 
-      if (!response.ok) {
-        const body = await response.text().catch(() => "");
-        throw new Error(`SearXNG error: ${response.status} ${response.statusText}\n${body}`);
-      }
+        const response = await fetch(`${searxngUrl}/search?${searchParams.toString()}`, {
+          method: "GET",
+          headers: { Accept: "application/json" },
+          signal,
+        });
+
+        if (!response.ok) {
+          const body = await response.text().catch(() => "");
+          throw new Error(`SearXNG error: ${response.status} ${response.statusText}\n${body}`);
+        }
+
+        const data = (await response.json()) as SearxResponse;
+        finalQuery = data.query;
+
+        if (data.suggestions && data.suggestions.length > 0 && !suggestions) {
+          suggestions = data.suggestions;
+        }
+
+        if (!data.results || data.results.length === 0) {
+          break;
+        }
+
+        for (const r of data.results) {
+          if (!seenUrls.has(r.url)) {
+            seenUrls.add(r.url);
+            allResults.push(r);
+          }
+        }
 
-      const data = (await response.json()) as SearxResponse;
+        if (allResults.length >= maxResults) {
+          break;
+        }
+      }
 
-      if (!data.results || data.results.length === 0) {
-        let text = `No results found for "${data.query}".`;
-        if (data.suggestions && data.suggestions.length > 0) {
-          text += `\n\nSuggestions:\n${data.suggestions.map((s) => `- ${s}`).join("\n")}`;
+      if (allResults.length === 0) {
+        let text = `No results found for "${finalQuery}".`;
+        if (suggestions && suggestions.length > 0) {
+          text += `\n\nSuggestions:\n${suggestions.map((s) => `- ${s}`).join("\n")}`;
         }
         return {
           content: [{ type: "text", text }],
-          details: { query: data.query, totalResults: 0, results: [], fullOutputPath: undefined },
+          details: { query: finalQuery, totalResults: 0, results: [] as SearxResult[], fullOutputPath: undefined as string | undefined },
         };
       }
 
       const lines: string[] = [
-        `Results for "${data.query}":`,
+        `Results for "${finalQuery}":`,
         "",
       ];
 
-      for (let i = 0; i < Math.min(maxResults, data.results.length); i++) {
-        const r = data.results[i];
+      for (let i = 0; i < Math.min(maxResults, allResults.length); i++) {
+        const r = allResults[i];
         lines.push(`${i + 1}. ${r.title}`);
         lines.push(`   URL: ${r.url}`);
         if (r.content) {
@@ -123,22 +152,15 @@ const webSearchTool = defineTool({
       }
 
       const rawText = lines.join("\n");
-      const truncation = truncateHead(rawText, {
-        maxLines: DEFAULT_MAX_LINES,
-        maxBytes: DEFAULT_MAX_BYTES,
+      const sink = await writeWithFallback(rawText, {
+        tmpPrefix: "pi-web-search-",
+        alwaysWriteFile: true,
       });
-
-      let finalText = truncation.content;
-      if (truncation.truncated) {
-        const tmpDir = await mkdtemp(path.join(os.tmpdir(), "pi-web-search-"));
-        fullOutputPath = path.join(tmpDir, "output.txt");
-        await writeFile(fullOutputPath, rawText, "utf-8");
-        finalText += `\n\n[Output truncated: ${truncation.outputLines} of ${truncation.totalLines} lines (${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)}). Full output saved to: ${fullOutputPath}]`;
-      }
+      fullOutputPath = sink.fullOutputPath;
 
       return {
-        content: [{ type: "text", text: finalText }],
-        details: { query: data.query, totalResults: data.results.length, results: data.results.slice(0, maxResults), fullOutputPath },
+        content: [{ type: "text", text: sink.text }],
+        details: { query: finalQuery, totalResults: allResults.length, results: allResults.slice(0, maxResults), fullOutputPath },
       };
     } catch (err: any) {
       throw new Error(`Failed to query SearXNG at ${searxngUrl}: ${err.message ?? err}`);
@@ -154,23 +176,82 @@ const webSearchTool = defineTool({
     return new Text(text, 0, 0);
   },
 
-  renderResult(result, { expanded, isPartial }, theme) {
+  renderResult(result, { expanded, isPartial }, theme, context) {
+    const isError = context?.isError ?? false;
+
     if (isPartial) {
-      return new Text(theme.fg("warning", "Searching..."), 0, 0);
+      const query = (result.details as any)?.query as string | undefined;
+      const label = query ? `Searching "${query}"...` : "Searching...";
+      return new Text(theme.fg("warning", label), 0, 0);
+    }
+
+    const details = result.details as {
+      query?: string;
+      totalResults?: number;
+      results?: Array<{ title?: string; url?: string; score?: number; engine?: string; content?: string }>;
+      fullOutputPath?: string;
+    } | undefined;
+
+    if (isError) {
+      const errText = getErrorText(result);
+      const query = details?.query;
+      let text = theme.fg("error", "✗ Search failed");
+      if (query) text += `  ${theme.fg("dim", query)}`;
+      text += `\n\n  ${theme.fg("toolOutput", errText)}`;
+      return new Text(text, 0, 0);
+    }
+
+    if (!details) {
+      return new Text(theme.fg("error", "No result details"), 0, 0);
+    }
+    const showing = details.results?.length ?? 0;
+    const total = details?.totalResults ?? 0;
+    let text = theme.fg("success", `✓ ${showing} unique results`);
+    if (total > showing) {
+      text += theme.fg("dim", ` (${total} total)`);
     }
-    const details = result.details as { query?: string; totalResults?: number; results?: Array<{ title?: string; url?: string }>; fullOutputPath?: string } | undefined;
-    let text = theme.fg("success", `✓ ${details?.totalResults ?? 0} results`);
-    if (details?.query) {
-      text += theme.fg("muted", ` for ${details.query}`);
+
+    if (!expanded && showing > 0) {
+      // Default: top 3 compact — [i] Title + domain + snippet
+      const top3 = (details.results ?? []).slice(0, 3);
+      for (let i = 0; i < top3.length; i++) {
+        const r = top3[i];
+        const domain = r.url ? theme.fg("dim", `  ${getDomain(r.url)}`) : "";
+        text += `\n  [${i + 1}] ${theme.fg("toolTitle", r.title ?? "(untitled)")}${domain}`;
+        if (r.content) {
+          const snippet = normalizeWhitespace(r.content);
+          const short = snippet.length > 90 ? snippet.slice(0, 90).replace(/\s+\S*$/, "") + "..." : snippet;
+          text += `\n    ${theme.fg("muted", short)}`;
+        }
+      }
+      if (showing > 3) {
+        text += `\n  ${theme.fg("muted", `... and ${showing - 3} more (Ctrl+O for full list)`)}`;
+      }
     }
+
     if (expanded && details?.results?.length) {
-      for (const r of details.results.slice(0, 10)) {
-        text += `\n  ${theme.fg("dim", `${r.title ?? "(untitled)"} — ${r.url ?? ""}`)}`;
+      // Expanded (Ctrl+O): top 10 cards — [i] Title|engine|score, URL, snippet
+      const top10 = (details.results ?? []).slice(0, 10);
+      for (let i = 0; i < top10.length; i++) {
+        const r = top10[i];
+        const scoreStr = r.score !== undefined ? r.score.toFixed(2) : "—";
+        const metaStr = r.engine ? ` | ${r.engine} | ${scoreStr}` : ` | ${scoreStr}`;
+        text += `\n  [${i + 1}] ${theme.fg("toolTitle", r.title ?? "(untitled)")}${theme.fg("dim", metaStr)}`;
+        text += `\n    ${theme.fg("dim", abbreviateUrl(r.url ?? ""))}`;
+        if (r.content) {
+          text += `\n    ${theme.fg("muted", normalizeWhitespace(r.content))}`;
+        }
+        text += "\n";
+      }
+      if (details.results.length > 10) {
+        text += `\n  ${theme.fg("muted", `... and ${details.results.length - 10} more results (see full output file)`)}`;
       }
     }
+
     if (expanded && details?.fullOutputPath) {
-      text += `\n${theme.fg("dim", `Full output: ${details.fullOutputPath}`)}`;
+      text += `\n${theme.fg("accent", `Full output: ${details.fullOutputPath}`)}`;
     }
+
     return new Text(text, 0, 0);
   },
 });
@@ -178,3 +259,5 @@ const webSearchTool = defineTool({
 export default function (pi: ExtensionAPI) {
   pi.registerTool(webSearchTool);
 }
+
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-web-toolkit",
-  "version": "0.1.2",
+  "version": "0.2.1",
   "description": "Web research toolkit for the pi coding agent. Search via SearXNG, fetch static pages with scrapling, browse interactively via agent-browser, and batch-read sources in parallel.",
   "author": "Wade Huang <fastwade11@gmail.com>",
   "license": "MIT",
@@ -17,6 +17,15 @@
   "engines": {
     "node": ">=22.0.0"
   },
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "test": "npx tsx test/content-preview/test.ts && npx tsx test/agent-browser/test.ts",
+    "test:agent-browser": "npx tsx test/agent-browser/test.ts",
+    "test:approve": "npx tsx test/content-preview/test.ts --approve"
+  },
+  "devDependencies": {
+    "typescript": "^5.7.0"
+  },
   "peerDependencies": {
     "@earendil-works/pi-ai": "*",
     "@earendil-works/pi-coding-agent": "*",