pi-web-toolkit 0.2.1 → 0.3.0

package/extensions/web_fetch.ts

@@ -25,6 +25,7 @@ import * as path from "node:path";
 import { runScraplingWithFallback } from "./utils/scrapling";
 import { extractPreview } from "./utils/content-preview";
 import { writeWithFallback } from "./utils/output-sink";
+import { scrapeKeyless } from "./utils/firecrawl";
 import { abbreviateUrl, getDomain, getErrorText, normalizeWhitespace, formatExtraction } from "./utils/render-helpers";
 
 export const WebFetchParamsSchema = Type.Object({
@@ -41,13 +42,13 @@ const webFetchTool = defineTool({
   description: [
     "Fetch and extract readable content from a web page URL.",
     "Uses scrapling to download the page and convert it to clean markdown.",
-    "Use web_fetch AFTER web_search to read the full content of a result page.",
-    "Respects robots.txt and site ToS.",
+    "Use web_fetch to read the full content of a specific result or user-provided URL.",
+    "Callers remain responsible for robots.txt and site terms; Scrapling extract commands do not enforce them automatically.",
     `Output is truncated to ${DEFAULT_MAX_LINES} lines or ${formatSize(DEFAULT_MAX_BYTES)}; if truncated, full output is saved to a temp file.`,
   ].join(" "),
   promptSnippet: "Fetch full page content from a URL as markdown",
   promptGuidelines: [
-    "Use web_fetch to read a single static page (article, doc, or blog) when given a specific URL.",
+    "Use web_fetch to read a single page (article, doc, or blog) that needs no interaction.",
     "For a single URL, always use web_fetch instead of web_batch_fetch.",
     "If the page is dynamic/JavaScript-heavy, the tool automatically uses browser automation.",
     "When reading multiple (2–5) pages at once (e.g., after web_search), prefer web_batch_fetch over repeated web_fetch calls.",
@@ -68,15 +69,31 @@ const webFetchTool = defineTool({
         signal,
       );
 
-      if (!ok) {
-        throw new Error(`Failed to fetch ${params.url}\n\nscrapling error:\n${stderr}`);
+      let content: string;
+      let bytes: number;
+      let viaFirecrawl = false;
+
+      if (ok) {
+        content = await fs.promises.readFile(tmpFile, "utf-8");
+        bytes = (await fs.promises.stat(tmpFile)).size;
+      } else {
+        // Local scrapling failed — try the Firecrawl keyless fallback.
+        const localError = `Failed to fetch ${params.url}\n\nscrapling error:\n${stderr}`;
+        const fb = await scrapeKeyless(params.url, {}, signal);
+        if (fb.ok) {
+          content = fb.content;
+          bytes = fb.bytes;
+          viaFirecrawl = true;
+        } else {
+          // Graceful skip (CLI absent / IP flagged / rate-limited / disabled):
+          // never leave the user worse off — surface the original local error.
+          throw new Error(localError);
+        }
       }
 
-      const content = await fs.promises.readFile(tmpFile, "utf-8");
-      const stats = await fs.promises.stat(tmpFile);
-
       const preview = extractPreview(content, 500);
-      const rawText = `Fetched: ${params.url}\nSize: ${stats.size} bytes\n\n---\n\n${content}`;
+      const viaTag = viaFirecrawl ? "\n(via Firecrawl keyless fallback)" : "";
+      const rawText = `Fetched: ${params.url}${viaTag}\nSize: ${bytes} bytes\n\n---\n\n${content}`;
       const sink = await writeWithFallback(rawText, {
         tmpPrefix: "pi-web-fetch-full-",
       });
@@ -86,11 +103,12 @@ const webFetchTool = defineTool({
         content: [{ type: "text", text: sink.text }],
         details: {
           url: params.url,
-          bytes: stats.size,
+          bytes,
           fullOutputPath: tmpFull,
           preview,
           selector: params.selector,
           stealthy: params.stealthy,
+          viaFirecrawl,
         },
       };
     } catch (err: any) {
@@ -128,6 +146,7 @@ const webFetchTool = defineTool({
       preview?: string;
       selector?: string;
       stealthy?: boolean;
+      viaFirecrawl?: boolean;
     } | undefined;
 
     if (isError) {
@@ -139,6 +158,9 @@ const webFetchTool = defineTool({
     }
 
     let text = theme.fg("success", "✓ Fetched");
+    if (details?.viaFirecrawl) {
+      text += theme.fg("accent", " [Firecrawl keyless]");
+    }
     if (details?.url) {
       text += `  ${theme.fg("dim", abbreviateUrl(details.url))}`;
     }

package/extensions/web_search.ts

@@ -20,6 +20,7 @@ import {
 import { Text } from "@earendil-works/pi-tui";
 import { Type, type Static } from "typebox";
 import { writeWithFallback } from "./utils/output-sink";
+import { searchKeyless, shouldFallbackSearch } from "./utils/firecrawl";
 import { abbreviateUrl, getDomain, getErrorText, normalizeWhitespace } from "./utils/render-helpers";
 
 
@@ -78,6 +79,9 @@ const webSearchTool = defineTool({
     let fullOutputPath: string | undefined;
     const MAX_PAGES = 3;
 
+    let localOk = true;
+    let localError: string | undefined;
+
     try {
       for (let page = 1; page <= MAX_PAGES; page++) {
         const searchParams = new URLSearchParams({
@@ -120,51 +124,89 @@ const webSearchTool = defineTool({
           break;
         }
       }
+    } catch (err: any) {
+      localOk = false;
+      localError = err.message ?? String(err);
+    }
 
-      if (allResults.length === 0) {
-        let text = `No results found for "${finalQuery}".`;
-        if (suggestions && suggestions.length > 0) {
-          text += `\n\nSuggestions:\n${suggestions.map((s) => `- ${s}`).join("\n")}`;
+    // Firecrawl keyless fallback: when SearXNG errored OR returned nothing.
+    if (shouldFallbackSearch(localOk, allResults.length)) {
+      const fb = await searchKeyless(params.query, { limit: Math.min(maxResults, 10) }, signal);
+      if (fb.ok && fb.results.length > 0) {
+        const fbResults: SearxResult[] = fb.results.slice(0, maxResults).map((r) => ({
+          title: r.title ?? "(untitled)",
+          url: r.url,
+          content: r.description,
+          engine: "firecrawl",
+        }));
+        const creditTag = fb.creditsUsed !== undefined ? `, ${fb.creditsUsed} credits` : "";
+        const lines: string[] = [`Results for "${params.query}" (via Firecrawl keyless${creditTag}):`, ""];
+        for (let i = 0; i < fbResults.length; i++) {
+          const r = fbResults[i];
+          lines.push(`${i + 1}. ${r.title}`);
+          lines.push(`   URL: ${r.url}`);
+          if (r.content) lines.push(`   ${r.content.replace(/\s+/g, " ").trim()}`);
+          if (r.engine) lines.push(`   [engine: ${r.engine}]`);
+          lines.push("");
         }
+        const rawText = lines.join("\n");
+        const sink = await writeWithFallback(rawText, {
+          tmpPrefix: "pi-web-search-firecrawl-",
+          alwaysWriteFile: true,
+        });
         return {
-          content: [{ type: "text", text }],
-          details: { query: finalQuery, totalResults: 0, results: [] as SearxResult[], fullOutputPath: undefined as string | undefined },
+          content: [{ type: "text", text: sink.text }],
+          details: { query: params.query, totalResults: fbResults.length, results: fbResults, fullOutputPath: sink.fullOutputPath, viaFirecrawl: true, creditsUsed: fb.creditsUsed },
         };
       }
+      // Graceful skip or empty Firecrawl: fall through to local handling.
+    }
 
-      const lines: string[] = [
-        `Results for "${finalQuery}":`,
-        "",
-      ];
+    if (!localOk) {
+      throw new Error(`Failed to query SearXNG at ${searxngUrl}: ${localError}`);
+    }
 
-      for (let i = 0; i < Math.min(maxResults, allResults.length); i++) {
-        const r = allResults[i];
-        lines.push(`${i + 1}. ${r.title}`);
-        lines.push(`   URL: ${r.url}`);
-        if (r.content) {
-          const snippet = r.content.replace(/\s+/g, " ").trim();
-          lines.push(`   ${snippet}`);
-        }
-        if (r.engine) {
-          lines.push(`   [engine: ${r.engine}]`);
-        }
-        lines.push("");
+    if (allResults.length === 0) {
+      let text = `No results found for "${finalQuery}".`;
+      if (suggestions && suggestions.length > 0) {
+        text += `\n\nSuggestions:\n${suggestions.map((s) => `- ${s}`).join("\n")}`;
       }
-
-      const rawText = lines.join("\n");
-      const sink = await writeWithFallback(rawText, {
-        tmpPrefix: "pi-web-search-",
-        alwaysWriteFile: true,
-      });
-      fullOutputPath = sink.fullOutputPath;
-
       return {
-        content: [{ type: "text", text: sink.text }],
-        details: { query: finalQuery, totalResults: allResults.length, results: allResults.slice(0, maxResults), fullOutputPath },
+        content: [{ type: "text", text }],
+        details: { query: finalQuery, totalResults: 0, results: [] as SearxResult[], fullOutputPath: undefined as string | undefined, viaFirecrawl: false, creditsUsed: undefined },
       };
-    } catch (err: any) {
-      throw new Error(`Failed to query SearXNG at ${searxngUrl}: ${err.message ?? err}`);
     }
+
+    const lines: string[] = [
+      `Results for "${finalQuery}":`,
+      "",
+    ];
+
+    for (let i = 0; i < Math.min(maxResults, allResults.length); i++) {
+      const r = allResults[i];
+      lines.push(`${i + 1}. ${r.title}`);
+      lines.push(`   URL: ${r.url}`);
+      if (r.content) {
+        const snippet = r.content.replace(/\s+/g, " ").trim();
+        lines.push(`   ${snippet}`);
+      }
+      if (r.engine) {
+        lines.push(`   [engine: ${r.engine}]`);
+      }
+      lines.push("");
+    }
+
+    const rawText = lines.join("\n");
+    const sink = await writeWithFallback(rawText, {
+      tmpPrefix: "pi-web-search-",
+      alwaysWriteFile: true,
+    });
+    fullOutputPath = sink.fullOutputPath;
+
+    return {
+      content: [{ type: "text", text: sink.text }],
+      details: { query: finalQuery, totalResults: allResults.length, results: allResults.slice(0, maxResults), fullOutputPath, viaFirecrawl: false, creditsUsed: undefined },
+    };
   },
 
   renderCall(args, theme) {
@@ -190,6 +232,8 @@ const webSearchTool = defineTool({
       totalResults?: number;
       results?: Array<{ title?: string; url?: string; score?: number; engine?: string; content?: string }>;
       fullOutputPath?: string;
+      viaFirecrawl?: boolean;
+      creditsUsed?: number;
     } | undefined;
 
     if (isError) {
@@ -207,7 +251,13 @@ const webSearchTool = defineTool({
     const showing = details.results?.length ?? 0;
     const total = details?.totalResults ?? 0;
     let text = theme.fg("success", `✓ ${showing} unique results`);
-    if (total > showing) {
+    if (details?.viaFirecrawl) {
+      text += theme.fg("accent", " [Firecrawl keyless]");
+    }
+    if (details?.creditsUsed !== undefined) {
+      text += theme.fg("muted", ` ${details.creditsUsed} credits`);
+    }
+    if (!details?.viaFirecrawl && total > showing) {
       text += theme.fg("dim", ` (${total} total)`);
     }
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pi-web-toolkit",
-  "version": "0.2.1",
-  "description": "Web research toolkit for the pi coding agent. Search via SearXNG, fetch static pages with scrapling, browse interactively via agent-browser, and batch-read sources in parallel.",
+  "version": "0.3.0",
+  "description": "Web research toolkit for the pi coding agent. Search via SearXNG, fetch pages with scrapling, browse interactively via agent-browser, batch-read sources in parallel, and optionally fall back to Firecrawl Keyless (no API key) when a local backend fails.",
   "author": "Wade Huang <fastwade11@gmail.com>",
   "license": "MIT",
   "repository": {
@@ -12,18 +12,20 @@
     "url": "https://github.com/Wade11s/pi-web-toolkit/issues"
   },
   "homepage": "https://github.com/Wade11s/pi-web-toolkit#readme",
-  "keywords": ["pi-package", "pi-extension", "web-search", "scrapling", "agent-browser"],
-  "files": ["extensions", "docs", "README.md", "package.json", "LICENSE"],
+  "keywords": ["pi-package", "pi-extension", "web-search", "scrapling", "agent-browser", "firecrawl"],
+  "files": ["extensions", "docs", "README.md", "CHANGELOG.md", "package.json", "LICENSE"],
   "engines": {
     "node": ">=22.0.0"
   },
   "scripts": {
     "typecheck": "tsc --noEmit",
-    "test": "npx tsx test/content-preview/test.ts && npx tsx test/agent-browser/test.ts",
-    "test:agent-browser": "npx tsx test/agent-browser/test.ts",
-    "test:approve": "npx tsx test/content-preview/test.ts --approve"
+    "test": "tsx test/content-preview/test.ts && tsx test/agent-browser/test.ts && tsx test/firecrawl/test.ts",
+    "test:agent-browser": "tsx test/agent-browser/test.ts",
+    "test:firecrawl": "tsx test/firecrawl/test.ts",
+    "test:approve": "tsx test/content-preview/test.ts --approve"
   },
   "devDependencies": {
+    "tsx": "^4.22.4",
     "typescript": "^5.7.0"
   },
   "peerDependencies": {