pi-web-toolkit 0.2.2 → 0.3.1

package/extensions/web_browse.ts

@@ -28,6 +28,7 @@ import {
   closeAgentBrowserSession,
 } from "./utils/agent-browser";
 import { writeWithFallback } from "./utils/output-sink";
+import { interactKeyless, shouldFallbackBrowse, isFirecrawlEnabled } from "./utils/firecrawl";
 import { abbreviateUrl, getErrorText, normalizeWhitespace } from "./utils/render-helpers";
 
 export const WebBrowseActionSchema = Type.Object({
@@ -82,6 +83,25 @@ function formatBrowseStep(action: BrowseAction): string {
   }
 }
 
+function synthesizeBrowsePrompt(params: { url: string; actions: BrowseAction[]; selector?: string }): string {
+  const parts: string[] = [];
+  for (const a of params.actions) {
+    switch (a.type) {
+      case "click": parts.push(`click the element "${a.selector ?? ""}"`); break;
+      case "fill": case "type": parts.push(`type "${a.value ?? ""}" into "${a.selector ?? ""}"`); break;
+      case "press": parts.push(`press ${a.key ?? ""}`); break;
+      case "scroll": parts.push(`scroll ${a.direction ?? "down"}`); break;
+      case "wait": parts.push("wait briefly"); break;
+      case "wait_selector": parts.push(`wait for "${a.selector ?? ""}" to appear`); break;
+    }
+  }
+  const actionText = parts.length ? `Perform these actions in order: ${parts.join("; ")}. ` : "";
+  const extract = params.selector
+    ? `Then return the text content of the element matching "${params.selector}".`
+    : "Then return the main textual content of the page.";
+  return `${actionText}${extract}`;
+}
+
 const webBrowseTool = defineTool({
   name: "web_browse",
   label: "Web Browse",
@@ -195,6 +215,38 @@ const webBrowseTool = defineTool({
         },
       };
     } catch (err: any) {
+      // Firecrawl keyless fallback: only on runtime failures (CLI missing /
+      // batch failure), never on local validation errors (bad caller actions).
+      if (isFirecrawlEnabled() && !signal?.aborted && shouldFallbackBrowse(err as Error)) {
+        const fb = await interactKeyless(
+          params.url,
+          { prompt: synthesizeBrowsePrompt({ url: params.url, actions: params.actions as BrowseAction[], selector: params.selector }), timeout: 60 },
+          signal,
+        );
+        if (fb.ok) {
+          const preview = (fb.output || "").replace(/\s+/g, " ").trim().slice(0, 500);
+          const creditTag = fb.creditsUsed !== undefined ? `, ${fb.creditsUsed} credits` : "";
+          const rawText = `URL: ${params.url}\n(via Firecrawl keyless interact fallback${creditTag})\n\n---\n\n${fb.output || "(no content extracted)"}`;
+          const sink = await writeWithFallback(rawText, { tmpPrefix: "pi-web-browse-firecrawl-" });
+          return {
+            content: [{ type: "text", text: sink.text }],
+            details: {
+              title: "",
+              url: params.url,
+              fullOutputPath: sink.fullOutputPath,
+              preview,
+              selector: params.selector,
+              headless: params.headless ?? true,
+              actionCount,
+              steps,
+              viaFirecrawl: true,
+              creditsUsed: fb.creditsUsed,
+            },
+          };
+        }
+        // Graceful skip (CLI absent / IP flagged / rate-limited / disabled):
+        // fall through to the original local error.
+      }
       throw new Error(`Error browsing ${params.url}: ${err.message ?? err}`);
     } finally {
       await closeAgentBrowserSession(session, signal);
@@ -249,6 +301,8 @@ const webBrowseTool = defineTool({
       headless?: boolean;
       actionCount?: number;
       steps?: string[];
+      viaFirecrawl?: boolean;
+      creditsUsed?: number;
     } | undefined;
 
     if (isError) {
@@ -266,6 +320,12 @@ const webBrowseTool = defineTool({
     }
 
     let text = theme.fg("success", "✓ Browsed");
+    if (details?.viaFirecrawl) {
+      text += theme.fg("accent", " [Firecrawl keyless]");
+    }
+    if (details?.creditsUsed !== undefined) {
+      text += theme.fg("muted", ` ${details.creditsUsed} credits`);
+    }
     if (details?.title) {
       text += `  ${theme.fg("toolTitle", details.title)}`;
     }

package/extensions/web_fetch.ts

@@ -25,6 +25,7 @@ import * as path from "node:path";
 import { runScraplingWithFallback } from "./utils/scrapling";
 import { extractPreview } from "./utils/content-preview";
 import { writeWithFallback } from "./utils/output-sink";
+import { scrapeKeyless } from "./utils/firecrawl";
 import { abbreviateUrl, getDomain, getErrorText, normalizeWhitespace, formatExtraction } from "./utils/render-helpers";
 
 export const WebFetchParamsSchema = Type.Object({
@@ -68,15 +69,31 @@ const webFetchTool = defineTool({
         signal,
       );
 
-      if (!ok) {
-        throw new Error(`Failed to fetch ${params.url}\n\nscrapling error:\n${stderr}`);
+      let content: string;
+      let bytes: number;
+      let viaFirecrawl = false;
+
+      if (ok) {
+        content = await fs.promises.readFile(tmpFile, "utf-8");
+        bytes = (await fs.promises.stat(tmpFile)).size;
+      } else {
+        // Local scrapling failed — try the Firecrawl keyless fallback.
+        const localError = `Failed to fetch ${params.url}\n\nscrapling error:\n${stderr}`;
+        const fb = await scrapeKeyless(params.url, {}, signal);
+        if (fb.ok) {
+          content = fb.content;
+          bytes = fb.bytes;
+          viaFirecrawl = true;
+        } else {
+          // Graceful skip (CLI absent / IP flagged / rate-limited / disabled):
+          // never leave the user worse off — surface the original local error.
+          throw new Error(localError);
+        }
       }
 
-      const content = await fs.promises.readFile(tmpFile, "utf-8");
-      const stats = await fs.promises.stat(tmpFile);
-
       const preview = extractPreview(content, 500);
-      const rawText = `Fetched: ${params.url}\nSize: ${stats.size} bytes\n\n---\n\n${content}`;
+      const viaTag = viaFirecrawl ? "\n(via Firecrawl keyless fallback)" : "";
+      const rawText = `Fetched: ${params.url}${viaTag}\nSize: ${bytes} bytes\n\n---\n\n${content}`;
       const sink = await writeWithFallback(rawText, {
         tmpPrefix: "pi-web-fetch-full-",
       });
@@ -86,11 +103,12 @@ const webFetchTool = defineTool({
         content: [{ type: "text", text: sink.text }],
         details: {
           url: params.url,
-          bytes: stats.size,
+          bytes,
           fullOutputPath: tmpFull,
           preview,
           selector: params.selector,
           stealthy: params.stealthy,
+          viaFirecrawl,
         },
       };
     } catch (err: any) {
@@ -128,6 +146,7 @@ const webFetchTool = defineTool({
       preview?: string;
       selector?: string;
       stealthy?: boolean;
+      viaFirecrawl?: boolean;
     } | undefined;
 
     if (isError) {
@@ -139,6 +158,9 @@ const webFetchTool = defineTool({
     }
 
     let text = theme.fg("success", "✓ Fetched");
+    if (details?.viaFirecrawl) {
+      text += theme.fg("accent", " [Firecrawl keyless]");
+    }
     if (details?.url) {
       text += `  ${theme.fg("dim", abbreviateUrl(details.url))}`;
     }

package/extensions/web_search.ts

@@ -20,6 +20,7 @@ import {
 import { Text } from "@earendil-works/pi-tui";
 import { Type, type Static } from "typebox";
 import { writeWithFallback } from "./utils/output-sink";
+import { searchKeyless, shouldFallbackSearch } from "./utils/firecrawl";
 import { abbreviateUrl, getDomain, getErrorText, normalizeWhitespace } from "./utils/render-helpers";
 
 
@@ -78,6 +79,9 @@ const webSearchTool = defineTool({
     let fullOutputPath: string | undefined;
     const MAX_PAGES = 3;
 
+    let localOk = true;
+    let localError: string | undefined;
+
     try {
       for (let page = 1; page <= MAX_PAGES; page++) {
         const searchParams = new URLSearchParams({
@@ -120,51 +124,89 @@ const webSearchTool = defineTool({
           break;
         }
       }
+    } catch (err: any) {
+      localOk = false;
+      localError = err.message ?? String(err);
+    }
 
-      if (allResults.length === 0) {
-        let text = `No results found for "${finalQuery}".`;
-        if (suggestions && suggestions.length > 0) {
-          text += `\n\nSuggestions:\n${suggestions.map((s) => `- ${s}`).join("\n")}`;
+    // Firecrawl keyless fallback: when SearXNG errored OR returned nothing.
+    if (shouldFallbackSearch(localOk, allResults.length)) {
+      const fb = await searchKeyless(params.query, { limit: Math.min(maxResults, 10) }, signal);
+      if (fb.ok && fb.results.length > 0) {
+        const fbResults: SearxResult[] = fb.results.slice(0, maxResults).map((r) => ({
+          title: r.title ?? "(untitled)",
+          url: r.url,
+          content: r.description,
+          engine: "firecrawl",
+        }));
+        const creditTag = fb.creditsUsed !== undefined ? `, ${fb.creditsUsed} credits` : "";
+        const lines: string[] = [`Results for "${params.query}" (via Firecrawl keyless${creditTag}):`, ""];
+        for (let i = 0; i < fbResults.length; i++) {
+          const r = fbResults[i];
+          lines.push(`${i + 1}. ${r.title}`);
+          lines.push(`   URL: ${r.url}`);
+          if (r.content) lines.push(`   ${r.content.replace(/\s+/g, " ").trim()}`);
+          if (r.engine) lines.push(`   [engine: ${r.engine}]`);
+          lines.push("");
         }
+        const rawText = lines.join("\n");
+        const sink = await writeWithFallback(rawText, {
+          tmpPrefix: "pi-web-search-firecrawl-",
+          alwaysWriteFile: true,
+        });
         return {
-          content: [{ type: "text", text }],
-          details: { query: finalQuery, totalResults: 0, results: [] as SearxResult[], fullOutputPath: undefined as string | undefined },
+          content: [{ type: "text", text: sink.text }],
+          details: { query: params.query, totalResults: fbResults.length, results: fbResults, fullOutputPath: sink.fullOutputPath, viaFirecrawl: true, creditsUsed: fb.creditsUsed },
         };
       }
+      // Graceful skip or empty Firecrawl: fall through to local handling.
+    }
 
-      const lines: string[] = [
-        `Results for "${finalQuery}":`,
-        "",
-      ];
+    if (!localOk) {
+      throw new Error(`Failed to query SearXNG at ${searxngUrl}: ${localError}`);
+    }
 
-      for (let i = 0; i < Math.min(maxResults, allResults.length); i++) {
-        const r = allResults[i];
-        lines.push(`${i + 1}. ${r.title}`);
-        lines.push(`   URL: ${r.url}`);
-        if (r.content) {
-          const snippet = r.content.replace(/\s+/g, " ").trim();
-          lines.push(`   ${snippet}`);
-        }
-        if (r.engine) {
-          lines.push(`   [engine: ${r.engine}]`);
-        }
-        lines.push("");
+    if (allResults.length === 0) {
+      let text = `No results found for "${finalQuery}".`;
+      if (suggestions && suggestions.length > 0) {
+        text += `\n\nSuggestions:\n${suggestions.map((s) => `- ${s}`).join("\n")}`;
       }
-
-      const rawText = lines.join("\n");
-      const sink = await writeWithFallback(rawText, {
-        tmpPrefix: "pi-web-search-",
-        alwaysWriteFile: true,
-      });
-      fullOutputPath = sink.fullOutputPath;
-
       return {
-        content: [{ type: "text", text: sink.text }],
-        details: { query: finalQuery, totalResults: allResults.length, results: allResults.slice(0, maxResults), fullOutputPath },
+        content: [{ type: "text", text }],
+        details: { query: finalQuery, totalResults: 0, results: [] as SearxResult[], fullOutputPath: undefined as string | undefined, viaFirecrawl: false, creditsUsed: undefined },
       };
-    } catch (err: any) {
-      throw new Error(`Failed to query SearXNG at ${searxngUrl}: ${err.message ?? err}`);
     }
+
+    const lines: string[] = [
+      `Results for "${finalQuery}":`,
+      "",
+    ];
+
+    for (let i = 0; i < Math.min(maxResults, allResults.length); i++) {
+      const r = allResults[i];
+      lines.push(`${i + 1}. ${r.title}`);
+      lines.push(`   URL: ${r.url}`);
+      if (r.content) {
+        const snippet = r.content.replace(/\s+/g, " ").trim();
+        lines.push(`   ${snippet}`);
+      }
+      if (r.engine) {
+        lines.push(`   [engine: ${r.engine}]`);
+      }
+      lines.push("");
+    }
+
+    const rawText = lines.join("\n");
+    const sink = await writeWithFallback(rawText, {
+      tmpPrefix: "pi-web-search-",
+      alwaysWriteFile: true,
+    });
+    fullOutputPath = sink.fullOutputPath;
+
+    return {
+      content: [{ type: "text", text: sink.text }],
+      details: { query: finalQuery, totalResults: allResults.length, results: allResults.slice(0, maxResults), fullOutputPath, viaFirecrawl: false, creditsUsed: undefined },
+    };
   },
 
   renderCall(args, theme) {
@@ -190,6 +232,8 @@ const webSearchTool = defineTool({
       totalResults?: number;
       results?: Array<{ title?: string; url?: string; score?: number; engine?: string; content?: string }>;
       fullOutputPath?: string;
+      viaFirecrawl?: boolean;
+      creditsUsed?: number;
     } | undefined;
 
     if (isError) {
@@ -207,7 +251,13 @@ const webSearchTool = defineTool({
     const showing = details.results?.length ?? 0;
     const total = details?.totalResults ?? 0;
     let text = theme.fg("success", `✓ ${showing} unique results`);
-    if (total > showing) {
+    if (details?.viaFirecrawl) {
+      text += theme.fg("accent", " [Firecrawl keyless]");
+    }
+    if (details?.creditsUsed !== undefined) {
+      text += theme.fg("muted", ` ${details.creditsUsed} credits`);
+    }
+    if (!details?.viaFirecrawl && total > showing) {
       text += theme.fg("dim", ` (${total} total)`);
     }
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pi-web-toolkit",
-  "version": "0.2.2",
-  "description": "Web research toolkit for the pi coding agent. Search via SearXNG, fetch pages with scrapling, browse interactively via agent-browser, and batch-read sources in parallel.",
+  "version": "0.3.1",
+  "description": "Web research toolkit for the pi coding agent. Search via SearXNG, fetch pages with scrapling, browse interactively via agent-browser, batch-read sources in parallel, and optionally fall back to Firecrawl Keyless (no API key) when a local backend fails.",
   "author": "Wade Huang <fastwade11@gmail.com>",
   "license": "MIT",
   "repository": {
@@ -12,15 +12,16 @@
     "url": "https://github.com/Wade11s/pi-web-toolkit/issues"
   },
   "homepage": "https://github.com/Wade11s/pi-web-toolkit#readme",
-  "keywords": ["pi-package", "pi-extension", "web-search", "scrapling", "agent-browser"],
+  "keywords": ["pi-package", "pi-extension", "web-search", "scrapling", "agent-browser", "firecrawl"],
   "files": ["extensions", "docs", "README.md", "CHANGELOG.md", "package.json", "LICENSE"],
   "engines": {
     "node": ">=22.0.0"
   },
   "scripts": {
     "typecheck": "tsc --noEmit",
-    "test": "tsx test/content-preview/test.ts && tsx test/agent-browser/test.ts",
+    "test": "tsx test/content-preview/test.ts && tsx test/agent-browser/test.ts && tsx test/firecrawl/test.ts",
     "test:agent-browser": "tsx test/agent-browser/test.ts",
+    "test:firecrawl": "tsx test/firecrawl/test.ts",
     "test:approve": "tsx test/content-preview/test.ts --approve"
   },
   "devDependencies": {