pi-web-toolkit 0.2.1 → 0.3.0

package/extensions/firecrawl_interact.ts

@@ -0,0 +1,147 @@
+/**
+ * Firecrawl Interact Extension — natural-language browser interaction (keyless)
+ *
+ * Provides a `firecrawl_interact` tool that scrapes a URL to start a live
+ * Firecrawl browser session, then drives the page with a natural-language
+ * prompt (or code) and returns the result. It is an escape hatch for
+ * interactive pages the local agent-browser tool cannot run (missing CLI,
+ * missing OS browser deps), and underpins the automatic `web_browse` fallback.
+ *
+ * Requires: `npm install -g firecrawl-cli` (optional; degrades gracefully).
+ * Privacy: the URL, page content, and prompt are sent to Firecrawl's cloud.
+ */
+
+import {
+  defineTool,
+  type ExtensionAPI,
+  formatSize,
+  DEFAULT_MAX_BYTES,
+  DEFAULT_MAX_LINES,
+} from "@earendil-works/pi-coding-agent";
+import { Text } from "@earendil-works/pi-tui";
+import { Type, type Static } from "typebox";
+import { StringEnum } from "@earendil-works/pi-ai";
+import { interactKeyless } from "./utils/firecrawl";
+import { writeWithFallback } from "./utils/output-sink";
+import { abbreviateUrl, getErrorText, normalizeWhitespace } from "./utils/render-helpers";
+
+export const FirecrawlInteractParamsSchema = Type.Object({
+  url: Type.String({ description: "Full URL to open and interact with" }),
+  prompt: Type.Optional(Type.String({ description: "Natural-language task for the AI agent (e.g. 'Click the pricing tab and return the price')" })),
+  code: Type.Optional(Type.String({ description: "Code to execute in the browser sandbox instead of a prompt" })),
+  language: Type.Optional(StringEnum(["node", "python", "bash"] as const)),
+  timeout: Type.Optional(Type.Integer({ description: "Timeout in seconds (1-300). Default: 30", minimum: 1, maximum: 300 })),
+});
+
+export type FirecrawlInteractInput = Static<typeof FirecrawlInteractParamsSchema>;
+
+const firecrawlInteractTool = defineTool({
+  name: "firecrawl_interact",
+  label: "Firecrawl Interact",
+  description: [
+    "Open a URL in a live Firecrawl browser session and drive it with a natural-language",
+    "prompt (or code), returning the result. Keyless — no API key, no signup.",
+    "Use firecrawl_interact when the local web_browse cannot run, or when you want",
+    "natural-language page interaction without CSS selectors.",
+    "Privacy: the URL, page content, and prompt are sent to Firecrawl's cloud.",
+    `Output is truncated to ${DEFAULT_MAX_LINES} lines or ${formatSize(DEFAULT_MAX_BYTES)}; if truncated, full output is saved to a temp file.`,
+  ].join(" "),
+  promptSnippet: "Drive a page via Firecrawl keyless (natural-language interaction)",
+  promptGuidelines: [
+    "Prefer web_browse first; reach for firecrawl_interact when web_browse can't run or you want NL interaction.",
+    "Write each prompt as a single, focused task; the session can be reused across calls.",
+    "Always pass the full URL including https://.",
+  ],
+  parameters: FirecrawlInteractParamsSchema,
+
+  async execute(_toolCallId, params, signal) {
+    if (!params.prompt && !params.code) {
+      throw new Error("firecrawl_interact requires either a prompt or code.");
+    }
+    const out = await interactKeyless(
+      params.url,
+      { prompt: params.prompt, code: params.code, language: params.language, timeout: params.timeout },
+      signal,
+    );
+
+    if (!out.ok) {
+      const reason = out.failure?.reason ?? "unknown error";
+      throw new Error(`Firecrawl interact failed (${out.failure?.kind}): ${reason}`);
+    }
+
+    const rawText = `Interacted: ${params.url}\n(via Firecrawl keyless${out.creditsUsed !== undefined ? `, ${out.creditsUsed} credits` : ""})\n${out.liveViewUrl ? `Live view: ${out.liveViewUrl}\n` : ""}\n---\n\n${out.output || "(no output)"}`;
+    const sink = await writeWithFallback(rawText, { tmpPrefix: "pi-firecrawl-interact-" });
+    const preview = (out.output || "").replace(/\s+/g, " ").trim().slice(0, 500);
+
+    return {
+      content: [{ type: "text", text: sink.text }],
+      details: {
+        url: params.url,
+        output: out.output,
+        preview,
+        fullOutputPath: sink.fullOutputPath,
+        liveViewUrl: out.liveViewUrl,
+        creditsUsed: out.creditsUsed,
+        viaFirecrawl: true,
+      },
+    };
+  },
+
+  renderCall(args, theme) {
+    let text = theme.fg("toolTitle", theme.bold("firecrawl_interact "));
+    text += theme.fg("muted", args.url);
+    if (args.prompt) text += theme.fg("dim", ` — ${args.prompt.slice(0, 60)}`);
+    return new Text(text, 0, 0);
+  },
+
+  renderResult(result, { expanded, isPartial }, theme, context) {
+    const isError = context?.isError ?? false;
+
+    if (isPartial) {
+      return new Text(theme.fg("warning", "Interacting via Firecrawl..."), 0, 0);
+    }
+
+    const details = result.details as {
+      url?: string;
+      output?: string;
+      preview?: string;
+      fullOutputPath?: string;
+      liveViewUrl?: string;
+      creditsUsed?: number;
+    } | undefined;
+
+    if (isError) {
+      const errText = getErrorText(result);
+      let text = theme.fg("error", "✗ Firecrawl interact failed");
+      if (details?.url) text += `  ${theme.fg("dim", abbreviateUrl(details.url))}`;
+      text += `\n\n  ${theme.fg("toolOutput", errText)}`;
+      return new Text(text, 0, 0);
+    }
+
+    let text = theme.fg("success", "✓ Interacted");
+    text += theme.fg("accent", " [Firecrawl keyless]");
+    if (details?.url) text += `  ${theme.fg("dim", abbreviateUrl(details.url))}`;
+    if (details?.creditsUsed !== undefined) text += theme.fg("muted", ` ${details.creditsUsed} credits`);
+
+    if (!expanded && details?.preview) {
+      const snippet = normalizeWhitespace(details.preview);
+      const short = snippet.length > 160 ? snippet.slice(0, 160).replace(/\s+\S*$/, "") + "..." : snippet;
+      text += `\n\n  ${theme.fg("muted", short)}`;
+    }
+
+    if (expanded) {
+      if (details?.output) {
+        text += `\n\n  ${theme.fg("muted", normalizeWhitespace(details.output))}`;
+      }
+      if (details?.fullOutputPath) {
+        text += `\n\n${theme.fg("accent", `Full output: ${details.fullOutputPath}`)}`;
+      }
+    }
+
+    return new Text(text, 0, 0);
+  },
+});
+
+export default function (pi: ExtensionAPI) {
+  pi.registerTool(firecrawlInteractTool);
+}

package/extensions/firecrawl_scrape.ts

@@ -0,0 +1,154 @@
+/**
+ * Firecrawl Scrape Extension — single-page fetch via firecrawl-cli (keyless)
+ *
+ * Provides a `firecrawl_scrape` tool that fetches a single URL as clean
+ * markdown through the official Firecrawl CLI in keyless mode (no API key,
+ * no signup). It is an explicit escape hatch for hard targets the local
+ * scrapling fetcher cannot handle (anti-bot, heavy JS, PDFs), and also
+ * underpins the automatic `web_fetch` fallback.
+ *
+ * Requires: `npm install -g firecrawl-cli` (optional; the tool degrades
+ * gracefully and reports when the CLI is unavailable).
+ *
+ * Privacy: the URL and page content are sent to Firecrawl's cloud.
+ */
+
+import {
+  defineTool,
+  type ExtensionAPI,
+  formatSize,
+  DEFAULT_MAX_BYTES,
+  DEFAULT_MAX_LINES,
+} from "@earendil-works/pi-coding-agent";
+import { Text } from "@earendil-works/pi-tui";
+import { Type, type Static } from "typebox";
+import { scrapeKeyless } from "./utils/firecrawl";
+import { extractPreview } from "./utils/content-preview";
+import { writeWithFallback } from "./utils/output-sink";
+import { abbreviateUrl, getErrorText, normalizeWhitespace, formatExtraction } from "./utils/render-helpers";
+
+export const FirecrawlScrapeParamsSchema = Type.Object({
+  url: Type.String({ description: "Full URL to fetch (e.g. https://example.com/article)" }),
+  waitFor: Type.Optional(Type.Integer({ description: "Wait (ms) before scraping for JS-rendered content", minimum: 0 })),
+  includeTags: Type.Optional(Type.Array(Type.String(), { description: "HTML tags to include (Firecrawl tag filter, not a CSS selector)" })),
+  excludeTags: Type.Optional(Type.Array(Type.String(), { description: "HTML tags to exclude" })),
+  onlyMainContent: Type.Optional(Type.Boolean({ description: "Extract only main content (drop nav/footer). Default: true", default: true })),
+});
+
+export type FirecrawlScrapeInput = Static<typeof FirecrawlScrapeParamsSchema>;
+
+const firecrawlScrapeTool = defineTool({
+  name: "firecrawl_scrape",
+  label: "Firecrawl Scrape",
+  description: [
+    "Fetch a single page as clean markdown via Firecrawl (keyless — no API key, no signup).",
+    "Use firecrawl_scrape when the local web_fetch fails on a hard target (anti-bot,",
+    "JavaScript-heavy pages, PDFs) or when you need Firecrawl's cloud rendering directly.",
+    "Privacy: the URL and page content are sent to Firecrawl's cloud.",
+    `Output is truncated to ${DEFAULT_MAX_LINES} lines or ${formatSize(DEFAULT_MAX_BYTES)}; if truncated, full output is saved to a temp file.`,
+  ].join(" "),
+  promptSnippet: "Fetch a single page via Firecrawl keyless (anti-bot / JS / PDF fallback)",
+  promptGuidelines: [
+    "Prefer web_fetch first; reach for firecrawl_scrape when web_fetch fails or you need cloud rendering.",
+    "firecrawl_scrape handles anti-bot protection, JS-heavy SPAs, and PDFs that scrapling may miss.",
+    "Always pass the full URL including https://.",
+  ],
+  parameters: FirecrawlScrapeParamsSchema,
+
+  async execute(_toolCallId, params, signal) {
+    const out = await scrapeKeyless(params.url, {
+      waitFor: params.waitFor,
+      includeTags: params.includeTags,
+      excludeTags: params.excludeTags,
+      onlyMainContent: params.onlyMainContent,
+    }, signal);
+
+    if (!out.ok) {
+      const reason = out.failure?.reason ?? "unknown error";
+      throw new Error(`Firecrawl scrape failed (${out.failure?.kind}): ${reason}`);
+    }
+
+    const preview = extractPreview(out.content, 500);
+    const rawText = `Fetched: ${params.url}\n(via Firecrawl keyless${out.creditsUsed !== undefined ? `, ${out.creditsUsed} credits` : ""})\nSize: ${out.bytes} bytes\n\n---\n\n${out.content}`;
+    const sink = await writeWithFallback(rawText, { tmpPrefix: "pi-firecrawl-scrape-full-" });
+
+    return {
+      content: [{ type: "text", text: sink.text }],
+      details: {
+        url: params.url,
+        bytes: out.bytes,
+        fullOutputPath: sink.fullOutputPath,
+        preview,
+        title: out.title,
+        creditsUsed: out.creditsUsed,
+        viaFirecrawl: true,
+      },
+    };
+  },
+
+  renderCall(args, theme) {
+    let text = theme.fg("toolTitle", theme.bold("firecrawl_scrape "));
+    text += theme.fg("muted", args.url);
+    if (args.waitFor) {
+      text += theme.fg("dim", ` [wait=${args.waitFor}]`);
+    }
+    return new Text(text, 0, 0);
+  },
+
+  renderResult(result, { expanded, isPartial }, theme, context) {
+    const isError = context?.isError ?? false;
+
+    if (isPartial) {
+      return new Text(theme.fg("warning", "Scraping via Firecrawl..."), 0, 0);
+    }
+
+    const details = result.details as {
+      url?: string;
+      bytes?: number;
+      fullOutputPath?: string;
+      preview?: string;
+      title?: string;
+      creditsUsed?: number;
+    } | undefined;
+
+    if (isError) {
+      const errText = getErrorText(result);
+      let text = theme.fg("error", "✗ Firecrawl scrape failed");
+      if (details?.url) text += `  ${theme.fg("dim", abbreviateUrl(details.url))}`;
+      text += `\n\n  ${theme.fg("toolOutput", errText)}`;
+      return new Text(text, 0, 0);
+    }
+
+    let text = theme.fg("success", "✓ Fetched");
+    text += theme.fg("accent", " [Firecrawl keyless]");
+    if (details?.title) {
+      text += `  ${theme.fg("toolTitle", details.title)}`;
+    } else if (details?.url) {
+      text += `  ${theme.fg("dim", abbreviateUrl(details.url))}`;
+    }
+    if (details?.bytes && details?.preview) {
+      text += `  ${theme.fg("muted", formatExtraction(details.bytes, details.preview.length))}`;
+    }
+
+    if (!expanded && details?.preview) {
+      const snippet = normalizeWhitespace(details.preview);
+      const short = snippet.length > 160 ? snippet.slice(0, 160).replace(/\s+\S*$/, "") + "..." : snippet;
+      text += `\n\n  ${theme.fg("muted", short)}`;
+    }
+
+    if (expanded) {
+      if (details?.preview) {
+        text += `\n\n  ${theme.fg("muted", normalizeWhitespace(details.preview))}`;
+      }
+      if (details?.fullOutputPath) {
+        text += `\n\n${theme.fg("accent", `Full output: ${details.fullOutputPath}`)}`;
+      }
+    }
+
+    return new Text(text, 0, 0);
+  },
+});
+
+export default function (pi: ExtensionAPI) {
+  pi.registerTool(firecrawlScrapeTool);
+}

package/extensions/firecrawl_search.ts

@@ -0,0 +1,165 @@
+/**
+ * Firecrawl Search Extension — web search via firecrawl-cli (keyless)
+ *
+ * Provides a `firecrawl_search` tool that searches the web through the
+ * official Firecrawl CLI in keyless mode (no API key, no signup). It exposes
+ * Firecrawl-specific capabilities the local SearXNG tool does not: sources
+ * (web/images/news), categories (github/research/pdf), and domain filters.
+ *
+ * Requires: `npm install -g firecrawl-cli` (optional; degrades gracefully).
+ * Privacy: the query is sent to Firecrawl's cloud.
+ */
+
+import {
+  defineTool,
+  type ExtensionAPI,
+  formatSize,
+  DEFAULT_MAX_BYTES,
+  DEFAULT_MAX_LINES,
+} from "@earendil-works/pi-coding-agent";
+import { Text } from "@earendil-works/pi-tui";
+import { Type, type Static } from "typebox";
+import { StringEnum } from "@earendil-works/pi-ai";
+import { searchKeyless, buildSearchQuery } from "./utils/firecrawl";
+import { writeWithFallback } from "./utils/output-sink";
+import { abbreviateUrl, getDomain, getErrorText, normalizeWhitespace } from "./utils/render-helpers";
+
+export const FirecrawlSearchParamsSchema = Type.Object({
+  query: Type.String({ description: "Search query" }),
+  limit: Type.Optional(Type.Integer({ description: "Max results (1-100). Default: 10", minimum: 1, maximum: 100 })),
+  sources: Type.Optional(Type.Array(StringEnum(["web", "images", "news"] as const), { description: "Sources to search. Default: web" })),
+  categories: Type.Optional(Type.Array(StringEnum(["github", "research", "pdf"] as const), { description: "Filter by GitHub / research papers / PDFs" })),
+  country: Type.Optional(Type.String({ description: "ISO country code for geo-targeting (e.g. US, DE)" })),
+  tbs: Type.Optional(Type.String({ description: "Time filter: qdr:h (hour), qdr:d (day), qdr:w (week), qdr:m (month), qdr:y (year)" })),
+  location: Type.Optional(Type.String({ description: "Geo-targeting location (e.g. 'Berlin,Germany')" })),
+  includeDomains: Type.Optional(Type.Array(Type.String(), { description: "Restrict results to these domains (hostnames)" })),
+  excludeDomains: Type.Optional(Type.Array(Type.String(), { description: "Exclude results from these domains (hostnames)" })),
+});
+
+export type FirecrawlSearchInput = Static<typeof FirecrawlSearchParamsSchema>;
+
+const firecrawlSearchTool = defineTool({
+  name: "firecrawl_search",
+  label: "Firecrawl Search",
+  description: [
+    "Search the web via Firecrawl (keyless — no API key, no signup).",
+    "Supports sources (web/images/news) and categories (github/research/pdf) that",
+    "SearXNG does not. Use as an escape hatch or when web_search returns nothing.",
+    "Privacy: the query is sent to Firecrawl's cloud.",
+    `Output is truncated to ${DEFAULT_MAX_LINES} lines or ${formatSize(DEFAULT_MAX_BYTES)}; if truncated, full output is saved to a temp file.`,
+  ].join(" "),
+  promptSnippet: "Search the web via Firecrawl keyless (categories, sources, domain filters)",
+  promptGuidelines: [
+    "Prefer web_search first; reach for firecrawl_search when web_search fails or returns nothing.",
+    "Use categories=[\"github\"], [\"research\"], or [\"pdf\"] for source-type-specific discovery.",
+    "Use includeDomains/excludeDomains to scope results to specific sites.",
+  ],
+  parameters: FirecrawlSearchParamsSchema,
+
+  async execute(_toolCallId, params, signal) {
+    const query = buildSearchQuery(params.query, params.includeDomains, params.excludeDomains);
+    const out = await searchKeyless(query, {
+      limit: params.limit,
+      sources: params.sources,
+      categories: params.categories,
+      country: params.country,
+      tbs: params.tbs,
+      location: params.location,
+    }, signal);
+
+    if (!out.ok) {
+      const reason = out.failure?.reason ?? "unknown error";
+      throw new Error(`Firecrawl search failed (${out.failure?.kind}): ${reason}`);
+    }
+
+    const lines: string[] = [`Results for "${params.query}" (via Firecrawl keyless${out.creditsUsed !== undefined ? `, ${out.creditsUsed} credits` : ""}):`, ""];
+    for (let i = 0; i < out.results.length; i++) {
+      const r = out.results[i];
+      lines.push(`${i + 1}. ${r.title ?? "(untitled)"}`);
+      lines.push(`   URL: ${r.url}`);
+      if (r.description) lines.push(`   ${r.description.replace(/\s+/g, " ").trim()}`);
+      if (r.category) lines.push(`   [category: ${r.category}]`);
+      lines.push("");
+    }
+
+    const rawText = lines.join("\n");
+    const sink = await writeWithFallback(rawText, { tmpPrefix: "pi-firecrawl-search-", alwaysWriteFile: true });
+
+    return {
+      content: [{ type: "text", text: sink.text }],
+      details: {
+        query: params.query,
+        totalResults: out.results.length,
+        results: out.results,
+        creditsUsed: out.creditsUsed,
+        fullOutputPath: sink.fullOutputPath,
+        viaFirecrawl: true,
+      },
+    };
+  },
+
+  renderCall(args, theme) {
+    let text = theme.fg("toolTitle", theme.bold("firecrawl_search "));
+    text += theme.fg("muted", args.query);
+    if (args.categories) text += theme.fg("dim", ` [${args.categories.join(",")}]`);
+    return new Text(text, 0, 0);
+  },
+
+  renderResult(result, { expanded, isPartial }, theme, context) {
+    const isError = context?.isError ?? false;
+
+    if (isPartial) {
+      const query = (result.details as any)?.query as string | undefined;
+      const label = query ? `Searching "${query}" via Firecrawl...` : "Searching via Firecrawl...";
+      return new Text(theme.fg("warning", label), 0, 0);
+    }
+
+    const details = result.details as {
+      query?: string;
+      totalResults?: number;
+      results?: Array<{ title?: string; url?: string; description?: string; category?: string }>;
+      creditsUsed?: number;
+      fullOutputPath?: string;
+    } | undefined;
+
+    if (isError) {
+      const errText = getErrorText(result);
+      let text = theme.fg("error", "✗ Firecrawl search failed");
+      if (details?.query) text += `  ${theme.fg("dim", details.query)}`;
+      text += `\n\n  ${theme.fg("toolOutput", errText)}`;
+      return new Text(text, 0, 0);
+    }
+
+    const showing = details?.results?.length ?? 0;
+    let text = theme.fg("success", `✓ ${showing} results`);
+    text += theme.fg("accent", " [Firecrawl keyless]");
+    if (details?.creditsUsed !== undefined) {
+      text += theme.fg("muted", ` ${details.creditsUsed} credits`);
+    }
+
+    const top = (details?.results ?? []).slice(0, expanded ? 10 : 3);
+    for (let i = 0; i < top.length; i++) {
+      const r = top[i];
+      const domain = r.url ? theme.fg("dim", `  ${getDomain(r.url)}`) : "";
+      text += `\n  [${i + 1}] ${theme.fg("toolTitle", r.title ?? "(untitled)")}${domain}`;
+      if (r.description) {
+        const snippet = normalizeWhitespace(r.description);
+        const short = snippet.length > 90 ? snippet.slice(0, 90).replace(/\s+\S*$/, "") + "..." : snippet;
+        text += `\n    ${theme.fg("muted", short)}`;
+      }
+    }
+    if (showing > top.length) {
+      text += `\n  ${theme.fg("muted", `... and ${showing - top.length} more${expanded ? "" : " (Ctrl+O for full list)"}`)}`;
+    }
+
+    if (expanded && details?.fullOutputPath) {
+      text += `\n${theme.fg("accent", `Full output: ${details.fullOutputPath}`)}`;
+    }
+
+    return new Text(text, 0, 0);
+  },
+});
+
+export default function (pi: ExtensionAPI) {
+  pi.registerTool(firecrawlSearchTool);
+}

package/extensions/index.ts

@@ -3,7 +3,7 @@
  *
  * Registers all web research tools as a single extension:
  *   - web_search: Search via SearXNG
- *   - web_fetch: Fetch static pages with scrapling
+ *   - web_fetch: Fetch a single page with scrapling
  *   - web_browse: Interactive browser automation via agent-browser
  *   - web_batch_fetch: Concurrent multi-page fetching
  */
@@ -13,10 +13,16 @@ import registerWebSearch from "./web_search";
 import registerWebFetch from "./web_fetch";
 import registerWebBrowse from "./web_browse";
 import registerWebBatchFetch from "./web_batch_fetch";
+import registerFirecrawlScrape from "./firecrawl_scrape";
+import registerFirecrawlSearch from "./firecrawl_search";
+import registerFirecrawlInteract from "./firecrawl_interact";
 
 export default function (pi: ExtensionAPI) {
   registerWebSearch(pi);
   registerWebFetch(pi);
   registerWebBrowse(pi);
   registerWebBatchFetch(pi);
+  registerFirecrawlScrape(pi);
+  registerFirecrawlSearch(pi);
+  registerFirecrawlInteract(pi);
 }

package/extensions/utils/cli-runner.ts

@@ -3,7 +3,7 @@
  *
  * Provides a single interface for running external CLI commands
  * with consistent signal handling, timeout support, and stdout/stderr
- * collection. Enables testability by allowing the runner to be swapped.
+ * collection.
  */
 
 import { spawn, type ChildProcess } from "node:child_process";
@@ -17,6 +17,8 @@ export interface CLIRunOptions {
   timeout?: number;
   /** AbortSignal for cancellation. */
   signal?: AbortSignal;
+  /** Optional environment override for the child process. */
+  env?: NodeJS.ProcessEnv;
 }
 
 export interface CLIRunResult {
@@ -44,6 +46,7 @@ export function runCLI(options: CLIRunOptions): Promise<CLIRunResult> {
     const proc = spawn(options.command, options.args, {
       shell: false,
       stdio: stdio as any,
+      env: options.env,
     }) as ChildProcess;
 
     let stdout = "";