pi-firecrawl 0.1.0

package/README.md

@@ -0,0 +1,68 @@
+# pi-firecrawl
+
+Firecrawl web scraping extension for [Pi coding agent](https://pi.dev).
+
+Provides 5 tools and a `/firecrawl` command for Firecrawl API v2 integration.
+
+## Install
+
+```bash
+pi install npm:pi-firecrawl
+```
+
+## Tools
+
+| Tool | Description |
+|------|-------------|
+| `firecrawl_scrape` | Scrape a single URL into markdown, HTML, raw HTML, links, screenshots, or JSON |
+| `firecrawl_crawl` | Start a crawl job to discover and scrape multiple pages |
+| `firecrawl_crawl_status` | Check the status and retrieve data of a crawl job |
+| `firecrawl_map` | Discover all URLs on a website using the map endpoint |
+| `firecrawl_search` | Search the web and return results |
+
+## Commands
+
+- `/firecrawl` — Select which Firecrawl tools to enable/disable
+
+## Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `FIRECRAWL_API_KEY` | — | Firecrawl API token (fallback if not set via `/firecrawl key`) |
+| `FIRECRAWL_API_URL` | `https://api.firecrawl.dev` | Base URL for Firecrawl API |
+
+## API Key Management
+
+You can set the Firecrawl API key from within pi (persisted across sessions):
+
+```
+/firecrawl key fc-your-api-key-here
+```
+
+Check or clear the key:
+
+```
+/firecrawl key          # shows current key status
+/firecrawl key --clear   # removes saved key
+```
+
+**Key resolution order:**
+1. Saved key (via `/firecrawl key`)
+2. `FIRECRAWL_API_KEY` env var
+3. `token` parameter passed per-tool
+
+## Usage
+
+```bash
+# Scrape a URL
+firecrawl_scrape url="https://example.com"
+
+# Search the web
+firecrawl_search query="pi coding agent"
+
+# Start a crawl
+firecrawl_crawl url="https://example.com"
+
+# Check crawl status
+firecrawl_crawl_status jobId="<job-id>"
+```

package/extensions/firecrawl.ts

@@ -0,0 +1,307 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+// Base API URL, can be overridden via FIRECRAWL_API_URL env var
+const FIRECRAWL_BASE_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
+const STATE_KEY = "firecrawl-settings";
+const AUTH_FILE = path.join(process.env.HOME || process.env.USERPROFILE || "~", ".pi", "agent", "auth.json");
+
+// Load API key from pi's auth.json
+function loadKey(): string | undefined {
+  try {
+    const raw = fs.readFileSync(AUTH_FILE, "utf-8");
+    const data = JSON.parse(raw);
+    const entry = data["firecrawl"];
+    if (entry && entry.type === "api_key" && typeof entry.key === "string") {
+      return entry.key;
+    }
+    return undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function saveKey(key: string) {
+  try {
+    const raw = fs.existsSync(AUTH_FILE) ? fs.readFileSync(AUTH_FILE, "utf-8") : "{}";
+    const data = JSON.parse(raw);
+    data["firecrawl"] = { type: "api_key", key };
+    const dir = path.dirname(AUTH_FILE);
+    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(AUTH_FILE, JSON.stringify(data, null, 2), "utf-8");
+  } catch { /* ignore file errors */ }
+}
+
+function clearKey() {
+  try {
+    if (!fs.existsSync(AUTH_FILE)) return;
+    const raw = fs.readFileSync(AUTH_FILE, "utf-8");
+    const data = JSON.parse(raw);
+    delete data["firecrawl"];
+    fs.writeFileSync(AUTH_FILE, JSON.stringify(data, null, 2), "utf-8");
+  } catch { /* ignore */ }
+}
+
+// Resolve API token: auth.json > env var > parameter
+function resolveToken(paramToken?: string): string | undefined {
+  return loadKey() ?? process.env.FIRECRAWL_API_KEY ?? paramToken;
+}
+
+async function firecrawlRequest(
+  endpoint: string,
+  token: string,
+  body: unknown,
+  signal?: AbortSignal
+) {
+  const response = await fetch(`${FIRECRAWL_BASE_URL}${endpoint}`, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${token}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(body),
+    signal,
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`Firecrawl API error ${response.status}: ${errorText}`);
+  }
+
+  return response.json();
+}
+
+async function firecrawlGet(
+  endpoint: string,
+  token: string,
+  signal?: AbortSignal
+) {
+  const response = await fetch(`${FIRECRAWL_BASE_URL}${endpoint}`, {
+    headers: {
+      Authorization: `Bearer ${token}`,
+    },
+    signal,
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`Firecrawl API error ${response.status}: ${errorText}`);
+  }
+
+  return response.json();
+}
+
+export default function (pi: ExtensionAPI) {
+  let activeFirecrawlTools: string[] = ["firecrawl_scrape", "firecrawl_crawl", "firecrawl_crawl_status", "firecrawl_map", "firecrawl_search"];
+
+  // Restore state on session start
+  pi.on("session_start", async (_event, ctx) => {
+    for (const entry of ctx.sessionManager.getEntries()) {
+      if (entry.type === "custom" && entry.customType === STATE_KEY) {
+        if (Array.isArray(entry.data)) {
+          activeFirecrawlTools = entry.data;
+        }
+      }
+    }
+    updateActiveTools();
+  });
+
+  function updateActiveTools() {
+    const allTools = pi.getAllTools();
+    const nonFirecrawlTools = allTools
+      .filter(t => !t.name.startsWith("firecrawl_"))
+      .map(t => t.name);
+
+    pi.setActiveTools([...nonFirecrawlTools, ...activeFirecrawlTools]);
+  }
+
+  // --- Tools ---
+
+  pi.registerTool({
+    name: "firecrawl_scrape",
+    label: "Firecrawl Scrape",
+    description: "Scrape a single URL into markdown, HTML, raw HTML, links, screenshots, or JSON",
+    parameters: Type.Object({
+      url: Type.String({ description: "URL to scrape" }),
+      formats: Type.Optional(
+        Type.Array(Type.String(), { description: "Output formats (markdown, html, rawHtml, links, screenshot, json)" })
+      ),
+      token: Type.Optional(Type.String({ description: "Firecrawl API token (defaults to saved key, then FIRECRAWL_API_KEY env var)" })),
+    }),
+    async execute(toolCallId, params, signal, onUpdate, ctx) {
+      const token = resolveToken(params.token);
+      if (!token) return { content: [{ type: "text", text: "Firecrawl API token not provided. Set one with `/firecrawl key <token>` or set FIRECRAWL_API_KEY env var." }], isError: true };
+
+      onUpdate?.({ content: [{ type: "text", text: `Scraping ${params.url}...` }] });
+
+      try {
+        const data = await firecrawlRequest("/v2/scrape", token, {
+          url: params.url,
+          formats: params.formats || ["markdown"],
+        }, signal);
+        return { content: [{ type: "text", text: JSON.stringify(data, null, 2) }] };
+      } catch (err: any) {
+        return { content: [{ type: "text", text: `Firecrawl scrape failed: ${err.message}` }], isError: true };
+      }
+    },
+  });
+
+  pi.registerTool({
+    name: "firecrawl_crawl",
+    label: "Firecrawl Crawl",
+    description: "Start a crawl job to discover and scrape multiple pages",
+    parameters: Type.Object({
+      url: Type.String({ description: "Base URL to start crawl from" }),
+      token: Type.Optional(Type.String({ description: "Firecrawl API token (defaults to saved key, then FIRECRAWL_API_KEY env var)" })),
+    }),
+    async execute(toolCallId, params, signal, onUpdate, ctx) {
+      const token = resolveToken(params.token);
+      if (!token) return { content: [{ type: "text", text: "Firecrawl API token not provided. Set one with `/firecrawl key <token>` or set FIRECRAWL_API_KEY env var." }], isError: true };
+
+      onUpdate?.({ content: [{ type: "text", text: `Starting crawl from ${params.url}...` }] });
+
+      try {
+        const data = await firecrawlRequest("/v2/crawl", token, { url: params.url }, signal);
+        return { content: [{ type: "text", text: JSON.stringify(data, null, 2) }] };
+      } catch (err: any) {
+        return { content: [{ type: "text", text: `Firecrawl crawl failed: ${err.message}` }], isError: true };
+      }
+    },
+  });
+
+  pi.registerTool({
+    name: "firecrawl_crawl_status",
+    label: "Firecrawl Crawl Status",
+    description: "Check the status and retrieve data of a crawl job",
+    parameters: Type.Object({
+      jobId: Type.String({ description: "The ID of the crawl job" }),
+      token: Type.Optional(Type.String({ description: "Firecrawl API token (defaults to saved key, then FIRECRAWL_API_KEY env var)" })),
+    }),
+    async execute(toolCallId, params, signal, onUpdate, ctx) {
+      const token = resolveToken(params.token);
+      if (!token) return { content: [{ type: "text", text: "Firecrawl API token not provided. Set one with `/firecrawl key <token>` or set FIRECRAWL_API_KEY env var." }], isError: true };
+
+      onUpdate?.({ content: [{ type: "text", text: `Checking status of job ${params.jobId}...` }] });
+
+      try {
+        const data = await firecrawlGet(`/v2/crawl/${params.jobId}`, token, signal);
+        return { content: [{ type: "text", text: JSON.stringify(data, null, 2) }] };
+      } catch (err: any) {
+        return { content: [{ type: "text", text: `Firecrawl status check failed: ${err.message}` }], isError: true };
+      }
+    },
+  });
+
+  pi.registerTool({
+    name: "firecrawl_map",
+    label: "Firecrawl Map",
+    description: "Discover all URLs on a website using the map endpoint",
+    parameters: Type.Object({
+      url: Type.String({ description: "Base URL to map" }),
+      token: Type.Optional(Type.String({ description: "Firecrawl API token (defaults to saved key, then FIRECRAWL_API_KEY env var)" })),
+    }),
+    async execute(toolCallId, params, signal, onUpdate, ctx) {
+      const token = resolveToken(params.token);
+      if (!token) return { content: [{ type: "text", text: "Firecrawl API token not provided. Set one with `/firecrawl key <token>` or set FIRECRAWL_API_KEY env var." }], isError: true };
+
+      onUpdate?.({ content: [{ type: "text", text: `Mapping ${params.url}...` }] });
+
+      try {
+        const data = await firecrawlRequest("/v2/map", token, { url: params.url }, signal);
+        return { content: [{ type: "text", text: JSON.stringify(data, null, 2) }] };
+      } catch (err: any) {
+        return { content: [{ type: "text", text: `Firecrawl map failed: ${err.message}` }], isError: true };
+      }
+    },
+  });
+
+  pi.registerTool({
+    name: "firecrawl_search",
+    label: "Firecrawl Search",
+    description: "Search the web and return results",
+    parameters: Type.Object({
+      query: Type.String({ description: "Search query" }),
+      limit: Type.Optional(Type.Number({ description: "Number of results to return" })),
+      token: Type.Optional(Type.String({ description: "Firecrawl API token (defaults to saved key, then FIRECRAWL_API_KEY env var)" })),
+    }),
+    async execute(toolCallId, params, signal, onUpdate, ctx) {
+      const token = resolveToken(params.token);
+      if (!token) return { content: [{ type: "text", text: "Firecrawl API token not provided. Set one with `/firecrawl key <token>` or set FIRECRAWL_API_KEY env var." }], isError: true };
+
+      onUpdate?.({ content: [{ type: "text", text: `Searching for "${params.query}"...` }] });
+
+      try {
+        const data = await firecrawlRequest("/v2/search", token, {
+          query: params.query,
+          limit: params.limit || 5,
+        }, signal);
+        return { content: [{ type: "text", text: JSON.stringify(data, null, 2) }] };
+      } catch (err: any) {
+        return { content: [{ type: "text", text: `Firecrawl search failed: ${err.message}` }], isError: true };
+      }
+    },
+  });
+
+  // --- Command ---
+
+  pi.registerCommand("firecrawl", {
+    description: "Firecrawl configuration. Use `/firecrawl` to select tools, `/firecrawl key <token>` to set API key, `/firecrawl key --clear` to remove it.",
+    handler: async (args, ctx) => {
+      const parts = args.trim().split(/\s+/);
+
+      // /firecrawl key <value>
+      if (parts[0] === "key") {
+        if (parts.length >= 2 && parts[1] === "--clear") {
+          clearKey();
+          ctx.ui.notify("Firecrawl API key cleared", "info");
+        } else if (parts.length >= 2) {
+          const key = parts.slice(1).join(" ");
+          saveKey(key);
+          ctx.ui.notify("Firecrawl API key saved", "info");
+        } else {
+          const currentKey = loadKey();
+          if (currentKey) {
+            ctx.ui.notify(`Firecrawl API key is set (${currentKey.slice(0, 8)}...)`, "info");
+          } else if (process.env.FIRECRAWL_API_KEY) {
+            ctx.ui.notify(`Using FIRECRAWL_API_KEY env var (${process.env.FIRECRAWL_API_KEY.slice(0, 8)}...)`, "info");
+          } else {
+            ctx.ui.notify("No Firecrawl API key set. Use `/firecrawl key <token>` to set one.", "info");
+          }
+        }
+        return;
+      }
+
+      // /firecrawl (tool selector)
+      const allFirecrawlTools = ["firecrawl_scrape", "firecrawl_crawl", "firecrawl_crawl_status", "firecrawl_map", "firecrawl_search"];
+
+      const choice = await ctx.ui.select(
+        "Select Firecrawl tools to enable",
+        allFirecrawlTools,
+        { multiple: true }
+      );
+
+      if (choice) {
+        activeFirecrawlTools = choice as string[];
+        pi.appendEntry(STATE_KEY, activeFirecrawlTools);
+        updateActiveTools();
+        ctx.ui.notify(`Firecrawl tools updated`, "info");
+      }
+    },
+  });
+
+  // --- Status Indicator ---
+
+  pi.on("tool_execution_start", async (event, ctx) => {
+    if (event.toolName?.startsWith("firecrawl_")) {
+      ctx.ui.setStatus("firecrawl", "🔥 Firecrawl active...");
+    }
+  });
+
+  pi.on("tool_execution_end", async (event, ctx) => {
+    if (event.toolName?.startsWith("firecrawl_")) {
+      ctx.ui.setStatus("firecrawl", "");
+    }
+  });
+}

package/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "pi-firecrawl",
+  "version": "0.1.0",
+  "description": "Firecrawl web scraping tools for Pi coding agent - scrape, crawl, map, and search with Firecrawl API v2",
+  "keywords": ["pi-package", "firecrawl", "web-scraping", "crawl", "pi-extension"],
+  "license": "MIT",
+  "type": "module",
+  "pi": {
+    "extensions": ["./extensions"],
+    "video": "",
+    "image": ""
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "typebox": "*"
+  }
+}