pi-cloudflare-browser-run 0.1.8 → 0.2.0

package/README.md

@@ -16,6 +16,8 @@ the v4 REST API directly.
 | `browse` | fetch a public URL, return clean **markdown** text (default; also `screenshot` / `pdf` actions) |
 | `screenshot` | save a PNG of the page locally, returns the file path |
 | `pdf` | save a PDF of the page locally, returns the file path |
+| `crawl` | multi-page crawl via Browser Run `/crawl` (markdown by default; waits for small limits) |
+| `crawl_status` | poll / fetch results for a crawl job id |
 
 ## Install
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pi-cloudflare-browser-run",
-  "version": "0.1.8",
-  "description": "Pi extension: web browsing tools (markdown / screenshot / pdf) powered by Cloudflare Browser Run. Headless Chrome on CF's network — JS-rendered pages, login sessions, WebMCP-capable.",
+  "version": "0.2.0",
+  "description": "Pi extension: web browsing tools (markdown / screenshot / pdf / crawl) powered by Cloudflare Browser Run. Headless Chrome on CF's network \u2014 JS-rendered pages, login sessions, WebMCP-capable.",
   "type": "module",
   "engines": {
     "node": ">=20"
@@ -31,7 +31,7 @@
   "devDependencies": {
     "@earendil-works/pi-coding-agent": "^0.86.0",
     "@types/node": "^26.6.2",
-    "tsx": "^4.23.13",
+    "tsx": "^4.23.15",
     "typescript": "^7.0.2"
   },
   "repository": {

package/src/api.ts

@@ -5,6 +5,8 @@
 //
 // Endpoints (v4 API):
 //   POST /accounts/{ACCOUNT_ID}/browser-rendering/{screenshot|markdown|pdf}
+//   POST /accounts/{ACCOUNT_ID}/browser-rendering/crawl
+//   GET  /accounts/{ACCOUNT_ID}/browser-rendering/crawl/{jobId}
 //
 // Security: URL strictness first — only public http(s), localhost / private
 // / reserved IPs / IPv6 literals / userinfo are rejected (SSRF guard).
@@ -133,3 +135,202 @@ export async function browserRunAction(config: BrowserRunConfig, action: Action,
     return { ok: false, error: e instanceof Error ? e.message : String(e) };
   }
 }
+
+// ---------------------------------------------------------------------------
+// /crawl — async multi-page scrape (start + status/poll)
+// Docs: https://developers.cloudflare.com/browser-rendering/rest-api/crawl-endpoint/
+// ---------------------------------------------------------------------------
+
+export type CrawlSource = "all" | "sitemaps" | "links";
+export type CrawlFormat = "html" | "markdown" | "json";
+
+export interface CrawlStartParams {
+  url: string;
+  limit?: number;
+  depth?: number;
+  formats?: CrawlFormat[];
+  render?: boolean;
+  source?: CrawlSource;
+  includePatterns?: string[];
+  excludePatterns?: string[];
+  includeExternalLinks?: boolean;
+  includeSubdomains?: boolean;
+  /** Content Signals purpose declarations. */
+  crawlPurposes?: Array<"search" | "ai-input" | "ai-train">;
+  contentUse?: "reference" | "full";
+}
+
+export interface CrawlStatusQuery {
+  jobId: string;
+  /** Max records to return (omit for a full page of results). */
+  limit?: number;
+  cursor?: string | number;
+  /** Filter records: queued|completed|disallowed|skipped|errored|cancelled */
+  status?: string;
+}
+
+export type CrawlJobStatus =
+  | "running"
+  | "completed"
+  | "errored"
+  | "cancelled_due_to_timeout"
+  | "cancelled_due_to_limits"
+  | "cancelled_by_user"
+  | string;
+
+export interface CrawlRecord {
+  url: string;
+  status: string;
+  markdown?: string;
+  html?: string;
+  json?: unknown;
+  metadata?: Record<string, unknown>;
+}
+
+export interface CrawlJobResult {
+  id: string;
+  status: CrawlJobStatus;
+  browserSecondsUsed?: number;
+  total?: number;
+  finished?: number;
+  records?: CrawlRecord[];
+  cursor?: string | number;
+}
+
+export type CrawlApiResult =
+  | { ok: true; jobId: string; job?: CrawlJobResult }
+  | { ok: false; error: string };
+
+const DEFAULT_CRAWL_FORMATS: CrawlFormat[] = ["markdown"];
+const CRAWL_POLL_INTERVAL_MS = 5_000;
+const CRAWL_POLL_MAX_ATTEMPTS = 60; // 5 min
+const SMALL_CRAWL_LIMIT = 20;
+
+function crawlEndpoint(config: BrowserRunConfig, jobId?: string): string {
+  const base = (config.apiBase ?? DEFAULT_API_BASE).replace(/\/$/, "");
+  const root = `${base}/accounts/${encodeURIComponent(config.accountId)}/browser-rendering/crawl`;
+  return jobId ? `${root}/${encodeURIComponent(jobId)}` : root;
+}
+
+function crawlAuthHeaders(config: BrowserRunConfig): Record<string, string> {
+  return {
+    Authorization: `Bearer ${config.apiToken}`,
+    "Content-Type": "application/json",
+    "user-agent":
+      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+  };
+}
+
+async function readCfJson(res: Response): Promise<{ success?: boolean; result?: unknown; errors?: unknown }> {
+  const text = await res.text();
+  try {
+    return JSON.parse(text) as { success?: boolean; result?: unknown; errors?: unknown };
+  } catch {
+    throw new Error(`non-JSON response (HTTP ${res.status}): ${text.slice(0, 200)}`);
+  }
+}
+
+/** Start a crawl job; returns the job id immediately. */
+export async function crawlStart(config: BrowserRunConfig, params: CrawlStartParams): Promise<CrawlApiResult> {
+  try {
+    const url = assertSafeUrl(params.url);
+    const body: Record<string, unknown> = {
+      url,
+      formats: params.formats?.length ? params.formats : DEFAULT_CRAWL_FORMATS,
+    };
+    if (params.limit !== undefined) body.limit = params.limit;
+    if (params.depth !== undefined) body.depth = params.depth;
+    if (params.render !== undefined) body.render = params.render;
+    if (params.source !== undefined) body.source = params.source;
+    if (params.crawlPurposes !== undefined) body.crawlPurposes = params.crawlPurposes;
+    if (params.contentUse !== undefined) body.contentUse = params.contentUse;
+    const options: Record<string, unknown> = {};
+    if (params.includePatterns?.length) options.includePatterns = params.includePatterns;
+    if (params.excludePatterns?.length) options.excludePatterns = params.excludePatterns;
+    if (params.includeExternalLinks !== undefined) options.includeExternalLinks = params.includeExternalLinks;
+    if (params.includeSubdomains !== undefined) options.includeSubdomains = params.includeSubdomains;
+    if (Object.keys(options).length) body.options = options;
+
+    const res = await fetch(crawlEndpoint(config), {
+      method: "POST",
+      headers: crawlAuthHeaders(config),
+      body: JSON.stringify(body),
+    });
+    const data = await readCfJson(res);
+    if (!res.ok || data.success === false) {
+      const err = data.errors ? JSON.stringify(data.errors).slice(0, 300) : `HTTP ${res.status}`;
+      return { ok: false, error: `crawl start failed: ${err}` };
+    }
+    const jobId = typeof data.result === "string" ? data.result : (data.result as { id?: string } | undefined)?.id;
+    if (!jobId) return { ok: false, error: "crawl start failed: missing job id in response" };
+    return { ok: true, jobId };
+  } catch (e) {
+    return { ok: false, error: e instanceof Error ? e.message : String(e) };
+  }
+}
+
+/** Fetch crawl job status / records. */
+export async function crawlStatus(config: BrowserRunConfig, query: CrawlStatusQuery): Promise<CrawlApiResult> {
+  try {
+    if (!query.jobId || !String(query.jobId).trim()) {
+      return { ok: false, error: "jobId is required" };
+    }
+    const u = new URL(crawlEndpoint(config, query.jobId));
+    if (query.limit !== undefined) u.searchParams.set("limit", String(query.limit));
+    if (query.cursor !== undefined) u.searchParams.set("cursor", String(query.cursor));
+    if (query.status !== undefined) u.searchParams.set("status", String(query.status));
+
+    const res = await fetch(u.toString(), {
+      method: "GET",
+      headers: crawlAuthHeaders(config),
+    });
+    const data = await readCfJson(res);
+    if (!res.ok || data.success === false) {
+      const err = data.errors ? JSON.stringify(data.errors).slice(0, 300) : `HTTP ${res.status}`;
+      return { ok: false, error: `crawl status failed: ${err}` };
+    }
+    const job = data.result as CrawlJobResult;
+    const jobId = job?.id ?? query.jobId;
+    return { ok: true, jobId, job };
+  } catch (e) {
+    return { ok: false, error: e instanceof Error ? e.message : String(e) };
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+/**
+ * Start a crawl and optionally poll until a terminal status.
+ * `wait` defaults to true when limit is undefined or ≤20, else false.
+ */
+export async function crawl(
+  config: BrowserRunConfig,
+  params: CrawlStartParams & { wait?: boolean },
+): Promise<CrawlApiResult> {
+  const limit = params.limit ?? 10;
+  const wait = params.wait ?? limit <= SMALL_CRAWL_LIMIT;
+  const started = await crawlStart(config, params);
+  if (!started.ok) return started;
+  if (!wait) return started;
+
+  for (let i = 0; i < CRAWL_POLL_MAX_ATTEMPTS; i++) {
+    const light = await crawlStatus(config, { jobId: started.jobId, limit: 1 });
+    if (!light.ok) return light;
+    const status = light.job?.status;
+    if (status && status !== "running") {
+      return crawlStatus(config, { jobId: started.jobId });
+    }
+    await sleep(CRAWL_POLL_INTERVAL_MS);
+  }
+  return {
+    ok: false,
+    error: `crawl job ${started.jobId} did not complete within timeout (still running)`,
+  };
+}
+
+/** Whether wait should default on for a given limit (exported for tests). */
+export function defaultCrawlWait(limit?: number): boolean {
+  return (limit ?? 10) <= SMALL_CRAWL_LIMIT;
+}

package/src/index.ts

@@ -5,6 +5,8 @@
 //   browse(url, action?)       read a page as clean markdown (default)
 //   screenshot(url)            save a PNG of the page, returns local path
 //   pdf(url)                   save a PDF of the page, returns local path
+//   crawl(url, ...)            multi-page crawl via /crawl (optional wait/poll)
+//   crawl_status(jobId, ...)   poll / fetch crawl job results
 //
 // Auth: CLOUDFLARE_API_TOKEN (Browser Rendering:Edit) + CLOUDFLARE_ACCOUNT_ID
 // env vars. No Workers, no proxy — direct v4 REST calls.
@@ -14,7 +16,17 @@ import { Type } from "typebox";
 import { mkdirSync, writeFileSync } from "fs";
 import { tmpdir } from "os";
 import { join } from "path";
-import { browserRunAction, loadConfig, type Action } from "./api.js";
+import {
+  browserRunAction,
+  crawl,
+  crawlStatus,
+  defaultCrawlWait,
+  loadConfig,
+  type Action,
+  type CrawlFormat,
+  type CrawlJobResult,
+  type CrawlRecord,
+} from "./api.js";
 
 const urlSchema = Type.Object({
   url: Type.String({ minLength: 1, description: "Public http(s) URL, e.g. https://example.com" }),
@@ -121,4 +133,141 @@ export default function (pi: ExtensionAPI) {
       return toolResult(`pdf saved to ${file}`);
     },
   });
+
+  const truncateRecord = (rec: CrawlRecord, maxMd = 4_000): CrawlRecord => {
+    const out: CrawlRecord = { url: rec.url, status: rec.status };
+    if (rec.metadata) out.metadata = rec.metadata;
+    if (typeof rec.markdown === "string") {
+      out.markdown =
+        rec.markdown.length > maxMd ? rec.markdown.slice(0, maxMd) + "\n…(truncated)" : rec.markdown;
+    }
+    if (typeof rec.html === "string") {
+      out.html = rec.html.length > maxMd ? rec.html.slice(0, maxMd) + "\n…(truncated)" : rec.html;
+    }
+    if (rec.json !== undefined) out.json = rec.json;
+    return out;
+  };
+
+  const formatCrawlJob = (job: CrawlJobResult, maxRecords = 50): string => {
+    const records = (job.records ?? []).slice(0, maxRecords).map((r) => truncateRecord(r));
+    return JSON.stringify(
+      {
+        id: job.id,
+        status: job.status,
+        total: job.total,
+        finished: job.finished,
+        browserSecondsUsed: job.browserSecondsUsed,
+        cursor: job.cursor,
+        recordCount: records.length,
+        records,
+      },
+      null,
+      2,
+    );
+  };
+
+  const crawlSchema = Type.Object({
+    url: Type.String({ minLength: 1, description: "Starting public http(s) URL to crawl" }),
+    limit: Type.Optional(Type.Number({ description: "Max pages to crawl (default 10)" })),
+    depth: Type.Optional(Type.Number({ description: "Max link depth from the start URL" })),
+    formats: Type.Optional(
+      Type.Array(Type.Union([Type.Literal("markdown"), Type.Literal("html"), Type.Literal("json")]), {
+        description: "Response formats (default [markdown])",
+      }),
+    ),
+    render: Type.Optional(Type.Boolean({ description: "Headless Chrome (true) or fast HTML fetch (false)" })),
+    source: Type.Optional(
+      Type.Union([Type.Literal("all"), Type.Literal("sitemaps"), Type.Literal("links")], {
+        description: "URL discovery source (default all)",
+      }),
+    ),
+    includePatterns: Type.Optional(Type.Array(Type.String(), { description: "Only visit matching URL patterns" })),
+    excludePatterns: Type.Optional(Type.Array(Type.String(), { description: "Skip matching URL patterns" })),
+    wait: Type.Optional(
+      Type.Boolean({
+        description:
+          "Poll until done (default true when limit≤20). If false, returns job id immediately.",
+      }),
+    ),
+  });
+
+  const crawlStatusSchema = Type.Object({
+    jobId: Type.String({ minLength: 1, description: "Crawl job id from crawl" }),
+    limit: Type.Optional(Type.Number({ description: "Max records to return" })),
+    cursor: Type.Optional(Type.Union([Type.String(), Type.Number()], { description: "Pagination cursor" })),
+    status: Type.Optional(
+      Type.String({ description: "Filter records: queued|completed|disallowed|skipped|errored|cancelled" }),
+    ),
+  });
+
+  pi.registerTool({
+    name: "crawl",
+    label: "Crawl",
+    description:
+      "Crawl a public site starting from a URL via Cloudflare Browser Run /crawl. " +
+      "Follows links up to limit/depth and returns markdown (default) per page. " +
+      "For small jobs (limit≤20) waits for completion by default; for larger jobs " +
+      "returns a job id — then use crawl_status.",
+    promptSnippet: "crawl a public site (multi-page markdown)",
+    promptGuidelines: [
+      "Use crawl when you need several pages from one site; use browse for a single URL.",
+      "Prefer small limits (≤20) so wait can finish in one call; use crawl_status for long jobs.",
+    ],
+    parameters: crawlSchema,
+    executionMode: "sequential",
+    async execute(_toolCallId, params) {
+      const cfg = typeof config === "object" && "apiToken" in config ? config : null;
+      if (!cfg) return toolResult(`crawl unavailable: ${(config as { error: string }).error}`);
+      const limit = typeof params.limit === "number" ? params.limit : undefined;
+      const wait = typeof params.wait === "boolean" ? params.wait : defaultCrawlWait(limit);
+      const r = await crawl(cfg, {
+        url: params.url,
+        limit,
+        depth: typeof params.depth === "number" ? params.depth : undefined,
+        formats: (params.formats as CrawlFormat[] | undefined) ?? ["markdown"],
+        render: typeof params.render === "boolean" ? params.render : undefined,
+        source: params.source as "all" | "sitemaps" | "links" | undefined,
+        includePatterns: params.includePatterns as string[] | undefined,
+        excludePatterns: params.excludePatterns as string[] | undefined,
+        wait,
+      });
+      if (!r.ok) return toolResult(`crawl failed: ${"error" in r ? r.error : "unknown"}`);
+      if (r.job) return toolResult(formatCrawlJob(r.job));
+      return toolResult(
+        JSON.stringify(
+          {
+            jobId: r.jobId,
+            status: "running",
+            message: "Crawl started. Poll with crawl_status(jobId) until status is completed.",
+          },
+          null,
+          2,
+        ),
+      );
+    },
+  });
+
+  pi.registerTool({
+    name: "crawl_status",
+    label: "Crawl status",
+    description:
+      "Check status or fetch results of a Cloudflare Browser Run crawl job by id. " +
+      "Supports limit/cursor pagination and status filters on records.",
+    promptSnippet: "poll crawl job status / results",
+    parameters: crawlStatusSchema,
+    executionMode: "sequential",
+    async execute(_toolCallId, params) {
+      const cfg = typeof config === "object" && "apiToken" in config ? config : null;
+      if (!cfg) return toolResult(`crawl_status unavailable: ${(config as { error: string }).error}`);
+      const r = await crawlStatus(cfg, {
+        jobId: params.jobId,
+        limit: typeof params.limit === "number" ? params.limit : undefined,
+        cursor: params.cursor as string | number | undefined,
+        status: typeof params.status === "string" ? params.status : undefined,
+      });
+      if (!r.ok) return toolResult(`crawl_status failed: ${"error" in r ? r.error : "unknown"}`);
+      if (r.job) return toolResult(formatCrawlJob(r.job));
+      return toolResult(JSON.stringify({ jobId: r.jobId }, null, 2));
+    },
+  });
 }