webcrawl 1.4.3 → 1.4.5

package/README.md

@@ -14,7 +14,7 @@
 | Target | Returns | Requirements |
 |---|---|---|
 | **Websites & JS SPAs** (`web`) | Clean Markdown, semantic headings, tables, code blocks, links, OpenGraph/JSON-LD, and SHA-256 content hashes | Zero-key local (`impit` TLS impersonation + local Chrome) or [`webcrawl.app`](https://webcrawl.app) cloud stealth browsers |
-| **PDF & Office Docs** (`pdf-office`) | Page-numbered Markdown, tables, slide notes, and automatic OCR on scanned PDF pages (`pagesOcr` + honest `pagesSkipped`) | Built-in (`@firecrawl/pdf-inspector`, `pdf-parse`, `mammoth`, `officeparser`, `tesseract.js`) |
+| **PDF & Office Docs** (`pdf-office`) | Page-numbered Markdown, tables, slide notes, and automatic OCR on scanned PDF pages (`pagesOcr` + honest `pagesSkipped`) | Built-in PDF inspector plus `pdf-parse`, `mammoth`, `officeparser`, and `tesseract.js` |
 | **YouTube & Media** (`youtube-media`) | Timestamped transcript Markdown, video title/channel/duration metadata, and spoken audio transcription | Built-in zero-key YouTube multi-client caption extractor; optional audio transcription provider for raw MP3/MP4 |
 | **Shopping & Products** (`shopping`) | Multi-retailer price comparison, currency-normalized offers, stock status, ratings, review counts, and shipping | Zero-key local HTML/JSON-LD product extractor (`-c shopping`); optional `SERPER_API_KEY` |
 | **Flights & Hotels** (`travel`) | Structured flight routes (`origin`, `destination`, `airline`, `stops`, `price`) and hotel nightly rates, stars, and guest ratings | Zero-key local search + schema.org extractor (`-c flights` / `-c hotels`) |
@@ -101,7 +101,7 @@ Webcrawl Anything uses the **exact same TypeScript engine (`src/`)** locally and
 | **Core Engine & Target Plugins** | 100% shared (`src/engine`, `src/targets`, `src/extract`) | 100% shared |
 | **PDF, Office, YouTube & CVE Parsers** | Built-in local execution | Hosted + background async workers (`/v1/runs/:id`) |
 | **Anti-Bot & Cloudflare Bypass** | Local `impit` TLS + local Chrome | Managed residential stealth browser fleet + Cloudflare rendering |
-| **Search Fan-Out** | Built-in DuckDuckGo/Bing + BYO keys | Managed multi-provider fan-out (Serper, Brave, Jina, Tavily) |
+| **Search Fan-Out** | Built-in DuckDuckGo/Bing plus optional API keys | Managed multi-provider fan-out |
 | **Global Content Cache** | Local in-memory LRU | Sub-50ms global SHA-256 content-addressed cache |
 
 To connect your CLI and MCP server to [`webcrawl.app`](https://webcrawl.app) cloud:

package/dist/cli.d.ts

@@ -17,7 +17,9 @@ export interface Deps {
     /** Working directory override for `webcrawl setup` tests. */
     cwd?: string;
 }
-export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  monitor [subcmd]    Watch pages/sites: create <url...>, list, get <id>, check <id>, pause <id>, resume <id>, delete <id>\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in via browser OAuth (default for login; no need to pass it)\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --scope <s|u|w>     Monitor scope: page (default), urls, or site\n  --interval <mins>   Monitor check interval in minutes (default 60)\n  --watch-for <text>  What to watch for (e.g. \"Pricing tiers and discounts\")\n  --condition <expr>  Trigger rule expression (e.g. 'price < 50', 'content contains \"In Stock\"')\n  --notify-email <e>  Alert email address when triggered (or 'true' for account email)\n  --webhook <url>     Alert webhook URL when triggered\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
+export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  monitor [subcmd]    Watch pages/sites: create <url...>, list, get <id>, check <id>, pause <id>, resume <id>, delete <id>\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in via browser OAuth (default for login; no need to pass it)\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --scope <s|u|w>     Monitor scope: page (default), urls, or site\n  --interval <mins>   Monitor check interval in minutes (default 60)\n  --watch-for <text>  What to watch for (e.g. \"Pricing tiers and discounts\")\n  --condition <expr>  Trigger rule expression (e.g. 'price < 50', 'content contains \"In Stock\"')\n  --notify-email <e>  Alert email address when triggered (or 'true' for account email)\n  --webhook <url>     Alert webhook URL when triggered\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -v, --version       Print the installed package version\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
+/** Installed `webcrawl` version, read from package.json at runtime. */
+export declare function packageVersion(): string;
 /** Self-contained (no network assets) page served on the loopback callback. */
 export declare function callbackPage(opts: {
     kind: "success" | "error" | "info";

package/dist/cli.js

@@ -2,7 +2,9 @@ import { execFile } from "node:child_process";
 import crypto from "node:crypto";
 import { readFileSync } from "node:fs";
 import { createServer } from "node:http";
+import { dirname, join } from "node:path";
 import { createInterface } from "node:readline";
+import { fileURLToPath } from "node:url";
 import { parseArgs } from "node:util";
 import { WebcrawlClient, WebcrawlError, configPath, describeError, readConfigFile, resolveConfig, writeConfigFile, } from "@webcrawl/client";
 import { formatCrawl, formatCredits, formatMap, formatMonitor, formatRerank, formatScrape, formatSearch, formatTargets, } from "./format.js";
@@ -45,12 +47,14 @@ Options
   --webhook <url>     Alert webhook URL when triggered
   --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)
   --schema-profile    MCP schema profile for setup: compact (default) or full
+  -v, --version       Print the installed package version
   -h, --help
 
 Config order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json`;
 const OPTIONS = {
     json: { type: "boolean" },
     help: { type: "boolean", short: "h" },
+    version: { type: "boolean", short: "v" },
     "api-key": { type: "string" },
     url: { type: "string" },
     mode: { type: "string" },
@@ -84,6 +88,15 @@ const OPTIONS = {
 };
 class UsageError extends Error {
 }
+/** Installed `webcrawl` version, read from package.json at runtime. */
+export function packageVersion() {
+    const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
+    const pkg = JSON.parse(readFileSync(pkgPath, "utf8"));
+    if (!pkg.version) {
+        throw new Error("package.json is missing a version field");
+    }
+    return pkg.version;
+}
 function int(v, name) {
     if (v === undefined) {
         return undefined;
@@ -356,6 +369,10 @@ export async function main(argv, io, deps = {}) {
     }
     const { values: v, positionals } = parsed;
     const [cmd, ...rest] = positionals;
+    if (v.version) {
+        io.out(`${packageVersion()}\n`);
+        return 0;
+    }
     if (v.help || !cmd || cmd === "help") {
         io.out(`${HELP}\n`);
         return cmd || v.help ? 0 : 2;
@@ -467,8 +484,15 @@ export async function main(argv, io, deps = {}) {
                     }
                 }
                 const res = await client.extract({ url: need("url"), prompt: v.prompt, schema });
+                if (res.success === false && !v.json) {
+                    const msg = typeof res.error === "string" && res.error.trim()
+                        ? res.error
+                        : JSON.stringify(res.data ?? res, null, 2);
+                    io.err(`${msg}\n`);
+                    return 1;
+                }
                 emit(res, JSON.stringify(res.data ?? res, null, 2));
-                return 0;
+                return res.success === false ? 1 : 0;
             }
             case "rerank": {
                 const query = need("query");

package/dist/core/engine/chrome-cdp.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Minimal Chrome DevTools Protocol client for local JS rendering.
+ *
+ * Drive a real Chrome over CDP instead of dump-dom, wait until the DOM is
+ * actually ready, and always bound the session with a hard timeout. No
+ * puppeteer/playwright dependency.
+ */
+import { spawn } from "node:child_process";
+import { type ChromeHeadlessMode } from "./chrome-launch.js";
+/** Chrome spawned but CDP never became ready. `hung` is true when the process was still alive. */
+export declare class ChromeConnectError extends Error {
+    readonly hung: boolean;
+    constructor(message: string, hung?: boolean);
+}
+export interface CdpSnapshot {
+    html: string;
+    title: string;
+    textLen: number;
+    readyState: string;
+}
+export interface CdpRenderOptions {
+    timeoutMs: number;
+    /** CDP launch/connect budget. Defaults to 4s (`WEBCRAWL_CHROME_CONNECT_TIMEOUT_MS`). */
+    connectTimeoutMs?: number;
+    headless: ChromeHeadlessMode;
+    /** Injected for tests. */
+    spawnChrome?: typeof spawn;
+}
+export declare function renderUrlViaCdp(chromePath: string, url: string, opts: CdpRenderOptions): Promise<CdpSnapshot>;

package/dist/core/engine/chrome-cdp.js

@@ -0,0 +1,339 @@
+/**
+ * Minimal Chrome DevTools Protocol client for local JS rendering.
+ *
+ * Drive a real Chrome over CDP instead of dump-dom, wait until the DOM is
+ * actually ready, and always bound the session with a hard timeout. No
+ * puppeteer/playwright dependency.
+ */
+import { spawn } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { chromeCdpArgs, killProcessGroup, resolveChromeConnectTimeoutMs, } from "./chrome-launch.js";
+/** Chrome spawned but CDP never became ready. `hung` is true when the process was still alive. */
+export class ChromeConnectError extends Error {
+    hung;
+    constructor(message, hung = false) {
+        super(message);
+        this.name = "ChromeConnectError";
+        this.hung = hung;
+    }
+}
+const SNAPSHOT_JS = `(() => {
+  const body = document.body;
+  const text = body && typeof body.innerText === "string" ? body.innerText.trim() : "";
+  return {
+    html: document.documentElement ? document.documentElement.outerHTML : "",
+    title: document.title || "",
+    textLen: text.length,
+    readyState: document.readyState || "",
+  };
+})()`;
+/** mutationIdle (200ms quiet) plus readyState complete, capped. */
+const WAIT_FOR_CONTENT_JS = `(async () => {
+  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+  const snap = () => {
+    const body = document.body;
+    const text = body && typeof body.innerText === "string" ? body.innerText.trim() : "";
+    return {
+      html: document.documentElement ? document.documentElement.outerHTML : "",
+      title: document.title || "",
+      textLen: text.length,
+      readyState: document.readyState || "",
+      elemCount: document.getElementsByTagName("*").length,
+    };
+  };
+  let lastMut = Date.now();
+  let observer = null;
+  try {
+    observer = new MutationObserver(() => { lastMut = Date.now(); });
+    if (document.documentElement) {
+      observer.observe(document.documentElement, {
+        subtree: true,
+        childList: true,
+        characterData: true,
+      });
+    }
+  } catch (_) { /* ignore */ }
+  const start = Date.now();
+  let last = snap();
+  while (Date.now() - start < 2500) {
+    await sleep(150);
+    const now = snap();
+    const quiet = Date.now() - lastMut >= 200;
+    const grown = now.elemCount > last.elemCount + 2 || now.textLen > last.textLen + 20;
+    last = now;
+    if (
+      now.readyState === "complete" &&
+      now.textLen >= 80 &&
+      quiet &&
+      !grown
+    ) break;
+  }
+  try { observer && observer.disconnect(); } catch (_) { /* ignore */ }
+  return last;
+})()`;
+function childStillRunning(child) {
+    return Boolean(child && child.pid && child.exitCode == null && !child.killed);
+}
+export async function renderUrlViaCdp(chromePath, url, opts) {
+    const timeoutMs = Math.max(1_500, opts.timeoutMs);
+    const connectTimeoutMs = resolveChromeConnectTimeoutMs(opts.connectTimeoutMs);
+    const profile = mkdtempSync(join(tmpdir(), "wc-cdp-"));
+    const port = 40000 + Math.floor(Math.random() * 20000);
+    const spawnFn = opts.spawnChrome ?? spawn;
+    const args = chromeCdpArgs({
+        port,
+        profile,
+        headless: opts.headless,
+        binaryPath: chromePath,
+    });
+    let child;
+    let cdpReady = false;
+    const ac = new AbortController();
+    // Page budget covers navigation after connect; connect has its own deadline.
+    const timer = setTimeout(() => ac.abort(), timeoutMs);
+    try {
+        child = spawnFn(chromePath, args, {
+            stdio: ["ignore", "ignore", "pipe"],
+            detached: true,
+        });
+        if (!child.pid) {
+            throw new Error("Chrome failed to start (no pid)");
+        }
+        child.once("exit", () => {
+            if (!cdpReady && !ac.signal.aborted)
+                ac.abort();
+        });
+        const wsUrl = await waitForPageWebSocket(port, connectTimeoutMs, ac.signal);
+        cdpReady = true;
+        const snapshot = await withCdpSession(wsUrl, url, timeoutMs, ac.signal);
+        if (!snapshot.html || snapshot.html.length < 40) {
+            throw new Error("CDP returned empty HTML");
+        }
+        return snapshot;
+    }
+    catch (err) {
+        if (!cdpReady) {
+            const hung = childStillRunning(child);
+            const detail = err instanceof Error ? err.message : String(err);
+            throw new ChromeConnectError(hung
+                ? `Chrome did not respond over CDP within ${connectTimeoutMs}ms`
+                : `Chrome CDP failed to start (${detail})`, hung);
+        }
+        if (ac.signal.aborted) {
+            throw new Error(`CDP timed out after ${timeoutMs}ms`);
+        }
+        throw err;
+    }
+    finally {
+        clearTimeout(timer);
+        if (child?.pid)
+            killProcessGroup(child.pid, child);
+        try {
+            rmSync(profile, { recursive: true, force: true });
+        }
+        catch {
+            /* leftover profile is in /tmp */
+        }
+    }
+}
+async function waitForPageWebSocket(port, connectTimeoutMs, signal) {
+    const deadline = Date.now() + connectTimeoutMs;
+    let lastErr = "CDP HTTP endpoint never became ready";
+    while (Date.now() < deadline) {
+        if (signal.aborted)
+            throw new Error("Chrome CDP timed out while starting");
+        try {
+            const res = await fetch(`http://127.0.0.1:${port}/json/list`, {
+                signal: AbortSignal.timeout(400),
+            });
+            if (res.ok) {
+                const json = (await res.json());
+                const page = (Array.isArray(json) ? json : []).find((t) => t.webSocketDebuggerUrl && (t.type === "page" || !t.type));
+                if (page?.webSocketDebuggerUrl)
+                    return page.webSocketDebuggerUrl;
+                lastErr = "CDP /json/list had no page websocket";
+            }
+            else {
+                lastErr = `CDP /json/list HTTP ${res.status}`;
+            }
+        }
+        catch (err) {
+            lastErr = err instanceof Error ? err.message : String(err);
+        }
+        await delay(80, signal);
+    }
+    throw new Error(lastErr);
+}
+async function withCdpSession(wsUrl, url, timeoutMs, signal) {
+    const client = await CdpClient.connect(wsUrl, signal);
+    const onAbort = () => client.close();
+    signal.addEventListener("abort", onAbort);
+    try {
+        await client.send("Page.enable");
+        await client.send("Runtime.enable");
+        const load = client.waitForEvent("Page.loadEventFired", Math.min(timeoutMs, 12_000));
+        const nav = await client.send("Page.navigate", { url });
+        if (nav.errorText) {
+            throw new Error(`Page.navigate failed: ${String(nav.errorText)}`);
+        }
+        try {
+            await load;
+        }
+        catch {
+            /* take a snapshot anyway if navigation load times out */
+        }
+        let snapshot = await evaluateSnapshot(client, WAIT_FOR_CONTENT_JS);
+        if (!snapshot || snapshot.textLen < 40 || snapshot.html.length < 80) {
+            snapshot = await evaluateSnapshot(client, SNAPSHOT_JS);
+        }
+        if (!snapshot)
+            throw new Error("Runtime.evaluate returned no snapshot");
+        return snapshot;
+    }
+    finally {
+        signal.removeEventListener("abort", onAbort);
+        client.close();
+    }
+}
+async function evaluateSnapshot(client, expression) {
+    const result = await client.send("Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true });
+    const value = result.result?.value;
+    if (!value || typeof value.html !== "string")
+        return null;
+    return {
+        html: value.html,
+        title: String(value.title ?? ""),
+        textLen: Number(value.textLen ?? 0),
+        readyState: String(value.readyState ?? ""),
+    };
+}
+class CdpClient {
+    ws;
+    nextId = 0;
+    pending = new Map();
+    eventWaiters = [];
+    constructor(ws) {
+        this.ws = ws;
+        ws.addEventListener("message", (ev) => {
+            let msg;
+            try {
+                msg = JSON.parse(String(ev.data));
+            }
+            catch {
+                return;
+            }
+            if (msg.id != null) {
+                const p = this.pending.get(msg.id);
+                if (p) {
+                    this.pending.delete(msg.id);
+                    if (msg.error) {
+                        p.reject(new Error(msg.error.message || `CDP error ${msg.error.code ?? ""}`));
+                    }
+                    else {
+                        p.resolve(msg.result ?? {});
+                    }
+                }
+                return;
+            }
+            if (msg.method) {
+                for (const w of [...this.eventWaiters]) {
+                    if (w.method === msg.method) {
+                        this.eventWaiters = this.eventWaiters.filter((x) => x !== w);
+                        w.resolve();
+                    }
+                }
+            }
+        });
+    }
+    static connect(url, signal) {
+        return new Promise((resolve, reject) => {
+            const ws = new WebSocket(url);
+            const onAbort = () => {
+                try {
+                    ws.close();
+                }
+                catch {
+                    /* ignore */
+                }
+                reject(new Error("Chrome CDP websocket timed out"));
+            };
+            if (signal.aborted) {
+                onAbort();
+                return;
+            }
+            signal.addEventListener("abort", onAbort, { once: true });
+            ws.addEventListener("open", () => {
+                signal.removeEventListener("abort", onAbort);
+                resolve(new CdpClient(ws));
+            });
+            ws.addEventListener("error", () => {
+                signal.removeEventListener("abort", onAbort);
+                reject(new Error("Chrome CDP websocket failed"));
+            });
+        });
+    }
+    send(method, params = {}) {
+        const id = ++this.nextId;
+        return new Promise((resolve, reject) => {
+            this.pending.set(id, { resolve, reject });
+            try {
+                this.ws.send(JSON.stringify({ id, method, params }));
+            }
+            catch (err) {
+                this.pending.delete(id);
+                reject(err instanceof Error ? err : new Error(String(err)));
+            }
+        });
+    }
+    waitForEvent(method, timeoutMs) {
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.eventWaiters = this.eventWaiters.filter((w) => w !== waiter);
+                reject(new Error(`Timed out waiting for ${method}`));
+            }, timeoutMs);
+            const waiter = {
+                method,
+                resolve: () => {
+                    clearTimeout(timer);
+                    resolve();
+                },
+                reject: (err) => {
+                    clearTimeout(timer);
+                    reject(err);
+                },
+            };
+            this.eventWaiters.push(waiter);
+        });
+    }
+    close() {
+        for (const p of this.pending.values()) {
+            p.reject(new Error("CDP session closed"));
+        }
+        this.pending.clear();
+        for (const w of this.eventWaiters) {
+            w.reject(new Error("CDP session closed"));
+        }
+        this.eventWaiters = [];
+        try {
+            this.ws.close();
+        }
+        catch {
+            /* ignore */
+        }
+    }
+}
+function delay(ms, signal) {
+    return new Promise((resolve, reject) => {
+        if (signal.aborted) {
+            reject(new Error("aborted"));
+            return;
+        }
+        const t = setTimeout(resolve, ms);
+        signal.addEventListener("abort", () => {
+            clearTimeout(t);
+            reject(new Error("aborted"));
+        }, { once: true });
+    });
+}

package/dist/core/engine/chrome-launch.d.ts

@@ -0,0 +1,49 @@
+import type { ChildProcess } from "node:child_process";
+export declare const DUMP_MAX_BYTES: number;
+/** Page navigation + DOM-wait budget once Chrome is answering over CDP. */
+export declare const DEFAULT_TIMEOUT_MS = 18000;
+/**
+ * How long to wait for Chrome to accept CDP after spawn. Kept short and
+ * separate from `DEFAULT_TIMEOUT_MS` so a hung binary fails fast while a
+ * slow but working JS page (often ~7s) still has time to render.
+ */
+export declare const DEFAULT_CONNECT_TIMEOUT_MS = 4000;
+export declare const VIRTUAL_TIME_MS = 10000;
+/** `opts.timeoutMs`, else `WEBCRAWL_CHROME_TIMEOUT_MS`, else 18s. */
+export declare function resolveChromeTimeoutMs(override?: number): number;
+/** `opts.connectTimeoutMs`, else `WEBCRAWL_CHROME_CONNECT_TIMEOUT_MS`, else 4s. */
+export declare function resolveChromeConnectTimeoutMs(override?: number): number;
+export type ChromeHeadlessMode = "new" | "old" | "shell";
+export declare function killProcessGroup(pid: number | undefined, child?: Pick<ChildProcess, "kill">): void;
+export declare function headlessFlag(mode: ChromeHeadlessMode, binaryPath: string): string[];
+export declare function chromeCdpArgs(opts: {
+    port: number;
+    profile: string;
+    headless: ChromeHeadlessMode;
+    binaryPath?: string;
+}): string[];
+export declare function chromeDumpDomArgs(opts: {
+    profile: string;
+    headless: ChromeHeadlessMode;
+    url: string;
+    timeoutMs: number;
+    binaryPath?: string;
+}): string[];
+/**
+ * First line of Chrome's DevToolsActivePort file is the chosen port when
+ * `--remote-debugging-port=0`.
+ */
+export declare function parseDevToolsActivePort(text: string): number | null;
+export declare function defaultChromeInstallPaths(): string[];
+/** Playwright's chrome-headless-shell, if a previous `playwright install` left it on disk. */
+export declare function findPlaywrightHeadlessShell(): string[];
+/**
+ * Path to a real installed Chrome/Chromium, or null.
+ * Priority: WEBCRAWL_CHROME_PATH, INTERACT_CHROME_PATH, CHROME_PATH, then per-OS
+ * defaults (same locations `webcrawl doctor` checks). Env overrides are returned
+ * even if the file is missing so tests can point at a stub binary.
+ */
+export declare function installedChromePath(): string | null;
+export declare function isLocalChromeAvailable(): boolean;
+/** Distinct binaries to try, env override first, then system Chrome, then headless-shell. */
+export declare function discoverChromeBinaries(): string[];