webcrawl 1.4.4 → 1.4.5

package/dist/cli.d.ts

@@ -17,7 +17,9 @@ export interface Deps {
     /** Working directory override for `webcrawl setup` tests. */
     cwd?: string;
 }
-export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  monitor [subcmd]    Watch pages/sites: create <url...>, list, get <id>, check <id>, pause <id>, resume <id>, delete <id>\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in via browser OAuth (default for login; no need to pass it)\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --scope <s|u|w>     Monitor scope: page (default), urls, or site\n  --interval <mins>   Monitor check interval in minutes (default 60)\n  --watch-for <text>  What to watch for (e.g. \"Pricing tiers and discounts\")\n  --condition <expr>  Trigger rule expression (e.g. 'price < 50', 'content contains \"In Stock\"')\n  --notify-email <e>  Alert email address when triggered (or 'true' for account email)\n  --webhook <url>     Alert webhook URL when triggered\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
+export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  monitor [subcmd]    Watch pages/sites: create <url...>, list, get <id>, check <id>, pause <id>, resume <id>, delete <id>\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in via browser OAuth (default for login; no need to pass it)\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --scope <s|u|w>     Monitor scope: page (default), urls, or site\n  --interval <mins>   Monitor check interval in minutes (default 60)\n  --watch-for <text>  What to watch for (e.g. \"Pricing tiers and discounts\")\n  --condition <expr>  Trigger rule expression (e.g. 'price < 50', 'content contains \"In Stock\"')\n  --notify-email <e>  Alert email address when triggered (or 'true' for account email)\n  --webhook <url>     Alert webhook URL when triggered\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -v, --version       Print the installed package version\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
+/** Installed `webcrawl` version, read from package.json at runtime. */
+export declare function packageVersion(): string;
 /** Self-contained (no network assets) page served on the loopback callback. */
 export declare function callbackPage(opts: {
     kind: "success" | "error" | "info";

package/dist/cli.js

@@ -2,7 +2,9 @@ import { execFile } from "node:child_process";
 import crypto from "node:crypto";
 import { readFileSync } from "node:fs";
 import { createServer } from "node:http";
+import { dirname, join } from "node:path";
 import { createInterface } from "node:readline";
+import { fileURLToPath } from "node:url";
 import { parseArgs } from "node:util";
 import { WebcrawlClient, WebcrawlError, configPath, describeError, readConfigFile, resolveConfig, writeConfigFile, } from "@webcrawl/client";
 import { formatCrawl, formatCredits, formatMap, formatMonitor, formatRerank, formatScrape, formatSearch, formatTargets, } from "./format.js";
@@ -45,12 +47,14 @@ Options
   --webhook <url>     Alert webhook URL when triggered
   --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)
   --schema-profile    MCP schema profile for setup: compact (default) or full
+  -v, --version       Print the installed package version
   -h, --help
 
 Config order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json`;
 const OPTIONS = {
     json: { type: "boolean" },
     help: { type: "boolean", short: "h" },
+    version: { type: "boolean", short: "v" },
     "api-key": { type: "string" },
     url: { type: "string" },
     mode: { type: "string" },
@@ -84,6 +88,15 @@ const OPTIONS = {
 };
 class UsageError extends Error {
 }
+/** Installed `webcrawl` version, read from package.json at runtime. */
+export function packageVersion() {
+    const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
+    const pkg = JSON.parse(readFileSync(pkgPath, "utf8"));
+    if (!pkg.version) {
+        throw new Error("package.json is missing a version field");
+    }
+    return pkg.version;
+}
 function int(v, name) {
     if (v === undefined) {
         return undefined;
@@ -356,6 +369,10 @@ export async function main(argv, io, deps = {}) {
     }
     const { values: v, positionals } = parsed;
     const [cmd, ...rest] = positionals;
+    if (v.version) {
+        io.out(`${packageVersion()}\n`);
+        return 0;
+    }
     if (v.help || !cmd || cmd === "help") {
         io.out(`${HELP}\n`);
         return cmd || v.help ? 0 : 2;

package/dist/core/engine/chrome-cdp.d.ts

@@ -7,6 +7,11 @@
  */
 import { spawn } from "node:child_process";
 import { type ChromeHeadlessMode } from "./chrome-launch.js";
+/** Chrome spawned but CDP never became ready. `hung` is true when the process was still alive. */
+export declare class ChromeConnectError extends Error {
+    readonly hung: boolean;
+    constructor(message: string, hung?: boolean);
+}
 export interface CdpSnapshot {
     html: string;
     title: string;
@@ -15,6 +20,8 @@ export interface CdpSnapshot {
 }
 export interface CdpRenderOptions {
     timeoutMs: number;
+    /** CDP launch/connect budget. Defaults to 4s (`WEBCRAWL_CHROME_CONNECT_TIMEOUT_MS`). */
+    connectTimeoutMs?: number;
     headless: ChromeHeadlessMode;
     /** Injected for tests. */
     spawnChrome?: typeof spawn;

package/dist/core/engine/chrome-cdp.js

@@ -9,7 +9,16 @@ import { spawn } from "node:child_process";
 import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { chromeCdpArgs, killProcessGroup } from "./chrome-launch.js";
+import { chromeCdpArgs, killProcessGroup, resolveChromeConnectTimeoutMs, } from "./chrome-launch.js";
+/** Chrome spawned but CDP never became ready. `hung` is true when the process was still alive. */
+export class ChromeConnectError extends Error {
+    hung;
+    constructor(message, hung = false) {
+        super(message);
+        this.name = "ChromeConnectError";
+        this.hung = hung;
+    }
+}
 const SNAPSHOT_JS = `(() => {
   const body = document.body;
   const text = body && typeof body.innerText === "string" ? body.innerText.trim() : "";
@@ -64,8 +73,12 @@ const WAIT_FOR_CONTENT_JS = `(async () => {
   try { observer && observer.disconnect(); } catch (_) { /* ignore */ }
   return last;
 })()`;
+function childStillRunning(child) {
+    return Boolean(child && child.pid && child.exitCode == null && !child.killed);
+}
 export async function renderUrlViaCdp(chromePath, url, opts) {
     const timeoutMs = Math.max(1_500, opts.timeoutMs);
+    const connectTimeoutMs = resolveChromeConnectTimeoutMs(opts.connectTimeoutMs);
     const profile = mkdtempSync(join(tmpdir(), "wc-cdp-"));
     const port = 40000 + Math.floor(Math.random() * 20000);
     const spawnFn = opts.spawnChrome ?? spawn;
@@ -76,7 +89,9 @@ export async function renderUrlViaCdp(chromePath, url, opts) {
         binaryPath: chromePath,
     });
     let child;
+    let cdpReady = false;
     const ac = new AbortController();
+    // Page budget covers navigation after connect; connect has its own deadline.
     const timer = setTimeout(() => ac.abort(), timeoutMs);
     try {
         child = spawnFn(chromePath, args, {
@@ -86,12 +101,11 @@ export async function renderUrlViaCdp(chromePath, url, opts) {
         if (!child.pid) {
             throw new Error("Chrome failed to start (no pid)");
         }
-        let cdpReady = false;
         child.once("exit", () => {
             if (!cdpReady && !ac.signal.aborted)
                 ac.abort();
         });
-        const wsUrl = await waitForPageWebSocket(port, timeoutMs, ac.signal);
+        const wsUrl = await waitForPageWebSocket(port, connectTimeoutMs, ac.signal);
         cdpReady = true;
         const snapshot = await withCdpSession(wsUrl, url, timeoutMs, ac.signal);
         if (!snapshot.html || snapshot.html.length < 40) {
@@ -100,6 +114,13 @@ export async function renderUrlViaCdp(chromePath, url, opts) {
         return snapshot;
     }
     catch (err) {
+        if (!cdpReady) {
+            const hung = childStillRunning(child);
+            const detail = err instanceof Error ? err.message : String(err);
+            throw new ChromeConnectError(hung
+                ? `Chrome did not respond over CDP within ${connectTimeoutMs}ms`
+                : `Chrome CDP failed to start (${detail})`, hung);
+        }
         if (ac.signal.aborted) {
             throw new Error(`CDP timed out after ${timeoutMs}ms`);
         }
@@ -117,8 +138,8 @@ export async function renderUrlViaCdp(chromePath, url, opts) {
         }
     }
 }
-async function waitForPageWebSocket(port, timeoutMs, signal) {
-    const deadline = Date.now() + Math.min(timeoutMs, 8_000);
+async function waitForPageWebSocket(port, connectTimeoutMs, signal) {
+    const deadline = Date.now() + connectTimeoutMs;
     let lastErr = "CDP HTTP endpoint never became ready";
     while (Date.now() < deadline) {
         if (signal.aborted)

package/dist/core/engine/chrome-launch.d.ts

@@ -1,7 +1,18 @@
 import type { ChildProcess } from "node:child_process";
 export declare const DUMP_MAX_BYTES: number;
+/** Page navigation + DOM-wait budget once Chrome is answering over CDP. */
 export declare const DEFAULT_TIMEOUT_MS = 18000;
+/**
+ * How long to wait for Chrome to accept CDP after spawn. Kept short and
+ * separate from `DEFAULT_TIMEOUT_MS` so a hung binary fails fast while a
+ * slow but working JS page (often ~7s) still has time to render.
+ */
+export declare const DEFAULT_CONNECT_TIMEOUT_MS = 4000;
 export declare const VIRTUAL_TIME_MS = 10000;
+/** `opts.timeoutMs`, else `WEBCRAWL_CHROME_TIMEOUT_MS`, else 18s. */
+export declare function resolveChromeTimeoutMs(override?: number): number;
+/** `opts.connectTimeoutMs`, else `WEBCRAWL_CHROME_CONNECT_TIMEOUT_MS`, else 4s. */
+export declare function resolveChromeConnectTimeoutMs(override?: number): number;
 export type ChromeHeadlessMode = "new" | "old" | "shell";
 export declare function killProcessGroup(pid: number | undefined, child?: Pick<ChildProcess, "kill">): void;
 export declare function headlessFlag(mode: ChromeHeadlessMode, binaryPath: string): string[];

package/dist/core/engine/chrome-launch.js

@@ -5,8 +5,38 @@ import { existsSync, readdirSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
 export const DUMP_MAX_BYTES = 2 * 1024 * 1024;
+/** Page navigation + DOM-wait budget once Chrome is answering over CDP. */
 export const DEFAULT_TIMEOUT_MS = 18_000;
+/**
+ * How long to wait for Chrome to accept CDP after spawn. Kept short and
+ * separate from `DEFAULT_TIMEOUT_MS` so a hung binary fails fast while a
+ * slow but working JS page (often ~7s) still has time to render.
+ */
+export const DEFAULT_CONNECT_TIMEOUT_MS = 4_000;
 export const VIRTUAL_TIME_MS = 10_000;
+function envPositiveMs(name, fallback, min) {
+    const raw = process.env[name];
+    if (raw == null || raw.trim() === "")
+        return fallback;
+    const n = Number(raw);
+    if (!Number.isFinite(n) || n <= 0)
+        return fallback;
+    return Math.max(min, n);
+}
+/** `opts.timeoutMs`, else `WEBCRAWL_CHROME_TIMEOUT_MS`, else 18s. */
+export function resolveChromeTimeoutMs(override) {
+    if (typeof override === "number" && Number.isFinite(override) && override > 0) {
+        return Math.max(1_500, override);
+    }
+    return envPositiveMs("WEBCRAWL_CHROME_TIMEOUT_MS", DEFAULT_TIMEOUT_MS, 1_500);
+}
+/** `opts.connectTimeoutMs`, else `WEBCRAWL_CHROME_CONNECT_TIMEOUT_MS`, else 4s. */
+export function resolveChromeConnectTimeoutMs(override) {
+    if (typeof override === "number" && Number.isFinite(override) && override > 0) {
+        return Math.max(500, override);
+    }
+    return envPositiveMs("WEBCRAWL_CHROME_CONNECT_TIMEOUT_MS", DEFAULT_CONNECT_TIMEOUT_MS, 500);
+}
 export function killProcessGroup(pid, child) {
     if (pid && pid > 0) {
         try {

package/dist/core/engine/chrome.d.ts

@@ -1,4 +1,5 @@
-export { DEFAULT_TIMEOUT_MS, VIRTUAL_TIME_MS, discoverChromeBinaries, findPlaywrightHeadlessShell, installedChromePath, isLocalChromeAvailable, parseDevToolsActivePort, chromeCdpArgs, chromeDumpDomArgs, killProcessGroup, type ChromeHeadlessMode, } from "./chrome-launch.js";
+export { DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_TIMEOUT_MS, VIRTUAL_TIME_MS, discoverChromeBinaries, findPlaywrightHeadlessShell, installedChromePath, isLocalChromeAvailable, parseDevToolsActivePort, chromeCdpArgs, chromeDumpDomArgs, killProcessGroup, resolveChromeConnectTimeoutMs, resolveChromeTimeoutMs, type ChromeHeadlessMode, } from "./chrome-launch.js";
+export { ChromeConnectError } from "./chrome-cdp.js";
 export interface ChromeRenderAttempt {
     ok: boolean;
     method: string;
@@ -17,6 +18,8 @@ export interface ChromeRenderResult {
  */
 export declare function renderViaLocalChrome(url: string, opts?: {
     timeoutMs?: number;
+    connectTimeoutMs?: number;
+    binaries?: string[];
 }): Promise<string | null>;
 /**
  * Same as `renderViaLocalChrome` but with the attempt log and an actionable
@@ -25,4 +28,6 @@ export declare function renderViaLocalChrome(url: string, opts?: {
  */
 export declare function renderViaLocalChromeDetailed(url: string, opts?: {
     timeoutMs?: number;
+    connectTimeoutMs?: number;
+    binaries?: string[];
 }): Promise<ChromeRenderResult>;

package/dist/core/engine/chrome.js

@@ -15,9 +15,10 @@ import { spawn } from "node:child_process";
 import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { renderUrlViaCdp } from "./chrome-cdp.js";
-import { DEFAULT_TIMEOUT_MS, DUMP_MAX_BYTES, chromeDumpDomArgs, discoverChromeBinaries, killProcessGroup, } from "./chrome-launch.js";
-export { DEFAULT_TIMEOUT_MS, VIRTUAL_TIME_MS, discoverChromeBinaries, findPlaywrightHeadlessShell, installedChromePath, isLocalChromeAvailable, parseDevToolsActivePort, chromeCdpArgs, chromeDumpDomArgs, killProcessGroup, } from "./chrome-launch.js";
+import { ChromeConnectError, renderUrlViaCdp } from "./chrome-cdp.js";
+import { DUMP_MAX_BYTES, chromeDumpDomArgs, discoverChromeBinaries, killProcessGroup, resolveChromeConnectTimeoutMs, resolveChromeTimeoutMs, } from "./chrome-launch.js";
+export { DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_TIMEOUT_MS, VIRTUAL_TIME_MS, discoverChromeBinaries, findPlaywrightHeadlessShell, installedChromePath, isLocalChromeAvailable, parseDevToolsActivePort, chromeCdpArgs, chromeDumpDomArgs, killProcessGroup, resolveChromeConnectTimeoutMs, resolveChromeTimeoutMs, } from "./chrome-launch.js";
+export { ChromeConnectError } from "./chrome-cdp.js";
 function attemptPlan(binary) {
     const isShell = /headless[-_]?shell/i.test(binary);
     if (isShell) {
@@ -51,7 +52,7 @@ export async function renderViaLocalChrome(url, opts = {}) {
  * failures should use this.
  */
 export async function renderViaLocalChromeDetailed(url, opts = {}) {
-    const binaries = discoverChromeBinaries();
+    const binaries = opts.binaries?.length ? opts.binaries : discoverChromeBinaries();
     if (binaries.length === 0) {
         return {
             html: null,
@@ -60,15 +61,20 @@ export async function renderViaLocalChromeDetailed(url, opts = {}) {
             attempts: [],
         };
     }
-    const budgetMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const budgetMs = resolveChromeTimeoutMs(opts.timeoutMs);
+    const connectTimeoutMs = resolveChromeConnectTimeoutMs(opts.connectTimeoutMs);
     const started = Date.now();
     const attempts = [];
     let lastError = "local Chrome produced no content";
     for (const binary of binaries) {
+        let hungConnect = false;
         for (const kind of attemptPlan(binary)) {
             const remaining = budgetMs - (Date.now() - started);
             if (remaining < 1_200)
                 break;
+            // A process that launched but never answered CDP will hang dump-dom too.
+            if (kind.driver === "dump-dom" && hungConnect)
+                continue;
             const method = methodName(binary, kind);
             const attemptBudget = kind.driver === "cdp"
                 ? Math.min(remaining, kind.headless === "new" ? 12_000 : 8_000)
@@ -78,6 +84,7 @@ export async function renderViaLocalChromeDetailed(url, opts = {}) {
                 const html = kind.driver === "cdp"
                     ? (await renderUrlViaCdp(binary, url, {
                         timeoutMs: attemptBudget,
+                        connectTimeoutMs: Math.min(connectTimeoutMs, remaining),
                         headless: kind.headless,
                     })).html
                     : await dumpDom(binary, url, kind.headless, attemptBudget);
@@ -93,6 +100,8 @@ export async function renderViaLocalChromeDetailed(url, opts = {}) {
                 const ms = Date.now() - t0;
                 lastError = `${method}: ${err instanceof Error ? err.message : String(err)}`;
                 attempts.push({ ok: false, method, ms, error: lastError });
+                if (err instanceof ChromeConnectError && err.hung)
+                    hungConnect = true;
             }
         }
     }

package/dist/index.d.ts

@@ -1,2 +1,2 @@
-export { main, HELP, type Deps, type Io } from "./cli.js";
+export { main, HELP, packageVersion, type Deps, type Io } from "./cli.js";
 export { SUPPORTED_AGENTS, WEBCRAWL_SKILL_MD, runDoctor, runSetup, type AgentTarget, type DoctorCheck, type DoctorReport, type SetupOptions, type SetupResult, type SupportedAgentId, } from "./setup.js";

package/dist/index.js

@@ -1,2 +1,2 @@
-export { main, HELP } from "./cli.js";
+export { main, HELP, packageVersion } from "./cli.js";
 export { SUPPORTED_AGENTS, WEBCRAWL_SKILL_MD, runDoctor, runSetup, } from "./setup.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.4.4",
+  "version": "1.4.5",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.4.4",
-    "@webcrawl/curl-impersonate": "^1.4.4",
+    "@webcrawl/client": "^1.4.5",
+    "@webcrawl/curl-impersonate": "^1.4.5",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",