webcrawl 0.0.0-stage → 1.2.0

package/README.md

@@ -1,3 +1,43 @@
-# Temporary Holding Version
+# webcrawl — Webcrawl Anything CLI
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+**Crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, shopping offer, travel fare, or CVE advisory — from a single CLI and MCP server.**
+
+## Quickstart
+
+One command registers the `@webcrawl/mcp` server and installs the `webcrawl-anything` Agent Skill (`SKILL.md`) for Claude Code, Cursor, and Codex:
+
+```bash
+npx webcrawl setup
+npx webcrawl doctor
+```
+
+## CLI Commands
+
+```bash
+# List all supported targets and their requirements
+npx webcrawl targets
+
+# Scrape any website, GitHub repo, ArXiv paper, PDF, or YouTube video
+npx webcrawl scrape https://github.com/torvalds/linux
+npx webcrawl scrape https://arxiv.org/abs/1706.03762
+
+# Search the web or structured verticals
+npx webcrawl search "react 19 compiler" -n 5
+npx webcrawl search "sony wh-1000xm5" -c shopping
+npx webcrawl search "boutique hotels in shibuya" -c hotels
+npx webcrawl search "CVE-2024-3094 xz utils" -c security --sub cve
+
+# Discover URLs and crawl multi-page documentation
+npx webcrawl map https://docs.stripe.com --filter webhook -n 50
+npx webcrawl crawl https://docs.stripe.com/webhooks -n 5
+
+# Connect to webcrawl.app cloud for managed stealth browsers & global cache
+npx webcrawl login
+npx webcrawl whoami
+```
+
+## Documentation & Source
+
+- **GitHub Repository:** [brainfish-ai/webcrawl-anything](https://github.com/brainfish-ai/webcrawl-anything)
+- **Cloud Platform:** [https://webcrawl.app](https://webcrawl.app)
+- **License:** MIT

package/dist/bin.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/bin.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+import { main } from "./cli.js";
+async function readStdin() {
+    const chunks = [];
+    for await (const c of process.stdin) {
+        chunks.push(c);
+    }
+    return Buffer.concat(chunks).toString("utf8");
+}
+const code = await main(process.argv.slice(2), {
+    out: (s) => process.stdout.write(s),
+    err: (s) => process.stderr.write(s),
+    readStdin,
+    isTTY: Boolean(process.stdin.isTTY),
+}, {});
+process.exitCode = code;

package/dist/cli.d.ts

@@ -0,0 +1,19 @@
+export interface Io {
+    out: (s: string) => void;
+    err: (s: string) => void;
+    /** Whole stdin as text (only called when needed). */
+    readStdin: () => Promise<string>;
+    isTTY: boolean;
+}
+export interface Deps {
+    env?: NodeJS.ProcessEnv;
+    fetch?: typeof fetch;
+    /** Config file path override (tests). */
+    configFile?: string;
+    /** Home directory override for `webcrawl setup` tests. */
+    homeDir?: string;
+    /** Working directory override for `webcrawl setup` tests. */
+    cwd?: string;
+}
+export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login               Save your webcrawl.app API key (--key <key>, or paste when asked)\n  whoami              Show execution mode (local vs cloud), key source, endpoint and credit usage\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
+export declare function main(argv: string[], io: Io, deps?: Deps): Promise<number>;

package/dist/cli.js

@@ -0,0 +1,308 @@
+import { readFileSync } from "node:fs";
+import { createInterface } from "node:readline";
+import { parseArgs } from "node:util";
+import { WebcrawlClient, WebcrawlError, configPath, describeError, readConfigFile, resolveConfig, writeConfigFile, } from "@webcrawl/client";
+import { formatCrawl, formatCredits, formatMap, formatRerank, formatScrape, formatSearch, formatTargets, } from "./format.js";
+import { SUPPORTED_AGENTS, runDoctor, runSetup, } from "./setup.js";
+export const HELP = `Webcrawl Anything: search, scrape, parse and crawl any target on the internet
+
+Usage: webcrawl <command> [options]
+
+Commands
+  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>
+  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)
+  map <url>           List a site's URLs. --filter <text>, -n limit
+  crawl <url>         Read several pages. -n max pages (default 5, max 25)
+  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>
+  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top
+  targets             List all supported crawl & search targets and their requirements
+  categories          List search categories and vertical filters
+  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)
+  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration
+  login               Save your webcrawl.app API key (--key <key>, or paste when asked)
+  whoami              Show execution mode (local vs cloud), key source, endpoint and credit usage
+
+Options
+  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud
+  --json              Print the raw JSON response
+  --api-key <key>     Override WEBCRAWL_API_KEY
+  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)
+  --country, --city   Where to search from (default: this machine's timezone)
+  --no-cache          Skip cached pages (scrape)
+  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)
+  --schema-profile    MCP schema profile for setup: compact (default) or full
+  -h, --help
+
+Config order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json`;
+const OPTIONS = {
+    json: { type: "boolean" },
+    help: { type: "boolean", short: "h" },
+    "api-key": { type: "string" },
+    url: { type: "string" },
+    mode: { type: "string" },
+    category: { type: "string", short: "c" },
+    sub: { type: "string" },
+    limit: { type: "string", short: "n" },
+    site: { type: "string", multiple: true },
+    country: { type: "string" },
+    city: { type: "string" },
+    filter: { type: "string" },
+    prompt: { type: "string" },
+    schema: { type: "string" },
+    doc: { type: "string", multiple: true },
+    file: { type: "string" },
+    key: { type: "string" },
+    "no-cache": { type: "boolean" },
+    content: { type: "boolean" },
+    agent: { type: "string", multiple: true },
+    "schema-profile": { type: "string" },
+};
+class UsageError extends Error {
+}
+function int(v, name) {
+    if (v === undefined) {
+        return undefined;
+    }
+    const n = Number(v);
+    if (!Number.isInteger(n) || n < 1) {
+        throw new UsageError(`${name} must be a positive whole number`);
+    }
+    return n;
+}
+async function readKeyInteractively(io) {
+    if (!io.isTTY) {
+        return (await io.readStdin()).trim();
+    }
+    const rl = createInterface({ input: process.stdin, output: process.stderr });
+    return new Promise((resolve) => {
+        rl.question("Paste your API key (wc_live_…): ", (a) => {
+            rl.close();
+            resolve(a.trim());
+        });
+    });
+}
+async function resolveBackendClient(cfg, deps) {
+    // When a custom fetch mock is injected (e.g. unit tests) or effectiveMode is cloud, use WebcrawlClient
+    if (cfg.effectiveMode === "cloud" || deps.fetch) {
+        return new WebcrawlClient({
+            apiKey: cfg.apiKey,
+            baseUrl: cfg.baseUrl,
+            fetch: deps.fetch,
+            userAgent: "webcrawl-cli/0.1",
+        });
+    }
+    try {
+        const localEngineSpec = ["..", "..", "..", "src", "local-engine.js"].join("/");
+        const localMod = (await import(localEngineSpec));
+        return localMod.createLocalEngineClient();
+    }
+    catch {
+        return new WebcrawlClient({
+            apiKey: cfg.apiKey,
+            baseUrl: cfg.baseUrl,
+            fetch: deps.fetch,
+            userAgent: "webcrawl-cli/0.1",
+        });
+    }
+}
+export async function main(argv, io, deps = {}) {
+    const env = deps.env ?? process.env;
+    let parsed;
+    try {
+        parsed = parseArgs({ args: argv, options: OPTIONS, allowPositionals: true });
+    }
+    catch (e) {
+        io.err(`${e instanceof Error ? e.message : String(e)}\nRun \`webcrawl --help\`.\n`);
+        return 2;
+    }
+    const { values: v, positionals } = parsed;
+    const [cmd, ...rest] = positionals;
+    if (v.help || !cmd || cmd === "help") {
+        io.out(`${HELP}\n`);
+        return cmd || v.help ? 0 : 2;
+    }
+    const file = deps.configFile ?? configPath(env);
+    const cfg = resolveConfig({ apiKey: v["api-key"], baseUrl: v.url, mode: v.mode }, env, readConfigFile(file));
+    const client = await resolveBackendClient(cfg, deps);
+    const arg = rest.join(" ").trim();
+    const need = (what) => {
+        if (!arg) {
+            throw new UsageError(`Missing ${what}. Usage: webcrawl ${cmd} <${what}>`);
+        }
+        return arg;
+    };
+    const emit = (res, pretty) => {
+        if (v.json) {
+            io.out(`${JSON.stringify(res, null, 2)}\n`);
+            return;
+        }
+        io.out(`${pretty}\n`);
+        const credits = formatCredits(res);
+        if (credits) {
+            io.err(`[${credits}]\n`);
+        }
+    };
+    try {
+        switch (cmd) {
+            case "search": {
+                const res = await client.search({
+                    query: need("query"),
+                    category: v.category,
+                    subcategory: v.sub,
+                    limit: int(v.limit, "--limit"),
+                    sites: v.site,
+                    country: v.country,
+                    city: v.city,
+                    scrapeContent: v.content,
+                });
+                emit(res, formatSearch(res));
+                return 0;
+            }
+            case "scrape": {
+                const res = await client.scrape({ url: need("url"), formats: ["markdown"], noCache: v["no-cache"] });
+                emit(res, formatScrape(res));
+                return 0;
+            }
+            case "map": {
+                const res = await client.map({ url: need("url"), search: v.filter, limit: int(v.limit, "--limit") });
+                emit(res, formatMap(res));
+                return 0;
+            }
+            case "crawl": {
+                const res = await client.crawl({ url: need("url"), maxPages: int(v.limit, "--limit") });
+                emit(res, formatCrawl(res));
+                return 0;
+            }
+            case "extract": {
+                let schema;
+                if (v.schema) {
+                    try {
+                        schema = JSON.parse(readFileSync(v.schema, "utf8"));
+                    }
+                    catch (e) {
+                        throw new UsageError(`Could not read schema ${v.schema}: ${e instanceof Error ? e.message : String(e)}`);
+                    }
+                }
+                const res = await client.extract({ url: need("url"), prompt: v.prompt, schema });
+                emit(res, JSON.stringify(res.data ?? res, null, 2));
+                return 0;
+            }
+            case "rerank": {
+                const query = need("query");
+                let docs = v.doc ?? [];
+                if (v.file) {
+                    docs = docs.concat(readFileSync(v.file, "utf8").split(/\r?\n/).filter(Boolean));
+                }
+                if (!docs.length && !io.isTTY) {
+                    docs = (await io.readStdin()).split(/\r?\n/).filter(Boolean);
+                }
+                if (!docs.length) {
+                    throw new UsageError("No documents. Pass --doc, --file, or pipe lines on stdin.");
+                }
+                const res = await client.rerank({ query, documents: docs, topK: int(v.limit, "--limit") });
+                emit(res, formatRerank(res));
+                return 0;
+            }
+            case "categories": {
+                const res = await client.categories();
+                emit(res, JSON.stringify(res, null, 2));
+                return 0;
+            }
+            case "targets": {
+                const res = await client.targets();
+                emit(res, formatTargets(res));
+                return 0;
+            }
+            case "setup": {
+                const rawAgents = v.agent ?? [];
+                let agents;
+                if (rawAgents.includes("all")) {
+                    agents = SUPPORTED_AGENTS.map((a) => a.id);
+                }
+                else if (rawAgents.length) {
+                    const valid = new Set(SUPPORTED_AGENTS.map((a) => a.id));
+                    for (const a of rawAgents) {
+                        if (!valid.has(a)) {
+                            throw new UsageError(`Unknown agent "${a}". Supported: ${SUPPORTED_AGENTS.map((x) => x.id).join(", ")}, all`);
+                        }
+                    }
+                    agents = rawAgents;
+                }
+                const schemaProfile = v["schema-profile"] === "full" ? "full" : "compact";
+                const setupRes = runSetup({
+                    agents,
+                    schemaProfile,
+                    apiKey: cfg.apiKey,
+                    baseUrl: cfg.source.baseUrl !== "default" ? cfg.baseUrl : undefined,
+                    homeDir: deps.homeDir,
+                    cwd: deps.cwd,
+                });
+                const prettyLines = [
+                    `Webcrawl Anything agent setup complete (MCP schema profile: ${setupRes.schemaProfile}):`,
+                    ...setupRes.configuredAgents.map((a) => `  ✓ ${a.name}\n    MCP:   ${a.mcpConfigPath}${a.backupPath ? ` (backup: ${a.backupPath})` : ""}\n    Skill: ${a.skillPath}`),
+                    "",
+                    "Restart your agent to load the `webcrawl` MCP server and `webcrawl-anything` skill.",
+                ].join("\n");
+                emit({ success: true, ...setupRes }, prettyLines);
+                return 0;
+            }
+            case "doctor": {
+                const report = runDoctor(env, cfg.effectiveMode, cfg.source.apiKey);
+                const prettyLines = [
+                    `Webcrawl Anything Doctor (mode: ${report.mode})`,
+                    ...report.checks.map((c) => {
+                        const badge = c.status === "ok" ? "✓" : c.status === "optional" ? "○" : "!";
+                        return `  ${badge} ${c.name.padEnd(26)} ${c.detail}`;
+                    }),
+                ].join("\n");
+                emit({ success: true, ...report }, prettyLines);
+                return 0;
+            }
+            case "login": {
+                const key = (v.key ?? (await readKeyInteractively(io))).trim();
+                if (!key) {
+                    throw new UsageError("No key provided.");
+                }
+                const existing = readConfigFile(file);
+                writeConfigFile({ ...existing, apiKey: key, ...(v.url ? { baseUrl: v.url } : {}) }, file);
+                io.out(`Saved your key to ${file}.\n`);
+                return 0;
+            }
+            case "whoami": {
+                const masked = cfg.apiKey ? `${cfg.apiKey.slice(0, 8)}…${cfg.apiKey.slice(-4)}` : "none";
+                const lines = [
+                    `endpoint: ${cfg.baseUrl} (${cfg.source.baseUrl})`,
+                    `mode:     ${cfg.effectiveMode} (${cfg.source.mode})`,
+                    `key:      ${masked}${cfg.apiKey ? ` (${cfg.source.apiKey})` : ". Run `webcrawl login` or set WEBCRAWL_API_KEY (local mode works with zero key)."}`,
+                ];
+                if (cfg.apiKey) {
+                    const u = await client.usage();
+                    if (v.json) {
+                        io.out(`${JSON.stringify({ endpoint: cfg.baseUrl, mode: cfg.effectiveMode, keySource: cfg.source.apiKey, usage: u }, null, 2)}\n`);
+                        return 0;
+                    }
+                    const credits = (u.credits ?? u);
+                    if (typeof credits.remaining === "number") {
+                        lines.push(`credits:  ${credits.remaining} remaining of ${credits.limit ?? "?"}`);
+                    }
+                }
+                io.out(`${lines.join("\n")}\n`);
+                return 0;
+            }
+            default:
+                throw new UsageError(`Unknown command "${cmd}".`);
+        }
+    }
+    catch (e) {
+        if (e instanceof UsageError) {
+            io.err(`${e.message}\nRun \`webcrawl --help\`.\n`);
+            return 2;
+        }
+        if (e instanceof WebcrawlError) {
+            io.err(`webcrawl: ${describeError(e, cfg.baseUrl)}\n`);
+            return 1;
+        }
+        io.err(`webcrawl: ${e instanceof Error ? e.message : String(e)}\n`);
+        return 1;
+    }
+}

package/dist/format.d.ts

@@ -0,0 +1,9 @@
+type Rec = Record<string, unknown>;
+export declare function formatSearch(res: Rec): string;
+export declare function formatScrape(res: Rec): string;
+export declare function formatMap(res: Rec): string;
+export declare function formatCrawl(res: Rec): string;
+export declare function formatRerank(res: Rec): string;
+export declare function formatCredits(res: Rec): string;
+export declare function formatTargets(res: Rec): string;
+export {};

package/dist/format.js

@@ -0,0 +1,113 @@
+const isRec = (v) => !!v && typeof v === "object" && !Array.isArray(v);
+const str = (v) => (typeof v === "string" ? v : typeof v === "number" ? String(v) : "");
+const first = (o, keys) => {
+    for (const k of keys) {
+        const v = str(o[k]);
+        if (v) {
+            return v;
+        }
+    }
+    return "";
+};
+const VERTICALS = ["products", "offers", "listings", "flights", "hotels", "businesses", "cves"];
+/** CVE records have no title/price; map them onto the shared row shape. */
+const cveRow = (o) => ({
+    title: `${str(o.id)} ${str(o.title) || str(o.summary)}`.trim(),
+    price: [str(o.cvssScore), str(o.severity), o.kev === true ? "KEV" : ""].filter(Boolean).join(" "),
+    source: str(o.source),
+    url: str(o.url),
+});
+const clip = (s, n) => (s.length > n ? `${s.slice(0, n - 1)}…` : s);
+function rows(items) {
+    const data = items.filter(isRec).map((o) => ({
+        title: clip(first(o, ["title", "name", "address", "airline", "productName"]) || "(untitled)", 52),
+        price: first(o, ["priceText", "price", "pricePerNight", "minPrice"]),
+        src: clip(first(o, ["provider", "portal", "vendor", "domain", "source"]), 24),
+        url: first(o, ["url", "link"]),
+    }));
+    const w = (k) => Math.max(...data.map((d) => d[k].length), 1);
+    const [wt, wp, ws] = [w("title"), w("price"), w("src")];
+    return data
+        .map((d, i) => `${String(i + 1).padStart(2)}  ${d.title.padEnd(wt)}  ${d.price.padEnd(wp)}  ${d.src.padEnd(ws)}${d.url ? `\n    ${d.url}` : ""}`)
+        .join("\n");
+}
+export function formatSearch(res) {
+    const parts = [];
+    for (const key of VERTICALS) {
+        const arr = res[key];
+        if (Array.isArray(arr) && arr.length) {
+            parts.push(`${key}\n${rows(key === "cves" ? arr.filter(isRec).map(cveRow) : arr)}`);
+        }
+    }
+    const results = Array.isArray(res.results) ? res.results : Array.isArray(res.data) ? res.data : [];
+    if (results.length) {
+        parts.push(`results\n${results
+            .filter(isRec)
+            .map((r, i) => {
+            const desc = first(r, ["description", "snippet"]).replace(/\s+/g, " ").slice(0, 160);
+            return `${String(i + 1).padStart(2)}  ${first(r, ["title"]) || "(untitled)"}\n    ${first(r, ["url"])}${desc ? `\n    ${desc}` : ""}`;
+        })
+            .join("\n")}`);
+    }
+    return parts.length ? parts.join("\n\n") : "No results.";
+}
+export function formatScrape(res) {
+    const d = isRec(res.data) ? res.data : res;
+    return str(d.markdown) || str(d.content) || "(no content returned)";
+}
+export function formatMap(res) {
+    const links = Array.isArray(res.links) ? res.links : Array.isArray(res.urls) ? res.urls : [];
+    return links.map((l) => (isRec(l) ? str(l.url) : str(l))).filter(Boolean).join("\n") || "No URLs found.";
+}
+export function formatCrawl(res) {
+    const pages = Array.isArray(res.data) ? res.data : Array.isArray(res.pages) ? res.pages : [];
+    if (!pages.length) {
+        return "No pages crawled.";
+    }
+    return pages
+        .filter(isRec)
+        .map((p) => {
+        const url = str(p.url) || (isRec(p.metadata) ? str(p.metadata.sourceURL) || str(p.metadata.url) : "");
+        return `# ${str(p.title) || url || "page"}\n${url}\n\n${str(p.markdown)}`;
+    })
+        .join("\n\n---\n\n");
+}
+export function formatRerank(res) {
+    const items = Array.isArray(res.results) ? res.results : Array.isArray(res.data) ? res.data : [];
+    if (!items.length) {
+        return JSON.stringify(res, null, 2);
+    }
+    return items
+        .map((r, i) => {
+        const o = isRec(r) ? r : {};
+        const score = typeof o.score === "number" ? o.score.toFixed(3) : "";
+        return `${String(i + 1).padStart(2)}  ${score.padEnd(6)}  ${clip(first(o, ["description", "markdown", "title"]).replace(/\s+/g, " "), 100)}`;
+    })
+        .join("\n");
+}
+export function formatCredits(res) {
+    const c = res.credits;
+    if (!isRec(c)) {
+        return "";
+    }
+    const bits = [
+        typeof c.charged === "number" ? `charged ${c.charged}` : "",
+        typeof c.remaining === "number" ? `${c.remaining} remaining` : "",
+    ].filter(Boolean);
+    return bits.join(", ");
+}
+export function formatTargets(res) {
+    const list = Array.isArray(res.targets) ? res.targets.filter(isRec) : [];
+    if (!list.length) {
+        return "No targets registered.";
+    }
+    return list
+        .map((t) => {
+        const id = str(t.id).padEnd(16);
+        const name = str(t.name);
+        const returns = str(t.returns);
+        const reqs = str(t.requirements);
+        return `${id}  ${name}\n                  Returns: ${returns}\n                  Requires: ${reqs}`;
+    })
+        .join("\n\n");
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { main, HELP, type Deps, type Io } from "./cli.js";
+export { SUPPORTED_AGENTS, WEBCRAWL_SKILL_MD, runDoctor, runSetup, type AgentTarget, type DoctorCheck, type DoctorReport, type SetupOptions, type SetupResult, type SupportedAgentId, } from "./setup.js";

package/dist/index.js

@@ -0,0 +1,2 @@
+export { main, HELP } from "./cli.js";
+export { SUPPORTED_AGENTS, WEBCRAWL_SKILL_MD, runDoctor, runSetup, } from "./setup.js";

package/dist/setup.d.ts

@@ -0,0 +1,38 @@
+export type SupportedAgentId = "claude" | "cursor" | "codex" | "gemini" | "windsurf";
+export interface AgentTarget {
+    id: SupportedAgentId;
+    name: string;
+    mcpConfigPath: (home: string, cwd: string) => string;
+    skillPath: (home: string, cwd: string) => string;
+}
+export declare const SUPPORTED_AGENTS: AgentTarget[];
+export declare const WEBCRAWL_SKILL_MD = "---\nname: webcrawl-anything\ndescription: Crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, shopping offer, travel fare, or CVE advisory.\n---\n\n# Webcrawl Anything \u2014 Agent Investigation Workflow\n\nUse **Webcrawl Anything** (`webcrawl` CLI or `webcrawl` MCP tools) whenever you need live web search, anti-bot page scraping, document/video parsing, or structured vertical extraction with provenance.\n\n## 1. Choose the Right Tool or Target\n\n1. **Structured Search (`web_search` / `webcrawl search`)**:\n   - Pass `category` when the query belongs to a vertical:\n     - `shopping`: Multi-retailer product prices, stock availability, ratings, shipping.\n     - `flights`: IATA routes, fares, airlines, stops, cabin class.\n     - `hotels`: Nightly rates, star ratings, guest reviews, amenities.\n     - `real-estate`: Property listings (`subcategory: \"buy\" | \"rent\" | \"sold\"`).\n     - `business`: Local businesses, clinics, phone numbers, hours, ratings.\n     - `security`: CVE / GHSA records with CVSS scores, CWEs, affected/fixed versions, and CISA KEV status.\n     - `academic`: ArXiv and research papers.\n     - `tech` / `news` / `general`: General and developer search.\n2. **Page & Developer Target Inspection (`web_scrape` / `webcrawl scrape <url>`)**:\n   - Works on any website, client-rendered SPA, GitHub repository/issue/release, ArXiv abstract page, or `llms.txt` manifest.\n3. **Binary Documents & Media (`web_parse` / `webcrawl scrape <pdf-or-youtube-url>`)**:\n   - Reads PDFs (native text + automatic OCR on scanned pages), Word (`.docx`), Excel (`.xlsx`), PowerPoint (`.pptx`), and YouTube URLs (timestamped transcripts).\n4. **Site Discovery & Multi-Page Crawl (`web_map` & `web_crawl`)**:\n   - Always run `web_map` first on unfamiliar domains to inspect available URLs before running `web_crawl` (capped at 25 pages).\n\n## 2. Verify Evidence & Provenance\n\n- Check the returned `evidence` block (`targetId`, `tier`, `contentHash`, `matchedSignals`, `limitations`) before stating claims as fact.\n- If `limitations` notes that a page was a client-rendered shell or blocked by a challenge in static mode, escalate to stealth browser or `webcrawl.app` cloud (`webcrawl login`).\n";
+export interface SetupOptions {
+    agents?: SupportedAgentId[];
+    schemaProfile?: "compact" | "full";
+    apiKey?: string;
+    baseUrl?: string;
+    homeDir?: string;
+    cwd?: string;
+}
+export interface SetupResult {
+    configuredAgents: Array<{
+        id: SupportedAgentId;
+        name: string;
+        mcpConfigPath: string;
+        backupPath?: string;
+        skillPath: string;
+    }>;
+    schemaProfile: "compact" | "full";
+}
+export declare function runSetup(opts?: SetupOptions): SetupResult;
+export interface DoctorCheck {
+    name: string;
+    status: "ok" | "optional" | "warn";
+    detail: string;
+}
+export interface DoctorReport {
+    mode: "local" | "cloud";
+    checks: DoctorCheck[];
+}
+export declare function runDoctor(env?: NodeJS.ProcessEnv, effectiveMode?: "local" | "cloud", apiKeySource?: string): DoctorReport;

package/dist/setup.js

@@ -0,0 +1,184 @@
+/**
+ * `webcrawl setup` and `webcrawl doctor` — REA-style one-command agent onboarding.
+ *
+ * Registers the `@webcrawl/mcp` server into Claude Code, Cursor, Codex, Windsurf,
+ * and Gemini CLI config files (with timestamped backup) and installs the matching
+ * `webcrawl-anything` Agent Skill (`SKILL.md`).
+ */
+import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+export const SUPPORTED_AGENTS = [
+    {
+        id: "claude",
+        name: "Claude Code",
+        mcpConfigPath: (home) => join(home, ".claude.json"),
+        skillPath: (home) => join(home, ".claude", "skills", "webcrawl-anything", "SKILL.md"),
+    },
+    {
+        id: "cursor",
+        name: "Cursor",
+        mcpConfigPath: (home) => join(home, ".cursor", "mcp.json"),
+        skillPath: (home) => join(home, ".cursor", "skills", "webcrawl-anything", "SKILL.md"),
+    },
+    {
+        id: "codex",
+        name: "Codex / OpenAI Agents",
+        mcpConfigPath: (home) => join(home, ".codex", "mcp.json"),
+        skillPath: (_home, cwd) => join(cwd, ".agents", "skills", "webcrawl-anything", "SKILL.md"),
+    },
+    {
+        id: "gemini",
+        name: "Gemini CLI",
+        mcpConfigPath: (home) => join(home, ".gemini", "settings.json"),
+        skillPath: (home) => join(home, ".gemini", "skills", "webcrawl-anything", "SKILL.md"),
+    },
+    {
+        id: "windsurf",
+        name: "Windsurf",
+        mcpConfigPath: (home) => join(home, ".codeium", "windsurf", "mcp_config.json"),
+        skillPath: (home) => join(home, ".codeium", "windsurf", "skills", "webcrawl-anything", "SKILL.md"),
+    },
+];
+export const WEBCRAWL_SKILL_MD = `---
+name: webcrawl-anything
+description: Crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, shopping offer, travel fare, or CVE advisory.
+---
+
+# Webcrawl Anything — Agent Investigation Workflow
+
+Use **Webcrawl Anything** (\`webcrawl\` CLI or \`webcrawl\` MCP tools) whenever you need live web search, anti-bot page scraping, document/video parsing, or structured vertical extraction with provenance.
+
+## 1. Choose the Right Tool or Target
+
+1. **Structured Search (\`web_search\` / \`webcrawl search\`)**:
+   - Pass \`category\` when the query belongs to a vertical:
+     - \`shopping\`: Multi-retailer product prices, stock availability, ratings, shipping.
+     - \`flights\`: IATA routes, fares, airlines, stops, cabin class.
+     - \`hotels\`: Nightly rates, star ratings, guest reviews, amenities.
+     - \`real-estate\`: Property listings (\`subcategory: "buy" | "rent" | "sold"\`).
+     - \`business\`: Local businesses, clinics, phone numbers, hours, ratings.
+     - \`security\`: CVE / GHSA records with CVSS scores, CWEs, affected/fixed versions, and CISA KEV status.
+     - \`academic\`: ArXiv and research papers.
+     - \`tech\` / \`news\` / \`general\`: General and developer search.
+2. **Page & Developer Target Inspection (\`web_scrape\` / \`webcrawl scrape <url>\`)**:
+   - Works on any website, client-rendered SPA, GitHub repository/issue/release, ArXiv abstract page, or \`llms.txt\` manifest.
+3. **Binary Documents & Media (\`web_parse\` / \`webcrawl scrape <pdf-or-youtube-url>\`)**:
+   - Reads PDFs (native text + automatic OCR on scanned pages), Word (\`.docx\`), Excel (\`.xlsx\`), PowerPoint (\`.pptx\`), and YouTube URLs (timestamped transcripts).
+4. **Site Discovery & Multi-Page Crawl (\`web_map\` & \`web_crawl\`)**:
+   - Always run \`web_map\` first on unfamiliar domains to inspect available URLs before running \`web_crawl\` (capped at 25 pages).
+
+## 2. Verify Evidence & Provenance
+
+- Check the returned \`evidence\` block (\`targetId\`, \`tier\`, \`contentHash\`, \`matchedSignals\`, \`limitations\`) before stating claims as fact.
+- If \`limitations\` notes that a page was a client-rendered shell or blocked by a challenge in static mode, escalate to stealth browser or \`webcrawl.app\` cloud (\`webcrawl login\`).
+`;
+export function runSetup(opts = {}) {
+    const home = opts.homeDir ?? homedir();
+    const cwd = opts.cwd ?? process.cwd();
+    const schemaProfile = opts.schemaProfile ?? "compact";
+    const selectedIds = opts.agents?.length ? new Set(opts.agents) : new Set(["claude", "cursor", "codex"]);
+    const targets = SUPPORTED_AGENTS.filter((a) => selectedIds.has(a.id));
+    const configuredAgents = [];
+    for (const agent of targets) {
+        const mcpPath = agent.mcpConfigPath(home, cwd);
+        let existing = {};
+        let backupPath;
+        if (existsSync(mcpPath)) {
+            try {
+                existing = JSON.parse(readFileSync(mcpPath, "utf8"));
+                backupPath = `${mcpPath}.bak`;
+                copyFileSync(mcpPath, backupPath);
+            }
+            catch {
+                existing = {};
+            }
+        }
+        const mcpServers = existing.mcpServers && typeof existing.mcpServers === "object" && !Array.isArray(existing.mcpServers)
+            ? { ...existing.mcpServers }
+            : {};
+        const envVars = {
+            WEBCRAWL_MCP_SCHEMA_PROFILE: schemaProfile,
+        };
+        if (opts.apiKey)
+            envVars.WEBCRAWL_API_KEY = opts.apiKey;
+        if (opts.baseUrl)
+            envVars.WEBCRAWL_URL = opts.baseUrl;
+        mcpServers.webcrawl = {
+            command: "npx",
+            args: ["-y", "@webcrawl/mcp@latest"],
+            env: envVars,
+        };
+        const updated = { ...existing, mcpServers };
+        mkdirSync(dirname(mcpPath), { recursive: true });
+        writeFileSync(mcpPath, `${JSON.stringify(updated, null, 2)}\n`, "utf8");
+        const skillPath = agent.skillPath(home, cwd);
+        mkdirSync(dirname(skillPath), { recursive: true });
+        writeFileSync(skillPath, WEBCRAWL_SKILL_MD, "utf8");
+        configuredAgents.push({
+            id: agent.id,
+            name: agent.name,
+            mcpConfigPath: mcpPath,
+            backupPath,
+            skillPath,
+        });
+    }
+    return { configuredAgents, schemaProfile };
+}
+export function runDoctor(env = process.env, effectiveMode = "local", apiKeySource = "none") {
+    const checks = [];
+    // 1. Node runtime
+    const nodeVer = process.versions.node;
+    const major = Number(nodeVer.split(".")[0]);
+    checks.push({
+        name: "Node.js runtime",
+        status: major >= 20 ? "ok" : "warn",
+        detail: `v${nodeVer} (requires >=20)`,
+    });
+    // 2. Execution mode & Cloud API key
+    checks.push({
+        name: "Execution mode",
+        status: "ok",
+        detail: effectiveMode === "cloud"
+            ? `cloud (https://webcrawl.app via ${apiKeySource} key)`
+            : "local zero-key engine (impit TLS + built-in parsers & vertical extractors)",
+    });
+    // 3. Local Chrome / Stealth Browser availability
+    const chromePaths = [
+        env.CHROME_PATH,
+        "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+        "/usr/bin/google-chrome",
+        "/usr/bin/chromium",
+        "/usr/bin/chromium-browser",
+    ].filter(Boolean);
+    const foundChrome = chromePaths.find((p) => existsSync(p));
+    checks.push({
+        name: "Local stealth browser",
+        status: foundChrome ? "ok" : "optional",
+        detail: foundChrome
+            ? `Found Chrome at ${foundChrome}`
+            : "No local Chrome binary detected; static/TLS fetch works out of the box, or run `webcrawl login` for cloud stealth browsers",
+    });
+    // 4. Search providers
+    const serper = Boolean(env.SERPER_API_KEY);
+    const brave = Boolean(env.BRAVE_SEARCH_API_KEY);
+    const jina = Boolean(env.JINA_API_KEY);
+    const configuredProviders = [
+        "DuckDuckGo/Bing (built-in zero-key)",
+        ...(serper ? ["Serper"] : []),
+        ...(brave ? ["Brave"] : []),
+        ...(jina ? ["Jina"] : []),
+    ];
+    checks.push({
+        name: "Search providers",
+        status: "ok",
+        detail: configuredProviders.join(", "),
+    });
+    // 5. Document & Media parsers
+    checks.push({
+        name: "Document & Media parsers",
+        status: "ok",
+        detail: "PDF (@firecrawl/pdf-inspector), Office (mammoth/officeparser), YouTube captions, NVD/OSV CVEs ready",
+    });
+    return { mode: effectiveMode, checks };
+}

package/package.json

@@ -1,6 +1,61 @@
 {
   "name": "webcrawl",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.2.0",
+  "description": "Webcrawl Anything CLI — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
+  "type": "module",
+  "license": "MIT",
+  "author": "Brainfish AI <support@webcrawl.app> (https://webcrawl.app)",
+  "homepage": "https://webcrawl.app",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/brainfish-ai/webcrawl-anything.git",
+    "directory": "packages/cli"
+  },
+  "bugs": {
+    "url": "https://github.com/brainfish-ai/webcrawl-anything/issues"
+  },
+  "keywords": [
+    "webcrawl",
+    "web-crawler",
+    "web-scraper",
+    "search-engine",
+    "mcp",
+    "claude-code",
+    "cursor",
+    "codex",
+    "pdf-parser",
+    "youtube-transcript",
+    "cve",
+    "ai-agents"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "bin": {
+    "webcrawl": "./dist/bin.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "development": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "prepack": "npm run build",
+    "test": "node --conditions=development --import tsx --test test/*.test.ts"
+  },
+  "dependencies": {
+    "@webcrawl/client": "^1.2.0"
+  },
+  "engines": {
+    "node": ">=20"
+  }
+}