webcrawl 1.2.1 → 1.2.2

package/dist/cli.d.ts

@@ -15,5 +15,5 @@ export interface Deps {
     /** Working directory override for `webcrawl setup` tests. */
     cwd?: string;
 }
-export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login               Save your webcrawl.app API key (--key <key>, or paste when asked)\n  whoami              Show execution mode (local vs cloud), key source, endpoint and credit usage\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
+export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in (--email <addr> [--code <6-digit>], --web, or --key <key>)\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in or create an account in your browser via OAuth PKCE redirect\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
 export declare function main(argv: string[], io: Io, deps?: Deps): Promise<number>;

package/dist/cli.js

@@ -1,4 +1,7 @@
+import { execFile } from "node:child_process";
+import crypto from "node:crypto";
 import { readFileSync } from "node:fs";
+import { createServer } from "node:http";
 import { createInterface } from "node:readline";
 import { parseArgs } from "node:util";
 import { WebcrawlClient, WebcrawlError, configPath, describeError, readConfigFile, resolveConfig, writeConfigFile, } from "@webcrawl/client";
@@ -19,11 +22,15 @@ Commands
   categories          List search categories and vertical filters
   setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)
   doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration
-  login               Save your webcrawl.app API key (--key <key>, or paste when asked)
-  whoami              Show execution mode (local vs cloud), key source, endpoint and credit usage
+  login / signup      Create an account or sign in (--email <addr> [--code <6-digit>], --web, or --key <key>)
+  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance
 
 Options
   --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud
+  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)
+  --code <6-digit>    6-digit email verification code from \`webcrawl login --email <addr>\`
+  --name <name>       Optional display name when creating an account with \`--email\`
+  --web               Sign in or create an account in your browser via OAuth PKCE redirect
   --json              Print the raw JSON response
   --api-key <key>     Override WEBCRAWL_API_KEY
   --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)
@@ -52,6 +59,11 @@ const OPTIONS = {
     doc: { type: "string", multiple: true },
     file: { type: "string" },
     key: { type: "string" },
+    email: { type: "string" },
+    code: { type: "string" },
+    name: { type: "string" },
+    web: { type: "boolean" },
+    "no-wait": { type: "boolean" },
     "no-cache": { type: "boolean" },
     content: { type: "boolean" },
     agent: { type: "string", multiple: true },
@@ -69,18 +81,155 @@ function int(v, name) {
     }
     return n;
 }
-async function readKeyInteractively(io) {
+async function promptLine(io, promptText) {
     if (!io.isTTY) {
         return (await io.readStdin()).trim();
     }
     const rl = createInterface({ input: process.stdin, output: process.stderr });
     return new Promise((resolve) => {
-        rl.question("Paste your API key (wc_live_…): ", (a) => {
+        rl.question(promptText, (a) => {
             rl.close();
             resolve(a.trim());
         });
     });
 }
+async function readKeyInteractively(io) {
+    return promptLine(io, "Paste your API key (wc_live_…): ");
+}
+function openBrowserUrl(url) {
+    const cmd = process.platform === "darwin"
+        ? "open"
+        : process.platform === "win32"
+            ? "cmd"
+            : "xdg-open";
+    const args = process.platform === "win32" ? ["/c", "start", "", url] : [url];
+    try {
+        execFile(cmd, args, () => {
+            // ignore browser launch errors in headless environments
+        });
+    }
+    catch {
+        // ignore
+    }
+}
+async function runWebOAuthLogin(baseUrl, fetchImpl, io) {
+    const verifier = crypto.randomBytes(32).toString("base64url");
+    const challenge = crypto.createHash("sha256").update(verifier).digest("base64url");
+    const state = crypto.randomBytes(16).toString("base64url");
+    return new Promise((resolve, reject) => {
+        const server = createServer();
+        const timeout = setTimeout(() => {
+            server.close();
+            reject(new Error("Browser sign-in timed out after 180 seconds."));
+        }, 180_000);
+        server.listen(0, "127.0.0.1", async () => {
+            try {
+                const addr = server.address();
+                const port = typeof addr === "object" && addr ? addr.port : 0;
+                const redirectUri = `http://127.0.0.1:${port}/callback`;
+                const regRes = await fetchImpl(`${baseUrl}/oauth/register`, {
+                    method: "POST",
+                    headers: { "content-type": "application/json", accept: "application/json" },
+                    body: JSON.stringify({
+                        client_name: "Webcrawl Anything CLI",
+                        redirect_uris: [redirectUri],
+                    }),
+                });
+                if (!regRes.ok) {
+                    throw new Error(`Could not register CLI OAuth client (HTTP ${regRes.status}). Try \`webcrawl login --email <you@domain.com>\`.`);
+                }
+                const { client_id: clientId } = (await regRes.json());
+                const authUrl = `${baseUrl}/oauth/authorize?${new URLSearchParams({
+                    response_type: "code",
+                    client_id: clientId,
+                    redirect_uri: redirectUri,
+                    code_challenge: challenge,
+                    code_challenge_method: "S256",
+                    state,
+                }).toString()}`;
+                io.out([
+                    "Open this URL in your browser to create your webcrawl.app account or sign in (1,000 free monthly credits):",
+                    `  ${authUrl}`,
+                    "",
+                    "Waiting for browser authorization...",
+                ].join("\n") + "\n");
+                openBrowserUrl(authUrl);
+                server.on("request", async (req, res) => {
+                    try {
+                        const u = new URL(req.url || "/", redirectUri);
+                        if (u.pathname !== "/callback") {
+                            res.writeHead(404).end("Not found");
+                            return;
+                        }
+                        const code = u.searchParams.get("code");
+                        const returnedState = u.searchParams.get("state");
+                        if (!code || returnedState !== state) {
+                            res.writeHead(400, { "content-type": "text/html; charset=utf-8" });
+                            res.end("<h3>Authorization failed or state mismatch. You can close this tab.</h3>");
+                            clearTimeout(timeout);
+                            server.close();
+                            reject(new Error("OAuth callback missing code or state mismatch."));
+                            return;
+                        }
+                        const tokenRes = await fetchImpl(`${baseUrl}/oauth/token`, {
+                            method: "POST",
+                            headers: { "content-type": "application/x-www-form-urlencoded", accept: "application/json" },
+                            body: new URLSearchParams({
+                                grant_type: "authorization_code",
+                                code,
+                                client_id: clientId,
+                                redirect_uri: redirectUri,
+                                code_verifier: verifier,
+                            }).toString(),
+                        });
+                        if (!tokenRes.ok) {
+                            throw new Error(`OAuth token exchange failed (HTTP ${tokenRes.status}).`);
+                        }
+                        const tokenData = (await tokenRes.json());
+                        let finalKey = tokenData.access_token;
+                        const authedClient = new WebcrawlClient({
+                            baseUrl,
+                            apiKey: finalKey,
+                            fetch: fetchImpl,
+                            userAgent: "webcrawl-cli/1.2",
+                        });
+                        try {
+                            const keyRes = await authedClient.request("POST", "/v1/account/keys", { name: "CLI API Key" });
+                            if (keyRes.apiKey) {
+                                finalKey = keyRes.apiKey;
+                            }
+                        }
+                        catch {
+                            // keep OAuth access_token (`wc_mcp_...`) which is also accepted on all /v1/* endpoints
+                        }
+                        const meRes = await authedClient.me().catch(() => undefined);
+                        res.writeHead(200, { "content-type": "text/html; charset=utf-8" });
+                        res.end("<html><body style='font-family:system-ui;padding:2rem'><h2>✓ Webcrawl CLI authenticated!</h2><p>Your account and 1,000 monthly credits are now linked. You can close this tab and return to your terminal or agent.</p></body></html>");
+                        clearTimeout(timeout);
+                        server.close();
+                        resolve({
+                            apiKey: finalKey,
+                            email: meRes?.user?.email,
+                            creditsRemaining: meRes?.user?.creditsRemaining,
+                            creditsLimit: meRes?.user?.creditsLimit,
+                            plan: meRes?.user?.plan,
+                        });
+                    }
+                    catch (err) {
+                        clearTimeout(timeout);
+                        server.close();
+                        reject(err);
+                    }
+                });
+            }
+            catch (err) {
+                clearTimeout(timeout);
+                server.close();
+                reject(err);
+            }
+        });
+    });
+}
 async function resolveBackendClient(cfg, deps) {
     // When a custom fetch mock is injected (e.g. unit tests) or effectiveMode is cloud, use WebcrawlClient
     if (cfg.effectiveMode === "cloud" || deps.fetch) {
@@ -142,6 +291,38 @@ export async function main(argv, io, deps = {}) {
             io.err(`[${credits}]\n`);
         }
     };
+    const METERED_COMMANDS = new Set(["search", "scrape", "map", "crawl", "extract", "rerank"]);
+    if (METERED_COMMANDS.has(cmd) && cfg.effectiveMode === "cloud" && !cfg.apiKey && !deps.fetch) {
+        const authHelp = [
+            "webcrawl: Cloud mode requires a logged-in webcrawl.app account so credits can be attributed to your user (1,000 free monthly credits on signup).",
+            "",
+            "Create an account or sign in from the CLI:",
+            "  1. Email OTP (agent & terminal friendly):",
+            "     webcrawl login --email <you@domain.com>",
+            "     webcrawl login --email <you@domain.com> --code <6-digit-code>",
+            "  2. Browser OAuth redirect:",
+            "     webcrawl login --web",
+            "  3. Or run in zero-key local mode:",
+            `     webcrawl --mode local ${cmd} ${rest.join(" ")}`,
+        ].join("\n");
+        if (v.json) {
+            io.out(`${JSON.stringify({
+                success: false,
+                error: "authentication_required",
+                message: "Cloud mode requires a logged-in webcrawl.app account (1,000 free monthly credits on signup).",
+                loginCommands: {
+                    sendOtp: "webcrawl login --email <you@domain.com>",
+                    verifyOtp: "webcrawl login --email <you@domain.com> --code <6-digit-code>",
+                    browserRedirect: "webcrawl login --web",
+                    signupUrl: `${cfg.baseUrl}/?signup=cli`,
+                },
+            }, null, 2)}\n`);
+        }
+        else {
+            io.err(`${authHelp}\n`);
+        }
+        return 1;
+    }
     try {
         switch (cmd) {
             case "search": {
@@ -258,10 +439,93 @@ export async function main(argv, io, deps = {}) {
                 emit({ success: true, ...report }, prettyLines);
                 return 0;
             }
-            case "login": {
+            case "login":
+            case "signup": {
+                const authClient = new WebcrawlClient({
+                    baseUrl: cfg.baseUrl,
+                    fetch: deps.fetch,
+                    userAgent: "webcrawl-cli/1.2",
+                });
+                // 1. Browser OAuth PKCE redirect (`webcrawl login --web`)
+                if (v.web) {
+                    const webRes = await runWebOAuthLogin(cfg.baseUrl, deps.fetch ?? globalThis.fetch, io);
+                    const existing = readConfigFile(file);
+                    writeConfigFile({ ...existing, apiKey: webRes.apiKey, ...(v.url ? { baseUrl: v.url } : {}) }, file);
+                    const payload = {
+                        success: true,
+                        method: "web_oauth",
+                        email: webRes.email,
+                        plan: webRes.plan ?? "free",
+                        creditsRemaining: webRes.creditsRemaining ?? 1000,
+                        creditsLimit: webRes.creditsLimit ?? 1000,
+                        configFile: file,
+                    };
+                    emit(payload, [
+                        `✓ Authenticated${webRes.email ? ` as ${webRes.email}` : ""} (${payload.plan} plan · ${payload.creditsRemaining} credits remaining of ${payload.creditsLimit}).`,
+                        `Saved API key to ${file}.`,
+                    ].join("\n"));
+                    return 0;
+                }
+                // 2. Email OTP account creation / sign-in (`webcrawl login --email <addr> [--code <6-digit>]`)
+                if (v.email) {
+                    const email = v.email.trim();
+                    let code = v.code?.trim();
+                    if (!code) {
+                        const otpRes = await authClient.sendOtp(email);
+                        if (!otpRes.success) {
+                            throw new WebcrawlError("bad_request", otpRes.error || "Could not send verification code.");
+                        }
+                        if (otpRes.devCode && !io.isTTY) {
+                            code = otpRes.devCode;
+                        }
+                        else if (io.isTTY && !v["no-wait"]) {
+                            io.err(`Sent a 6-digit verification code to ${email} (expires in 10 minutes).\n`);
+                            code = await promptLine(io, `Enter the 6-digit code sent to ${email}: `);
+                        }
+                        else {
+                            const stepPayload = {
+                                success: true,
+                                step: "otp_sent",
+                                email: otpRes.email,
+                                expiresInSeconds: otpRes.expiresInSeconds,
+                                nextCommand: `webcrawl login --email ${email} --code <6-digit-code>`,
+                                browserFallbackUrl: `${cfg.baseUrl}/?signup=cli&email=${encodeURIComponent(email)}`,
+                            };
+                            emit(stepPayload, [
+                                `Verification code sent to ${otpRes.email} (expires in ${Math.round(otpRes.expiresInSeconds / 60)} minutes).`,
+                                `Ask the user for the 6-digit code from their email, then complete sign-in with:`,
+                                `  webcrawl login --email ${email} --code <6-digit-code>`,
+                                `Or redirect the user to sign in on the web: ${stepPayload.browserFallbackUrl}`,
+                            ].join("\n"));
+                            return 0;
+                        }
+                    }
+                    const verified = await authClient.verifyOtp({ email, code, name: v.name });
+                    if (!verified.success || !verified.apiKey) {
+                        throw new WebcrawlError("auth", verified.error || "Could not verify the 6-digit code.");
+                    }
+                    const existing = readConfigFile(file);
+                    writeConfigFile({ ...existing, apiKey: verified.apiKey, ...(v.url ? { baseUrl: v.url } : {}) }, file);
+                    const u = verified.user;
+                    const payload = {
+                        success: true,
+                        isNewUser: verified.isNewUser ?? false,
+                        email: u?.email ?? email,
+                        plan: u?.plan ?? "free",
+                        creditsRemaining: u?.creditsRemaining ?? 1000,
+                        creditsLimit: u?.creditsLimit ?? 1000,
+                        configFile: file,
+                    };
+                    emit(payload, [
+                        `✓ ${verified.isNewUser ? "Created webcrawl.app account" : "Signed in"} as ${payload.email} (${payload.plan} plan · ${payload.creditsRemaining} credits remaining of ${payload.creditsLimit}).`,
+                        `Saved API key (${verified.apiKey.slice(0, 8)}…${verified.apiKey.slice(-4)}) to ${file}.`,
+                    ].join("\n"));
+                    return 0;
+                }
+                // 3. Direct API key (`webcrawl login --key wc_live_...` or interactive paste)
                 const key = (v.key ?? (await readKeyInteractively(io))).trim();
                 if (!key) {
-                    throw new UsageError("No key provided.");
+                    throw new UsageError("No key provided. Pass --key <wc_live_...>, --email <you@domain.com>, or --web.");
                 }
                 const existing = readConfigFile(file);
                 writeConfigFile({ ...existing, apiKey: key, ...(v.url ? { baseUrl: v.url } : {}) }, file);
@@ -273,7 +537,7 @@ export async function main(argv, io, deps = {}) {
                 const lines = [
                     `endpoint: ${cfg.baseUrl} (${cfg.source.baseUrl})`,
                     `mode:     ${cfg.effectiveMode} (${cfg.source.mode})`,
-                    `key:      ${masked}${cfg.apiKey ? ` (${cfg.source.apiKey})` : ". Run `webcrawl login` or set WEBCRAWL_API_KEY (local mode works with zero key)."}`,
+                    `key:      ${masked}${cfg.apiKey ? ` (${cfg.source.apiKey})` : ". Run `webcrawl login` (`--email <you@domain.com>` or `--web`) or set WEBCRAWL_API_KEY (local mode works with zero key)."}`,
                 ];
                 if (cfg.apiKey) {
                     const u = await client.usage();
@@ -281,6 +545,10 @@ export async function main(argv, io, deps = {}) {
                         io.out(`${JSON.stringify({ endpoint: cfg.baseUrl, mode: cfg.effectiveMode, keySource: cfg.source.apiKey, usage: u }, null, 2)}\n`);
                         return 0;
                     }
+                    const userObj = (u.user ?? {});
+                    if (typeof userObj.email === "string") {
+                        lines.push(`account:  ${userObj.email} (${userObj.plan ?? "free"} plan)`);
+                    }
                     const credits = (u.credits ?? u);
                     if (typeof credits.remaining === "number") {
                         lines.push(`credits:  ${credits.remaining} remaining of ${credits.limit ?? "?"}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "Webcrawl Anything CLI — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -53,7 +53,7 @@
     "test": "node --conditions=development --import tsx --test test/*.test.ts"
   },
   "dependencies": {
-    "@webcrawl/client": "^1.2.0"
+    "@webcrawl/client": "^1.2.2"
   },
   "engines": {
     "node": ">=20"