webcrawl 1.3.4 → 1.3.6

package/dist/cli.d.ts

@@ -10,10 +10,12 @@ export interface Deps {
     fetch?: typeof fetch;
     /** Config file path override (tests). */
     configFile?: string;
+    /** Browser opener override (tests). */
+    openUrl?: (url: string) => void;
     /** Home directory override for `webcrawl setup` tests. */
     homeDir?: string;
     /** Working directory override for `webcrawl setup` tests. */
     cwd?: string;
 }
-export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in (--email <addr> [--code <6-digit>], --web, or --key <key>)\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in or create an account in your browser via OAuth PKCE redirect\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
+export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in via browser OAuth (default for login; no need to pass it)\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
 export declare function main(argv: string[], io: Io, deps?: Deps): Promise<number>;

package/dist/cli.js

@@ -22,7 +22,7 @@ Commands
   categories          List search categories and vertical filters
   setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)
   doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration
-  login / signup      Create an account or sign in (--email <addr> [--code <6-digit>], --web, or --key <key>)
+  login / signup      Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>
   whoami              Show execution mode (local vs cloud), account email, key source, and credit balance
 
 Options
@@ -30,7 +30,7 @@ Options
   --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)
   --code <6-digit>    6-digit email verification code from \`webcrawl login --email <addr>\`
   --name <name>       Optional display name when creating an account with \`--email\`
-  --web               Sign in or create an account in your browser via OAuth PKCE redirect
+  --web               Sign in via browser OAuth (default for login; no need to pass it)
   --json              Print the raw JSON response
   --api-key <key>     Override WEBCRAWL_API_KEY
   --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)
@@ -112,7 +112,7 @@ function openBrowserUrl(url) {
         // ignore
     }
 }
-async function runWebOAuthLogin(baseUrl, fetchImpl, io) {
+async function runWebOAuthLogin(baseUrl, fetchImpl, io, openUrl = openBrowserUrl) {
     const verifier = crypto.randomBytes(32).toString("base64url");
     const challenge = crypto.createHash("sha256").update(verifier).digest("base64url");
     const state = crypto.randomBytes(16).toString("base64url");
@@ -153,7 +153,8 @@ async function runWebOAuthLogin(baseUrl, fetchImpl, io) {
                     "",
                     "Waiting for browser authorization...",
                 ].join("\n") + "\n");
-                openBrowserUrl(authUrl);
+                openUrl(authUrl);
+                let handled = false;
                 server.on("request", async (req, res) => {
                     try {
                         const u = new URL(req.url || "/", redirectUri);
@@ -163,14 +164,29 @@ async function runWebOAuthLogin(baseUrl, fetchImpl, io) {
                         }
                         const code = u.searchParams.get("code");
                         const returnedState = u.searchParams.get("state");
-                        if (!code || returnedState !== state) {
+                        if (handled) {
+                            // Browser retry / prefetch of the same callback: the code is single-use, so never exchange it twice.
+                            res.writeHead(200, { "content-type": "text/html; charset=utf-8" });
+                            res.end("<h3>Already received. You can close this tab and return to your terminal.</h3>");
+                            return;
+                        }
+                        const oauthError = u.searchParams.get("error");
+                        if (oauthError && returnedState === state) {
+                            handled = true;
                             res.writeHead(400, { "content-type": "text/html; charset=utf-8" });
-                            res.end("<h3>Authorization failed or state mismatch. You can close this tab.</h3>");
+                            res.end("<h3>Authorization was cancelled. You can close this tab.</h3>");
                             clearTimeout(timeout);
                             server.close();
-                            reject(new Error("OAuth callback missing code or state mismatch."));
+                            reject(new Error(`OAuth authorization failed: ${oauthError}.`));
+                            return;
+                        }
+                        if (!code || returnedState !== state) {
+                            // Stray or forged request: reject it but keep waiting for the real callback.
+                            res.writeHead(400, { "content-type": "text/html; charset=utf-8" });
+                            res.end("<h3>Authorization failed or state mismatch.</h3>");
                             return;
                         }
+                        handled = true;
                         const tokenRes = await fetchImpl(`${baseUrl}/oauth/token`, {
                             method: "POST",
                             headers: { "content-type": "application/x-www-form-urlencoded", accept: "application/json" },
@@ -183,7 +199,11 @@ async function runWebOAuthLogin(baseUrl, fetchImpl, io) {
                             }).toString(),
                         });
                         if (!tokenRes.ok) {
-                            throw new Error(`OAuth token exchange failed (HTTP ${tokenRes.status}).`);
+                            const detail = (await tokenRes.json().catch(() => ({})));
+                            const reason = detail.error_description || detail.error;
+                            res.writeHead(400, { "content-type": "text/html; charset=utf-8" });
+                            res.end("<h3>Webcrawl sign-in failed. Return to your terminal for details.</h3>");
+                            throw new Error(`OAuth token exchange failed (HTTP ${tokenRes.status}${reason ? `: ${reason}` : ""}). Run \`webcrawl login\` again, or use \`webcrawl login --email <you@domain.com>\`.`);
                         }
                         const tokenData = (await tokenRes.json());
                         let finalKey = tokenData.access_token;
@@ -302,7 +322,7 @@ export async function main(argv, io, deps = {}) {
             "     webcrawl login --email <you@domain.com>",
             "     webcrawl login --email <you@domain.com> --code <6-digit-code>",
             "  2. Browser OAuth redirect:",
-            "     webcrawl login --web",
+            "     webcrawl login",
             "  3. Or run in zero-key local mode:",
             `     webcrawl --mode local ${cmd} ${rest.join(" ")}`,
         ].join("\n");
@@ -314,7 +334,7 @@ export async function main(argv, io, deps = {}) {
                 loginCommands: {
                     sendOtp: "webcrawl login --email <you@domain.com>",
                     verifyOtp: "webcrawl login --email <you@domain.com> --code <6-digit-code>",
-                    browserRedirect: "webcrawl login --web",
+                    browserRedirect: "webcrawl login",
                     signupUrl: `${cfg.baseUrl}/?signup=cli`,
                 },
             }, null, 2)}\n`);
@@ -447,9 +467,9 @@ export async function main(argv, io, deps = {}) {
                     fetch: deps.fetch,
                     userAgent: "webcrawl-cli/1.2",
                 });
-                // 1. Browser OAuth PKCE redirect (`webcrawl login --web`)
-                if (v.web) {
-                    const webRes = await runWebOAuthLogin(cfg.baseUrl, deps.fetch ?? globalThis.fetch, io);
+                // 1. Browser OAuth PKCE redirect (default for `webcrawl login`; `--web` kept for compatibility)
+                if (v.web || (!v.email && !v.key)) {
+                    const webRes = await runWebOAuthLogin(cfg.baseUrl, deps.fetch ?? globalThis.fetch, io, deps.openUrl);
                     const existing = readConfigFile(file);
                     writeConfigFile({ ...existing, apiKey: webRes.apiKey, ...(v.url ? { baseUrl: v.url } : {}) }, file);
                     const payload = {
@@ -524,9 +544,9 @@ export async function main(argv, io, deps = {}) {
                     return 0;
                 }
                 // 3. Direct API key (`webcrawl login --key wc_live_...` or interactive paste)
-                const key = (v.key ?? (await readKeyInteractively(io))).trim();
+                const key = v.key?.trim() ?? (await readKeyInteractively(io)).trim();
                 if (!key) {
-                    throw new UsageError("No key provided. Pass --key <wc_live_...>, --email <you@domain.com>, or --web.");
+                    throw new UsageError("No key provided. Pass --key <wc_live_...> or --email <you@domain.com>, or run `webcrawl login` to sign in via your browser.");
                 }
                 const existing = readConfigFile(file);
                 writeConfigFile({ ...existing, apiKey: key, ...(v.url ? { baseUrl: v.url } : {}) }, file);
@@ -538,7 +558,7 @@ export async function main(argv, io, deps = {}) {
                 const lines = [
                     `endpoint: ${cfg.baseUrl} (${cfg.source.baseUrl})`,
                     `mode:     ${cfg.effectiveMode} (${cfg.source.mode})`,
-                    `key:      ${masked}${cfg.apiKey ? ` (${cfg.source.apiKey})` : ". Run `webcrawl login` (`--email <you@domain.com>` or `--web`) or set WEBCRAWL_API_KEY (local mode works with zero key)."}`,
+                    `key:      ${masked}${cfg.apiKey ? ` (${cfg.source.apiKey})` : ". Run `webcrawl login` to sign in via your browser (or `--email <you@domain.com>`), or set WEBCRAWL_API_KEY (local mode works with zero key)."}`,
                 ];
                 if (cfg.apiKey) {
                     const u = await client.usage();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.3.4",
+  "version": "1.3.6",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.3.4",
-    "@webcrawl/curl-impersonate": "^1.3.4",
+    "@webcrawl/client": "^1.3.6",
+    "@webcrawl/curl-impersonate": "^1.3.6",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",