webcrawl 1.3.4 → 1.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +9 -9
- package/package.json +3 -3
package/dist/cli.d.ts
CHANGED
|
@@ -15,5 +15,5 @@ export interface Deps {
|
|
|
15
15
|
/** Working directory override for `webcrawl setup` tests. */
|
|
16
16
|
cwd?: string;
|
|
17
17
|
}
|
|
18
|
-
export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n search <query> Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n scrape <url> Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n map <url> List a site's URLs. --filter <text>, -n limit\n crawl <url> Read several pages. -n max pages (default 5, max 25)\n extract <url> Structured JSON. --prompt <text> or --schema <file.json>\n rerank <query> Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n targets List all supported crawl & search targets and their requirements\n categories List search categories and vertical filters\n setup Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n doctor Check local crawl engine, parsers, stealth browser, and cloud configuration\n login / signup Create an account or sign in (--email <addr> [--code <6-digit>],
|
|
18
|
+
export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n search <query> Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n scrape <url> Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n map <url> List a site's URLs. --filter <text>, -n limit\n crawl <url> Read several pages. -n max pages (default 5, max 25)\n extract <url> Structured JSON. --prompt <text> or --schema <file.json>\n rerank <query> Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n targets List all supported crawl & search targets and their requirements\n categories List search categories and vertical filters\n setup Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n doctor Check local crawl engine, parsers, stealth browser, and cloud configuration\n login / signup Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>\n whoami Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n --mode <m> Execution mode: auto (default: local without key, cloud with key), local, or cloud\n --email <addr> Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n --code <6-digit> 6-digit email verification code from `webcrawl login --email <addr>`\n --name <name> Optional display name when creating an account with `--email`\n --web Sign in via browser OAuth (default for login; no need to pass it)\n --json Print the raw JSON response\n --api-key <key> Override WEBCRAWL_API_KEY\n --url <base> Override WEBCRAWL_URL (default https://webcrawl.app)\n --country, --city Where to search from (default: this machine's timezone)\n --no-cache Skip cached pages (scrape)\n --agent <name> Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n --schema-profile MCP schema profile for setup: compact (default) or full\n -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
|
|
19
19
|
export declare function main(argv: string[], io: Io, deps?: Deps): Promise<number>;
|
package/dist/cli.js
CHANGED
|
@@ -22,7 +22,7 @@ Commands
|
|
|
22
22
|
categories List search categories and vertical filters
|
|
23
23
|
setup Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)
|
|
24
24
|
doctor Check local crawl engine, parsers, stealth browser, and cloud configuration
|
|
25
|
-
login / signup Create an account or sign in (--email <addr> [--code <6-digit>],
|
|
25
|
+
login / signup Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>
|
|
26
26
|
whoami Show execution mode (local vs cloud), account email, key source, and credit balance
|
|
27
27
|
|
|
28
28
|
Options
|
|
@@ -30,7 +30,7 @@ Options
|
|
|
30
30
|
--email <addr> Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)
|
|
31
31
|
--code <6-digit> 6-digit email verification code from \`webcrawl login --email <addr>\`
|
|
32
32
|
--name <name> Optional display name when creating an account with \`--email\`
|
|
33
|
-
--web Sign in
|
|
33
|
+
--web Sign in via browser OAuth (default for login; no need to pass it)
|
|
34
34
|
--json Print the raw JSON response
|
|
35
35
|
--api-key <key> Override WEBCRAWL_API_KEY
|
|
36
36
|
--url <base> Override WEBCRAWL_URL (default https://webcrawl.app)
|
|
@@ -302,7 +302,7 @@ export async function main(argv, io, deps = {}) {
|
|
|
302
302
|
" webcrawl login --email <you@domain.com>",
|
|
303
303
|
" webcrawl login --email <you@domain.com> --code <6-digit-code>",
|
|
304
304
|
" 2. Browser OAuth redirect:",
|
|
305
|
-
" webcrawl login
|
|
305
|
+
" webcrawl login",
|
|
306
306
|
" 3. Or run in zero-key local mode:",
|
|
307
307
|
` webcrawl --mode local ${cmd} ${rest.join(" ")}`,
|
|
308
308
|
].join("\n");
|
|
@@ -314,7 +314,7 @@ export async function main(argv, io, deps = {}) {
|
|
|
314
314
|
loginCommands: {
|
|
315
315
|
sendOtp: "webcrawl login --email <you@domain.com>",
|
|
316
316
|
verifyOtp: "webcrawl login --email <you@domain.com> --code <6-digit-code>",
|
|
317
|
-
browserRedirect: "webcrawl login
|
|
317
|
+
browserRedirect: "webcrawl login",
|
|
318
318
|
signupUrl: `${cfg.baseUrl}/?signup=cli`,
|
|
319
319
|
},
|
|
320
320
|
}, null, 2)}\n`);
|
|
@@ -447,8 +447,8 @@ export async function main(argv, io, deps = {}) {
|
|
|
447
447
|
fetch: deps.fetch,
|
|
448
448
|
userAgent: "webcrawl-cli/1.2",
|
|
449
449
|
});
|
|
450
|
-
// 1. Browser OAuth PKCE redirect (`webcrawl login
|
|
451
|
-
if (v.web) {
|
|
450
|
+
// 1. Browser OAuth PKCE redirect (default for `webcrawl login`; `--web` kept for compatibility)
|
|
451
|
+
if (v.web || (!v.email && !v.key)) {
|
|
452
452
|
const webRes = await runWebOAuthLogin(cfg.baseUrl, deps.fetch ?? globalThis.fetch, io);
|
|
453
453
|
const existing = readConfigFile(file);
|
|
454
454
|
writeConfigFile({ ...existing, apiKey: webRes.apiKey, ...(v.url ? { baseUrl: v.url } : {}) }, file);
|
|
@@ -524,9 +524,9 @@ export async function main(argv, io, deps = {}) {
|
|
|
524
524
|
return 0;
|
|
525
525
|
}
|
|
526
526
|
// 3. Direct API key (`webcrawl login --key wc_live_...` or interactive paste)
|
|
527
|
-
const key =
|
|
527
|
+
const key = v.key?.trim() ?? (await readKeyInteractively(io)).trim();
|
|
528
528
|
if (!key) {
|
|
529
|
-
throw new UsageError("No key provided. Pass --key <wc_live_
|
|
529
|
+
throw new UsageError("No key provided. Pass --key <wc_live_...> or --email <you@domain.com>, or run `webcrawl login` to sign in via your browser.");
|
|
530
530
|
}
|
|
531
531
|
const existing = readConfigFile(file);
|
|
532
532
|
writeConfigFile({ ...existing, apiKey: key, ...(v.url ? { baseUrl: v.url } : {}) }, file);
|
|
@@ -538,7 +538,7 @@ export async function main(argv, io, deps = {}) {
|
|
|
538
538
|
const lines = [
|
|
539
539
|
`endpoint: ${cfg.baseUrl} (${cfg.source.baseUrl})`,
|
|
540
540
|
`mode: ${cfg.effectiveMode} (${cfg.source.mode})`,
|
|
541
|
-
`key: ${masked}${cfg.apiKey ? ` (${cfg.source.apiKey})` : ". Run `webcrawl login` (`--email <you@domain.com>`
|
|
541
|
+
`key: ${masked}${cfg.apiKey ? ` (${cfg.source.apiKey})` : ". Run `webcrawl login` to sign in via your browser (or `--email <you@domain.com>`), or set WEBCRAWL_API_KEY (local mode works with zero key)."}`,
|
|
542
542
|
];
|
|
543
543
|
if (cfg.apiKey) {
|
|
544
544
|
const u = await client.usage();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "webcrawl",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.5",
|
|
4
4
|
"description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -127,8 +127,8 @@
|
|
|
127
127
|
"@ai-sdk/gateway": "^4.0.123",
|
|
128
128
|
"@firecrawl/pdf-inspector": "^1.25.2",
|
|
129
129
|
"@vercel/sandbox": "^1.0.0",
|
|
130
|
-
"@webcrawl/client": "^1.3.
|
|
131
|
-
"@webcrawl/curl-impersonate": "^1.3.
|
|
130
|
+
"@webcrawl/client": "^1.3.5",
|
|
131
|
+
"@webcrawl/curl-impersonate": "^1.3.5",
|
|
132
132
|
"ai": "^7.0.150",
|
|
133
133
|
"impit": "^0.14.7",
|
|
134
134
|
"mammoth": "^1.13.0",
|