webcrawl 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +133 -14
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +275 -7
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1,43 +1,162 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Webcrawl Anything
|
|
2
2
|
|
|
3
3
|
**Crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, shopping offer, travel fare, or CVE advisory — from a single CLI and MCP server.**
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
[](https://www.npmjs.com/package/webcrawl)
|
|
6
|
+
[](https://www.npmjs.com/package/@webcrawl/mcp)
|
|
7
|
+
[](https://webcrawl.app)
|
|
8
|
+
[](https://github.com/brainfish-ai/webcrawl-anything/blob/main/LICENSE)
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
### What You Can Crawl & Search
|
|
13
|
+
|
|
14
|
+
| Target | Returns | Requirements |
|
|
15
|
+
|---|---|---|
|
|
16
|
+
| **Websites & JS SPAs** (`web`) | Clean Markdown, semantic headings, tables, code blocks, links, OpenGraph/JSON-LD, and SHA-256 content hashes | Zero-key local (`impit` TLS impersonation + local Chrome) or [`webcrawl.app`](https://webcrawl.app) cloud stealth browsers |
|
|
17
|
+
| **PDF & Office Docs** (`pdf-office`) | Page-numbered Markdown, tables, slide notes, and automatic OCR on scanned PDF pages (`pagesOcr` + honest `pagesSkipped`) | Built-in (`@firecrawl/pdf-inspector`, `pdf-parse`, `mammoth`, `officeparser`, `tesseract.js`) |
|
|
18
|
+
| **YouTube & Media** (`youtube-media`) | Timestamped transcript Markdown, video title/channel/duration metadata, and spoken audio transcription | Built-in zero-key YouTube multi-client caption extractor; optional audio transcription provider for raw MP3/MP4 |
|
|
19
|
+
| **Shopping & Products** (`shopping`) | Multi-retailer price comparison, currency-normalized offers, stock status, ratings, review counts, and shipping | Zero-key local HTML/JSON-LD product extractor (`-c shopping`); optional `SERPER_API_KEY` |
|
|
20
|
+
| **Flights & Hotels** (`travel`) | Structured flight routes (`origin`, `destination`, `airline`, `stops`, `price`) and hotel nightly rates, stars, and guest ratings | Zero-key local search + schema.org extractor (`-c flights` / `-c hotels`) |
|
|
21
|
+
| **Real Estate & Local Business** (`real-estate`) | Property listings (`bedrooms`, `bathrooms`, `parking`, `suburb`, `portal`) and local business hours, phone, address, and reviews | Zero-key local extractor (`-c real-estate --sub buy\|rent\|sold` / `-c business`) |
|
|
22
|
+
| **Security & CVEs** (`cve`) | Authoritative CVE & GHSA records with CVSS v3.1/v4.0 scores, CWEs, affected/fixed package versions, and CISA KEV flags | Built-in live NVD 2.0 + OSV.dev + CISA KEV enrichment (`-c security --sub cve`) |
|
|
23
|
+
| **GitHub Repos, Issues & Releases** (`github`) | Structured repo metadata (stars, forks, license, topics, language), README Markdown, or issue/PR/release discussion threads | Built-in zero-key HTML & GitHub extractor (`npx webcrawl scrape https://github.com/owner/repo`) |
|
|
24
|
+
| **ArXiv & Academic Papers** (`arxiv`) | Paper title, authors, abstract, primary category, submission date, direct PDF link, and full parsed paper Markdown | Built-in ArXiv abstract & PDF inspector (`npx webcrawl scrape https://arxiv.org/abs/...`) |
|
|
25
|
+
| **llms.txt & Sitemaps** (`llms-txt`) | Structured documentation manifest, section hierarchy, and crawlable doc URLs | Built-in (`npx webcrawl scrape https://docs.example.com/llms.txt` or `npx webcrawl map`) |
|
|
26
|
+
|
|
27
|
+
Run `npx webcrawl targets` at any time to inspect all registered targets from your terminal.
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
### Quickstart
|
|
6
32
|
|
|
7
33
|
One command registers the `@webcrawl/mcp` server and installs the `webcrawl-anything` Agent Skill (`SKILL.md`) for Claude Code, Cursor, and Codex:
|
|
8
34
|
|
|
9
35
|
```bash
|
|
10
36
|
npx webcrawl setup
|
|
11
|
-
npx webcrawl doctor
|
|
12
37
|
```
|
|
13
38
|
|
|
14
|
-
|
|
39
|
+
Or target specific coding agents:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
npx webcrawl setup --agent claude --agent cursor --agent codex --agent gemini --agent windsurf
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Check your local engine, stealth browser, and parser readiness:
|
|
15
46
|
|
|
16
47
|
```bash
|
|
17
|
-
|
|
18
|
-
|
|
48
|
+
npx webcrawl doctor
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
### Zero-Key CLI Usage
|
|
19
54
|
|
|
20
|
-
|
|
55
|
+
Webcrawl Anything runs **out-of-the-box locally with zero API keys** using the built-in `impit` TLS-impersonated fetcher, document/media parsers, BM25 + semantic ranker, and vertical extractors:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
# Scrape any website, GitHub repo, ArXiv paper, PDF, or YouTube video into clean Markdown
|
|
21
59
|
npx webcrawl scrape https://github.com/torvalds/linux
|
|
22
60
|
npx webcrawl scrape https://arxiv.org/abs/1706.03762
|
|
61
|
+
npx webcrawl scrape https://www.youtube.com/watch?v=dQw4w9WgXcQ
|
|
23
62
|
|
|
24
|
-
# Search the web or structured verticals
|
|
63
|
+
# Search the live web or structured verticals
|
|
25
64
|
npx webcrawl search "react 19 compiler" -n 5
|
|
26
65
|
npx webcrawl search "sony wh-1000xm5" -c shopping
|
|
27
66
|
npx webcrawl search "boutique hotels in shibuya" -c hotels
|
|
67
|
+
npx webcrawl search "3 bedroom house in marrickville" -c real-estate --sub buy
|
|
28
68
|
npx webcrawl search "CVE-2024-3094 xz utils" -c security --sub cve
|
|
29
69
|
|
|
30
|
-
# Discover
|
|
70
|
+
# Discover every URL on a domain (sitemap.xml + robots.txt + link graph)
|
|
31
71
|
npx webcrawl map https://docs.stripe.com --filter webhook -n 50
|
|
72
|
+
|
|
73
|
+
# Crawl up to 25 pages in parallel with target evidence
|
|
32
74
|
npx webcrawl crawl https://docs.stripe.com/webhooks -n 5
|
|
33
75
|
|
|
34
|
-
#
|
|
76
|
+
# Extract structured JSON from any page
|
|
77
|
+
npx webcrawl extract https://news.ycombinator.com --prompt "top 5 stories with points"
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
### Example Prompts (Once Connected to Your Agent)
|
|
83
|
+
|
|
84
|
+
Once `npx webcrawl setup` has registered the MCP server and skill, ask Claude Code, Cursor, or Codex directly:
|
|
85
|
+
|
|
86
|
+
- *"Search for CVE-2024-3094 in the security category and show me the CVSS score, CISA KEV status, and fixed versions."*
|
|
87
|
+
- *"Scrape `https://arxiv.org/abs/2305.18290` and summarize the DPO objective and experimental results."*
|
|
88
|
+
- *"Compare retail prices and stock availability for the Apple Mac Mini M4 across US retailers using `web_search` with `category: shopping`."*
|
|
89
|
+
- *"Map `https://modelcontextprotocol.io`, pick the specification pages, and crawl them into Markdown."*
|
|
90
|
+
- *"Parse the PDF at `https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf` and verify whether any pages required OCR."*
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
### Local Engine vs. [`webcrawl.app`](https://webcrawl.app) Cloud
|
|
95
|
+
|
|
96
|
+
Webcrawl Anything uses the **exact same TypeScript engine (`src/`)** locally and in the hosted [`webcrawl.app`](https://webcrawl.app) cloud API. You can start with zero keys locally and upgrade to cloud with one command whenever you need managed stealth browsers or sub-50ms global cache hits:
|
|
97
|
+
|
|
98
|
+
| Capability | Open-Source Local (`--mode local`) | [`webcrawl.app`](https://webcrawl.app) Cloud (`--mode cloud`) |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| **API Key Required** | **None (Zero-Key)** | `WEBCRAWL_API_KEY` (`npx webcrawl login`) |
|
|
101
|
+
| **Core Engine & Target Plugins** | 100% shared (`src/engine`, `src/targets`, `src/extract`) | 100% shared |
|
|
102
|
+
| **PDF, Office, YouTube & CVE Parsers** | Built-in local execution | Hosted + background async workers (`/v1/runs/:id`) |
|
|
103
|
+
| **Anti-Bot & Cloudflare Bypass** | Local `impit` TLS + local Chrome | Managed residential stealth browser fleet + Cloudflare rendering |
|
|
104
|
+
| **Search Fan-Out** | Built-in DuckDuckGo/Bing + BYO keys | Managed multi-provider fan-out (Serper, Brave, Jina, Tavily) |
|
|
105
|
+
| **Global Content Cache** | Local in-memory LRU | Sub-50ms global SHA-256 content-addressed cache |
|
|
106
|
+
|
|
107
|
+
To connect your CLI and MCP server to [`webcrawl.app`](https://webcrawl.app) cloud:
|
|
108
|
+
|
|
109
|
+
```bash
|
|
35
110
|
npx webcrawl login
|
|
36
111
|
npx webcrawl whoami
|
|
37
112
|
```
|
|
38
113
|
|
|
39
|
-
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
### Compact MCP Schema Profile for Context-Constrained Agents
|
|
117
|
+
|
|
118
|
+
By default, `npx webcrawl setup` configures `WEBCRAWL_MCP_SCHEMA_PROFILE=compact` so MCP tool schemas use minimal token overhead inside Claude Code, Cursor, and local LLM context windows while keeping full server-side validation.
|
|
119
|
+
|
|
120
|
+
To switch to verbose tool parameter descriptions:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
npx webcrawl setup --schema-profile full
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
---
|
|
127
|
+
|
|
128
|
+
### Architecture & Shared Engine Flywheel
|
|
129
|
+
|
|
130
|
+
```text
|
|
131
|
+
webcrawl/
|
|
132
|
+
├── src/
|
|
133
|
+
│ ├── targets/ # Pluggable Target & Extractor Registry (web, pdf-office, youtube, github, arxiv, llms-txt, shopping, travel, real-estate, cve)
|
|
134
|
+
│ ├── engine/ # Multi-tier fetcher (impit TLS impersonation → stealth browser → HTML-to-Markdown + SHA-256 cache)
|
|
135
|
+
│ ├── parse/ # Native PDF inspector + OCR fallback, DOCX/XLSX/PPTX parser, YouTube caption extractor
|
|
136
|
+
│ ├── extract/ # Vertical structured extractors (shopping, flights, hotels, real-estate, business, CVE/NVD/OSV)
|
|
137
|
+
│ ├── search/ # Multi-provider search fan-out + vertical pipelines
|
|
138
|
+
│ ├── ranker/ # Hybrid BM25 + semantic vector + RRF + MMR diversity reranker
|
|
139
|
+
│ └── local-engine.ts # In-process zero-key backend powering CLI & MCP locally
|
|
140
|
+
├── packages/
|
|
141
|
+
│ ├── cli/ # `webcrawl` CLI (`npx webcrawl search|scrape|map|crawl|extract|rerank|targets|setup|doctor`)
|
|
142
|
+
│ ├── mcp/ # `@webcrawl/mcp` stdio & Streamable HTTP MCP server
|
|
143
|
+
│ └── client/ # `@webcrawl/client` zero-dependency TypeScript SDK
|
|
144
|
+
└── api/
|
|
145
|
+
└── index.ts # Production `webcrawl.app` API powered by the exact same `src/` engine
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Every community improvement to a target plugin (`src/targets/`), vertical extractor (`src/extract/`), or document parser (`src/parse/`) immediately powers both local CLI/MCP users and the hosted [`webcrawl.app`](https://webcrawl.app) platform.
|
|
149
|
+
|
|
150
|
+
---
|
|
151
|
+
|
|
152
|
+
### Contributing a New Target or Extractor
|
|
153
|
+
|
|
154
|
+
Want to add a new target (e.g., Reddit threads, Hacker News discussions, SEC EDGAR filings, npm/PyPI packages, or a regional real-estate/shopping portal)?
|
|
155
|
+
|
|
156
|
+
See **[CONTRIBUTING.md](https://github.com/brainfish-ai/webcrawl-anything/blob/main/CONTRIBUTING.md)** for the 3-step guide to registering a `CrawlTargetPlugin` in `src/targets/` with a fixture test in `src/__tests__/targets.test.ts`.
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
### License
|
|
40
161
|
|
|
41
|
-
|
|
42
|
-
- **Cloud Platform:** [https://webcrawl.app](https://webcrawl.app)
|
|
43
|
-
- **License:** MIT
|
|
162
|
+
MIT
|
package/dist/cli.d.ts
CHANGED
|
@@ -15,5 +15,5 @@ export interface Deps {
|
|
|
15
15
|
/** Working directory override for `webcrawl setup` tests. */
|
|
16
16
|
cwd?: string;
|
|
17
17
|
}
|
|
18
|
-
export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n search <query> Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n scrape <url> Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n map <url> List a site's URLs. --filter <text>, -n limit\n crawl <url> Read several pages. -n max pages (default 5, max 25)\n extract <url> Structured JSON. --prompt <text> or --schema <file.json>\n rerank <query> Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n targets List all supported crawl & search targets and their requirements\n categories List search categories and vertical filters\n setup Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n doctor Check local crawl engine, parsers, stealth browser, and cloud configuration\n login
|
|
18
|
+
export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n search <query> Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n scrape <url> Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n map <url> List a site's URLs. --filter <text>, -n limit\n crawl <url> Read several pages. -n max pages (default 5, max 25)\n extract <url> Structured JSON. --prompt <text> or --schema <file.json>\n rerank <query> Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n targets List all supported crawl & search targets and their requirements\n categories List search categories and vertical filters\n setup Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n doctor Check local crawl engine, parsers, stealth browser, and cloud configuration\n login / signup Create an account or sign in (--email <addr> [--code <6-digit>], --web, or --key <key>)\n whoami Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n --mode <m> Execution mode: auto (default: local without key, cloud with key), local, or cloud\n --email <addr> Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n --code <6-digit> 6-digit email verification code from `webcrawl login --email <addr>`\n --name <name> Optional display name when creating an account with `--email`\n --web Sign in or create an account in your browser via OAuth PKCE redirect\n --json Print the raw JSON response\n --api-key <key> Override WEBCRAWL_API_KEY\n --url <base> Override WEBCRAWL_URL (default https://webcrawl.app)\n --country, --city Where to search from (default: this machine's timezone)\n --no-cache Skip cached pages (scrape)\n --agent <name> Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n --schema-profile MCP schema profile for setup: compact (default) or full\n -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
|
|
19
19
|
export declare function main(argv: string[], io: Io, deps?: Deps): Promise<number>;
|
package/dist/cli.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import crypto from "node:crypto";
|
|
1
3
|
import { readFileSync } from "node:fs";
|
|
4
|
+
import { createServer } from "node:http";
|
|
2
5
|
import { createInterface } from "node:readline";
|
|
3
6
|
import { parseArgs } from "node:util";
|
|
4
7
|
import { WebcrawlClient, WebcrawlError, configPath, describeError, readConfigFile, resolveConfig, writeConfigFile, } from "@webcrawl/client";
|
|
@@ -19,11 +22,15 @@ Commands
|
|
|
19
22
|
categories List search categories and vertical filters
|
|
20
23
|
setup Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)
|
|
21
24
|
doctor Check local crawl engine, parsers, stealth browser, and cloud configuration
|
|
22
|
-
login
|
|
23
|
-
whoami Show execution mode (local vs cloud), key source,
|
|
25
|
+
login / signup Create an account or sign in (--email <addr> [--code <6-digit>], --web, or --key <key>)
|
|
26
|
+
whoami Show execution mode (local vs cloud), account email, key source, and credit balance
|
|
24
27
|
|
|
25
28
|
Options
|
|
26
29
|
--mode <m> Execution mode: auto (default: local without key, cloud with key), local, or cloud
|
|
30
|
+
--email <addr> Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)
|
|
31
|
+
--code <6-digit> 6-digit email verification code from \`webcrawl login --email <addr>\`
|
|
32
|
+
--name <name> Optional display name when creating an account with \`--email\`
|
|
33
|
+
--web Sign in or create an account in your browser via OAuth PKCE redirect
|
|
27
34
|
--json Print the raw JSON response
|
|
28
35
|
--api-key <key> Override WEBCRAWL_API_KEY
|
|
29
36
|
--url <base> Override WEBCRAWL_URL (default https://webcrawl.app)
|
|
@@ -52,6 +59,11 @@ const OPTIONS = {
|
|
|
52
59
|
doc: { type: "string", multiple: true },
|
|
53
60
|
file: { type: "string" },
|
|
54
61
|
key: { type: "string" },
|
|
62
|
+
email: { type: "string" },
|
|
63
|
+
code: { type: "string" },
|
|
64
|
+
name: { type: "string" },
|
|
65
|
+
web: { type: "boolean" },
|
|
66
|
+
"no-wait": { type: "boolean" },
|
|
55
67
|
"no-cache": { type: "boolean" },
|
|
56
68
|
content: { type: "boolean" },
|
|
57
69
|
agent: { type: "string", multiple: true },
|
|
@@ -69,18 +81,155 @@ function int(v, name) {
|
|
|
69
81
|
}
|
|
70
82
|
return n;
|
|
71
83
|
}
|
|
72
|
-
async function
|
|
84
|
+
async function promptLine(io, promptText) {
|
|
73
85
|
if (!io.isTTY) {
|
|
74
86
|
return (await io.readStdin()).trim();
|
|
75
87
|
}
|
|
76
88
|
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
77
89
|
return new Promise((resolve) => {
|
|
78
|
-
rl.question(
|
|
90
|
+
rl.question(promptText, (a) => {
|
|
79
91
|
rl.close();
|
|
80
92
|
resolve(a.trim());
|
|
81
93
|
});
|
|
82
94
|
});
|
|
83
95
|
}
|
|
96
|
+
async function readKeyInteractively(io) {
|
|
97
|
+
return promptLine(io, "Paste your API key (wc_live_…): ");
|
|
98
|
+
}
|
|
99
|
+
function openBrowserUrl(url) {
|
|
100
|
+
const cmd = process.platform === "darwin"
|
|
101
|
+
? "open"
|
|
102
|
+
: process.platform === "win32"
|
|
103
|
+
? "cmd"
|
|
104
|
+
: "xdg-open";
|
|
105
|
+
const args = process.platform === "win32" ? ["/c", "start", "", url] : [url];
|
|
106
|
+
try {
|
|
107
|
+
execFile(cmd, args, () => {
|
|
108
|
+
// ignore browser launch errors in headless environments
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
// ignore
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
async function runWebOAuthLogin(baseUrl, fetchImpl, io) {
|
|
116
|
+
const verifier = crypto.randomBytes(32).toString("base64url");
|
|
117
|
+
const challenge = crypto.createHash("sha256").update(verifier).digest("base64url");
|
|
118
|
+
const state = crypto.randomBytes(16).toString("base64url");
|
|
119
|
+
return new Promise((resolve, reject) => {
|
|
120
|
+
const server = createServer();
|
|
121
|
+
const timeout = setTimeout(() => {
|
|
122
|
+
server.close();
|
|
123
|
+
reject(new Error("Browser sign-in timed out after 180 seconds."));
|
|
124
|
+
}, 180_000);
|
|
125
|
+
server.listen(0, "127.0.0.1", async () => {
|
|
126
|
+
try {
|
|
127
|
+
const addr = server.address();
|
|
128
|
+
const port = typeof addr === "object" && addr ? addr.port : 0;
|
|
129
|
+
const redirectUri = `http://127.0.0.1:${port}/callback`;
|
|
130
|
+
const regRes = await fetchImpl(`${baseUrl}/oauth/register`, {
|
|
131
|
+
method: "POST",
|
|
132
|
+
headers: { "content-type": "application/json", accept: "application/json" },
|
|
133
|
+
body: JSON.stringify({
|
|
134
|
+
client_name: "Webcrawl Anything CLI",
|
|
135
|
+
redirect_uris: [redirectUri],
|
|
136
|
+
}),
|
|
137
|
+
});
|
|
138
|
+
if (!regRes.ok) {
|
|
139
|
+
throw new Error(`Could not register CLI OAuth client (HTTP ${regRes.status}). Try \`webcrawl login --email <you@domain.com>\`.`);
|
|
140
|
+
}
|
|
141
|
+
const { client_id: clientId } = (await regRes.json());
|
|
142
|
+
const authUrl = `${baseUrl}/oauth/authorize?${new URLSearchParams({
|
|
143
|
+
response_type: "code",
|
|
144
|
+
client_id: clientId,
|
|
145
|
+
redirect_uri: redirectUri,
|
|
146
|
+
code_challenge: challenge,
|
|
147
|
+
code_challenge_method: "S256",
|
|
148
|
+
state,
|
|
149
|
+
}).toString()}`;
|
|
150
|
+
io.out([
|
|
151
|
+
"Open this URL in your browser to create your webcrawl.app account or sign in (1,000 free monthly credits):",
|
|
152
|
+
` ${authUrl}`,
|
|
153
|
+
"",
|
|
154
|
+
"Waiting for browser authorization...",
|
|
155
|
+
].join("\n") + "\n");
|
|
156
|
+
openBrowserUrl(authUrl);
|
|
157
|
+
server.on("request", async (req, res) => {
|
|
158
|
+
try {
|
|
159
|
+
const u = new URL(req.url || "/", redirectUri);
|
|
160
|
+
if (u.pathname !== "/callback") {
|
|
161
|
+
res.writeHead(404).end("Not found");
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
const code = u.searchParams.get("code");
|
|
165
|
+
const returnedState = u.searchParams.get("state");
|
|
166
|
+
if (!code || returnedState !== state) {
|
|
167
|
+
res.writeHead(400, { "content-type": "text/html; charset=utf-8" });
|
|
168
|
+
res.end("<h3>Authorization failed or state mismatch. You can close this tab.</h3>");
|
|
169
|
+
clearTimeout(timeout);
|
|
170
|
+
server.close();
|
|
171
|
+
reject(new Error("OAuth callback missing code or state mismatch."));
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
const tokenRes = await fetchImpl(`${baseUrl}/oauth/token`, {
|
|
175
|
+
method: "POST",
|
|
176
|
+
headers: { "content-type": "application/x-www-form-urlencoded", accept: "application/json" },
|
|
177
|
+
body: new URLSearchParams({
|
|
178
|
+
grant_type: "authorization_code",
|
|
179
|
+
code,
|
|
180
|
+
client_id: clientId,
|
|
181
|
+
redirect_uri: redirectUri,
|
|
182
|
+
code_verifier: verifier,
|
|
183
|
+
}).toString(),
|
|
184
|
+
});
|
|
185
|
+
if (!tokenRes.ok) {
|
|
186
|
+
throw new Error(`OAuth token exchange failed (HTTP ${tokenRes.status}).`);
|
|
187
|
+
}
|
|
188
|
+
const tokenData = (await tokenRes.json());
|
|
189
|
+
let finalKey = tokenData.access_token;
|
|
190
|
+
const authedClient = new WebcrawlClient({
|
|
191
|
+
baseUrl,
|
|
192
|
+
apiKey: finalKey,
|
|
193
|
+
fetch: fetchImpl,
|
|
194
|
+
userAgent: "webcrawl-cli/1.2",
|
|
195
|
+
});
|
|
196
|
+
try {
|
|
197
|
+
const keyRes = await authedClient.request("POST", "/v1/account/keys", { name: "CLI API Key" });
|
|
198
|
+
if (keyRes.apiKey) {
|
|
199
|
+
finalKey = keyRes.apiKey;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
// keep OAuth access_token (`wc_mcp_...`) which is also accepted on all /v1/* endpoints
|
|
204
|
+
}
|
|
205
|
+
const meRes = await authedClient.me().catch(() => undefined);
|
|
206
|
+
res.writeHead(200, { "content-type": "text/html; charset=utf-8" });
|
|
207
|
+
res.end("<html><body style='font-family:system-ui;padding:2rem'><h2>✓ Webcrawl CLI authenticated!</h2><p>Your account and 1,000 monthly credits are now linked. You can close this tab and return to your terminal or agent.</p></body></html>");
|
|
208
|
+
clearTimeout(timeout);
|
|
209
|
+
server.close();
|
|
210
|
+
resolve({
|
|
211
|
+
apiKey: finalKey,
|
|
212
|
+
email: meRes?.user?.email,
|
|
213
|
+
creditsRemaining: meRes?.user?.creditsRemaining,
|
|
214
|
+
creditsLimit: meRes?.user?.creditsLimit,
|
|
215
|
+
plan: meRes?.user?.plan,
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
catch (err) {
|
|
219
|
+
clearTimeout(timeout);
|
|
220
|
+
server.close();
|
|
221
|
+
reject(err);
|
|
222
|
+
}
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
catch (err) {
|
|
226
|
+
clearTimeout(timeout);
|
|
227
|
+
server.close();
|
|
228
|
+
reject(err);
|
|
229
|
+
}
|
|
230
|
+
});
|
|
231
|
+
});
|
|
232
|
+
}
|
|
84
233
|
async function resolveBackendClient(cfg, deps) {
|
|
85
234
|
// When a custom fetch mock is injected (e.g. unit tests) or effectiveMode is cloud, use WebcrawlClient
|
|
86
235
|
if (cfg.effectiveMode === "cloud" || deps.fetch) {
|
|
@@ -142,6 +291,38 @@ export async function main(argv, io, deps = {}) {
|
|
|
142
291
|
io.err(`[${credits}]\n`);
|
|
143
292
|
}
|
|
144
293
|
};
|
|
294
|
+
const METERED_COMMANDS = new Set(["search", "scrape", "map", "crawl", "extract", "rerank"]);
|
|
295
|
+
if (METERED_COMMANDS.has(cmd) && cfg.effectiveMode === "cloud" && !cfg.apiKey && !deps.fetch) {
|
|
296
|
+
const authHelp = [
|
|
297
|
+
"webcrawl: Cloud mode requires a logged-in webcrawl.app account so credits can be attributed to your user (1,000 free monthly credits on signup).",
|
|
298
|
+
"",
|
|
299
|
+
"Create an account or sign in from the CLI:",
|
|
300
|
+
" 1. Email OTP (agent & terminal friendly):",
|
|
301
|
+
" webcrawl login --email <you@domain.com>",
|
|
302
|
+
" webcrawl login --email <you@domain.com> --code <6-digit-code>",
|
|
303
|
+
" 2. Browser OAuth redirect:",
|
|
304
|
+
" webcrawl login --web",
|
|
305
|
+
" 3. Or run in zero-key local mode:",
|
|
306
|
+
` webcrawl --mode local ${cmd} ${rest.join(" ")}`,
|
|
307
|
+
].join("\n");
|
|
308
|
+
if (v.json) {
|
|
309
|
+
io.out(`${JSON.stringify({
|
|
310
|
+
success: false,
|
|
311
|
+
error: "authentication_required",
|
|
312
|
+
message: "Cloud mode requires a logged-in webcrawl.app account (1,000 free monthly credits on signup).",
|
|
313
|
+
loginCommands: {
|
|
314
|
+
sendOtp: "webcrawl login --email <you@domain.com>",
|
|
315
|
+
verifyOtp: "webcrawl login --email <you@domain.com> --code <6-digit-code>",
|
|
316
|
+
browserRedirect: "webcrawl login --web",
|
|
317
|
+
signupUrl: `${cfg.baseUrl}/?signup=cli`,
|
|
318
|
+
},
|
|
319
|
+
}, null, 2)}\n`);
|
|
320
|
+
}
|
|
321
|
+
else {
|
|
322
|
+
io.err(`${authHelp}\n`);
|
|
323
|
+
}
|
|
324
|
+
return 1;
|
|
325
|
+
}
|
|
145
326
|
try {
|
|
146
327
|
switch (cmd) {
|
|
147
328
|
case "search": {
|
|
@@ -258,10 +439,93 @@ export async function main(argv, io, deps = {}) {
|
|
|
258
439
|
emit({ success: true, ...report }, prettyLines);
|
|
259
440
|
return 0;
|
|
260
441
|
}
|
|
261
|
-
case "login":
|
|
442
|
+
case "login":
|
|
443
|
+
case "signup": {
|
|
444
|
+
const authClient = new WebcrawlClient({
|
|
445
|
+
baseUrl: cfg.baseUrl,
|
|
446
|
+
fetch: deps.fetch,
|
|
447
|
+
userAgent: "webcrawl-cli/1.2",
|
|
448
|
+
});
|
|
449
|
+
// 1. Browser OAuth PKCE redirect (`webcrawl login --web`)
|
|
450
|
+
if (v.web) {
|
|
451
|
+
const webRes = await runWebOAuthLogin(cfg.baseUrl, deps.fetch ?? globalThis.fetch, io);
|
|
452
|
+
const existing = readConfigFile(file);
|
|
453
|
+
writeConfigFile({ ...existing, apiKey: webRes.apiKey, ...(v.url ? { baseUrl: v.url } : {}) }, file);
|
|
454
|
+
const payload = {
|
|
455
|
+
success: true,
|
|
456
|
+
method: "web_oauth",
|
|
457
|
+
email: webRes.email,
|
|
458
|
+
plan: webRes.plan ?? "free",
|
|
459
|
+
creditsRemaining: webRes.creditsRemaining ?? 1000,
|
|
460
|
+
creditsLimit: webRes.creditsLimit ?? 1000,
|
|
461
|
+
configFile: file,
|
|
462
|
+
};
|
|
463
|
+
emit(payload, [
|
|
464
|
+
`✓ Authenticated${webRes.email ? ` as ${webRes.email}` : ""} (${payload.plan} plan · ${payload.creditsRemaining} credits remaining of ${payload.creditsLimit}).`,
|
|
465
|
+
`Saved API key to ${file}.`,
|
|
466
|
+
].join("\n"));
|
|
467
|
+
return 0;
|
|
468
|
+
}
|
|
469
|
+
// 2. Email OTP account creation / sign-in (`webcrawl login --email <addr> [--code <6-digit>]`)
|
|
470
|
+
if (v.email) {
|
|
471
|
+
const email = v.email.trim();
|
|
472
|
+
let code = v.code?.trim();
|
|
473
|
+
if (!code) {
|
|
474
|
+
const otpRes = await authClient.sendOtp(email);
|
|
475
|
+
if (!otpRes.success) {
|
|
476
|
+
throw new WebcrawlError("bad_request", otpRes.error || "Could not send verification code.");
|
|
477
|
+
}
|
|
478
|
+
if (otpRes.devCode && !io.isTTY) {
|
|
479
|
+
code = otpRes.devCode;
|
|
480
|
+
}
|
|
481
|
+
else if (io.isTTY && !v["no-wait"]) {
|
|
482
|
+
io.err(`Sent a 6-digit verification code to ${email} (expires in 10 minutes).\n`);
|
|
483
|
+
code = await promptLine(io, `Enter the 6-digit code sent to ${email}: `);
|
|
484
|
+
}
|
|
485
|
+
else {
|
|
486
|
+
const stepPayload = {
|
|
487
|
+
success: true,
|
|
488
|
+
step: "otp_sent",
|
|
489
|
+
email: otpRes.email,
|
|
490
|
+
expiresInSeconds: otpRes.expiresInSeconds,
|
|
491
|
+
nextCommand: `webcrawl login --email ${email} --code <6-digit-code>`,
|
|
492
|
+
browserFallbackUrl: `${cfg.baseUrl}/?signup=cli&email=${encodeURIComponent(email)}`,
|
|
493
|
+
};
|
|
494
|
+
emit(stepPayload, [
|
|
495
|
+
`Verification code sent to ${otpRes.email} (expires in ${Math.round(otpRes.expiresInSeconds / 60)} minutes).`,
|
|
496
|
+
`Ask the user for the 6-digit code from their email, then complete sign-in with:`,
|
|
497
|
+
` webcrawl login --email ${email} --code <6-digit-code>`,
|
|
498
|
+
`Or redirect the user to sign in on the web: ${stepPayload.browserFallbackUrl}`,
|
|
499
|
+
].join("\n"));
|
|
500
|
+
return 0;
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
const verified = await authClient.verifyOtp({ email, code, name: v.name });
|
|
504
|
+
if (!verified.success || !verified.apiKey) {
|
|
505
|
+
throw new WebcrawlError("auth", verified.error || "Could not verify the 6-digit code.");
|
|
506
|
+
}
|
|
507
|
+
const existing = readConfigFile(file);
|
|
508
|
+
writeConfigFile({ ...existing, apiKey: verified.apiKey, ...(v.url ? { baseUrl: v.url } : {}) }, file);
|
|
509
|
+
const u = verified.user;
|
|
510
|
+
const payload = {
|
|
511
|
+
success: true,
|
|
512
|
+
isNewUser: verified.isNewUser ?? false,
|
|
513
|
+
email: u?.email ?? email,
|
|
514
|
+
plan: u?.plan ?? "free",
|
|
515
|
+
creditsRemaining: u?.creditsRemaining ?? 1000,
|
|
516
|
+
creditsLimit: u?.creditsLimit ?? 1000,
|
|
517
|
+
configFile: file,
|
|
518
|
+
};
|
|
519
|
+
emit(payload, [
|
|
520
|
+
`✓ ${verified.isNewUser ? "Created webcrawl.app account" : "Signed in"} as ${payload.email} (${payload.plan} plan · ${payload.creditsRemaining} credits remaining of ${payload.creditsLimit}).`,
|
|
521
|
+
`Saved API key (${verified.apiKey.slice(0, 8)}…${verified.apiKey.slice(-4)}) to ${file}.`,
|
|
522
|
+
].join("\n"));
|
|
523
|
+
return 0;
|
|
524
|
+
}
|
|
525
|
+
// 3. Direct API key (`webcrawl login --key wc_live_...` or interactive paste)
|
|
262
526
|
const key = (v.key ?? (await readKeyInteractively(io))).trim();
|
|
263
527
|
if (!key) {
|
|
264
|
-
throw new UsageError("No key provided.");
|
|
528
|
+
throw new UsageError("No key provided. Pass --key <wc_live_...>, --email <you@domain.com>, or --web.");
|
|
265
529
|
}
|
|
266
530
|
const existing = readConfigFile(file);
|
|
267
531
|
writeConfigFile({ ...existing, apiKey: key, ...(v.url ? { baseUrl: v.url } : {}) }, file);
|
|
@@ -273,7 +537,7 @@ export async function main(argv, io, deps = {}) {
|
|
|
273
537
|
const lines = [
|
|
274
538
|
`endpoint: ${cfg.baseUrl} (${cfg.source.baseUrl})`,
|
|
275
539
|
`mode: ${cfg.effectiveMode} (${cfg.source.mode})`,
|
|
276
|
-
`key: ${masked}${cfg.apiKey ? ` (${cfg.source.apiKey})` : ". Run `webcrawl login` or set WEBCRAWL_API_KEY (local mode works with zero key)."}`,
|
|
540
|
+
`key: ${masked}${cfg.apiKey ? ` (${cfg.source.apiKey})` : ". Run `webcrawl login` (`--email <you@domain.com>` or `--web`) or set WEBCRAWL_API_KEY (local mode works with zero key)."}`,
|
|
277
541
|
];
|
|
278
542
|
if (cfg.apiKey) {
|
|
279
543
|
const u = await client.usage();
|
|
@@ -281,6 +545,10 @@ export async function main(argv, io, deps = {}) {
|
|
|
281
545
|
io.out(`${JSON.stringify({ endpoint: cfg.baseUrl, mode: cfg.effectiveMode, keySource: cfg.source.apiKey, usage: u }, null, 2)}\n`);
|
|
282
546
|
return 0;
|
|
283
547
|
}
|
|
548
|
+
const userObj = (u.user ?? {});
|
|
549
|
+
if (typeof userObj.email === "string") {
|
|
550
|
+
lines.push(`account: ${userObj.email} (${userObj.plan ?? "free"} plan)`);
|
|
551
|
+
}
|
|
284
552
|
const credits = (u.credits ?? u);
|
|
285
553
|
if (typeof credits.remaining === "number") {
|
|
286
554
|
lines.push(`credits: ${credits.remaining} remaining of ${credits.limit ?? "?"}`);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "webcrawl",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.2",
|
|
4
4
|
"description": "Webcrawl Anything CLI — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -53,7 +53,7 @@
|
|
|
53
53
|
"test": "node --conditions=development --import tsx --test test/*.test.ts"
|
|
54
54
|
},
|
|
55
55
|
"dependencies": {
|
|
56
|
-
"@webcrawl/client": "^1.2.
|
|
56
|
+
"@webcrawl/client": "^1.2.2"
|
|
57
57
|
},
|
|
58
58
|
"engines": {
|
|
59
59
|
"node": ">=20"
|