webpeel 0.3.5 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -4
- package/dist/cache.d.ts +31 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +132 -0
- package/dist/cache.js.map +1 -0
- package/dist/cli-auth.d.ts +0 -16
- package/dist/cli-auth.d.ts.map +1 -1
- package/dist/cli-auth.js +0 -76
- package/dist/cli-auth.js.map +1 -1
- package/dist/cli.js +131 -4
- package/dist/cli.js.map +1 -1
- package/dist/core/actions.d.ts +15 -0
- package/dist/core/actions.d.ts.map +1 -0
- package/dist/core/actions.js +67 -0
- package/dist/core/actions.js.map +1 -0
- package/dist/core/crawler.d.ts +19 -0
- package/dist/core/crawler.d.ts.map +1 -1
- package/dist/core/crawler.js +68 -4
- package/dist/core/crawler.js.map +1 -1
- package/dist/core/extract.d.ts +6 -0
- package/dist/core/extract.d.ts.map +1 -0
- package/dist/core/extract.js +77 -0
- package/dist/core/extract.js.map +1 -0
- package/dist/core/fetcher.d.ts +9 -0
- package/dist/core/fetcher.d.ts.map +1 -1
- package/dist/core/fetcher.js +12 -4
- package/dist/core/fetcher.js.map +1 -1
- package/dist/core/map.d.ts +30 -0
- package/dist/core/map.d.ts.map +1 -0
- package/dist/core/map.js +61 -0
- package/dist/core/map.js.map +1 -0
- package/dist/core/markdown.d.ts +5 -0
- package/dist/core/markdown.d.ts.map +1 -1
- package/dist/core/markdown.js +40 -0
- package/dist/core/markdown.js.map +1 -1
- package/dist/core/pdf.d.ts +9 -0
- package/dist/core/pdf.d.ts.map +1 -0
- package/dist/core/pdf.js +24 -0
- package/dist/core/pdf.js.map +1 -0
- package/dist/core/sitemap.d.ts +24 -0
- package/dist/core/sitemap.d.ts.map +1 -0
- package/dist/core/sitemap.js +106 -0
- package/dist/core/sitemap.js.map +1 -0
- package/dist/core/strategies.d.ts +10 -0
- package/dist/core/strategies.d.ts.map +1 -1
- package/dist/core/strategies.js +4 -1
- package/dist/core/strategies.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +20 -2
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +221 -4
- package/dist/mcp/server.js.map +1 -1
- package/dist/types.d.ts +25 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +9 -4
package/README.md
CHANGED
|
@@ -86,9 +86,24 @@ webpeel usage
|
|
|
86
86
|
# Stealth mode (bypass bot detection)
|
|
87
87
|
npx webpeel https://protected-site.com --stealth
|
|
88
88
|
|
|
89
|
+
# Page actions: click, scroll, type before extraction (v0.4.0)
|
|
90
|
+
npx webpeel https://example.com --action "click:.cookie-accept" --action "wait:2000" --action "scroll:bottom"
|
|
91
|
+
|
|
92
|
+
# Structured data extraction with CSS selectors (v0.4.0)
|
|
93
|
+
npx webpeel https://example.com --extract '{"title": "h1", "price": ".price", "description": ".desc"}'
|
|
94
|
+
|
|
95
|
+
# Token budget: truncate output to max tokens (v0.4.0)
|
|
96
|
+
npx webpeel https://example.com --max-tokens 2000
|
|
97
|
+
|
|
98
|
+
# Map discovery: find all URLs on a domain via sitemap & crawling (v0.4.0)
|
|
99
|
+
npx webpeel map https://example.com --max-urls 5000
|
|
100
|
+
|
|
89
101
|
# Crawl a website (follow links, respect robots.txt)
|
|
90
102
|
npx webpeel crawl https://example.com --max-pages 20 --max-depth 2
|
|
91
103
|
|
|
104
|
+
# Sitemap-first crawl with content deduplication (v0.4.0)
|
|
105
|
+
npx webpeel crawl https://example.com --sitemap-first --max-pages 100
|
|
106
|
+
|
|
92
107
|
# JSON output with metadata
|
|
93
108
|
npx webpeel https://example.com --json
|
|
94
109
|
|
|
@@ -231,10 +246,12 @@ Or add to your project's `.mcp.json` for team sharing:
|
|
|
231
246
|
```
|
|
232
247
|
|
|
233
248
|
This gives Claude Code access to:
|
|
234
|
-
- **webpeel_fetch** — Fetch any URL as clean markdown (with stealth mode
|
|
249
|
+
- **webpeel_fetch** — Fetch any URL as clean markdown (with stealth mode, actions, extraction & token budget)
|
|
235
250
|
- **webpeel_search** — Search the web via DuckDuckGo
|
|
236
251
|
- **webpeel_batch** — Fetch multiple URLs concurrently
|
|
237
|
-
- **webpeel_crawl** — Crawl websites following links
|
|
252
|
+
- **webpeel_crawl** — Crawl websites following links (with sitemap-first & deduplication)
|
|
253
|
+
- **webpeel_map** — Discover all URLs on a domain via sitemap.xml & link crawling
|
|
254
|
+
- **webpeel_extract** — Extract structured data using CSS selectors or JSON schema
|
|
238
255
|
|
|
239
256
|
---
|
|
240
257
|
|
|
@@ -506,9 +523,13 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
|
|
|
506
523
|
- [x] Weekly reset usage model with extra usage
|
|
507
524
|
- [x] Stealth mode (playwright-extra + anti-detect)
|
|
508
525
|
- [x] Crawl mode (follow links, respect robots.txt)
|
|
509
|
-
- [
|
|
526
|
+
- [x] PDF extraction (v0.4.0)
|
|
527
|
+
- [x] Structured data extraction with CSS selectors and JSON schema (v0.4.0)
|
|
528
|
+
- [x] Page actions: click, scroll, type, fill, select, press, hover (v0.4.0)
|
|
529
|
+
- [x] Map/sitemap discovery for full site URL mapping (v0.4.0)
|
|
530
|
+
- [x] Token budget for output truncation (v0.4.0)
|
|
531
|
+
- [x] Advanced crawl: sitemap-first, BFS/DFS, content deduplication (v0.4.0)
|
|
510
532
|
- [ ] Webhook notifications for monitoring
|
|
511
|
-
- [ ] Structured data extraction (JSON output)
|
|
512
533
|
|
|
513
534
|
Vote on features and roadmap at [GitHub Discussions](https://github.com/JakeLiuMe/webpeel/discussions).
|
|
514
535
|
|
package/dist/cache.d.ts
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local response cache for WebPeel CLI
|
|
3
|
+
*
|
|
4
|
+
* Caches fetch results in ~/.webpeel/cache/ with TTL support.
|
|
5
|
+
* Cache key is a hash of URL + relevant options.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Parse a TTL string like "5m", "1h", "30s", "1d" into milliseconds
|
|
9
|
+
*/
|
|
10
|
+
export declare function parseTTL(ttl: string): number;
|
|
11
|
+
/**
|
|
12
|
+
* Get a cached result if it exists and hasn't expired
|
|
13
|
+
*/
|
|
14
|
+
export declare function getCache(url: string, options?: Record<string, any>): any | null;
|
|
15
|
+
/**
|
|
16
|
+
* Store a result in the cache
|
|
17
|
+
*/
|
|
18
|
+
export declare function setCache(url: string, result: any, ttlMs: number, options?: Record<string, any>): void;
|
|
19
|
+
/**
|
|
20
|
+
* Clear expired cache entries (or all entries)
|
|
21
|
+
*/
|
|
22
|
+
export declare function clearCache(all?: boolean): number;
|
|
23
|
+
/**
|
|
24
|
+
* Get cache stats
|
|
25
|
+
*/
|
|
26
|
+
export declare function cacheStats(): {
|
|
27
|
+
entries: number;
|
|
28
|
+
sizeBytes: number;
|
|
29
|
+
dir: string;
|
|
30
|
+
};
|
|
31
|
+
//# sourceMappingURL=cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAiBH;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAc5C;AAiBD;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,GAAG,GAAG,IAAI,CAoB/E;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAerG;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,GAAG,UAAQ,GAAG,MAAM,CAyB9C;AAED;;GAEG;AACH,wBAAgB,UAAU,IAAI;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAchF"}
|
package/dist/cache.js
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local response cache for WebPeel CLI
|
|
3
|
+
*
|
|
4
|
+
* Caches fetch results in ~/.webpeel/cache/ with TTL support.
|
|
5
|
+
* Cache key is a hash of URL + relevant options.
|
|
6
|
+
*/
|
|
7
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, unlinkSync, statSync } from 'fs';
|
|
8
|
+
import { homedir } from 'os';
|
|
9
|
+
import { join } from 'path';
|
|
10
|
+
import { createHash } from 'crypto';
|
|
11
|
+
const CACHE_DIR = join(homedir(), '.webpeel', 'cache');
|
|
12
|
+
/**
|
|
13
|
+
* Parse a TTL string like "5m", "1h", "30s", "1d" into milliseconds
|
|
14
|
+
*/
|
|
15
|
+
export function parseTTL(ttl) {
|
|
16
|
+
const match = ttl.match(/^(\d+)(s|m|h|d)$/);
|
|
17
|
+
if (!match) {
|
|
18
|
+
throw new Error(`Invalid TTL format: "${ttl}". Use: 30s, 5m, 1h, 1d`);
|
|
19
|
+
}
|
|
20
|
+
const value = parseInt(match[1]);
|
|
21
|
+
const unit = match[2];
|
|
22
|
+
switch (unit) {
|
|
23
|
+
case 's': return value * 1000;
|
|
24
|
+
case 'm': return value * 60 * 1000;
|
|
25
|
+
case 'h': return value * 60 * 60 * 1000;
|
|
26
|
+
case 'd': return value * 24 * 60 * 60 * 1000;
|
|
27
|
+
default: throw new Error(`Unknown TTL unit: ${unit}`);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Generate a cache key from URL + options
|
|
32
|
+
*/
|
|
33
|
+
function cacheKey(url, options) {
|
|
34
|
+
const relevant = {
|
|
35
|
+
url,
|
|
36
|
+
render: options?.render || false,
|
|
37
|
+
stealth: options?.stealth || false,
|
|
38
|
+
selector: options?.selector || null,
|
|
39
|
+
format: options?.format || 'markdown',
|
|
40
|
+
};
|
|
41
|
+
const hash = createHash('sha256').update(JSON.stringify(relevant)).digest('hex').slice(0, 16);
|
|
42
|
+
return hash;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Get a cached result if it exists and hasn't expired
|
|
46
|
+
*/
|
|
47
|
+
export function getCache(url, options) {
|
|
48
|
+
const key = cacheKey(url, options);
|
|
49
|
+
const filePath = join(CACHE_DIR, `${key}.json`);
|
|
50
|
+
if (!existsSync(filePath))
|
|
51
|
+
return null;
|
|
52
|
+
try {
|
|
53
|
+
const entry = JSON.parse(readFileSync(filePath, 'utf-8'));
|
|
54
|
+
const age = Date.now() - entry.cachedAt;
|
|
55
|
+
if (age > entry.ttlMs) {
|
|
56
|
+
// Expired — delete and return null
|
|
57
|
+
try {
|
|
58
|
+
unlinkSync(filePath);
|
|
59
|
+
}
|
|
60
|
+
catch { }
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
return entry.result;
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Store a result in the cache
|
|
71
|
+
*/
|
|
72
|
+
export function setCache(url, result, ttlMs, options) {
|
|
73
|
+
if (!existsSync(CACHE_DIR)) {
|
|
74
|
+
mkdirSync(CACHE_DIR, { recursive: true });
|
|
75
|
+
}
|
|
76
|
+
const key = cacheKey(url, options);
|
|
77
|
+
const entry = {
|
|
78
|
+
url,
|
|
79
|
+
result,
|
|
80
|
+
cachedAt: Date.now(),
|
|
81
|
+
ttlMs,
|
|
82
|
+
options: options ? JSON.stringify(options) : undefined,
|
|
83
|
+
};
|
|
84
|
+
writeFileSync(join(CACHE_DIR, `${key}.json`), JSON.stringify(entry));
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Clear expired cache entries (or all entries)
|
|
88
|
+
*/
|
|
89
|
+
export function clearCache(all = false) {
|
|
90
|
+
if (!existsSync(CACHE_DIR))
|
|
91
|
+
return 0;
|
|
92
|
+
const files = readdirSync(CACHE_DIR).filter(f => f.endsWith('.json'));
|
|
93
|
+
let cleared = 0;
|
|
94
|
+
for (const file of files) {
|
|
95
|
+
const filePath = join(CACHE_DIR, file);
|
|
96
|
+
try {
|
|
97
|
+
if (all) {
|
|
98
|
+
unlinkSync(filePath);
|
|
99
|
+
cleared++;
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
const entry = JSON.parse(readFileSync(filePath, 'utf-8'));
|
|
103
|
+
if (Date.now() - entry.cachedAt > entry.ttlMs) {
|
|
104
|
+
unlinkSync(filePath);
|
|
105
|
+
cleared++;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
// Skip corrupt files
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return cleared;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Get cache stats
|
|
117
|
+
*/
|
|
118
|
+
export function cacheStats() {
|
|
119
|
+
if (!existsSync(CACHE_DIR))
|
|
120
|
+
return { entries: 0, sizeBytes: 0, dir: CACHE_DIR };
|
|
121
|
+
const files = readdirSync(CACHE_DIR).filter(f => f.endsWith('.json'));
|
|
122
|
+
let sizeBytes = 0;
|
|
123
|
+
for (const file of files) {
|
|
124
|
+
try {
|
|
125
|
+
const stat = statSync(join(CACHE_DIR, file));
|
|
126
|
+
sizeBytes += stat.size;
|
|
127
|
+
}
|
|
128
|
+
catch { }
|
|
129
|
+
}
|
|
130
|
+
return { entries: files.length, sizeBytes, dir: CACHE_DIR };
|
|
131
|
+
}
|
|
132
|
+
//# sourceMappingURL=cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAC3G,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;AAUvD;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,yBAAyB,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACtB,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,IAAI,CAAC;QAC9B,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC;QACnC,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QACxC,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAC7C,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,QAAQ,CAAC,GAAW,EAAE,OAA6B;IAC1D,MAAM,QAAQ,GAAG;QACf,GAAG;QACH,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,KAAK;QAChC,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,KAAK;QAClC,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,IAAI;QACnC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,UAAU;KACtC,CAAC;IACF,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9F,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,OAA6B;IACjE,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IAEhD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;QAExC,IAAI,GAAG,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YACtB,mCAAmC;YACnC,IAAI,CAAC;gBAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;YACtC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,MAAW,EAAE,KAAa,EAAE,OAA6B;IAC7F,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,KAAK,GAAe;QACxB,GAAG;QACH,MAAM;QACN,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;QACpB,KAAK;QACL,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KACvD,CAAC;IAEF,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AACvE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,GAAG,GAAG,KAAK;IACpC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,CAAC,CAAC;IAErC,MAAM,KAAK,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC;YACH,IAAI,GAAG,EAAE,CAAC;gBACR,UAAU,CAAC,QAAQ,CAAC,CAAC;gBACrB,OAAO,EAAE,CAAC;YACZ,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBACtE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;oBAC9C,UAAU,CAAC,QAAQ,CAAC,CAAC;oBACrB,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;IAEhF,MAAM,KAAK,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7C,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;IACZ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;AAC9D,CAAC"}
|
package/dist/cli-auth.d.ts
CHANGED
|
@@ -40,22 +40,6 @@ export declare function deleteConfig(): void;
|
|
|
40
40
|
* Check usage quota before making a request
|
|
41
41
|
*/
|
|
42
42
|
export declare function checkUsage(): Promise<UsageCheckResult>;
|
|
43
|
-
export type PremiumFeature = 'stealth' | 'crawl' | 'batch';
|
|
44
|
-
/**
|
|
45
|
-
* Check if user has access to a premium feature.
|
|
46
|
-
* Returns { allowed: true } for paid users, or a helpful upgrade message.
|
|
47
|
-
*
|
|
48
|
-
* Priority:
|
|
49
|
-
* 1. No API key → blocked (must sign up)
|
|
50
|
-
* 2. Has API key + cached plan → check plan tier
|
|
51
|
-
* 3. Has API key + no cache → check API, then cache
|
|
52
|
-
* 4. API unreachable + cached plan within 7 days → use cache
|
|
53
|
-
* 5. API unreachable + stale cache → allow gracefully (trust the user)
|
|
54
|
-
*/
|
|
55
|
-
export declare function checkFeatureAccess(feature: PremiumFeature): Promise<{
|
|
56
|
-
allowed: boolean;
|
|
57
|
-
message?: string;
|
|
58
|
-
}>;
|
|
59
43
|
/**
|
|
60
44
|
* Show usage footer after successful fetch (for free/anonymous users only)
|
|
61
45
|
*/
|
package/dist/cli-auth.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli-auth.d.ts","sourceRoot":"","sources":["../src/cli-auth.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAcH,UAAU,SAAS;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,UAAU,gBAAgB;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAwBD;;GAEG;AACH,wBAAgB,UAAU,IAAI,SAAS,CAyBtC;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAWlD;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,IAAI,CAQnC;AAwBD;;GAEG;AACH,wBAAsB,UAAU,IAAI,OAAO,CAAC,gBAAgB,CAAC,CA+F5D;
|
|
1
|
+
{"version":3,"file":"cli-auth.d.ts","sourceRoot":"","sources":["../src/cli-auth.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAcH,UAAU,SAAS;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,UAAU,gBAAgB;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAwBD;;GAEG;AACH,wBAAgB,UAAU,IAAI,SAAS,CAyBtC;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAWlD;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,IAAI,CAQnC;AAwBD;;GAEG;AACH,wBAAsB,UAAU,IAAI,OAAO,CAAC,gBAAgB,CAAC,CA+F5D;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,SAAS,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,EACzE,WAAW,EAAE,OAAO,EACpB,OAAO,GAAE,OAAe,GACvB,IAAI,CAaN;AAED;;GAEG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC,CAYvD;AAED;;GAEG;AACH,wBAAsB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAgEjD;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,IAAI,CAUnC;AAED;;GAEG;AACH,wBAAsB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAoEjD"}
|
package/dist/cli-auth.js
CHANGED
|
@@ -179,82 +179,6 @@ export async function checkUsage() {
|
|
|
179
179
|
return { allowed: true };
|
|
180
180
|
}
|
|
181
181
|
}
|
|
182
|
-
const FEATURE_LABELS = {
|
|
183
|
-
stealth: 'Stealth mode (anti-bot bypass)',
|
|
184
|
-
crawl: 'Crawl mode (multi-page)',
|
|
185
|
-
batch: 'Batch mode (bulk URLs)',
|
|
186
|
-
};
|
|
187
|
-
/**
|
|
188
|
-
* Check if user has access to a premium feature.
|
|
189
|
-
* Returns { allowed: true } for paid users, or a helpful upgrade message.
|
|
190
|
-
*
|
|
191
|
-
* Priority:
|
|
192
|
-
* 1. No API key → blocked (must sign up)
|
|
193
|
-
* 2. Has API key + cached plan → check plan tier
|
|
194
|
-
* 3. Has API key + no cache → check API, then cache
|
|
195
|
-
* 4. API unreachable + cached plan within 7 days → use cache
|
|
196
|
-
* 5. API unreachable + stale cache → allow gracefully (trust the user)
|
|
197
|
-
*/
|
|
198
|
-
export async function checkFeatureAccess(feature) {
|
|
199
|
-
const config = loadConfig();
|
|
200
|
-
// No API key → must sign up and subscribe
|
|
201
|
-
if (!config.apiKey) {
|
|
202
|
-
return {
|
|
203
|
-
allowed: false,
|
|
204
|
-
message: `⚡ ${FEATURE_LABELS[feature]} requires a Pro plan ($9/mo).\n\n` +
|
|
205
|
-
` Basic fetch is free — stealth, crawl, and batch are Pro features.\n\n` +
|
|
206
|
-
` Sign up: https://app.webpeel.dev/signup\n` +
|
|
207
|
-
` Pricing: https://webpeel.dev/#pricing\n` +
|
|
208
|
-
` Login: webpeel login\n`,
|
|
209
|
-
};
|
|
210
|
-
}
|
|
211
|
-
// Try to get fresh plan info from API
|
|
212
|
-
try {
|
|
213
|
-
const response = await fetch(`${API_BASE_URL}/v1/cli/usage`, {
|
|
214
|
-
headers: { 'Authorization': `Bearer ${config.apiKey}` },
|
|
215
|
-
signal: AbortSignal.timeout(5000),
|
|
216
|
-
});
|
|
217
|
-
if (response.ok) {
|
|
218
|
-
const data = await response.json();
|
|
219
|
-
const planName = (data.plan?.tier || 'free').toLowerCase();
|
|
220
|
-
// Cache for offline use
|
|
221
|
-
config.planTier = planName;
|
|
222
|
-
config.planCachedAt = new Date().toISOString();
|
|
223
|
-
saveConfig(config);
|
|
224
|
-
if (planName === 'free') {
|
|
225
|
-
return {
|
|
226
|
-
allowed: false,
|
|
227
|
-
message: `⚡ ${FEATURE_LABELS[feature]} requires a Pro plan ($9/mo).\n\n` +
|
|
228
|
-
` You're on the Free plan. Upgrade to unlock stealth, crawl, and batch.\n\n` +
|
|
229
|
-
` Upgrade: https://app.webpeel.dev/billing\n` +
|
|
230
|
-
` Pricing: https://webpeel.dev/#pricing\n`,
|
|
231
|
-
};
|
|
232
|
-
}
|
|
233
|
-
// Pro or Max — allowed
|
|
234
|
-
return { allowed: true };
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
catch {
|
|
238
|
-
// API unreachable — fall through to cache check
|
|
239
|
-
}
|
|
240
|
-
// API unreachable — use cached plan if recent (within 7 days)
|
|
241
|
-
if (config.planTier && config.planCachedAt) {
|
|
242
|
-
const cacheAge = Date.now() - new Date(config.planCachedAt).getTime();
|
|
243
|
-
const SEVEN_DAYS = 7 * 24 * 60 * 60 * 1000;
|
|
244
|
-
if (cacheAge < SEVEN_DAYS) {
|
|
245
|
-
if (config.planTier === 'free') {
|
|
246
|
-
return {
|
|
247
|
-
allowed: false,
|
|
248
|
-
message: `⚡ ${FEATURE_LABELS[feature]} requires a Pro plan ($9/mo).\n\n` +
|
|
249
|
-
` Upgrade: https://app.webpeel.dev/billing\n`,
|
|
250
|
-
};
|
|
251
|
-
}
|
|
252
|
-
return { allowed: true };
|
|
253
|
-
}
|
|
254
|
-
}
|
|
255
|
-
// Stale cache or no cache, API unreachable — allow gracefully (trust paying users)
|
|
256
|
-
return { allowed: true };
|
|
257
|
-
}
|
|
258
182
|
/**
|
|
259
183
|
* Show usage footer after successful fetch (for free/anonymous users only)
|
|
260
184
|
*/
|
package/dist/cli-auth.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli-auth.js","sourceRoot":"","sources":["../src/cli-auth.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AACpF,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,+CAA+C;AAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,CAAC,CAAC;AAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAEpD,0CAA0C;AAC1C,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,yBAAyB,CAAC;AA2C9E;;GAEG;AACH,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC;QACH,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7B,OAAO;gBACL,cAAc,EAAE,CAAC;gBACjB,SAAS,EAAE,aAAa,EAAE,CAAC,WAAW,EAAE;aACzC,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAc,CAAC;QAEhD,0BAA0B;QAC1B,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtB,MAAM,CAAC,SAAS,GAAG,aAAa,EAAE,CAAC,WAAW,EAAE,CAAC;QACnD,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sCAAsC;QACtC,OAAO;YACL,cAAc,EAAE,CAAC;YACjB,SAAS,EAAE,aAAa,EAAE,CAAC,WAAW,EAAE;SACzC,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,MAAiB;IAC1C,IAAI,CAAC;QACH,0BAA0B;QAC1B,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5B,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,CAAC;QAED,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACvE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,mCAAmC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC/G,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,IAAI,CAAC;QACH,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,WAAW,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,qCAAqC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IACjH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,aAAa;IACpB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,yBAAyB;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;IACjC,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;IAC/C,UAAU,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAAC,MAAiB;IAClD,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,gBAAgB,GAAG,aAAa,EAAE,CAAC;IACzC,OAAO,SAAS,GAAG,gBAAgB,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,uCAAuC;IACvC,IAAI,yBAAyB,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC;QAC1B,MAAM,CAAC,SAAS,GAAG,aAAa,EAAE,CAAC,WAAW,EAAE,CAAC;QACjD,UAAU,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,0CAA0C;IAC1C,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,MAAM,CAAC,cAAc,CAAC;QACnC,MAAM,SAAS,GAAG,KAAK,GAAG,IAAI,CAAC;QAE/B,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;YAClB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,oBAAoB,KAAK,wHAAwH;aAC3J,CAAC;QACJ,CAAC;QAED,0BAA0B;QAC1B,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,UAAU,CAAC,MAAM,CAAC,CAAC;QAEnB,OAAO;YACL,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE;gBACT,IAAI,EAAE,IAAI,GAAG,CAAC;gBACd,KAAK;gBACL,SAAS,EAAE,SAAS,GAAG,CAAC;aACzB;SACF,CAAC;IACJ,CAAC;IAED,sCAAsC;IACtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,YAAY,eAAe,EAAE;YAC3D,OAAO,EAAE;gBACP,eAAe,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE;aAC3C;YACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,OAAO,EAAE,mGAAmG;iBAC7G,CAAC;YACJ,CAAC;YACD,gDAAgD;YAChD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC3B,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAsB,CAAC;QAEvD,oBAAoB;QACpB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACxC,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,wBAAwB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,gBAAgB,IAAI,CAAC,KAAK,CAAC,QAAQ,eAAe,IAAI,CAAC,UAAU,EAAE;aACxI,CAAC;QACJ,CAAC;QAED,gCAAgC;QAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,yBAAyB,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,eAAe,IAAI,CAAC,MAAM,CAAC,QAAQ,cAAc,IAAI,CAAC,UAAU,EAAE;aAC1I,CAAC;QACJ,CAAC;QAED,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,IAAI,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;QAC3D,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC3B,MAAM,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC/C,UAAU,CAAC,MAAM,CAAC,CAAC;QAEnB,OAAO;YACL,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE;gBACT,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;gBACtB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;gBACxB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;aACjC;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,kEAAkE;QAClE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC;AAKD,MAAM,cAAc,GAAmC;IACrD,OAAO,EAAE,gCAAgC;IACzC,KAAK,EAAE,yBAAyB;IAChC,KAAK,EAAE,wBAAwB;CAChC,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAuB;IAC9D,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,0CAA0C;IAC1C,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,KAAK,cAAc,CAAC,OAAO,CAAC,mCAAmC;gBACtE,yEAAyE;gBACzE,8CAA8C;gBAC9C,4CAA4C;gBAC5C,6BAA6B;SAChC,CAAC;IACJ,CAAC;IAED,sCAAsC;IACtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,YAAY,eAAe,EAAE;YAC3D,OAAO,EAAE,EAAE,eAAe,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE,EAAE;YACvD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YAChB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAsB,CAAC;YACvD,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,IAAI,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAE3D,wBAAwB;YACxB,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;YAC3B,MAAM,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC/C,UAAU,CAAC,MAAM,CAAC,CAAC;YAEnB,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;gBACxB,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,OAAO,EAAE,KAAK,cAAc,CAAC,OAAO,CAAC,mCAAmC;wBACtE,6EAA6E;wBAC7E,+CAA+C;wBAC/C,4CAA4C;iBAC/C,CAAC;YACJ,CAAC;YAED,uBAAuB;YACvB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gDAAgD;IAClD,CAAC;IAED,8DAA8D;IAC9D,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;QACtE,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAE3C,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;gBAC/B,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,OAAO,EAAE,KAAK,cAAc,CAAC,OAAO,CAAC,mCAAmC;wBACtE,+CAA+C;iBAClD,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,mFAAmF;IACnF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,SAAyE,EACzE,WAAoB,EACpB,UAAmB,KAAK;IAExB,IAAI,CAAC,SAAS;QAAE,OAAO;IAEvB,+CAA+C;IAC/C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,KAAK,0BAA0B,QAAQ,+CAA+C,CAAC,CAAC;IAC9I,CAAC;SAAM,IAAI,SAAS,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC;QAClC,gCAAgC;QAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,KAAK,+BAA+B,QAAQ,GAAG,CAAC,CAAC;IACvG,CAAC;IACD,mCAAmC;AACrC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IAEH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,EAAE,CAAC,QAAQ,CAAC,gEAAgE,EAAE,CAAC,MAAM,EAAE,EAAE;YACvF,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,oEAAoE,CAAC,CAAC;QAClF,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IAEnE,MAAM,MAAM,GAAG,MAAM,eAAe,EAAE,CAAC;IAEvC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,kDAAkD;IAClD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAC;IACjG,CAAC;IAED,gDAAgD;IAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,YAAY,eAAe,EAAE;YAC3D,OAAO,EAAE,EAAE,eAAe,EAAE,UAAU,MAAM,EAAE,EAAE;YAChD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YAChB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAsB,CAAC;YACvD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAEnF,gCAAgC;YAChC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;YACvB,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YACjC,MAAM,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC/C,UAAU,CAAC,MAAM,CAAC,CAAC;YAEnB,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,SAAS,SAAS,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,gBAAgB,CAAC,CAAC;YACtE,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC3E,CAAC;aAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACnC,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,uEAAuE;YACvE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;YACvB,UAAU,CAAC,MAAM,CAAC,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,+EAA+E,CAAC,CAAC;QAC/F,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;QAC7C,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;QACvB,UAAU,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;IACvE,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;AAC1D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO;IACT,CAAC;IAED,YAAY,EAAE,CAAC;IACf,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,yBAAyB;IACzB,IAAI,yBAAyB,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC;QAC1B,MAAM,CAAC,SAAS,GAAG,aAAa,EAAE,CAAC,WAAW,EAAE,CAAC;QACjD,UAAU,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,iBAAiB;IACjB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,MAAM,CAAC,cAAc,CAAC;QACnC,MAAM,SAAS,GAAG,KAAK,GAAG,IAAI,CAAC;QAC/B,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QAC7C,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;QAEnD,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,4DAA4D,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;QACjE,OAAO;IACT,CAAC;IAED,sCAAsC;IACtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,YAAY,eAAe,EAAE;YAC3D,OAAO,EAAE;gBACP,eAAe,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE;aAC3C;YACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,OAAO,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;gBAC5E,OAAO,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;gBAC9E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAsB,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAEnF,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,SAAS,SAAS,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QACtG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,yBAAyB,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC1G,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAE7E,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,4CAA4C,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QACxG,OAAO,CAAC,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC1E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"cli-auth.js","sourceRoot":"","sources":["../src/cli-auth.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AACpF,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,+CAA+C;AAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,CAAC,CAAC;AAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAEpD,0CAA0C;AAC1C,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,yBAAyB,CAAC;AA2C9E;;GAEG;AACH,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC;QACH,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7B,OAAO;gBACL,cAAc,EAAE,CAAC;gBACjB,SAAS,EAAE,aAAa,EAAE,CAAC,WAAW,EAAE;aACzC,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAc,CAAC;QAEhD,0BAA0B;QAC1B,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtB,MAAM,CAAC,SAAS,GAAG,aAAa,EAAE,CAAC,WAAW,EAAE,CAAC;QACnD,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sCAAsC;QACtC,OAAO;YACL,cAAc,EAAE,CAAC;YACjB,SAAS,EAAE,aAAa,EAAE,CAAC,WAAW,EAAE;SACzC,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,MAAiB;IAC1C,IAAI,CAAC;QACH,0BAA0B;QAC1B,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5B,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,CAAC;QAED,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACvE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,mCAAmC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC/G,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,IAAI,CAAC;QACH,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,WAAW,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,qCAAqC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IACjH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,aAAa;IACpB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,yBAAyB;IAC/D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;IACjC,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;IAC/C,UAAU,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAAC,MAAiB;IAClD,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,gBAAgB,GAAG,aAAa,EAAE,CAAC;IACzC,OAAO,SAAS,GAAG,gBAAgB,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,uCAAuC;IACvC,IAAI,yBAAyB,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC;QAC1B,MAAM,CAAC,SAAS,GAAG,aAAa,EAAE,CAAC,WAAW,EAAE,CAAC;QACjD,UAAU,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,0CAA0C;IAC1C,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,MAAM,CAAC,cAAc,CAAC;QACnC,MAAM,SAAS,GAAG,KAAK,GAAG,IAAI,CAAC;QAE/B,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;YAClB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,oBAAoB,KAAK,wHAAwH;aAC3J,CAAC;QACJ,CAAC;QAED,0BAA0B;QAC1B,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,UAAU,CAAC,MAAM,CAAC,CAAC;QAEnB,OAAO;YACL,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE;gBACT,IAAI,EAAE,IAAI,GAAG,CAAC;gBACd,KAAK;gBACL,SAAS,EAAE,SAAS,GAAG,CAAC;aACzB;SACF,CAAC;IACJ,CAAC;IAED,sCAAsC;IACtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,YAAY,eAAe,EAAE;YAC3D,OAAO,EAAE;gBACP,eAAe,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE;aAC3C;YACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,OAAO,EAAE,mGAAmG;iBAC7G,CAAC;YACJ,CAAC;YACD,gDAAgD;YAChD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC3B,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAsB,CAAC;QAEvD,oBAAoB;QACpB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACxC,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,wBAAwB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,gBAAgB,IAAI,CAAC,KAAK,CAAC,QAAQ,eAAe,IAAI,CAAC,UAAU,EAAE;aACxI,CAAC;QACJ,CAAC;QAED,gCAAgC;QAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,yBAAyB,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,eAAe,IAAI,CAAC,MAAM,CAAC,QAAQ,cAAc,IAAI,CAAC,UAAU,EAAE;aAC1I,CAAC;QACJ,CAAC;QAED,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,IAAI,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;QAC3D,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC3B,MAAM,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC/C,UAAU,CAAC,MAAM,CAAC,CAAC;QAEnB,OAAO;YACL,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE;gBACT,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;gBACtB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;gBACxB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;aACjC;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,kEAAkE;QAClE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,SAAyE,EACzE,WAAoB,EACpB,UAAmB,KAAK;IAExB,IAAI,CAAC,SAAS;QAAE,OAAO;IAEvB,+CAA+C;IAC/C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,KAAK,0BAA0B,QAAQ,+CAA+C,CAAC,CAAC;IAC9I,CAAC;SAAM,IAAI,SAAS,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC;QAClC,gCAAgC;QAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,KAAK,+BAA+B,QAAQ,GAAG,CAAC,CAAC;IACvG,CAAC;IACD,mCAAmC;AACrC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IAEH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,EAAE,CAAC,QAAQ,CAAC,gEAAgE,EAAE,CAAC,MAAM,EAAE,EAAE;YACvF,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,oEAAoE,CAAC,CAAC;QAClF,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IAEnE,MAAM,MAAM,GAAG,MAAM,eAAe,EAAE,CAAC;IAEvC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,kDAAkD;IAClD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAC;IACjG,CAAC;IAED,gDAAgD;IAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,YAAY,eAAe,EAAE;YAC3D,OAAO,EAAE,EAAE,eAAe,EAAE,UAAU,MAAM,EAAE,EAAE;YAChD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YAChB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAsB,CAAC;YACvD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAEnF,gCAAgC;YAChC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;YACvB,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YACjC,MAAM,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC/C,UAAU,CAAC,MAAM,CAAC,CAAC;YAEnB,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,SAAS,SAAS,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,gBAAgB,CAAC,CAAC;YACtE,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC3E,CAAC;aAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACnC,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,uEAAuE;YACvE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;YACvB,UAAU,CAAC,MAAM,CAAC,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,+EAA+E,CAAC,CAAC;QAC/F,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;QAC7C,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;QACvB,UAAU,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;IACvE,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;AAC1D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO;IACT,CAAC;IAED,YAAY,EAAE,CAAC;IACf,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,yBAAyB;IACzB,IAAI,yBAAyB,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC;QAC1B,MAAM,CAAC,SAAS,GAAG,aAAa,EAAE,CAAC,WAAW,EAAE,CAAC;QACjD,UAAU,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,iBAAiB;IACjB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,MAAM,CAAC,cAAc,CAAC;QACnC,MAAM,SAAS,GAAG,KAAK,GAAG,IAAI,CAAC;QAC/B,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QAC7C,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;QAEnD,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,4DAA4D,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;QACjE,OAAO;IACT,CAAC;IAED,sCAAsC;IACtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,YAAY,eAAe,EAAE;YAC3D,OAAO,EAAE;gBACP,eAAe,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE;aAC3C;YACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,OAAO,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;gBAC5E,OAAO,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;gBAC9E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAsB,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAEnF,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,SAAS,SAAS,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QACtG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,yBAAyB,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC1G,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAE7E,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,4CAA4C,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QACxG,OAAO,CAAC,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC1E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
|
package/dist/cli.js
CHANGED
|
@@ -19,11 +19,56 @@ import { peel, peelBatch, cleanup } from './index.js';
|
|
|
19
19
|
import { checkUsage, showUsageFooter, handleLogin, handleLogout, handleUsage, loadConfig } from './cli-auth.js';
|
|
20
20
|
import { getCache, setCache, parseTTL, clearCache, cacheStats } from './cache.js';
|
|
21
21
|
const program = new Command();
|
|
22
|
+
// Read version from package.json dynamically
|
|
23
|
+
import { fileURLToPath } from 'url';
|
|
24
|
+
import { dirname, resolve } from 'path';
|
|
25
|
+
let cliVersion = '0.0.0';
|
|
26
|
+
try {
|
|
27
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
28
|
+
const pkgPath = resolve(__dirname, '..', 'package.json');
|
|
29
|
+
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
|
|
30
|
+
cliVersion = pkg.version;
|
|
31
|
+
}
|
|
32
|
+
catch { /* fallback */ }
|
|
22
33
|
program
|
|
23
34
|
.name('webpeel')
|
|
24
35
|
.description('Fast web fetcher for AI agents')
|
|
25
|
-
.version(
|
|
36
|
+
.version(cliVersion)
|
|
26
37
|
.enablePositionalOptions();
|
|
38
|
+
/**
|
|
39
|
+
* Parse action strings into PageAction array
|
|
40
|
+
* Format: "type:value" where type is wait|click|scroll|type|fill|press|hover|waitFor
|
|
41
|
+
*/
|
|
42
|
+
function parseActions(actionStrings) {
|
|
43
|
+
return actionStrings.map(str => {
|
|
44
|
+
const [type, ...rest] = str.split(':');
|
|
45
|
+
const value = rest.join(':');
|
|
46
|
+
switch (type) {
|
|
47
|
+
case 'wait':
|
|
48
|
+
return { type: 'wait', ms: parseInt(value) || 1000 };
|
|
49
|
+
case 'click':
|
|
50
|
+
return { type: 'click', selector: value };
|
|
51
|
+
case 'scroll':
|
|
52
|
+
return { type: 'scroll', to: value === 'top' ? 'top' : value === 'bottom' ? 'bottom' : parseInt(value) };
|
|
53
|
+
case 'type': {
|
|
54
|
+
const [sel, ...text] = value.split('=');
|
|
55
|
+
return { type: 'type', selector: sel, value: text.join('=') };
|
|
56
|
+
}
|
|
57
|
+
case 'fill': {
|
|
58
|
+
const [sel, ...text] = value.split('=');
|
|
59
|
+
return { type: 'fill', selector: sel, value: text.join('=') };
|
|
60
|
+
}
|
|
61
|
+
case 'press':
|
|
62
|
+
return { type: 'press', key: value };
|
|
63
|
+
case 'hover':
|
|
64
|
+
return { type: 'hover', selector: value };
|
|
65
|
+
case 'waitFor':
|
|
66
|
+
return { type: 'waitForSelector', selector: value };
|
|
67
|
+
default:
|
|
68
|
+
throw new Error(`Unknown action type: ${type}`);
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
}
|
|
27
72
|
program
|
|
28
73
|
.argument('[url]', 'URL to fetch')
|
|
29
74
|
.option('-r, --render', 'Use headless browser (for JS-heavy sites)')
|
|
@@ -45,6 +90,9 @@ program
|
|
|
45
90
|
.option('--links', 'Output only the links found on the page')
|
|
46
91
|
.option('--meta', 'Output only the page metadata (title, description, author, etc.)')
|
|
47
92
|
.option('--raw', 'Return full page without smart content extraction')
|
|
93
|
+
.option('--action <actions...>', 'Page actions before scraping (e.g., "click:.btn" "wait:2000" "scroll:bottom")')
|
|
94
|
+
.option('--extract <json>', 'Extract structured data using CSS selectors (JSON object of field:selector pairs)')
|
|
95
|
+
.option('--max-tokens <n>', 'Maximum token count for output (truncate if exceeded)', parseInt)
|
|
48
96
|
.action(async (url, options) => {
|
|
49
97
|
if (!url) {
|
|
50
98
|
console.error('Error: URL is required\n');
|
|
@@ -122,9 +170,39 @@ program
|
|
|
122
170
|
headers[key] = value;
|
|
123
171
|
}
|
|
124
172
|
}
|
|
173
|
+
// Parse actions
|
|
174
|
+
let actions;
|
|
175
|
+
if (options.action && options.action.length > 0) {
|
|
176
|
+
try {
|
|
177
|
+
actions = parseActions(options.action);
|
|
178
|
+
}
|
|
179
|
+
catch (e) {
|
|
180
|
+
console.error(`Error: ${e.message}`);
|
|
181
|
+
process.exit(1);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
// Parse extract
|
|
185
|
+
let extract;
|
|
186
|
+
if (options.extract) {
|
|
187
|
+
try {
|
|
188
|
+
extract = { selectors: JSON.parse(options.extract) };
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
console.error('Error: --extract must be valid JSON (e.g., \'{"title": "h1", "price": ".price"}\')');
|
|
192
|
+
process.exit(1);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
// Validate maxTokens
|
|
196
|
+
if (options.maxTokens !== undefined) {
|
|
197
|
+
if (isNaN(options.maxTokens) || options.maxTokens < 100) {
|
|
198
|
+
console.error('Error: --max-tokens must be at least 100');
|
|
199
|
+
process.exit(1);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
125
202
|
// Build peel options
|
|
126
203
|
// --stealth auto-enables --render (stealth requires browser)
|
|
127
|
-
|
|
204
|
+
// --action auto-enables --render (actions require browser)
|
|
205
|
+
const useRender = options.render || options.stealth || (actions && actions.length > 0) || false;
|
|
128
206
|
const peelOptions = {
|
|
129
207
|
render: useRender,
|
|
130
208
|
stealth: options.stealth || false,
|
|
@@ -138,6 +216,9 @@ program
|
|
|
138
216
|
headers,
|
|
139
217
|
cookies: options.cookie,
|
|
140
218
|
raw: options.raw || false,
|
|
219
|
+
actions,
|
|
220
|
+
maxTokens: options.maxTokens,
|
|
221
|
+
extract,
|
|
141
222
|
};
|
|
142
223
|
// Determine format
|
|
143
224
|
if (options.html) {
|
|
@@ -312,7 +393,7 @@ program
|
|
|
312
393
|
// Batch command
|
|
313
394
|
program
|
|
314
395
|
.command('batch [file]')
|
|
315
|
-
.description('Fetch multiple URLs from file or stdin pipe
|
|
396
|
+
.description('Fetch multiple URLs from file or stdin pipe')
|
|
316
397
|
.option('-c, --concurrency <n>', 'Max concurrent fetches (default: 3)', '3')
|
|
317
398
|
.option('-o, --output <dir>', 'Output directory (one file per URL)')
|
|
318
399
|
.option('--json', 'Output as JSON array')
|
|
@@ -443,7 +524,7 @@ program
|
|
|
443
524
|
});
|
|
444
525
|
program
|
|
445
526
|
.command('crawl <url>')
|
|
446
|
-
.description('Crawl a website starting from a URL
|
|
527
|
+
.description('Crawl a website starting from a URL')
|
|
447
528
|
.option('--max-pages <number>', 'Maximum number of pages to crawl (default: 10, max: 100)', parseInt, 10)
|
|
448
529
|
.option('--max-depth <number>', 'Maximum depth to crawl (default: 2, max: 5)', parseInt, 2)
|
|
449
530
|
.option('--allowed-domains <domains...>', 'Only crawl these domains (default: same as starting URL)')
|
|
@@ -517,6 +598,52 @@ program
|
|
|
517
598
|
process.exit(1);
|
|
518
599
|
}
|
|
519
600
|
});
|
|
601
|
+
program
|
|
602
|
+
.command('map <url>')
|
|
603
|
+
.description('Discover all URLs on a domain (sitemap + crawl)')
|
|
604
|
+
.option('--no-sitemap', 'Skip sitemap.xml discovery')
|
|
605
|
+
.option('--no-crawl', 'Skip homepage crawl')
|
|
606
|
+
.option('--max <n>', 'Maximum URLs to discover (default: 5000)', parseInt, 5000)
|
|
607
|
+
.option('--include <patterns...>', 'Include only URLs matching these regex patterns')
|
|
608
|
+
.option('--exclude <patterns...>', 'Exclude URLs matching these regex patterns')
|
|
609
|
+
.option('--json', 'Output as JSON')
|
|
610
|
+
.option('-s, --silent', 'Silent mode')
|
|
611
|
+
.action(async (url, options) => {
|
|
612
|
+
const { mapDomain } = await import('./core/map.js');
|
|
613
|
+
const spinner = options.silent ? null : ora('Discovering URLs...').start();
|
|
614
|
+
try {
|
|
615
|
+
const result = await mapDomain(url, {
|
|
616
|
+
useSitemap: options.sitemap !== false,
|
|
617
|
+
crawlHomepage: options.crawl !== false,
|
|
618
|
+
maxUrls: options.max,
|
|
619
|
+
includePatterns: options.include,
|
|
620
|
+
excludePatterns: options.exclude,
|
|
621
|
+
});
|
|
622
|
+
if (spinner)
|
|
623
|
+
spinner.succeed(`Found ${result.total} URLs in ${result.elapsed}ms`);
|
|
624
|
+
if (options.json) {
|
|
625
|
+
console.log(JSON.stringify(result, null, 2));
|
|
626
|
+
}
|
|
627
|
+
else {
|
|
628
|
+
for (const url of result.urls) {
|
|
629
|
+
console.log(url);
|
|
630
|
+
}
|
|
631
|
+
if (!options.silent) {
|
|
632
|
+
console.error(`\nTotal: ${result.total} URLs`);
|
|
633
|
+
if (result.sitemapUrls.length > 0) {
|
|
634
|
+
console.error(`Sitemaps used: ${result.sitemapUrls.join(', ')}`);
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
process.exit(0);
|
|
639
|
+
}
|
|
640
|
+
catch (error) {
|
|
641
|
+
if (spinner)
|
|
642
|
+
spinner.fail('URL discovery failed');
|
|
643
|
+
console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
644
|
+
process.exit(1);
|
|
645
|
+
}
|
|
646
|
+
});
|
|
520
647
|
program
|
|
521
648
|
.command('login')
|
|
522
649
|
.description('Authenticate the CLI with your API key')
|