webpeel 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +145 -42
- package/dist/cli-auth.d.ts +84 -0
- package/dist/cli-auth.d.ts.map +1 -0
- package/dist/cli-auth.js +392 -0
- package/dist/cli-auth.js.map +1 -0
- package/dist/cli.js +171 -2
- package/dist/cli.js.map +1 -1
- package/dist/core/crawler.d.ts +58 -0
- package/dist/core/crawler.d.ts.map +1 -0
- package/dist/core/crawler.js +205 -0
- package/dist/core/crawler.js.map +1 -0
- package/dist/core/fetcher.d.ts +1 -0
- package/dist/core/fetcher.d.ts.map +1 -1
- package/dist/core/fetcher.js +35 -4
- package/dist/core/fetcher.js.map +1 -1
- package/dist/core/markdown.d.ts.map +1 -1
- package/dist/core/markdown.js +31 -22
- package/dist/core/markdown.js.map +1 -1
- package/dist/core/strategies.d.ts +6 -3
- package/dist/core/strategies.d.ts.map +1 -1
- package/dist/core/strategies.js +33 -7
- package/dist/core/strategies.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +163 -7
- package/dist/mcp/server.js.map +1 -1
- package/dist/types.d.ts +4 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/llms.txt +4 -2
- package/package.json +11 -4
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,cAAc,GAAG;IACrB,
|
|
1
|
+
{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,cAAc,GAAG;IACrB,4BAA4B;IAC5B,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,wBAAwB;IACjE,yBAAyB;IACzB,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO;IAClC,qBAAqB,EAAE,iBAAiB,EAAE,sBAAsB;IAChE,wBAAwB,EAAE,iBAAiB;IAC3C,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW;IAC3D,aAAa,EAAE,cAAc,EAAE,uBAAuB;IACtD,aAAa,EAAE,uBAAuB;IACtC,iBAAiB;IACjB,gBAAgB,EAAE,KAAK,EAAE,gBAAgB,EAAE,aAAa;IACxD,mBAAmB,EAAE,oBAAoB,EAAE,kBAAkB;IAC7D,mBAAmB;IACnB,gBAAgB,EAAE,gBAAgB,EAAE,iBAAiB;IACrD,mBAAmB,EAAE,gBAAgB;IACrC,oBAAoB,EAAE,iBAAiB;IACvC,0BAA0B;IAC1B,mBAAmB,EAAE,kBAAkB,EAAE,kBAAkB;IAC3D,oBAAoB,EAAE,6BAA6B;IACnD,mBAAmB;IACnB,eAAe,EAAE,kBAAkB,EAAE,mBAAmB;IACxD,mBAAmB;IACnB,oBAAoB,EAAE,uBAAuB,EAAE,sBAAsB;IACrE,gBAAgB,EAAE,2BAA2B,EAAE,mBAAmB;IAClE,kBAAkB;IAClB,gBAAgB,EAAE,oBAAoB,EAAE,wBAAwB;IAChE,yBAAyB,EAAE,yBAAyB;IACpD,WAAW;IACX,WAAW,EAAE,WAAW,EAAE,oBAAoB;IAC9C,+CAA+C;IAC/C,kBAAkB;CACnB,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,QAAgB,EAAE,OAAkB;IAC9E,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,uBAAuB;IACvB,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,2BAA2B;IAC3B,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,oDAAoD;QACpD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0CAA0C;IAC1C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAAC,IAAY;IAC7B,2CAA2C;IAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,OAAO;QAC3C,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,uBAAuB;IACvB,cAAc,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAClC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,mCAAmC;IACnC,CAAC,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC;IAEjC,uCAAuC;IACvC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACtB,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACtB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAEpC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;QACnC,YAAY,EAAE,KAAK;QACnB,cAAc,EAAE,QAAQ;QACxB,gBAAgB,EAAE,GAAG;QACrB,WAAW,EAAE,GAAG;QAChB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IAEH,kBAAkB;IAClB,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAE7D,kDAAkD;IAClD,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE;QACzB,MAAM,EAAE,KAAK;QACb,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,GAAG,GAAI,IAAY,CAAC,GAAG,CAAC;YAC9B,MAAM,GAAG,GAAI,IAAY,CAAC,GAAG,CAAC;YAC9B,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;YAC7B,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC;KACF,CAAC,CAAC;IAEH,oCAAoC;IACpC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE;QAC7B,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACf,OAAO,IAAI,CAAC,QAAQ,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,QAAQ,KAAK,MAAM,CAAC;QACzE,CAAC;QACD,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAiB,CAAC;YACxC,MAAM,SAAS,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACvD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9D,OAAO,SAAS,GAAG,QAAQ,GAAG,IAAI,GAAG,QAAQ,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1E,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAE9C,kEAAkE;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,yBAAyB;QAC5D,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED,8DAA8D;IAC9D,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE;QAC3D,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;QACrC,IAAI,SAAS,IAAI,SAAS;YAAE,OAAO,GAAG,CAAC;QACvC,OAAO,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;IAC3B,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,qCAAqC;IACrC,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAE3B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEpC,8CAA8C;IAC9C,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,CAAC,CAAC,+BAA+B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,IAAI,OAAO,GAAG,MAAM,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,yDAAyD;IACzD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACjB,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,gCAAgC;IAChC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAEpC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC"}
|
|
@@ -5,6 +5,8 @@ import { type FetchResult } from './fetcher.js';
|
|
|
5
5
|
export interface StrategyOptions {
|
|
6
6
|
/** Force browser mode (skip simple fetch) */
|
|
7
7
|
forceBrowser?: boolean;
|
|
8
|
+
/** Use stealth mode to bypass bot detection */
|
|
9
|
+
stealth?: boolean;
|
|
8
10
|
/** Wait time after page load in browser mode (ms) */
|
|
9
11
|
waitMs?: number;
|
|
10
12
|
/** Custom user agent */
|
|
@@ -21,8 +23,8 @@ export interface StrategyOptions {
|
|
|
21
23
|
cookies?: string[];
|
|
22
24
|
}
|
|
23
25
|
export interface StrategyResult extends FetchResult {
|
|
24
|
-
/** Which strategy succeeded: 'simple' | 'browser' */
|
|
25
|
-
method: 'simple' | 'browser';
|
|
26
|
+
/** Which strategy succeeded: 'simple' | 'browser' | 'stealth' */
|
|
27
|
+
method: 'simple' | 'browser' | 'stealth';
|
|
26
28
|
}
|
|
27
29
|
/**
|
|
28
30
|
* Smart fetch with automatic escalation
|
|
@@ -30,7 +32,8 @@ export interface StrategyResult extends FetchResult {
|
|
|
30
32
|
* Strategy:
|
|
31
33
|
* 1. Try simple HTTP fetch first (fast, ~200ms)
|
|
32
34
|
* 2. If blocked (403, 503, Cloudflare, empty body) → try browser
|
|
33
|
-
* 3. If browser
|
|
35
|
+
* 3. If browser gets blocked (403, CAPTCHA) → try stealth mode
|
|
36
|
+
* 4. If stealth mode is explicitly requested → skip to stealth
|
|
34
37
|
*
|
|
35
38
|
* Returns the result along with which method worked
|
|
36
39
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAyC,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AAGvF,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,cAAe,SAAQ,WAAW;IACjD,
|
|
1
|
+
{"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAyC,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AAGvF,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,cAAe,SAAQ,WAAW;IACjD,iEAAiE;IACjE,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;CAC1C;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,cAAc,CAAC,CAqGpG"}
|
package/dist/core/strategies.js
CHANGED
|
@@ -9,14 +9,15 @@ import { BlockedError, NetworkError } from '../types.js';
|
|
|
9
9
|
* Strategy:
|
|
10
10
|
* 1. Try simple HTTP fetch first (fast, ~200ms)
|
|
11
11
|
* 2. If blocked (403, 503, Cloudflare, empty body) → try browser
|
|
12
|
-
* 3. If browser
|
|
12
|
+
* 3. If browser gets blocked (403, CAPTCHA) → try stealth mode
|
|
13
|
+
* 4. If stealth mode is explicitly requested → skip to stealth
|
|
13
14
|
*
|
|
14
15
|
* Returns the result along with which method worked
|
|
15
16
|
*/
|
|
16
17
|
export async function smartFetch(url, options = {}) {
|
|
17
|
-
const { forceBrowser = false, waitMs = 0, userAgent, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies } = options;
|
|
18
|
-
// If
|
|
19
|
-
const shouldUseBrowser = forceBrowser || screenshot;
|
|
18
|
+
const { forceBrowser = false, stealth = false, waitMs = 0, userAgent, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies } = options;
|
|
19
|
+
// If stealth is requested, force browser mode (stealth requires browser)
|
|
20
|
+
const shouldUseBrowser = forceBrowser || screenshot || stealth;
|
|
20
21
|
// Strategy 1: Simple fetch (unless browser is forced or screenshot is requested)
|
|
21
22
|
if (!shouldUseBrowser) {
|
|
22
23
|
try {
|
|
@@ -37,7 +38,7 @@ export async function smartFetch(url, options = {}) {
|
|
|
37
38
|
}
|
|
38
39
|
}
|
|
39
40
|
}
|
|
40
|
-
// Strategy 2: Browser fetch
|
|
41
|
+
// Strategy 2: Browser fetch (with or without stealth)
|
|
41
42
|
try {
|
|
42
43
|
const result = await browserFetch(url, {
|
|
43
44
|
userAgent,
|
|
@@ -47,13 +48,37 @@ export async function smartFetch(url, options = {}) {
|
|
|
47
48
|
screenshotFullPage,
|
|
48
49
|
headers,
|
|
49
50
|
cookies,
|
|
51
|
+
stealth,
|
|
50
52
|
});
|
|
51
53
|
return {
|
|
52
54
|
...result,
|
|
53
|
-
method: 'browser',
|
|
55
|
+
method: stealth ? 'stealth' : 'browser',
|
|
54
56
|
};
|
|
55
57
|
}
|
|
56
58
|
catch (error) {
|
|
59
|
+
// Strategy 3: If browser gets blocked, try stealth mode as fallback (unless already using stealth)
|
|
60
|
+
if (!stealth && error instanceof BlockedError) {
|
|
61
|
+
try {
|
|
62
|
+
const result = await browserFetch(url, {
|
|
63
|
+
userAgent,
|
|
64
|
+
waitMs,
|
|
65
|
+
timeoutMs,
|
|
66
|
+
screenshot,
|
|
67
|
+
screenshotFullPage,
|
|
68
|
+
headers,
|
|
69
|
+
cookies,
|
|
70
|
+
stealth: true, // Escalate to stealth mode
|
|
71
|
+
});
|
|
72
|
+
return {
|
|
73
|
+
...result,
|
|
74
|
+
method: 'stealth',
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
catch (stealthError) {
|
|
78
|
+
// If stealth also fails, throw the original error
|
|
79
|
+
throw stealthError;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
57
82
|
// If browser encounters Cloudflare, retry with extra wait time
|
|
58
83
|
if (error instanceof NetworkError &&
|
|
59
84
|
error.message.toLowerCase().includes('cloudflare')) {
|
|
@@ -65,10 +90,11 @@ export async function smartFetch(url, options = {}) {
|
|
|
65
90
|
screenshotFullPage,
|
|
66
91
|
headers,
|
|
67
92
|
cookies,
|
|
93
|
+
stealth, // Keep stealth setting
|
|
68
94
|
});
|
|
69
95
|
return {
|
|
70
96
|
...result,
|
|
71
|
-
method: 'browser',
|
|
97
|
+
method: stealth ? 'stealth' : 'browser',
|
|
72
98
|
};
|
|
73
99
|
}
|
|
74
100
|
throw error;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"strategies.js","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAoB,MAAM,cAAc,CAAC;AACvF,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"strategies.js","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAoB,MAAM,cAAc,CAAC;AACvF,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA4BzD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW,EAAE,UAA2B,EAAE;IACzE,MAAM,EACJ,YAAY,GAAG,KAAK,EACpB,OAAO,GAAG,KAAK,EACf,MAAM,GAAG,CAAC,EACV,SAAS,EACT,SAAS,GAAG,KAAK,EACjB,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,OAAO,EACP,OAAO,EACR,GAAG,OAAO,CAAC;IAEZ,yEAAyE;IACzE,MAAM,gBAAgB,GAAG,YAAY,IAAI,UAAU,IAAI,OAAO,CAAC;IAE/D,iFAAiF;IACjF,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAC7B,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,EACrD,CAAC,CACF,CAAC;YACF,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,QAAQ;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,8CAA8C;YAC9C,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBAClC,mCAAmC;YACrC,CAAC;iBAAM,CAAC;gBACN,kDAAkD;gBAClD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,sDAAsD;IACtD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;YACrC,SAAS;YACT,MAAM;YACN,SAAS;YACT,UAAU;YACV,kBAAkB;YAClB,OAAO;YACP,OAAO;YACP,OAAO;SACR,CAAC,CAAC;QACH,OAAO;YACL,GAAG,MAAM;YACT,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;SACxC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mGAAmG;QACnG,IAAI,CAAC,OAAO,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;YAC9C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;oBACrC,SAAS;oBACT,MAAM;oBACN,SAAS;oBACT,UAAU;oBACV,kBAAkB;oBAClB,OAAO;oBACP,OAAO;oBACP,OAAO,EAAE,IAAI,EAAE,2BAA2B;iBAC3C,CAAC,CAAC;gBACH,OAAO;oBACL,GAAG,MAAM;oBACT,MAAM,EAAE,SAAS;iBAClB,CAAC;YACJ,CAAC;YAAC,OAAO,YAAY,EAAE,CAAC;gBACtB,kDAAkD;gBAClD,MAAM,YAAY,CAAC;YACrB,CAAC;QACH,CAAC;QAED,+DAA+D;QAC/D,IACE,KAAK,YAAY,YAAY;YAC7B,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,EAClD,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;gBACrC,SAAS;gBACT,MAAM,EAAE,IAAI,EAAE,mCAAmC;gBACjD,SAAS;gBACT,UAAU;gBACV,kBAAkB;gBAClB,OAAO;gBACP,OAAO;gBACP,OAAO,EAAE,uBAAuB;aACjC,CAAC,CAAC;YACH,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;aACxC,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { cleanup } from './core/fetcher.js';
|
|
7
7
|
import type { PeelOptions, PeelResult } from './types.js';
|
|
8
8
|
export * from './types.js';
|
|
9
|
+
export { crawl, type CrawlOptions, type CrawlResult } from './core/crawler.js';
|
|
9
10
|
/**
|
|
10
11
|
* Fetch and extract content from a URL
|
|
11
12
|
*
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1D,cAAc,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1D,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE/E;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAkGtF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,WAAW,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACnD,OAAO,CAAC,CAAC,UAAU,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EAAE,CAAC,CAwB1D;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -8,6 +8,7 @@ import { htmlToMarkdown, htmlToText, estimateTokens, selectContent } from './cor
|
|
|
8
8
|
import { extractMetadata, extractLinks } from './core/metadata.js';
|
|
9
9
|
import { cleanup } from './core/fetcher.js';
|
|
10
10
|
export * from './types.js';
|
|
11
|
+
export { crawl } from './core/crawler.js';
|
|
11
12
|
/**
|
|
12
13
|
* Fetch and extract content from a URL
|
|
13
14
|
*
|
|
@@ -26,7 +27,7 @@ export * from './types.js';
|
|
|
26
27
|
*/
|
|
27
28
|
export async function peel(url, options = {}) {
|
|
28
29
|
const startTime = Date.now();
|
|
29
|
-
let { render = false, wait = 0, format = 'markdown', timeout = 30000, userAgent, screenshot = false, screenshotFullPage = false, selector, exclude, headers, cookies, } = options;
|
|
30
|
+
let { render = false, stealth = false, wait = 0, format = 'markdown', timeout = 30000, userAgent, screenshot = false, screenshotFullPage = false, selector, exclude, headers, cookies, } = options;
|
|
30
31
|
// Detect PDF URLs and force browser rendering
|
|
31
32
|
const isPdf = url.toLowerCase().endsWith('.pdf');
|
|
32
33
|
if (isPdf) {
|
|
@@ -36,10 +37,15 @@ export async function peel(url, options = {}) {
|
|
|
36
37
|
if (screenshot) {
|
|
37
38
|
render = true;
|
|
38
39
|
}
|
|
40
|
+
// If stealth is requested, force render mode
|
|
41
|
+
if (stealth) {
|
|
42
|
+
render = true;
|
|
43
|
+
}
|
|
39
44
|
try {
|
|
40
45
|
// Fetch the page
|
|
41
46
|
const fetchResult = await smartFetch(url, {
|
|
42
47
|
forceBrowser: render,
|
|
48
|
+
stealth,
|
|
43
49
|
waitMs: wait,
|
|
44
50
|
userAgent,
|
|
45
51
|
timeoutMs: timeout,
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAC/F,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAG5C,cAAc,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAC/F,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAG5C,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAuC,MAAM,mBAAmB,CAAC;AAE/E;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,UAAuB,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,EACF,MAAM,GAAG,KAAK,EACd,OAAO,GAAG,KAAK,EACf,IAAI,GAAG,CAAC,EACR,MAAM,GAAG,UAAU,EACnB,OAAO,GAAG,KAAK,EACf,SAAS,EACT,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,QAAQ,EACR,OAAO,EACP,OAAO,EACP,OAAO,GACR,GAAG,OAAO,CAAC;IAEZ,8CAA8C;IAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,gDAAgD;IAChD,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,IAAI,CAAC;QACH,iBAAiB;QACjB,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE;YACxC,YAAY,EAAE,MAAM;YACpB,OAAO;YACP,MAAM,EAAE,IAAI;YACZ,SAAS;YACT,SAAS,EAAE,OAAO;YAClB,UAAU;YACV,kBAAkB;YAClB,OAAO;YACP,OAAO;SACR,CAAC,CAAC;QAEH,wCAAwC;QACxC,IAAI,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;QAC5B,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,CAAC;QAED,6BAA6B;QAC7B,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;QAEnE,gBAAgB;QAChB,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;QAElD,sCAAsC;QACtC,IAAI,OAAe,CAAC;QACpB,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,MAAM;gBACT,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM;YACR,KAAK,MAAM;gBACT,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC3B,MAAM;YACR,KAAK,UAAU,CAAC;YAChB;gBACE,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;gBAC/B,MAAM;QACV,CAAC;QAED,4CAA4C;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACvC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QAEvC,iDAAiD;QACjD,MAAM,gBAAgB,GAAG,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEpE,OAAO;YACL,GAAG,EAAE,WAAW,CAAC,GAAG;YACpB,KAAK;YACL,OAAO;YACP,QAAQ;YACR,KAAK;YACL,MAAM;YACN,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,OAAO;YACP,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sCAAsC;QACtC,MAAM,OAAO,EAAE,CAAC;QAChB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAc,EACd,UAAkD,EAAE;IAEpD,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC;IACjD,MAAM,OAAO,GAAoD,EAAE,CAAC;IAEpE,qBAAqB;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CACtC,CAAC;QAEF,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACjC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;oBACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI,eAAe;iBACjD,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
|
package/dist/mcp/server.js
CHANGED
|
@@ -11,7 +11,7 @@ import { fetch as undiciFetch } from 'undici';
|
|
|
11
11
|
import { load } from 'cheerio';
|
|
12
12
|
const server = new Server({
|
|
13
13
|
name: 'webpeel',
|
|
14
|
-
version: '0.
|
|
14
|
+
version: '0.3.1',
|
|
15
15
|
}, {
|
|
16
16
|
capabilities: {
|
|
17
17
|
tools: {},
|
|
@@ -39,17 +39,29 @@ async function searchWeb(query, count = 5) {
|
|
|
39
39
|
return;
|
|
40
40
|
const $result = $(elem);
|
|
41
41
|
let title = $result.find('.result__title').text().trim();
|
|
42
|
-
|
|
42
|
+
const rawUrl = $result.find('.result__a').attr('href') || '';
|
|
43
43
|
let snippet = $result.find('.result__snippet').text().trim();
|
|
44
|
-
|
|
45
|
-
if (!title || !url)
|
|
44
|
+
if (!title || !rawUrl)
|
|
46
45
|
return;
|
|
47
|
-
//
|
|
46
|
+
// Extract actual URL from DuckDuckGo redirect
|
|
47
|
+
let url = rawUrl;
|
|
48
|
+
try {
|
|
49
|
+
const ddgUrl = new URL(rawUrl, 'https://duckduckgo.com');
|
|
50
|
+
const uddg = ddgUrl.searchParams.get('uddg');
|
|
51
|
+
if (uddg) {
|
|
52
|
+
url = decodeURIComponent(uddg);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
// Use raw URL if parsing fails
|
|
57
|
+
}
|
|
58
|
+
// SECURITY: Validate and sanitize results — only allow HTTP/HTTPS URLs
|
|
48
59
|
try {
|
|
49
60
|
const parsed = new URL(url);
|
|
50
61
|
if (!['http:', 'https:'].includes(parsed.protocol)) {
|
|
51
62
|
return;
|
|
52
63
|
}
|
|
64
|
+
url = parsed.href;
|
|
53
65
|
}
|
|
54
66
|
catch {
|
|
55
67
|
return;
|
|
@@ -69,7 +81,7 @@ async function searchWeb(query, count = 5) {
|
|
|
69
81
|
const tools = [
|
|
70
82
|
{
|
|
71
83
|
name: 'webpeel_fetch',
|
|
72
|
-
description: 'Fetch a URL and return clean, AI-ready markdown content. Handles JavaScript rendering and anti-bot protections automatically. Use this when you need to read the content of a web page.',
|
|
84
|
+
description: 'Fetch a URL and return clean, AI-ready markdown content. Handles JavaScript rendering and anti-bot protections automatically. Use this when you need to read the content of a web page. For protected sites, use stealth=true to bypass bot detection.',
|
|
73
85
|
annotations: {
|
|
74
86
|
title: 'Fetch Web Page',
|
|
75
87
|
readOnlyHint: true,
|
|
@@ -89,6 +101,11 @@ const tools = [
|
|
|
89
101
|
description: 'Force browser rendering (slower but handles JavaScript-heavy sites)',
|
|
90
102
|
default: false,
|
|
91
103
|
},
|
|
104
|
+
stealth: {
|
|
105
|
+
type: 'boolean',
|
|
106
|
+
description: 'Use stealth mode to bypass bot detection (auto-enables render=true)',
|
|
107
|
+
default: false,
|
|
108
|
+
},
|
|
92
109
|
wait: {
|
|
93
110
|
type: 'number',
|
|
94
111
|
description: 'Milliseconds to wait for dynamic content (only used with render=true)',
|
|
@@ -203,6 +220,76 @@ const tools = [
|
|
|
203
220
|
required: ['urls'],
|
|
204
221
|
},
|
|
205
222
|
},
|
|
223
|
+
{
|
|
224
|
+
name: 'webpeel_crawl',
|
|
225
|
+
description: 'Crawl a website starting from a URL, following links and extracting content. Respects robots.txt and rate limits. Perfect for gathering documentation or site content.',
|
|
226
|
+
annotations: {
|
|
227
|
+
title: 'Crawl Website',
|
|
228
|
+
readOnlyHint: true,
|
|
229
|
+
destructiveHint: false,
|
|
230
|
+
idempotentHint: true,
|
|
231
|
+
openWorldHint: true,
|
|
232
|
+
},
|
|
233
|
+
inputSchema: {
|
|
234
|
+
type: 'object',
|
|
235
|
+
properties: {
|
|
236
|
+
url: {
|
|
237
|
+
type: 'string',
|
|
238
|
+
description: 'Starting URL to crawl from',
|
|
239
|
+
},
|
|
240
|
+
maxPages: {
|
|
241
|
+
type: 'number',
|
|
242
|
+
description: 'Maximum number of pages to crawl (default: 10, max: 100)',
|
|
243
|
+
default: 10,
|
|
244
|
+
minimum: 1,
|
|
245
|
+
maximum: 100,
|
|
246
|
+
},
|
|
247
|
+
maxDepth: {
|
|
248
|
+
type: 'number',
|
|
249
|
+
description: 'Maximum depth to crawl (default: 2, max: 5)',
|
|
250
|
+
default: 2,
|
|
251
|
+
minimum: 1,
|
|
252
|
+
maximum: 5,
|
|
253
|
+
},
|
|
254
|
+
allowedDomains: {
|
|
255
|
+
type: 'array',
|
|
256
|
+
items: {
|
|
257
|
+
type: 'string',
|
|
258
|
+
},
|
|
259
|
+
description: 'Only crawl URLs from these domains (default: same domain as starting URL)',
|
|
260
|
+
},
|
|
261
|
+
excludePatterns: {
|
|
262
|
+
type: 'array',
|
|
263
|
+
items: {
|
|
264
|
+
type: 'string',
|
|
265
|
+
},
|
|
266
|
+
description: 'Exclude URLs matching these regex patterns',
|
|
267
|
+
},
|
|
268
|
+
respectRobotsTxt: {
|
|
269
|
+
type: 'boolean',
|
|
270
|
+
description: 'Respect robots.txt (default: true)',
|
|
271
|
+
default: true,
|
|
272
|
+
},
|
|
273
|
+
rateLimitMs: {
|
|
274
|
+
type: 'number',
|
|
275
|
+
description: 'Rate limit between requests in milliseconds (default: 1000)',
|
|
276
|
+
default: 1000,
|
|
277
|
+
minimum: 100,
|
|
278
|
+
},
|
|
279
|
+
render: {
|
|
280
|
+
type: 'boolean',
|
|
281
|
+
description: 'Use browser rendering for all pages',
|
|
282
|
+
default: false,
|
|
283
|
+
},
|
|
284
|
+
stealth: {
|
|
285
|
+
type: 'boolean',
|
|
286
|
+
description: 'Use stealth mode for all pages',
|
|
287
|
+
default: false,
|
|
288
|
+
},
|
|
289
|
+
},
|
|
290
|
+
required: ['url'],
|
|
291
|
+
},
|
|
292
|
+
},
|
|
206
293
|
];
|
|
207
294
|
server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
|
208
295
|
tools,
|
|
@@ -211,7 +298,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
211
298
|
const { name, arguments: args } = request.params;
|
|
212
299
|
try {
|
|
213
300
|
if (name === 'webpeel_fetch') {
|
|
214
|
-
const { url, render, wait, format, screenshot, screenshotFullPage, selector, exclude, headers } = args;
|
|
301
|
+
const { url, render, stealth, wait, format, screenshot, screenshotFullPage, selector, exclude, headers } = args;
|
|
215
302
|
// SECURITY: Validate input parameters
|
|
216
303
|
if (!url || typeof url !== 'string') {
|
|
217
304
|
throw new Error('Invalid URL parameter');
|
|
@@ -238,6 +325,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
238
325
|
}
|
|
239
326
|
const options = {
|
|
240
327
|
render: render || false,
|
|
328
|
+
stealth: stealth || false,
|
|
241
329
|
wait: wait || 0,
|
|
242
330
|
format: format || 'markdown',
|
|
243
331
|
screenshot: screenshot || false,
|
|
@@ -376,6 +464,74 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
376
464
|
],
|
|
377
465
|
};
|
|
378
466
|
}
|
|
467
|
+
if (name === 'webpeel_crawl') {
|
|
468
|
+
const { crawl } = await import('../core/crawler.js');
|
|
469
|
+
const { url, maxPages, maxDepth, allowedDomains, excludePatterns, respectRobotsTxt, rateLimitMs, render, stealth, } = args;
|
|
470
|
+
// SECURITY: Validate input parameters
|
|
471
|
+
if (!url || typeof url !== 'string') {
|
|
472
|
+
throw new Error('Invalid URL parameter');
|
|
473
|
+
}
|
|
474
|
+
if (url.length > 2048) {
|
|
475
|
+
throw new Error('URL too long (max 2048 characters)');
|
|
476
|
+
}
|
|
477
|
+
if (maxPages !== undefined) {
|
|
478
|
+
if (typeof maxPages !== 'number' || isNaN(maxPages) || maxPages < 1 || maxPages > 100) {
|
|
479
|
+
throw new Error('Invalid maxPages parameter: must be between 1 and 100');
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
if (maxDepth !== undefined) {
|
|
483
|
+
if (typeof maxDepth !== 'number' || isNaN(maxDepth) || maxDepth < 1 || maxDepth > 5) {
|
|
484
|
+
throw new Error('Invalid maxDepth parameter: must be between 1 and 5');
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
if (allowedDomains !== undefined && !Array.isArray(allowedDomains)) {
|
|
488
|
+
throw new Error('Invalid allowedDomains parameter: must be an array');
|
|
489
|
+
}
|
|
490
|
+
if (excludePatterns !== undefined && !Array.isArray(excludePatterns)) {
|
|
491
|
+
throw new Error('Invalid excludePatterns parameter: must be an array');
|
|
492
|
+
}
|
|
493
|
+
if (rateLimitMs !== undefined) {
|
|
494
|
+
if (typeof rateLimitMs !== 'number' || isNaN(rateLimitMs) || rateLimitMs < 100) {
|
|
495
|
+
throw new Error('Invalid rateLimitMs parameter: must be at least 100ms');
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
// SECURITY: Wrap in timeout (10 minutes max for crawling)
|
|
499
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
500
|
+
setTimeout(() => reject(new Error('Crawl operation timed out after 10 minutes')), 600000);
|
|
501
|
+
});
|
|
502
|
+
const results = await Promise.race([
|
|
503
|
+
crawl(url, {
|
|
504
|
+
maxPages,
|
|
505
|
+
maxDepth,
|
|
506
|
+
allowedDomains,
|
|
507
|
+
excludePatterns,
|
|
508
|
+
respectRobotsTxt,
|
|
509
|
+
rateLimitMs,
|
|
510
|
+
render,
|
|
511
|
+
stealth,
|
|
512
|
+
}),
|
|
513
|
+
timeoutPromise,
|
|
514
|
+
]);
|
|
515
|
+
// SECURITY: Handle JSON serialization errors
|
|
516
|
+
let resultText;
|
|
517
|
+
try {
|
|
518
|
+
resultText = JSON.stringify(results, null, 2);
|
|
519
|
+
}
|
|
520
|
+
catch (jsonError) {
|
|
521
|
+
resultText = JSON.stringify({
|
|
522
|
+
error: 'serialization_error',
|
|
523
|
+
message: 'Failed to serialize crawl results',
|
|
524
|
+
});
|
|
525
|
+
}
|
|
526
|
+
return {
|
|
527
|
+
content: [
|
|
528
|
+
{
|
|
529
|
+
type: 'text',
|
|
530
|
+
text: resultText,
|
|
531
|
+
},
|
|
532
|
+
],
|
|
533
|
+
};
|
|
534
|
+
}
|
|
379
535
|
throw new Error(`Unknown tool: ${name}`);
|
|
380
536
|
}
|
|
381
537
|
catch (error) {
|
package/dist/mcp/server.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.js","sourceRoot":"","sources":["../../src/mcp/server.ts"],"names":[],"mappings":";AAEA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EACL,qBAAqB,EACrB,sBAAsB,GAEvB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAE/B,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB;IACE,IAAI,EAAE,SAAS;IACf,OAAO,EAAE,OAAO;CACjB,EACD;IACE,YAAY,EAAE;QACZ,KAAK,EAAE,EAAE;KACV;CACF,CACF,CAAC;AAEF;;GAEG;AACH,KAAK,UAAU,SAAS,CAAC,KAAa,EAAE,QAAgB,CAAC;IAKvD,MAAM,SAAS,GAAG,uCAAuC,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;IAErF,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE;YAC5C,OAAO,EAAE;gBACP,YAAY,EAAE,oEAAoE;aACnF;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QAErB,MAAM,OAAO,GAA2D,EAAE,CAAC;QAE3E,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE;YAC7B,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;gBAAE,OAAO;YAEpC,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YACxB,IAAI,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACzD,IAAI,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC1D,IAAI,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAE7D,0CAA0C;YAC1C,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG;gBAAE,OAAO;YAE3B,6BAA6B;YAC7B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC5B,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACnD,OAAO;gBACT,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;YACT,CAAC;YAED,sCAAsC;YACtC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC5B,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAEhC,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAc,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;AACH,CAAC;AAED,MAAM,KAAK,GAAW;IACpB;QACE,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,yLAAyL;QACtM,WAAW,EAAE;YACX,KAAK,EAAE,gBAAgB;YACvB,YAAY,EAAE,IAAI;YAClB,eAAe,EAAE,KAAK;YACtB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;QACD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,kBAAkB;iBAChC;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,qEAAqE;oBAClF,OAAO,EAAE,KAAK;iBACf;gBACD,IAAI,EAAE;oBACJ,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,uEAAuE;oBACpF,OAAO,EAAE,CAAC;iBACX;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC;oBAClC,WAAW,EAAE,kDAAkD;oBAC/D,OAAO,EAAE,UAAU;iBACpB;gBACD,UAAU,EAAE;oBACV,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,+DAA+D;oBAC5E,OAAO,EAAE,KAAK;iBACf;gBACD,kBAAkB,EAAE;oBAClB,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,+CAA+C;oBAC5D,OAAO,EAAE,KAAK;iBACf;gBACD,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,6EAA6E;iBAC3F;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;qBACf;oBACD,WAAW,EAAE,oEAAoE;iBAClF;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,6BAA6B;iBAC3C;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,CAAC;SAClB;KACF;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,WAAW,EAAE,+IAA+I;QAC5J,WAAW,EAAE;YACX,KAAK,EAAE,gBAAgB;YACvB,YAAY,EAAE,IAAI;YAClB,eAAe,EAAE,KAAK;YACtB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;QACD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,cAAc;iBAC5B;gBACD,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,oCAAoC;oBACjD,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,EAAE;iBACZ;aACF;YACD,QAAQ,EAAE,CAAC,OAAO,CAAC;SACpB;KACF;IACD;QACE,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,+FAA+F;QAC5G,WAAW,EAAE;YACX,KAAK,EAAE,kBAAkB;YACzB,YAAY,EAAE,IAAI;YAClB,eAAe,EAAE,KAAK;YACtB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;QACD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,IAAI,EAAE;oBACJ,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;qBACf;oBACD,WAAW,EAAE,wBAAwB;iBACtC;gBACD,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,qCAAqC;oBAClD,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,EAAE;iBACZ;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,sCAAsC;oBACnD,OAAO,EAAE,KAAK;iBACf;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC;oBAClC,WAAW,EAAE,4BAA4B;oBACzC,OAAO,EAAE,UAAU;iBACpB;gBACD,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,0CAA0C;iBACxD;aACF;YACD,QAAQ,EAAE,CAAC,MAAM,CAAC;SACnB;KACF;CACF,CAAC;AAEF,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;IAC5D,KAAK;CACN,CAAC,CAAC,CAAC;AAEJ,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;IAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAEjD,IAAI,CAAC;QACH,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;YAC7B,MAAM,EACJ,GAAG,EACH,MAAM,EACN,IAAI,EACJ,MAAM,EACN,UAAU,EACV,kBAAkB,EAClB,QAAQ,EACR,OAAO,EACP,OAAO,EACR,GAAG,IAUH,CAAC;YAEF,sCAAsC;YACtC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAC3C,CAAC;YAED,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACxD,CAAC;YAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;oBACxE,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;gBAC3E,CAAC;YACH,CAAC;YAED,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3E,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;YACrF,CAAC;YAED,IAAI,QAAQ,KAAK,SAAS,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAC3D,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAClE,CAAC;YAED,IAAI,OAAO,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBACrD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;YACjE,CAAC;YAED,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACzD,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAClE,CAAC;YAED,MAAM,OAAO,GAAgB;gBAC3B,MAAM,EAAE,MAAM,IAAI,KAAK;gBACvB,IAAI,EAAE,IAAI,IAAI,CAAC;gBACf,MAAM,EAAE,MAAM,IAAI,UAAU;gBAC5B,UAAU,EAAE,UAAU,IAAI,KAAK;gBAC/B,kBAAkB,EAAE,kBAAkB,IAAI,KAAK;gBAC/C,QAAQ;gBACR,OAAO;gBACP,OAAO;aACR,CAAC;YAEF,6CAA6C;YAC7C,MAAM,cAAc,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/C,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAClF,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBAChC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC;gBAClB,cAAc;aACf,CAAQ,CAAC;YAEV,6CAA6C;YAC7C,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/C,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;oBAC1B,KAAK,EAAE,qBAAqB;oBAC5B,OAAO,EAAE,4BAA4B;iBACtC,CAAC,CAAC;YACL,CAAC;YAED,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;YAC9B,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAGxB,CAAC;YAEF,sCAAsC;YACtC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACxC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC7C,CAAC;YAED,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;oBACzE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC;YAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAE1D,6CAA6C;YAC7C,MAAM,cAAc,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/C,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACrF,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjC,SAAS,CAAC,KAAK,EAAE,WAAW,CAAC;gBAC7B,cAAc;aACf,CAAQ,CAAC;YAEV,6CAA6C;YAC7C,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;oBAC1B,KAAK,EAAE,qBAAqB;oBAC5B,OAAO,EAAE,6BAA6B;iBACvC,CAAC,CAAC;YACL,CAAC;YAED,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;YAC7B,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAMvD,CAAC;YAEF,sCAAsC;YACtC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;YAC9D,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAChD,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC5C,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBAC1C,CAAC;gBACD,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;oBACtB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;YAED,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC9B,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,EAAE,EAAE,CAAC;oBACjG,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;YAED,MAAM,OAAO,GAA2C;gBACtD,WAAW,EAAE,WAAW,IAAI,CAAC;gBAC7B,MAAM,EAAE,MAAM,IAAI,KAAK;gBACvB,MAAM,EAAE,MAAM,IAAI,UAAU;gBAC5B,QAAQ;aACT,CAAC;YAEF,sDAAsD;YACtD,MAAM,cAAc,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/C,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YAC3F,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC;gBACxB,cAAc;aACf,CAAQ,CAAC;YAEV,6CAA6C;YAC7C,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;oBAC1B,KAAK,EAAE,qBAAqB;oBAC5B,OAAO,EAAE,mCAAmC;iBAC7C,CAAC,CAAC;YACL,CAAC;YAED,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;IAC3C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAc,CAAC;QAC3B,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,OAAO;wBAC1B,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,wBAAwB;qBACjD,EAAE,IAAI,EAAE,CAAC,CAAC;iBACZ;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,KAAK,UAAU,IAAI;IACjB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAEhC,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;AACvD,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../../src/mcp/server.ts"],"names":[],"mappings":";AAEA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EACL,qBAAqB,EACrB,sBAAsB,GAEvB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAE/B,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB;IACE,IAAI,EAAE,SAAS;IACf,OAAO,EAAE,OAAO;CACjB,EACD;IACE,YAAY,EAAE;QACZ,KAAK,EAAE,EAAE;KACV;CACF,CACF,CAAC;AAEF;;GAEG;AACH,KAAK,UAAU,SAAS,CAAC,KAAa,EAAE,QAAgB,CAAC;IAKvD,MAAM,SAAS,GAAG,uCAAuC,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;IAErF,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE;YAC5C,OAAO,EAAE;gBACP,YAAY,EAAE,oEAAoE;aACnF;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QAErB,MAAM,OAAO,GAA2D,EAAE,CAAC;QAE3E,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE;YAC7B,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;gBAAE,OAAO;YAEpC,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YACxB,IAAI,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACzD,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC7D,IAAI,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAE7D,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM;gBAAE,OAAO;YAE9B,8CAA8C;YAC9C,IAAI,GAAG,GAAG,MAAM,CAAC;YACjB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;gBACzD,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC7C,IAAI,IAAI,EAAE,CAAC;oBACT,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,+BAA+B;YACjC,CAAC;YAED,uEAAuE;YACvE,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC5B,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACnD,OAAO;gBACT,CAAC;gBACD,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC;YACpB,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;YACT,CAAC;YAED,sCAAsC;YACtC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC5B,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAEhC,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAc,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;AACH,CAAC;AAED,MAAM,KAAK,GAAW;IACpB;QACE,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,wPAAwP;QACrQ,WAAW,EAAE;YACX,KAAK,EAAE,gBAAgB;YACvB,YAAY,EAAE,IAAI;YAClB,eAAe,EAAE,KAAK;YACtB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;QACD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,kBAAkB;iBAChC;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,qEAAqE;oBAClF,OAAO,EAAE,KAAK;iBACf;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,qEAAqE;oBAClF,OAAO,EAAE,KAAK;iBACf;gBACD,IAAI,EAAE;oBACJ,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,uEAAuE;oBACpF,OAAO,EAAE,CAAC;iBACX;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC;oBAClC,WAAW,EAAE,kDAAkD;oBAC/D,OAAO,EAAE,UAAU;iBACpB;gBACD,UAAU,EAAE;oBACV,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,+DAA+D;oBAC5E,OAAO,EAAE,KAAK;iBACf;gBACD,kBAAkB,EAAE;oBAClB,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,+CAA+C;oBAC5D,OAAO,EAAE,KAAK;iBACf;gBACD,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,6EAA6E;iBAC3F;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;qBACf;oBACD,WAAW,EAAE,oEAAoE;iBAClF;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,6BAA6B;iBAC3C;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,CAAC;SAClB;KACF;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,WAAW,EAAE,+IAA+I;QAC5J,WAAW,EAAE;YACX,KAAK,EAAE,gBAAgB;YACvB,YAAY,EAAE,IAAI;YAClB,eAAe,EAAE,KAAK;YACtB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;QACD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,cAAc;iBAC5B;gBACD,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,oCAAoC;oBACjD,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,EAAE;iBACZ;aACF;YACD,QAAQ,EAAE,CAAC,OAAO,CAAC;SACpB;KACF;IACD;QACE,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,+FAA+F;QAC5G,WAAW,EAAE;YACX,KAAK,EAAE,kBAAkB;YACzB,YAAY,EAAE,IAAI;YAClB,eAAe,EAAE,KAAK;YACtB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;QACD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,IAAI,EAAE;oBACJ,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;qBACf;oBACD,WAAW,EAAE,wBAAwB;iBACtC;gBACD,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,qCAAqC;oBAClD,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,EAAE;iBACZ;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,sCAAsC;oBACnD,OAAO,EAAE,KAAK;iBACf;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC;oBAClC,WAAW,EAAE,4BAA4B;oBACzC,OAAO,EAAE,UAAU;iBACpB;gBACD,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,0CAA0C;iBACxD;aACF;YACD,QAAQ,EAAE,CAAC,MAAM,CAAC;SACnB;KACF;IACD;QACE,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,wKAAwK;QACrL,WAAW,EAAE;YACX,KAAK,EAAE,eAAe;YACtB,YAAY,EAAE,IAAI;YAClB,eAAe,EAAE,KAAK;YACtB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;QACD,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,4BAA4B;iBAC1C;gBACD,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,0DAA0D;oBACvE,OAAO,EAAE,EAAE;oBACX,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,GAAG;iBACb;gBACD,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,6CAA6C;oBAC1D,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,CAAC;iBACX;gBACD,cAAc,EAAE;oBACd,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;qBACf;oBACD,WAAW,EAAE,2EAA2E;iBACzF;gBACD,eAAe,EAAE;oBACf,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;qBACf;oBACD,WAAW,EAAE,4CAA4C;iBAC1D;gBACD,gBAAgB,EAAE;oBAChB,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,oCAAoC;oBACjD,OAAO,EAAE,IAAI;iBACd;gBACD,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,6DAA6D;oBAC1E,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,GAAG;iBACb;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,qCAAqC;oBAClD,OAAO,EAAE,KAAK;iBACf;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,gCAAgC;oBAC7C,OAAO,EAAE,KAAK;iBACf;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,CAAC;SAClB;KACF;CACF,CAAC;AAEF,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;IAC5D,KAAK;CACN,CAAC,CAAC,CAAC;AAEJ,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;IAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAEjD,IAAI,CAAC;QACH,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;YAC7B,MAAM,EACJ,GAAG,EACH,MAAM,EACN,OAAO,EACP,IAAI,EACJ,MAAM,EACN,UAAU,EACV,kBAAkB,EAClB,QAAQ,EACR,OAAO,EACP,OAAO,EACR,GAAG,IAWH,CAAC;YAEF,sCAAsC;YACtC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAC3C,CAAC;YAED,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACxD,CAAC;YAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;oBACxE,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;gBAC3E,CAAC;YACH,CAAC;YAED,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3E,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;YACrF,CAAC;YAED,IAAI,QAAQ,KAAK,SAAS,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAC3D,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAClE,CAAC;YAED,IAAI,OAAO,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBACrD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;YACjE,CAAC;YAED,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACzD,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAClE,CAAC;YAED,MAAM,OAAO,GAAgB;gBAC3B,MAAM,EAAE,MAAM,IAAI,KAAK;gBACvB,OAAO,EAAE,OAAO,IAAI,KAAK;gBACzB,IAAI,EAAE,IAAI,IAAI,CAAC;gBACf,MAAM,EAAE,MAAM,IAAI,UAAU;gBAC5B,UAAU,EAAE,UAAU,IAAI,KAAK;gBAC/B,kBAAkB,EAAE,kBAAkB,IAAI,KAAK;gBAC/C,QAAQ;gBACR,OAAO;gBACP,OAAO;aACR,CAAC;YAEF,6CAA6C;YAC7C,MAAM,cAAc,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/C,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAClF,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBAChC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC;gBAClB,cAAc;aACf,CAAQ,CAAC;YAEV,6CAA6C;YAC7C,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/C,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;oBAC1B,KAAK,EAAE,qBAAqB;oBAC5B,OAAO,EAAE,4BAA4B;iBACtC,CAAC,CAAC;YACL,CAAC;YAED,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;YAC9B,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAGxB,CAAC;YAEF,sCAAsC;YACtC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACxC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC7C,CAAC;YAED,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;oBACzE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC;YAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAE1D,6CAA6C;YAC7C,MAAM,cAAc,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/C,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACrF,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjC,SAAS,CAAC,KAAK,EAAE,WAAW,CAAC;gBAC7B,cAAc;aACf,CAAQ,CAAC;YAEV,6CAA6C;YAC7C,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;oBAC1B,KAAK,EAAE,qBAAqB;oBAC5B,OAAO,EAAE,6BAA6B;iBACvC,CAAC,CAAC;YACL,CAAC;YAED,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;YAC7B,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAMvD,CAAC;YAEF,sCAAsC;YACtC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;YAC9D,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAChD,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC5C,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBAC1C,CAAC;gBACD,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;oBACtB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;YAED,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC9B,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,EAAE,EAAE,CAAC;oBACjG,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;YAED,MAAM,OAAO,GAA2C;gBACtD,WAAW,EAAE,WAAW,IAAI,CAAC;gBAC7B,MAAM,EAAE,MAAM,IAAI,KAAK;gBACvB,MAAM,EAAE,MAAM,IAAI,UAAU;gBAC5B,QAAQ;aACT,CAAC;YAEF,sDAAsD;YACtD,MAAM,cAAc,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/C,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YAC3F,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC;gBACxB,cAAc;aACf,CAAQ,CAAC;YAEV,6CAA6C;YAC7C,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;oBAC1B,KAAK,EAAE,qBAAqB;oBAC5B,OAAO,EAAE,mCAAmC;iBAC7C,CAAC,CAAC;YACL,CAAC;YAED,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;YAC7B,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;YAErD,MAAM,EACJ,GAAG,EACH,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,WAAW,EACX,MAAM,EACN,OAAO,GACR,GAAG,IAUH,CAAC;YAEF,sCAAsC;YACtC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAC3C,CAAC;YAED,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACxD,CAAC;YAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;oBACtF,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;gBAC3E,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;oBACpF,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;YAED,IAAI,cAAc,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;gBACnE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;YACxE,CAAC;YAED,IAAI,eAAe,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACrE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;YACzE,CAAC;YAED,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC9B,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;oBAC/E,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;gBAC3E,CAAC;YACH,CAAC;YAED,0DAA0D;YAC1D,MAAM,cAAc,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/C,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YAC5F,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjC,KAAK,CAAC,GAAG,EAAE;oBACT,QAAQ;oBACR,QAAQ;oBACR,cAAc;oBACd,eAAe;oBACf,gBAAgB;oBAChB,WAAW;oBACX,MAAM;oBACN,OAAO;iBACR,CAAC;gBACF,cAAc;aACf,CAAQ,CAAC;YAEV,6CAA6C;YAC7C,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;oBAC1B,KAAK,EAAE,qBAAqB;oBAC5B,OAAO,EAAE,mCAAmC;iBAC7C,CAAC,CAAC;YACL,CAAC;YAED,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;IAC3C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAc,CAAC;QAC3B,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,OAAO;wBAC1B,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,wBAAwB;qBACjD,EAAE,IAAI,EAAE,CAAC,CAAC;iBACZ;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,KAAK,UAAU,IAAI;IACjB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAEhC,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;AACvD,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
export interface PeelOptions {
|
|
5
5
|
/** Use headless browser instead of simple HTTP fetch */
|
|
6
6
|
render?: boolean;
|
|
7
|
+
/** Use stealth mode to bypass bot detection (requires render=true, auto-enables if not set) */
|
|
8
|
+
stealth?: boolean;
|
|
7
9
|
/** Wait time in milliseconds after page load (only with render=true) */
|
|
8
10
|
wait?: number;
|
|
9
11
|
/** Output format */
|
|
@@ -38,8 +40,8 @@ export interface PeelResult {
|
|
|
38
40
|
links: string[];
|
|
39
41
|
/** Estimated token count (rough: content.length / 4) */
|
|
40
42
|
tokens: number;
|
|
41
|
-
/** Method used: 'simple' | 'browser' */
|
|
42
|
-
method: 'simple' | 'browser';
|
|
43
|
+
/** Method used: 'simple' | 'browser' | 'stealth' */
|
|
44
|
+
method: 'simple' | 'browser' | 'stealth';
|
|
43
45
|
/** Time elapsed in milliseconds */
|
|
44
46
|
elapsed: number;
|
|
45
47
|
/** Base64-encoded screenshot (PNG), only if screenshot option was set */
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,WAAW;IAC1B,wDAAwD;IACxD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wEAAwE;IACxE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB;IACpB,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;IACtC,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,yBAAyB;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,gEAAgE;IAChE,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,WAAW;IAC1B,wDAAwD;IACxD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+FAA+F;IAC/F,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,wEAAwE;IACxE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB;IACpB,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;IACtC,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,yBAAyB;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,gEAAgE;IAChE,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,oDAAoD;IACpD,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;IACzC,mCAAmC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,uBAAuB;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oBAAoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAa,SAAQ,KAAK;IACD,IAAI,CAAC,EAAE,MAAM;gBAArC,OAAO,EAAE,MAAM,EAAS,IAAI,CAAC,EAAE,MAAM,YAAA;CAIlD;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,YAAa,SAAQ,YAAY;gBAChC,OAAO,EAAE,MAAM;CAI5B"}
|
package/dist/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA+DH,MAAM,OAAO,YAAa,SAAQ,KAAK;IACD;IAApC,YAAY,OAAe,EAAS,IAAa;QAC/C,KAAK,CAAC,OAAO,CAAC,CAAC;QADmB,SAAI,GAAJ,IAAI,CAAS;QAE/C,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,YAAY;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF"}
|
package/llms.txt
CHANGED
|
@@ -45,7 +45,9 @@ Works with: Claude Desktop, Cursor, VS Code (Cline), Windsurf, Continue.dev, Ope
|
|
|
45
45
|
- Token-optimized: strips navigation, ads, scripts, cookie banners
|
|
46
46
|
- Metadata extraction: title, description, author, published date, links
|
|
47
47
|
- Search: built-in DuckDuckGo integration
|
|
48
|
-
-
|
|
48
|
+
- Usage model: Free tier (125 fetches/week), Pro (1,250/week), Max (6,250/week)
|
|
49
|
+
- First 25 fetches: No account needed — try instantly
|
|
50
|
+
- Local-first: runs on your machine, MIT licensed, can self-host
|
|
49
51
|
- SSRF protection: blocks private IPs, IPv6 mapped addresses, redirect attacks
|
|
50
52
|
- TypeScript-native: full type definitions, autocomplete, type safety
|
|
51
53
|
|
|
@@ -83,4 +85,4 @@ curl "https://api.webpeel.dev/v1/search?q=your+query"
|
|
|
83
85
|
- Website: https://webpeel.dev
|
|
84
86
|
- GitHub: https://github.com/JakeLiuMe/webpeel
|
|
85
87
|
- npm: https://www.npmjs.com/package/webpeel
|
|
86
|
-
- MCP Registry:
|
|
88
|
+
- MCP Registry: io.github.JakeLiuMe/webpeel
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "webpeel",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Fast web fetcher for AI agents - smart escalation from simple HTTP to headless browser",
|
|
3
|
+
"version": "0.3.1",
|
|
4
|
+
"description": "Fast web fetcher for AI agents - stealth mode, crawl mode, smart escalation from simple HTTP to headless browser",
|
|
5
5
|
"author": "Jake Liu",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"type": "module",
|
|
@@ -33,7 +33,11 @@
|
|
|
33
33
|
"dist/mcp",
|
|
34
34
|
"README.md",
|
|
35
35
|
"LICENSE",
|
|
36
|
-
"llms.txt"
|
|
36
|
+
"llms.txt",
|
|
37
|
+
"dist/cli-auth.js",
|
|
38
|
+
"dist/cli-auth.js.map",
|
|
39
|
+
"dist/cli-auth.d.ts",
|
|
40
|
+
"dist/cli-auth.d.ts.map"
|
|
37
41
|
],
|
|
38
42
|
"scripts": {
|
|
39
43
|
"build": "tsc",
|
|
@@ -85,6 +89,8 @@
|
|
|
85
89
|
"lru-cache": "^11.0.2",
|
|
86
90
|
"ora": "^8.0.1",
|
|
87
91
|
"playwright": "^1.48.0",
|
|
92
|
+
"playwright-extra": "^4.3.6",
|
|
93
|
+
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
|
88
94
|
"turndown": "^7.2.0",
|
|
89
95
|
"undici": "^7.2.0"
|
|
90
96
|
},
|
|
@@ -110,5 +116,6 @@
|
|
|
110
116
|
},
|
|
111
117
|
"engines": {
|
|
112
118
|
"node": ">=20.0.0"
|
|
113
|
-
}
|
|
119
|
+
},
|
|
120
|
+
"mcpName": "io.github.JakeLiuMe/webpeel"
|
|
114
121
|
}
|