webpeel 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -33
- package/dist/cli.js +69 -1
- package/dist/cli.js.map +1 -1
- package/dist/core/crawler.d.ts +58 -0
- package/dist/core/crawler.d.ts.map +1 -0
- package/dist/core/crawler.js +205 -0
- package/dist/core/crawler.js.map +1 -0
- package/dist/core/fetcher.d.ts +1 -0
- package/dist/core/fetcher.d.ts.map +1 -1
- package/dist/core/fetcher.js +27 -2
- package/dist/core/fetcher.js.map +1 -1
- package/dist/core/markdown.d.ts.map +1 -1
- package/dist/core/markdown.js +31 -22
- package/dist/core/markdown.js.map +1 -1
- package/dist/core/strategies.d.ts +6 -3
- package/dist/core/strategies.d.ts.map +1 -1
- package/dist/core/strategies.js +33 -7
- package/dist/core/strategies.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +163 -7
- package/dist/mcp/server.js.map +1 -1
- package/dist/types.d.ts +4 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/llms.txt +1 -1
- package/package.json +6 -3
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Web crawler functionality
|
|
3
|
+
* Crawls a starting URL and follows links matching specified patterns
|
|
4
|
+
*/
|
|
5
|
+
import { peel } from '../index.js';
|
|
6
|
+
import { fetch as undiciFetch } from 'undici';
|
|
7
|
+
/**
|
|
8
|
+
* Parse robots.txt and return disallowed paths for User-agent: *
|
|
9
|
+
*/
|
|
10
|
+
async function fetchRobotsTxt(domain) {
|
|
11
|
+
const robotsUrl = `https://${domain}/robots.txt`;
|
|
12
|
+
try {
|
|
13
|
+
const response = await undiciFetch(robotsUrl, {
|
|
14
|
+
headers: {
|
|
15
|
+
'User-Agent': 'WebPeel/0.3.0 (+https://webpeel.dev)',
|
|
16
|
+
},
|
|
17
|
+
signal: AbortSignal.timeout(5000), // 5 second timeout
|
|
18
|
+
});
|
|
19
|
+
if (!response.ok) {
|
|
20
|
+
// If robots.txt doesn't exist, allow everything
|
|
21
|
+
return { disallowedPaths: [] };
|
|
22
|
+
}
|
|
23
|
+
const text = await response.text();
|
|
24
|
+
const lines = text.split('\n');
|
|
25
|
+
const disallowedPaths = [];
|
|
26
|
+
let crawlDelay;
|
|
27
|
+
let relevantSection = false;
|
|
28
|
+
for (const line of lines) {
|
|
29
|
+
const trimmed = line.trim();
|
|
30
|
+
// Check for User-agent: *
|
|
31
|
+
if (trimmed.toLowerCase().startsWith('user-agent:')) {
|
|
32
|
+
const agent = trimmed.substring('user-agent:'.length).trim();
|
|
33
|
+
relevantSection = agent === '*';
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
if (!relevantSection)
|
|
37
|
+
continue;
|
|
38
|
+
// Parse Disallow directives
|
|
39
|
+
if (trimmed.toLowerCase().startsWith('disallow:')) {
|
|
40
|
+
const path = trimmed.substring('disallow:'.length).trim();
|
|
41
|
+
if (path) {
|
|
42
|
+
disallowedPaths.push(path);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
// Parse Crawl-delay directive
|
|
46
|
+
if (trimmed.toLowerCase().startsWith('crawl-delay:')) {
|
|
47
|
+
const delay = parseInt(trimmed.substring('crawl-delay:'.length).trim());
|
|
48
|
+
if (!isNaN(delay)) {
|
|
49
|
+
crawlDelay = delay * 1000; // Convert to milliseconds
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return { disallowedPaths, crawlDelay };
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
// If we can't fetch robots.txt, allow everything
|
|
57
|
+
return { disallowedPaths: [] };
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Check if a URL is allowed by robots.txt rules
|
|
62
|
+
*/
|
|
63
|
+
function isAllowedByRobots(url, rules) {
|
|
64
|
+
const urlObj = new URL(url);
|
|
65
|
+
const path = urlObj.pathname;
|
|
66
|
+
for (const disallowed of rules.disallowedPaths) {
|
|
67
|
+
// Simple prefix matching (proper robots.txt parsing would handle wildcards)
|
|
68
|
+
if (path.startsWith(disallowed)) {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return true;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Crawl a website starting from a URL
|
|
76
|
+
*
|
|
77
|
+
* @param startUrl - Starting URL to crawl from
|
|
78
|
+
* @param options - Crawl options
|
|
79
|
+
* @returns Array of crawl results
|
|
80
|
+
*
|
|
81
|
+
* @example
|
|
82
|
+
* ```typescript
|
|
83
|
+
* import { crawl } from 'webpeel';
|
|
84
|
+
*
|
|
85
|
+
* const results = await crawl('https://example.com', {
|
|
86
|
+
* maxPages: 20,
|
|
87
|
+
* maxDepth: 2,
|
|
88
|
+
* });
|
|
89
|
+
*
|
|
90
|
+
* console.log(`Crawled ${results.length} pages`);
|
|
91
|
+
* ```
|
|
92
|
+
*/
|
|
93
|
+
export async function crawl(startUrl, options = {}) {
|
|
94
|
+
const { maxPages = 10, maxDepth = 2, allowedDomains, excludePatterns = [], respectRobotsTxt = true, rateLimitMs = 1000, ...peelOptions } = options;
|
|
95
|
+
// Validate limits
|
|
96
|
+
const validatedMaxPages = Math.min(Math.max(maxPages, 1), 100);
|
|
97
|
+
const validatedMaxDepth = Math.min(Math.max(maxDepth, 1), 5);
|
|
98
|
+
const validatedRateLimit = Math.max(rateLimitMs, 100); // Min 100ms between requests
|
|
99
|
+
// Parse starting URL
|
|
100
|
+
const startUrlObj = new URL(startUrl);
|
|
101
|
+
const startDomain = startUrlObj.hostname;
|
|
102
|
+
// Default: only crawl same domain as starting URL
|
|
103
|
+
const validatedAllowedDomains = allowedDomains && allowedDomains.length > 0
|
|
104
|
+
? allowedDomains
|
|
105
|
+
: [startDomain];
|
|
106
|
+
// Compile exclude patterns
|
|
107
|
+
const excludeRegexes = excludePatterns.map(pattern => new RegExp(pattern));
|
|
108
|
+
// Fetch robots.txt if needed
|
|
109
|
+
let robotsRules = { disallowedPaths: [] };
|
|
110
|
+
if (respectRobotsTxt) {
|
|
111
|
+
robotsRules = await fetchRobotsTxt(startDomain);
|
|
112
|
+
// Use crawl-delay from robots.txt if it's larger than our rate limit
|
|
113
|
+
if (robotsRules.crawlDelay && robotsRules.crawlDelay > validatedRateLimit) {
|
|
114
|
+
console.error(`[Crawler] Using Crawl-delay from robots.txt: ${robotsRules.crawlDelay}ms`);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
const effectiveRateLimit = robotsRules.crawlDelay || validatedRateLimit;
|
|
118
|
+
// State tracking
|
|
119
|
+
const results = [];
|
|
120
|
+
const visited = new Set();
|
|
121
|
+
const queue = [
|
|
122
|
+
{ url: startUrl, depth: 0, parent: null },
|
|
123
|
+
];
|
|
124
|
+
while (queue.length > 0 && results.length < validatedMaxPages) {
|
|
125
|
+
const item = queue.shift();
|
|
126
|
+
const { url, depth, parent } = item;
|
|
127
|
+
// Skip if already visited
|
|
128
|
+
if (visited.has(url))
|
|
129
|
+
continue;
|
|
130
|
+
visited.add(url);
|
|
131
|
+
// Skip if depth exceeded
|
|
132
|
+
if (depth > validatedMaxDepth)
|
|
133
|
+
continue;
|
|
134
|
+
// Validate URL
|
|
135
|
+
let urlObj;
|
|
136
|
+
try {
|
|
137
|
+
urlObj = new URL(url);
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
continue; // Skip invalid URLs
|
|
141
|
+
}
|
|
142
|
+
// Check if domain is allowed
|
|
143
|
+
if (!validatedAllowedDomains.includes(urlObj.hostname)) {
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
// Check exclude patterns
|
|
147
|
+
if (excludeRegexes.some(regex => regex.test(url))) {
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
// Check robots.txt
|
|
151
|
+
if (respectRobotsTxt && !isAllowedByRobots(url, robotsRules)) {
|
|
152
|
+
console.error(`[Crawler] Skipping ${url} (disallowed by robots.txt)`);
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
// Fetch the page
|
|
156
|
+
try {
|
|
157
|
+
const result = await peel(url, {
|
|
158
|
+
...peelOptions,
|
|
159
|
+
format: 'markdown',
|
|
160
|
+
});
|
|
161
|
+
results.push({
|
|
162
|
+
url: result.url,
|
|
163
|
+
title: result.title,
|
|
164
|
+
markdown: result.content,
|
|
165
|
+
links: result.links,
|
|
166
|
+
depth,
|
|
167
|
+
parent,
|
|
168
|
+
elapsed: result.elapsed,
|
|
169
|
+
});
|
|
170
|
+
// Add discovered links to queue
|
|
171
|
+
if (depth < validatedMaxDepth) {
|
|
172
|
+
for (const link of result.links) {
|
|
173
|
+
if (!visited.has(link)) {
|
|
174
|
+
queue.push({
|
|
175
|
+
url: link,
|
|
176
|
+
depth: depth + 1,
|
|
177
|
+
parent: url,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
// Rate limiting
|
|
183
|
+
if (results.length < validatedMaxPages) {
|
|
184
|
+
await new Promise(resolve => setTimeout(resolve, effectiveRateLimit));
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
// Log error and continue
|
|
189
|
+
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
|
190
|
+
console.error(`[Crawler] Failed to fetch ${url}: ${errorMessage}`);
|
|
191
|
+
results.push({
|
|
192
|
+
url,
|
|
193
|
+
title: '',
|
|
194
|
+
markdown: '',
|
|
195
|
+
links: [],
|
|
196
|
+
depth,
|
|
197
|
+
parent,
|
|
198
|
+
elapsed: 0,
|
|
199
|
+
error: errorMessage,
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return results;
|
|
204
|
+
}
|
|
205
|
+
//# sourceMappingURL=crawler.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/core/crawler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;AAyC9C;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,MAAc;IAC1C,MAAM,SAAS,GAAG,WAAW,MAAM,aAAa,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE;YAC5C,OAAO,EAAE;gBACP,YAAY,EAAE,sCAAsC;aACrD;YACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,mBAAmB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,gDAAgD;YAChD,OAAO,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE/B,MAAM,eAAe,GAAa,EAAE,CAAC;QACrC,IAAI,UAA8B,CAAC;QACnC,IAAI,eAAe,GAAG,KAAK,CAAC;QAE5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAE5B,0BAA0B;YAC1B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC7D,eAAe,GAAG,KAAK,KAAK,GAAG,CAAC;gBAChC,SAAS;YACX,CAAC;YAED,IAAI,CAAC,eAAe;gBAAE,SAAS;YAE/B,4BAA4B;YAC5B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC1D,IAAI,IAAI,EAAE,CAAC;oBACT,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YAED,8BAA8B;YAC9B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;gBACrD,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACxE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBAClB,UAAU,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC,0BAA0B;gBACvD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,iDAAiD;QACjD,OAAO,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,GAAW,EAAE,KAAkB;IACxD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;IAE7B,KAAK,MAAM,UAAU,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;QAC/C,4EAA4E;QAC5E,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAChC,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,QAAgB,EAChB,UAAwB,EAAE;IAE1B,MAAM,EACJ,QAAQ,GAAG,EAAE,EACb,QAAQ,GAAG,CAAC,EACZ,cAAc,EACd,eAAe,GAAG,EAAE,EACpB,gBAAgB,GAAG,IAAI,EACvB,WAAW,GAAG,IAAI,EAClB,GAAG,WAAW,EACf,GAAG,OAAO,CAAC;IAEZ,kBAAkB;IAClB,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC/D,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7D,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,6BAA6B;IAEpF,qBAAqB;IACrB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC;IAEzC,kDAAkD;IAClD,MAAM,uBAAuB,GAAG,cAAc,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC;QACzE,CAAC,CAAC,cAAc;QAChB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAElB,2BAA2B;IAC3B,MAAM,cAAc,GAAG,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IAE3E,6BAA6B;IAC7B,IAAI,WAAW,GAAgB,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;IACvD,IAAI,gBAAgB,EAAE,CAAC;QACrB,WAAW,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC;QAEhD,qEAAqE;QACrE,IAAI,WAAW,CAAC,UAAU,IAAI,WAAW,CAAC,UAAU,GAAG,kBAAkB,EAAE,CAAC;YAC1E,OAAO,CAAC,KAAK,CAAC,gDAAgD,WAAW,CAAC,UAAU,IAAI,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;IAED,MAAM,kBAAkB,GAAG,WAAW,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAExE,iBAAiB;IACjB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAAiE;QAC1E,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;KAC1C,CAAC;IAEF,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;QAC9D,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;QAC5B,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAEpC,0BAA0B;QAC1B,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjB,yBAAyB;QACzB,IAAI,KAAK,GAAG,iBAAiB;YAAE,SAAS;QAExC,eAAe;QACf,IAAI,MAAW,CAAC;QAChB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS,CAAC,oBAAoB;QAChC,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvD,SAAS;QACX,CAAC;QAED,yBAAyB;QACzB,IAAI,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAClD,SAAS;QACX,CAAC;QAED,mBAAmB;QACnB,IAAI,gBAAgB,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,CAAC;YAC7D,OAAO,CAAC,KAAK,CAAC,sBAAsB,GAAG,6BAA6B,CAAC,CAAC;YACtE,SAAS;QACX,CAAC;QAED,iBAAiB;QACjB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE;gBAC7B,GAAG,WAAW;gBACd,MAAM,EAAE,UAAU;aACnB,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,QAAQ,EAAE,MAAM,CAAC,OAAO;gBACxB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK;gBACL,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAC,CAAC;YAEH,gCAAgC;YAChC,IAAI,KAAK,GAAG,iBAAiB,EAAE,CAAC;gBAC9B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;oBAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;wBACvB,KAAK,CAAC,IAAI,CAAC;4BACT,GAAG,EAAE,IAAI;4BACT,KAAK,EAAE,KAAK,GAAG,CAAC;4BAChB,MAAM,EAAE,GAAG;yBACZ,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,IAAI,OAAO,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;gBACvC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,yBAAyB;YACzB,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC9E,OAAO,CAAC,KAAK,CAAC,6BAA6B,GAAG,KAAK,YAAY,EAAE,CAAC,CAAC;YAEnE,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG;gBACH,KAAK,EAAE,EAAE;gBACT,QAAQ,EAAE,EAAE;gBACZ,KAAK,EAAE,EAAE;gBACT,KAAK;gBACL,MAAM;gBACN,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,YAAY;aACpB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/core/fetcher.d.ts
CHANGED
|
@@ -26,6 +26,7 @@ export declare function browserFetch(url: string, options?: {
|
|
|
26
26
|
screenshotFullPage?: boolean;
|
|
27
27
|
headers?: Record<string, string>;
|
|
28
28
|
cookies?: string[];
|
|
29
|
+
stealth?: boolean;
|
|
29
30
|
}): Promise<FetchResult>;
|
|
30
31
|
/**
|
|
31
32
|
* Retry a fetch operation with exponential backoff
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AAgQH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,MAAM,EACX,SAAS,CAAC,EAAE,MAAM,EAClB,SAAS,GAAE,MAAc,EACzB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACrC,OAAO,CAAC,WAAW,CAAC,CA2JtB;AAyCD;;;GAGG;AACH,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,GACL,OAAO,CAAC,WAAW,CAAC,CAkKtB;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,CAAC,EAChC,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,WAAW,GAAE,MAAU,EACvB,WAAW,GAAE,MAAa,GACzB,OAAO,CAAC,CAAC,CAAC,CAsBZ;AAED;;GAEG;AACH,wBAAsB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAS7C"}
|
package/dist/core/fetcher.js
CHANGED
|
@@ -2,7 +2,11 @@
|
|
|
2
2
|
* Core fetching logic: simple HTTP and browser-based fetching
|
|
3
3
|
*/
|
|
4
4
|
import { chromium } from 'playwright';
|
|
5
|
+
import { chromium as stealthChromium } from 'playwright-extra';
|
|
6
|
+
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
|
|
5
7
|
import { TimeoutError, BlockedError, NetworkError, WebPeelError } from '../types.js';
|
|
8
|
+
// Add stealth plugin to playwright-extra
|
|
9
|
+
stealthChromium.use(StealthPlugin());
|
|
6
10
|
const USER_AGENTS = [
|
|
7
11
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
|
8
12
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
|
@@ -354,6 +358,7 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
|
|
|
354
358
|
throw new WebPeelError(`Too many redirects (max ${MAX_REDIRECTS})`);
|
|
355
359
|
}
|
|
356
360
|
let sharedBrowser = null;
|
|
361
|
+
let sharedStealthBrowser = null;
|
|
357
362
|
let activePagesCount = 0;
|
|
358
363
|
const MAX_CONCURRENT_PAGES = 5;
|
|
359
364
|
async function getBrowser() {
|
|
@@ -372,6 +377,22 @@ async function getBrowser() {
|
|
|
372
377
|
sharedBrowser = await chromium.launch({ headless: true });
|
|
373
378
|
return sharedBrowser;
|
|
374
379
|
}
|
|
380
|
+
async function getStealthBrowser() {
|
|
381
|
+
// SECURITY: Check if stealth browser is still connected and healthy
|
|
382
|
+
if (sharedStealthBrowser) {
|
|
383
|
+
try {
|
|
384
|
+
if (sharedStealthBrowser.isConnected()) {
|
|
385
|
+
return sharedStealthBrowser;
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
catch {
|
|
389
|
+
// Browser is dead, recreate
|
|
390
|
+
sharedStealthBrowser = null;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
sharedStealthBrowser = await stealthChromium.launch({ headless: true });
|
|
394
|
+
return sharedStealthBrowser;
|
|
395
|
+
}
|
|
375
396
|
/**
|
|
376
397
|
* Fetch using headless Chromium via Playwright
|
|
377
398
|
* Slower but can handle JavaScript-heavy sites and bypass some bot detection
|
|
@@ -379,7 +400,7 @@ async function getBrowser() {
|
|
|
379
400
|
export async function browserFetch(url, options = {}) {
|
|
380
401
|
// SECURITY: Validate URL to prevent SSRF
|
|
381
402
|
validateUrl(url);
|
|
382
|
-
const { userAgent, waitMs = 0, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies } = options;
|
|
403
|
+
const { userAgent, waitMs = 0, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, stealth = false } = options;
|
|
383
404
|
// Validate user agent if provided
|
|
384
405
|
const validatedUserAgent = userAgent ? validateUserAgent(userAgent) : getRandomUserAgent();
|
|
385
406
|
// Validate wait time
|
|
@@ -410,7 +431,7 @@ export async function browserFetch(url, options = {}) {
|
|
|
410
431
|
activePagesCount++;
|
|
411
432
|
let page = null;
|
|
412
433
|
try {
|
|
413
|
-
const browser = await getBrowser();
|
|
434
|
+
const browser = stealth ? await getStealthBrowser() : await getBrowser();
|
|
414
435
|
page = await browser.newPage({
|
|
415
436
|
userAgent: validatedUserAgent,
|
|
416
437
|
});
|
|
@@ -537,5 +558,9 @@ export async function cleanup() {
|
|
|
537
558
|
await sharedBrowser.close();
|
|
538
559
|
sharedBrowser = null;
|
|
539
560
|
}
|
|
561
|
+
if (sharedStealthBrowser) {
|
|
562
|
+
await sharedStealthBrowser.close();
|
|
563
|
+
sharedStealthBrowser = null;
|
|
564
|
+
}
|
|
540
565
|
}
|
|
541
566
|
//# sourceMappingURL=fetcher.js.map
|
package/dist/core/fetcher.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAA2B,MAAM,YAAY,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAErF,MAAM,WAAW,GAAG;IAClB,uHAAuH;IACvH,iHAAiH;IACjH,uHAAuH;IACvH,kFAAkF;IAClF,uGAAuG;CACxG,CAAC;AAEF,SAAS,kBAAkB;IACzB,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,SAAiB;IACpC,eAAe;IACf,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,uDAAuD;IACvD,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAED,qBAAqB;IACrB,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;IACtE,CAAC;IAED,iCAAiC;IACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,YAAY,CAAC,kBAAkB,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,2BAA2B;IAC3B,MAAM,iBAAiB,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACnD,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,iFAAiF;IACjF,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,QAAQ,EAAE,CAAC;QACb,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,0CAA0C;IAC1C,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,6BAA6B;IAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAEjD,wCAAwC;IACxC,MAAM,WAAW,GAAG,8CAA8C,CAAC;IACnE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1C,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,CAAC;IACjD,CAAC;IAED,2BAA2B;IAC3B,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO;YACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;YAClB,GAAG,GAAG,IAAI;SACX,CAAC;IACJ,CAAC;IAED,wDAAwD;IACxD,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,0BAA0B;QAC1B,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;gBACtB,OAAO;oBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;YACJ,CAAC;QACH,CAAC;QACD,kCAAkC;QAClC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;YACtB,OAAO;gBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;gBAClB,GAAG,GAAG,IAAI;aACX,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAgB;IAC3C,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;IAE5B,wBAAwB;IACxB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,sBAAsB;IACtB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,4CAA4C,CAAC,CAAC;IACvE,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,MAAM,IAAI,YAAY,CAAC,mDAAmD,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,kBAAkB;IAClB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAE5D,gBAAgB;IAChB,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;QACjD,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,2DAA2D;IAC3D,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,wBAAwB;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAEnC,iEAAiE;QACjE,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,oBAAoB;YACpB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC/C,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBACvD,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACjC,MAAM,MAAM,GAAG;oBACb,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;gBACF,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,qDAAqD,CAAC,CAAC;IAChF,CAAC;IAED,sDAAsD;IACtD,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,YAAY,CAAC,sDAAsD,CAAC,CAAC;IACjF,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,0CAA0C,CAAC,CAAC;IACrE,CAAC;IACD,wCAAwC;IACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAUD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAW,EACX,SAAkB,EAClB,YAAoB,KAAK,EACzB,aAAsC;IAEtC,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,2EAA2E;IAC3E,MAAM,cAAc,GAA2B;QAC7C,YAAY,EAAE,kBAAkB;QAChC,QAAQ,EAAE,4EAA4E;QACtF,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,KAAK,EAAE,GAAG;QACV,YAAY,EAAE,YAAY;QAC1B,2BAA2B,EAAE,GAAG;KACjC,CAAC;IAEF,MAAM,aAAa,GAAG,EAAE,GAAG,cAAc,EAAE,CAAC;IAE5C,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;YACzD,uCAAuC;YACvC,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBACjC,MAAM,IAAI,YAAY,CAAC,mCAAmC,CAAC,CAAC;YAC9D,CAAC;YACD,aAAa,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,GAAG,CAAC;IACrB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,OAAO,aAAa,IAAI,aAAa,EAAE,CAAC;QACtC,wBAAwB;QACxB,IAAI,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,YAAY,CAAC,wBAAwB,CAAC,CAAC;QACnD,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEzB,+BAA+B;QAC/B,WAAW,CAAC,UAAU,CAAC,CAAC;QAExB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;QAE9D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;gBACvC,OAAO,EAAE,aAAa;gBACtB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ,EAAE,qCAAqC;aAC1D,CAAC,CAAC;YAEH,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,4BAA4B;YAC5B,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;gBACtE,CAAC;gBAED,wBAAwB;gBACxB,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC;gBAChD,aAAa,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACvD,MAAM,IAAI,YAAY,CACpB,QAAQ,QAAQ,CAAC,MAAM,iEAAiE,CACzF,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,YAAY,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC5E,CAAC;YAED,kCAAkC;YAClC,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAClC,CAAC,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC;gBAC9C,CAAC,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,YAAY,CAAC,6BAA6B,WAAW,wBAAwB,CAAC,CAAC;YAC3F,CAAC;YAED,wEAAwE;YACxE,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;YAE1C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,IAAI,EAAE,CAAC;oBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,MAAM;oBAEhB,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;oBAC1B,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;wBACzB,MAAM,CAAC,MAAM,EAAE,CAAC;wBAChB,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;oBAC1D,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,CAAC;YAED,iBAAiB;YACjB,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEhD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC/B,MAAM,IAAI,YAAY,CAAC,oEAAoE,CAAC,CAAC;YAC/F,CAAC;YAED,iCAAiC;YACjC,IAAI,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,YAAY,CAAC,+DAA+D,CAAC,CAAC;YAC1F,CAAC;YAED,OAAO;gBACL,IAAI;gBACJ,GAAG,EAAE,UAAU;gBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;gBAC3B,WAAW;aACZ,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACpG,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,YAAY,CAAC,2BAA2B,SAAS,IAAI,CAAC,CAAC;YACnE,CAAC;YAED,MAAM,IAAI,YAAY,CAAC,oBAAoB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;IAED,MAAM,IAAI,YAAY,CAAC,2BAA2B,aAAa,GAAG,CAAC,CAAC;AACtE,CAAC;AAED,IAAI,aAAa,GAAmB,IAAI,CAAC;AACzC,IAAI,gBAAgB,GAAG,CAAC,CAAC;AACzB,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,KAAK,UAAU,UAAU;IACvB,4DAA4D;IAC5D,IAAI,aAAa,EAAE,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC;gBAChC,OAAO,aAAa,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;YAC5B,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,aAAa,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,UAQI,EAAE;IAEN,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,MAAM,EACJ,SAAS,EACT,MAAM,GAAG,CAAC,EACV,SAAS,GAAG,KAAK,EACjB,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,OAAO,EACP,OAAO,EACR,GAAG,OAAO,CAAC;IAEZ,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,qBAAqB;IACrB,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;QACjC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,gDAAgD;IAChD,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,6BAA6B;YAC7B,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBACjC,MAAM,IAAI,YAAY,CAAC,mCAAmC,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,gBAAgB,GAAG,KAAK,CAAC,CAAC,qBAAqB;IAErD,OAAO,gBAAgB,IAAI,oBAAoB,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,GAAG,gBAAgB,EAAE,CAAC;YACnD,MAAM,IAAI,YAAY,CAAC,2DAA2D,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,gBAAgB,EAAE,CAAC;IACnB,IAAI,IAAI,GAAgB,IAAI,CAAC;IAE7B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;QACnC,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC;YAC3B,SAAS,EAAE,kBAAkB;SAC9B,CAAC,CAAC;QAEH,iCAAiC;QACjC,IAAI,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,0BAA0B;QAC1B,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;gBACzC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACzD,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE3C,IAAI,CAAC,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACjC,MAAM,IAAI,YAAY,CAAC,0BAA0B,MAAM,EAAE,CAAC,CAAC;gBAC7D,CAAC;gBAED,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;oBACjB,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;oBACnB,GAAG;iBACJ,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACjD,CAAC;QAED,4FAA4F;QAC5F,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;gBACjC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;gBACpD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACpE,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,uCAAuC;YACvC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,6CAA6C;QAC7C,MAAM,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YAC/B,MAAM,IAAK,CAAC,IAAI,CAAC,GAAG,EAAE;gBACpB,SAAS,EAAE,kBAAkB;gBAC7B,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,MAAM,IAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAK,CAAC,GAAG,EAAE,CAAC;YAE7B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,6BAA6B,SAAS,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;QAEH,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;QAE9E,4BAA4B;QAC5B,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,aAAa;YACjD,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC/B,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;QAC/E,CAAC;QAED,kCAAkC;QAClC,IAAI,gBAAoC,CAAC;QACzC,IAAI,UAAU,EAAE,CAAC;YACf,gBAAgB,GAAG,MAAM,IAAK,CAAC,UAAU,CAAC;gBACxC,QAAQ,EAAE,kBAAkB;gBAC5B,IAAI,EAAE,KAAK;aACZ,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,IAAI;YACJ,GAAG,EAAE,QAAQ;YACb,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;YACpG,MAAM,KAAK,CAAC;QACd,CAAC;QAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,YAAY,CAAC,8BAA8B,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,IAAI,YAAY,CACpB,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CACpF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,oDAAoD;QACpD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrC,CAAC;QACD,gBAAgB,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,EAAoB,EACpB,cAAsB,CAAC,EACvB,cAAsB,IAAI;IAE1B,IAAI,SAAS,GAAiB,IAAI,CAAC;IAEnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;YAExE,4CAA4C;YAC5C,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACnE,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;gBACrD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,YAAY,CAAC,cAAc,CAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;QAC5B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAA2B,MAAM,YAAY,CAAC;AAC/D,OAAO,EAAE,QAAQ,IAAI,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,aAAa,MAAM,gCAAgC,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAErF,yCAAyC;AACzC,eAAe,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC;AAErC,MAAM,WAAW,GAAG;IAClB,uHAAuH;IACvH,iHAAiH;IACjH,uHAAuH;IACvH,kFAAkF;IAClF,uGAAuG;CACxG,CAAC;AAEF,SAAS,kBAAkB;IACzB,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,SAAiB;IACpC,eAAe;IACf,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,uDAAuD;IACvD,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAED,qBAAqB;IACrB,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;IACtE,CAAC;IAED,iCAAiC;IACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,YAAY,CAAC,kBAAkB,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,2BAA2B;IAC3B,MAAM,iBAAiB,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACnD,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,iFAAiF;IACjF,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,QAAQ,EAAE,CAAC;QACb,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,0CAA0C;IAC1C,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,6BAA6B;IAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAEjD,wCAAwC;IACxC,MAAM,WAAW,GAAG,8CAA8C,CAAC;IACnE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1C,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,CAAC;IACjD,CAAC;IAED,2BAA2B;IAC3B,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO;YACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;YAClB,GAAG,GAAG,IAAI;SACX,CAAC;IACJ,CAAC;IAED,wDAAwD;IACxD,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,0BAA0B;QAC1B,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;gBACtB,OAAO;oBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;YACJ,CAAC;QACH,CAAC;QACD,kCAAkC;QAClC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;YACtB,OAAO;gBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;gBAClB,GAAG,GAAG,IAAI;aACX,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAgB;IAC3C,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;IAE5B,wBAAwB;IACxB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,sBAAsB;IACtB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,4CAA4C,CAAC,CAAC;IACvE,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,MAAM,IAAI,YAAY,CAAC,mDAAmD,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,kBAAkB;IAClB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAE5D,gBAAgB;IAChB,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;QACjD,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,2DAA2D;IAC3D,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,wBAAwB;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAEnC,iEAAiE;QACjE,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,oBAAoB;YACpB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC/C,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBACvD,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACjC,MAAM,MAAM,GAAG;oBACb,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;gBACF,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,qDAAqD,CAAC,CAAC;IAChF,CAAC;IAED,sDAAsD;IACtD,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,YAAY,CAAC,sDAAsD,CAAC,CAAC;IACjF,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,0CAA0C,CAAC,CAAC;IACrE,CAAC;IACD,wCAAwC;IACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAUD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAW,EACX,SAAkB,EAClB,YAAoB,KAAK,EACzB,aAAsC;IAEtC,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,2EAA2E;IAC3E,MAAM,cAAc,GAA2B;QAC7C,YAAY,EAAE,kBAAkB;QAChC,QAAQ,EAAE,4EAA4E;QACtF,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,KAAK,EAAE,GAAG;QACV,YAAY,EAAE,YAAY;QAC1B,2BAA2B,EAAE,GAAG;KACjC,CAAC;IAEF,MAAM,aAAa,GAAG,EAAE,GAAG,cAAc,EAAE,CAAC;IAE5C,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;YACzD,uCAAuC;YACvC,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBACjC,MAAM,IAAI,YAAY,CAAC,mCAAmC,CAAC,CAAC;YAC9D,CAAC;YACD,aAAa,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,GAAG,CAAC;IACrB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,OAAO,aAAa,IAAI,aAAa,EAAE,CAAC;QACtC,wBAAwB;QACxB,IAAI,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,YAAY,CAAC,wBAAwB,CAAC,CAAC;QACnD,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEzB,+BAA+B;QAC/B,WAAW,CAAC,UAAU,CAAC,CAAC;QAExB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;QAE9D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;gBACvC,OAAO,EAAE,aAAa;gBACtB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ,EAAE,qCAAqC;aAC1D,CAAC,CAAC;YAEH,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,4BAA4B;YAC5B,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;gBACtE,CAAC;gBAED,wBAAwB;gBACxB,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC;gBAChD,aAAa,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACvD,MAAM,IAAI,YAAY,CACpB,QAAQ,QAAQ,CAAC,MAAM,iEAAiE,CACzF,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,YAAY,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC5E,CAAC;YAED,kCAAkC;YAClC,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAClC,CAAC,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC;gBAC9C,CAAC,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,YAAY,CAAC,6BAA6B,WAAW,wBAAwB,CAAC,CAAC;YAC3F,CAAC;YAED,wEAAwE;YACxE,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;YAE1C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,IAAI,EAAE,CAAC;oBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,MAAM;oBAEhB,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;oBAC1B,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;wBACzB,MAAM,CAAC,MAAM,EAAE,CAAC;wBAChB,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;oBAC1D,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,CAAC;YAED,iBAAiB;YACjB,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEhD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC/B,MAAM,IAAI,YAAY,CAAC,oEAAoE,CAAC,CAAC;YAC/F,CAAC;YAED,iCAAiC;YACjC,IAAI,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,YAAY,CAAC,+DAA+D,CAAC,CAAC;YAC1F,CAAC;YAED,OAAO;gBACL,IAAI;gBACJ,GAAG,EAAE,UAAU;gBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;gBAC3B,WAAW;aACZ,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACpG,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,YAAY,CAAC,2BAA2B,SAAS,IAAI,CAAC,CAAC;YACnE,CAAC;YAED,MAAM,IAAI,YAAY,CAAC,oBAAoB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;IAED,MAAM,IAAI,YAAY,CAAC,2BAA2B,aAAa,GAAG,CAAC,CAAC;AACtE,CAAC;AAED,IAAI,aAAa,GAAmB,IAAI,CAAC;AACzC,IAAI,oBAAoB,GAAmB,IAAI,CAAC;AAChD,IAAI,gBAAgB,GAAG,CAAC,CAAC;AACzB,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,KAAK,UAAU,UAAU;IACvB,4DAA4D;IAC5D,IAAI,aAAa,EAAE,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC;gBAChC,OAAO,aAAa,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;YAC5B,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,aAAa,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,KAAK,UAAU,iBAAiB;IAC9B,oEAAoE;IACpE,IAAI,oBAAoB,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,IAAI,oBAAoB,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvC,OAAO,oBAAoB,CAAC;YAC9B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;YAC5B,oBAAoB,GAAG,IAAI,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,oBAAoB,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,UASI,EAAE;IAEN,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,MAAM,EACJ,SAAS,EACT,MAAM,GAAG,CAAC,EACV,SAAS,GAAG,KAAK,EACjB,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,OAAO,EACP,OAAO,EACP,OAAO,GAAG,KAAK,EAChB,GAAG,OAAO,CAAC;IAEZ,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,qBAAqB;IACrB,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;QACjC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,gDAAgD;IAChD,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,6BAA6B;YAC7B,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBACjC,MAAM,IAAI,YAAY,CAAC,mCAAmC,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,gBAAgB,GAAG,KAAK,CAAC,CAAC,qBAAqB;IAErD,OAAO,gBAAgB,IAAI,oBAAoB,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,GAAG,gBAAgB,EAAE,CAAC;YACnD,MAAM,IAAI,YAAY,CAAC,2DAA2D,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,gBAAgB,EAAE,CAAC;IACnB,IAAI,IAAI,GAAgB,IAAI,CAAC;IAE7B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC,CAAC,MAAM,UAAU,EAAE,CAAC;QACzE,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC;YAC3B,SAAS,EAAE,kBAAkB;SAC9B,CAAC,CAAC;QAEH,iCAAiC;QACjC,IAAI,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,0BAA0B;QAC1B,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;gBACzC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACzD,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE3C,IAAI,CAAC,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACjC,MAAM,IAAI,YAAY,CAAC,0BAA0B,MAAM,EAAE,CAAC,CAAC;gBAC7D,CAAC;gBAED,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;oBACjB,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;oBACnB,GAAG;iBACJ,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACjD,CAAC;QAED,4FAA4F;QAC5F,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;gBACjC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;gBACpD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACpE,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,uCAAuC;YACvC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,6CAA6C;QAC7C,MAAM,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YAC/B,MAAM,IAAK,CAAC,IAAI,CAAC,GAAG,EAAE;gBACpB,SAAS,EAAE,kBAAkB;gBAC7B,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,MAAM,IAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAK,CAAC,GAAG,EAAE,CAAC;YAE7B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,6BAA6B,SAAS,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;QAEH,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;QAE9E,4BAA4B;QAC5B,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,aAAa;YACjD,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC/B,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;QAC/E,CAAC;QAED,kCAAkC;QAClC,IAAI,gBAAoC,CAAC;QACzC,IAAI,UAAU,EAAE,CAAC;YACf,gBAAgB,GAAG,MAAM,IAAK,CAAC,UAAU,CAAC;gBACxC,QAAQ,EAAE,kBAAkB;gBAC5B,IAAI,EAAE,KAAK;aACZ,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,IAAI;YACJ,GAAG,EAAE,QAAQ;YACb,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;YACpG,MAAM,KAAK,CAAC;QACd,CAAC;QAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,YAAY,CAAC,8BAA8B,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,IAAI,YAAY,CACpB,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CACpF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,oDAAoD;QACpD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrC,CAAC;QACD,gBAAgB,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,EAAoB,EACpB,cAAsB,CAAC,EACvB,cAAsB,IAAI;IAE1B,IAAI,SAAS,GAAiB,IAAI,CAAC;IAEnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;YAExE,4CAA4C;YAC5C,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACnE,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;gBACrD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,YAAY,CAAC,cAAc,CAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;QAC5B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;IACD,IAAI,oBAAoB,EAAE,CAAC;QACzB,MAAM,oBAAoB,CAAC,KAAK,EAAE,CAAC;QACnC,oBAAoB,GAAG,IAAI,CAAC;IAC9B,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AAuCH;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAiBxF;AAkCD;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CA4DnD;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAuB/C;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD"}
|
package/dist/core/markdown.js
CHANGED
|
@@ -4,28 +4,37 @@
|
|
|
4
4
|
import TurndownService from 'turndown';
|
|
5
5
|
import * as cheerio from 'cheerio';
|
|
6
6
|
const JUNK_SELECTORS = [
|
|
7
|
-
|
|
8
|
-
'style',
|
|
9
|
-
|
|
10
|
-
'footer',
|
|
11
|
-
'
|
|
12
|
-
'
|
|
13
|
-
'.sidebar',
|
|
14
|
-
'.
|
|
15
|
-
'.
|
|
16
|
-
|
|
17
|
-
'.
|
|
18
|
-
'
|
|
19
|
-
|
|
20
|
-
'.
|
|
21
|
-
'
|
|
22
|
-
'
|
|
23
|
-
|
|
24
|
-
'[class*="
|
|
25
|
-
'[
|
|
26
|
-
|
|
27
|
-
'[class*="
|
|
28
|
-
|
|
7
|
+
// Scripts, styles, metadata
|
|
8
|
+
'script', 'style', 'noscript', 'iframe', 'link[rel="stylesheet"]',
|
|
9
|
+
// Navigation & structure
|
|
10
|
+
'nav', 'header', 'footer', 'aside',
|
|
11
|
+
'[role="navigation"]', '[role="banner"]', '[role="contentinfo"]',
|
|
12
|
+
'[role="complementary"]', '[role="search"]',
|
|
13
|
+
'.sidebar', '.topbar', '.top-bar', '.site-nav', '.main-nav',
|
|
14
|
+
'.breadcrumb', '.breadcrumbs', '[class*="breadcrumb"]',
|
|
15
|
+
'.pagination', '[class*="pagination"]',
|
|
16
|
+
// Ads & tracking
|
|
17
|
+
'.advertisement', '.ad', '[class*="ad-"]', '[id*="ad-"]',
|
|
18
|
+
'[class*="advert"]', '[class*="sponsor"]', '[class*="promo"]',
|
|
19
|
+
// Cookie & consent
|
|
20
|
+
'.cookie-banner', '.cookie-notice', '.cookie-consent',
|
|
21
|
+
'[class*="cookie"]', '[id*="cookie"]',
|
|
22
|
+
'[class*="consent"]', '[class*="gdpr"]',
|
|
23
|
+
// Popups, modals, banners
|
|
24
|
+
'[class*="banner"]', '[class*="popup"]', '[class*="modal"]',
|
|
25
|
+
'[class*="overlay"]', '[class*="notification-bar"]',
|
|
26
|
+
// Social & sharing
|
|
27
|
+
'.social-share', '[class*="share"]', '[class*="social"]',
|
|
28
|
+
// Newsletter & CTA
|
|
29
|
+
'.newsletter-signup', '[class*="newsletter"]', '[class*="subscribe"]',
|
|
30
|
+
'[class*="cta"]', '[class*="call-to-action"]', '[class*="signup"]',
|
|
31
|
+
// Related content
|
|
32
|
+
'.related-posts', '[class*="related"]', '[class*="recommended"]',
|
|
33
|
+
'[class*="you-may-also"]', '[class*="more-stories"]',
|
|
34
|
+
// Comments
|
|
35
|
+
'.comments', '#comments', '[class*="comment"]',
|
|
36
|
+
// SVG decorations (icons, decorative elements)
|
|
37
|
+
'svg:not(img svg)',
|
|
29
38
|
];
|
|
30
39
|
/**
|
|
31
40
|
* Extract content matching a CSS selector
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,cAAc,GAAG;IACrB,
|
|
1
|
+
{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,cAAc,GAAG;IACrB,4BAA4B;IAC5B,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,wBAAwB;IACjE,yBAAyB;IACzB,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO;IAClC,qBAAqB,EAAE,iBAAiB,EAAE,sBAAsB;IAChE,wBAAwB,EAAE,iBAAiB;IAC3C,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW;IAC3D,aAAa,EAAE,cAAc,EAAE,uBAAuB;IACtD,aAAa,EAAE,uBAAuB;IACtC,iBAAiB;IACjB,gBAAgB,EAAE,KAAK,EAAE,gBAAgB,EAAE,aAAa;IACxD,mBAAmB,EAAE,oBAAoB,EAAE,kBAAkB;IAC7D,mBAAmB;IACnB,gBAAgB,EAAE,gBAAgB,EAAE,iBAAiB;IACrD,mBAAmB,EAAE,gBAAgB;IACrC,oBAAoB,EAAE,iBAAiB;IACvC,0BAA0B;IAC1B,mBAAmB,EAAE,kBAAkB,EAAE,kBAAkB;IAC3D,oBAAoB,EAAE,6BAA6B;IACnD,mBAAmB;IACnB,eAAe,EAAE,kBAAkB,EAAE,mBAAmB;IACxD,mBAAmB;IACnB,oBAAoB,EAAE,uBAAuB,EAAE,sBAAsB;IACrE,gBAAgB,EAAE,2BAA2B,EAAE,mBAAmB;IAClE,kBAAkB;IAClB,gBAAgB,EAAE,oBAAoB,EAAE,wBAAwB;IAChE,yBAAyB,EAAE,yBAAyB;IACpD,WAAW;IACX,WAAW,EAAE,WAAW,EAAE,oBAAoB;IAC9C,+CAA+C;IAC/C,kBAAkB;CACnB,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,QAAgB,EAAE,OAAkB;IAC9E,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,uBAAuB;IACvB,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,2BAA2B;IAC3B,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,oDAAoD;QACpD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0CAA0C;IAC1C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAAC,IAAY;IAC7B,2CAA2C;IAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,OAAO;QAC3C,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,uBAAuB;IACvB,cAAc,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAClC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,mCAAmC;IACnC,CAAC,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC;IAEjC,uCAAuC;IACvC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACtB,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACtB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAEpC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;QACnC,YAAY,EAAE,KAAK;QACnB,cAAc,EAAE,QAAQ;QACxB,gBAAgB,EAAE,GAAG;QACrB,WAAW,EAAE,GAAG;QAChB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IAEH,kBAAkB;IAClB,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAE7D,kDAAkD;IAClD,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE;QACzB,MAAM,EAAE,KAAK;QACb,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,GAAG,GAAI,IAAY,CAAC,GAAG,CAAC;YAC9B,MAAM,GAAG,GAAI,IAAY,CAAC,GAAG,CAAC;YAC9B,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;YAC7B,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC;KACF,CAAC,CAAC;IAEH,oCAAoC;IACpC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE;QAC7B,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACf,OAAO,IAAI,CAAC,QAAQ,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,QAAQ,KAAK,MAAM,CAAC;QACzE,CAAC;QACD,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAiB,CAAC;YACxC,MAAM,SAAS,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACvD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9D,OAAO,SAAS,GAAG,QAAQ,GAAG,IAAI,GAAG,QAAQ,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1E,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAE9C,kEAAkE;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,yBAAyB;QAC5D,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED,8DAA8D;IAC9D,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE;QAC3D,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;QACrC,IAAI,SAAS,IAAI,SAAS;YAAE,OAAO,GAAG,CAAC;QACvC,OAAO,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;IAC3B,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,qCAAqC;IACrC,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAE3B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEpC,8CAA8C;IAC9C,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,CAAC,CAAC,+BAA+B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,IAAI,OAAO,GAAG,MAAM,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,yDAAyD;IACzD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACjB,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,gCAAgC;IAChC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAEpC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC"}
|
|
@@ -5,6 +5,8 @@ import { type FetchResult } from './fetcher.js';
|
|
|
5
5
|
export interface StrategyOptions {
|
|
6
6
|
/** Force browser mode (skip simple fetch) */
|
|
7
7
|
forceBrowser?: boolean;
|
|
8
|
+
/** Use stealth mode to bypass bot detection */
|
|
9
|
+
stealth?: boolean;
|
|
8
10
|
/** Wait time after page load in browser mode (ms) */
|
|
9
11
|
waitMs?: number;
|
|
10
12
|
/** Custom user agent */
|
|
@@ -21,8 +23,8 @@ export interface StrategyOptions {
|
|
|
21
23
|
cookies?: string[];
|
|
22
24
|
}
|
|
23
25
|
export interface StrategyResult extends FetchResult {
|
|
24
|
-
/** Which strategy succeeded: 'simple' | 'browser' */
|
|
25
|
-
method: 'simple' | 'browser';
|
|
26
|
+
/** Which strategy succeeded: 'simple' | 'browser' | 'stealth' */
|
|
27
|
+
method: 'simple' | 'browser' | 'stealth';
|
|
26
28
|
}
|
|
27
29
|
/**
|
|
28
30
|
* Smart fetch with automatic escalation
|
|
@@ -30,7 +32,8 @@ export interface StrategyResult extends FetchResult {
|
|
|
30
32
|
* Strategy:
|
|
31
33
|
* 1. Try simple HTTP fetch first (fast, ~200ms)
|
|
32
34
|
* 2. If blocked (403, 503, Cloudflare, empty body) → try browser
|
|
33
|
-
* 3. If browser
|
|
35
|
+
* 3. If browser gets blocked (403, CAPTCHA) → try stealth mode
|
|
36
|
+
* 4. If stealth mode is explicitly requested → skip to stealth
|
|
34
37
|
*
|
|
35
38
|
* Returns the result along with which method worked
|
|
36
39
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAyC,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AAGvF,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,cAAe,SAAQ,WAAW;IACjD,
|
|
1
|
+
{"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAyC,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AAGvF,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,cAAe,SAAQ,WAAW;IACjD,iEAAiE;IACjE,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;CAC1C;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,cAAc,CAAC,CAqGpG"}
|
package/dist/core/strategies.js
CHANGED
|
@@ -9,14 +9,15 @@ import { BlockedError, NetworkError } from '../types.js';
|
|
|
9
9
|
* Strategy:
|
|
10
10
|
* 1. Try simple HTTP fetch first (fast, ~200ms)
|
|
11
11
|
* 2. If blocked (403, 503, Cloudflare, empty body) → try browser
|
|
12
|
-
* 3. If browser
|
|
12
|
+
* 3. If browser gets blocked (403, CAPTCHA) → try stealth mode
|
|
13
|
+
* 4. If stealth mode is explicitly requested → skip to stealth
|
|
13
14
|
*
|
|
14
15
|
* Returns the result along with which method worked
|
|
15
16
|
*/
|
|
16
17
|
export async function smartFetch(url, options = {}) {
|
|
17
|
-
const { forceBrowser = false, waitMs = 0, userAgent, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies } = options;
|
|
18
|
-
// If
|
|
19
|
-
const shouldUseBrowser = forceBrowser || screenshot;
|
|
18
|
+
const { forceBrowser = false, stealth = false, waitMs = 0, userAgent, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies } = options;
|
|
19
|
+
// If stealth is requested, force browser mode (stealth requires browser)
|
|
20
|
+
const shouldUseBrowser = forceBrowser || screenshot || stealth;
|
|
20
21
|
// Strategy 1: Simple fetch (unless browser is forced or screenshot is requested)
|
|
21
22
|
if (!shouldUseBrowser) {
|
|
22
23
|
try {
|
|
@@ -37,7 +38,7 @@ export async function smartFetch(url, options = {}) {
|
|
|
37
38
|
}
|
|
38
39
|
}
|
|
39
40
|
}
|
|
40
|
-
// Strategy 2: Browser fetch
|
|
41
|
+
// Strategy 2: Browser fetch (with or without stealth)
|
|
41
42
|
try {
|
|
42
43
|
const result = await browserFetch(url, {
|
|
43
44
|
userAgent,
|
|
@@ -47,13 +48,37 @@ export async function smartFetch(url, options = {}) {
|
|
|
47
48
|
screenshotFullPage,
|
|
48
49
|
headers,
|
|
49
50
|
cookies,
|
|
51
|
+
stealth,
|
|
50
52
|
});
|
|
51
53
|
return {
|
|
52
54
|
...result,
|
|
53
|
-
method: 'browser',
|
|
55
|
+
method: stealth ? 'stealth' : 'browser',
|
|
54
56
|
};
|
|
55
57
|
}
|
|
56
58
|
catch (error) {
|
|
59
|
+
// Strategy 3: If browser gets blocked, try stealth mode as fallback (unless already using stealth)
|
|
60
|
+
if (!stealth && error instanceof BlockedError) {
|
|
61
|
+
try {
|
|
62
|
+
const result = await browserFetch(url, {
|
|
63
|
+
userAgent,
|
|
64
|
+
waitMs,
|
|
65
|
+
timeoutMs,
|
|
66
|
+
screenshot,
|
|
67
|
+
screenshotFullPage,
|
|
68
|
+
headers,
|
|
69
|
+
cookies,
|
|
70
|
+
stealth: true, // Escalate to stealth mode
|
|
71
|
+
});
|
|
72
|
+
return {
|
|
73
|
+
...result,
|
|
74
|
+
method: 'stealth',
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
catch (stealthError) {
|
|
78
|
+
// If stealth also fails, throw the original error
|
|
79
|
+
throw stealthError;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
57
82
|
// If browser encounters Cloudflare, retry with extra wait time
|
|
58
83
|
if (error instanceof NetworkError &&
|
|
59
84
|
error.message.toLowerCase().includes('cloudflare')) {
|
|
@@ -65,10 +90,11 @@ export async function smartFetch(url, options = {}) {
|
|
|
65
90
|
screenshotFullPage,
|
|
66
91
|
headers,
|
|
67
92
|
cookies,
|
|
93
|
+
stealth, // Keep stealth setting
|
|
68
94
|
});
|
|
69
95
|
return {
|
|
70
96
|
...result,
|
|
71
|
-
method: 'browser',
|
|
97
|
+
method: stealth ? 'stealth' : 'browser',
|
|
72
98
|
};
|
|
73
99
|
}
|
|
74
100
|
throw error;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"strategies.js","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAoB,MAAM,cAAc,CAAC;AACvF,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"strategies.js","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAoB,MAAM,cAAc,CAAC;AACvF,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA4BzD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW,EAAE,UAA2B,EAAE;IACzE,MAAM,EACJ,YAAY,GAAG,KAAK,EACpB,OAAO,GAAG,KAAK,EACf,MAAM,GAAG,CAAC,EACV,SAAS,EACT,SAAS,GAAG,KAAK,EACjB,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,OAAO,EACP,OAAO,EACR,GAAG,OAAO,CAAC;IAEZ,yEAAyE;IACzE,MAAM,gBAAgB,GAAG,YAAY,IAAI,UAAU,IAAI,OAAO,CAAC;IAE/D,iFAAiF;IACjF,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAC7B,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,EACrD,CAAC,CACF,CAAC;YACF,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,QAAQ;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,8CAA8C;YAC9C,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBAClC,mCAAmC;YACrC,CAAC;iBAAM,CAAC;gBACN,kDAAkD;gBAClD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,sDAAsD;IACtD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;YACrC,SAAS;YACT,MAAM;YACN,SAAS;YACT,UAAU;YACV,kBAAkB;YAClB,OAAO;YACP,OAAO;YACP,OAAO;SACR,CAAC,CAAC;QACH,OAAO;YACL,GAAG,MAAM;YACT,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;SACxC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mGAAmG;QACnG,IAAI,CAAC,OAAO,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;YAC9C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;oBACrC,SAAS;oBACT,MAAM;oBACN,SAAS;oBACT,UAAU;oBACV,kBAAkB;oBAClB,OAAO;oBACP,OAAO;oBACP,OAAO,EAAE,IAAI,EAAE,2BAA2B;iBAC3C,CAAC,CAAC;gBACH,OAAO;oBACL,GAAG,MAAM;oBACT,MAAM,EAAE,SAAS;iBAClB,CAAC;YACJ,CAAC;YAAC,OAAO,YAAY,EAAE,CAAC;gBACtB,kDAAkD;gBAClD,MAAM,YAAY,CAAC;YACrB,CAAC;QACH,CAAC;QAED,+DAA+D;QAC/D,IACE,KAAK,YAAY,YAAY;YAC7B,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,EAClD,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;gBACrC,SAAS;gBACT,MAAM,EAAE,IAAI,EAAE,mCAAmC;gBACjD,SAAS;gBACT,UAAU;gBACV,kBAAkB;gBAClB,OAAO;gBACP,OAAO;gBACP,OAAO,EAAE,uBAAuB;aACjC,CAAC,CAAC;YACH,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;aACxC,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { cleanup } from './core/fetcher.js';
|
|
7
7
|
import type { PeelOptions, PeelResult } from './types.js';
|
|
8
8
|
export * from './types.js';
|
|
9
|
+
export { crawl, type CrawlOptions, type CrawlResult } from './core/crawler.js';
|
|
9
10
|
/**
|
|
10
11
|
* Fetch and extract content from a URL
|
|
11
12
|
*
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1D,cAAc,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1D,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE/E;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAkGtF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,WAAW,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACnD,OAAO,CAAC,CAAC,UAAU,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EAAE,CAAC,CAwB1D;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
|