webpeel 0.3.6 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -4
- package/dist/cli-auth.d.ts +0 -16
- package/dist/cli-auth.d.ts.map +1 -1
- package/dist/cli-auth.js +0 -76
- package/dist/cli-auth.js.map +1 -1
- package/dist/cli.js +131 -4
- package/dist/cli.js.map +1 -1
- package/dist/core/actions.d.ts +15 -0
- package/dist/core/actions.d.ts.map +1 -0
- package/dist/core/actions.js +67 -0
- package/dist/core/actions.js.map +1 -0
- package/dist/core/crawler.d.ts +19 -0
- package/dist/core/crawler.d.ts.map +1 -1
- package/dist/core/crawler.js +68 -4
- package/dist/core/crawler.js.map +1 -1
- package/dist/core/extract.d.ts +6 -0
- package/dist/core/extract.d.ts.map +1 -0
- package/dist/core/extract.js +77 -0
- package/dist/core/extract.js.map +1 -0
- package/dist/core/fetcher.d.ts +9 -0
- package/dist/core/fetcher.d.ts.map +1 -1
- package/dist/core/fetcher.js +12 -4
- package/dist/core/fetcher.js.map +1 -1
- package/dist/core/map.d.ts +30 -0
- package/dist/core/map.d.ts.map +1 -0
- package/dist/core/map.js +61 -0
- package/dist/core/map.js.map +1 -0
- package/dist/core/markdown.d.ts +5 -0
- package/dist/core/markdown.d.ts.map +1 -1
- package/dist/core/markdown.js +40 -0
- package/dist/core/markdown.js.map +1 -1
- package/dist/core/pdf.d.ts +9 -0
- package/dist/core/pdf.d.ts.map +1 -0
- package/dist/core/pdf.js +24 -0
- package/dist/core/pdf.js.map +1 -0
- package/dist/core/sitemap.d.ts +24 -0
- package/dist/core/sitemap.d.ts.map +1 -0
- package/dist/core/sitemap.js +106 -0
- package/dist/core/sitemap.js.map +1 -0
- package/dist/core/strategies.d.ts +10 -0
- package/dist/core/strategies.d.ts.map +1 -1
- package/dist/core/strategies.js +4 -1
- package/dist/core/strategies.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +20 -2
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +221 -4
- package/dist/mcp/server.js.map +1 -1
- package/dist/types.d.ts +25 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +4 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"actions.js","sourceRoot":"","sources":["../../src/core/actions.ts"],"names":[],"mappings":"AAAA;;GAEG;AAcH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAU,EAAE,OAAqB;IACpE,IAAI,cAAkC,CAAC;IAEvC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;YACpB,KAAK,MAAM;gBACT,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;gBAC7C,MAAM;YACR,KAAK,OAAO;gBACV,IAAI,CAAC,MAAM,CAAC,QAAQ;oBAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;gBACxE,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,QAAQ;gBACX,IAAI,MAAM,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;oBAC3B,MAAM,IAAI,CAAC,QAAQ,CAAC,gDAAgD,CAAC,CAAC;gBACxE,CAAC;qBAAM,IAAI,MAAM,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;gBAC/C,CAAC;qBAAM,IAAI,OAAO,MAAM,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;oBACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,sBAAsB,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC1D,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,CAAC,QAAQ,CAAC,gDAAgD,CAAC,CAAC;gBACxE,CAAC;gBACD,MAAM;YACR,KAAK,MAAM;gBACT,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK;oBAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;gBAClG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC/C,MAAM;YACR,KAAK,MAAM;gBACT,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK;oBAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;gBAClG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC/C,MAAM;YACR,KAAK,QAAQ;gBACX,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK;oBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;gBACpG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;gBACvD,MAAM;YACR,KAAK,OAAO;gBACV,IAAI,CAAC,MAAM,CAAC,GAAG;oBAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;gBAC9D,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACtC,MAAM;YACR,KAAK,OAAO;gBACV,IAAI,CAAC,MAAM,CAAC,QAAQ;oBAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;gBACxE,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAClC,MAAM;YACR,KAAK,iBAAiB;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ;oBAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;gBAClF,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC;gBAClF,MAAM;YACR,KAAK,YAAY;gBACf,cAAc,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBACxE,MAAM;QACV,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC;AACxB,CAAC"}
|
package/dist/core/crawler.d.ts
CHANGED
|
@@ -16,6 +16,23 @@ export interface CrawlOptions extends Omit<PeelOptions, 'format'> {
|
|
|
16
16
|
respectRobotsTxt?: boolean;
|
|
17
17
|
/** Rate limit between requests in milliseconds (default: 1000ms = 1 req/sec) */
|
|
18
18
|
rateLimitMs?: number;
|
|
19
|
+
/** Try sitemap.xml first to discover URLs (default: false) */
|
|
20
|
+
sitemapFirst?: boolean;
|
|
21
|
+
/** Crawl strategy: breadth-first or depth-first (default: 'bfs') */
|
|
22
|
+
strategy?: 'bfs' | 'dfs';
|
|
23
|
+
/** Skip duplicate content using fingerprinting (default: true) */
|
|
24
|
+
deduplication?: boolean;
|
|
25
|
+
/** Only crawl URLs matching these regex patterns */
|
|
26
|
+
includePatterns?: string[];
|
|
27
|
+
/** Progress callback called after each page */
|
|
28
|
+
onProgress?: (status: CrawlProgress) => void;
|
|
29
|
+
}
|
|
30
|
+
export interface CrawlProgress {
|
|
31
|
+
crawled: number;
|
|
32
|
+
queued: number;
|
|
33
|
+
failed: number;
|
|
34
|
+
currentUrl: string;
|
|
35
|
+
elapsed: number;
|
|
19
36
|
}
|
|
20
37
|
export interface CrawlResult {
|
|
21
38
|
/** URL of the crawled page */
|
|
@@ -34,6 +51,8 @@ export interface CrawlResult {
|
|
|
34
51
|
elapsed: number;
|
|
35
52
|
/** Error message if page failed to fetch */
|
|
36
53
|
error?: string;
|
|
54
|
+
/** Content fingerprint for deduplication */
|
|
55
|
+
fingerprint?: string;
|
|
37
56
|
}
|
|
38
57
|
/**
|
|
39
58
|
* Crawl a website starting from a URL
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../src/core/crawler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../src/core/crawler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAK/C,MAAM,WAAW,YAAa,SAAQ,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC;IAC/D,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gFAAgF;IAChF,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,2DAA2D;IAC3D,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,gFAAgF;IAChF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8DAA8D;IAC9D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC;IACzB,kEAAkE;IAClE,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,oDAAoD;IACpD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,+CAA+C;IAC/C,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,IAAI,CAAC;CAC9C;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,8BAA8B;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,uBAAuB;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,qCAAqC;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,2CAA2C;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAsFD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,KAAK,CACzB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,WAAW,EAAE,CAAC,CAkNxB"}
|
package/dist/core/crawler.js
CHANGED
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { peel } from '../index.js';
|
|
6
6
|
import { fetch as undiciFetch } from 'undici';
|
|
7
|
+
import { createHash } from 'crypto';
|
|
8
|
+
import { discoverSitemap } from './sitemap.js';
|
|
7
9
|
/**
|
|
8
10
|
* Parse robots.txt and return disallowed paths for User-agent: *
|
|
9
11
|
*/
|
|
@@ -91,7 +93,8 @@ function isAllowedByRobots(url, rules) {
|
|
|
91
93
|
* ```
|
|
92
94
|
*/
|
|
93
95
|
export async function crawl(startUrl, options = {}) {
|
|
94
|
-
const { maxPages = 10, maxDepth = 2, allowedDomains, excludePatterns = [], respectRobotsTxt = true, rateLimitMs = 1000, ...peelOptions } = options;
|
|
96
|
+
const { maxPages = 10, maxDepth = 2, allowedDomains, excludePatterns = [], respectRobotsTxt = true, rateLimitMs = 1000, sitemapFirst = false, strategy = 'bfs', deduplication = true, includePatterns = [], onProgress, ...peelOptions } = options;
|
|
97
|
+
const crawlStartTime = Date.now();
|
|
95
98
|
// Validate limits
|
|
96
99
|
const validatedMaxPages = Math.min(Math.max(maxPages, 1), 100);
|
|
97
100
|
const validatedMaxDepth = Math.min(Math.max(maxDepth, 1), 5);
|
|
@@ -105,6 +108,8 @@ export async function crawl(startUrl, options = {}) {
|
|
|
105
108
|
: [startDomain];
|
|
106
109
|
// Compile exclude patterns
|
|
107
110
|
const excludeRegexes = excludePatterns.map(pattern => new RegExp(pattern));
|
|
111
|
+
// Compile include patterns
|
|
112
|
+
const includeRegexes = includePatterns.map(pattern => new RegExp(pattern));
|
|
108
113
|
// Fetch robots.txt if needed
|
|
109
114
|
let robotsRules = { disallowedPaths: [] };
|
|
110
115
|
if (respectRobotsTxt) {
|
|
@@ -118,11 +123,31 @@ export async function crawl(startUrl, options = {}) {
|
|
|
118
123
|
// State tracking
|
|
119
124
|
const results = [];
|
|
120
125
|
const visited = new Set();
|
|
126
|
+
const contentFingerprints = new Set();
|
|
127
|
+
let failedCount = 0;
|
|
121
128
|
const queue = [
|
|
122
129
|
{ url: startUrl, depth: 0, parent: null },
|
|
123
130
|
];
|
|
131
|
+
// Sitemap-first: Discover URLs from sitemap before crawling
|
|
132
|
+
if (sitemapFirst) {
|
|
133
|
+
try {
|
|
134
|
+
const sitemap = await discoverSitemap(startDomain, { timeout: 10000, maxUrls: validatedMaxPages });
|
|
135
|
+
for (const entry of sitemap.urls) {
|
|
136
|
+
const entryUrl = entry.url;
|
|
137
|
+
try {
|
|
138
|
+
const entryUrlObj = new URL(entryUrl);
|
|
139
|
+
if (validatedAllowedDomains.includes(entryUrlObj.hostname)) {
|
|
140
|
+
queue.push({ url: entryUrl, depth: 1, parent: startUrl });
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
catch { /* skip invalid URLs */ }
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
catch { /* skip sitemap errors */ }
|
|
147
|
+
}
|
|
124
148
|
while (queue.length > 0 && results.length < validatedMaxPages) {
|
|
125
|
-
|
|
149
|
+
// Use DFS (stack) or BFS (queue) strategy
|
|
150
|
+
const item = strategy === 'dfs' ? queue.pop() : queue.shift();
|
|
126
151
|
const { url, depth, parent } = item;
|
|
127
152
|
// Skip if already visited
|
|
128
153
|
if (visited.has(url))
|
|
@@ -147,6 +172,10 @@ export async function crawl(startUrl, options = {}) {
|
|
|
147
172
|
if (excludeRegexes.some(regex => regex.test(url))) {
|
|
148
173
|
continue;
|
|
149
174
|
}
|
|
175
|
+
// Check include patterns
|
|
176
|
+
if (includeRegexes.length > 0 && !includeRegexes.some(regex => regex.test(url))) {
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
150
179
|
// Check robots.txt
|
|
151
180
|
if (respectRobotsTxt && !isAllowedByRobots(url, robotsRules)) {
|
|
152
181
|
console.error(`[Crawler] Skipping ${url} (disallowed by robots.txt)`);
|
|
@@ -158,7 +187,17 @@ export async function crawl(startUrl, options = {}) {
|
|
|
158
187
|
...peelOptions,
|
|
159
188
|
format: 'markdown',
|
|
160
189
|
});
|
|
161
|
-
|
|
190
|
+
// Deduplication: compute content fingerprint
|
|
191
|
+
let fingerprint;
|
|
192
|
+
if (deduplication) {
|
|
193
|
+
fingerprint = createHash('sha256').update(result.content).digest('hex');
|
|
194
|
+
if (contentFingerprints.has(fingerprint)) {
|
|
195
|
+
// Skip duplicate content
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
contentFingerprints.add(fingerprint);
|
|
199
|
+
}
|
|
200
|
+
const crawlResult = {
|
|
162
201
|
url: result.url,
|
|
163
202
|
title: result.title,
|
|
164
203
|
markdown: result.content,
|
|
@@ -166,7 +205,21 @@ export async function crawl(startUrl, options = {}) {
|
|
|
166
205
|
depth,
|
|
167
206
|
parent,
|
|
168
207
|
elapsed: result.elapsed,
|
|
169
|
-
}
|
|
208
|
+
};
|
|
209
|
+
if (fingerprint) {
|
|
210
|
+
crawlResult.fingerprint = fingerprint;
|
|
211
|
+
}
|
|
212
|
+
results.push(crawlResult);
|
|
213
|
+
// Call progress callback
|
|
214
|
+
if (onProgress) {
|
|
215
|
+
onProgress({
|
|
216
|
+
crawled: results.length,
|
|
217
|
+
queued: queue.length,
|
|
218
|
+
failed: failedCount,
|
|
219
|
+
currentUrl: url,
|
|
220
|
+
elapsed: Date.now() - crawlStartTime,
|
|
221
|
+
});
|
|
222
|
+
}
|
|
170
223
|
// Add discovered links to queue
|
|
171
224
|
if (depth < validatedMaxDepth) {
|
|
172
225
|
for (const link of result.links) {
|
|
@@ -186,6 +239,7 @@ export async function crawl(startUrl, options = {}) {
|
|
|
186
239
|
}
|
|
187
240
|
catch (error) {
|
|
188
241
|
// Log error and continue
|
|
242
|
+
failedCount++;
|
|
189
243
|
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
|
190
244
|
console.error(`[Crawler] Failed to fetch ${url}: ${errorMessage}`);
|
|
191
245
|
results.push({
|
|
@@ -198,6 +252,16 @@ export async function crawl(startUrl, options = {}) {
|
|
|
198
252
|
elapsed: 0,
|
|
199
253
|
error: errorMessage,
|
|
200
254
|
});
|
|
255
|
+
// Call progress callback even for failed pages
|
|
256
|
+
if (onProgress) {
|
|
257
|
+
onProgress({
|
|
258
|
+
crawled: results.length,
|
|
259
|
+
queued: queue.length,
|
|
260
|
+
failed: failedCount,
|
|
261
|
+
currentUrl: url,
|
|
262
|
+
elapsed: Date.now() - crawlStartTime,
|
|
263
|
+
});
|
|
264
|
+
}
|
|
201
265
|
}
|
|
202
266
|
}
|
|
203
267
|
return results;
|
package/dist/core/crawler.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/core/crawler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/core/crawler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AA6D/C;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,MAAc;IAC1C,MAAM,SAAS,GAAG,WAAW,MAAM,aAAa,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE;YAC5C,OAAO,EAAE;gBACP,YAAY,EAAE,sCAAsC;aACrD;YACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,mBAAmB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,gDAAgD;YAChD,OAAO,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE/B,MAAM,eAAe,GAAa,EAAE,CAAC;QACrC,IAAI,UAA8B,CAAC;QACnC,IAAI,eAAe,GAAG,KAAK,CAAC;QAE5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAE5B,0BAA0B;YAC1B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC7D,eAAe,GAAG,KAAK,KAAK,GAAG,CAAC;gBAChC,SAAS;YACX,CAAC;YAED,IAAI,CAAC,eAAe;gBAAE,SAAS;YAE/B,4BAA4B;YAC5B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC1D,IAAI,IAAI,EAAE,CAAC;oBACT,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YAED,8BAA8B;YAC9B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;gBACrD,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACxE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBAClB,UAAU,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC,0BAA0B;gBACvD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,iDAAiD;QACjD,OAAO,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,GAAW,EAAE,KAAkB;IACxD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;IAE7B,KAAK,MAAM,UAAU,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;QAC/C,4EAA4E;QAC5E,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAChC,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,QAAgB,EAChB,UAAwB,EAAE;IAE1B,MAAM,EACJ,QAAQ,GAAG,EAAE,EACb,QAAQ,GAAG,CAAC,EACZ,cAAc,EACd,eAAe,GAAG,EAAE,EACpB,gBAAgB,GAAG,IAAI,EACvB,WAAW,GAAG,IAAI,EAClB,YAAY,GAAG,KAAK,EACpB,QAAQ,GAAG,KAAK,EAChB,aAAa,GAAG,IAAI,EACpB,eAAe,GAAG,EAAE,EACpB,UAAU,EACV,GAAG,WAAW,EACf,GAAG,OAAO,CAAC;IAEZ,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,kBAAkB;IAClB,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC/D,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7D,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,6BAA6B;IAEpF,qBAAqB;IACrB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC;IAEzC,kDAAkD;IAClD,MAAM,uBAAuB,GAAG,cAAc,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC;QACzE,CAAC,CAAC,cAAc;QAChB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAElB,2BAA2B;IAC3B,MAAM,cAAc,GAAG,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IAE3E,2BAA2B;IAC3B,MAAM,cAAc,GAAG,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IAE3E,6BAA6B;IAC7B,IAAI,WAAW,GAAgB,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;IACvD,IAAI,gBAAgB,EAAE,CAAC;QACrB,WAAW,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC;QAEhD,qEAAqE;QACrE,IAAI,WAAW,CAAC,UAAU,IAAI,WAAW,CAAC,UAAU,GAAG,kBAAkB,EAAE,CAAC;YAC1E,OAAO,CAAC,KAAK,CAAC,gDAAgD,WAAW,CAAC,UAAU,IAAI,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;IAED,MAAM,kBAAkB,GAAG,WAAW,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAExE,iBAAiB;IACjB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9C,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,MAAM,KAAK,GAAiE;QAC1E,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;KAC1C,CAAC;IAEF,4DAA4D;IAC5D,IAAI,YAAY,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACnG,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC;gBAC3B,IAAI,CAAC;oBACH,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;oBACtC,IAAI,uBAAuB,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC3D,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;oBAC5D,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC,CAAC,uBAAuB,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,yBAAyB,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;QAC9D,0CAA0C;QAC1C,MAAM,IAAI,GAAG,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;QAChE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAEpC,0BAA0B;QAC1B,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjB,yBAAyB;QACzB,IAAI,KAAK,GAAG,iBAAiB;YAAE,SAAS;QAExC,eAAe;QACf,IAAI,MAAW,CAAC;QAChB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS,CAAC,oBAAoB;QAChC,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvD,SAAS;QACX,CAAC;QAED,yBAAyB;QACzB,IAAI,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAClD,SAAS;QACX,CAAC;QAED,yBAAyB;QACzB,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAChF,SAAS;QACX,CAAC;QAED,mBAAmB;QACnB,IAAI,gBAAgB,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,CAAC;YAC7D,OAAO,CAAC,KAAK,CAAC,sBAAsB,GAAG,6BAA6B,CAAC,CAAC;YACtE,SAAS;QACX,CAAC;QAED,iBAAiB;QACjB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE;gBAC7B,GAAG,WAAW;gBACd,MAAM,EAAE,UAAU;aACnB,CAAC,CAAC;YAEH,6CAA6C;YAC7C,IAAI,WAA+B,CAAC;YACpC,IAAI,aAAa,EAAE,CAAC;gBAClB,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACxE,IAAI,mBAAmB,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;oBACzC,yBAAyB;oBACzB,SAAS;gBACX,CAAC;gBACD,mBAAmB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACvC,CAAC;YAED,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,QAAQ,EAAE,MAAM,CAAC,OAAO;gBACxB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK;gBACL,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAC;YAEF,IAAI,WAAW,EAAE,CAAC;gBAChB,WAAW,CAAC,WAAW,GAAG,WAAW,CAAC;YACxC,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAE1B,yBAAyB;YACzB,IAAI,UAAU,EAAE,CAAC;gBACf,UAAU,CAAC;oBACT,OAAO,EAAE,OAAO,CAAC,MAAM;oBACvB,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,WAAW;oBACnB,UAAU,EAAE,GAAG;oBACf,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;iBACrC,CAAC,CAAC;YACL,CAAC;YAED,gCAAgC;YAChC,IAAI,KAAK,GAAG,iBAAiB,EAAE,CAAC;gBAC9B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;oBAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;wBACvB,KAAK,CAAC,IAAI,CAAC;4BACT,GAAG,EAAE,IAAI;4BACT,KAAK,EAAE,KAAK,GAAG,CAAC;4BAChB,MAAM,EAAE,GAAG;yBACZ,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,IAAI,OAAO,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;gBACvC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,yBAAyB;YACzB,WAAW,EAAE,CAAC;YACd,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC9E,OAAO,CAAC,KAAK,CAAC,6BAA6B,GAAG,KAAK,YAAY,EAAE,CAAC,CAAC;YAEnE,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG;gBACH,KAAK,EAAE,EAAE;gBACT,QAAQ,EAAE,EAAE;gBACZ,KAAK,EAAE,EAAE;gBACT,KAAK;gBACL,MAAM;gBACN,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,YAAY;aACpB,CAAC,CAAC;YAEH,+CAA+C;YAC/C,IAAI,UAAU,EAAE,CAAC;gBACf,UAAU,CAAC;oBACT,OAAO,EAAE,OAAO,CAAC,MAAM;oBACvB,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,WAAW;oBACnB,UAAU,EAAE,GAAG;oBACf,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;iBACrC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Structured data extraction using CSS selectors and heuristics
|
|
3
|
+
*/
|
|
4
|
+
import type { ExtractOptions } from '../types.js';
|
|
5
|
+
export declare function extractStructured(html: string, options: ExtractOptions): Record<string, any>;
|
|
6
|
+
//# sourceMappingURL=extract.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/core/extract.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAuE5F"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Structured data extraction using CSS selectors and heuristics
|
|
3
|
+
*/
|
|
4
|
+
import { load } from 'cheerio';
|
|
5
|
+
export function extractStructured(html, options) {
|
|
6
|
+
const $ = load(html);
|
|
7
|
+
const result = {};
|
|
8
|
+
if (options.selectors) {
|
|
9
|
+
// Direct CSS selector extraction
|
|
10
|
+
for (const [field, selector] of Object.entries(options.selectors)) {
|
|
11
|
+
const elements = $(selector);
|
|
12
|
+
if (elements.length === 0) {
|
|
13
|
+
result[field] = null;
|
|
14
|
+
}
|
|
15
|
+
else if (elements.length === 1) {
|
|
16
|
+
result[field] = elements.first().text().trim();
|
|
17
|
+
}
|
|
18
|
+
else {
|
|
19
|
+
result[field] = elements.map((_, el) => $(el).text().trim()).get();
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
if (options.schema) {
|
|
24
|
+
// Schema-based extraction using heuristics
|
|
25
|
+
const properties = options.schema.properties || options.schema;
|
|
26
|
+
for (const [field, spec] of Object.entries(properties)) {
|
|
27
|
+
if (result[field] !== undefined)
|
|
28
|
+
continue; // Already extracted by selector
|
|
29
|
+
// Try common CSS patterns based on field name
|
|
30
|
+
const fieldLower = field.toLowerCase();
|
|
31
|
+
const candidates = [
|
|
32
|
+
`[itemprop="${fieldLower}"]`,
|
|
33
|
+
`[data-${fieldLower}]`,
|
|
34
|
+
`.${fieldLower}`,
|
|
35
|
+
`#${fieldLower}`,
|
|
36
|
+
`[class*="${fieldLower}"]`,
|
|
37
|
+
`meta[name="${fieldLower}"]`,
|
|
38
|
+
`meta[property="og:${fieldLower}"]`,
|
|
39
|
+
];
|
|
40
|
+
for (const sel of candidates) {
|
|
41
|
+
const el = $(sel).first();
|
|
42
|
+
if (el.length > 0) {
|
|
43
|
+
let value = el.attr('content') || el.text().trim();
|
|
44
|
+
if (value) {
|
|
45
|
+
// Type coercion based on schema
|
|
46
|
+
if (spec?.type === 'number') {
|
|
47
|
+
const num = parseFloat(value.replace(/[^0-9.-]/g, ''));
|
|
48
|
+
if (!isNaN(num)) {
|
|
49
|
+
result[field] = num;
|
|
50
|
+
break;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
else if (spec?.type === 'boolean') {
|
|
54
|
+
result[field] = ['true', 'yes', '1'].includes(value.toLowerCase());
|
|
55
|
+
break;
|
|
56
|
+
}
|
|
57
|
+
else if (spec?.type === 'array') {
|
|
58
|
+
// For arrays, get all matches
|
|
59
|
+
const allValues = $(sel).map((_, e) => $(e).text().trim()).get();
|
|
60
|
+
result[field] = allValues;
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
result[field] = value;
|
|
65
|
+
break;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (result[field] === undefined) {
|
|
71
|
+
result[field] = null;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return result;
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=extract.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/core/extract.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAG/B,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,OAAuB;IACrE,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IACrB,MAAM,MAAM,GAAwB,EAAE,CAAC;IAEvC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,iCAAiC;QACjC,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YAClE,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACjD,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;YACrE,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,2CAA2C;QAC3C,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;QAC/D,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAoB,EAAE,CAAC;YAC1E,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS;gBAAE,SAAS,CAAC,gCAAgC;YAE3E,8CAA8C;YAC9C,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;YACvC,MAAM,UAAU,GAAG;gBACjB,cAAc,UAAU,IAAI;gBAC5B,SAAS,UAAU,GAAG;gBACtB,IAAI,UAAU,EAAE;gBAChB,IAAI,UAAU,EAAE;gBAChB,YAAY,UAAU,IAAI;gBAC1B,cAAc,UAAU,IAAI;gBAC5B,qBAAqB,UAAU,IAAI;aACpC,CAAC;YAEF,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC7B,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;gBAC1B,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;oBACnD,IAAI,KAAK,EAAE,CAAC;wBACV,gCAAgC;wBAChC,IAAI,IAAI,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;4BAC5B,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;4BACvD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gCAChB,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;gCACpB,MAAM;4BACR,CAAC;wBACH,CAAC;6BAAM,IAAI,IAAI,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;4BACpC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;4BACnE,MAAM;wBACR,CAAC;6BAAM,IAAI,IAAI,EAAE,IAAI,KAAK,OAAO,EAAE,CAAC;4BAClC,8BAA8B;4BAC9B,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;4BACjE,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;4BAC1B,MAAM;wBACR,CAAC;6BAAM,CAAC;4BACN,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;4BACtB,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,EAAE,CAAC;gBAChC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/core/fetcher.d.ts
CHANGED
|
@@ -27,6 +27,15 @@ export declare function browserFetch(url: string, options?: {
|
|
|
27
27
|
headers?: Record<string, string>;
|
|
28
28
|
cookies?: string[];
|
|
29
29
|
stealth?: boolean;
|
|
30
|
+
actions?: Array<{
|
|
31
|
+
type: 'wait' | 'click' | 'scroll' | 'type' | 'fill' | 'select' | 'press' | 'hover' | 'waitForSelector' | 'screenshot';
|
|
32
|
+
selector?: string;
|
|
33
|
+
value?: string;
|
|
34
|
+
key?: string;
|
|
35
|
+
ms?: number;
|
|
36
|
+
to?: 'top' | 'bottom' | number;
|
|
37
|
+
timeout?: number;
|
|
38
|
+
}>;
|
|
30
39
|
}): Promise<FetchResult>;
|
|
31
40
|
/**
|
|
32
41
|
* Retry a fetch operation with exponential backoff
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AAiQH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,MAAM,EACX,SAAS,CAAC,EAAE,MAAM,EAClB,SAAS,GAAE,MAAc,EACzB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACrC,OAAO,CAAC,WAAW,CAAC,CAgMtB;AAyCD;;;GAGG;AACH,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AAiQH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,MAAM,EACX,SAAS,CAAC,EAAE,MAAM,EAClB,SAAS,GAAE,MAAc,EACzB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACrC,OAAO,CAAC,WAAW,CAAC,CAgMtB;AAyCD;;;GAGG;AACH,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,iBAAiB,GAAG,YAAY,CAAC;QACtH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,EAAE,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;QAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;CACC,GACL,OAAO,CAAC,WAAW,CAAC,CA6KtB;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,CAAC,EAChC,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,WAAW,GAAE,MAAU,EACvB,WAAW,GAAE,MAAa,GACzB,OAAO,CAAC,CAAC,CAAC,CAsBZ;AAED;;GAEG;AACH,wBAAsB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAS7C"}
|
package/dist/core/fetcher.js
CHANGED
|
@@ -436,7 +436,7 @@ async function getStealthBrowser() {
|
|
|
436
436
|
export async function browserFetch(url, options = {}) {
|
|
437
437
|
// SECURITY: Validate URL to prevent SSRF
|
|
438
438
|
validateUrl(url);
|
|
439
|
-
const { userAgent, waitMs = 0, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, stealth = false } = options;
|
|
439
|
+
const { userAgent, waitMs = 0, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, stealth = false, actions } = options;
|
|
440
440
|
// Validate user agent if provided
|
|
441
441
|
const validatedUserAgent = userAgent ? validateUserAgent(userAgent) : getRandomUserAgent();
|
|
442
442
|
// Validate wait time
|
|
@@ -508,6 +508,7 @@ export async function browserFetch(url, options = {}) {
|
|
|
508
508
|
await page.route('**/*', (route) => route.continue());
|
|
509
509
|
}
|
|
510
510
|
// SECURITY: Wrap entire operation in timeout
|
|
511
|
+
let screenshotBuffer;
|
|
511
512
|
const fetchPromise = (async () => {
|
|
512
513
|
await page.goto(url, {
|
|
513
514
|
waitUntil: 'domcontentloaded',
|
|
@@ -517,6 +518,14 @@ export async function browserFetch(url, options = {}) {
|
|
|
517
518
|
if (waitMs > 0) {
|
|
518
519
|
await page.waitForTimeout(waitMs);
|
|
519
520
|
}
|
|
521
|
+
// Execute page actions if provided
|
|
522
|
+
if (actions && actions.length > 0) {
|
|
523
|
+
const { executeActions } = await import('./actions.js');
|
|
524
|
+
const actionScreenshot = await executeActions(page, actions);
|
|
525
|
+
if (actionScreenshot) {
|
|
526
|
+
screenshotBuffer = actionScreenshot;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
520
529
|
const html = await page.content();
|
|
521
530
|
const finalUrl = page.url();
|
|
522
531
|
return { html, finalUrl };
|
|
@@ -532,9 +541,8 @@ export async function browserFetch(url, options = {}) {
|
|
|
532
541
|
if (!html || html.length < 100) {
|
|
533
542
|
throw new BlockedError('Empty or suspiciously small response from browser.');
|
|
534
543
|
}
|
|
535
|
-
// Capture screenshot if requested
|
|
536
|
-
|
|
537
|
-
if (screenshot) {
|
|
544
|
+
// Capture screenshot if requested (and not already captured by actions)
|
|
545
|
+
if (screenshot && !screenshotBuffer) {
|
|
538
546
|
screenshotBuffer = await page.screenshot({
|
|
539
547
|
fullPage: screenshotFullPage,
|
|
540
548
|
type: 'png'
|
package/dist/core/fetcher.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAA2B,MAAM,YAAY,CAAC;AAC/D,OAAO,EAAE,QAAQ,IAAI,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,aAAa,MAAM,gCAAgC,CAAC;AAC3D,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAErF,yCAAyC;AACzC,eAAe,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC;AAErC,MAAM,WAAW,GAAG;IAClB,uHAAuH;IACvH,iHAAiH;IACjH,uHAAuH;IACvH,kFAAkF;IAClF,uGAAuG;CACxG,CAAC;AAEF,SAAS,kBAAkB;IACzB,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,SAAiB;IACpC,eAAe;IACf,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,uDAAuD;IACvD,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAED,qBAAqB;IACrB,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;IACtE,CAAC;IAED,iCAAiC;IACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,YAAY,CAAC,kBAAkB,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,2BAA2B;IAC3B,MAAM,iBAAiB,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACnD,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,iFAAiF;IACjF,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,QAAQ,EAAE,CAAC;QACb,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,0CAA0C;IAC1C,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,6BAA6B;IAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAEjD,wCAAwC;IACxC,MAAM,WAAW,GAAG,8CAA8C,CAAC;IACnE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1C,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,CAAC;IACjD,CAAC;IAED,2BAA2B;IAC3B,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO;YACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;YAClB,GAAG,GAAG,IAAI;SACX,CAAC;IACJ,CAAC;IAED,wDAAwD;IACxD,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,0BAA0B;QAC1B,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;gBACtB,OAAO;oBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;YACJ,CAAC;QACH,CAAC;QACD,kCAAkC;QAClC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;YACtB,OAAO;gBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;gBAClB,GAAG,GAAG,IAAI;aACX,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAgB;IAC3C,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;IAE5B,wBAAwB;IACxB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,sBAAsB;IACtB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,4CAA4C,CAAC,CAAC;IACvE,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,MAAM,IAAI,YAAY,CAAC,mDAAmD,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,kBAAkB;IAClB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAE5D,gBAAgB;IAChB,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;QACjD,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,2DAA2D;IAC3D,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,wBAAwB;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAEnC,iEAAiE;QACjE,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,oBAAoB;YACpB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC/C,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBACvD,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACjC,MAAM,MAAM,GAAG;oBACb,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;gBACF,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,qDAAqD,CAAC,CAAC;IAChF,CAAC;IAED,sDAAsD;IACtD,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,YAAY,CAAC,sDAAsD,CAAC,CAAC;IACjF,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,0CAA0C,CAAC,CAAC;IACrE,CAAC;IACD,wCAAwC;IACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAUD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAW,EACX,SAAkB,EAClB,YAAoB,KAAK,EACzB,aAAsC;IAEtC,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,2EAA2E;IAC3E,MAAM,cAAc,GAA2B;QAC7C,YAAY,EAAE,kBAAkB;QAChC,QAAQ,EAAE,4EAA4E;QACtF,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,KAAK,EAAE,GAAG;QACV,YAAY,EAAE,YAAY;QAC1B,2BAA2B,EAAE,GAAG;KACjC,CAAC;IAEF,MAAM,aAAa,GAAG,EAAE,GAAG,cAAc,EAAE,CAAC;IAE5C,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;YACzD,uCAAuC;YACvC,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBACjC,MAAM,IAAI,YAAY,CAAC,mCAAmC,CAAC,CAAC;YAC9D,CAAC;YACD,aAAa,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,GAAG,CAAC;IACrB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,OAAO,aAAa,IAAI,aAAa,EAAE,CAAC;QACtC,wBAAwB;QACxB,IAAI,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,YAAY,CAAC,wBAAwB,CAAC,CAAC;QACnD,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEzB,+BAA+B;QAC/B,WAAW,CAAC,UAAU,CAAC,CAAC;QAExB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;QAE9D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE;gBAC7C,OAAO,EAAE,aAAa;gBACtB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ,EAAE,qCAAqC;aAC1D,CAAC,CAAC;YAEH,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,4BAA4B;YAC5B,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;gBACtE,CAAC;gBAED,wBAAwB;gBACxB,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC;gBAChD,aAAa,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACvD,MAAM,IAAI,YAAY,CACpB,QAAQ,QAAQ,CAAC,MAAM,iEAAiE,CACzF,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,YAAY,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC5E,CAAC;YAED,qEAAqE;YACrE,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,MAAM,aAAa,GAAG;gBACpB,WAAW,EAAE,uBAAuB,EAAE,iBAAiB;gBACvD,YAAY,EAAE,eAAe,EAAE,UAAU;gBACzC,kBAAkB,EAAE,WAAW;gBAC/B,UAAU,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,sBAAsB;gBAC5E,wBAAwB,EAAE,iBAAiB,EAAE,UAAU;aACxD,CAAC;YACF,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC;YACnF,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,oCAAoC;gBACpC,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBACtG,MAAM,IAAI,YAAY,CAAC,wBAAwB,WAAW,4CAA4C,CAAC,CAAC;gBAC1G,CAAC;YACH,CAAC;YAED,wEAAwE;YACxE,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;YAE1C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,IAAI,EAAE,CAAC;oBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,MAAM;oBAEhB,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;oBAC1B,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;wBACzB,MAAM,CAAC,MAAM,EAAE,CAAC;wBAChB,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;oBAC1D,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,CAAC;YAED,iBAAiB;YACjB,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEhD,wEAAwE;YACxE,+DAA+D;YAC/D,MAAM,aAAa,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACpF,IAAI,aAAa,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,YAAY,CAAC,oEAAoE,CAAC,CAAC;YAC/F,CAAC;YACD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,YAAY,CAAC,qBAAqB,CAAC,CAAC;YAChD,CAAC;YAED,0DAA0D;YAC1D,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;gBACrG,MAAM,IAAI,YAAY,CAAC,+DAA+D,CAAC,CAAC;YAC1F,CAAC;YAED,OAAO;gBACL,IAAI;gBACJ,GAAG,EAAE,UAAU;gBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;gBAC3B,WAAW;aACZ,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACpG,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,YAAY,CAAC,2BAA2B,SAAS,IAAI,CAAC,CAAC;YACnE,CAAC;YAED,4DAA4D;YAC5D,MAAM,KAAK,GAAG,KAAK,YAAY,KAAK,IAAK,KAAa,CAAC,KAAK,CAAC;YAC7D,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC;YAErD,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1H,MAAM,IAAI,YAAY,CAAC,iCAAiC,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,qEAAqE,CAAC,CAAC;YAC7J,CAAC;YACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;gBACvE,MAAM,IAAI,YAAY,CAAC,0BAA0B,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,uDAAuD,CAAC,CAAC;YACxI,CAAC;YACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBACtC,MAAM,IAAI,YAAY,CAAC,yBAAyB,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,2BAA2B,CAAC,CAAC;YAC3G,CAAC;YACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAClE,MAAM,IAAI,YAAY,CAAC,uBAAuB,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,8BAA8B,CAAC,CAAC;YAC5G,CAAC;YACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;gBACvE,MAAM,IAAI,YAAY,CAAC,mDAAmD,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;YAC7G,CAAC;YAED,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YACrE,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,YAAY,CAAC,oBAAoB,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,MAAM,IAAI,YAAY,CAAC,2BAA2B,aAAa,GAAG,CAAC,CAAC;AACtE,CAAC;AAED,IAAI,aAAa,GAAmB,IAAI,CAAC;AACzC,IAAI,oBAAoB,GAAmB,IAAI,CAAC;AAChD,IAAI,gBAAgB,GAAG,CAAC,CAAC;AACzB,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,KAAK,UAAU,UAAU;IACvB,4DAA4D;IAC5D,IAAI,aAAa,EAAE,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC;gBAChC,OAAO,aAAa,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;YAC5B,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,aAAa,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,KAAK,UAAU,iBAAiB;IAC9B,oEAAoE;IACpE,IAAI,oBAAoB,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,IAAI,oBAAoB,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvC,OAAO,oBAAoB,CAAC;YAC9B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;YAC5B,oBAAoB,GAAG,IAAI,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,oBAAoB,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,UASI,EAAE;IAEN,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,MAAM,EACJ,SAAS,EACT,MAAM,GAAG,CAAC,EACV,SAAS,GAAG,KAAK,EACjB,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,OAAO,EACP,OAAO,EACP,OAAO,GAAG,KAAK,EAChB,GAAG,OAAO,CAAC;IAEZ,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,qBAAqB;IACrB,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;QACjC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,gDAAgD;IAChD,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,6BAA6B;YAC7B,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBACjC,MAAM,IAAI,YAAY,CAAC,mCAAmC,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,gBAAgB,GAAG,KAAK,CAAC,CAAC,qBAAqB;IAErD,OAAO,gBAAgB,IAAI,oBAAoB,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,GAAG,gBAAgB,EAAE,CAAC;YACnD,MAAM,IAAI,YAAY,CAAC,2DAA2D,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,gBAAgB,EAAE,CAAC;IACnB,IAAI,IAAI,GAAgB,IAAI,CAAC;IAE7B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC,CAAC,MAAM,UAAU,EAAE,CAAC;QACzE,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC;YAC3B,SAAS,EAAE,kBAAkB;SAC9B,CAAC,CAAC;QAEH,iCAAiC;QACjC,IAAI,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,0BAA0B;QAC1B,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;gBACzC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACzD,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE3C,IAAI,CAAC,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACjC,MAAM,IAAI,YAAY,CAAC,0BAA0B,MAAM,EAAE,CAAC,CAAC;gBAC7D,CAAC;gBAED,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;oBACjB,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;oBACnB,GAAG;iBACJ,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACjD,CAAC;QAED,4FAA4F;QAC5F,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;gBACjC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;gBACpD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACpE,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,uCAAuC;YACvC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,6CAA6C;QAC7C,MAAM,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YAC/B,MAAM,IAAK,CAAC,IAAI,CAAC,GAAG,EAAE;gBACpB,SAAS,EAAE,kBAAkB;gBAC7B,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,MAAM,IAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAK,CAAC,GAAG,EAAE,CAAC;YAE7B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,6BAA6B,SAAS,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;QAEH,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;QAE9E,4BAA4B;QAC5B,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,aAAa;YACjD,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC/B,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;QAC/E,CAAC;QAED,kCAAkC;QAClC,IAAI,gBAAoC,CAAC;QACzC,IAAI,UAAU,EAAE,CAAC;YACf,gBAAgB,GAAG,MAAM,IAAK,CAAC,UAAU,CAAC;gBACxC,QAAQ,EAAE,kBAAkB;gBAC5B,IAAI,EAAE,KAAK;aACZ,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,IAAI;YACJ,GAAG,EAAE,QAAQ;YACb,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;YACpG,MAAM,KAAK,CAAC;QACd,CAAC;QAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,YAAY,CAAC,8BAA8B,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,IAAI,YAAY,CACpB,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CACpF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,oDAAoD;QACpD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrC,CAAC;QACD,gBAAgB,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,EAAoB,EACpB,cAAsB,CAAC,EACvB,cAAsB,IAAI;IAE1B,IAAI,SAAS,GAAiB,IAAI,CAAC;IAEnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;YAExE,4CAA4C;YAC5C,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACnE,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;gBACrD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,YAAY,CAAC,cAAc,CAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;QAC5B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;IACD,IAAI,oBAAoB,EAAE,CAAC;QACzB,MAAM,oBAAoB,CAAC,KAAK,EAAE,CAAC;QACnC,oBAAoB,GAAG,IAAI,CAAC;IAC9B,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAA2B,MAAM,YAAY,CAAC;AAC/D,OAAO,EAAE,QAAQ,IAAI,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,aAAa,MAAM,gCAAgC,CAAC;AAC3D,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAErF,yCAAyC;AACzC,eAAe,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC;AAErC,MAAM,WAAW,GAAG;IAClB,uHAAuH;IACvH,iHAAiH;IACjH,uHAAuH;IACvH,kFAAkF;IAClF,uGAAuG;CACxG,CAAC;AAEF,SAAS,kBAAkB;IACzB,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,SAAiB;IACpC,eAAe;IACf,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,uDAAuD;IACvD,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAED,qBAAqB;IACrB,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;IACtE,CAAC;IAED,iCAAiC;IACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,YAAY,CAAC,kBAAkB,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,2BAA2B;IAC3B,MAAM,iBAAiB,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACnD,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,iFAAiF;IACjF,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,QAAQ,EAAE,CAAC;QACb,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,0CAA0C;IAC1C,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,6BAA6B;IAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAEjD,wCAAwC;IACxC,MAAM,WAAW,GAAG,8CAA8C,CAAC;IACnE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1C,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,CAAC;IACjD,CAAC;IAED,2BAA2B;IAC3B,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO;YACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;YAClB,GAAG,GAAG,IAAI;SACX,CAAC;IACJ,CAAC;IAED,wDAAwD;IACxD,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,0BAA0B;QAC1B,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;gBACtB,OAAO;oBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;YACJ,CAAC;QACH,CAAC;QACD,kCAAkC;QAClC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;YACtB,OAAO;gBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;gBAClB,GAAG,GAAG,IAAI;aACX,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAgB;IAC3C,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;IAE5B,wBAAwB;IACxB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,sBAAsB;IACtB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,4CAA4C,CAAC,CAAC;IACvE,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,MAAM,IAAI,YAAY,CAAC,mDAAmD,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,kBAAkB;IAClB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAE5D,gBAAgB;IAChB,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;QACjD,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,2DAA2D;IAC3D,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,wBAAwB;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAEnC,iEAAiE;QACjE,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,oBAAoB;YACpB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC/C,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBACvD,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACjC,MAAM,MAAM,GAAG;oBACb,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;gBACF,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,qDAAqD,CAAC,CAAC;IAChF,CAAC;IAED,sDAAsD;IACtD,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,YAAY,CAAC,sDAAsD,CAAC,CAAC;IACjF,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,0CAA0C,CAAC,CAAC;IACrE,CAAC;IACD,wCAAwC;IACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAUD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAW,EACX,SAAkB,EAClB,YAAoB,KAAK,EACzB,aAAsC;IAEtC,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,2EAA2E;IAC3E,MAAM,cAAc,GAA2B;QAC7C,YAAY,EAAE,kBAAkB;QAChC,QAAQ,EAAE,4EAA4E;QACtF,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,KAAK,EAAE,GAAG;QACV,YAAY,EAAE,YAAY;QAC1B,2BAA2B,EAAE,GAAG;KACjC,CAAC;IAEF,MAAM,aAAa,GAAG,EAAE,GAAG,cAAc,EAAE,CAAC;IAE5C,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;YACzD,uCAAuC;YACvC,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBACjC,MAAM,IAAI,YAAY,CAAC,mCAAmC,CAAC,CAAC;YAC9D,CAAC;YACD,aAAa,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,GAAG,CAAC;IACrB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,OAAO,aAAa,IAAI,aAAa,EAAE,CAAC;QACtC,wBAAwB;QACxB,IAAI,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,YAAY,CAAC,wBAAwB,CAAC,CAAC;QACnD,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEzB,+BAA+B;QAC/B,WAAW,CAAC,UAAU,CAAC,CAAC;QAExB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;QAE9D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE;gBAC7C,OAAO,EAAE,aAAa;gBACtB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ,EAAE,qCAAqC;aAC1D,CAAC,CAAC;YAEH,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,4BAA4B;YAC5B,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;gBACtE,CAAC;gBAED,wBAAwB;gBACxB,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC;gBAChD,aAAa,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACvD,MAAM,IAAI,YAAY,CACpB,QAAQ,QAAQ,CAAC,MAAM,iEAAiE,CACzF,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,YAAY,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC5E,CAAC;YAED,qEAAqE;YACrE,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,MAAM,aAAa,GAAG;gBACpB,WAAW,EAAE,uBAAuB,EAAE,iBAAiB;gBACvD,YAAY,EAAE,eAAe,EAAE,UAAU;gBACzC,kBAAkB,EAAE,WAAW;gBAC/B,UAAU,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,sBAAsB;gBAC5E,wBAAwB,EAAE,iBAAiB,EAAE,UAAU;aACxD,CAAC;YACF,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC;YACnF,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,oCAAoC;gBACpC,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBACtG,MAAM,IAAI,YAAY,CAAC,wBAAwB,WAAW,4CAA4C,CAAC,CAAC;gBAC1G,CAAC;YACH,CAAC;YAED,wEAAwE;YACxE,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;YAE1C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,IAAI,EAAE,CAAC;oBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,MAAM;oBAEhB,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;oBAC1B,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;wBACzB,MAAM,CAAC,MAAM,EAAE,CAAC;wBAChB,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;oBAC1D,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,CAAC;YAED,iBAAiB;YACjB,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEhD,wEAAwE;YACxE,+DAA+D;YAC/D,MAAM,aAAa,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACpF,IAAI,aAAa,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,YAAY,CAAC,oEAAoE,CAAC,CAAC;YAC/F,CAAC;YACD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,YAAY,CAAC,qBAAqB,CAAC,CAAC;YAChD,CAAC;YAED,0DAA0D;YAC1D,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;gBACrG,MAAM,IAAI,YAAY,CAAC,+DAA+D,CAAC,CAAC;YAC1F,CAAC;YAED,OAAO;gBACL,IAAI;gBACJ,GAAG,EAAE,UAAU;gBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;gBAC3B,WAAW;aACZ,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACpG,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,YAAY,CAAC,2BAA2B,SAAS,IAAI,CAAC,CAAC;YACnE,CAAC;YAED,4DAA4D;YAC5D,MAAM,KAAK,GAAG,KAAK,YAAY,KAAK,IAAK,KAAa,CAAC,KAAK,CAAC;YAC7D,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC;YAErD,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1H,MAAM,IAAI,YAAY,CAAC,iCAAiC,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,qEAAqE,CAAC,CAAC;YAC7J,CAAC;YACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;gBACvE,MAAM,IAAI,YAAY,CAAC,0BAA0B,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,uDAAuD,CAAC,CAAC;YACxI,CAAC;YACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBACtC,MAAM,IAAI,YAAY,CAAC,yBAAyB,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,2BAA2B,CAAC,CAAC;YAC3G,CAAC;YACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAClE,MAAM,IAAI,YAAY,CAAC,uBAAuB,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,8BAA8B,CAAC,CAAC;YAC5G,CAAC;YACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;gBACvE,MAAM,IAAI,YAAY,CAAC,mDAAmD,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;YAC7G,CAAC;YAED,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YACrE,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,YAAY,CAAC,oBAAoB,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,MAAM,IAAI,YAAY,CAAC,2BAA2B,aAAa,GAAG,CAAC,CAAC;AACtE,CAAC;AAED,IAAI,aAAa,GAAmB,IAAI,CAAC;AACzC,IAAI,oBAAoB,GAAmB,IAAI,CAAC;AAChD,IAAI,gBAAgB,GAAG,CAAC,CAAC;AACzB,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,KAAK,UAAU,UAAU;IACvB,4DAA4D;IAC5D,IAAI,aAAa,EAAE,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC;gBAChC,OAAO,aAAa,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;YAC5B,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,aAAa,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,KAAK,UAAU,iBAAiB;IAC9B,oEAAoE;IACpE,IAAI,oBAAoB,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,IAAI,oBAAoB,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvC,OAAO,oBAAoB,CAAC;YAC9B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;YAC5B,oBAAoB,GAAG,IAAI,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,oBAAoB,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,UAkBI,EAAE;IAEN,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,MAAM,EACJ,SAAS,EACT,MAAM,GAAG,CAAC,EACV,SAAS,GAAG,KAAK,EACjB,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,OAAO,EACP,OAAO,EACP,OAAO,GAAG,KAAK,EACf,OAAO,EACR,GAAG,OAAO,CAAC;IAEZ,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,qBAAqB;IACrB,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;QACjC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,gDAAgD;IAChD,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,6BAA6B;YAC7B,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBACjC,MAAM,IAAI,YAAY,CAAC,mCAAmC,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,gBAAgB,GAAG,KAAK,CAAC,CAAC,qBAAqB;IAErD,OAAO,gBAAgB,IAAI,oBAAoB,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,GAAG,gBAAgB,EAAE,CAAC;YACnD,MAAM,IAAI,YAAY,CAAC,2DAA2D,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,gBAAgB,EAAE,CAAC;IACnB,IAAI,IAAI,GAAgB,IAAI,CAAC;IAE7B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC,CAAC,MAAM,UAAU,EAAE,CAAC;QACzE,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC;YAC3B,SAAS,EAAE,kBAAkB;SAC9B,CAAC,CAAC;QAEH,iCAAiC;QACjC,IAAI,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,0BAA0B;QAC1B,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;gBACzC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACzD,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE3C,IAAI,CAAC,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACjC,MAAM,IAAI,YAAY,CAAC,0BAA0B,MAAM,EAAE,CAAC,CAAC;gBAC7D,CAAC;gBAED,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;oBACjB,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;oBACnB,GAAG;iBACJ,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACjD,CAAC;QAED,4FAA4F;QAC5F,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;gBACjC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;gBACpD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACpE,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,uCAAuC;YACvC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,6CAA6C;QAC7C,IAAI,gBAAoC,CAAC;QAEzC,MAAM,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YAC/B,MAAM,IAAK,CAAC,IAAI,CAAC,GAAG,EAAE;gBACpB,SAAS,EAAE,kBAAkB;gBAC7B,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,MAAM,IAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;YAED,mCAAmC;YACnC,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;gBACxD,MAAM,gBAAgB,GAAG,MAAM,cAAc,CAAC,IAAK,EAAE,OAAO,CAAC,CAAC;gBAC9D,IAAI,gBAAgB,EAAE,CAAC;oBACrB,gBAAgB,GAAG,gBAAgB,CAAC;gBACtC,CAAC;YACH,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAK,CAAC,GAAG,EAAE,CAAC;YAE7B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,6BAA6B,SAAS,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;QAEH,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;QAE9E,4BAA4B;QAC5B,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,aAAa;YACjD,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC/B,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;QAC/E,CAAC;QAED,wEAAwE;QACxE,IAAI,UAAU,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpC,gBAAgB,GAAG,MAAM,IAAK,CAAC,UAAU,CAAC;gBACxC,QAAQ,EAAE,kBAAkB;gBAC5B,IAAI,EAAE,KAAK;aACZ,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,IAAI;YACJ,GAAG,EAAE,QAAQ;YACb,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;YACpG,MAAM,KAAK,CAAC;QACd,CAAC;QAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,YAAY,CAAC,8BAA8B,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,IAAI,YAAY,CACpB,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CACpF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,oDAAoD;QACpD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrC,CAAC;QACD,gBAAgB,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,EAAoB,EACpB,cAAsB,CAAC,EACvB,cAAsB,IAAI;IAE1B,IAAI,SAAS,GAAiB,IAAI,CAAC;IAEnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;YAExE,4CAA4C;YAC5C,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACnE,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;gBACrD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,YAAY,CAAC,cAAc,CAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;QAC5B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;IACD,IAAI,oBAAoB,EAAE,CAAC;QACzB,MAAM,oBAAoB,CAAC,KAAK,EAAE,CAAC;QACnC,oBAAoB,GAAG,IAAI,CAAC;IAC9B,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Domain URL mapping
|
|
3
|
+
* Combines sitemap discovery with link crawling to discover all URLs on a domain
|
|
4
|
+
*/
|
|
5
|
+
export interface MapOptions {
|
|
6
|
+
/** Include sitemap URLs (default: true) */
|
|
7
|
+
useSitemap?: boolean;
|
|
8
|
+
/** Crawl the homepage for additional links (default: true) */
|
|
9
|
+
crawlHomepage?: boolean;
|
|
10
|
+
/** Maximum URLs to discover (default: 5000) */
|
|
11
|
+
maxUrls?: number;
|
|
12
|
+
/** Timeout per request in ms (default: 10000) */
|
|
13
|
+
timeout?: number;
|
|
14
|
+
/** Include URL patterns matching these regexes only */
|
|
15
|
+
includePatterns?: string[];
|
|
16
|
+
/** Exclude URL patterns matching these regexes */
|
|
17
|
+
excludePatterns?: string[];
|
|
18
|
+
}
|
|
19
|
+
export interface MapResult {
|
|
20
|
+
/** All discovered URLs (deduplicated) */
|
|
21
|
+
urls: string[];
|
|
22
|
+
/** Sitemap URLs used */
|
|
23
|
+
sitemapUrls: string[];
|
|
24
|
+
/** Total URLs discovered */
|
|
25
|
+
total: number;
|
|
26
|
+
/** Time elapsed in ms */
|
|
27
|
+
elapsed: number;
|
|
28
|
+
}
|
|
29
|
+
export declare function mapDomain(startUrl: string, options?: MapOptions): Promise<MapResult>;
|
|
30
|
+
//# sourceMappingURL=map.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"map.d.ts","sourceRoot":"","sources":["../../src/core/map.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,MAAM,WAAW,UAAU;IACzB,2CAA2C;IAC3C,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,8DAA8D;IAC9D,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,kDAAkD;IAClD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,SAAS;IACxB,yCAAyC;IACzC,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,wBAAwB;IACxB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,SAAS,CAAC,CA4D9F"}
|
package/dist/core/map.js
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Domain URL mapping
|
|
3
|
+
* Combines sitemap discovery with link crawling to discover all URLs on a domain
|
|
4
|
+
*/
|
|
5
|
+
import { discoverSitemap } from './sitemap.js';
|
|
6
|
+
import { peel } from '../index.js';
|
|
7
|
+
export async function mapDomain(startUrl, options = {}) {
|
|
8
|
+
const startTime = Date.now();
|
|
9
|
+
const { useSitemap = true, crawlHomepage = true, maxUrls = 5000, timeout = 10000, includePatterns = [], excludePatterns = [], } = options;
|
|
10
|
+
const urlObj = new URL(startUrl);
|
|
11
|
+
const domain = urlObj.hostname;
|
|
12
|
+
const allUrls = new Set();
|
|
13
|
+
let sitemapUrls = [];
|
|
14
|
+
// Compile filter patterns
|
|
15
|
+
const includeRegexes = includePatterns.map(p => new RegExp(p));
|
|
16
|
+
const excludeRegexes = excludePatterns.map(p => new RegExp(p));
|
|
17
|
+
function shouldInclude(url) {
|
|
18
|
+
if (excludeRegexes.some(r => r.test(url)))
|
|
19
|
+
return false;
|
|
20
|
+
if (includeRegexes.length > 0 && !includeRegexes.some(r => r.test(url)))
|
|
21
|
+
return false;
|
|
22
|
+
return true;
|
|
23
|
+
}
|
|
24
|
+
// Step 1: Sitemap discovery
|
|
25
|
+
if (useSitemap) {
|
|
26
|
+
const sitemap = await discoverSitemap(domain, { timeout, maxUrls });
|
|
27
|
+
sitemapUrls = sitemap.sitemapUrls;
|
|
28
|
+
for (const entry of sitemap.urls) {
|
|
29
|
+
if (allUrls.size >= maxUrls)
|
|
30
|
+
break;
|
|
31
|
+
if (shouldInclude(entry.url)) {
|
|
32
|
+
allUrls.add(entry.url);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
// Step 2: Crawl homepage for additional links
|
|
37
|
+
if (crawlHomepage && allUrls.size < maxUrls) {
|
|
38
|
+
try {
|
|
39
|
+
const result = await peel(startUrl, { timeout });
|
|
40
|
+
for (const link of result.links) {
|
|
41
|
+
if (allUrls.size >= maxUrls)
|
|
42
|
+
break;
|
|
43
|
+
try {
|
|
44
|
+
const linkUrl = new URL(link);
|
|
45
|
+
if (linkUrl.hostname === domain && shouldInclude(link)) {
|
|
46
|
+
allUrls.add(link);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
catch { /* skip invalid URLs */ }
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
catch { /* skip homepage crawl errors */ }
|
|
53
|
+
}
|
|
54
|
+
return {
|
|
55
|
+
urls: Array.from(allUrls).sort(),
|
|
56
|
+
sitemapUrls,
|
|
57
|
+
total: allUrls.size,
|
|
58
|
+
elapsed: Date.now() - startTime,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=map.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"map.js","sourceRoot":"","sources":["../../src/core/map.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AA4BnC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,UAAsB,EAAE;IACxE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,EACJ,UAAU,GAAG,IAAI,EACjB,aAAa,GAAG,IAAI,EACpB,OAAO,GAAG,IAAI,EACd,OAAO,GAAG,KAAK,EACf,eAAe,GAAG,EAAE,EACpB,eAAe,GAAG,EAAE,GACrB,GAAG,OAAO,CAAC;IAEZ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,IAAI,WAAW,GAAa,EAAE,CAAC;IAE/B,0BAA0B;IAC1B,MAAM,cAAc,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,cAAc,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/D,SAAS,aAAa,CAAC,GAAW;QAChC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACxD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACtF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4BAA4B;IAC5B,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QACpE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO;gBAAE,MAAM;YACnC,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,IAAI,aAAa,IAAI,OAAO,CAAC,IAAI,GAAG,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACjD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO;oBAAE,MAAM;gBACnC,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC9B,IAAI,OAAO,CAAC,QAAQ,KAAK,MAAM,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;wBACvD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACpB,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC,CAAC,uBAAuB,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,gCAAgC,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QAChC,WAAW;QACX,KAAK,EAAE,OAAO,CAAC,IAAI;QACnB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;KAChC,CAAC;AACJ,CAAC"}
|
package/dist/core/markdown.d.ts
CHANGED
|
@@ -35,4 +35,9 @@ export declare function htmlToText(html: string): string;
|
|
|
35
35
|
* Rule of thumb: 1 token ≈ 4 characters for English text
|
|
36
36
|
*/
|
|
37
37
|
export declare function estimateTokens(text: string): number;
|
|
38
|
+
/**
|
|
39
|
+
* Truncate content to fit within a token budget
|
|
40
|
+
* Intelligently preserves structure (headings, first paragraph)
|
|
41
|
+
*/
|
|
42
|
+
export declare function truncateToTokenBudget(content: string, maxTokens: number): string;
|
|
38
43
|
//# sourceMappingURL=markdown.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AAuCH;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAiBxF;AAmDD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAE,CAqCnF;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,CAqC9E;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE;IAAE,GAAG,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,MAAM,CA4DjF;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAuB/C;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD"}
|
|
1
|
+
{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AAuCH;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAiBxF;AAmDD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAE,CAqCnF;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,CAqC9E;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE;IAAE,GAAG,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,MAAM,CA4DjF;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAuB/C;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CA4ChF"}
|