webpeel 0.1.0

package/dist/core/fetcher.js

@@ -0,0 +1,479 @@
+/**
+ * Core fetching logic: simple HTTP and browser-based fetching
+ */
+import { chromium } from 'playwright';
+import { TimeoutError, BlockedError, NetworkError, WebPeelError } from '../types.js';
+const USER_AGENTS = [
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+];
+function getRandomUserAgent() {
+    return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
+}
+/**
+ * SECURITY: Validate URL to prevent SSRF attacks
+ * Blocks localhost, private IPs, link-local, and various bypass techniques
+ */
+function validateUrl(urlString) {
+    // Length check
+    if (urlString.length > 2048) {
+        throw new WebPeelError('URL too long (max 2048 characters)');
+    }
+    // Check for control characters and suspicious encoding
+    if (/[\x00-\x1F\x7F]/.test(urlString)) {
+        throw new WebPeelError('URL contains invalid control characters');
+    }
+    let url;
+    try {
+        url = new URL(urlString);
+    }
+    catch {
+        throw new WebPeelError('Invalid URL format');
+    }
+    // Only allow HTTP(S)
+    if (!['http:', 'https:'].includes(url.protocol)) {
+        throw new WebPeelError('Only HTTP and HTTPS protocols are allowed');
+    }
+    // Validate hostname is not empty
+    if (!url.hostname) {
+        throw new WebPeelError('Invalid hostname');
+    }
+    const hostname = url.hostname.toLowerCase();
+    // Block localhost patterns
+    const localhostPatterns = ['localhost', '0.0.0.0'];
+    if (localhostPatterns.some(pattern => hostname === pattern || hostname.endsWith('.' + pattern))) {
+        throw new WebPeelError('Access to localhost is not allowed');
+    }
+    // ENHANCED: Parse and validate IP addresses (handles hex, octal, decimal, mixed)
+    const ipv4Info = parseAndValidateIPv4(hostname);
+    if (ipv4Info) {
+        validateIPv4Address(ipv4Info);
+    }
+    // ENHANCED: Comprehensive IPv6 validation
+    if (hostname.includes(':')) {
+        validateIPv6Address(hostname);
+    }
+}
+/**
+ * Parse IPv4 address in any format (dotted, hex, octal, decimal, mixed)
+ * Returns null if not an IPv4 address
+ */
+function parseAndValidateIPv4(hostname) {
+    // Remove brackets if present
+    const cleaned = hostname.replace(/^\[|\]$/g, '');
+    // Standard dotted notation: 192.168.1.1
+    const dottedRegex = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/;
+    const dottedMatch = cleaned.match(dottedRegex);
+    if (dottedMatch) {
+        const octets = dottedMatch.slice(1).map(Number);
+        if (octets.every(o => o >= 0 && o <= 255)) {
+            return octets;
+        }
+        throw new WebPeelError('Invalid IPv4 address');
+    }
+    // Hex notation: 0x7f000001
+    if (/^0x[0-9a-fA-F]+$/.test(cleaned)) {
+        const num = parseInt(cleaned, 16);
+        return [
+            (num >>> 24) & 0xff,
+            (num >>> 16) & 0xff,
+            (num >>> 8) & 0xff,
+            num & 0xff,
+        ];
+    }
+    // Octal notation: 0177.0.0.1 or full octal 017700000001
+    if (/^0[0-7]/.test(cleaned)) {
+        // Full octal (all digits)
+        if (/^0[0-7]+$/.test(cleaned)) {
+            const num = parseInt(cleaned, 8);
+            if (num <= 0xffffffff) {
+                return [
+                    (num >>> 24) & 0xff,
+                    (num >>> 16) & 0xff,
+                    (num >>> 8) & 0xff,
+                    num & 0xff,
+                ];
+            }
+        }
+        // Mixed octal-decimal: 0177.0.0.1
+        const parts = cleaned.split('.');
+        if (parts.length === 4) {
+            const octets = parts.map(p => parseInt(p, /^0[0-7]/.test(p) ? 8 : 10));
+            if (octets.every(o => o >= 0 && o <= 255)) {
+                return octets;
+            }
+        }
+    }
+    // Decimal notation: 2130706433
+    if (/^\d+$/.test(cleaned)) {
+        const num = parseInt(cleaned, 10);
+        if (num <= 0xffffffff) {
+            return [
+                (num >>> 24) & 0xff,
+                (num >>> 16) & 0xff,
+                (num >>> 8) & 0xff,
+                num & 0xff,
+            ];
+        }
+    }
+    return null;
+}
+/**
+ * Validate IPv4 address against private/reserved ranges
+ */
+function validateIPv4Address(octets) {
+    const [a, b, c, d] = octets;
+    // Loopback: 127.0.0.0/8
+    if (a === 127) {
+        throw new WebPeelError('Access to loopback addresses is not allowed');
+    }
+    // Private: 10.0.0.0/8
+    if (a === 10) {
+        throw new WebPeelError('Access to private IP addresses is not allowed');
+    }
+    // Private: 172.16.0.0/12
+    if (a === 172 && b >= 16 && b <= 31) {
+        throw new WebPeelError('Access to private IP addresses is not allowed');
+    }
+    // Private: 192.168.0.0/16
+    if (a === 192 && b === 168) {
+        throw new WebPeelError('Access to private IP addresses is not allowed');
+    }
+    // Link-local: 169.254.0.0/16
+    if (a === 169 && b === 254) {
+        throw new WebPeelError('Access to link-local addresses is not allowed');
+    }
+    // Broadcast: 255.255.255.255
+    if (a === 255 && b === 255 && c === 255 && d === 255) {
+        throw new WebPeelError('Access to broadcast address is not allowed');
+    }
+    // This network: 0.0.0.0/8
+    if (a === 0) {
+        throw new WebPeelError('Access to "this network" addresses is not allowed');
+    }
+}
+/**
+ * Validate IPv6 address against private/reserved ranges
+ */
+function validateIPv6Address(hostname) {
+    // Remove brackets
+    const addr = hostname.replace(/^\[|\]$/g, '').toLowerCase();
+    // Loopback: ::1
+    if (addr === '::1' || addr === '0:0:0:0:0:0:0:1') {
+        throw new WebPeelError('Access to loopback addresses is not allowed');
+    }
+    // IPv6 mapped IPv4: ::ffff:192.168.1.1 or ::ffff:c0a8:0101
+    if (addr.startsWith('::ffff:')) {
+        // Extract the IPv4 part
+        const ipv4Part = addr.substring(7);
+        // Could be dotted (::ffff:192.168.1.1) or hex (::ffff:c0a8:0101)
+        if (ipv4Part.includes('.')) {
+            // Parse dotted IPv4
+            const parts = ipv4Part.split('.');
+            if (parts.length === 4) {
+                const octets = parts.map(p => parseInt(p, 10));
+                if (octets.every(o => !isNaN(o) && o >= 0 && o <= 255)) {
+                    validateIPv4Address(octets);
+                }
+            }
+        }
+        else {
+            // Parse hex IPv4 (e.g., c0a80101 = 192.168.1.1)
+            const hexStr = ipv4Part.replace(/:/g, '');
+            if (/^[0-9a-f]{1,8}$/.test(hexStr)) {
+                const num = parseInt(hexStr, 16);
+                const octets = [
+                    (num >>> 24) & 0xff,
+                    (num >>> 16) & 0xff,
+                    (num >>> 8) & 0xff,
+                    num & 0xff,
+                ];
+                validateIPv4Address(octets);
+            }
+        }
+        throw new WebPeelError('Access to IPv6-mapped IPv4 addresses is not allowed');
+    }
+    // Unique local addresses: fc00::/7 (fc00:: to fdff::)
+    if (addr.startsWith('fc') || addr.startsWith('fd')) {
+        throw new WebPeelError('Access to unique local IPv6 addresses is not allowed');
+    }
+    // Link-local: fe80::/10
+    if (addr.startsWith('fe8') || addr.startsWith('fe9') ||
+        addr.startsWith('fea') || addr.startsWith('feb')) {
+        throw new WebPeelError('Access to link-local IPv6 addresses is not allowed');
+    }
+}
+/**
+ * Validate and sanitize user agent string
+ */
+function validateUserAgent(userAgent) {
+    if (userAgent.length > 500) {
+        throw new WebPeelError('User agent too long (max 500 characters)');
+    }
+    // Allow only printable ASCII characters
+    if (!/^[\x20-\x7E]*$/.test(userAgent)) {
+        throw new WebPeelError('User agent contains invalid characters');
+    }
+    return userAgent;
+}
+/**
+ * Simple HTTP fetch using native fetch + Cheerio
+ * Fast and lightweight, but can be blocked by Cloudflare/bot detection
+ * SECURITY: Manual redirect handling with SSRF re-validation
+ */
+export async function simpleFetch(url, userAgent, timeoutMs = 30000) {
+    // SECURITY: Validate URL to prevent SSRF
+    validateUrl(url);
+    // Validate user agent if provided
+    const validatedUserAgent = userAgent ? validateUserAgent(userAgent) : getRandomUserAgent();
+    const MAX_REDIRECTS = 10;
+    let redirectCount = 0;
+    let currentUrl = url;
+    const seenUrls = new Set();
+    while (redirectCount <= MAX_REDIRECTS) {
+        // Detect redirect loops
+        if (seenUrls.has(currentUrl)) {
+            throw new WebPeelError('Redirect loop detected');
+        }
+        seenUrls.add(currentUrl);
+        // Re-validate on each redirect
+        validateUrl(currentUrl);
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), timeoutMs);
+        try {
+            const response = await fetch(currentUrl, {
+                headers: {
+                    'User-Agent': validatedUserAgent,
+                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+                    'Accept-Language': 'en-US,en;q=0.9',
+                    'Accept-Encoding': 'gzip, deflate, br',
+                    'DNT': '1',
+                    'Connection': 'keep-alive',
+                    'Upgrade-Insecure-Requests': '1',
+                },
+                signal: controller.signal,
+                redirect: 'manual', // SECURITY: Manual redirect handling
+            });
+            clearTimeout(timer);
+            // Handle redirects manually
+            if (response.status >= 300 && response.status < 400) {
+                const location = response.headers.get('location');
+                if (!location) {
+                    throw new NetworkError('Redirect response missing Location header');
+                }
+                // Resolve relative URLs
+                currentUrl = new URL(location, currentUrl).href;
+                redirectCount++;
+                continue;
+            }
+            if (!response.ok) {
+                if (response.status === 403 || response.status === 503) {
+                    throw new BlockedError(`HTTP ${response.status}: Site may be blocking requests. Try --render for browser mode.`);
+                }
+                throw new NetworkError(`HTTP ${response.status}: ${response.statusText}`);
+            }
+            // SECURITY: Validate Content-Type
+            const contentType = response.headers.get('content-type') || '';
+            if (!contentType.includes('text/html') && !contentType.includes('application/xhtml+xml')) {
+                throw new WebPeelError('Unsupported content type. Only HTML is supported.');
+            }
+            // SECURITY: Stream response with size limit (prevent memory exhaustion)
+            const chunks = [];
+            let totalSize = 0;
+            const MAX_SIZE = 10 * 1024 * 1024; // 10MB
+            const reader = response.body?.getReader();
+            if (!reader) {
+                throw new NetworkError('Response body is not readable');
+            }
+            try {
+                while (true) {
+                    const { done, value } = await reader.read();
+                    if (done)
+                        break;
+                    totalSize += value.length;
+                    if (totalSize > MAX_SIZE) {
+                        reader.cancel();
+                        throw new WebPeelError('Response too large (max 10MB)');
+                    }
+                    chunks.push(value);
+                }
+            }
+            finally {
+                reader.releaseLock();
+            }
+            // Combine chunks
+            const combined = new Uint8Array(totalSize);
+            let offset = 0;
+            for (const chunk of chunks) {
+                combined.set(chunk, offset);
+                offset += chunk.length;
+            }
+            const html = new TextDecoder().decode(combined);
+            if (!html || html.length < 100) {
+                throw new BlockedError('Empty or suspiciously small response. Site may require JavaScript.');
+            }
+            // Check for Cloudflare challenge
+            if (html.includes('cf-browser-verification') || html.includes('Just a moment...')) {
+                throw new BlockedError('Cloudflare challenge detected. Try --render for browser mode.');
+            }
+            return {
+                html,
+                url: currentUrl,
+                statusCode: response.status,
+            };
+        }
+        catch (error) {
+            clearTimeout(timer);
+            if (error instanceof BlockedError || error instanceof NetworkError || error instanceof WebPeelError) {
+                throw error;
+            }
+            if (error instanceof Error && error.name === 'AbortError') {
+                throw new TimeoutError(`Request timed out after ${timeoutMs}ms`);
+            }
+            throw new NetworkError(`Failed to fetch: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    throw new WebPeelError(`Too many redirects (max ${MAX_REDIRECTS})`);
+}
+let sharedBrowser = null;
+let activePagesCount = 0;
+const MAX_CONCURRENT_PAGES = 5;
+async function getBrowser() {
+    // SECURITY: Check if browser is still connected and healthy
+    if (sharedBrowser) {
+        try {
+            if (sharedBrowser.isConnected()) {
+                return sharedBrowser;
+            }
+        }
+        catch {
+            // Browser is dead, recreate
+            sharedBrowser = null;
+        }
+    }
+    sharedBrowser = await chromium.launch({ headless: true });
+    return sharedBrowser;
+}
+/**
+ * Fetch using headless Chromium via Playwright
+ * Slower but can handle JavaScript-heavy sites and bypass some bot detection
+ */
+export async function browserFetch(url, options = {}) {
+    // SECURITY: Validate URL to prevent SSRF
+    validateUrl(url);
+    const { userAgent, waitMs = 0, timeoutMs = 30000 } = options;
+    // Validate user agent if provided
+    const validatedUserAgent = userAgent ? validateUserAgent(userAgent) : getRandomUserAgent();
+    // Validate wait time
+    if (waitMs < 0 || waitMs > 60000) {
+        throw new WebPeelError('Wait time must be between 0 and 60000ms');
+    }
+    // SECURITY: Limit concurrent browser pages with timeout
+    const queueStartTime = Date.now();
+    const QUEUE_TIMEOUT_MS = 30000; // 30 second max wait
+    while (activePagesCount >= MAX_CONCURRENT_PAGES) {
+        if (Date.now() - queueStartTime > QUEUE_TIMEOUT_MS) {
+            throw new TimeoutError('Browser page queue timeout - too many concurrent requests');
+        }
+        await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    activePagesCount++;
+    let page = null;
+    try {
+        const browser = await getBrowser();
+        page = await browser.newPage({
+            userAgent: validatedUserAgent,
+        });
+        // Block images, fonts, and other heavy resources for speed
+        await page.route('**/*', (route) => {
+            const resourceType = route.request().resourceType();
+            if (['image', 'font', 'media', 'stylesheet'].includes(resourceType)) {
+                route.abort();
+            }
+            else {
+                route.continue();
+            }
+        });
+        // SECURITY: Wrap entire operation in timeout
+        const fetchPromise = (async () => {
+            await page.goto(url, {
+                waitUntil: 'domcontentloaded',
+                timeout: timeoutMs,
+            });
+            // Wait for additional time if requested (for dynamic content)
+            if (waitMs > 0) {
+                await page.waitForTimeout(waitMs);
+            }
+            const html = await page.content();
+            const finalUrl = page.url();
+            return { html, finalUrl };
+        })();
+        const timeoutPromise = new Promise((_, reject) => {
+            setTimeout(() => reject(new TimeoutError(`Operation timed out after ${timeoutMs}ms`)), timeoutMs);
+        });
+        const { html, finalUrl } = await Promise.race([fetchPromise, timeoutPromise]);
+        // SECURITY: Limit HTML size
+        if (html.length > 10 * 1024 * 1024) { // 10MB limit
+            throw new WebPeelError('Response too large (max 10MB)');
+        }
+        if (!html || html.length < 100) {
+            throw new BlockedError('Empty or suspiciously small response from browser.');
+        }
+        return {
+            html,
+            url: finalUrl,
+        };
+    }
+    catch (error) {
+        if (error instanceof BlockedError || error instanceof WebPeelError || error instanceof TimeoutError) {
+            throw error;
+        }
+        if (error instanceof Error && error.message.includes('Timeout')) {
+            throw new TimeoutError(`Browser navigation timed out`);
+        }
+        throw new NetworkError(`Browser fetch failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+    finally {
+        // CRITICAL: Always close page and decrement counter
+        if (page) {
+            await page.close().catch(() => { });
+        }
+        activePagesCount--;
+    }
+}
+/**
+ * Retry a fetch operation with exponential backoff
+ */
+export async function retryFetch(fn, maxAttempts = 3, baseDelayMs = 1000) {
+    let lastError = null;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new Error('Unknown error');
+            // Don't retry on blocked errors or timeouts
+            if (error instanceof BlockedError || error instanceof TimeoutError) {
+                throw error;
+            }
+            if (attempt < maxAttempts) {
+                const delay = baseDelayMs * Math.pow(2, attempt - 1);
+                await new Promise((resolve) => setTimeout(resolve, delay));
+            }
+        }
+    }
+    throw lastError || new NetworkError('Retry failed');
+}
+/**
+ * Clean up browser resources
+ */
+export async function cleanup() {
+    if (sharedBrowser) {
+        await sharedBrowser.close();
+        sharedBrowser = null;
+    }
+}
+//# sourceMappingURL=fetcher.js.map

package/dist/core/fetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAA2B,MAAM,YAAY,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAErF,MAAM,WAAW,GAAG;IAClB,uHAAuH;IACvH,iHAAiH;IACjH,uHAAuH;IACvH,kFAAkF;IAClF,uGAAuG;CACxG,CAAC;AAEF,SAAS,kBAAkB;IACzB,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,SAAiB;IACpC,eAAe;IACf,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,uDAAuD;IACvD,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAED,qBAAqB;IACrB,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;IACtE,CAAC;IAED,iCAAiC;IACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,YAAY,CAAC,kBAAkB,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,2BAA2B;IAC3B,MAAM,iBAAiB,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACnD,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,iFAAiF;IACjF,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,QAAQ,EAAE,CAAC;QACb,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,0CAA0C;IAC1C,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,6BAA6B;IAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAEjD,wCAAwC;IACxC,MAAM,WAAW,GAAG,8CAA8C,CAAC;IACnE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1C,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,CAAC;IACjD,CAAC;IAED,2BAA2B;IAC3B,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO;YACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;YAClB,GAAG,GAAG,IAAI;SACX,CAAC;IACJ,CAAC;IAED,wDAAwD;IACxD,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,0BAA0B;QAC1B,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;gBACtB,OAAO;oBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;YACJ,CAAC;QACH,CAAC;QACD,kCAAkC;QAClC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;YACtB,OAAO;gBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;gBAClB,GAAG,GAAG,IAAI;aACX,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAgB;IAC3C,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;IAE5B,wBAAwB;IACxB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,sBAAsB;IACtB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,4CAA4C,CAAC,CAAC;IACvE,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,MAAM,IAAI,YAAY,CAAC,mDAAmD,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,kBAAkB;IAClB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAE5D,gBAAgB;IAChB,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;QACjD,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,2DAA2D;IAC3D,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,wBAAwB;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAEnC,iEAAiE;QACjE,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,oBAAoB;YACpB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC/C,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBACvD,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACjC,MAAM,MAAM,GAAG;oBACb,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;gBACF,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,qDAAqD,CAAC,CAAC;IAChF,CAAC;IAED,sDAAsD;IACtD,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,YAAY,CAAC,sDAAsD,CAAC,CAAC;IACjF,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,0CAA0C,CAAC,CAAC;IACrE,CAAC;IACD,wCAAwC;IACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAQD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAW,EACX,SAAkB,EAClB,YAAoB,KAAK;IAEzB,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,GAAG,CAAC;IACrB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,OAAO,aAAa,IAAI,aAAa,EAAE,CAAC;QACtC,wBAAwB;QACxB,IAAI,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,YAAY,CAAC,wBAAwB,CAAC,CAAC;QACnD,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEzB,+BAA+B;QAC/B,WAAW,CAAC,UAAU,CAAC,CAAC;QAExB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;QAE9D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;gBACvC,OAAO,EAAE;oBACP,YAAY,EAAE,kBAAkB;oBAChC,QAAQ,EAAE,4EAA4E;oBACtF,iBAAiB,EAAE,gBAAgB;oBACnC,iBAAiB,EAAE,mBAAmB;oBACtC,KAAK,EAAE,GAAG;oBACV,YAAY,EAAE,YAAY;oBAC1B,2BAA2B,EAAE,GAAG;iBACjC;gBACD,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ,EAAE,qCAAqC;aAC1D,CAAC,CAAC;YAEH,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,4BAA4B;YAC5B,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;gBACtE,CAAC;gBAED,wBAAwB;gBACxB,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC;gBAChD,aAAa,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACvD,MAAM,IAAI,YAAY,CACpB,QAAQ,QAAQ,CAAC,MAAM,iEAAiE,CACzF,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,YAAY,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC5E,CAAC;YAED,kCAAkC;YAClC,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC;gBACzF,MAAM,IAAI,YAAY,CAAC,mDAAmD,CAAC,CAAC;YAC9E,CAAC;YAED,wEAAwE;YACxE,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;YAE1C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,IAAI,EAAE,CAAC;oBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,MAAM;oBAEhB,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;oBAC1B,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;wBACzB,MAAM,CAAC,MAAM,EAAE,CAAC;wBAChB,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;oBAC1D,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,CAAC;YAED,iBAAiB;YACjB,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEhD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC/B,MAAM,IAAI,YAAY,CAAC,oEAAoE,CAAC,CAAC;YAC/F,CAAC;YAED,iCAAiC;YACjC,IAAI,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,YAAY,CAAC,+DAA+D,CAAC,CAAC;YAC1F,CAAC;YAED,OAAO;gBACL,IAAI;gBACJ,GAAG,EAAE,UAAU;gBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;aAC5B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACpG,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,YAAY,CAAC,2BAA2B,SAAS,IAAI,CAAC,CAAC;YACnE,CAAC;YAED,MAAM,IAAI,YAAY,CAAC,oBAAoB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;IAED,MAAM,IAAI,YAAY,CAAC,2BAA2B,aAAa,GAAG,CAAC,CAAC;AACtE,CAAC;AAED,IAAI,aAAa,GAAmB,IAAI,CAAC;AACzC,IAAI,gBAAgB,GAAG,CAAC,CAAC;AACzB,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,KAAK,UAAU,UAAU;IACvB,4DAA4D;IAC5D,IAAI,aAAa,EAAE,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC;gBAChC,OAAO,aAAa,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;YAC5B,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,aAAa,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,UAII,EAAE;IAEN,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAE7D,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,qBAAqB;IACrB,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;QACjC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,wDAAwD;IACxD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,gBAAgB,GAAG,KAAK,CAAC,CAAC,qBAAqB;IAErD,OAAO,gBAAgB,IAAI,oBAAoB,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,GAAG,gBAAgB,EAAE,CAAC;YACnD,MAAM,IAAI,YAAY,CAAC,2DAA2D,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,gBAAgB,EAAE,CAAC;IACnB,IAAI,IAAI,GAAgB,IAAI,CAAC;IAE7B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;QACnC,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC;YAC3B,SAAS,EAAE,kBAAkB;SAC9B,CAAC,CAAC;QAEH,2DAA2D;QAC3D,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;YACpD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACpE,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,6CAA6C;QAC7C,MAAM,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YAC/B,MAAM,IAAK,CAAC,IAAI,CAAC,GAAG,EAAE;gBACpB,SAAS,EAAE,kBAAkB;gBAC7B,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,MAAM,IAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAK,CAAC,GAAG,EAAE,CAAC;YAE7B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,6BAA6B,SAAS,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;QAEH,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;QAE9E,4BAA4B;QAC5B,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,aAAa;YACjD,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC/B,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;QAC/E,CAAC;QAED,OAAO;YACL,IAAI;YACJ,GAAG,EAAE,QAAQ;SACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;YACpG,MAAM,KAAK,CAAC;QACd,CAAC;QAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,YAAY,CAAC,8BAA8B,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,IAAI,YAAY,CACpB,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CACpF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,oDAAoD;QACpD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrC,CAAC;QACD,gBAAgB,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,EAAoB,EACpB,cAAsB,CAAC,EACvB,cAAsB,IAAI;IAE1B,IAAI,SAAS,GAAiB,IAAI,CAAC;IAEnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;YAExE,4CAA4C;YAC5C,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACnE,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;gBACrD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,YAAY,CAAC,cAAc,CAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;QAC5B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;AACH,CAAC"}

package/dist/core/markdown.d.ts

@@ -0,0 +1,17 @@
+/**
+ * HTML to Markdown conversion with smart cleanup
+ */
+/**
+ * Convert HTML to clean, readable Markdown
+ */
+export declare function htmlToMarkdown(html: string): string;
+/**
+ * Convert HTML to plain text (strip all formatting)
+ */
+export declare function htmlToText(html: string): string;
+/**
+ * Estimate token count (very rough approximation)
+ * Rule of thumb: 1 token ≈ 4 characters for English text
+ */
+export declare function estimateTokens(text: string): number;
+//# sourceMappingURL=markdown.d.ts.map

package/dist/core/markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AA8DH;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CA4DnD;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAuB/C;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD"}

package/dist/core/markdown.js

@@ -0,0 +1,143 @@
+/**
+ * HTML to Markdown conversion with smart cleanup
+ */
+import TurndownService from 'turndown';
+import * as cheerio from 'cheerio';
+const JUNK_SELECTORS = [
+    'script',
+    'style',
+    'nav',
+    'footer',
+    'header.site-header',
+    'aside',
+    '.sidebar',
+    '.advertisement',
+    '.ad',
+    '.cookie-banner',
+    '.cookie-notice',
+    '.newsletter-signup',
+    '.social-share',
+    '.related-posts',
+    '.comments',
+    '#comments',
+    '.cookie-consent',
+    '[class*="cookie"]',
+    '[id*="cookie"]',
+    '[class*="banner"]',
+    '[class*="popup"]',
+    '[class*="modal"]',
+];
+/**
+ * Clean HTML before conversion
+ * Remove navigation, ads, cookie banners, and other junk
+ */
+function cleanHTML(html) {
+    // SECURITY: Limit HTML size to prevent DoS
+    if (html.length > 10 * 1024 * 1024) { // 10MB
+        throw new Error('HTML too large to process (max 10MB)');
+    }
+    const $ = cheerio.load(html);
+    // Remove junk elements
+    JUNK_SELECTORS.forEach((selector) => {
+        $(selector).remove();
+    });
+    // Remove empty paragraphs and divs
+    $('p:empty, div:empty').remove();
+    // Remove elements with only whitespace
+    $('*').each((_, elem) => {
+        const $elem = $(elem);
+        const text = $elem.text().trim();
+        if (!text && $elem.children().length === 0) {
+            $elem.remove();
+        }
+    });
+    return $.html();
+}
+/**
+ * Convert HTML to clean, readable Markdown
+ */
+export function htmlToMarkdown(html) {
+    const cleanedHTML = cleanHTML(html);
+    const turndown = new TurndownService({
+        headingStyle: 'atx',
+        codeBlockStyle: 'fenced',
+        bulletListMarker: '-',
+        emDelimiter: '_',
+        strongDelimiter: '**',
+    });
+    // Preserve tables
+    turndown.keep(['table', 'thead', 'tbody', 'tr', 'th', 'td']);
+    // Custom rule: convert images to alt text or skip
+    turndown.addRule('images', {
+        filter: 'img',
+        replacement: (_content, node) => {
+            const alt = node.alt;
+            const src = node.src;
+            if (alt) {
+                return `![${alt}](${src})`;
+            }
+            return '';
+        },
+    });
+    // Custom rule: preserve code blocks
+    turndown.addRule('codeBlocks', {
+        filter: (node) => {
+            return node.nodeName === 'PRE' && node.firstChild?.nodeName === 'CODE';
+        },
+        replacement: (_content, node) => {
+            const codeNode = node.firstChild;
+            const className = codeNode.getAttribute('class') || '';
+            const language = className.match(/language-(\w+)/)?.[1] || '';
+            return '\n\n```' + language + '\n' + codeNode.textContent + '\n```\n\n';
+        },
+    });
+    let markdown = turndown.turndown(cleanedHTML);
+    // SECURITY: Protect against ReDoS - limit input size before regex
+    if (markdown.length > 1024 * 1024) { // 1MB limit for markdown
+        markdown = markdown.slice(0, 1024 * 1024);
+    }
+    // Clean up excessive newlines (use non-backtracking approach)
+    markdown = markdown.split('\n').reduce((acc, line, i, arr) => {
+        if (i === 0)
+            return line;
+        const prevEmpty = arr[i - 1].trim() === '';
+        const currEmpty = line.trim() === '';
+        if (prevEmpty && currEmpty)
+            return acc;
+        return acc + '\n' + line;
+    }, '');
+    // Remove leading/trailing whitespace
+    markdown = markdown.trim();
+    return markdown;
+}
+/**
+ * Convert HTML to plain text (strip all formatting)
+ */
+export function htmlToText(html) {
+    const cleanedHTML = cleanHTML(html);
+    const $ = cheerio.load(cleanedHTML);
+    // Get text content, preserving some structure
+    let text = '';
+    $('h1, h2, h3, h4, h5, h6, p, li').each((_, elem) => {
+        const content = $(elem).text().trim();
+        if (content) {
+            text += content + '\n\n';
+        }
+    });
+    // Fallback: if no structured content found, get all text
+    if (!text.trim()) {
+        text = $('body').text();
+    }
+    // Clean up excessive whitespace
+    text = text.replace(/\n{3,}/g, '\n\n');
+    text = text.replace(/[ \t]+/g, ' ');
+    return text.trim();
+}
+/**
+ * Estimate token count (very rough approximation)
+ * Rule of thumb: 1 token ≈ 4 characters for English text
+ */
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+//# sourceMappingURL=markdown.js.map