webcontext-ai 1.0.0

package/dist/core/pipeline.js

@@ -0,0 +1,476 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CrawlPipeline = void 0;
+const manager_1 = require("../browser/manager");
+const content_1 = require("../extractors/content");
+const pdf_1 = require("../extractors/pdf");
+const github_1 = require("../extractors/github");
+const markdown_1 = require("../transformers/markdown");
+const chunker_1 = require("../chunking/chunker");
+const cache_1 = require("../cache/cache");
+const sitemap_1 = require("../utils/sitemap");
+const dedup_1 = require("../utils/dedup");
+const p_queue_1 = __importDefault(require("p-queue"));
+const crypto_1 = require("crypto");
+const cheerio = __importStar(require("cheerio"));
+const fs_1 = require("fs");
+const path_1 = require("path");
+/**
+ * Main crawling pipeline that orchestrates the full extraction workflow:
+ * URL → Fetch → Extract → Transform → Chunk → Output
+ */
+class CrawlPipeline {
+    browser;
+    extractor;
+    pdfExtractor;
+    githubExtractor;
+    transformer;
+    chunker;
+    cache;
+    sitemapParser;
+    dedup;
+    config;
+    constructor(config = {}) {
+        this.config = config;
+        this.browser = new manager_1.BrowserManager(config.browser, config.rateLimit);
+        this.extractor = new content_1.ContentExtractor();
+        this.pdfExtractor = new pdf_1.PdfExtractor();
+        this.githubExtractor = new github_1.GitHubExtractor();
+        this.transformer = new markdown_1.MarkdownTransformer({
+            preserveImages: config.extraction?.preserveImages,
+        });
+        this.chunker = new chunker_1.ContentChunker(config.chunking);
+        this.cache = new cache_1.CrawlCache({
+            enabled: config.cache?.enabled ?? true,
+            ttl: config.cache?.ttl ?? 3600,
+            maxSize: config.cache?.maxSize ?? 500,
+            directory: config.cache?.directory,
+            contentHashing: config.cache?.contentHashing ?? true,
+        });
+        this.sitemapParser = new sitemap_1.SitemapParser();
+        this.dedup = new dedup_1.Deduplicator();
+    }
+    async crawl(options) {
+        const startTime = Date.now();
+        const pages = [];
+        const errors = [];
+        const diffs = [];
+        const visited = new Set();
+        const maxPages = options.maxPages ?? 50;
+        const depth = options.depth ?? 0;
+        let cacheHits = 0;
+        let cacheMisses = 0;
+        let retries = 0;
+        // Load checkpoint if resuming
+        let checkpoint = null;
+        if (options.checkpoint && options.checkpointDir) {
+            checkpoint = this.loadCheckpoint(options.checkpointDir);
+            if (checkpoint) {
+                checkpoint.visitedUrls.forEach(u => visited.add(u));
+                pages.push(...checkpoint.pages);
+                errors.push(...checkpoint.errors);
+            }
+        }
+        // Discover URLs: sitemap or link-based crawling
+        let queue = [];
+        if (options.sitemapUrl) {
+            queue = await this.parseSitemap(options.sitemapUrl);
+        }
+        else if (checkpoint?.pendingUrls?.length) {
+            queue = checkpoint.pendingUrls;
+        }
+        else {
+            // Auto-discover sitemap before crawling
+            if (depth > 0) {
+                const sitemapUrl = await this.sitemapParser.discover(options.url).catch(() => null);
+                if (sitemapUrl) {
+                    const entries = await this.sitemapParser.parse(sitemapUrl).catch(() => []);
+                    if (entries.length > 0) {
+                        queue = entries
+                            .map(e => e.url)
+                            .filter(u => this.matchesPatterns(u, options.includePatterns, options.excludePatterns))
+                            .slice(0, maxPages);
+                    }
+                }
+            }
+            if (!queue.length)
+                queue = [options.url];
+        }
+        // Handle special sources: PDF and GitHub
+        if (this.pdfExtractor.isPdf(options.url)) {
+            return this.handlePdf(options, startTime);
+        }
+        if (this.githubExtractor.isGitHubUrl(options.url)) {
+            return this.handleGitHub(options, startTime);
+        }
+        // Initialize browser only if JS rendering is explicitly requested
+        if (options.javascript === true) {
+            await this.browser.launch();
+        }
+        const concurrency = this.config.concurrency ?? 3;
+        const pQueue = new p_queue_1.default({ concurrency });
+        const processUrl = async (url, currentDepth) => {
+            if (visited.has(url) || pages.length >= maxPages)
+                return;
+            visited.add(url);
+            // Progress callback
+            options.onProgress?.({
+                pagesProcessed: pages.length,
+                totalDiscovered: visited.size + queue.length,
+                currentUrl: url,
+                status: 'crawling',
+            });
+            try {
+                // Run pre-fetch plugins
+                let ctx = { url };
+                ctx = await this.runPlugins('pre-fetch', ctx, options.plugins);
+                // Check cache first
+                if (options.cache !== false) {
+                    const cached = this.cache.get(url);
+                    if (cached) {
+                        cacheHits++;
+                        pages.push(cached);
+                        // Check for content changes if hashing enabled
+                        if (this.config.cache?.contentHashing) {
+                            // We'll compare on next fresh fetch
+                        }
+                        // Still discover links for recursive crawl
+                        if (currentDepth < depth) {
+                            const newUrls = cached.links
+                                .filter(l => l.isInternal)
+                                .filter(l => this.matchesPatterns(l.href, options.includePatterns, options.excludePatterns))
+                                .filter(l => !visited.has(l.href))
+                                .map(l => l.href);
+                            for (const newUrl of newUrls.slice(0, maxPages - pages.length)) {
+                                pQueue.add(() => processUrl(newUrl, currentDepth + 1));
+                            }
+                        }
+                        return;
+                    }
+                    cacheMisses++;
+                }
+                // Fetch page
+                let html;
+                let status;
+                if (options.javascript === true) {
+                    const result = await this.browser.fetchPage(url, {
+                        respectRobots: options.respectRobotsTxt,
+                        cookies: options.cookies,
+                        headers: options.headers,
+                        retryConfig: options.retry,
+                    });
+                    html = result.content;
+                    status = result.status;
+                }
+                else {
+                    const result = await this.browser.fetchStatic(url, {
+                        respectRobots: options.respectRobotsTxt,
+                        headers: options.headers,
+                        retryConfig: options.retry,
+                    });
+                    html = result.body.toString('utf-8');
+                    status = result.status;
+                }
+                if (status >= 400) {
+                    errors.push({ url, error: `HTTP ${status}`, statusCode: status });
+                    return;
+                }
+                // Run post-fetch plugins
+                ctx = await this.runPlugins('post-fetch', { ...ctx, html }, options.plugins);
+                const finalHtml = ctx.html ?? html;
+                // Run pre-extract plugins
+                ctx = await this.runPlugins('pre-extract', ctx, options.plugins);
+                // Extract content
+                const extracted = this.extractor.extract(finalHtml, url, options.focusMode);
+                // Run post-extract plugins
+                ctx = await this.runPlugins('post-extract', { ...ctx, extracted }, options.plugins);
+                const finalExtracted = ctx.extracted ?? extracted;
+                // Run pre-transform plugins
+                ctx = await this.runPlugins('pre-transform', ctx, options.plugins);
+                // Transform to markdown
+                finalExtracted.markdown = this.transformer.transform(finalExtracted.html || '');
+                // Run post-transform plugins
+                ctx = await this.runPlugins('post-transform', { ...ctx, markdown: finalExtracted.markdown }, options.plugins);
+                if (ctx.markdown)
+                    finalExtracted.markdown = ctx.markdown;
+                // Resolve relative links to absolute URLs
+                finalExtracted.markdown = this.resolveLinks(finalExtracted.markdown, url);
+                // Deduplication check
+                const dupOf = this.dedup.isDuplicate(url, finalExtracted.text);
+                if (dupOf) {
+                    return; // Skip duplicate content
+                }
+                // Check content diff
+                if (options.cache !== false && this.config.cache?.contentHashing) {
+                    const diff = this.cache.hasChanged(url, finalExtracted.markdown);
+                    if (diff.changed)
+                        diffs.push(diff);
+                }
+                // Cache result
+                if (options.cache !== false) {
+                    this.cache.set(url, finalExtracted);
+                }
+                pages.push(finalExtracted);
+                // Queue internal links for recursive crawl
+                if (currentDepth < depth) {
+                    const newUrls = finalExtracted.links
+                        .filter(l => l.isInternal)
+                        .filter(l => this.matchesPatterns(l.href, options.includePatterns, options.excludePatterns))
+                        .filter(l => !visited.has(l.href))
+                        .map(l => l.href);
+                    for (const newUrl of newUrls.slice(0, maxPages - pages.length)) {
+                        pQueue.add(() => processUrl(newUrl, currentDepth + 1));
+                    }
+                }
+                // Respect delay
+                if (options.delay) {
+                    await new Promise(r => setTimeout(r, options.delay));
+                }
+                // Save checkpoint
+                if (options.checkpoint && options.checkpointDir) {
+                    this.saveCheckpoint({
+                        visitedUrls: [...visited],
+                        pendingUrls: queue.filter(u => !visited.has(u)),
+                        pages,
+                        errors,
+                        timestamp: new Date().toISOString(),
+                    }, options.checkpointDir);
+                }
+            }
+            catch (err) {
+                errors.push({ url, error: err.message });
+            }
+        };
+        // Process initial queue
+        for (const url of queue) {
+            if (pages.length >= maxPages)
+                break;
+            pQueue.add(() => processUrl(url, 0));
+        }
+        await pQueue.onIdle();
+        // Build chunks from all pages
+        const allChunks = [];
+        for (const page of pages) {
+            // Run pre-chunk plugins
+            let ctx = { page };
+            ctx = await this.runPlugins('pre-chunk', ctx, options.plugins);
+            const chunks = this.chunker.chunk(page.markdown, page.url, page.title, page.headings, this.config.chunking);
+            // Run post-chunk plugins
+            ctx = await this.runPlugins('post-chunk', { ...ctx, chunks }, options.plugins);
+            allChunks.push(...(ctx.chunks ?? chunks));
+        }
+        const relationships = this.buildRelationships(pages);
+        const totalTokens = allChunks.reduce((sum, c) => sum + c.tokens, 0);
+        const summary = this.generateSummary(pages);
+        const context = {
+            id: (0, crypto_1.createHash)('sha256').update(options.url + Date.now()).digest('hex').slice(0, 16),
+            source: options.url,
+            chunks: allChunks,
+            summary,
+            totalTokens,
+            metadata: {
+                crawledAt: new Date().toISOString(),
+                pageCount: pages.length,
+                contentType: pages[0]?.metadata.type || 'unknown',
+                framework: pages[0]?.metadata.framework,
+                version: pages[0]?.metadata.version,
+                relationships,
+            },
+            format: this.config.output?.format || 'markdown',
+        };
+        const stats = {
+            pagesProcessed: pages.length,
+            totalTokens,
+            duration: Date.now() - startTime,
+            errors,
+            cached: cacheHits,
+            cacheHits,
+            cacheMisses,
+            retries,
+        };
+        // Notify completion
+        options.onProgress?.({
+            pagesProcessed: pages.length,
+            totalDiscovered: visited.size,
+            currentUrl: '',
+            status: 'complete',
+        });
+        // Cleanup browser (chunker is reusable, disposed via WebContext.dispose())
+        await this.browser.close();
+        return { pages, context, stats, diffs: diffs.length ? diffs : undefined };
+    }
+    async parseSitemap(sitemapUrl) {
+        try {
+            const result = await this.browser.fetchStatic(sitemapUrl);
+            const xml = result.body.toString('utf-8');
+            const $ = cheerio.load(xml, { xmlMode: true });
+            return $('url > loc').map((_, el) => $(el).text()).get();
+        }
+        catch {
+            return [];
+        }
+    }
+    saveCheckpoint(state, dir) {
+        if (!(0, fs_1.existsSync)(dir))
+            (0, fs_1.mkdirSync)(dir, { recursive: true });
+        (0, fs_1.writeFileSync)((0, path_1.join)(dir, 'checkpoint.json'), JSON.stringify(state));
+    }
+    loadCheckpoint(dir) {
+        const file = (0, path_1.join)(dir, 'checkpoint.json');
+        if (!(0, fs_1.existsSync)(file))
+            return null;
+        try {
+            return JSON.parse((0, fs_1.readFileSync)(file, 'utf-8'));
+        }
+        catch {
+            return null;
+        }
+    }
+    async runPlugins(phase, ctx, plugins) {
+        if (!plugins?.length)
+            return ctx;
+        for (const plugin of plugins) {
+            if (plugin.hooks[phase]) {
+                ctx = (await plugin.hooks[phase](ctx)) || ctx;
+            }
+        }
+        return ctx;
+    }
+    generateSummary(pages) {
+        const combined = pages.map(p => p.text).join(' ');
+        const sentences = combined.match(/[^.!?]+[.!?]+/g) || [];
+        return sentences.slice(0, 3).join(' ').trim();
+    }
+    buildRelationships(pages) {
+        const relationships = [];
+        const urls = new Set(pages.map(p => p.url));
+        for (const page of pages) {
+            for (const link of page.links) {
+                if (urls.has(link.href) && link.href !== page.url) {
+                    relationships.push({ from: page.url, to: link.href, type: 'links-to' });
+                }
+            }
+        }
+        return relationships;
+    }
+    matchesPatterns(url, include, exclude) {
+        if (exclude?.some(p => url.includes(p)))
+            return false;
+        if (include?.length && !include.some(p => url.includes(p)))
+            return false;
+        return true;
+    }
+    /** Resolve relative markdown links to absolute URLs */
+    resolveLinks(markdown, baseUrl) {
+        return markdown.replace(/\[([^\]]*)\]\(([^)]+)\)/g, (match, text, href) => {
+            if (href.startsWith('http') || href.startsWith('#') || href.startsWith('mailto:'))
+                return match;
+            try {
+                const resolved = new URL(href, baseUrl).href;
+                return `[${text}](${resolved})`;
+            }
+            catch {
+                return match;
+            }
+        });
+    }
+    /** Handle PDF extraction */
+    async handlePdf(options, startTime) {
+        const extracted = await this.pdfExtractor.extract(options.url);
+        const chunks = this.chunker.chunk(extracted.markdown, extracted.url, extracted.title, extracted.headings, this.config.chunking);
+        const totalTokens = chunks.reduce((s, c) => s + c.tokens, 0);
+        return {
+            pages: [extracted],
+            context: {
+                id: (0, crypto_1.createHash)('sha256').update(options.url).digest('hex').slice(0, 16),
+                source: options.url,
+                chunks,
+                summary: extracted.description,
+                totalTokens,
+                metadata: {
+                    crawledAt: new Date().toISOString(),
+                    pageCount: 1,
+                    contentType: 'documentation',
+                    relationships: [],
+                },
+                format: 'markdown',
+            },
+            stats: {
+                pagesProcessed: 1, totalTokens, duration: Date.now() - startTime,
+                errors: [], cached: 0, cacheHits: 0, cacheMisses: 0, retries: 0,
+            },
+        };
+    }
+    /** Handle GitHub repository extraction */
+    async handleGitHub(options, startTime) {
+        const pages = [];
+        // Always get README
+        const readme = await this.githubExtractor.extractReadme(options.url);
+        pages.push(readme);
+        // If depth > 0, also get docs folder
+        if ((options.depth ?? 0) > 0) {
+            const docs = await this.githubExtractor.extractDocs(options.url);
+            pages.push(...docs.slice(0, (options.maxPages ?? 50) - 1));
+        }
+        const allChunks = [];
+        for (const page of pages) {
+            const chunks = this.chunker.chunk(page.markdown, page.url, page.title, page.headings, this.config.chunking);
+            allChunks.push(...chunks);
+        }
+        const totalTokens = allChunks.reduce((s, c) => s + c.tokens, 0);
+        return {
+            pages,
+            context: {
+                id: (0, crypto_1.createHash)('sha256').update(options.url).digest('hex').slice(0, 16),
+                source: options.url,
+                chunks: allChunks,
+                summary: pages[0]?.description,
+                totalTokens,
+                metadata: {
+                    crawledAt: new Date().toISOString(),
+                    pageCount: pages.length,
+                    contentType: 'readme',
+                    relationships: this.buildRelationships(pages),
+                },
+                format: 'markdown',
+            },
+            stats: {
+                pagesProcessed: pages.length, totalTokens, duration: Date.now() - startTime,
+                errors: [], cached: 0, cacheHits: 0, cacheMisses: 0, retries: 0,
+            },
+        };
+    }
+    /** Free resources (tiktoken encoder) */
+    dispose() {
+        this.chunker.dispose();
+        this.dedup.clear();
+    }
+}
+exports.CrawlPipeline = CrawlPipeline;
+//# sourceMappingURL=pipeline.js.map

package/dist/core/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/core/pipeline.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,gDAAoD;AACpD,mDAAyD;AACzD,2CAAiD;AACjD,iDAAuD;AACvD,uDAA+D;AAC/D,iDAAqD;AACrD,0CAA4C;AAC5C,8CAAiD;AACjD,0CAA8C;AAC9C,sDAA6B;AAC7B,mCAAoC;AACpC,iDAAmC;AACnC,2BAAwE;AACxE,+BAA4B;AAE5B;;;GAGG;AACH,MAAa,aAAa;IAChB,OAAO,CAAiB;IACxB,SAAS,CAAmB;IAC5B,YAAY,CAAe;IAC3B,eAAe,CAAkB;IACjC,WAAW,CAAsB;IACjC,OAAO,CAAiB;IACxB,KAAK,CAAa;IAClB,aAAa,CAAgB;IAC7B,KAAK,CAAe;IACpB,MAAM,CAAmB;IAEjC,YAAY,SAA2B,EAAE;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,IAAI,wBAAc,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QACpE,IAAI,CAAC,SAAS,GAAG,IAAI,0BAAgB,EAAE,CAAC;QACxC,IAAI,CAAC,YAAY,GAAG,IAAI,kBAAY,EAAE,CAAC;QACvC,IAAI,CAAC,eAAe,GAAG,IAAI,wBAAe,EAAE,CAAC;QAC7C,IAAI,CAAC,WAAW,GAAG,IAAI,8BAAmB,CAAC;YACzC,cAAc,EAAE,MAAM,CAAC,UAAU,EAAE,cAAc;SAClD,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,GAAG,IAAI,wBAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,kBAAU,CAAC;YAC1B,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI;YACtC,GAAG,EAAE,MAAM,CAAC,KAAK,EAAE,GAAG,IAAI,IAAI;YAC9B,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,GAAG;YACrC,SAAS,EAAE,MAAM,CAAC,KAAK,EAAE,SAAS;YAClC,cAAc,EAAE,MAAM,CAAC,KAAK,EAAE,cAAc,IAAI,IAAI;SACrD,CAAC,CAAC;QACH,IAAI,CAAC,aAAa,GAAG,IAAI,uBAAa,EAAE,CAAC;QACzC,IAAI,CAAC,KAAK,GAAG,IAAI,oBAAY,EAAE,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAqB;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAuB,EAAE,CAAC;QACrC,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC;QACjC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,8BAA8B;QAC9B,IAAI,UAAU,GAA2B,IAAI,CAAC;QAC9C,IAAI,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YAChD,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YACxD,IAAI,UAAU,EAAE,CAAC;gBACf,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpD,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QAED,gDAAgD;QAChD,IAAI,KAAK,GAAa,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACtD,CAAC;aAAM,IAAI,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;YAC3C,KAAK,GAAG,UAAU,CAAC,WAAW,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,wCAAwC;YACxC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;gBACpF,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC3E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACvB,KAAK,GAAG,OAAO;6BACZ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;6BACf,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,EAAE,OAAO,CAAC,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;6BACtF,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;oBACxB,CAAC;gBACH,CAAC;YACH,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,MAAM;gBAAE,KAAK,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC3C,CAAC;QAED,yCAAyC;QACzC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YACzC,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC/C,CAAC;QAED,kEAAkE;QAClE,IAAI,OAAO,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9B,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,iBAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC;QAE3C,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAAE,YAAoB,EAAE,EAAE;YAC7D,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;gBAAE,OAAO;YACzD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjB,oBAAoB;YACpB,OAAO,CAAC,UAAU,EAAE,CAAC;gBACnB,cAAc,EAAE,KAAK,CAAC,MAAM;gBAC5B,eAAe,EAAE,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,MAAM;gBAC5C,UAAU,EAAE,GAAG;gBACf,MAAM,EAAE,UAAU;aACnB,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,wBAAwB;gBACxB,IAAI,GAAG,GAAQ,EAAE,GAAG,EAAE,CAAC;gBACvB,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;gBAE/D,oBAAoB;gBACpB,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;oBAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnC,IAAI,MAAM,EAAE,CAAC;wBACX,SAAS,EAAE,CAAC;wBACZ,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBAEnB,+CAA+C;wBAC/C,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,cAAc,EAAE,CAAC;4BACtC,oCAAoC;wBACtC,CAAC;wBAED,2CAA2C;wBAC3C,IAAI,YAAY,GAAG,KAAK,EAAE,CAAC;4BACzB,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK;iCACzB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;iCACzB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;iCAC3F,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;iCACjC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;4BACpB,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;gCAC/D,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC;4BACzD,CAAC;wBACH,CAAC;wBACD,OAAO;oBACT,CAAC;oBACD,WAAW,EAAE,CAAC;gBAChB,CAAC;gBAED,aAAa;gBACb,IAAI,IAAY,CAAC;gBACjB,IAAI,MAAc,CAAC;gBACnB,IAAI,OAAO,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;oBAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE;wBAC/C,aAAa,EAAE,OAAO,CAAC,gBAAgB;wBACvC,OAAO,EAAE,OAAO,CAAC,OAAO;wBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;wBACxB,WAAW,EAAE,OAAO,CAAC,KAAK;qBAC3B,CAAC,CAAC;oBACH,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC;oBACtB,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;gBACzB,CAAC;qBAAM,CAAC;oBACN,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE;wBACjD,aAAa,EAAE,OAAO,CAAC,gBAAgB;wBACvC,OAAO,EAAE,OAAO,CAAC,OAAO;wBACxB,WAAW,EAAE,OAAO,CAAC,KAAK;qBAC3B,CAAC,CAAC;oBACH,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;oBACrC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;gBACzB,CAAC;gBAED,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;oBAClB,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;oBAClE,OAAO;gBACT,CAAC;gBAED,yBAAyB;gBACzB,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC7E,MAAM,SAAS,GAAG,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC;gBAEnC,0BAA0B;gBAC1B,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;gBAEjE,kBAAkB;gBAClB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;gBAE5E,2BAA2B;gBAC3B,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,EAAE,GAAG,GAAG,EAAE,SAAS,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;gBACpF,MAAM,cAAc,GAAqB,GAAG,CAAC,SAAS,IAAI,SAAS,CAAC;gBAEpE,4BAA4B;gBAC5B,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,eAAe,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;gBAEnE,wBAAwB;gBACxB,cAAc,CAAC,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;gBAEhF,6BAA6B;gBAC7B,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,gBAAgB,EAAE,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,cAAc,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC9G,IAAI,GAAG,CAAC,QAAQ;oBAAE,cAAc,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBAEzD,0CAA0C;gBAC1C,cAAc,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAE1E,sBAAsB;gBACtB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;gBAC/D,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,CAAC,yBAAyB;gBACnC,CAAC;gBAED,qBAAqB;gBACrB,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,cAAc,EAAE,CAAC;oBACjE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;oBACjE,IAAI,IAAI,CAAC,OAAO;wBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACrC,CAAC;gBAED,eAAe;gBACf,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;oBAC5B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;gBACtC,CAAC;gBAED,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAE3B,2CAA2C;gBAC3C,IAAI,YAAY,GAAG,KAAK,EAAE,CAAC;oBACzB,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK;yBACjC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;yBACzB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;yBAC3F,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;yBACjC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBACpB,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC/D,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC;oBACzD,CAAC;gBACH,CAAC;gBAED,gBAAgB;gBAChB,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBAClB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;gBACvD,CAAC;gBAED,kBAAkB;gBAClB,IAAI,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;oBAChD,IAAI,CAAC,cAAc,CAAC;wBAClB,WAAW,EAAE,CAAC,GAAG,OAAO,CAAC;wBACzB,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;wBAC/C,KAAK;wBACL,MAAM;wBACN,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACpC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC,CAAC;QAEF,wBAAwB;QACxB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;gBAAE,MAAM;YACpC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;QAED,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;QAEtB,8BAA8B;QAC9B,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,wBAAwB;YACxB,IAAI,GAAG,GAAQ,EAAE,IAAI,EAAE,CAAC;YACxB,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;YAE/D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAC/B,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,MAAM,CAAC,QAAQ,CACrB,CAAC;YAEF,yBAAyB;YACzB,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,GAAG,GAAG,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;YAC/E,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAE5C,MAAM,OAAO,GAAkB;YAC7B,EAAE,EAAE,IAAA,mBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YACpF,MAAM,EAAE,OAAO,CAAC,GAAG;YACnB,MAAM,EAAE,SAAS;YACjB,OAAO;YACP,WAAW;YACX,QAAQ,EAAE;gBACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,SAAS,EAAE,KAAK,CAAC,MAAM;gBACvB,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,IAAI,SAAS;gBACjD,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,SAAS;gBACvC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,OAAO;gBACnC,aAAa;aACd;YACD,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,IAAI,UAAU;SACjD,CAAC;QAEF,MAAM,KAAK,GAAe;YACxB,cAAc,EAAE,KAAK,CAAC,MAAM;YAC5B,WAAW;YACX,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAChC,MAAM;YACN,MAAM,EAAE,SAAS;YACjB,SAAS;YACT,WAAW;YACX,OAAO;SACR,CAAC;QAEF,oBAAoB;QACpB,OAAO,CAAC,UAAU,EAAE,CAAC;YACnB,cAAc,EAAE,KAAK,CAAC,MAAM;YAC5B,eAAe,EAAE,OAAO,CAAC,IAAI;YAC7B,UAAU,EAAE,EAAE;YACd,MAAM,EAAE,UAAU;SACnB,CAAC,CAAC;QAEH,2EAA2E;QAC3E,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAE3B,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,UAAkB;QAC3C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YAC1D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC1C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/C,OAAO,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,KAAsB,EAAE,GAAW;QACxD,IAAI,CAAC,IAAA,eAAU,EAAC,GAAG,CAAC;YAAE,IAAA,cAAS,EAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAA,kBAAa,EAAC,IAAA,WAAI,EAAC,GAAG,EAAE,iBAAiB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACrE,CAAC;IAEO,cAAc,CAAC,GAAW;QAChC,MAAM,IAAI,GAAG,IAAA,WAAI,EAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;QAC1C,IAAI,CAAC,IAAA,eAAU,EAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACnC,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAA,iBAAY,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAa,EAAE,GAAQ,EAAE,OAA4B;QAC5E,IAAI,CAAC,OAAO,EAAE,MAAM;YAAE,OAAO,GAAG,CAAC;QACjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxB,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC;YAChD,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,eAAe,CAAC,KAAyB;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClD,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QACzD,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAChD,CAAC;IAEO,kBAAkB,CAAC,KAAyB;QAClD,MAAM,aAAa,GAAuB,EAAE,CAAC;QAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;oBAClD,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;gBAC1E,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAEO,eAAe,CAAC,GAAW,EAAE,OAAkB,EAAE,OAAkB;QACzE,IAAI,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACtD,IAAI,OAAO,EAAE,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACzE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,uDAAuD;IAC/C,YAAY,CAAC,QAAgB,EAAE,OAAe;QACpD,OAAO,QAAQ,CAAC,OAAO,CAAC,0BAA0B,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;YACxE,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;gBAAE,OAAO,KAAK,CAAC;YAChG,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;gBAC7C,OAAO,IAAI,IAAI,KAAK,QAAQ,GAAG,CAAC;YAClC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4BAA4B;IACpB,KAAK,CAAC,SAAS,CAAC,OAAqB,EAAE,SAAiB;QAC9D,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAC/B,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAC7F,CAAC;QACF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC7D,OAAO;YACL,KAAK,EAAE,CAAC,SAAS,CAAC;YAClB,OAAO,EAAE;gBACP,EAAE,EAAE,IAAA,mBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;gBACvE,MAAM,EAAE,OAAO,CAAC,GAAG;gBACnB,MAAM;gBACN,OAAO,EAAE,SAAS,CAAC,WAAW;gBAC9B,WAAW;gBACX,QAAQ,EAAE;oBACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,SAAS,EAAE,CAAC;oBACZ,WAAW,EAAE,eAAe;oBAC5B,aAAa,EAAE,EAAE;iBAClB;gBACD,MAAM,EAAE,UAAU;aACnB;YACD,KAAK,EAAE;gBACL,cAAc,EAAE,CAAC,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAChE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC;aAChE;SACF,CAAC;IACJ,CAAC;IAED,0CAA0C;IAClC,KAAK,CAAC,YAAY,CAAC,OAAqB,EAAE,SAAiB;QACjE,MAAM,KAAK,GAAuB,EAAE,CAAC;QAErC,oBAAoB;QACpB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEnB,qCAAqC;QACrC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACjE,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAC/B,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CACzE,CAAC;YACF,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QAC5B,CAAC;QAED,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChE,OAAO;YACL,KAAK;YACL,OAAO,EAAE;gBACP,EAAE,EAAE,IAAA,mBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;gBACvE,MAAM,EAAE,OAAO,CAAC,GAAG;gBACnB,MAAM,EAAE,SAAS;gBACjB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW;gBAC9B,WAAW;gBACX,QAAQ,EAAE;oBACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,SAAS,EAAE,KAAK,CAAC,MAAM;oBACvB,WAAW,EAAE,QAAQ;oBACrB,aAAa,EAAE,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC;iBAC9C;gBACD,MAAM,EAAE,UAAU;aACnB;YACD,KAAK,EAAE;gBACL,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAC3E,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC;aAChE;SACF,CAAC;IACJ,CAAC;IAED,wCAAwC;IACxC,OAAO;QACL,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACvB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;CACF;AA1dD,sCA0dC"}

package/dist/core/stream.d.ts

@@ -0,0 +1,48 @@
+/// <reference types="node" />
+import { EventEmitter } from 'events';
+import { ExtractedContent, ContentChunk, CrawlProgress, CrawlResult } from './types';
+/**
+ * Streaming interface for crawl results.
+ * Emits events as pages are processed, enabling real-time consumption.
+ */
+export declare class CrawlStream extends EventEmitter {
+    private _pages;
+    private _chunks;
+    private _errors;
+    private _done;
+    /** Emitted when a page is extracted */
+    onPage(handler: (page: ExtractedContent) => void): this;
+    /** Emitted when chunks are generated from a page */
+    onChunks(handler: (chunks: ContentChunk[]) => void): this;
+    /** Emitted on progress updates */
+    onProgress(handler: (progress: CrawlProgress) => void): this;
+    /** Emitted on errors (non-fatal) */
+    onError(handler: (error: {
+        url: string;
+        error: string;
+    }) => void): this;
+    /** Emitted when crawl is complete */
+    onDone(handler: (result: CrawlResult) => void): this;
+    /** @internal */
+    emitPage(page: ExtractedContent): void;
+    /** @internal */
+    emitChunks(chunks: ContentChunk[]): void;
+    /** @internal */
+    emitProgress(progress: CrawlProgress): void;
+    /** @internal */
+    emitError(error: {
+        url: string;
+        error: string;
+    }): void;
+    /** @internal */
+    emitDone(result: CrawlResult): void;
+    /** Get all pages collected so far */
+    get pages(): ExtractedContent[];
+    /** Get all chunks collected so far */
+    get chunks(): ContentChunk[];
+    /** Check if crawl is complete */
+    get done(): boolean;
+    /** Wait for completion and return final result */
+    toPromise(): Promise<CrawlResult>;
+}
+//# sourceMappingURL=stream.d.ts.map

package/dist/core/stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/core/stream.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAErF;;;GAGG;AACH,qBAAa,WAAY,SAAQ,YAAY;IAC3C,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,OAAO,CAA6C;IAC5D,OAAO,CAAC,KAAK,CAAS;IAEtB,uCAAuC;IACvC,MAAM,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,IAAI;IAIvD,oDAAoD;IACpD,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,IAAI,GAAG,IAAI;IAIzD,kCAAkC;IAClC,UAAU,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI;IAI5D,oCAAoC;IACpC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,GAAG,IAAI;IAIvE,qCAAqC;IACrC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI;IAIpD,gBAAgB;IAChB,QAAQ,CAAC,IAAI,EAAE,gBAAgB,GAAG,IAAI;IAKtC,gBAAgB;IAChB,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI;IAKxC,gBAAgB;IAChB,YAAY,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI;IAI3C,gBAAgB;IAChB,SAAS,CAAC,KAAK,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAKtD,gBAAgB;IAChB,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IAKnC,qCAAqC;IACrC,IAAI,KAAK,IAAI,gBAAgB,EAAE,CAAwB;IAEvD,sCAAsC;IACtC,IAAI,MAAM,IAAI,YAAY,EAAE,CAAyB;IAErD,iCAAiC;IACjC,IAAI,IAAI,IAAI,OAAO,CAAuB;IAE1C,kDAAkD;IAClD,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC;CAIlC"}

package/dist/core/stream.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CrawlStream = void 0;
+const events_1 = require("events");
+/**
+ * Streaming interface for crawl results.
+ * Emits events as pages are processed, enabling real-time consumption.
+ */
+class CrawlStream extends events_1.EventEmitter {
+    _pages = [];
+    _chunks = [];
+    _errors = [];
+    _done = false;
+    /** Emitted when a page is extracted */
+    onPage(handler) {
+        return this.on('page', handler);
+    }
+    /** Emitted when chunks are generated from a page */
+    onChunks(handler) {
+        return this.on('chunks', handler);
+    }
+    /** Emitted on progress updates */
+    onProgress(handler) {
+        return this.on('progress', handler);
+    }
+    /** Emitted on errors (non-fatal) */
+    onError(handler) {
+        return this.on('error', handler);
+    }
+    /** Emitted when crawl is complete */
+    onDone(handler) {
+        return this.on('done', handler);
+    }
+    /** @internal */
+    emitPage(page) {
+        this._pages.push(page);
+        this.emit('page', page);
+    }
+    /** @internal */
+    emitChunks(chunks) {
+        this._chunks.push(...chunks);
+        this.emit('chunks', chunks);
+    }
+    /** @internal */
+    emitProgress(progress) {
+        this.emit('progress', progress);
+    }
+    /** @internal */
+    emitError(error) {
+        this._errors.push(error);
+        this.emit('error', error);
+    }
+    /** @internal */
+    emitDone(result) {
+        this._done = true;
+        this.emit('done', result);
+    }
+    /** Get all pages collected so far */
+    get pages() { return this._pages; }
+    /** Get all chunks collected so far */
+    get chunks() { return this._chunks; }
+    /** Check if crawl is complete */
+    get done() { return this._done; }
+    /** Wait for completion and return final result */
+    toPromise() {
+        if (this._done)
+            return Promise.resolve(this.listenerCount('done') > 0 ? undefined : undefined);
+        return new Promise((resolve) => this.once('done', resolve));
+    }
+}
+exports.CrawlStream = CrawlStream;
+//# sourceMappingURL=stream.js.map

package/dist/core/stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream.js","sourceRoot":"","sources":["../../src/core/stream.ts"],"names":[],"mappings":";;;AAAA,mCAAsC;AAGtC;;;GAGG;AACH,MAAa,WAAY,SAAQ,qBAAY;IACnC,MAAM,GAAuB,EAAE,CAAC;IAChC,OAAO,GAAmB,EAAE,CAAC;IAC7B,OAAO,GAA0C,EAAE,CAAC;IACpD,KAAK,GAAG,KAAK,CAAC;IAEtB,uCAAuC;IACvC,MAAM,CAAC,OAAyC;QAC9C,OAAO,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,CAAC;IAED,oDAAoD;IACpD,QAAQ,CAAC,OAAyC;QAChD,OAAO,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED,kCAAkC;IAClC,UAAU,CAAC,OAA0C;QACnD,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACtC,CAAC;IAED,oCAAoC;IACpC,OAAO,CAAC,OAAwD;QAC9D,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACnC,CAAC;IAED,qCAAqC;IACrC,MAAM,CAAC,OAAsC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,CAAC;IAED,gBAAgB;IAChB,QAAQ,CAAC,IAAsB;QAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,gBAAgB;IAChB,UAAU,CAAC,MAAsB;QAC/B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED,gBAAgB;IAChB,YAAY,CAAC,QAAuB;QAClC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,gBAAgB;IAChB,SAAS,CAAC,KAAqC;QAC7C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,gBAAgB;IAChB,QAAQ,CAAC,MAAmB;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,qCAAqC;IACrC,IAAI,KAAK,KAAyB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAEvD,sCAAsC;IACtC,IAAI,MAAM,KAAqB,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAErD,iCAAiC;IACjC,IAAI,IAAI,KAAc,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAE1C,kDAAkD;IAClD,SAAS;QACP,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAgB,CAAC,CAAC,CAAC,SAAgB,CAAC,CAAC;QAC7G,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC9D,CAAC;CACF;AA1ED,kCA0EC"}