webcontext-ai 1.0.0

package/dist/extractors/content.js

@@ -0,0 +1,328 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ContentExtractor = void 0;
+const cheerio = __importStar(require("cheerio"));
+/**
+ * Content extractor that cleans HTML and extracts structured content.
+ * Uses Readability algorithm + custom heuristics for developer content.
+ */
+class ContentExtractor {
+    // Selectors for noise elements to remove
+    static NOISE_SELECTORS = [
+        'nav', 'header', 'footer', 'aside',
+        '.sidebar', '.navigation', '.nav',
+        '.cookie-banner', '.cookie-consent', '.gdpr',
+        '.advertisement', '.ad', '.ads', '[class*="ad-"]',
+        '.social-share', '.share-buttons',
+        '.comments', '#comments', '.disqus',
+        '.newsletter', '.subscribe',
+        '.popup', '.modal', '.overlay',
+        '.breadcrumb', '.pagination',
+        'script', 'style', 'noscript', 'iframe',
+        '[role="banner"]', '[role="navigation"]', '[role="complementary"]',
+    ];
+    // Content-likely selectors (priority order)
+    static CONTENT_SELECTORS = [
+        'article', 'main', '[role="main"]',
+        '.markdown-body', '.documentation', '.doc-content',
+        '.post-content', '.article-content', '.entry-content',
+        '.readme', '#readme',
+        '.content', '#content',
+    ];
+    extract(html, url, focusMode = 'full') {
+        const $ = cheerio.load(html);
+        // Remove noise
+        ContentExtractor.NOISE_SELECTORS.forEach(sel => $(sel).remove());
+        // Find main content area
+        const contentEl = this.findContentElement($, focusMode);
+        const title = this.extractTitle($);
+        const description = $('meta[name="description"]').attr('content') ||
+            $('meta[property="og:description"]').attr('content') || '';
+        // Extract structured data before converting
+        let codeBlocks = this.extractCodeBlocks($, contentEl);
+        const headings = this.extractHeadings($, contentEl);
+        const links = this.extractLinks($, contentEl, url);
+        const metadata = this.extractMetadata($, url);
+        // Extract OpenAPI endpoints in API focus mode
+        if (focusMode === 'api') {
+            const apiBlocks = this.extractOpenAPIEndpoints($, contentEl);
+            codeBlocks = codeBlocks.concat(apiBlocks);
+        }
+        // Get clean text
+        const text = contentEl.text().replace(/\s+/g, ' ').trim();
+        // Get clean HTML for markdown conversion
+        const cleanHtml = contentEl.html() || '';
+        return {
+            url,
+            title,
+            description,
+            markdown: '', // Filled by transformer pipeline
+            html: cleanHtml,
+            text,
+            codeBlocks,
+            headings,
+            links,
+            metadata,
+            timestamp: new Date().toISOString(),
+        };
+    }
+    findContentElement($, focusMode) {
+        if (focusMode === 'code') {
+            const codeContainer = $('pre, .highlight, .code-block').parent();
+            if (codeContainer.length)
+                return codeContainer;
+        }
+        if (focusMode === 'api') {
+            const apiContent = $('.api-content, .endpoint, .method-section, .operation').first();
+            if (apiContent.length)
+                return apiContent;
+        }
+        for (const selector of ContentExtractor.CONTENT_SELECTORS) {
+            const el = $(selector).first();
+            if (el.length && (el.text().length > 200)) {
+                return el;
+            }
+        }
+        // Score remaining candidates by content density
+        const body = $('body');
+        const candidates = body.find('div, section').toArray();
+        let best = null;
+        let bestScore = 0;
+        for (const candidate of candidates) {
+            const $c = $(candidate);
+            const textLen = $c.text().trim().length;
+            if (textLen < 200)
+                continue;
+            const htmlLen = ($c.html() || '').length;
+            if (htmlLen === 0)
+                continue;
+            const density = textLen / htmlLen;
+            const paragraphs = $c.find('p').length;
+            const codeEls = $c.find('pre, code').length;
+            const score = density * (1 + paragraphs * 0.1 + codeEls * 0.2);
+            if (score > bestScore) {
+                bestScore = score;
+                best = $c;
+            }
+        }
+        return best || body;
+    }
+    extractTitle($) {
+        return $('h1').first().text().trim() ||
+            $('title').text().trim() ||
+            $('meta[property="og:title"]').attr('content') || 'Untitled';
+    }
+    extractCodeBlocks($, container) {
+        const blocks = [];
+        container.find('pre code, pre').each((_, el) => {
+            const $el = $(el);
+            const code = $el.text().trim();
+            if (!code)
+                return;
+            // Detect language from class
+            const classes = ($el.attr('class') || '') + ' ' + ($el.parent().attr('class') || '');
+            const langMatch = classes.match(/(?:language-|lang-|highlight-)(\w+)/);
+            const language = langMatch?.[1] || this.detectLanguage(code);
+            // Get surrounding context (previous heading or paragraph)
+            const prevHeading = $el.closest('section, div').find('h1,h2,h3,h4').last().text().trim();
+            blocks.push({ language, code, context: prevHeading || undefined });
+        });
+        return blocks;
+    }
+    extractHeadings($, container) {
+        const headings = [];
+        container.find('h1, h2, h3, h4, h5, h6').each((_, el) => {
+            const $el = $(el);
+            headings.push({
+                level: parseInt(el.tagName?.[1] || '1'),
+                text: $el.text().trim(),
+                id: $el.attr('id') || undefined,
+            });
+        });
+        return headings;
+    }
+    extractLinks($, container, baseUrl) {
+        const links = [];
+        const baseHost = new URL(baseUrl).hostname;
+        container.find('a[href]').each((_, el) => {
+            const $el = $(el);
+            const href = $el.attr('href') || '';
+            const text = $el.text().trim();
+            if (!href || !text || href.startsWith('#'))
+                return;
+            try {
+                const resolved = new URL(href, baseUrl).href;
+                links.push({
+                    href: resolved,
+                    text,
+                    isInternal: new URL(resolved).hostname === baseHost,
+                });
+            }
+            catch { }
+        });
+        return links;
+    }
+    extractMetadata($, url) {
+        const meta = {};
+        meta.author = $('meta[name="author"]').attr('content') ||
+            $('[rel="author"]').text().trim() || undefined;
+        meta.publishedDate = $('meta[property="article:published_time"]').attr('content') ||
+            $('time[datetime]').first().attr('datetime') || undefined;
+        meta.language = $('html').attr('lang') || undefined;
+        meta.ogImage = $('meta[property="og:image"]').attr('content') || undefined;
+        meta.canonical = $('link[rel="canonical"]').attr('href') || undefined;
+        meta.siteName = $('meta[property="og:site_name"]').attr('content') || undefined;
+        meta.type = this.detectContentType($, url);
+        meta.framework = this.detectFramework($, url);
+        meta.version = this.detectVersion($);
+        return meta;
+    }
+    detectContentType($, url) {
+        const urlLower = url.toLowerCase();
+        if (urlLower.includes('/api') || urlLower.includes('/reference'))
+            return 'api-reference';
+        if (urlLower.includes('/docs') || urlLower.includes('/documentation'))
+            return 'documentation';
+        if (urlLower.includes('/blog') || $('article').length)
+            return 'blog-post';
+        if (urlLower.includes('github.com') && urlLower.includes('/readme'))
+            return 'readme';
+        if ($('#readme').length || $('.markdown-body').length)
+            return 'readme';
+        if (urlLower.includes('/tutorial') || urlLower.includes('/guide'))
+            return 'tutorial';
+        if (urlLower.includes('/changelog') || urlLower.includes('/releases'))
+            return 'changelog';
+        return 'unknown';
+    }
+    detectFramework($, url) {
+        const text = $('body').text().toLowerCase();
+        const frameworks = [
+            'react', 'vue', 'angular', 'svelte', 'next.js', 'nuxt',
+            'express', 'fastapi', 'django', 'flask', 'spring', 'rails',
+            'tailwind', 'bootstrap', 'pytorch', 'tensorflow',
+        ];
+        return frameworks.find(f => text.includes(f) || url.includes(f));
+    }
+    detectVersion($) {
+        // Check meta tags first
+        const metaVersion = $('meta[name="version"]').attr('content') ||
+            $('meta[name="doc-version"]').attr('content');
+        if (metaVersion)
+            return metaVersion;
+        // Search visible text for version patterns
+        const text = $('body').text();
+        const match = text.match(/(?:v|version\s*)(\d+\.\d+(?:\.\d+)?(?:-[\w.]+)?)/i);
+        return match ? match[0].trim() : undefined;
+    }
+    extractOpenAPIEndpoints($, container) {
+        const blocks = [];
+        const methods = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS'];
+        // Look for structured endpoint elements (Swagger UI, Redoc, etc.)
+        container.find('.opblock, .operation, [class*="endpoint"], [class*="method"]').each((_, el) => {
+            const $el = $(el);
+            const text = $el.text();
+            const methodMatch = text.match(new RegExp(`\\b(${methods.join('|')})\\b`));
+            const pathMatch = text.match(/\/[\w\-\/.{}:]+/);
+            if (methodMatch && pathMatch) {
+                const description = $el.find('.opblock-summary-description, .description, p').first().text().trim();
+                const endpoint = `${methodMatch[1]} ${pathMatch[0]}`;
+                blocks.push({
+                    language: 'http',
+                    code: description ? `${endpoint}\n# ${description}` : endpoint,
+                    context: 'API Endpoint',
+                });
+            }
+        });
+        // Fallback: scan for HTTP method + path patterns in text nodes
+        if (!blocks.length) {
+            const text = container.text();
+            const endpointRegex = new RegExp(`\\b(${methods.join('|')})\\s+(\/[\\w\\-\\/.{}:?&=]+)`, 'g');
+            let match;
+            while ((match = endpointRegex.exec(text)) !== null) {
+                blocks.push({
+                    language: 'http',
+                    code: `${match[1]} ${match[2]}`,
+                    context: 'API Endpoint',
+                });
+            }
+        }
+        return blocks;
+    }
+    detectLanguage(code) {
+        // Python
+        if (code.includes('import ') && code.includes('from '))
+            return 'python';
+        // TypeScript (check before JS due to overlap)
+        if (code.includes('interface ') || code.includes(': string'))
+            return 'typescript';
+        // JavaScript
+        if (code.includes('const ') || code.includes('let ') || code.includes('=>'))
+            return 'javascript';
+        // Go
+        if (code.includes('func ') && code.includes(':='))
+            return 'go';
+        // Rust
+        if (code.includes('fn ') && code.includes('->'))
+            return 'rust';
+        // Java
+        if (code.includes('public class') || code.includes('System.out'))
+            return 'java';
+        // C#
+        if (code.includes('using System') || (code.includes('namespace') && code.includes('public static')))
+            return 'csharp';
+        // Kotlin
+        if (code.includes('fun ') && (code.includes('val ') || code.includes('package')))
+            return 'kotlin';
+        // Swift
+        if (code.includes('import Foundation') || (code.includes('func ') && code.includes('let ')))
+            return 'swift';
+        // Ruby
+        if (code.includes('def ') && code.includes('end'))
+            return 'ruby';
+        // PHP
+        if (code.includes('<?php'))
+            return 'php';
+        // SQL
+        if (/\b(SELECT|INSERT|UPDATE|CREATE TABLE)\b/.test(code))
+            return 'sql';
+        // Shell
+        if (code.startsWith('#!/bin/bash') || /^\s*\$/m.test(code))
+            return 'bash';
+        // HTML
+        if (code.trimStart().startsWith('<') && /<\w+[\s>]/.test(code))
+            return 'html';
+        // CSS
+        if (/\{[^}]*(color|margin|padding|display|font)\s*:/.test(code))
+            return 'css';
+        // YAML
+        if (/^[\w-]+\s*:(?:\s|$)/m.test(code) && !code.includes('{'))
+            return 'yaml';
+        return 'text';
+    }
+}
+exports.ContentExtractor = ContentExtractor;
+//# sourceMappingURL=content.js.map

package/dist/extractors/content.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content.js","sourceRoot":"","sources":["../../src/extractors/content.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAmC;AAGnC;;;GAGG;AACH,MAAa,gBAAgB;IAC3B,yCAAyC;IACjC,MAAM,CAAC,eAAe,GAAG;QAC/B,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO;QAClC,UAAU,EAAE,aAAa,EAAE,MAAM;QACjC,gBAAgB,EAAE,iBAAiB,EAAE,OAAO;QAC5C,gBAAgB,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB;QACjD,eAAe,EAAE,gBAAgB;QACjC,WAAW,EAAE,WAAW,EAAE,SAAS;QACnC,aAAa,EAAE,YAAY;QAC3B,QAAQ,EAAE,QAAQ,EAAE,UAAU;QAC9B,aAAa,EAAE,aAAa;QAC5B,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ;QACvC,iBAAiB,EAAE,qBAAqB,EAAE,wBAAwB;KACnE,CAAC;IAEF,4CAA4C;IACpC,MAAM,CAAC,iBAAiB,GAAG;QACjC,SAAS,EAAE,MAAM,EAAE,eAAe;QAClC,gBAAgB,EAAE,gBAAgB,EAAE,cAAc;QAClD,eAAe,EAAE,kBAAkB,EAAE,gBAAgB;QACrD,SAAS,EAAE,SAAS;QACpB,UAAU,EAAE,UAAU;KACvB,CAAC;IAEF,OAAO,CAAC,IAAY,EAAE,GAAW,EAAE,YAAuB,MAAM;QAC9D,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,eAAe;QACf,gBAAgB,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAEjE,yBAAyB;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,WAAW,GAAG,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;YAC9C,CAAC,CAAC,iCAAiC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAE9E,4CAA4C;QAC5C,IAAI,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAE9C,8CAA8C;QAC9C,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;YACxB,MAAM,SAAS,GAAG,IAAI,CAAC,uBAAuB,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YAC7D,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC5C,CAAC;QAED,iBAAiB;QACjB,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAE1D,yCAAyC;QACzC,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;QAEzC,OAAO;YACL,GAAG;YACH,KAAK;YACL,WAAW;YACX,QAAQ,EAAE,EAAE,EAAE,iCAAiC;YAC/C,IAAI,EAAE,SAAS;YACf,IAAI;YACJ,UAAU;YACV,QAAQ;YACR,KAAK;YACL,QAAQ;YACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC;IACJ,CAAC;IAEO,kBAAkB,CAAC,CAAqB,EAAE,SAAoB;QACpE,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;YACzB,MAAM,aAAa,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,MAAM,EAAE,CAAC;YACjE,IAAI,aAAa,CAAC,MAAM;gBAAE,OAAO,aAAa,CAAC;QACjD,CAAC;QAED,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,CAAC,CAAC,sDAAsD,CAAC,CAAC,KAAK,EAAE,CAAC;YACrF,IAAI,UAAU,CAAC,MAAM;gBAAE,OAAO,UAAU,CAAC;QAC3C,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;YAC1D,MAAM,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;gBAC1C,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,gDAAgD;QAChD,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QACvB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;QACvD,IAAI,IAAI,GAAgC,IAAI,CAAC;QAC7C,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;YACxB,MAAM,OAAO,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;YACxC,IAAI,OAAO,GAAG,GAAG;gBAAE,SAAS;YAE5B,MAAM,OAAO,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YACzC,IAAI,OAAO,KAAK,CAAC;gBAAE,SAAS;YAE5B,MAAM,OAAO,GAAG,OAAO,GAAG,OAAO,CAAC;YAClC,MAAM,UAAU,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;YACvC,MAAM,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC;YAC5C,MAAM,KAAK,GAAG,OAAO,GAAG,CAAC,CAAC,GAAG,UAAU,GAAG,GAAG,GAAG,OAAO,GAAG,GAAG,CAAC,CAAC;YAE/D,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;gBACtB,SAAS,GAAG,KAAK,CAAC;gBAClB,IAAI,GAAG,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,OAAO,IAAI,IAAI,IAAI,CAAC;IACtB,CAAC;IAEO,YAAY,CAAC,CAAqB;QACxC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;YAC7B,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;YACxB,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,UAAU,CAAC;IACtE,CAAC;IAEO,iBAAiB,CAAC,CAAqB,EAAE,SAA+B;QAC9E,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC7C,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAClB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI;gBAAE,OAAO;YAElB,6BAA6B;YAC7B,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YACrF,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACvE,MAAM,QAAQ,GAAG,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAE7D,0DAA0D;YAC1D,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAEzF,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,IAAI,SAAS,EAAE,CAAC,CAAC;QACrE,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CAAC,CAAqB,EAAE,SAA+B;QAC5E,MAAM,QAAQ,GAAc,EAAE,CAAC;QAC/B,SAAS,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACtD,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAClB,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;gBACvC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;gBACvB,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS;aAChC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,YAAY,CAAC,CAAqB,EAAE,SAA+B,EAAE,OAAe;QAC1F,MAAM,KAAK,GAAe,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC;QAE3C,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACvC,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAClB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,OAAO;YAEnD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;gBAC7C,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,QAAQ;oBACd,IAAI;oBACJ,UAAU,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,QAAQ,KAAK,QAAQ;iBACpD,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,eAAe,CAAC,CAAqB,EAAE,GAAW;QACxD,MAAM,IAAI,GAAiB,EAAE,CAAC;QAE9B,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;YACxC,CAAC,CAAC,gBAAgB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QAC7D,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,yCAAyC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;YAC5D,CAAC,CAAC,gBAAgB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,SAAS,CAAC;QAC/E,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;QACpD,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;QAC3E,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;QACtE,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,+BAA+B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;QAChF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QAErC,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,iBAAiB,CAAC,CAAqB,EAAE,GAAW;QAC1D,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACnC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC;YAAE,OAAO,eAAe,CAAC;QACzF,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC;YAAE,OAAO,eAAe,CAAC;QAC9F,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM;YAAE,OAAO,WAAW,CAAC;QAC1E,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC;YAAE,OAAO,QAAQ,CAAC;QACrF,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,gBAAgB,CAAC,CAAC,MAAM;YAAE,OAAO,QAAQ,CAAC;QACvE,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO,UAAU,CAAC;QACrF,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC;YAAE,OAAO,WAAW,CAAC;QAC1F,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,eAAe,CAAC,CAAqB,EAAE,GAAW;QACxD,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAG;YACjB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM;YACtD,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO;YAC1D,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY;SACjD,CAAC;QACF,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,aAAa,CAAC,CAAqB;QACzC,wBAAwB;QACxB,MAAM,WAAW,GAAG,CAAC,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;YACzC,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAClE,IAAI,WAAW;YAAE,OAAO,WAAW,CAAC;QAEpC,2CAA2C;QAC3C,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAC9E,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7C,CAAC;IAED,uBAAuB,CAAC,CAAqB,EAAE,SAA+B;QAC5E,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAE7E,kEAAkE;QAClE,SAAS,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC5F,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAClB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;YACxB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;YAC3E,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAChD,IAAI,WAAW,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,WAAW,GAAG,GAAG,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACpG,MAAM,QAAQ,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,MAAM,CAAC,IAAI,CAAC;oBACV,QAAQ,EAAE,MAAM;oBAChB,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,QAAQ,OAAO,WAAW,EAAE,CAAC,CAAC,CAAC,QAAQ;oBAC9D,OAAO,EAAE,cAAc;iBACxB,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,+DAA+D;QAC/D,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,aAAa,GAAG,IAAI,MAAM,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,8BAA8B,EAAE,GAAG,CAAC,CAAC;YAC9F,IAAI,KAA6B,CAAC;YAClC,OAAO,CAAC,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBACnD,MAAM,CAAC,IAAI,CAAC;oBACV,QAAQ,EAAE,MAAM;oBAChB,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE;oBAC/B,OAAO,EAAE,cAAc;iBACxB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,cAAc,CAAC,IAAY;QACjC,SAAS;QACT,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,OAAO,QAAQ,CAAC;QACxE,8CAA8C;QAC9C,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;YAAE,OAAO,YAAY,CAAC;QAClF,aAAa;QACb,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,OAAO,YAAY,CAAC;QACjG,KAAK;QACL,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QAC/D,OAAO;QACP,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAC/D,OAAO;QACP,IAAI,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;YAAE,OAAO,MAAM,CAAC;QAChF,KAAK;QACL,IAAI,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;YAAE,OAAO,QAAQ,CAAC;QACrH,SAAS;QACT,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAAE,OAAO,QAAQ,CAAC;QAClG,QAAQ;QACR,IAAI,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAAE,OAAO,OAAO,CAAC;QAC5G,OAAO;QACP,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC;QACjE,MAAM;QACN,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,OAAO,KAAK,CAAC;QACzC,MAAM;QACN,IAAI,yCAAyC,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QACvE,QAAQ;QACR,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAC1E,OAAO;QACP,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAC9E,MAAM;QACN,IAAI,gDAAgD,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QAC9E,OAAO;QACP,IAAI,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,MAAM,CAAC;QAC5E,OAAO,MAAM,CAAC;IAChB,CAAC;;AA7SH,4CA8SC"}

package/dist/extractors/github.d.ts

@@ -0,0 +1,19 @@
+import { ExtractedContent } from '../core/types';
+/**
+ * GitHub repository extractor.
+ * Fetches README and optionally docs folder from GitHub repos.
+ */
+export declare class GitHubExtractor {
+    constructor();
+    isGitHubUrl(url: string): boolean;
+    parseRepoUrl(url: string): {
+        owner: string;
+        repo: string;
+        branch?: string;
+        path?: string;
+    } | null;
+    extractReadme(url: string): Promise<ExtractedContent>;
+    extractDocs(url: string, docsPath?: string): Promise<ExtractedContent[]>;
+    private extractCodeBlocks;
+}
+//# sourceMappingURL=github.d.ts.map

package/dist/extractors/github.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/extractors/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAqB,MAAM,eAAe,CAAC;AAEpE;;;GAGG;AACH,qBAAa,eAAe;;IAG1B,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAIjC,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAM3F,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAgFrD,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAe,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAiDtF,OAAO,CAAC,iBAAiB;CAS1B"}

package/dist/extractors/github.js

@@ -0,0 +1,150 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GitHubExtractor = void 0;
+/**
+ * GitHub repository extractor.
+ * Fetches README and optionally docs folder from GitHub repos.
+ */
+class GitHubExtractor {
+    constructor() { }
+    isGitHubUrl(url) {
+        return /^https?:\/\/(www\.)?github\.com\/[\w.-]+\/[\w.-]+/.test(url);
+    }
+    parseRepoUrl(url) {
+        const match = url.match(/github\.com\/([\w.-]+)\/([\w.-]+)(?:\/(?:tree|blob)\/([\w.-]+)(?:\/(.+))?)?/);
+        if (!match)
+            return null;
+        return { owner: match[1], repo: match[2], branch: match[3], path: match[4] };
+    }
+    async extractReadme(url) {
+        const parsed = this.parseRepoUrl(url);
+        if (!parsed)
+            throw new Error(`Invalid GitHub URL: ${url}`);
+        const { owner, repo, branch } = parsed;
+        const ref = branch || 'main';
+        // Try fetching README
+        const readmeUrls = [
+            `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/README.md`,
+            `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/readme.md`,
+            `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/Readme.md`,
+            `https://raw.githubusercontent.com/${owner}/${repo}/master/README.md`,
+        ];
+        let markdown = '';
+        let fetchedUrl = '';
+        for (const readmeUrl of readmeUrls) {
+            try {
+                const res = await fetch(readmeUrl, { signal: AbortSignal.timeout(10000) });
+                if (res.ok) {
+                    markdown = await res.text();
+                    fetchedUrl = readmeUrl;
+                    break;
+                }
+            }
+            catch { }
+        }
+        if (!markdown)
+            throw new Error(`Could not find README for ${owner}/${repo}`);
+        // Extract headings from markdown
+        const headings = [];
+        const headingRegex = /^(#{1,6})\s+(.+)$/gm;
+        let match;
+        while ((match = headingRegex.exec(markdown)) !== null) {
+            headings.push({ level: match[1].length, text: match[2].trim() });
+        }
+        // Extract links
+        const links = [];
+        const linkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
+        while ((match = linkRegex.exec(markdown)) !== null) {
+            const href = match[2].startsWith('http') ? match[2] : `https://github.com/${owner}/${repo}/blob/${ref}/${match[2]}`;
+            links.push({ href, text: match[1], isInternal: !match[2].startsWith('http') });
+        }
+        // Fetch repo metadata from API (no auth needed for public repos)
+        let description = '';
+        let tags = [];
+        try {
+            const apiRes = await fetch(`https://api.github.com/repos/${owner}/${repo}`, {
+                headers: { 'Accept': 'application/vnd.github.v3+json' },
+                signal: AbortSignal.timeout(5000),
+            });
+            if (apiRes.ok) {
+                const repoData = await apiRes.json();
+                description = repoData.description || '';
+                tags = repoData.topics || [];
+            }
+        }
+        catch { }
+        return {
+            url,
+            title: `${owner}/${repo}`,
+            description,
+            markdown,
+            text: markdown.replace(/[#*`\[\]()>-]/g, ' ').replace(/\s+/g, ' ').trim(),
+            codeBlocks: this.extractCodeBlocks(markdown),
+            headings,
+            links,
+            metadata: {
+                author: owner,
+                type: 'readme',
+                tags,
+                siteName: 'GitHub',
+            },
+            timestamp: new Date().toISOString(),
+        };
+    }
+    async extractDocs(url, docsPath = 'docs') {
+        const parsed = this.parseRepoUrl(url);
+        if (!parsed)
+            throw new Error(`Invalid GitHub URL: ${url}`);
+        const { owner, repo, branch } = parsed;
+        const ref = branch || 'main';
+        // Fetch directory listing from GitHub API
+        const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${docsPath}?ref=${ref}`;
+        const res = await fetch(apiUrl, {
+            headers: { 'Accept': 'application/vnd.github.v3+json' },
+            signal: AbortSignal.timeout(10000),
+        });
+        if (!res.ok)
+            return [];
+        const files = await res.json();
+        const mdFiles = files.filter((f) => f.name.endsWith('.md') && f.type === 'file');
+        const results = [];
+        for (const file of mdFiles) {
+            try {
+                const contentRes = await fetch(file.download_url, { signal: AbortSignal.timeout(10000) });
+                if (!contentRes.ok)
+                    continue;
+                const markdown = await contentRes.text();
+                const headings = [];
+                const headingRegex = /^(#{1,6})\s+(.+)$/gm;
+                let match;
+                while ((match = headingRegex.exec(markdown)) !== null) {
+                    headings.push({ level: match[1].length, text: match[2].trim() });
+                }
+                results.push({
+                    url: file.html_url,
+                    title: headings[0]?.text || file.name.replace('.md', ''),
+                    markdown,
+                    text: markdown.replace(/[#*`\[\]()>-]/g, ' ').replace(/\s+/g, ' ').trim(),
+                    codeBlocks: this.extractCodeBlocks(markdown),
+                    headings,
+                    links: [],
+                    metadata: { type: 'documentation', siteName: 'GitHub' },
+                    timestamp: new Date().toISOString(),
+                });
+            }
+            catch { }
+        }
+        return results;
+    }
+    extractCodeBlocks(markdown) {
+        const blocks = [];
+        const regex = /```(\w*)\n([\s\S]*?)```/g;
+        let match;
+        while ((match = regex.exec(markdown)) !== null) {
+            blocks.push({ language: match[1] || 'text', code: match[2].trim() });
+        }
+        return blocks;
+    }
+}
+exports.GitHubExtractor = GitHubExtractor;
+//# sourceMappingURL=github.js.map

package/dist/extractors/github.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/extractors/github.ts"],"names":[],"mappings":";;;AAEA;;;GAGG;AACH,MAAa,eAAe;IAC1B,gBAAe,CAAC;IAEhB,WAAW,CAAC,GAAW;QACrB,OAAO,mDAAmD,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACvE,CAAC;IAED,YAAY,CAAC,GAAW;QACtB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,6EAA6E,CAAC,CAAC;QACvG,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QACxB,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/E,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,GAAW;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;QAE3D,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,IAAI,MAAM,CAAC;QAE7B,sBAAsB;QACtB,MAAM,UAAU,GAAG;YACjB,qCAAqC,KAAK,IAAI,IAAI,IAAI,GAAG,YAAY;YACrE,qCAAqC,KAAK,IAAI,IAAI,IAAI,GAAG,YAAY;YACrE,qCAAqC,KAAK,IAAI,IAAI,IAAI,GAAG,YAAY;YACrE,qCAAqC,KAAK,IAAI,IAAI,mBAAmB;SACtE,CAAC;QAEF,IAAI,QAAQ,GAAG,EAAE,CAAC;QAClB,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC3E,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;oBACX,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;oBAC5B,UAAU,GAAG,SAAS,CAAC;oBACvB,MAAM;gBACR,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC;QAE7E,iCAAiC;QACjC,MAAM,QAAQ,GAAc,EAAE,CAAC;QAC/B,MAAM,YAAY,GAAG,qBAAqB,CAAC;QAC3C,IAAI,KAA6B,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACtD,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,gBAAgB;QAChB,MAAM,KAAK,GAAe,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,0BAA0B,CAAC;QAC7C,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,sBAAsB,KAAK,IAAI,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACpH,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,iEAAiE;QACjE,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,IAAI,GAAa,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,gCAAgC,KAAK,IAAI,IAAI,EAAE,EAAE;gBAC1E,OAAO,EAAE,EAAE,QAAQ,EAAE,gCAAgC,EAAE;gBACvD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;aAClC,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACd,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBACrC,WAAW,GAAG,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC;gBACzC,IAAI,GAAG,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC;YAC/B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAEV,OAAO;YACL,GAAG;YACH,KAAK,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE;YACzB,WAAW;YACX,QAAQ;YACR,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE;YACzE,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC;YAC5C,QAAQ;YACR,KAAK;YACL,QAAQ,EAAE;gBACR,MAAM,EAAE,KAAK;gBACb,IAAI,EAAE,QAAQ;gBACd,IAAI;gBACJ,QAAQ,EAAE,QAAQ;aACnB;YACD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,WAAmB,MAAM;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;QAE3D,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,IAAI,MAAM,CAAC;QAE7B,0CAA0C;QAC1C,MAAM,MAAM,GAAG,gCAAgC,KAAK,IAAI,IAAI,aAAa,QAAQ,QAAQ,GAAG,EAAE,CAAC;QAC/F,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;YAC9B,OAAO,EAAE,EAAE,QAAQ,EAAE,gCAAgC,EAAE;YACvD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;SACnC,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,EAAE,CAAC;QACvB,MAAM,KAAK,GAAU,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QAEtF,MAAM,OAAO,GAAuB,EAAE,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC1F,IAAI,CAAC,UAAU,CAAC,EAAE;oBAAE,SAAS;gBAC7B,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;gBAEzC,MAAM,QAAQ,GAAc,EAAE,CAAC;gBAC/B,MAAM,YAAY,GAAG,qBAAqB,CAAC;gBAC3C,IAAI,KAA6B,CAAC;gBAClC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;oBACtD,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACnE,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,IAAI,CAAC,QAAQ;oBAClB,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;oBACxD,QAAQ;oBACR,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE;oBACzE,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC;oBAC5C,QAAQ;oBACR,KAAK,EAAE,EAAE;oBACT,QAAQ,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE;oBACvD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACpC,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,iBAAiB,CAAC,QAAgB;QACxC,MAAM,MAAM,GAA2D,EAAE,CAAC;QAC1E,MAAM,KAAK,GAAG,0BAA0B,CAAC;QACzC,IAAI,KAA6B,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAvJD,0CAuJC"}

package/dist/extractors/images.d.ts

@@ -0,0 +1,20 @@
+export interface ExtractedImage {
+    src: string;
+    alt: string;
+    title?: string;
+    width?: number;
+    height?: number;
+    context?: string;
+}
+/**
+ * Extracts images and their alt text/context from HTML.
+ * Useful for understanding diagrams, charts, and visual documentation.
+ */
+export declare class ImageExtractor {
+    extract(html: string, baseUrl: string): ExtractedImage[];
+    /** Convert extracted images to markdown references */
+    toMarkdown(images: ExtractedImage[]): string;
+    /** Get image descriptions as plain text (for LLM context) */
+    toDescriptions(images: ExtractedImage[]): string[];
+}
+//# sourceMappingURL=images.d.ts.map

package/dist/extractors/images.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"images.d.ts","sourceRoot":"","sources":["../../src/extractors/images.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,cAAc,EAAE;IA2BxD,sDAAsD;IACtD,UAAU,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM;IAU5C,6DAA6D;IAC7D,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,EAAE;CAKnD"}

package/dist/extractors/images.js

@@ -0,0 +1,73 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ImageExtractor = void 0;
+const cheerio = __importStar(require("cheerio"));
+/**
+ * Extracts images and their alt text/context from HTML.
+ * Useful for understanding diagrams, charts, and visual documentation.
+ */
+class ImageExtractor {
+    extract(html, baseUrl) {
+        const $ = cheerio.load(html);
+        const images = [];
+        $('img').each((_, el) => {
+            const $el = $(el);
+            const src = $el.attr('src') || '';
+            if (!src || src.startsWith('data:'))
+                return;
+            const resolvedSrc = src.startsWith('http') ? src : new URL(src, baseUrl).href;
+            const alt = $el.attr('alt') || '';
+            const title = $el.attr('title');
+            const width = parseInt($el.attr('width') || '0') || undefined;
+            const height = parseInt($el.attr('height') || '0') || undefined;
+            // Get surrounding context
+            const parent = $el.closest('figure, p, section, div');
+            const caption = parent.find('figcaption').text().trim();
+            const prevHeading = $el.closest('section, div').find('h1,h2,h3,h4').last().text().trim();
+            const context = caption || prevHeading || undefined;
+            images.push({ src: resolvedSrc, alt, title, width, height, context });
+        });
+        return images;
+    }
+    /** Convert extracted images to markdown references */
+    toMarkdown(images) {
+        return images
+            .filter(img => img.alt || img.context)
+            .map(img => {
+            const desc = img.alt || img.context || 'Image';
+            return `![${desc}](${img.src}${img.title ? ` "${img.title}"` : ''})`;
+        })
+            .join('\n\n');
+    }
+    /** Get image descriptions as plain text (for LLM context) */
+    toDescriptions(images) {
+        return images
+            .filter(img => img.alt || img.context)
+            .map(img => `[Image: ${img.alt || img.context || 'untitled'}] (${img.src})`);
+    }
+}
+exports.ImageExtractor = ImageExtractor;
+//# sourceMappingURL=images.js.map

package/dist/extractors/images.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"images.js","sourceRoot":"","sources":["../../src/extractors/images.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAmC;AAWnC;;;GAGG;AACH,MAAa,cAAc;IACzB,OAAO,CAAC,IAAY,EAAE,OAAe;QACnC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAqB,EAAE,CAAC;QAEpC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACtB,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAClB,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,OAAO;YAE5C,MAAM,WAAW,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAC9E,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,SAAS,CAAC;YAC9D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,IAAI,SAAS,CAAC;YAEhE,0BAA0B;YAC1B,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;YACtD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACxD,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACzF,MAAM,OAAO,GAAG,OAAO,IAAI,WAAW,IAAI,SAAS,CAAC;YAEpD,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,sDAAsD;IACtD,UAAU,CAAC,MAAwB;QACjC,OAAO,MAAM;aACV,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC;aACrC,GAAG,CAAC,GAAG,CAAC,EAAE;YACT,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC;YAC/C,OAAO,KAAK,IAAI,KAAK,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;QACvE,CAAC,CAAC;aACD,IAAI,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAED,6DAA6D;IAC7D,cAAc,CAAC,MAAwB;QACrC,OAAO,MAAM;aACV,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC;aACrC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,IAAI,UAAU,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;IACjF,CAAC;CACF;AA7CD,wCA6CC"}

package/dist/extractors/pdf.d.ts

@@ -0,0 +1,11 @@
+import { ExtractedContent } from '../core/types';
+/**
+ * PDF content extractor. Requires optional dependency: npm install pdf-parse
+ */
+export declare class PdfExtractor {
+    private pdfParse;
+    private loadParser;
+    extract(source: string): Promise<ExtractedContent>;
+    isPdf(url: string): boolean;
+}
+//# sourceMappingURL=pdf.d.ts.map