webcontext-ai 1.0.0

package/dist/extractors/pdf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf.d.ts","sourceRoot":"","sources":["../../src/extractors/pdf.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAW,MAAM,eAAe,CAAC;AAE1D;;GAEG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAa;YAEf,UAAU;IAclB,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuDxD,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;CAK5B"}

package/dist/extractors/pdf.js

@@ -0,0 +1,107 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PdfExtractor = void 0;
+const fs_1 = require("fs");
+/**
+ * PDF content extractor. Requires optional dependency: npm install pdf-parse
+ */
+class PdfExtractor {
+    pdfParse = null;
+    async loadParser() {
+        if (this.pdfParse)
+            return this.pdfParse;
+        try {
+            // @ts-ignore
+            this.pdfParse = (await Promise.resolve().then(() => __importStar(require('pdf-parse')))).default;
+            return this.pdfParse;
+        }
+        catch {
+            throw new Error('pdf-parse is required for PDF extraction but is not installed.\n' +
+                'Install it with: npm install pdf-parse');
+        }
+    }
+    async extract(source) {
+        const parser = await this.loadParser();
+        let buffer;
+        if (source.startsWith('http://') || source.startsWith('https://')) {
+            const response = await fetch(source, { signal: AbortSignal.timeout(60000) });
+            if (!response.ok)
+                throw new Error(`Failed to fetch PDF: HTTP ${response.status}`);
+            buffer = Buffer.from(await response.arrayBuffer());
+        }
+        else if ((0, fs_1.existsSync)(source)) {
+            buffer = (0, fs_1.readFileSync)(source);
+        }
+        else {
+            throw new Error(`PDF source not found: ${source}`);
+        }
+        const data = await parser(buffer);
+        const text = data.text || '';
+        const title = data.info?.Title || source.split('/').pop()?.replace('.pdf', '') || 'Untitled PDF';
+        const author = data.info?.Author;
+        const pages = data.numpages || 0;
+        // Extract headings (lines that look like headings: short, no period, often uppercase or title case)
+        const headings = [];
+        const lines = text.split('\n');
+        for (const line of lines) {
+            const trimmed = line.trim();
+            if (!trimmed || trimmed.length > 100 || trimmed.endsWith('.'))
+                continue;
+            if (/^\d+\.\s+[A-Z]/.test(trimmed) || /^[A-Z][A-Z\s]{3,}$/.test(trimmed)) {
+                headings.push({ level: /^\d+\.\d+/.test(trimmed) ? 2 : 1, text: trimmed });
+            }
+        }
+        // Convert to markdown
+        let markdown = `# ${title}\n\n`;
+        if (author)
+            markdown += `> Author: ${author}\n\n`;
+        markdown += `> ${pages} pages\n\n`;
+        markdown += text;
+        return {
+            url: source,
+            title,
+            description: `PDF document: ${title} (${pages} pages)`,
+            markdown,
+            text,
+            codeBlocks: [],
+            headings,
+            links: [],
+            metadata: {
+                author,
+                type: 'documentation',
+                tags: ['pdf'],
+            },
+            timestamp: new Date().toISOString(),
+        };
+    }
+    isPdf(url) {
+        return url.toLowerCase().endsWith('.pdf') ||
+            url.includes('/pdf/') ||
+            url.includes('application/pdf');
+    }
+}
+exports.PdfExtractor = PdfExtractor;
+//# sourceMappingURL=pdf.js.map

package/dist/extractors/pdf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../src/extractors/pdf.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2BAA8C;AAG9C;;GAEG;AACH,MAAa,YAAY;IACf,QAAQ,GAAQ,IAAI,CAAC;IAErB,KAAK,CAAC,UAAU;QACtB,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC;QACxC,IAAI,CAAC;YACH,aAAa;YACb,IAAI,CAAC,QAAQ,GAAG,CAAC,wDAAa,WAAW,GAAC,CAAC,CAAC,OAAO,CAAC;YACpD,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,kEAAkE;gBAClE,wCAAwC,CACzC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACvC,IAAI,MAAc,CAAC;QAEnB,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAClE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC7E,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACrD,CAAC;aAAM,IAAI,IAAA,eAAU,EAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,IAAA,iBAAY,EAAC,MAAM,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,yBAAyB,MAAM,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,IAAI,GAAW,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,cAAc,CAAC;QACjG,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QAEjC,oGAAoG;QACpG,MAAM,QAAQ,GAAc,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,SAAS;YACxE,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;QAED,sBAAsB;QACtB,IAAI,QAAQ,GAAG,KAAK,KAAK,MAAM,CAAC;QAChC,IAAI,MAAM;YAAE,QAAQ,IAAI,aAAa,MAAM,MAAM,CAAC;QAClD,QAAQ,IAAI,KAAK,KAAK,YAAY,CAAC;QACnC,QAAQ,IAAI,IAAI,CAAC;QAEjB,OAAO;YACL,GAAG,EAAE,MAAM;YACX,KAAK;YACL,WAAW,EAAE,iBAAiB,KAAK,KAAK,KAAK,SAAS;YACtD,QAAQ;YACR,IAAI;YACJ,UAAU,EAAE,EAAE;YACd,QAAQ;YACR,KAAK,EAAE,EAAE;YACT,QAAQ,EAAE;gBACR,MAAM;gBACN,IAAI,EAAE,eAAe;gBACrB,IAAI,EAAE,CAAC,KAAK,CAAC;aACd;YACD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAW;QACf,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;YACvC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;YACrB,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IACpC,CAAC;CACF;AA7ED,oCA6EC"}

package/dist/extractors/screenshot.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Screenshot capture for web pages. Requires Playwright (optional dependency).
+ */
+export declare class ScreenshotCapture {
+    private outputDir;
+    constructor(outputDir?: string);
+    capture(url: string, options?: {
+        fullPage?: boolean;
+        width?: number;
+        height?: number;
+        format?: 'png' | 'jpeg';
+        quality?: number;
+    }): Promise<string>;
+    captureMultiple(urls: string[], options?: {
+        fullPage?: boolean;
+        width?: number;
+        height?: number;
+        format?: 'png' | 'jpeg';
+    }): Promise<string[]>;
+}
+//# sourceMappingURL=screenshot.d.ts.map

package/dist/extractors/screenshot.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/extractors/screenshot.ts"],"names":[],"mappings":"AAGA;;GAEG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,SAAS,CAAS;gBAEd,SAAS,GAAE,MAAwB;IAIzC,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE;QAClC,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QACxB,OAAO,CAAC,EAAE,MAAM,CAAC;KACb,GAAG,OAAO,CAAC,MAAM,CAAC;IAuClB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE;QAC7C,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAY3B"}

package/dist/extractors/screenshot.js

@@ -0,0 +1,85 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScreenshotCapture = void 0;
+const fs_1 = require("fs");
+const path_1 = require("path");
+/**
+ * Screenshot capture for web pages. Requires Playwright (optional dependency).
+ */
+class ScreenshotCapture {
+    outputDir;
+    constructor(outputDir = './screenshots') {
+        this.outputDir = outputDir;
+    }
+    async capture(url, options = {}) {
+        let chromium;
+        try {
+            const pw = await Promise.resolve().then(() => __importStar(require('playwright')));
+            chromium = pw.chromium;
+        }
+        catch {
+            throw new Error('Playwright is required for screenshots but is not installed.\n' +
+                'Install it with: npm install playwright && npx playwright install chromium');
+        }
+        if (!(0, fs_1.existsSync)(this.outputDir))
+            (0, fs_1.mkdirSync)(this.outputDir, { recursive: true });
+        const browser = await chromium.launch({ headless: true });
+        const context = await browser.newContext({
+            viewport: { width: options.width || 1280, height: options.height || 720 },
+        });
+        const page = await context.newPage();
+        try {
+            await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
+            const filename = `${url.replace(/[^a-z0-9]/gi, '_').slice(0, 100)}.${options.format || 'png'}`;
+            const filepath = (0, path_1.join)(this.outputDir, filename);
+            await page.screenshot({
+                path: filepath,
+                fullPage: options.fullPage ?? true,
+                type: options.format || 'png',
+                ...(options.format === 'jpeg' && options.quality ? { quality: options.quality } : {}),
+            });
+            return filepath;
+        }
+        finally {
+            await browser.close();
+        }
+    }
+    async captureMultiple(urls, options = {}) {
+        const results = [];
+        for (const url of urls) {
+            try {
+                const path = await this.capture(url, options);
+                results.push(path);
+            }
+            catch {
+                results.push('');
+            }
+        }
+        return results;
+    }
+}
+exports.ScreenshotCapture = ScreenshotCapture;
+//# sourceMappingURL=screenshot.js.map

package/dist/extractors/screenshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../src/extractors/screenshot.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2BAA2C;AAC3C,+BAA4B;AAE5B;;GAEG;AACH,MAAa,iBAAiB;IACpB,SAAS,CAAS;IAE1B,YAAY,YAAoB,eAAe;QAC7C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAW,EAAE,UAMvB,EAAE;QACJ,IAAI,QAAa,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,wDAAa,YAAY,GAAC,CAAC;YACtC,QAAQ,GAAG,EAAE,CAAC,QAAQ,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,gEAAgE;gBAChE,4EAA4E,CAC7E,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,IAAA,eAAU,EAAC,IAAI,CAAC,SAAS,CAAC;YAAE,IAAA,cAAS,EAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEhF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;YACvC,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,GAAG,EAAE;SAC1E,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAEnE,MAAM,QAAQ,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;YAC/F,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAEhD,MAAM,IAAI,CAAC,UAAU,CAAC;gBACpB,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI;gBAClC,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;gBAC7B,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACtF,CAAC,CAAC;YAEH,OAAO,QAAQ,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAAc,EAAE,UAKlC,EAAE;QACJ,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAC9C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AArED,8CAqEC"}

package/dist/index.d.ts

@@ -0,0 +1,70 @@
+export * from './core/types';
+export { CrawlPipeline } from './core/pipeline';
+export { CrawlStream } from './core/stream';
+export { BrowserManager } from './browser/manager';
+export { ContentExtractor } from './extractors/content';
+export { PdfExtractor } from './extractors/pdf';
+export { GitHubExtractor } from './extractors/github';
+export { ImageExtractor } from './extractors/images';
+export { ScreenshotCapture } from './extractors/screenshot';
+export { MarkdownTransformer } from './transformers/markdown';
+export { ContentChunker } from './chunking/chunker';
+export { CrawlCache } from './cache/cache';
+export { VectorSearch } from './search/vector';
+export { VectorDBExporter, VectorDBExportOptions, OutputFormatter, OutputTemplate } from './export';
+export { SitemapParser } from './utils/sitemap';
+export { MetricsCollector } from './utils/metrics';
+export { CrawlScheduler } from './utils/scheduler';
+export { Deduplicator } from './utils/dedup';
+export { WebhookNotifier, WebhookConfig } from './utils/webhook';
+export { validateUrl, validateCrawlOptions } from './utils/validation';
+import { WebContextConfig, CrawlOptions, CrawlResult, ContentChunk, SearchResult, MetricsData } from './core/types';
+import { CrawlStream } from './core/stream';
+import { VectorDBExportOptions } from './export';
+import { WebhookConfig } from './utils/webhook';
+/**
+ * WebContext - Turn any web content into clean AI-ready context instantly.
+ */
+export declare class WebContext {
+    private pipeline;
+    private vectorSearch;
+    private metrics;
+    private webhooks;
+    private config;
+    constructor(config?: WebContextConfig);
+    /** Extract content from a single URL */
+    extract(url: string, options?: Partial<CrawlOptions>): Promise<CrawlResult>;
+    /** Crawl a documentation site recursively */
+    crawlDocs(url: string, options?: Partial<CrawlOptions>): Promise<CrawlResult>;
+    /** Extract and return only markdown */
+    toMarkdown(url: string, options?: Partial<CrawlOptions>): Promise<string>;
+    /** Extract and return chunked content for RAG */
+    toChunks(url: string, options?: Partial<CrawlOptions>): Promise<ContentChunk[]>;
+    /** Extract and return a context packet optimized for LLM consumption */
+    toContext(url: string, options?: Partial<CrawlOptions> & {
+        maxTokens?: number;
+    }): Promise<string>;
+    /** Semantic search within a page's content */
+    search(url: string, query: string, topK?: number): Promise<SearchResult[]>;
+    /** Extract GitHub README */
+    extractReadme(repoUrl: string): Promise<CrawlResult>;
+    /** Extract GitHub repo with docs */
+    extractGitHub(repoUrl: string, options?: Partial<CrawlOptions>): Promise<CrawlResult>;
+    /** Extract content from a PDF (URL or local path) */
+    extractPdf(source: string): Promise<CrawlResult>;
+    /** Extract API reference */
+    extractAPI(url: string): Promise<CrawlResult>;
+    /** Export chunks in vector DB format */
+    exportForVectorDB(url: string, options?: VectorDBExportOptions & Partial<CrawlOptions>): Promise<string>;
+    /** Stream crawl results in real-time */
+    extractStream(url: string, options?: Partial<CrawlOptions>): CrawlStream;
+    /** Register a webhook for crawl notifications */
+    registerWebhook(config: WebhookConfig): void;
+    /** Get collected metrics */
+    getMetrics(): MetricsData | null;
+    /** Cleanup resources */
+    dispose(): void;
+    private formatForLLM;
+}
+export default WebContext;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AACpG,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAEvE,OAAO,EACL,gBAAgB,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAC1C,YAAY,EAAE,WAAW,EACzC,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,OAAO,EAAoB,qBAAqB,EAAE,MAAM,UAAU,CAAC;AACnE,OAAO,EAAmB,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAIjE;;GAEG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAgB;IAChC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,QAAQ,CAAgC;IAChD,OAAO,CAAC,MAAM,CAAmB;gBAErB,MAAM,GAAE,gBAAqB;IAOzC,wCAAwC;IAClC,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAUrF,6CAA6C;IACvC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAUvF,uCAAuC;IACjC,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKnF,iDAAiD;IAC3C,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAKzF,wEAAwE;IAClE,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,YAAY,CAAC,GAAG;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAM3G,8CAA8C;IACxC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,MAAU,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAMnF,4BAA4B;IACtB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAK1D,oCAAoC;IAC9B,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAI/F,qDAAqD;IAC/C,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAStD,4BAA4B;IACtB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAInD,wCAAwC;IAClC,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;IAOlI,wCAAwC;IACxC,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,WAAW;IAe5E,iDAAiD;IACjD,eAAe,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAK5C,4BAA4B;IAC5B,UAAU,IAAI,WAAW,GAAG,IAAI;IAIhC,wBAAwB;IACxB,OAAO,IAAI,IAAI;IAKf,OAAO,CAAC,YAAY;CAiBrB;AAED,eAAe,UAAU,CAAC"}

package/dist/index.js

@@ -0,0 +1,206 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebContext = exports.validateCrawlOptions = exports.validateUrl = exports.WebhookNotifier = exports.Deduplicator = exports.CrawlScheduler = exports.MetricsCollector = exports.SitemapParser = exports.OutputFormatter = exports.VectorDBExporter = exports.VectorSearch = exports.CrawlCache = exports.ContentChunker = exports.MarkdownTransformer = exports.ScreenshotCapture = exports.ImageExtractor = exports.GitHubExtractor = exports.PdfExtractor = exports.ContentExtractor = exports.BrowserManager = exports.CrawlStream = exports.CrawlPipeline = void 0;
+__exportStar(require("./core/types"), exports);
+var pipeline_1 = require("./core/pipeline");
+Object.defineProperty(exports, "CrawlPipeline", { enumerable: true, get: function () { return pipeline_1.CrawlPipeline; } });
+var stream_1 = require("./core/stream");
+Object.defineProperty(exports, "CrawlStream", { enumerable: true, get: function () { return stream_1.CrawlStream; } });
+var manager_1 = require("./browser/manager");
+Object.defineProperty(exports, "BrowserManager", { enumerable: true, get: function () { return manager_1.BrowserManager; } });
+var content_1 = require("./extractors/content");
+Object.defineProperty(exports, "ContentExtractor", { enumerable: true, get: function () { return content_1.ContentExtractor; } });
+var pdf_1 = require("./extractors/pdf");
+Object.defineProperty(exports, "PdfExtractor", { enumerable: true, get: function () { return pdf_1.PdfExtractor; } });
+var github_1 = require("./extractors/github");
+Object.defineProperty(exports, "GitHubExtractor", { enumerable: true, get: function () { return github_1.GitHubExtractor; } });
+var images_1 = require("./extractors/images");
+Object.defineProperty(exports, "ImageExtractor", { enumerable: true, get: function () { return images_1.ImageExtractor; } });
+var screenshot_1 = require("./extractors/screenshot");
+Object.defineProperty(exports, "ScreenshotCapture", { enumerable: true, get: function () { return screenshot_1.ScreenshotCapture; } });
+var markdown_1 = require("./transformers/markdown");
+Object.defineProperty(exports, "MarkdownTransformer", { enumerable: true, get: function () { return markdown_1.MarkdownTransformer; } });
+var chunker_1 = require("./chunking/chunker");
+Object.defineProperty(exports, "ContentChunker", { enumerable: true, get: function () { return chunker_1.ContentChunker; } });
+var cache_1 = require("./cache/cache");
+Object.defineProperty(exports, "CrawlCache", { enumerable: true, get: function () { return cache_1.CrawlCache; } });
+var vector_1 = require("./search/vector");
+Object.defineProperty(exports, "VectorSearch", { enumerable: true, get: function () { return vector_1.VectorSearch; } });
+var export_1 = require("./export");
+Object.defineProperty(exports, "VectorDBExporter", { enumerable: true, get: function () { return export_1.VectorDBExporter; } });
+Object.defineProperty(exports, "OutputFormatter", { enumerable: true, get: function () { return export_1.OutputFormatter; } });
+var sitemap_1 = require("./utils/sitemap");
+Object.defineProperty(exports, "SitemapParser", { enumerable: true, get: function () { return sitemap_1.SitemapParser; } });
+var metrics_1 = require("./utils/metrics");
+Object.defineProperty(exports, "MetricsCollector", { enumerable: true, get: function () { return metrics_1.MetricsCollector; } });
+var scheduler_1 = require("./utils/scheduler");
+Object.defineProperty(exports, "CrawlScheduler", { enumerable: true, get: function () { return scheduler_1.CrawlScheduler; } });
+var dedup_1 = require("./utils/dedup");
+Object.defineProperty(exports, "Deduplicator", { enumerable: true, get: function () { return dedup_1.Deduplicator; } });
+var webhook_1 = require("./utils/webhook");
+Object.defineProperty(exports, "WebhookNotifier", { enumerable: true, get: function () { return webhook_1.WebhookNotifier; } });
+var validation_1 = require("./utils/validation");
+Object.defineProperty(exports, "validateUrl", { enumerable: true, get: function () { return validation_1.validateUrl; } });
+Object.defineProperty(exports, "validateCrawlOptions", { enumerable: true, get: function () { return validation_1.validateCrawlOptions; } });
+const pipeline_2 = require("./core/pipeline");
+const stream_2 = require("./core/stream");
+const vector_2 = require("./search/vector");
+const export_2 = require("./export");
+const webhook_2 = require("./utils/webhook");
+const metrics_2 = require("./utils/metrics");
+const validation_2 = require("./utils/validation");
+/**
+ * WebContext - Turn any web content into clean AI-ready context instantly.
+ */
+class WebContext {
+    pipeline;
+    vectorSearch;
+    metrics;
+    webhooks = null;
+    config;
+    constructor(config = {}) {
+        this.config = config;
+        this.pipeline = new pipeline_2.CrawlPipeline(config);
+        this.vectorSearch = new vector_2.VectorSearch();
+        this.metrics = config.metrics ? new metrics_2.MetricsCollector() : null;
+    }
+    /** Extract content from a single URL */
+    async extract(url, options = {}) {
+        (0, validation_2.validateUrl)(url);
+        const start = Date.now();
+        const result = await this.pipeline.crawl({ url, depth: 0, ...options });
+        if (this.metrics) {
+            this.metrics.recordCrawl(result.stats.pagesProcessed, result.stats.totalTokens, Date.now() - start);
+        }
+        return result;
+    }
+    /** Crawl a documentation site recursively */
+    async crawlDocs(url, options = {}) {
+        (0, validation_2.validateUrl)(url);
+        const start = Date.now();
+        const result = await this.pipeline.crawl({ url, depth: options.depth ?? 3, ...options });
+        if (this.metrics) {
+            this.metrics.recordCrawl(result.stats.pagesProcessed, result.stats.totalTokens, Date.now() - start);
+        }
+        return result;
+    }
+    /** Extract and return only markdown */
+    async toMarkdown(url, options = {}) {
+        const result = await this.extract(url, options);
+        return result.pages.map(p => p.markdown).join('\n\n---\n\n');
+    }
+    /** Extract and return chunked content for RAG */
+    async toChunks(url, options = {}) {
+        const result = await this.extract(url, options);
+        return result.context.chunks;
+    }
+    /** Extract and return a context packet optimized for LLM consumption */
+    async toContext(url, options = {}) {
+        const { maxTokens, ...crawlOptions } = options;
+        const result = await this.extract(url, crawlOptions);
+        return this.formatForLLM(result.context, maxTokens);
+    }
+    /** Semantic search within a page's content */
+    async search(url, query, topK = 5) {
+        const chunks = await this.toChunks(url);
+        this.vectorSearch.index(chunks);
+        return this.vectorSearch.search(query, topK);
+    }
+    /** Extract GitHub README */
+    async extractReadme(repoUrl) {
+        (0, validation_2.validateUrl)(repoUrl);
+        return this.extract(repoUrl.replace(/\/$/, ''));
+    }
+    /** Extract GitHub repo with docs */
+    async extractGitHub(repoUrl, options = {}) {
+        return this.crawlDocs(repoUrl, { depth: 1, ...options });
+    }
+    /** Extract content from a PDF (URL or local path) */
+    async extractPdf(source) {
+        if (source.startsWith('http')) {
+            return this.extract(source);
+        }
+        // Local file — bypass URL validation, call pipeline directly
+        const result = await this.pipeline.crawl({ url: source, depth: 0 });
+        return result;
+    }
+    /** Extract API reference */
+    async extractAPI(url) {
+        return this.extract(url, { focusMode: 'api' });
+    }
+    /** Export chunks in vector DB format */
+    async exportForVectorDB(url, options = { format: 'json' }) {
+        const { format, namespace, collection, includeMetadata, ...crawlOptions } = options;
+        const result = await this.extract(url, crawlOptions);
+        const exporter = new export_2.VectorDBExporter();
+        return exporter.exportChunks(result.context.chunks, { format, namespace, collection, includeMetadata });
+    }
+    /** Stream crawl results in real-time */
+    extractStream(url, options = {}) {
+        const stream = new stream_2.CrawlStream();
+        (0, validation_2.validateUrl)(url);
+        // Run crawl async and emit events
+        this.pipeline.crawl({ url, depth: 0, ...options, onProgress: (p) => stream.emitProgress(p) })
+            .then((result) => {
+            for (const page of result.pages)
+                stream.emitPage(page);
+            stream.emitChunks(result.context.chunks);
+            if (result.diffs?.length)
+                stream.emitDone(result);
+            else
+                stream.emitDone(result);
+        })
+            .catch((err) => stream.emitError({ url, error: err.message }));
+        return stream;
+    }
+    /** Register a webhook for crawl notifications */
+    registerWebhook(config) {
+        if (!this.webhooks)
+            this.webhooks = new webhook_2.WebhookNotifier();
+        this.webhooks.register(config);
+    }
+    /** Get collected metrics */
+    getMetrics() {
+        return this.metrics?.getMetrics() ?? null;
+    }
+    /** Cleanup resources */
+    dispose() {
+        this.vectorSearch.clear();
+        this.pipeline.dispose();
+    }
+    formatForLLM(packet, maxTokens) {
+        const budget = maxTokens ?? 8000;
+        let output = `# ${packet.source}\n\n`;
+        if (packet.summary)
+            output += `> ${packet.summary}\n\n`;
+        output += `> ${packet.metadata.pageCount} pages | ${packet.totalTokens} tokens | ${packet.metadata.contentType}\n\n`;
+        let usedTokens = Math.ceil(output.length / 4);
+        for (const chunk of packet.chunks) {
+            if (usedTokens + chunk.tokens > budget)
+                break;
+            if (chunk.metadata.headingPath.length) {
+                output += `## ${chunk.metadata.headingPath.join(' > ')}\n\n`;
+            }
+            output += chunk.content + '\n\n';
+            usedTokens += chunk.tokens;
+        }
+        return output.trim();
+    }
+}
+exports.WebContext = WebContext;
+exports.default = WebContext;
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,+CAA6B;AAC7B,4CAAgD;AAAvC,yGAAA,aAAa,OAAA;AACtB,wCAA4C;AAAnC,qGAAA,WAAW,OAAA;AACpB,6CAAmD;AAA1C,yGAAA,cAAc,OAAA;AACvB,gDAAwD;AAA/C,2GAAA,gBAAgB,OAAA;AACzB,wCAAgD;AAAvC,mGAAA,YAAY,OAAA;AACrB,8CAAsD;AAA7C,yGAAA,eAAe,OAAA;AACxB,8CAAqD;AAA5C,wGAAA,cAAc,OAAA;AACvB,sDAA4D;AAAnD,+GAAA,iBAAiB,OAAA;AAC1B,oDAA8D;AAArD,+GAAA,mBAAmB,OAAA;AAC5B,8CAAoD;AAA3C,yGAAA,cAAc,OAAA;AACvB,uCAA2C;AAAlC,mGAAA,UAAU,OAAA;AACnB,0CAA+C;AAAtC,sGAAA,YAAY,OAAA;AACrB,mCAAoG;AAA3F,0GAAA,gBAAgB,OAAA;AAAyB,yGAAA,eAAe,OAAA;AACjE,2CAAgD;AAAvC,wGAAA,aAAa,OAAA;AACtB,2CAAmD;AAA1C,2GAAA,gBAAgB,OAAA;AACzB,+CAAmD;AAA1C,2GAAA,cAAc,OAAA;AACvB,uCAA6C;AAApC,qGAAA,YAAY,OAAA;AACrB,2CAAiE;AAAxD,0GAAA,eAAe,OAAA;AACxB,iDAAuE;AAA9D,yGAAA,WAAW,OAAA;AAAE,kHAAA,oBAAoB,OAAA;AAM1C,8CAAgD;AAChD,0CAA4C;AAC5C,4CAA+C;AAC/C,qCAAmE;AACnE,6CAAiE;AACjE,6CAAmD;AACnD,mDAAiD;AAEjD;;GAEG;AACH,MAAa,UAAU;IACb,QAAQ,CAAgB;IACxB,YAAY,CAAe;IAC3B,OAAO,CAA0B;IACjC,QAAQ,GAA2B,IAAI,CAAC;IACxC,MAAM,CAAmB;IAEjC,YAAY,SAA2B,EAAE;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,IAAI,wBAAa,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,YAAY,GAAG,IAAI,qBAAY,EAAE,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,0BAAgB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAChE,CAAC;IAED,wCAAwC;IACxC,KAAK,CAAC,OAAO,CAAC,GAAW,EAAE,UAAiC,EAAE;QAC5D,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;QACjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QACxE,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;QACtG,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,UAAiC,EAAE;QAC9D,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;QACjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QACzF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;QACtG,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,uCAAuC;IACvC,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,UAAiC,EAAE;QAC/D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/D,CAAC;IAED,iDAAiD;IACjD,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,UAAiC,EAAE;QAC7D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,UAA0D,EAAE;QACvF,MAAM,EAAE,SAAS,EAAE,GAAG,YAAY,EAAE,GAAG,OAAO,CAAC;QAC/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,CAAC;IAED,8CAA8C;IAC9C,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,KAAa,EAAE,OAAe,CAAC;QACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC/C,CAAC;IAED,4BAA4B;IAC5B,KAAK,CAAC,aAAa,CAAC,OAAe;QACjC,IAAA,wBAAW,EAAC,OAAO,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,oCAAoC;IACpC,KAAK,CAAC,aAAa,CAAC,OAAe,EAAE,UAAiC,EAAE;QACtE,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,qDAAqD;IACrD,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC;QACD,6DAA6D;QAC7D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,4BAA4B;IAC5B,KAAK,CAAC,UAAU,CAAC,GAAW;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,wCAAwC;IACxC,KAAK,CAAC,iBAAiB,CAAC,GAAW,EAAE,UAAyD,EAAE,MAAM,EAAE,MAAM,EAAE;QAC9G,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,GAAG,YAAY,EAAE,GAAG,OAAO,CAAC;QACpF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,IAAI,yBAAgB,EAAE,CAAC;QACxC,OAAO,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,CAAC,CAAC;IAC1G,CAAC;IAED,wCAAwC;IACxC,aAAa,CAAC,GAAW,EAAE,UAAiC,EAAE;QAC5D,MAAM,MAAM,GAAG,IAAI,oBAAW,EAAE,CAAC;QACjC,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;QACjB,kCAAkC;QAClC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1F,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YACf,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK;gBAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACvD,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,MAAM,CAAC,KAAK,EAAE,MAAM;gBAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;;gBAC7C,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACjE,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,iDAAiD;IACjD,eAAe,CAAC,MAAqB;QACnC,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,QAAQ,GAAG,IAAI,yBAAe,EAAE,CAAC;QAC1D,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IAED,4BAA4B;IAC5B,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,IAAI,CAAC;IAC5C,CAAC;IAED,wBAAwB;IACxB,OAAO;QACL,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAEO,YAAY,CAAC,MAAqB,EAAE,SAAkB;QAC5D,MAAM,MAAM,GAAG,SAAS,IAAI,IAAI,CAAC;QACjC,IAAI,MAAM,GAAG,KAAK,MAAM,CAAC,MAAM,MAAM,CAAC;QACtC,IAAI,MAAM,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,MAAM,CAAC,OAAO,MAAM,CAAC;QACxD,MAAM,IAAI,KAAK,MAAM,CAAC,QAAQ,CAAC,SAAS,YAAY,MAAM,CAAC,WAAW,aAAa,MAAM,CAAC,QAAQ,CAAC,WAAW,MAAM,CAAC;QAErH,IAAI,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC9C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM;gBAAE,MAAM;YAC9C,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;gBACtC,MAAM,IAAI,MAAM,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YAC/D,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC;YACjC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;QAC7B,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;CACF;AAlJD,gCAkJC;AAED,kBAAe,UAAU,CAAC"}

package/dist/mcp-server.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=mcp-server.d.ts.map

package/dist/mcp-server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp-server.d.ts","sourceRoot":"","sources":["../src/mcp-server.ts"],"names":[],"mappings":""}

package/dist/mcp-server.js

@@ -0,0 +1,108 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * WebContext MCP Server for Amazon Q Developer / Claude Desktop / any MCP client.
+ *
+ * Usage:
+ *   node dist/mcp-server.js
+ *
+ * This runs as a stdio MCP server that exposes WebContext tools to AI agents.
+ */
+const mcp_1 = require("./sdk/mcp");
+const tools = (0, mcp_1.createMCPTools)({
+    cache: { enabled: true, ttl: 3600, maxSize: 500, contentHashing: true },
+    retry: { maxRetries: 3, backoffMs: 1000, backoffMultiplier: 2, retryOn: [429, 500, 502, 503, 504] },
+    rateLimit: { requestsPerSecond: 2, burstSize: 5 },
+});
+// MCP stdio protocol implementation
+const server = {
+    name: 'webcontext',
+    version: '1.0.0',
+    tools: tools.map(t => ({
+        name: t.name,
+        description: t.description,
+        inputSchema: t.inputSchema,
+    })),
+};
+function sendResponse(id, result) {
+    const msg = JSON.stringify({ jsonrpc: '2.0', id, result });
+    process.stdout.write(`Content-Length: ${Buffer.byteLength(msg)}\r\n\r\n${msg}`);
+}
+function sendError(id, code, message) {
+    const msg = JSON.stringify({ jsonrpc: '2.0', id, error: { code, message } });
+    process.stdout.write(`Content-Length: ${Buffer.byteLength(msg)}\r\n\r\n${msg}`);
+}
+async function handleRequest(request) {
+    const { id, method, params } = request;
+    switch (method) {
+        case 'initialize':
+            sendResponse(id, {
+                protocolVersion: '2024-11-05',
+                capabilities: { tools: {} },
+                serverInfo: { name: server.name, version: server.version },
+            });
+            break;
+        case 'tools/list':
+            sendResponse(id, { tools: server.tools });
+            break;
+        case 'tools/call': {
+            const tool = tools.find(t => t.name === params.name);
+            if (!tool) {
+                sendError(id, -32602, `Unknown tool: ${params.name}`);
+                return;
+            }
+            try {
+                const result = await tool.handler(params.arguments || {});
+                sendResponse(id, {
+                    content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+                });
+            }
+            catch (err) {
+                sendResponse(id, {
+                    content: [{ type: 'text', text: `Error: ${err.message}` }],
+                    isError: true,
+                });
+            }
+            break;
+        }
+        case 'notifications/initialized':
+            // No response needed for notifications
+            break;
+        default:
+            if (id)
+                sendError(id, -32601, `Method not found: ${method}`);
+    }
+}
+// Read stdio input (Content-Length header framing)
+let buffer = '';
+process.stdin.setEncoding('utf-8');
+process.stdin.on('data', (chunk) => {
+    buffer += chunk;
+    while (true) {
+        const headerEnd = buffer.indexOf('\r\n\r\n');
+        if (headerEnd === -1)
+            break;
+        const header = buffer.slice(0, headerEnd);
+        const match = header.match(/Content-Length:\s*(\d+)/i);
+        if (!match) {
+            buffer = buffer.slice(headerEnd + 4);
+            continue;
+        }
+        const contentLength = parseInt(match[1], 10);
+        const bodyStart = headerEnd + 4;
+        if (buffer.length < bodyStart + contentLength)
+            break;
+        const body = buffer.slice(bodyStart, bodyStart + contentLength);
+        buffer = buffer.slice(bodyStart + contentLength);
+        try {
+            const request = JSON.parse(body);
+            handleRequest(request);
+        }
+        catch (err) {
+            sendError(null, -32700, 'Parse error');
+        }
+    }
+});
+process.stderr.write('WebContext MCP Server started\n');
+//# sourceMappingURL=mcp-server.js.map