webcontext-ai 1.0.0

package/dist/chunking/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/chunking/chunker.ts"],"names":[],"mappings":";;;AAAA,uCAAkD;AAClD,mCAAoC;AAGpC;;;GAGG;AACH,MAAa,cAAc;IACjB,OAAO,CAAW;IAClB,OAAO,CAAyB;IAExC,YAAY,UAAwB,EAAE;QACpC,IAAI,CAAC,OAAO,GAAG,IAAA,uBAAY,EAAC,aAAa,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO,GAAG;YACb,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI;YACpC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,GAAG;YAC/B,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,UAAU;YACxC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,IAAI,IAAI;YACtD,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,IAAI;SACnD,CAAC;IACJ,CAAC;IAED,kCAAkC;IAClC,WAAW,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC1C,CAAC;IAED,8CAA8C;IAC9C,KAAK,CAAC,QAAgB,EAAE,SAAiB,EAAE,KAAa,EAAE,QAAmB,EAAE,OAAsB;QACnG,MAAM,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;QAC7C,IAAI,SAAmB,CAAC;QAExB,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,KAAK,SAAS;gBACZ,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;gBAC1C,MAAM;YACR,KAAK,OAAO;gBACV,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;gBACtD,MAAM;YACR,KAAK,WAAW;gBACd,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC5D,MAAM;YACR,KAAK,UAAU,CAAC;YAChB;gBACE,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;gBACzD,MAAM;QACV,CAAC;QAED,+CAA+C;QAC/C,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9D,CAAC;QAED,yBAAyB;QACzB,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAChC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAChF,CAAC;QAEF,gBAAgB;QAChB,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC;QACrC,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;YAC/B,MAAM,EAAE,GAAG,IAAA,mBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,SAAS,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5G,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;YAClE,OAAO;gBACL,EAAE;gBACF,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;gBAC9B,QAAQ,EAAE;oBACR,SAAS;oBACT,KAAK;oBACL,WAAW;oBACX,UAAU,EAAE,CAAC;oBACb,WAAW;oBACX,OAAO,EAAE,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;oBACpC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC;iBACzC;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,mCAAmC;IACnC,OAAO;QACL,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC;IAEO,aAAa,CAAC,OAAe,EAAE,SAAiB;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;YAC9D,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,SAAS,IAAI,MAAM,EAAE,CAAC;gBACrD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3B,MAAM,GAAG,OAAO,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,QAAQ,CAAC;YACpB,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,EAAE;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,cAAc,CAAC,OAAe;QACpC,OAAO,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5E,CAAC;IAEO,UAAU,CAAC,OAAe,EAAE,SAAiB;QACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACvD,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,SAAS,IAAI,MAAM,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3B,MAAM,GAAG,IAAI,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,SAAS,CAAC;YACrB,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,EAAE;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,gBAAgB,CAAC,OAAe,EAAE,SAAiB;QACzD,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAChE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACxD,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,SAAS,IAAI,MAAM,EAAE,CAAC;gBACrD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3B,MAAM,GAAG,IAAI,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,QAAQ,CAAC;YACpB,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,EAAE;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CAAC,QAAgB;QACtC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,WAAW,GAAG,KAAK,CAAC;QAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,WAAW,GAAG,CAAC,WAAW,CAAC;YACvD,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC9D,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3B,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,OAAO,IAAI,IAAI,GAAG,IAAI,CAAC;YACzB,CAAC;QACH,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/C,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,eAAe,CAAC,MAAgB,EAAE,SAAiB;QACzD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YACzD,IAAI,SAAS,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;gBACjD,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAChD,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,SAAS,GAAG,GAAG,EAAE,CAAC;oBAChD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACpB,CAAC,EAAE,CAAC;oBACJ,SAAS;gBACX,CAAC;YACH,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,YAAY,CAAC,MAAgB,EAAE,aAAqB;QAC1D,MAAM,MAAM,GAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC7C,IAAI,OAAO,GAAG,EAAE,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/C,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC/C,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,aAAa;oBAAE,MAAM;gBACvD,OAAO,GAAG,SAAS,CAAC;YACtB,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,cAAc,CAAC,WAAmB,EAAE,SAAiB,EAAE,QAAmB;QAChF,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC/D,IAAI,UAAU,KAAK,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QAChD,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC1D,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC;oBAAE,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;YAC/C,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAEO,mBAAmB,CAAC,OAAe;QACzC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACxC,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;CACF;AAhND,wCAgNC"}

package/dist/cli/index.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/cli/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":""}

package/dist/cli/index.js

@@ -0,0 +1,406 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const chalk_1 = __importDefault(require("chalk"));
+const ora_1 = __importDefault(require("ora"));
+const index_1 = require("../index");
+const vectordb_1 = require("../export/vectordb");
+const scheduler_1 = require("../utils/scheduler");
+const validation_1 = require("../utils/validation");
+const fs_1 = require("fs");
+const program = new commander_1.Command();
+program
+    .name('webcontext')
+    .description('Turn any web content into clean AI-ready context')
+    .version('1.0.0');
+program
+    .command('extract')
+    .description('Extract content from a URL')
+    .argument('<url>', 'URL to extract')
+    .option('-f, --format <format>', 'Output format: markdown|json|chunks', 'markdown')
+    .option('-o, --output <file>', 'Output file path')
+    .option('--focus <mode>', 'Focus mode: full|article|code|api|readme', 'full')
+    .option('--no-js', 'Disable JavaScript rendering')
+    .option('--selector <css>', 'Wait for CSS selector before extraction')
+    .action(async (url, opts) => {
+    try {
+        (0, validation_1.validateUrl)(url);
+        const spinner = (0, ora_1.default)(`Extracting content from ${url}`).start();
+        const wc = new index_1.WebContext();
+        const result = await wc.extract(url, {
+            focusMode: opts.focus,
+            javascript: opts.js !== false,
+            waitForSelector: opts.selector,
+        });
+        spinner.succeed(`Extracted ${result.stats.pagesProcessed} pages (${result.stats.totalTokens} tokens)`);
+        let output;
+        switch (opts.format) {
+            case 'json':
+                output = JSON.stringify(result.context, null, 2);
+                break;
+            case 'chunks':
+                output = JSON.stringify(result.context.chunks, null, 2);
+                break;
+            default:
+                output = result.pages.map(p => p.markdown).join('\n\n---\n\n');
+        }
+        if (opts.output) {
+            (0, fs_1.writeFileSync)(opts.output, output);
+            console.log(chalk_1.default.green(`✓ Written to ${opts.output}`));
+        }
+        else {
+            console.log(output);
+        }
+    }
+    catch (err) {
+        console.error(chalk_1.default.red(`Error: ${err.message}`));
+        process.exit(1);
+    }
+});
+program
+    .command('crawl')
+    .description('Recursively crawl a documentation site')
+    .argument('<url>', 'Base URL to crawl')
+    .option('-d, --depth <n>', 'Crawl depth', '2')
+    .option('-m, --max-pages <n>', 'Max pages to crawl', '50')
+    .option('-f, --format <format>', 'Output format: markdown|json', 'markdown')
+    .option('-o, --output <file>', 'Output file path')
+    .option('--include <patterns...>', 'URL patterns to include')
+    .option('--exclude <patterns...>', 'URL patterns to exclude')
+    .option('--delay <ms>', 'Delay between requests in ms', '500')
+    .option('--sitemap <url>', 'Sitemap URL to use for discovery')
+    .action(async (url, opts) => {
+    try {
+        (0, validation_1.validateUrl)(url);
+        const spinner = (0, ora_1.default)(`Crawling ${url} (depth: ${opts.depth}, max: ${opts.maxPages} pages)`).start();
+        const wc = new index_1.WebContext();
+        const result = await wc.crawlDocs(url, {
+            depth: parseInt(opts.depth),
+            maxPages: parseInt(opts.maxPages),
+            includePatterns: opts.include,
+            excludePatterns: opts.exclude,
+            delay: parseInt(opts.delay),
+            sitemapUrl: opts.sitemap,
+        });
+        spinner.succeed(`${result.stats.pagesProcessed} pages | ${result.stats.totalTokens} tokens | ${result.stats.duration}ms`);
+        if (result.stats.errors.length) {
+            console.log(chalk_1.default.yellow(`⚠ ${result.stats.errors.length} errors`));
+        }
+        const output = opts.format === 'json'
+            ? JSON.stringify(result.context, null, 2)
+            : result.pages.map(p => `# ${p.title}\n\nSource: ${p.url}\n\n${p.markdown}`).join('\n\n---\n\n');
+        if (opts.output) {
+            (0, fs_1.writeFileSync)(opts.output, output);
+            console.log(chalk_1.default.green(`✓ Written to ${opts.output}`));
+        }
+        else {
+            console.log(output);
+        }
+    }
+    catch (err) {
+        console.error(chalk_1.default.red(`Error: ${err.message}`));
+        process.exit(1);
+    }
+});
+program
+    .command('context')
+    .description('Generate LLM-ready context from a URL')
+    .argument('<url>', 'URL to process')
+    .option('--budget <tokens>', 'Token budget for output', '8000')
+    .option('--focus <mode>', 'Focus mode: full|article|code|api|readme', 'full')
+    .action(async (url, opts) => {
+    try {
+        (0, validation_1.validateUrl)(url);
+        const spinner = (0, ora_1.default)('Generating LLM context...').start();
+        const wc = new index_1.WebContext();
+        const context = await wc.toContext(url, {
+            focusMode: opts.focus,
+            maxTokens: parseInt(opts.budget),
+        });
+        spinner.succeed('Context generated');
+        console.log(context);
+    }
+    catch (err) {
+        console.error(chalk_1.default.red(`Error: ${err.message}`));
+        process.exit(1);
+    }
+});
+program
+    .command('search')
+    .description('Semantic search within a page\'s content')
+    .argument('<url>', 'URL to search within')
+    .argument('<query>', 'Search query')
+    .option('-k, --top-k <n>', 'Number of results', '5')
+    .action(async (url, query, opts) => {
+    try {
+        (0, validation_1.validateUrl)(url);
+        const spinner = (0, ora_1.default)(`Extracting and searching ${url}...`).start();
+        const wc = new index_1.WebContext();
+        const results = await wc.search(url, query, parseInt(opts.topK));
+        spinner.succeed(`Found ${results.length} results`);
+        results.forEach((r, i) => {
+            console.log(chalk_1.default.cyan(`\n--- Result ${i + 1} (score: ${r.score.toFixed(3)}) ---`));
+            console.log(r.chunk.content.slice(0, 500));
+        });
+    }
+    catch (err) {
+        console.error(chalk_1.default.red(`Error: ${err.message}`));
+        process.exit(1);
+    }
+});
+program
+    .command('schedule')
+    .description('Schedule a recurring crawl')
+    .argument('<url>', 'URL to crawl')
+    .requiredOption('--cron <expr>', 'Cron expression (e.g., "*/30 * * * *" for every 30 min)')
+    .option('-d, --depth <n>', 'Crawl depth', '2')
+    .option('-o, --output <dir>', 'Output directory for results')
+    .action(async (url, opts) => {
+    try {
+        (0, validation_1.validateUrl)(url);
+        const scheduler = new scheduler_1.CrawlScheduler();
+        const wc = new index_1.WebContext();
+        console.log(chalk_1.default.green(`Scheduled crawl for ${url}`));
+        console.log(chalk_1.default.dim(`  Cron: ${opts.cron}`));
+        console.log(chalk_1.default.dim(`  Depth: ${opts.depth}`));
+        scheduler.schedule('cli-job', {
+            cron: opts.cron,
+            urls: [url],
+            options: { depth: parseInt(opts.depth) },
+            onComplete: (result) => {
+                console.log(chalk_1.default.green(`✓ Crawl complete: ${result.stats.pagesProcessed} pages, ${result.stats.totalTokens} tokens`));
+                if (opts.output) {
+                    (0, fs_1.mkdirSync)(opts.output, { recursive: true });
+                    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+                    (0, fs_1.writeFileSync)(`${opts.output}/crawl-${timestamp}.json`, JSON.stringify(result.context, null, 2));
+                }
+            },
+        }, (crawlUrl, crawlOpts) => wc.crawlDocs(crawlUrl, crawlOpts));
+        console.log(chalk_1.default.green('Scheduler running. Press Ctrl+C to stop.'));
+        await new Promise(() => { }); // Keep alive
+    }
+    catch (err) {
+        console.error(chalk_1.default.red(`Error: ${err.message}`));
+        process.exit(1);
+    }
+});
+program
+    .command('metrics')
+    .description('Show metrics from a running server')
+    .option('-p, --port <port>', 'Server port', '3456')
+    .option('--host <host>', 'Server host', 'localhost')
+    .action(async (opts) => {
+    try {
+        const res = await fetch(`http://${opts.host}:${opts.port}/metrics`);
+        if (!res.ok)
+            throw new Error(`Server returned ${res.status}`);
+        const data = await res.json();
+        console.log(chalk_1.default.cyan('\n--- Server Metrics ---'));
+        for (const [key, value] of Object.entries(data)) {
+            console.log(`  ${chalk_1.default.bold(key)}: ${value}`);
+        }
+    }
+    catch (err) {
+        console.error(chalk_1.default.red(`Error: ${err.message}`));
+        process.exit(1);
+    }
+});
+program
+    .command('validate')
+    .description('Validate a URL is accessible and show page info')
+    .argument('<url>', 'URL to validate')
+    .option('--timeout <ms>', 'Request timeout in ms', '10000')
+    .action(async (url, opts) => {
+    try {
+        (0, validation_1.validateUrl)(url);
+        const spinner = (0, ora_1.default)(`Validating ${url}...`).start();
+        const start = Date.now();
+        const res = await fetch(url, {
+            method: 'HEAD',
+            signal: AbortSignal.timeout(parseInt(opts.timeout)),
+        });
+        const elapsed = Date.now() - start;
+        if (res.ok) {
+            spinner.succeed(`URL is accessible (${res.status})`);
+        }
+        else {
+            spinner.warn(`URL returned status ${res.status}`);
+        }
+        console.log(chalk_1.default.cyan('\n--- Page Info ---'));
+        console.log(`  ${chalk_1.default.bold('Status')}: ${res.status} ${res.statusText}`);
+        console.log(`  ${chalk_1.default.bold('Content-Type')}: ${res.headers.get('content-type') || 'unknown'}`);
+        console.log(`  ${chalk_1.default.bold('Response Time')}: ${elapsed}ms`);
+        console.log(`  ${chalk_1.default.bold('Server')}: ${res.headers.get('server') || 'unknown'}`);
+    }
+    catch (err) {
+        console.error(chalk_1.default.red(`Error: ${err.message}`));
+        process.exit(1);
+    }
+});
+program
+    .command('serve')
+    .description('Start the WebContext API server')
+    .option('-p, --port <port>', 'Port number', '3456')
+    .action(async (opts) => {
+    const { startServer } = await Promise.resolve().then(() => __importStar(require('../sdk/server')));
+    startServer(parseInt(opts.port));
+});
+program
+    .command('github')
+    .description('Extract README and docs from a GitHub repository')
+    .argument('<url>', 'GitHub repository URL')
+    .option('-f, --format <format>', 'Output format: markdown|json', 'markdown')
+    .option('-o, --output <file>', 'Output file path')
+    .option('--docs-only', 'Only extract /docs folder, skip README')
+    .action(async (url, opts) => {
+    try {
+        const spinner = (0, ora_1.default)(`Extracting GitHub repo: ${url}`).start();
+        const wc = new index_1.WebContext();
+        const result = await wc.extractGitHub(url, { depth: opts.docsOnly ? 1 : 1 });
+        spinner.succeed(`${result.stats.pagesProcessed} files | ${result.stats.totalTokens} tokens`);
+        const output = opts.format === 'json'
+            ? JSON.stringify(result.context, null, 2)
+            : result.pages.map(p => `# ${p.title}\n\n${p.markdown}`).join('\n\n---\n\n');
+        if (opts.output) {
+            (0, fs_1.writeFileSync)(opts.output, output);
+            console.log(chalk_1.default.green(`✓ Written to ${opts.output}`));
+        }
+        else {
+            console.log(output);
+        }
+    }
+    catch (err) {
+        console.error(chalk_1.default.red(`Error: ${err.message}`));
+        process.exit(1);
+    }
+});
+program
+    .command('pdf')
+    .description('Extract content from a PDF file')
+    .argument('<source>', 'PDF URL or local file path')
+    .option('-f, --format <format>', 'Output format: markdown|json|chunks', 'markdown')
+    .option('-o, --output <file>', 'Output file path')
+    .action(async (source, opts) => {
+    try {
+        const spinner = (0, ora_1.default)(`Extracting PDF: ${source}`).start();
+        const wc = new index_1.WebContext();
+        const result = await wc.extractPdf(source);
+        spinner.succeed(`Extracted ${result.stats.totalTokens} tokens from PDF`);
+        let output;
+        switch (opts.format) {
+            case 'json':
+                output = JSON.stringify(result.context, null, 2);
+                break;
+            case 'chunks':
+                output = JSON.stringify(result.context.chunks, null, 2);
+                break;
+            default:
+                output = result.pages[0]?.markdown || '';
+        }
+        if (opts.output) {
+            (0, fs_1.writeFileSync)(opts.output, output);
+            console.log(chalk_1.default.green(`✓ Written to ${opts.output}`));
+        }
+        else {
+            console.log(output);
+        }
+    }
+    catch (err) {
+        console.error(chalk_1.default.red(`Error: ${err.message}`));
+        process.exit(1);
+    }
+});
+program
+    .command('export')
+    .description('Export content as vector DB-ready format')
+    .argument('<url>', 'URL to extract and export')
+    .requiredOption('--to <format>', 'Target format: pinecone|chroma|weaviate|qdrant|json')
+    .option('-o, --output <file>', 'Output file path')
+    .option('--namespace <ns>', 'Namespace/collection name')
+    .option('-d, --depth <n>', 'Crawl depth', '0')
+    .option('-m, --max-pages <n>', 'Max pages', '20')
+    .action(async (url, opts) => {
+    try {
+        (0, validation_1.validateUrl)(url);
+        const spinner = (0, ora_1.default)(`Extracting and exporting to ${opts.to} format...`).start();
+        const wc = new index_1.WebContext();
+        const result = parseInt(opts.depth) > 0
+            ? await wc.crawlDocs(url, { depth: parseInt(opts.depth), maxPages: parseInt(opts.maxPages) })
+            : await wc.extract(url);
+        const exporter = new vectordb_1.VectorDBExporter();
+        const output = exporter.exportChunks(result.context.chunks, {
+            format: opts.to,
+            namespace: opts.namespace,
+            collection: opts.namespace,
+        });
+        spinner.succeed(`Exported ${result.context.chunks.length} chunks in ${opts.to} format`);
+        if (opts.output) {
+            (0, fs_1.writeFileSync)(opts.output, output);
+            console.log(chalk_1.default.green(`✓ Written to ${opts.output}`));
+        }
+        else {
+            console.log(output);
+        }
+    }
+    catch (err) {
+        console.error(chalk_1.default.red(`Error: ${err.message}`));
+        process.exit(1);
+    }
+});
+program
+    .command('screenshot')
+    .description('Capture a screenshot of a web page')
+    .argument('<url>', 'URL to screenshot')
+    .option('-o, --output <dir>', 'Output directory', './screenshots')
+    .option('--full-page', 'Capture full page (not just viewport)')
+    .option('--width <px>', 'Viewport width', '1280')
+    .option('--height <px>', 'Viewport height', '720')
+    .option('--format <fmt>', 'Image format: png|jpeg', 'png')
+    .action(async (url, opts) => {
+    try {
+        (0, validation_1.validateUrl)(url);
+        const spinner = (0, ora_1.default)(`Capturing screenshot: ${url}`).start();
+        const { ScreenshotCapture } = await Promise.resolve().then(() => __importStar(require('../extractors/screenshot')));
+        const capture = new ScreenshotCapture(opts.output);
+        const filepath = await capture.capture(url, {
+            fullPage: opts.fullPage,
+            width: parseInt(opts.width),
+            height: parseInt(opts.height),
+            format: opts.format,
+        });
+        spinner.succeed(`Screenshot saved: ${filepath}`);
+    }
+    catch (err) {
+        console.error(chalk_1.default.red(`Error: ${err.message}`));
+        process.exit(1);
+    }
+});
+program.parse();
+//# sourceMappingURL=index.js.map

package/dist/cli/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,yCAAoC;AACpC,kDAA0B;AAC1B,8CAAsB;AACtB,oCAAsC;AACtC,iDAAsD;AACtD,kDAAoD;AACpD,oDAAkD;AAClD,2BAA8C;AAE9C,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,YAAY,CAAC;KAClB,WAAW,CAAC,kDAAkD,CAAC;KAC/D,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,SAAS,CAAC;KAClB,WAAW,CAAC,4BAA4B,CAAC;KACzC,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC;KACnC,MAAM,CAAC,uBAAuB,EAAE,qCAAqC,EAAE,UAAU,CAAC;KAClF,MAAM,CAAC,qBAAqB,EAAE,kBAAkB,CAAC;KACjD,MAAM,CAAC,gBAAgB,EAAE,0CAA0C,EAAE,MAAM,CAAC;KAC5E,MAAM,CAAC,SAAS,EAAE,8BAA8B,CAAC;KACjD,MAAM,CAAC,kBAAkB,EAAE,yCAAyC,CAAC;KACrE,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,IAAI,EAAE,EAAE;IAClC,IAAI,CAAC;QACH,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;QACjB,MAAM,OAAO,GAAG,IAAA,aAAG,EAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAC9D,MAAM,EAAE,GAAG,IAAI,kBAAU,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACnC,SAAS,EAAE,IAAI,CAAC,KAAK;YACrB,UAAU,EAAE,IAAI,CAAC,EAAE,KAAK,KAAK;YAC7B,eAAe,EAAE,IAAI,CAAC,QAAQ;SAC/B,CAAC,CAAC;QACH,OAAO,CAAC,OAAO,CAAC,aAAa,MAAM,CAAC,KAAK,CAAC,cAAc,WAAW,MAAM,CAAC,KAAK,CAAC,WAAW,UAAU,CAAC,CAAC;QAEvG,IAAI,MAAc,CAAC;QACnB,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;YACpB,KAAK,MAAM;gBACT,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBACjD,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBACxD,MAAM;YACR;gBACE,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAA,kBAAa,EAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,wCAAwC,CAAC;KACrD,QAAQ,CAAC,OAAO,EAAE,mBAAmB,CAAC;KACtC,MAAM,CAAC,iBAAiB,EAAE,aAAa,EAAE,GAAG,CAAC;KAC7C,MAAM,CAAC,qBAAqB,EAAE,oBAAoB,EAAE,IAAI,CAAC;KACzD,MAAM,CAAC,uBAAuB,EAAE,8BAA8B,EAAE,UAAU,CAAC;KAC3E,MAAM,CAAC,qBAAqB,EAAE,kBAAkB,CAAC;KACjD,MAAM,CAAC,yBAAyB,EAAE,yBAAyB,CAAC;KAC5D,MAAM,CAAC,yBAAyB,EAAE,yBAAyB,CAAC;KAC5D,MAAM,CAAC,cAAc,EAAE,8BAA8B,EAAE,KAAK,CAAC;KAC7D,MAAM,CAAC,iBAAiB,EAAE,kCAAkC,CAAC;KAC7D,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,IAAI,EAAE,EAAE;IAClC,IAAI,CAAC;QACH,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;QACjB,MAAM,OAAO,GAAG,IAAA,aAAG,EAAC,YAAY,GAAG,YAAY,IAAI,CAAC,KAAK,UAAU,IAAI,CAAC,QAAQ,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;QACnG,MAAM,EAAE,GAAG,IAAI,kBAAU,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE;YACrC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;YAC3B,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;YACjC,eAAe,EAAE,IAAI,CAAC,OAAO;YAC7B,eAAe,EAAE,IAAI,CAAC,OAAO;YAC7B,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;YAC3B,UAAU,EAAE,IAAI,CAAC,OAAO;SACzB,CAAC,CAAC;QACH,OAAO,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,cAAc,YAAY,MAAM,CAAC,KAAK,CAAC,WAAW,aAAa,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;QAE1H,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,KAAK,MAAM;YACnC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,eAAe,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAEnG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAA,kBAAa,EAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,SAAS,CAAC;KAClB,WAAW,CAAC,uCAAuC,CAAC;KACpD,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC;KACnC,MAAM,CAAC,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,CAAC;KAC9D,MAAM,CAAC,gBAAgB,EAAE,0CAA0C,EAAE,MAAM,CAAC;KAC5E,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,IAAI,EAAE,EAAE;IAClC,IAAI,CAAC;QACH,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;QACjB,MAAM,OAAO,GAAG,IAAA,aAAG,EAAC,2BAA2B,CAAC,CAAC,KAAK,EAAE,CAAC;QACzD,MAAM,EAAE,GAAG,IAAI,kBAAU,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE;YACtC,SAAS,EAAE,IAAI,CAAC,KAAK;YACrB,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;SACjC,CAAC,CAAC;QACH,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,0CAA0C,CAAC;KACvD,QAAQ,CAAC,OAAO,EAAE,sBAAsB,CAAC;KACzC,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;KACnC,MAAM,CAAC,iBAAiB,EAAE,mBAAmB,EAAE,GAAG,CAAC;KACnD,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,KAAa,EAAE,IAAI,EAAE,EAAE;IACjD,IAAI,CAAC;QACH,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;QACjB,MAAM,OAAO,GAAG,IAAA,aAAG,EAAC,4BAA4B,GAAG,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;QAClE,MAAM,EAAE,GAAG,IAAI,kBAAU,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACjE,OAAO,CAAC,OAAO,CAAC,SAAS,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QAEnD,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACpF,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,UAAU,CAAC;KACnB,WAAW,CAAC,4BAA4B,CAAC;KACzC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;KACjC,cAAc,CAAC,eAAe,EAAE,yDAAyD,CAAC;KAC1F,MAAM,CAAC,iBAAiB,EAAE,aAAa,EAAE,GAAG,CAAC;KAC7C,MAAM,CAAC,oBAAoB,EAAE,8BAA8B,CAAC;KAC5D,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,IAAI,EAAE,EAAE;IAClC,IAAI,CAAC;QACH,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;QACjB,MAAM,SAAS,GAAG,IAAI,0BAAc,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,IAAI,kBAAU,EAAE,CAAC;QAE5B,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAEjD,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE;YAC5B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,CAAC,GAAG,CAAC;YACX,OAAO,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;YACxC,UAAU,EAAE,CAAC,MAAM,EAAE,EAAE;gBACrB,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,qBAAqB,MAAM,CAAC,KAAK,CAAC,cAAc,WAAW,MAAM,CAAC,KAAK,CAAC,WAAW,SAAS,CAAC,CAAC,CAAC;gBACvH,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBAChB,IAAA,cAAS,EAAC,IAAI,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC5C,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;oBACjE,IAAA,kBAAa,EAAC,GAAG,IAAI,CAAC,MAAM,UAAU,SAAS,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;gBACnG,CAAC;YACH,CAAC;SACF,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QAE/D,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC,CAAC;QACrE,MAAM,IAAI,OAAO,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,aAAa;IAC5C,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,SAAS,CAAC;KAClB,WAAW,CAAC,oCAAoC,CAAC;KACjD,MAAM,CAAC,mBAAmB,EAAE,aAAa,EAAE,MAAM,CAAC;KAClD,MAAM,CAAC,eAAe,EAAE,aAAa,EAAE,WAAW,CAAC;KACnD,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACrB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,UAAU,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,UAAU,CAAC,CAAC;QACpE,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC;QACpD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,CAAC,GAAG,CAAC,KAAK,eAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,UAAU,CAAC;KACnB,WAAW,CAAC,iDAAiD,CAAC;KAC9D,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC;KACpC,MAAM,CAAC,gBAAgB,EAAE,uBAAuB,EAAE,OAAO,CAAC;KAC1D,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,IAAI,EAAE,EAAE;IAClC,IAAI,CAAC;QACH,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;QACjB,MAAM,OAAO,GAAG,IAAA,aAAG,EAAC,cAAc,GAAG,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,MAAM,EAAE,MAAM;YACd,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SACpD,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEnC,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,OAAO,CAAC,OAAO,CAAC,sBAAsB,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QACvD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,uBAAuB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QACpD,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,KAAK,eAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CAAC,KAAK,eAAK,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,SAAS,EAAE,CAAC,CAAC;QAChG,OAAO,CAAC,GAAG,CAAC,KAAK,eAAK,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,OAAO,IAAI,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,eAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,SAAS,EAAE,CAAC,CAAC;IACtF,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,iCAAiC,CAAC;KAC9C,MAAM,CAAC,mBAAmB,EAAE,aAAa,EAAE,MAAM,CAAC;KAClD,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACrB,MAAM,EAAE,WAAW,EAAE,GAAG,wDAAa,eAAe,GAAC,CAAC;IACtD,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AACnC,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,kDAAkD,CAAC;KAC/D,QAAQ,CAAC,OAAO,EAAE,uBAAuB,CAAC;KAC1C,MAAM,CAAC,uBAAuB,EAAE,8BAA8B,EAAE,UAAU,CAAC;KAC3E,MAAM,CAAC,qBAAqB,EAAE,kBAAkB,CAAC;KACjD,MAAM,CAAC,aAAa,EAAE,wCAAwC,CAAC;KAC/D,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,IAAI,EAAE,EAAE;IAClC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAA,aAAG,EAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAC9D,MAAM,EAAE,GAAG,IAAI,kBAAU,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7E,OAAO,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,cAAc,YAAY,MAAM,CAAC,KAAK,CAAC,WAAW,SAAS,CAAC,CAAC;QAE7F,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,KAAK,MAAM;YACnC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE/E,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAA,kBAAa,EAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,iCAAiC,CAAC;KAC9C,QAAQ,CAAC,UAAU,EAAE,4BAA4B,CAAC;KAClD,MAAM,CAAC,uBAAuB,EAAE,qCAAqC,EAAE,UAAU,CAAC;KAClF,MAAM,CAAC,qBAAqB,EAAE,kBAAkB,CAAC;KACjD,MAAM,CAAC,KAAK,EAAE,MAAc,EAAE,IAAI,EAAE,EAAE;IACrC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAA,aAAG,EAAC,mBAAmB,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QACzD,MAAM,EAAE,GAAG,IAAI,kBAAU,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,CAAC,OAAO,CAAC,aAAa,MAAM,CAAC,KAAK,CAAC,WAAW,kBAAkB,CAAC,CAAC;QAEzE,IAAI,MAAc,CAAC;QACnB,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;YACpB,KAAK,MAAM;gBACT,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBACjD,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBACxD,MAAM;YACR;gBACE,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAA,kBAAa,EAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,0CAA0C,CAAC;KACvD,QAAQ,CAAC,OAAO,EAAE,2BAA2B,CAAC;KAC9C,cAAc,CAAC,eAAe,EAAE,qDAAqD,CAAC;KACtF,MAAM,CAAC,qBAAqB,EAAE,kBAAkB,CAAC;KACjD,MAAM,CAAC,kBAAkB,EAAE,2BAA2B,CAAC;KACvD,MAAM,CAAC,iBAAiB,EAAE,aAAa,EAAE,GAAG,CAAC;KAC7C,MAAM,CAAC,qBAAqB,EAAE,WAAW,EAAE,IAAI,CAAC;KAChD,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,IAAI,EAAE,EAAE;IAClC,IAAI,CAAC;QACH,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;QACjB,MAAM,OAAO,GAAG,IAAA,aAAG,EAAC,+BAA+B,IAAI,CAAC,EAAE,YAAY,CAAC,CAAC,KAAK,EAAE,CAAC;QAChF,MAAM,EAAE,GAAG,IAAI,kBAAU,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;YACrC,CAAC,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7F,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAE1B,MAAM,QAAQ,GAAG,IAAI,2BAAgB,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE;YAC1D,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,UAAU,EAAE,IAAI,CAAC,SAAS;SAC3B,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,YAAY,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,cAAc,IAAI,CAAC,EAAE,SAAS,CAAC,CAAC;QAExF,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAA,kBAAa,EAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,YAAY,CAAC;KACrB,WAAW,CAAC,oCAAoC,CAAC;KACjD,QAAQ,CAAC,OAAO,EAAE,mBAAmB,CAAC;KACtC,MAAM,CAAC,oBAAoB,EAAE,kBAAkB,EAAE,eAAe,CAAC;KACjE,MAAM,CAAC,aAAa,EAAE,uCAAuC,CAAC;KAC9D,MAAM,CAAC,cAAc,EAAE,gBAAgB,EAAE,MAAM,CAAC;KAChD,MAAM,CAAC,eAAe,EAAE,iBAAiB,EAAE,KAAK,CAAC;KACjD,MAAM,CAAC,gBAAgB,EAAE,wBAAwB,EAAE,KAAK,CAAC;KACzD,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,IAAI,EAAE,EAAE;IAClC,IAAI,CAAC;QACH,IAAA,wBAAW,EAAC,GAAG,CAAC,CAAC;QACjB,MAAM,OAAO,GAAG,IAAA,aAAG,EAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAC5D,MAAM,EAAE,iBAAiB,EAAE,GAAG,wDAAa,0BAA0B,GAAC,CAAC;QACvE,MAAM,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE;YAC1C,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;YAC3B,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;YAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,OAAO,CAAC,OAAO,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/dist/core/pipeline.d.ts

@@ -0,0 +1,35 @@
+import { CrawlOptions, CrawlResult, WebContextConfig } from './types';
+/**
+ * Main crawling pipeline that orchestrates the full extraction workflow:
+ * URL → Fetch → Extract → Transform → Chunk → Output
+ */
+export declare class CrawlPipeline {
+    private browser;
+    private extractor;
+    private pdfExtractor;
+    private githubExtractor;
+    private transformer;
+    private chunker;
+    private cache;
+    private sitemapParser;
+    private dedup;
+    private config;
+    constructor(config?: WebContextConfig);
+    crawl(options: CrawlOptions): Promise<CrawlResult>;
+    private parseSitemap;
+    private saveCheckpoint;
+    private loadCheckpoint;
+    private runPlugins;
+    private generateSummary;
+    private buildRelationships;
+    private matchesPatterns;
+    /** Resolve relative markdown links to absolute URLs */
+    private resolveLinks;
+    /** Handle PDF extraction */
+    private handlePdf;
+    /** Handle GitHub repository extraction */
+    private handleGitHub;
+    /** Free resources (tiktoken encoder) */
+    dispose(): void;
+}
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/core/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/core/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,YAAY,EAAE,WAAW,EAEO,gBAAgB,EACjD,MAAM,SAAS,CAAC;AAgBjB;;;GAGG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,SAAS,CAAmB;IACpC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,WAAW,CAAsB;IACzC,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,MAAM,CAAmB;gBAErB,MAAM,GAAE,gBAAqB;IAqBnC,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC;YA8R1C,YAAY;IAW1B,OAAO,CAAC,cAAc;IAKtB,OAAO,CAAC,cAAc;YAUR,UAAU;IAUxB,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAC,kBAAkB;IAa1B,OAAO,CAAC,eAAe;IAMvB,uDAAuD;IACvD,OAAO,CAAC,YAAY;IAYpB,4BAA4B;YACd,SAAS;IA6BvB,0CAA0C;YAC5B,YAAY;IA6C1B,wCAAwC;IACxC,OAAO,IAAI,IAAI;CAIhB"}