webcontext-ai 1.0.0

package/dist/browser/manager.js

@@ -0,0 +1,215 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BrowserManager = void 0;
+const robots_parser_1 = __importDefault(require("robots-parser"));
+const DEFAULT_RETRY = { maxRetries: 3, backoffMs: 1000, backoffMultiplier: 2, retryOn: [429, 500, 502, 503, 504] };
+const DEFAULT_RATE_LIMIT = { requestsPerSecond: 2, burstSize: 5 };
+/**
+ * Browser manager using Playwright for JS-heavy page rendering.
+ * Handles rate limiting, retry with backoff, and robots.txt compliance.
+ */
+class BrowserManager {
+    browser = null;
+    context = null;
+    config;
+    rateLimitConfig;
+    robotsCache = new Map();
+    tokens;
+    lastRefill;
+    constructor(config = {}, rateLimitConfig) {
+        this.config = {
+            headless: true,
+            userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+            viewport: { width: 1280, height: 720 },
+            ...config,
+        };
+        this.rateLimitConfig = rateLimitConfig || DEFAULT_RATE_LIMIT;
+        this.tokens = this.rateLimitConfig.burstSize;
+        this.lastRefill = Date.now();
+    }
+    refillTokens() {
+        const now = Date.now();
+        const elapsed = (now - this.lastRefill) / 1000;
+        this.tokens = Math.min(this.rateLimitConfig.burstSize, this.tokens + elapsed * this.rateLimitConfig.requestsPerSecond);
+        this.lastRefill = now;
+    }
+    async waitForToken() {
+        this.refillTokens();
+        if (this.tokens >= 1) {
+            this.tokens -= 1;
+            return;
+        }
+        const waitMs = ((1 - this.tokens) / this.rateLimitConfig.requestsPerSecond) * 1000;
+        await new Promise(resolve => setTimeout(resolve, waitMs));
+        this.refillTokens();
+        this.tokens -= 1;
+    }
+    async launch() {
+        if (this.browser)
+            return;
+        let chromium;
+        try {
+            const pw = await Promise.resolve().then(() => __importStar(require('playwright')));
+            chromium = pw.chromium;
+        }
+        catch {
+            throw new Error('Playwright is required for JavaScript rendering but is not installed.\n' +
+                'Install it with: npm install playwright && npx playwright install chromium\n' +
+                'Or use { javascript: false } to extract without a browser.');
+        }
+        this.browser = await chromium.launch({
+            headless: this.config.headless,
+            args: this.config.proxy ? [`--proxy-server=${this.config.proxy}`] : [],
+        });
+        this.context = await this.browser.newContext({
+            userAgent: this.config.userAgent,
+            viewport: this.config.viewport,
+        });
+    }
+    async checkRobots(url) {
+        const origin = new URL(url).origin;
+        if (!this.robotsCache.has(origin)) {
+            try {
+                const robotsUrl = `${origin}/robots.txt`;
+                const response = await fetch(robotsUrl, {
+                    headers: { 'User-Agent': this.config.userAgent || 'Mozilla/5.0' },
+                    signal: AbortSignal.timeout(5000),
+                });
+                const body = response.ok ? await response.text() : '';
+                this.robotsCache.set(origin, (0, robots_parser_1.default)(robotsUrl, body));
+            }
+            catch {
+                this.robotsCache.set(origin, (0, robots_parser_1.default)(`${origin}/robots.txt`, ''));
+            }
+        }
+        return this.robotsCache.get(origin).isAllowed(url, this.config.userAgent || '*') ?? true;
+    }
+    async fetchWithRetry(fn, retryConfig = DEFAULT_RETRY) {
+        let lastError = null;
+        for (let attempt = 0; attempt <= retryConfig.maxRetries; attempt++) {
+            try {
+                return await fn();
+            }
+            catch (err) {
+                lastError = err;
+                if (attempt === retryConfig.maxRetries)
+                    break;
+                const delay = retryConfig.backoffMs * Math.pow(retryConfig.backoffMultiplier, attempt);
+                await new Promise(resolve => setTimeout(resolve, delay));
+            }
+        }
+        throw lastError;
+    }
+    async fetchPage(url, options = {}) {
+        if (options.respectRobots !== false) {
+            const allowed = await this.checkRobots(url);
+            if (!allowed)
+                throw new Error(`Blocked by robots.txt: ${url}`);
+        }
+        await this.waitForToken();
+        if (!this.context)
+            await this.launch();
+        return this.fetchWithRetry(async () => {
+            if (options.cookies?.length) {
+                await this.context.addCookies(options.cookies.map((c) => ({ ...c, path: c.path || '/' })));
+            }
+            const page = await this.context.newPage();
+            if (options.headers) {
+                await page.setExtraHTTPHeaders(options.headers);
+            }
+            // Block unnecessary resources for speed
+            await page.route('**/*', (route) => {
+                const type = route.request().resourceType();
+                if (['image', 'media', 'font', 'stylesheet'].includes(type)) {
+                    route.abort();
+                }
+                else {
+                    route.continue();
+                }
+            });
+            try {
+                const response = await page.goto(url, {
+                    waitUntil: 'domcontentloaded',
+                    timeout: options.timeout || 30000,
+                });
+                if (options.waitForSelector) {
+                    await page.waitForSelector(options.waitForSelector, { timeout: 10000 }).catch(() => { });
+                }
+                // Wait for dynamic content
+                await page.waitForTimeout(1000);
+                const content = await page.content();
+                const status = response?.status() || 200;
+                // Throw on retryable status codes to trigger retry
+                if ((options.retryConfig?.retryOn || DEFAULT_RETRY.retryOn).includes(status)) {
+                    throw new Error(`HTTP ${status}`);
+                }
+                return { content, status };
+            }
+            finally {
+                await page.close();
+            }
+        }, options.retryConfig);
+    }
+    async fetchStatic(url, options = {}) {
+        if (options.respectRobots !== false) {
+            const allowed = await this.checkRobots(url);
+            if (!allowed)
+                throw new Error(`Blocked by robots.txt: ${url}`);
+        }
+        await this.waitForToken();
+        return this.fetchWithRetry(async () => {
+            const response = await fetch(url, {
+                headers: {
+                    'User-Agent': this.config.userAgent || 'WebContext/2.0',
+                    ...options.headers,
+                },
+                signal: AbortSignal.timeout(30000),
+            });
+            const buffer = Buffer.from(await response.arrayBuffer());
+            const status = response.status;
+            if ((options.retryConfig?.retryOn || DEFAULT_RETRY.retryOn).includes(status)) {
+                throw new Error(`HTTP ${status}`);
+            }
+            return { body: buffer, status };
+        }, options.retryConfig);
+    }
+    async close() {
+        if (this.context) {
+            await this.context.close();
+            this.context = null;
+        }
+        if (this.browser) {
+            await this.browser.close();
+            this.browser = null;
+        }
+        this.robotsCache.clear();
+    }
+}
+exports.BrowserManager = BrowserManager;
+//# sourceMappingURL=manager.js.map

package/dist/browser/manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"manager.js","sourceRoot":"","sources":["../../src/browser/manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kEAAyC;AAGzC,MAAM,aAAa,GAAgB,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;AAChI,MAAM,kBAAkB,GAAoB,EAAE,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AAEnF;;;GAGG;AACH,MAAa,cAAc;IACjB,OAAO,GAAQ,IAAI,CAAC;IACpB,OAAO,GAAQ,IAAI,CAAC;IACpB,MAAM,CAAgB;IACtB,eAAe,CAAkB;IACjC,WAAW,GAAqB,IAAI,GAAG,EAAE,CAAC;IAC1C,MAAM,CAAS;IACf,UAAU,CAAS;IAE3B,YAAY,SAAwB,EAAE,EAAE,eAAiC;QACvE,IAAI,CAAC,MAAM,GAAG;YACZ,QAAQ,EAAE,IAAI;YACd,SAAS,EAAE,iHAAiH;YAC5H,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;YACtC,GAAG,MAAM;SACV,CAAC;QACF,IAAI,CAAC,eAAe,GAAG,eAAe,IAAI,kBAAkB,CAAC;QAC7D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC;QAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC/B,CAAC;IAEO,YAAY;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;QAC/C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC;QACvH,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YACjB,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,GAAG,IAAI,CAAC;QACnF,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,MAAM;QACV,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,QAAa,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,wDAAa,YAAY,GAAC,CAAC;YACtC,QAAQ,GAAG,EAAE,CAAC,QAAQ,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,yEAAyE;gBACzE,8EAA8E;gBAC9E,4DAA4D,CAC7D,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;YACnC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;YAC9B,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE;SACvE,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;YAC3C,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YAChC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,GAAW;QAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACnC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,GAAG,MAAM,aAAa,CAAC;gBACzC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;oBACtC,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,aAAa,EAAE;oBACjE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;iBAClC,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAA,uBAAY,EAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;YAC9D,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAA,uBAAY,EAAC,GAAG,MAAM,aAAa,EAAE,EAAE,CAAC,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC;IAC5F,CAAC;IAED,KAAK,CAAC,cAAc,CAAI,EAAoB,EAAE,cAA2B,aAAa;QACpF,IAAI,SAAS,GAAiB,IAAI,CAAC;QACnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACnE,IAAI,CAAC;gBACH,OAAO,MAAM,EAAE,EAAE,CAAC;YACpB,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,SAAS,GAAG,GAAG,CAAC;gBAChB,IAAI,OAAO,KAAK,WAAW,CAAC,UAAU;oBAAE,MAAM;gBAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;gBACvF,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QACD,MAAM,SAAS,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,UAOzB,EAAE;QACJ,IAAI,OAAO,CAAC,aAAa,KAAK,KAAK,EAAE,CAAC;YACpC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QAEvC,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,IAAI,EAAE;YACpC,IAAI,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YAClG,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC1C,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAClD,CAAC;YAED,wCAAwC;YACxC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAU,EAAE,EAAE;gBACtC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;gBAC5C,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC5D,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;oBACpC,SAAS,EAAE,kBAAkB;oBAC7B,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,KAAK;iBAClC,CAAC,CAAC;gBAEH,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;oBAC5B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAC1F,CAAC;gBAED,2BAA2B;gBAC3B,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAEhC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBACrC,MAAM,MAAM,GAAG,QAAQ,EAAE,MAAM,EAAE,IAAI,GAAG,CAAC;gBAEzC,mDAAmD;gBACnD,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,IAAI,aAAa,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC7E,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,EAAE,CAAC,CAAC;gBACpC,CAAC;gBAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;YAC7B,CAAC;oBAAS,CAAC;gBACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;QACH,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,UAI3B,EAAE;QACJ,IAAI,OAAO,CAAC,aAAa,KAAK,KAAK,EAAE,CAAC;YACpC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,IAAI,EAAE;YACpC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO,EAAE;oBACP,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,gBAAgB;oBACvD,GAAG,OAAO,CAAC,OAAO;iBACnB;gBACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;aACnC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;YACzD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE/B,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,IAAI,aAAa,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7E,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,EAAE,CAAC,CAAC;YACpC,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAClC,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;CACF;AA1MD,wCA0MC"}

package/dist/cache/cache.d.ts

@@ -0,0 +1,22 @@
+import { CacheConfig, ExtractedContent, ContentDiff } from '../core/types';
+/**
+ * Dual-layer cache (LRU memory + file-based) with content hashing for diff detection.
+ */
+export declare class CrawlCache {
+    private memoryCache;
+    private hashCache;
+    private config;
+    private cacheDir;
+    constructor(config?: Partial<CacheConfig>);
+    private hash;
+    private urlToKey;
+    private readFileEntry;
+    get(url: string): ExtractedContent | undefined;
+    set(url: string, content: ExtractedContent): void;
+    has(url: string): boolean;
+    invalidate(url: string): void;
+    clear(): void;
+    getContentHash(url: string): string | undefined;
+    hasChanged(url: string, newContent: string): ContentDiff;
+}
+//# sourceMappingURL=cache.d.ts.map

package/dist/cache/cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/cache/cache.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAU3E;;GAEG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,WAAW,CAAqC;IACxD,OAAO,CAAC,SAAS,CAA2B;IAC5C,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,QAAQ,CAAS;gBAEb,MAAM,GAAE,OAAO,CAAC,WAAW,CAAM;IAsB7C,OAAO,CAAC,IAAI;IAIZ,OAAO,CAAC,QAAQ;IAIhB,OAAO,CAAC,aAAa;IAgBrB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAgB9C,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAajD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAMzB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAO7B,KAAK,IAAI,IAAI;IAUb,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAO/C,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,WAAW;CAsBzD"}

package/dist/cache/cache.js

@@ -0,0 +1,150 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CrawlCache = void 0;
+const crypto_1 = require("crypto");
+const fs_1 = require("fs");
+const path_1 = require("path");
+const lru_cache_1 = require("lru-cache");
+const zod_1 = require("zod");
+const urlSchema = zod_1.z.string().url();
+/**
+ * Dual-layer cache (LRU memory + file-based) with content hashing for diff detection.
+ */
+class CrawlCache {
+    memoryCache;
+    hashCache;
+    config;
+    cacheDir;
+    constructor(config = {}) {
+        this.config = {
+            enabled: config.enabled ?? true,
+            ttl: config.ttl ?? 3600,
+            maxSize: config.maxSize ?? 500,
+            directory: config.directory,
+            contentHashing: config.contentHashing ?? true,
+        };
+        this.cacheDir = this.config.directory || (0, path_1.join)(process.cwd(), '.webcontext-cache');
+        if (this.config.enabled && !(0, fs_1.existsSync)(this.cacheDir)) {
+            (0, fs_1.mkdirSync)(this.cacheDir, { recursive: true });
+        }
+        this.memoryCache = new lru_cache_1.LRUCache({
+            max: this.config.maxSize,
+            ttl: this.config.ttl * 1000,
+        });
+        this.hashCache = new lru_cache_1.LRUCache({
+            max: this.config.maxSize,
+            ttl: this.config.ttl * 1000,
+        });
+    }
+    hash(content) {
+        return (0, crypto_1.createHash)('sha256').update(content).digest('hex');
+    }
+    urlToKey(url) {
+        return (0, crypto_1.createHash)('md5').update(url).digest('hex');
+    }
+    readFileEntry(url) {
+        const filepath = (0, path_1.join)(this.cacheDir, this.urlToKey(url) + '.json');
+        if (!(0, fs_1.existsSync)(filepath))
+            return null;
+        try {
+            const entry = JSON.parse((0, fs_1.readFileSync)(filepath, 'utf-8'));
+            const age = (Date.now() - entry.timestamp) / 1000;
+            if (age > this.config.ttl) {
+                (0, fs_1.rmSync)(filepath);
+                return null;
+            }
+            return entry;
+        }
+        catch {
+            return null;
+        }
+    }
+    get(url) {
+        if (!this.config.enabled)
+            return undefined;
+        try {
+            urlSchema.parse(url);
+        }
+        catch {
+            return undefined;
+        }
+        const memResult = this.memoryCache.get(url);
+        if (memResult)
+            return memResult;
+        const fileEntry = this.readFileEntry(url);
+        if (fileEntry) {
+            this.memoryCache.set(url, fileEntry.data);
+            this.hashCache.set(url, fileEntry.hash);
+            return fileEntry.data;
+        }
+        return undefined;
+    }
+    set(url, content) {
+        if (!this.config.enabled)
+            return;
+        try {
+            urlSchema.parse(url);
+        }
+        catch {
+            return;
+        }
+        const contentHash = this.hash(JSON.stringify(content));
+        this.memoryCache.set(url, content);
+        this.hashCache.set(url, contentHash);
+        const entry = { hash: contentHash, timestamp: Date.now(), data: content };
+        const filepath = (0, path_1.join)(this.cacheDir, this.urlToKey(url) + '.json');
+        (0, fs_1.writeFileSync)(filepath, JSON.stringify(entry));
+    }
+    has(url) {
+        if (!this.config.enabled)
+            return false;
+        if (this.memoryCache.has(url))
+            return true;
+        return this.readFileEntry(url) !== null;
+    }
+    invalidate(url) {
+        this.memoryCache.delete(url);
+        this.hashCache.delete(url);
+        const filepath = (0, path_1.join)(this.cacheDir, this.urlToKey(url) + '.json');
+        if ((0, fs_1.existsSync)(filepath))
+            (0, fs_1.rmSync)(filepath);
+    }
+    clear() {
+        this.memoryCache.clear();
+        this.hashCache.clear();
+        if ((0, fs_1.existsSync)(this.cacheDir)) {
+            for (const file of (0, fs_1.readdirSync)(this.cacheDir)) {
+                if (file.endsWith('.json'))
+                    (0, fs_1.rmSync)((0, path_1.join)(this.cacheDir, file));
+            }
+        }
+    }
+    getContentHash(url) {
+        const memHash = this.hashCache.get(url);
+        if (memHash)
+            return memHash;
+        const fileEntry = this.readFileEntry(url);
+        return fileEntry?.hash;
+    }
+    hasChanged(url, newContent) {
+        const currentHash = this.hash(newContent);
+        const previousHash = this.getContentHash(url) || '';
+        const changed = previousHash !== '' && previousHash !== currentHash;
+        let addedSections = [];
+        let removedSections = [];
+        if (changed) {
+            const cached = this.get(url);
+            if (cached) {
+                const oldHeadings = cached.headings.map(h => h.text);
+                const newHeadings = newContent.match(/^#{1,6}\s+(.+)$/gm)?.map(h => h.replace(/^#+\s+/, '')) || [];
+                const oldSet = new Set(oldHeadings);
+                const newSet = new Set(newHeadings);
+                addedSections = newHeadings.filter(h => !oldSet.has(h));
+                removedSections = oldHeadings.filter(h => !newSet.has(h));
+            }
+        }
+        return { url, previousHash, currentHash, changed, addedSections, removedSections };
+    }
+}
+exports.CrawlCache = CrawlCache;
+//# sourceMappingURL=cache.js.map

package/dist/cache/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/cache/cache.ts"],"names":[],"mappings":";;;AAAA,mCAAoC;AACpC,2BAA6F;AAC7F,+BAA4B;AAC5B,yCAAqC;AACrC,6BAAwB;AAGxB,MAAM,SAAS,GAAG,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC;AAQnC;;GAEG;AACH,MAAa,UAAU;IACb,WAAW,CAAqC;IAChD,SAAS,CAA2B;IACpC,MAAM,CAAc;IACpB,QAAQ,CAAS;IAEzB,YAAY,SAA+B,EAAE;QAC3C,IAAI,CAAC,MAAM,GAAG;YACZ,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;YAC/B,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,IAAI;YACvB,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,GAAG;YAC9B,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,IAAI;SAC9C,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,IAAA,WAAI,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,mBAAmB,CAAC,CAAC;QAClF,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,IAAA,eAAU,EAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtD,IAAA,cAAS,EAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,oBAAQ,CAA2B;YACxD,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YACxB,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI;SAC5B,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,GAAG,IAAI,oBAAQ,CAAiB;YAC5C,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YACxB,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI;SAC5B,CAAC,CAAC;IACL,CAAC;IAEO,IAAI,CAAC,OAAe;QAC1B,OAAO,IAAA,mBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5D,CAAC;IAEO,QAAQ,CAAC,GAAW;QAC1B,OAAO,IAAA,mBAAU,EAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACrD,CAAC;IAEO,aAAa,CAAC,GAAW;QAC/B,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;QACnE,IAAI,CAAC,IAAA,eAAU,EAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC;QACvC,IAAI,CAAC;YACH,MAAM,KAAK,GAAmB,IAAI,CAAC,KAAK,CAAC,IAAA,iBAAY,EAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;YAC1E,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;YAClD,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAC1B,IAAA,WAAM,EAAC,QAAQ,CAAC,CAAC;gBACjB,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,GAAG,CAAC,GAAW;QACb,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,SAAS,CAAC;QAC3C,IAAI,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,SAAS,CAAC;QAAC,CAAC;QAEzD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,SAAS;YAAE,OAAO,SAAS,CAAC;QAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;YACxC,OAAO,SAAS,CAAC,IAAI,CAAC;QACxB,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,GAAG,CAAC,GAAW,EAAE,OAAyB;QACxC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO;QACjC,IAAI,CAAC;YAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO;QAAC,CAAC;QAE/C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAErC,MAAM,KAAK,GAAmB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC1F,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;QACnE,IAAA,kBAAa,EAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,GAAG,CAAC,GAAW;QACb,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QACvC,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAC3C,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC;IAC1C,CAAC;IAED,UAAU,CAAC,GAAW;QACpB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;QACnE,IAAI,IAAA,eAAU,EAAC,QAAQ,CAAC;YAAE,IAAA,WAAM,EAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK;QACH,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,IAAA,eAAU,EAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,KAAK,MAAM,IAAI,IAAI,IAAA,gBAAW,EAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9C,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAAE,IAAA,WAAM,EAAC,IAAA,WAAI,EAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;IACH,CAAC;IAED,cAAc,CAAC,GAAW;QACxB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO,SAAS,EAAE,IAAI,CAAC;IACzB,CAAC;IAED,UAAU,CAAC,GAAW,EAAE,UAAkB;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACpD,MAAM,OAAO,GAAG,YAAY,KAAK,EAAE,IAAI,YAAY,KAAK,WAAW,CAAC;QAEpE,IAAI,aAAa,GAAa,EAAE,CAAC;QACjC,IAAI,eAAe,GAAa,EAAE,CAAC;QAEnC,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACrD,MAAM,WAAW,GAAG,UAAU,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBACnG,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;gBACpC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;gBACpC,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxD,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAED,OAAO,EAAE,GAAG,EAAE,YAAY,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,CAAC;IACrF,CAAC;CACF;AArID,gCAqIC"}

package/dist/chunking/chunker.d.ts

@@ -0,0 +1,26 @@
+import { ContentChunk, ChunkOptions, Heading } from '../core/types';
+/**
+ * Token-aware content chunking for RAG/LLM pipelines.
+ * Uses actual tiktoken for accurate token counting.
+ */
+export declare class ContentChunker {
+    private encoder;
+    private options;
+    constructor(options?: ChunkOptions);
+    /** Count tokens using tiktoken */
+    countTokens(text: string): number;
+    /** Chunk content using configured strategy */
+    chunk(markdown: string, sourceUrl: string, title: string, headings: Heading[], options?: ChunkOptions): ContentChunk[];
+    /** Free tiktoken encoder memory */
+    dispose(): void;
+    private splitSemantic;
+    private splitByHeading;
+    private splitFixed;
+    private splitByParagraph;
+    private splitBySections;
+    private mergeCodeBlocks;
+    private applyOverlap;
+    private getHeadingPath;
+    private detectChunkLanguage;
+}
+//# sourceMappingURL=chunker.d.ts.map

package/dist/chunking/chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/chunking/chunker.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,YAAY,EAAgC,OAAO,EAAE,MAAM,eAAe,CAAC;AAElG;;;GAGG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,OAAO,CAAyB;gBAE5B,OAAO,GAAE,YAAiB;IAWtC,kCAAkC;IAClC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAIjC,8CAA8C;IAC9C,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,YAAY,EAAE;IAwDtH,mCAAmC;IACnC,OAAO,IAAI,IAAI;IAIf,OAAO,CAAC,aAAa;IAkBrB,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,UAAU;IAkBlB,OAAO,CAAC,gBAAgB;IAkBxB,OAAO,CAAC,eAAe;IAmBvB,OAAO,CAAC,eAAe;IAiBvB,OAAO,CAAC,YAAY;IAepB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,mBAAmB;CAI5B"}

package/dist/chunking/chunker.js

@@ -0,0 +1,208 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ContentChunker = void 0;
+const tiktoken_1 = require("tiktoken");
+const crypto_1 = require("crypto");
+/**
+ * Token-aware content chunking for RAG/LLM pipelines.
+ * Uses actual tiktoken for accurate token counting.
+ */
+class ContentChunker {
+    encoder;
+    options;
+    constructor(options = {}) {
+        this.encoder = (0, tiktoken_1.get_encoding)('cl100k_base');
+        this.options = {
+            maxTokens: options.maxTokens ?? 1500,
+            overlap: options.overlap ?? 100,
+            strategy: options.strategy ?? 'semantic',
+            preserveCodeBlocks: options.preserveCodeBlocks ?? true,
+            preserveHeadings: options.preserveHeadings ?? true,
+        };
+    }
+    /** Count tokens using tiktoken */
+    countTokens(text) {
+        return this.encoder.encode(text).length;
+    }
+    /** Chunk content using configured strategy */
+    chunk(markdown, sourceUrl, title, headings, options) {
+        const opts = { ...this.options, ...options };
+        let rawChunks;
+        switch (opts.strategy) {
+            case 'heading':
+                rawChunks = this.splitByHeading(markdown);
+                break;
+            case 'fixed':
+                rawChunks = this.splitFixed(markdown, opts.maxTokens);
+                break;
+            case 'paragraph':
+                rawChunks = this.splitByParagraph(markdown, opts.maxTokens);
+                break;
+            case 'semantic':
+            default:
+                rawChunks = this.splitSemantic(markdown, opts.maxTokens);
+                break;
+        }
+        // Preserve code blocks by merging split blocks
+        if (opts.preserveCodeBlocks) {
+            rawChunks = this.mergeCodeBlocks(rawChunks, opts.maxTokens);
+        }
+        // Enforce max token size
+        rawChunks = rawChunks.flatMap(c => this.countTokens(c) > opts.maxTokens ? this.splitFixed(c, opts.maxTokens) : [c]);
+        // Apply overlap
+        if (opts.overlap > 0 && rawChunks.length > 1) {
+            rawChunks = this.applyOverlap(rawChunks, opts.overlap);
+        }
+        const totalChunks = rawChunks.length;
+        return rawChunks.map((text, i) => {
+            const id = (0, crypto_1.createHash)('sha256').update(`${sourceUrl}:${i}:${text.slice(0, 64)}`).digest('hex').slice(0, 16);
+            const headingPath = this.getHeadingPath(markdown, text, headings);
+            return {
+                id,
+                content: text,
+                tokens: this.countTokens(text),
+                metadata: {
+                    sourceUrl,
+                    title,
+                    headingPath,
+                    chunkIndex: i,
+                    totalChunks,
+                    hasCode: /```[\s\S]*?```/.test(text),
+                    language: this.detectChunkLanguage(text),
+                },
+            };
+        });
+    }
+    /** Free tiktoken encoder memory */
+    dispose() {
+        this.encoder.free();
+    }
+    splitSemantic(content, maxTokens) {
+        const sections = this.splitBySections(content);
+        const chunks = [];
+        let buffer = '';
+        for (const section of sections) {
+            const combined = buffer ? buffer + '\n\n' + section : section;
+            if (this.countTokens(combined) > maxTokens && buffer) {
+                chunks.push(buffer.trim());
+                buffer = section;
+            }
+            else {
+                buffer = combined;
+            }
+        }
+        if (buffer.trim())
+            chunks.push(buffer.trim());
+        return chunks;
+    }
+    splitByHeading(content) {
+        return content.split(/(?=^#{1,3}\s)/m).map(s => s.trim()).filter(Boolean);
+    }
+    splitFixed(content, maxTokens) {
+        const lines = content.split('\n');
+        const chunks = [];
+        let buffer = '';
+        for (const line of lines) {
+            const candidate = buffer ? buffer + '\n' + line : line;
+            if (this.countTokens(candidate) > maxTokens && buffer) {
+                chunks.push(buffer.trim());
+                buffer = line;
+            }
+            else {
+                buffer = candidate;
+            }
+        }
+        if (buffer.trim())
+            chunks.push(buffer.trim());
+        return chunks;
+    }
+    splitByParagraph(content, maxTokens) {
+        const paragraphs = content.split(/\n\n+/).filter(p => p.trim());
+        const chunks = [];
+        let buffer = '';
+        for (const para of paragraphs) {
+            const combined = buffer ? buffer + '\n\n' + para : para;
+            if (this.countTokens(combined) > maxTokens && buffer) {
+                chunks.push(buffer.trim());
+                buffer = para;
+            }
+            else {
+                buffer = combined;
+            }
+        }
+        if (buffer.trim())
+            chunks.push(buffer.trim());
+        return chunks;
+    }
+    splitBySections(markdown) {
+        const parts = [];
+        const lines = markdown.split('\n');
+        let current = '';
+        let inCodeBlock = false;
+        for (const line of lines) {
+            if (line.startsWith('```'))
+                inCodeBlock = !inCodeBlock;
+            if (!inCodeBlock && line.match(/^#{1,4}\s/) && current.trim()) {
+                parts.push(current.trim());
+                current = line + '\n';
+            }
+            else {
+                current += line + '\n';
+            }
+        }
+        if (current.trim())
+            parts.push(current.trim());
+        return parts;
+    }
+    mergeCodeBlocks(chunks, maxTokens) {
+        const result = [];
+        for (let i = 0; i < chunks.length; i++) {
+            const openCount = (chunks[i].match(/```/g) || []).length;
+            if (openCount % 2 !== 0 && i + 1 < chunks.length) {
+                const merged = chunks[i] + '\n' + chunks[i + 1];
+                if (this.countTokens(merged) <= maxTokens * 1.5) {
+                    result.push(merged);
+                    i++;
+                    continue;
+                }
+            }
+            result.push(chunks[i]);
+        }
+        return result;
+    }
+    applyOverlap(chunks, overlapTokens) {
+        const result = [chunks[0]];
+        for (let i = 1; i < chunks.length; i++) {
+            const prevWords = chunks[i - 1].split(/\s+/);
+            let overlap = '';
+            for (let j = prevWords.length - 1; j >= 0; j--) {
+                const candidate = prevWords.slice(j).join(' ');
+                if (this.countTokens(candidate) > overlapTokens)
+                    break;
+                overlap = candidate;
+            }
+            result.push(overlap ? overlap + '\n' + chunks[i] : chunks[i]);
+        }
+        return result;
+    }
+    getHeadingPath(fullContent, chunkText, headings) {
+        const chunkStart = fullContent.indexOf(chunkText.slice(0, 50));
+        if (chunkStart === -1)
+            return [];
+        const before = fullContent.slice(0, chunkStart);
+        const path = [];
+        for (const h of headings) {
+            if (before.includes(h.text) || chunkText.includes(h.text)) {
+                if (h.level <= 3)
+                    path[h.level - 1] = h.text;
+            }
+        }
+        return path.filter(Boolean);
+    }
+    detectChunkLanguage(content) {
+        const match = content.match(/```(\w+)/);
+        return match?.[1];
+    }
+}
+exports.ContentChunker = ContentChunker;
+//# sourceMappingURL=chunker.js.map