webpeel 0.10.0 → 0.12.0

package/dist/core/bm25-filter.d.ts

@@ -0,0 +1,57 @@
+/**
+ * BM25 Query-Focused Content Filter
+ *
+ * Filters markdown content by BM25 relevance to a query, keeping only the
+ * blocks that are most relevant. This can reduce token usage by 70-90% for
+ * focused tasks (e.g., "find hotel prices" should not return navigation menus,
+ * footer text, or unrelated article sections).
+ *
+ * Algorithm: BM25 (Best Matching 25) — Okapi BM25
+ *   score(D, Q) = Σ IDF(qi) * tf(qi,D)*(k1+1) / (tf(qi,D) + k1*(1 - b + b*|D|/avgdl))
+ */
+export interface BM25FilterOptions {
+    /** Query to rank content against */
+    query: string;
+    /** BM25 threshold score. Blocks below this are removed. Default: auto-calculated */
+    threshold?: number;
+    /** Whether to return scores in output. Default: false */
+    includeScores?: boolean;
+}
+export interface BM25FilterResult {
+    /** Filtered content (relevant paragraphs only) */
+    content: string;
+    /** Number of blocks kept */
+    kept: number;
+    /** Total number of blocks */
+    total: number;
+    /** Percentage of content removed */
+    reductionPercent: number;
+}
+interface ContentBlock {
+    /** Original markdown text (preserved verbatim in output) */
+    raw: string;
+    /** Index in the original block list (for order preservation) */
+    index: number;
+}
+/**
+ * Split markdown content into logical blocks for scoring:
+ * - Code fences (``` ... ```) → single block
+ * - Heading + immediately following paragraph → single block
+ * - Lists (contiguous lines starting with - / * / + / number.) → single block
+ * - Tables → single block
+ * - Paragraphs → one block each
+ */
+export declare function splitIntoBlocks(content: string): ContentBlock[];
+/**
+ * Calculate BM25 scores for all blocks against a query.
+ * Returns array of scores in same order as blocks.
+ */
+export declare function scoreBM25(blocks: ContentBlock[], queryTerms: string[]): number[];
+/**
+ * Filter markdown content by BM25 relevance to a query.
+ * Splits content into blocks (paragraphs, headings+body, list items),
+ * scores each by BM25, and returns only blocks above threshold.
+ */
+export declare function filterByRelevance(content: string, options: BM25FilterOptions): BM25FilterResult;
+export {};
+//# sourceMappingURL=bm25-filter.d.ts.map

package/dist/core/bm25-filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25-filter.d.ts","sourceRoot":"","sources":["../../src/core/bm25-filter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,iBAAiB;IAChC,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,oFAAoF;IACpF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,kDAAkD;IAClD,OAAO,EAAE,MAAM,CAAC;IAChB,4BAA4B;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAiDD,UAAU,YAAY;IACpB,4DAA4D;IAC5D,GAAG,EAAE,MAAM,CAAC;IACZ,gEAAgE;IAChE,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,YAAY,EAAE,CA2E/D;AAMD;;;GAGG;AACH,wBAAgB,SAAS,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CA8ChF;AAMD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,gBAAgB,CAqF/F"}

package/dist/core/bm25-filter.js

@@ -0,0 +1,249 @@
+/**
+ * BM25 Query-Focused Content Filter
+ *
+ * Filters markdown content by BM25 relevance to a query, keeping only the
+ * blocks that are most relevant. This can reduce token usage by 70-90% for
+ * focused tasks (e.g., "find hotel prices" should not return navigation menus,
+ * footer text, or unrelated article sections).
+ *
+ * Algorithm: BM25 (Best Matching 25) — Okapi BM25
+ *   score(D, Q) = Σ IDF(qi) * tf(qi,D)*(k1+1) / (tf(qi,D) + k1*(1 - b + b*|D|/avgdl))
+ */
+// BM25 tuning parameters
+const K1 = 1.5; // term frequency saturation
+const B = 0.75; // length normalization
+// ---------------------------------------------------------------------------
+// Tokenization
+// ---------------------------------------------------------------------------
+/**
+ * Tokenize text into lowercase terms, stripping punctuation.
+ * Markdown formatting characters are also stripped.
+ */
+function tokenize(text) {
+    return text
+        .toLowerCase()
+        // Strip markdown formatting (bold, italic, code, links, images, headings)
+        .replace(/!\[.*?\]\(.*?\)/g, ' ') // images
+        .replace(/\[.*?\]\(.*?\)/g, ' ') // links
+        .replace(/`{1,3}[^`]*`{1,3}/g, ' ') // inline code
+        .replace(/[#*_~`>|\\]/g, ' ') // formatting chars
+        .replace(/[^\w\s]/g, ' ') // remaining punctuation
+        .split(/\s+/)
+        .filter(t => t.length > 0);
+}
+/**
+ * Strip markdown formatting from text for scoring purposes.
+ * Preserves words but removes symbols.
+ */
+function stripMarkdown(text) {
+    return text
+        .replace(/```[\s\S]*?```/g, ' ') // fenced code blocks
+        .replace(/`[^`]+`/g, ' ') // inline code
+        .replace(/!\[.*?\]\(.*?\)/g, ' ') // images
+        .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') // links → text
+        .replace(/^#{1,6}\s+/gm, '') // headings
+        .replace(/[*_~`>|\\]/g, ' ') // formatting
+        .replace(/^\s*[-*+]\s+/gm, ' ') // list bullets
+        .replace(/^\s*\d+\.\s+/gm, ' ') // numbered list
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+/**
+ * Split markdown content into logical blocks for scoring:
+ * - Code fences (``` ... ```) → single block
+ * - Heading + immediately following paragraph → single block
+ * - Lists (contiguous lines starting with - / * / + / number.) → single block
+ * - Tables → single block
+ * - Paragraphs → one block each
+ */
+export function splitIntoBlocks(content) {
+    // Normalise line endings
+    const text = content.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+    // First, extract fenced code blocks so they aren't broken apart
+    // We'll replace them with placeholders, split, then restore.
+    const codeBlocks = [];
+    const withPlaceholders = text.replace(/```[\s\S]*?```/g, (match) => {
+        const id = codeBlocks.length;
+        codeBlocks.push(match);
+        return `\x00CODE_BLOCK_${id}\x00`;
+    });
+    // Split on double newlines
+    const rawChunks = withPlaceholders.split(/\n{2,}/);
+    // Re-join heading with its following paragraph
+    const merged = [];
+    for (let i = 0; i < rawChunks.length; i++) {
+        const chunk = rawChunks[i].trim();
+        if (!chunk)
+            continue;
+        const isHeading = /^#{1,6}\s/.test(chunk);
+        const nextChunk = rawChunks[i + 1]?.trim();
+        if (isHeading && nextChunk && !/^#{1,6}\s/.test(nextChunk)) {
+            // Merge heading + following paragraph
+            merged.push(chunk + '\n\n' + nextChunk);
+            i++; // skip next
+        }
+        else {
+            merged.push(chunk);
+        }
+    }
+    // Now merge contiguous list lines that got split
+    const regrouped = [];
+    for (const chunk of merged) {
+        const lines = chunk.split('\n');
+        const isListBlock = lines.every(l => l.trim() === '' || /^\s*[-*+]\s/.test(l) || /^\s*\d+\.\s/.test(l) || /^\s*\d+\)\s/.test(l)) && lines.some(l => /^\s*[-*+]\s/.test(l) || /^\s*\d+[.)]\s/.test(l));
+        const isTableBlock = lines.some(l => /^\|/.test(l.trim()));
+        if (isListBlock || isTableBlock) {
+            // Check if previous block was the same type (adjacent lists should merge)
+            const prev = regrouped[regrouped.length - 1];
+            if (prev) {
+                const prevLines = prev.split('\n');
+                const prevIsListOrTable = prevLines.some(l => /^\s*[-*+]\s/.test(l) || /^\s*\d+[.)]\s/.test(l) || /^\|/.test(l.trim()));
+                if (prevIsListOrTable && isListBlock === prevIsListOrTable) {
+                    regrouped[regrouped.length - 1] = prev + '\n' + chunk;
+                    continue;
+                }
+            }
+        }
+        regrouped.push(chunk);
+    }
+    // Restore code blocks and build final ContentBlock array
+    const blocks = [];
+    for (let i = 0; i < regrouped.length; i++) {
+        let raw = regrouped[i];
+        // Restore code block placeholders
+        raw = raw.replace(/\x00CODE_BLOCK_(\d+)\x00/g, (_m, idx) => codeBlocks[Number(idx)]);
+        if (raw.trim()) {
+            blocks.push({ raw: raw.trim(), index: i });
+        }
+    }
+    return blocks;
+}
+// ---------------------------------------------------------------------------
+// BM25 Scoring
+// ---------------------------------------------------------------------------
+/**
+ * Calculate BM25 scores for all blocks against a query.
+ * Returns array of scores in same order as blocks.
+ */
+export function scoreBM25(blocks, queryTerms) {
+    if (blocks.length === 0 || queryTerms.length === 0) {
+        return blocks.map(() => 0);
+    }
+    const N = blocks.length;
+    // Tokenize each block (strip markdown for scoring)
+    const blockTokens = blocks.map(b => tokenize(stripMarkdown(b.raw)));
+    const blockLengths = blockTokens.map(t => t.length);
+    const avgdl = blockLengths.reduce((s, l) => s + l, 0) / N || 1;
+    // Build term frequency maps for each block
+    const tfMaps = blockTokens.map(tokens => {
+        const tf = new Map();
+        for (const t of tokens) {
+            tf.set(t, (tf.get(t) ?? 0) + 1);
+        }
+        return tf;
+    });
+    // For each query term, compute IDF and score contribution
+    const scores = new Array(N).fill(0);
+    for (const term of queryTerms) {
+        // n(qi) = number of documents containing the term
+        let nqi = 0;
+        for (const tf of tfMaps) {
+            if (tf.has(term))
+                nqi++;
+        }
+        // IDF(qi) = log((N - n(qi) + 0.5) / (n(qi) + 0.5) + 1)
+        const idf = Math.log((N - nqi + 0.5) / (nqi + 0.5) + 1);
+        for (let d = 0; d < N; d++) {
+            const tf = tfMaps[d].get(term) ?? 0;
+            if (tf === 0)
+                continue;
+            const dl = blockLengths[d];
+            // BM25 term score
+            const termScore = idf * (tf * (K1 + 1)) / (tf + K1 * (1 - B + B * dl / avgdl));
+            scores[d] += termScore;
+        }
+    }
+    return scores;
+}
+// ---------------------------------------------------------------------------
+// Main filter function
+// ---------------------------------------------------------------------------
+/**
+ * Filter markdown content by BM25 relevance to a query.
+ * Splits content into blocks (paragraphs, headings+body, list items),
+ * scores each by BM25, and returns only blocks above threshold.
+ */
+export function filterByRelevance(content, options) {
+    const { query, threshold, includeScores = false } = options;
+    // Empty query → return full content
+    if (!query || !query.trim()) {
+        return {
+            content,
+            kept: 0,
+            total: 0,
+            reductionPercent: 0,
+        };
+    }
+    const blocks = splitIntoBlocks(content);
+    const total = blocks.length;
+    if (total === 0) {
+        return { content, kept: 0, total: 0, reductionPercent: 0 };
+    }
+    const queryTerms = tokenize(query);
+    if (queryTerms.length === 0) {
+        return { content, kept: total, total, reductionPercent: 0 };
+    }
+    const scores = scoreBM25(blocks, queryTerms);
+    // Determine threshold
+    let effectiveThreshold;
+    if (threshold !== undefined) {
+        effectiveThreshold = threshold;
+    }
+    else {
+        const meanScore = scores.reduce((s, v) => s + v, 0) / scores.length;
+        effectiveThreshold = meanScore * 0.5;
+    }
+    // Select blocks above threshold
+    let keptIndices = scores
+        .map((score, i) => ({ score, i }))
+        .filter(({ score }) => score >= effectiveThreshold)
+        .map(({ i }) => i);
+    // Fallback: never return empty — keep top 3
+    if (keptIndices.length === 0) {
+        keptIndices = scores
+            .map((score, i) => ({ score, i }))
+            .sort((a, b) => b.score - a.score)
+            .slice(0, 3)
+            .map(({ i }) => i)
+            .sort((a, b) => a - b); // restore document order
+    }
+    // Preserve original document order
+    keptIndices.sort((a, b) => a - b);
+    const keptBlocks = keptIndices.map(i => blocks[i]);
+    const kept = keptBlocks.length;
+    // Build output content
+    let outputParts;
+    if (includeScores) {
+        outputParts = keptBlocks.map(b => {
+            const score = scores[b.index];
+            return `<!-- BM25: ${score.toFixed(4)} -->\n${b.raw}`;
+        });
+    }
+    else {
+        outputParts = keptBlocks.map(b => b.raw);
+    }
+    const filteredContent = outputParts.join('\n\n');
+    // Calculate reduction percent based on character count
+    const originalLen = content.length;
+    const filteredLen = filteredContent.length;
+    const reductionPercent = originalLen > 0
+        ? Math.round(((originalLen - filteredLen) / originalLen) * 100)
+        : 0;
+    return {
+        content: filteredContent,
+        kept,
+        total,
+        reductionPercent,
+    };
+}
+//# sourceMappingURL=bm25-filter.js.map

package/dist/core/bm25-filter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25-filter.js","sourceRoot":"","sources":["../../src/core/bm25-filter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAsBH,yBAAyB;AACzB,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,4BAA4B;AAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,uBAAuB;AAEvC,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E;;;GAGG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI;SACR,WAAW,EAAE;QACd,0EAA0E;SACzE,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAE,SAAS;SAC3C,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAG,QAAQ;SAC1C,OAAO,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC,cAAc;SACjD,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,CAAM,mBAAmB;SACrD,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAU,wBAAwB;SAC1D,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAC/B,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,IAAI;SACR,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAI,qBAAqB;SACxD,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAW,cAAc;SACjD,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAG,SAAS;SAC5C,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAC,eAAe;SACvD,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAQ,WAAW;SAC9C,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAQ,aAAa;SAChD,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAK,eAAe;SAClD,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAK,gBAAgB;SACnD,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAaD;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,yBAAyB;IACzB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAEjE,gEAAgE;IAChE,6DAA6D;IAC7D,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,KAAK,EAAE,EAAE;QACjE,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC;QAC7B,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvB,OAAO,kBAAkB,EAAE,MAAM,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,2BAA2B;IAC3B,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAEnD,+CAA+C;IAC/C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClC,IAAI,CAAC,KAAK;YAAE,SAAS;QAErB,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QAE3C,IAAI,SAAS,IAAI,SAAS,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3D,sCAAsC;YACtC,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC;YACxC,CAAC,EAAE,CAAC,CAAC,YAAY;QACnB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,iDAAiD;IACjD,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAC7B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAChG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvE,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAE3D,IAAI,WAAW,IAAI,YAAY,EAAE,CAAC;YAChC,0EAA0E;YAC1E,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7C,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACnC,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CACtC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAC9E,CAAC;gBACF,IAAI,iBAAiB,IAAI,WAAW,KAAK,iBAAiB,EAAE,CAAC;oBAC3D,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,CAAC;oBACtD,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC;IAED,yDAAyD;IACzD,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,IAAI,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAEvB,kCAAkC;QAClC,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,2BAA2B,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAErF,IAAI,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,MAAsB,EAAE,UAAoB;IACpE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAExB,mDAAmD;IACnD,MAAM,WAAW,GAAe,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAChF,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAE/D,2CAA2C;IAC3C,MAAM,MAAM,GAA0B,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;QAC7D,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;QACrC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IAEH,0DAA0D;IAC1D,MAAM,MAAM,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE5C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,kDAAkD;QAClD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;YACxB,IAAI,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,GAAG,EAAE,CAAC;QAC1B,CAAC;QAED,uDAAuD;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAExD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,EAAE,KAAK,CAAC;gBAAE,SAAS;YAEvB,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAC3B,kBAAkB;YAClB,MAAM,SAAS,GAAG,GAAG,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC;YAC/E,MAAM,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAe,EAAE,OAA0B;IAC3E,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAE5D,oCAAoC;IACpC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC5B,OAAO;YACL,OAAO;YACP,IAAI,EAAE,CAAC;YACP,KAAK,EAAE,CAAC;YACR,gBAAgB,EAAE,CAAC;SACpB,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAE5B,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEnC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IAC9D,CAAC;IAED,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAE7C,sBAAsB;IACtB,IAAI,kBAA0B,CAAC;IAC/B,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,kBAAkB,GAAG,SAAS,CAAC;IACjC,CAAC;SAAM,CAAC;QACN,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACpE,kBAAkB,GAAG,SAAS,GAAG,GAAG,CAAC;IACvC,CAAC;IAED,gCAAgC;IAChC,IAAI,WAAW,GAAG,MAAM;SACrB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;SACjC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,IAAI,kBAAkB,CAAC;SAClD,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAErB,4CAA4C;IAC5C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,WAAW,GAAG,MAAM;aACjB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;aACjC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;aACjC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;aACX,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;aACjB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,yBAAyB;IACrD,CAAC;IAED,mCAAmC;IACnC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAElC,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC;IAE/B,uBAAuB;IACvB,IAAI,WAAqB,CAAC;IAC1B,IAAI,aAAa,EAAE,CAAC;QAClB,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,cAAc,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,eAAe,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEjD,uDAAuD;IACvD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IACnC,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC;IAC3C,MAAM,gBAAgB,GAAG,WAAW,GAAG,CAAC;QACtC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC;QAC/D,CAAC,CAAC,CAAC,CAAC;IAEN,OAAO;QACL,OAAO,EAAE,eAAe;QACxB,IAAI;QACJ,KAAK;QACL,gBAAgB;KACjB,CAAC;AACJ,CAAC"}

package/dist/core/chunking.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Smart content chunking for LLM processing.
+ *
+ * Splits content into manageable pieces with configurable overlap and strategy.
+ * Zero external dependencies; target <1ms for typical page content.
+ */
+export interface ChunkOptions {
+    /** Target tokens per chunk. Default: 4000 */
+    chunkSize?: number;
+    /** Overlap tokens between chunks. Default: 200 */
+    overlap?: number;
+    /** Chunking strategy. Default: 'semantic' */
+    strategy?: 'fixed' | 'semantic' | 'paragraph';
+}
+export interface Chunk {
+    /** Chunk index (0-based) */
+    index: number;
+    /** Chunk content */
+    content: string;
+    /** Estimated tokens in this chunk */
+    tokens: number;
+    /** Character offset in original content */
+    startOffset: number;
+    /** Whether this is the last chunk */
+    isLast: boolean;
+}
+export interface ChunkResult {
+    /** Array of content chunks */
+    chunks: Chunk[];
+    /** Total chunks */
+    totalChunks: number;
+    /** Total tokens across all chunks */
+    totalTokens: number;
+    /** Original content tokens */
+    originalTokens: number;
+}
+/** Estimate token count using chars/4 heuristic. Accurate within ±10%. */
+export declare function estimateTokens(text: string): number;
+/**
+ * Split content into chunks suitable for LLM processing.
+ */
+export declare function chunkContent(content: string, options?: ChunkOptions): ChunkResult;
+//# sourceMappingURL=chunking.d.ts.map

package/dist/core/chunking.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../../src/core/chunking.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,YAAY;IAC3B,6CAA6C;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,WAAW,CAAC;CAC/C;AAED,MAAM,WAAW,KAAK;IACpB,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,2CAA2C;IAC3C,WAAW,EAAE,MAAM,CAAC;IACpB,qCAAqC;IACrC,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,8BAA8B;IAC9B,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,mBAAmB;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,qCAAqC;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,0EAA0E;AAC1E,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,WAAW,CA0CjF"}

package/dist/core/chunking.js

@@ -0,0 +1,182 @@
+/**
+ * Smart content chunking for LLM processing.
+ *
+ * Splits content into manageable pieces with configurable overlap and strategy.
+ * Zero external dependencies; target <1ms for typical page content.
+ */
+/** Estimate token count using chars/4 heuristic. Accurate within ±10%. */
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Split content into chunks suitable for LLM processing.
+ */
+export function chunkContent(content, options) {
+    const chunkSize = options?.chunkSize ?? 4000;
+    const overlap = options?.overlap ?? 200;
+    const strategy = options?.strategy ?? 'semantic';
+    const originalTokens = estimateTokens(content);
+    if (content.length === 0) {
+        return { chunks: [], totalChunks: 0, totalTokens: 0, originalTokens: 0 };
+    }
+    let rawChunks;
+    switch (strategy) {
+        case 'fixed':
+            rawChunks = chunkFixed(content, chunkSize, overlap);
+            break;
+        case 'paragraph':
+            rawChunks = chunkParagraph(content, chunkSize);
+            break;
+        case 'semantic':
+        default:
+            rawChunks = chunkSemantic(content, chunkSize, overlap);
+            break;
+    }
+    const chunks = rawChunks.map((raw, i) => ({
+        index: i,
+        content: raw.content,
+        tokens: estimateTokens(raw.content),
+        startOffset: raw.startOffset,
+        isLast: i === rawChunks.length - 1,
+    }));
+    const totalTokens = chunks.reduce((sum, c) => sum + c.tokens, 0);
+    return {
+        chunks,
+        totalChunks: chunks.length,
+        totalTokens,
+        originalTokens,
+    };
+}
+// ---------------------------------------------------------------------------
+// Fixed strategy
+// ---------------------------------------------------------------------------
+function chunkFixed(content, chunkSize, overlap) {
+    const chunkChars = chunkSize * 4;
+    const overlapChars = overlap * 4;
+    const result = [];
+    let start = 0;
+    while (start < content.length) {
+        const end = Math.min(start + chunkChars, content.length);
+        result.push({ content: content.slice(start, end), startOffset: start });
+        if (end >= content.length)
+            break;
+        start = end - overlapChars;
+    }
+    return result;
+}
+// ---------------------------------------------------------------------------
+// Semantic strategy
+// ---------------------------------------------------------------------------
+function findNaturalBreak(content, target, tolerance) {
+    const min = target - tolerance;
+    const max = Math.min(target + tolerance, content.length);
+    // 1. Heading break: newline immediately before ##
+    const headingRe = /\n(?=#{1,6} )/g;
+    let best = null;
+    headingRe.lastIndex = min;
+    let m = headingRe.exec(content);
+    while (m && m.index <= max) {
+        const dist = Math.abs(m.index - target);
+        if (!best || best.priority < 4 || dist < Math.abs(best.pos - target)) {
+            best = { pos: m.index, priority: 4 };
+        }
+        m = headingRe.exec(content);
+    }
+    // 2. Paragraph break: \n\n
+    const paraRe = /\n\n/g;
+    paraRe.lastIndex = min;
+    m = paraRe.exec(content);
+    while (m && m.index <= max) {
+        const dist = Math.abs(m.index - target);
+        if (!best || best.priority < 3 || (best.priority === 3 && dist < Math.abs(best.pos - target))) {
+            best = { pos: m.index + 2, priority: 3 };
+        }
+        m = paraRe.exec(content);
+    }
+    // 3. Sentence end: '. ', '! ', '? ' followed by capital or newline
+    const sentRe = /[.!?](?:\s+(?=[A-Z\n])|(?=\n))/g;
+    sentRe.lastIndex = min;
+    m = sentRe.exec(content);
+    while (m && m.index <= max) {
+        const pos = m.index + m[0].length;
+        const dist = Math.abs(pos - target);
+        if (!best || best.priority < 2 || (best.priority === 2 && dist < Math.abs(best.pos - target))) {
+            best = { pos, priority: 2 };
+        }
+        m = sentRe.exec(content);
+    }
+    // 4. Word boundary (space)
+    if (!best || best.priority < 1) {
+        const spaceRe = / /g;
+        spaceRe.lastIndex = min;
+        m = spaceRe.exec(content);
+        while (m && m.index <= max) {
+            const pos = m.index + 1;
+            const dist = Math.abs(pos - target);
+            if (!best || (best.priority < 2 && dist < Math.abs(best.pos - target))) {
+                best = { pos, priority: 1 };
+            }
+            m = spaceRe.exec(content);
+        }
+    }
+    return best ? best.pos : Math.min(target, content.length);
+}
+function chunkSemantic(content, chunkSize, overlap) {
+    const chunkChars = chunkSize * 4;
+    const overlapChars = overlap * 4;
+    const tolerance = Math.floor(chunkChars * 0.2);
+    const result = [];
+    let start = 0;
+    while (start < content.length) {
+        const remaining = content.length - start;
+        // If the rest fits, take it all
+        if (remaining <= chunkChars + tolerance) {
+            result.push({ content: content.slice(start), startOffset: start });
+            break;
+        }
+        const breakPos = findNaturalBreak(content, start + chunkChars, tolerance);
+        const end = Math.max(breakPos, start + 1); // always advance
+        result.push({ content: content.slice(start, end), startOffset: start });
+        // Next chunk starts with overlap from end of this chunk
+        const nextStart = Math.max(start + 1, end - overlapChars);
+        start = nextStart;
+    }
+    return result;
+}
+// ---------------------------------------------------------------------------
+// Paragraph strategy
+// ---------------------------------------------------------------------------
+function chunkParagraph(content, chunkSize) {
+    const chunkChars = chunkSize * 4;
+    const paragraphs = content.split(/\n\n/);
+    const result = [];
+    let currentParts = [];
+    let currentLen = 0;
+    let currentOffset = 0;
+    let offsetTracker = 0;
+    for (let i = 0; i < paragraphs.length; i++) {
+        const para = paragraphs[i];
+        // +2 for the \n\n separator (except for first paragraph)
+        const paraLen = para.length + (currentParts.length > 0 ? 2 : 0);
+        if (currentParts.length > 0 && currentLen + paraLen > chunkChars) {
+            // Flush current group
+            result.push({ content: currentParts.join('\n\n'), startOffset: currentOffset });
+            currentOffset = offsetTracker;
+            currentParts = [para];
+            currentLen = para.length;
+        }
+        else {
+            if (currentParts.length === 0) {
+                currentOffset = offsetTracker;
+            }
+            currentParts.push(para);
+            currentLen += paraLen;
+        }
+        offsetTracker += para.length + 2; // account for \n\n
+    }
+    if (currentParts.length > 0) {
+        result.push({ content: currentParts.join('\n\n'), startOffset: currentOffset });
+    }
+    return result;
+}
+//# sourceMappingURL=chunking.js.map

package/dist/core/chunking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../../src/core/chunking.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAmCH,0EAA0E;AAC1E,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,OAAsB;IAClE,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC;IAC7C,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,GAAG,CAAC;IACxC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,UAAU,CAAC;IAEjD,MAAM,cAAc,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;IAC3E,CAAC;IAED,IAAI,SAA0D,CAAC;IAE/D,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,OAAO;YACV,SAAS,GAAG,UAAU,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM;QACR,KAAK,WAAW;YACd,SAAS,GAAG,cAAc,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YAC/C,MAAM;QACR,KAAK,UAAU,CAAC;QAChB;YACE,SAAS,GAAG,aAAa,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM;IACV,CAAC;IAED,MAAM,MAAM,GAAY,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACjD,KAAK,EAAE,CAAC;QACR,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC;QACnC,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,MAAM,EAAE,CAAC,KAAK,SAAS,CAAC,MAAM,GAAG,CAAC;KACnC,CAAC,CAAC,CAAC;IAEJ,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEjE,OAAO;QACL,MAAM;QACN,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,WAAW;QACX,cAAc;KACf,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,SAAS,UAAU,CACjB,OAAe,EACf,SAAiB,EACjB,OAAe;IAEf,MAAM,UAAU,GAAG,SAAS,GAAG,CAAC,CAAC;IACjC,MAAM,YAAY,GAAG,OAAO,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAoD,EAAE,CAAC;IAEnE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACzD,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QACxE,IAAI,GAAG,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM;QACjC,KAAK,GAAG,GAAG,GAAG,YAAY,CAAC;IAC7B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E,SAAS,gBAAgB,CAAC,OAAe,EAAE,MAAc,EAAE,SAAiB;IAC1E,MAAM,GAAG,GAAG,MAAM,GAAG,SAAS,CAAC;IAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEzD,kDAAkD;IAClD,MAAM,SAAS,GAAG,gBAAgB,CAAC;IACnC,IAAI,IAAI,GAA6C,IAAI,CAAC;IAE1D,SAAS,CAAC,SAAS,GAAG,GAAG,CAAC;IAC1B,IAAI,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC;YACrE,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QACvC,CAAC;QACD,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAED,2BAA2B;IAC3B,MAAM,MAAM,GAAG,OAAO,CAAC;IACvB,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;IACvB,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzB,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC,EAAE,CAAC;YAC9F,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAC3C,CAAC;QACD,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,mEAAmE;IACnE,MAAM,MAAM,GAAG,iCAAiC,CAAC;IACjD,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;IACvB,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzB,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC,EAAE,CAAC;YAC9F,IAAI,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAC9B,CAAC;QACD,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,2BAA2B;IAC3B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC;QACrB,OAAO,CAAC,SAAS,GAAG,GAAG,CAAC;QACxB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;YACxB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC;YACpC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC,EAAE,CAAC;gBACvE,IAAI,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC9B,CAAC;YACD,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,aAAa,CACpB,OAAe,EACf,SAAiB,EACjB,OAAe;IAEf,MAAM,UAAU,GAAG,SAAS,GAAG,CAAC,CAAC;IACjC,MAAM,YAAY,GAAG,OAAO,GAAG,CAAC,CAAC;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAoD,EAAE,CAAC;IAEnE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;QAEzC,gCAAgC;QAChC,IAAI,SAAS,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YACnE,MAAM;QACR,CAAC;QAED,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,EAAE,KAAK,GAAG,UAAU,EAAE,SAAS,CAAC,CAAC;QAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB;QAE5D,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QAExE,wDAAwD;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,GAAG,YAAY,CAAC,CAAC;QAC1D,KAAK,GAAG,SAAS,CAAC;IACpB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,SAAS,cAAc,CACrB,OAAe,EACf,SAAiB;IAEjB,MAAM,UAAU,GAAG,SAAS,GAAG,CAAC,CAAC;IACjC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,MAAM,GAAoD,EAAE,CAAC;IAEnE,IAAI,YAAY,GAAa,EAAE,CAAC;IAChC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,yDAAyD;QACzD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEhE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,GAAG,OAAO,GAAG,UAAU,EAAE,CAAC;YACjE,sBAAsB;YACtB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC,CAAC;YAChF,aAAa,GAAG,aAAa,CAAC;YAC9B,YAAY,GAAG,CAAC,IAAI,CAAC,CAAC;YACtB,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,aAAa,GAAG,aAAa,CAAC;YAChC,CAAC;YACD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,UAAU,IAAI,OAAO,CAAC;QACxB,CAAC;QAED,aAAa,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,mBAAmB;IACvD,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/core/content-pruner.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Content Density Pruner
+ *
+ * Scores HTML block elements by text quality and removes low-value blocks
+ * (sidebars, footers, navigation, ads) that CSS selectors miss.
+ *
+ * Inspired by Crawl4AI's fit_markdown approach — typical 40-60% token savings.
+ */
+export interface PruneOptions {
+    /** Score threshold (0-1). Blocks below this are removed. Default: 0.4 */
+    threshold?: number;
+    /** Minimum word count for a block to be considered. Default: 3 */
+    minWords?: number;
+    /** Whether threshold adapts to content distribution. Default: true */
+    dynamic?: boolean;
+}
+export interface PruneResult {
+    /** Pruned HTML */
+    html: string;
+    /** Number of nodes removed */
+    nodesRemoved: number;
+    /** Percentage of content removed (by character count) */
+    reductionPercent: number;
+}
+/**
+ * Prune low-value HTML blocks using content density scoring.
+ *
+ * @param html - Raw HTML to prune
+ * @param options - Pruning configuration
+ * @returns Pruned HTML with stats
+ */
+export declare function pruneContent(html: string, options?: PruneOptions): PruneResult;
+//# sourceMappingURL=content-pruner.d.ts.map

package/dist/core/content-pruner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-pruner.d.ts","sourceRoot":"","sources":["../../src/core/content-pruner.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,MAAM,WAAW,YAAY;IAC3B,yEAAyE;IACzE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,kBAAkB;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AA2LD;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,WAAW,CAoGlF"}