webpeel 0.12.0 → 0.12.2

package/dist/core/content-pruner.js

@@ -1,240 +1,297 @@
 /**
  * Content Density Pruner
  *
- * Scores HTML block elements by text quality and removes low-value blocks
- * (sidebars, footers, navigation, ads) that CSS selectors miss.
+ * Two-pass pruning to reduce HTML before markdown conversion:
  *
- * Inspired by Crawl4AI's fit_markdown approach — typical 40-60% token savings.
+ *   Pass 1 — Semantic removal: strip elements whose tag or class/id clearly
+ *            mark them as page chrome (nav, footer, sidebar, cookie banners, ads).
+ *
+ *   Pass 2 — Density scoring: score remaining block elements by text density,
+ *            link density, tag importance, and word count. Remove low-scorers.
+ *
+ * Inspired by Crawl4AI's PruningContentFilter — targets 40-60% token savings.
  */
 import * as cheerio from 'cheerio';
-/** Block-level elements we score */
-const BLOCK_ELEMENTS = new Set([
-    'div', 'section', 'article', 'aside', 'nav', 'footer', 'header',
-    'main', 'p', 'ul', 'ol', 'table', 'blockquote', 'figure', 'form', 'details',
+// -----------------------------------------------------------------------
+// Pass 1 — Semantic removal: tags and class/id patterns
+// -----------------------------------------------------------------------
+/** Tags that are almost always page chrome, not article content. */
+const CHROME_TAGS = new Set([
+    'nav', 'footer', 'aside', 'noscript',
 ]);
 /**
- * Elements that should NEVER be removed — they are content containers.
- * Scoring them would be wrong: if we remove <main>, we lose everything.
+ * Class/id patterns that indicate page chrome.
+ * Tested against lowercased class/id strings.
  */
-const PROTECTED_ELEMENTS = new Set(['main', 'article', 'body']);
+const CHROME_PATTERNS = [
+    /\bsidebar\b/,
+    /\bcookie/,
+    /\bbanner\b/,
+    /\b(ad|ads|advert)\b/,
+    /\bpopup\b/,
+    /\bmodal\b/,
+    /\boverlay\b/,
+    /\bsocial/,
+    /\bshare\b/,
+    /\bbreadcrumb/,
+    /\bskip-?link/,
+    /\bfootnote/,
+    /\brelated-?(post|article)/,
+    /\bnewsletter/,
+    /\bsubscri/,
+    /\bcomment/,
+    /\b(sign-?up|sign-?in|log-?in)\b/,
+    /\btoc\b/,
+    /\btable-?of-?contents\b/,
+    /\bgdpr\b/,
+    /\bconsent\b/,
+];
 /**
- * Tag importance scores (-2 to +3).
- * These reflect semantic value of the element type.
+ * Tags we never remove (they likely wrap main content).
+ * We recurse into them but never strip the element itself.
  */
-const TAG_IMPORTANCE = {
-    article: 3,
-    main: 3,
-    p: 2,
-    h1: 2, h2: 2, h3: 2, h4: 2, h5: 2, h6: 2,
-    blockquote: 2,
-    pre: 2,
-    code: 2,
-    figure: 2,
-    figcaption: 2,
-    section: 1,
-    td: 1,
-    th: 1,
-    li: 1,
-    dd: 1,
-    dt: 1,
-    div: 0,
-    span: 0,
-    aside: -1,
-    header: -1,
-    form: -1,
-    nav: -2,
-    footer: -2,
-};
-/** Normalize tag importance (-2..+3) to 0..1 range */
-function normalizeTagScore(rawScore) {
-    // Range is 5 units (-2 to +3), shift by +2 and divide
-    return (rawScore + 2) / 5;
-}
-function getTagImportance(tagName) {
-    return TAG_IMPORTANCE[tagName.toLowerCase()] ?? 0;
-}
-/** Word count bonus using log scale (0-1) */
-function wordCountBonus(text) {
-    const words = text.trim().split(/\s+/).filter((w) => w.length > 0);
-    if (words.length === 0)
-        return 0;
-    return Math.min(Math.log(words.length + 1) / Math.log(1000), 1.0);
-}
+const PROTECTED_TAGS = new Set(['main', 'article', 'body']);
 /**
- * Position weight based on normalized position in document (0-1).
- * Middle 60% of the page (0.2–0.8) scores 1.0.
- * Top/bottom 20% scores linearly from 0 to 1.
+ * Tags we never remove during density scoring (Pass 2).
+ * Headings, paragraphs, and semantic content elements should survive
+ * even if they're small — they carry essential meaning.
  */
-function positionWeight(normalizedPos) {
-    if (normalizedPos >= 0.2 && normalizedPos <= 0.8)
-        return 1.0;
-    if (normalizedPos < 0.2)
-        return normalizedPos / 0.2;
-    // normalizedPos > 0.8
-    return (1.0 - normalizedPos) / 0.2;
-}
-/** Max HTML length for a "leaf" block — blocks larger than this are recursed into */
-const MAX_LEAF_BLOCK_HTML = 5000;
+const DENSITY_SAFE_TAGS = new Set([
+    'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+    'p', 'pre', 'code', 'blockquote', 'figcaption',
+    'main', 'article', 'body',
+]);
 /**
- * Score a single element and return its BlockData.
+ * Class/id patterns that protect an element from removal.
  */
-function scoreElement($, el) {
+const CONTENT_PATTERNS = [
+    /\barticle/,
+    /\bpost-?content/,
+    /\bentry-?content/,
+    /\bmain-?content/,
+    /\bstory/,
+    /\bblog/,
+    /\bpage-?content/,
+    /\bcontent-?area/,
+];
+function isChromeBySemantic(el, $) {
     const tagName = el.tagName?.toLowerCase() ?? '';
-    const $el = $(el);
-    const outerHtml = $.html($el) ?? '';
-    // Clone to compute visible text (strip scripts/styles)
-    const clone = $el.clone();
-    clone.find('script, style, noscript').remove();
-    const visibleText = clone.text() ?? '';
-    const visibleTextLen = visibleText.trim().length;
-    const totalHtmlLen = Math.max(outerHtml.length, 1);
-    // Text density: ratio of visible text to total HTML length
-    const textDensity = Math.min(visibleTextLen / totalHtmlLen, 1.0);
-    // Link density: ratio of link text to visible text
-    let linkTextLen = 0;
-    $el.find('a').each((_i, aEl) => {
-        linkTextLen += ($(aEl).text() ?? '').trim().length;
-    });
-    const linkDensity = visibleTextLen > 0
-        ? Math.min(linkTextLen / visibleTextLen, 1.0)
-        : 0;
-    return {
-        element: el,
-        tagName,
-        htmlLength: outerHtml.length,
-        visibleText,
-        textDensity,
-        linkDensity,
-        normalizedTagScore: normalizeTagScore(getTagImportance(tagName)),
-        wordBonus: wordCountBonus(visibleText),
-        score: 0,
-    };
+    if (CHROME_TAGS.has(tagName))
+        return true;
+    const cls = ($(el).attr('class') ?? '').toLowerCase();
+    const id = ($(el).attr('id') ?? '').toLowerCase();
+    const combined = cls + ' ' + id;
+    // Don't remove if it matches a content pattern
+    for (const p of CONTENT_PATTERNS) {
+        if (p.test(combined))
+            return false;
+    }
+    for (const p of CHROME_PATTERNS) {
+        if (p.test(combined))
+            return true;
+    }
+    // Role attribute
+    const role = ($(el).attr('role') ?? '').toLowerCase();
+    if (['navigation', 'banner', 'complementary', 'contentinfo', 'search'].includes(role)) {
+        return true;
+    }
+    return false;
+}
+// -----------------------------------------------------------------------
+// Pass 2 — Density scoring
+// -----------------------------------------------------------------------
+/** Tag importance scores for density scoring (-2 to +3) */
+const TAG_IMPORTANCE = {
+    article: 3, main: 3,
+    p: 2, h1: 2, h2: 2, h3: 2, h4: 2, h5: 2, h6: 2,
+    blockquote: 2, pre: 2, code: 2, figure: 2, figcaption: 2,
+    section: 1, td: 1, th: 1, li: 1, dd: 1, dt: 1,
+    div: 0, span: 0, table: 0, ul: 0, ol: 0, dl: 0,
+    aside: -1, header: -1, form: -1,
+    nav: -2, footer: -2,
+};
+function normalizeTagScore(rawScore) {
+    return (rawScore + 2) / 5; // -2..+3 → 0..1
 }
 /**
- * Recursively collect block elements for scoring.
+ * Collect scoreable blocks from a DOM tree.
  *
- * Key insight: if a block is very large (>MAX_LEAF_BLOCK_HTML chars), we recurse
- * into its children instead of treating it as one unit. This handles sites like HN
- * (table-based layout) and sites wrapped in a single <div>.
+ * Strategy: walk the tree top-down. For each element:
+ *   - If it's a "leaf-ish" block (< threshold size), score it as one unit.
+ *   - If it's large and a wrapper (div/section/table), recurse into children.
+ *   - Protected elements are always recursed.
  *
- * Protected elements (main, article, body) are always recursed into.
+ * This finds the right granularity: not scoring a 200KB wrapper div,
+ * but scoring the divs/sections/p's nested 3-4 levels deep that carry
+ * actual content or chrome.
  */
-function collectBlocks($, parent, blocks, totalHtmlLength, depth = 0) {
+function collectBlocks($, parent, blocks, maxLeafSize) {
     const children = 'children' in parent ? parent.children : [];
     for (const child of children) {
         if (child.type !== 'tag')
             continue;
         const el = child;
         const tagName = el.tagName?.toLowerCase() ?? '';
-        if (BLOCK_ELEMENTS.has(tagName)) {
-            const data = scoreElement($, el);
-            // Recurse into large blocks, protected elements, and layout containers
-            // to find the actual content sub-blocks
-            const isLarge = data.htmlLength > MAX_LEAF_BLOCK_HTML;
-            const isProtected = PROTECTED_ELEMENTS.has(tagName);
-            const isLayoutContainer = tagName === 'div' || tagName === 'section' || tagName === 'table';
-            if ((isLarge && isLayoutContainer) || isProtected) {
-                // Recurse into children to find sub-blocks
-                collectBlocks($, el, blocks, totalHtmlLength, depth + 1);
-            }
-            else {
-                // Score this block as a leaf
-                blocks.push(data);
-            }
-        }
-        else if (tagName === 'tr' || tagName === 'td' || tagName === 'th' || tagName === 'tbody' || tagName === 'thead') {
-            // Table layout elements — recurse through them to find block content
-            collectBlocks($, el, blocks, totalHtmlLength, depth + 1);
+        // Skip script/style
+        if (tagName === 'script' || tagName === 'style' || tagName === 'link' || tagName === 'meta')
+            continue;
+        const $el = $(el);
+        const outerHtml = $.html($el) ?? '';
+        const htmlLen = outerHtml.length;
+        // Skip extremely tiny elements (bare tags like <br>)
+        if (htmlLen < 10)
+            continue;
+        const isProtected = PROTECTED_TAGS.has(tagName);
+        const isWrapper = ['div', 'section', 'table', 'tbody', 'thead', 'tr',
+            'center', 'details', 'summary'].includes(tagName);
+        if (isProtected || (isWrapper && htmlLen > maxLeafSize)) {
+            // Too large or protected — recurse deeper
+            collectBlocks($, el, blocks, maxLeafSize);
         }
-        else {
-            // Non-block element — recurse to find nested blocks
-            collectBlocks($, el, blocks, totalHtmlLength, depth + 1);
+        else if (htmlLen > 0) {
+            // Score this element
+            const clone = $el.clone();
+            clone.find('script, style, noscript, svg, path').remove();
+            const visibleText = clone.text() ?? '';
+            const visibleTextLen = visibleText.trim().length;
+            const textDensity = Math.min(visibleTextLen / Math.max(htmlLen, 1), 1.0);
+            let linkTextLen = 0;
+            $el.find('a').each((_i, a) => {
+                linkTextLen += ($(a).text() ?? '').trim().length;
+            });
+            const linkDensity = visibleTextLen > 0
+                ? Math.min(linkTextLen / visibleTextLen, 1.0)
+                : 0;
+            const rawTagScore = TAG_IMPORTANCE[tagName] ?? 0;
+            const normalizedTag = normalizeTagScore(rawTagScore);
+            const words = visibleText.trim().split(/\s+/).filter(w => w.length > 0);
+            const wordBonus = words.length > 0
+                ? Math.min(Math.log(words.length + 1) / Math.log(1000), 1.0)
+                : 0;
+            const score = (textDensity * 0.35 +
+                (1 - linkDensity) * 0.25 +
+                normalizedTag * 0.2 +
+                wordBonus * 0.1 +
+                0.1 // baseline position score (removed position bias — not useful for deep nesting)
+            );
+            blocks.push({
+                element: el,
+                tagName,
+                htmlLength: htmlLen,
+                visibleText,
+                score,
+            });
         }
     }
 }
+// -----------------------------------------------------------------------
+// Main export
+// -----------------------------------------------------------------------
 /**
- * Compute the max of an array of numbers.
- */
-function maxValue(values) {
-    if (values.length === 0)
-        return 0;
-    return Math.max(...values);
-}
-/**
- * Prune low-value HTML blocks using content density scoring.
+ * Prune low-value HTML blocks using two-pass approach:
+ *   1. Semantic tag/class removal
+ *   2. Density scoring of remaining blocks
  *
  * @param html - Raw HTML to prune
  * @param options - Pruning configuration
  * @returns Pruned HTML with stats
  */
 export function pruneContent(html, options = {}) {
-    const { threshold = 0.4, minWords = 3, dynamic = true, } = options;
+    const { threshold = 0.3, minWords = 3, dynamic = true, } = options;
     const originalLength = html.length;
     if (!html.trim()) {
         return { html, nodesRemoved: 0, reductionPercent: 0 };
     }
     const $ = cheerio.load(html);
-    // Collect top-level block elements from the body
-    const blocks = [];
-    const bodyEl = $('body').get(0);
-    if (bodyEl) {
-        collectBlocks($, bodyEl, blocks, originalLength);
-    }
-    // If no blocks found (very sparse HTML), return as-is
-    if (blocks.length === 0) {
-        return { html, nodesRemoved: 0, reductionPercent: 0 };
+    let nodesRemoved = 0;
+    // =====================================================================
+    // Pass 1: Semantic removal
+    // =====================================================================
+    // Walk top-down; remove entire subtrees that are clearly chrome.
+    // We look at direct children of body, and one level deeper, to catch
+    // both <body> <nav> and <body> <div> <nav> patterns.
+    const toRemoveSemantic = [];
+    function walkForChrome(parent, depth) {
+        const children = 'children' in parent ? parent.children : [];
+        for (const child of children) {
+            if (child.type !== 'tag')
+                continue;
+            const el = child;
+            const tagName = el.tagName?.toLowerCase() ?? '';
+            if (tagName === 'script' || tagName === 'style')
+                continue;
+            if (PROTECTED_TAGS.has(tagName)) {
+                // Recurse into protected — there might be chrome inside <article>
+                walkForChrome(el, depth + 1);
+                continue;
+            }
+            if (isChromeBySemantic(el, $)) {
+                toRemoveSemantic.push(el);
+                continue; // don't recurse into something we'll remove
+            }
+            // Recurse up to a reasonable depth
+            if (depth < 6) {
+                walkForChrome(el, depth + 1);
+            }
+        }
     }
-    // Assign position weights and compute composite scores
-    const n = blocks.length;
-    for (let i = 0; i < n; i++) {
-        const block = blocks[i];
-        const normalizedPos = n > 1 ? i / (n - 1) : 0.5;
-        const posWeight = positionWeight(normalizedPos);
-        block.score = (block.textDensity * 0.35 +
-            (1 - block.linkDensity) * 0.25 +
-            block.normalizedTagScore * 0.2 +
-            block.wordBonus * 0.1 +
-            posWeight * 0.1);
+    const body = $('body').get(0);
+    if (body) {
+        walkForChrome(body, 0);
     }
-    // Determine effective threshold
-    let effectiveThreshold = threshold;
-    if (dynamic) {
-        // Use the best-block score as the reference: remove blocks that score below
-        // 40% of the highest-quality block. This handles the common bimodal case
-        // (one great article block + several low-quality nav/sidebar blocks) much
-        // better than median/mean approaches.
-        const scores = blocks.map((b) => b.score);
-        const best = maxValue(scores);
-        effectiveThreshold = best * 0.4;
+    for (const el of toRemoveSemantic) {
+        $(el).remove();
+        nodesRemoved++;
     }
-    // Safety floor: we must retain at least 30% of the original HTML
-    const minRetainLength = Math.ceil(originalLength * 0.3);
-    // Sort ascending by score so we remove worst blocks first
-    const sortedAsc = [...blocks].sort((a, b) => a.score - b.score);
-    const toRemove = new Set();
-    let removedLength = 0;
-    for (const block of sortedAsc) {
-        // Never remove protected containers
-        if (PROTECTED_ELEMENTS.has(block.tagName))
-            continue;
-        const words = block.visibleText.trim().split(/\s+/).filter((w) => w.length > 0);
-        const isTinyBlock = words.length < minWords;
-        const isLowScore = block.score < effectiveThreshold;
-        // Keep blocks that pass both checks
-        if (!isTinyBlock && !isLowScore)
-            continue;
-        // Always check safety floor before removing — even for empty blocks.
-        // This prevents over-pruning when every block is low quality.
-        const remainingLength = originalLength - (removedLength + block.htmlLength);
-        if (remainingLength >= minRetainLength) {
-            toRemove.add(block.element);
-            removedLength += block.htmlLength;
+    // =====================================================================
+    // Pass 2: Density scoring (on the remaining HTML)
+    // =====================================================================
+    const postPass1Html = $.html();
+    const postPass1Length = postPass1Html.length;
+    // Run density scoring on remaining content
+    if (postPass1Length > 100 && body) {
+        const blocks = [];
+        // Max leaf size: ~5KB or 30% of remaining content (whichever is smaller)
+        // This ensures we find leaf blocks even in small documents.
+        const maxLeafSize = Math.min(5000, Math.ceil(postPass1Length * 0.3));
+        collectBlocks($, body, blocks, maxLeafSize);
+        if (blocks.length >= 2) {
+            const scores = blocks.map(b => b.score);
+            const bestScore = Math.max(...scores);
+            let effectiveThreshold = threshold;
+            if (dynamic) {
+                // Blocks scoring below 50% of the best block are candidates for removal
+                effectiveThreshold = bestScore * 0.5;
+            }
+            // Safety: retain at least 40% of post-pass1 content
+            const minRetainLength = Math.ceil(postPass1Length * 0.4);
+            // Sort ascending by score — remove worst first
+            const sorted = blocks
+                .map((b, i) => ({ b, i, score: b.score }))
+                .sort((a, b) => a.score - b.score);
+            const toRemoveDensity = new Set();
+            let removedLength = 0;
+            for (const { b } of sorted) {
+                if (PROTECTED_TAGS.has(b.tagName) || DENSITY_SAFE_TAGS.has(b.tagName))
+                    continue;
+                const words = b.visibleText.trim().split(/\s+/).filter(w => w.length > 0);
+                const isTiny = words.length < minWords;
+                const isLow = b.score < effectiveThreshold;
+                if (!isTiny && !isLow)
+                    continue;
+                // Check safety floor
+                const remaining = postPass1Length - (removedLength + b.htmlLength);
+                if (remaining < minRetainLength)
+                    continue;
+                toRemoveDensity.add(b.element);
+                removedLength += b.htmlLength;
+            }
+            for (const el of toRemoveDensity) {
+                $(el).remove();
+                nodesRemoved++;
+            }
         }
     }
-    // Remove selected elements from the DOM
-    for (const el of toRemove) {
-        $(el).remove();
-    }
     const resultHtml = $.html() ?? html;
     const resultLength = resultHtml.length;
     const reductionPercent = originalLength > 0
@@ -242,7 +299,7 @@ export function pruneContent(html, options = {}) {
         : 0;
     return {
         html: resultHtml,
-        nodesRemoved: toRemove.size,
+        nodesRemoved,
         reductionPercent,
     };
 }

package/dist/core/content-pruner.js.map

@@ -1 +1 @@
-{"version":3,"file":"content-pruner.js","sourceRoot":"","sources":["../../src/core/content-pruner.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAqBnC,oCAAoC;AACpC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ;IAC/D,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS;CAC5E,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;AAEhE;;;GAGG;AACH,MAAM,cAAc,GAA2B;IAC7C,OAAO,EAAE,CAAC;IACV,IAAI,EAAE,CAAC;IACP,CAAC,EAAE,CAAC;IACJ,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;IACxC,UAAU,EAAE,CAAC;IACb,GAAG,EAAE,CAAC;IACN,IAAI,EAAE,CAAC;IACP,MAAM,EAAE,CAAC;IACT,UAAU,EAAE,CAAC;IACb,OAAO,EAAE,CAAC;IACV,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,CAAC;IACL,GAAG,EAAE,CAAC;IACN,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC,CAAC;IACT,MAAM,EAAE,CAAC,CAAC;IACV,IAAI,EAAE,CAAC,CAAC;IACR,GAAG,EAAE,CAAC,CAAC;IACP,MAAM,EAAE,CAAC,CAAC;CACX,CAAC;AAEF,sDAAsD;AACtD,SAAS,iBAAiB,CAAC,QAAgB;IACzC,sDAAsD;IACtD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,OAAO,cAAc,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC;AACpD,CAAC;AAED,6CAA6C;AAC7C,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;AACpE,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,aAAqB;IAC3C,IAAI,aAAa,IAAI,GAAG,IAAI,aAAa,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAC7D,IAAI,aAAa,GAAG,GAAG;QAAE,OAAO,aAAa,GAAG,GAAG,CAAC;IACpD,sBAAsB;IACtB,OAAO,CAAC,GAAG,GAAG,aAAa,CAAC,GAAG,GAAG,CAAC;AACrC,CAAC;AAeD,qFAAqF;AACrF,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC;;GAEG;AACH,SAAS,YAAY,CACnB,CAAqB,EACrB,EAAW;IAEX,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAChD,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IAClB,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IAEpC,uDAAuD;IACvD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,EAAE,CAAC;IAC1B,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,MAAM,EAAE,CAAC;IAC/C,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IACvC,MAAM,cAAc,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;IACjD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEnD,2DAA2D;IAC3D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,GAAG,YAAY,EAAE,GAAG,CAAC,CAAC;IAEjE,mDAAmD;IACnD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE;QAC7B,WAAW,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;IACrD,CAAC,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,cAAc,GAAG,CAAC;QACpC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,cAAc,EAAE,GAAG,CAAC;QAC7C,CAAC,CAAC,CAAC,CAAC;IAEN,OAAO;QACL,OAAO,EAAE,EAAE;QACX,OAAO;QACP,UAAU,EAAE,SAAS,CAAC,MAAM;QAC5B,WAAW;QACX,WAAW;QACX,WAAW;QACX,kBAAkB,EAAE,iBAAiB,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAChE,SAAS,EAAE,cAAc,CAAC,WAAW,CAAC;QACtC,KAAK,EAAE,CAAC;KACT,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,aAAa,CACpB,CAAqB,EACrB,MAAe,EACf,MAAmB,EACnB,eAAuB,EACvB,QAAgB,CAAC;IAEjB,MAAM,QAAQ,GAAG,UAAU,IAAI,MAAM,CAAC,CAAC,CAAE,MAAM,CAAC,QAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5E,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK;YAAE,SAAS;QACnC,MAAM,EAAE,GAAG,KAAgB,CAAC;QAC5B,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAEhD,IAAI,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAEjC,uEAAuE;YACvE,wCAAwC;YACxC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,GAAG,mBAAmB,CAAC;YACtD,MAAM,WAAW,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpD,MAAM,iBAAiB,GAAG,OAAO,KAAK,KAAK,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,OAAO,CAAC;YAE5F,IAAI,CAAC,OAAO,IAAI,iBAAiB,CAAC,IAAI,WAAW,EAAE,CAAC;gBAClD,2CAA2C;gBAC3C,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAC3D,CAAC;iBAAM,CAAC;gBACN,6BAA6B;gBAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,OAAO,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;YAClH,qEAAqE;YACrE,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QAC3D,CAAC;aAAM,CAAC;YACN,oDAAoD;YACpD,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,QAAQ,CAAC,MAAgB;IAChC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;AAC7B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,UAAwB,EAAE;IACnE,MAAM,EACJ,SAAS,GAAG,GAAG,EACf,QAAQ,GAAG,CAAC,EACZ,OAAO,GAAG,IAAI,GACf,GAAG,OAAO,CAAC;IAEZ,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;IAEnC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACjB,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,iDAAiD;IACjD,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAChC,IAAI,MAAM,EAAE,CAAC;QACX,aAAa,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,CAAC,CAAC;IACnD,CAAC;IAED,sDAAsD;IACtD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IACxD,CAAC;IAED,uDAAuD;IACvD,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACzB,MAAM,aAAa,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAChD,MAAM,SAAS,GAAG,cAAc,CAAC,aAAa,CAAC,CAAC;QAEhD,KAAK,CAAC,KAAK,GAAG,CACZ,KAAK,CAAC,WAAW,GAAG,IAAI;YACxB,CAAC,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI;YAC9B,KAAK,CAAC,kBAAkB,GAAG,GAAG;YAC9B,KAAK,CAAC,SAAS,GAAG,GAAG;YACrB,SAAS,GAAG,GAAG,CAChB,CAAC;IACJ,CAAC;IAED,gCAAgC;IAChC,IAAI,kBAAkB,GAAG,SAAS,CAAC;IACnC,IAAI,OAAO,EAAE,CAAC;QACZ,4EAA4E;QAC5E,yEAAyE;QACzE,0EAA0E;QAC1E,sCAAsC;QACtC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC9B,kBAAkB,GAAG,IAAI,GAAG,GAAG,CAAC;IAClC,CAAC;IAED,iEAAiE;IACjE,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC;IAExD,0DAA0D;IAC1D,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAEhE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAW,CAAC;IACpC,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,oCAAoC;QACpC,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,SAAS;QAEpD,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChF,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,GAAG,kBAAkB,CAAC;QAEpD,oCAAoC;QACpC,IAAI,CAAC,WAAW,IAAI,CAAC,UAAU;YAAE,SAAS;QAE1C,qEAAqE;QACrE,8DAA8D;QAC9D,MAAM,eAAe,GAAG,cAAc,GAAG,CAAC,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;QAC5E,IAAI,eAAe,IAAI,eAAe,EAAE,CAAC;YACvC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5B,aAAa,IAAI,KAAK,CAAC,UAAU,CAAC;QACpC,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;IACjB,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;IACpC,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC;IACvC,MAAM,gBAAgB,GAAG,cAAc,GAAG,CAAC;QACzC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,GAAG,YAAY,CAAC,GAAG,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC,CAAC;IAEN,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,gBAAgB;KACjB,CAAC;AACJ,CAAC"}
+{"version":3,"file":"content-pruner.js","sourceRoot":"","sources":["../../src/core/content-pruner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAqBnC,0EAA0E;AAC1E,wDAAwD;AACxD,0EAA0E;AAE1E,oEAAoE;AACpE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC;IAC1B,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU;CACrC,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,eAAe,GAAG;IACtB,aAAa;IACb,UAAU;IACV,YAAY;IACZ,qBAAqB;IACrB,WAAW;IACX,WAAW;IACX,aAAa;IACb,UAAU;IACV,WAAW;IACX,cAAc;IACd,cAAc;IACd,YAAY;IACZ,2BAA2B;IAC3B,cAAc;IACd,WAAW;IACX,WAAW;IACX,iCAAiC;IACjC,SAAS;IACT,yBAAyB;IACzB,UAAU;IACV,aAAa;CACd,CAAC;AAEF;;;GAGG;AACH,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;AAE5D;;;;GAIG;AACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;IAClC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY;IAC9C,MAAM,EAAE,SAAS,EAAE,MAAM;CAC1B,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,gBAAgB,GAAG;IACvB,WAAW;IACX,iBAAiB;IACjB,kBAAkB;IAClB,iBAAiB;IACjB,SAAS;IACT,QAAQ;IACR,iBAAiB;IACjB,iBAAiB;CAClB,CAAC;AAEF,SAAS,kBAAkB,CAAC,EAAW,EAAE,CAAqB;IAC5D,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAChD,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1C,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IACtD,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAClD,MAAM,QAAQ,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC;IAEhC,+CAA+C;IAC/C,KAAK,MAAM,CAAC,IAAI,gBAAgB,EAAE,CAAC;QACjC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,OAAO,KAAK,CAAC;IACrC,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,eAAe,EAAE,CAAC;QAChC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC;IACpC,CAAC;IAED,iBAAiB;IACjB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IACtD,IAAI,CAAC,YAAY,EAAE,QAAQ,EAAE,eAAe,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACtF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0EAA0E;AAC1E,2BAA2B;AAC3B,0EAA0E;AAE1E,2DAA2D;AAC3D,MAAM,cAAc,GAA2B;IAC7C,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC;IACnB,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;IAC9C,UAAU,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC;IACxD,OAAO,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;IAC7C,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;IAC9C,KAAK,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/B,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;CACpB,CAAC;AAEF,SAAS,iBAAiB,CAAC,QAAgB;IACzC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB;AAC7C,CAAC;AAUD;;;;;;;;;;;GAWG;AACH,SAAS,aAAa,CACpB,CAAqB,EACrB,MAAe,EACf,MAAqB,EACrB,WAAmB;IAEnB,MAAM,QAAQ,GAAG,UAAU,IAAI,MAAM,CAAC,CAAC,CAAE,MAAM,CAAC,QAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;IAE5E,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK;YAAE,SAAS;QACnC,MAAM,EAAE,GAAG,KAAgB,CAAC;QAC5B,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAEhD,oBAAoB;QACpB,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,OAAO,IAAI,OAAO,KAAK,MAAM,IAAI,OAAO,KAAK,MAAM;YAAE,SAAS;QAEtG,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC;QAEjC,qDAAqD;QACrD,IAAI,OAAO,GAAG,EAAE;YAAE,SAAS;QAE3B,MAAM,WAAW,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,SAAS,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI;YACjD,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAErE,IAAI,WAAW,IAAI,CAAC,SAAS,IAAI,OAAO,GAAG,WAAW,CAAC,EAAE,CAAC;YACxD,0CAA0C;YAC1C,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAC5C,CAAC;aAAM,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACvB,qBAAqB;YACrB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC,MAAM,EAAE,CAAC;YAC1D,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;YACvC,MAAM,cAAc,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;YAEjD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAEzE,IAAI,WAAW,GAAG,CAAC,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE;gBAC3B,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;YACnD,CAAC,CAAC,CAAC;YACH,MAAM,WAAW,GAAG,cAAc,GAAG,CAAC;gBACpC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,cAAc,EAAE,GAAG,CAAC;gBAC7C,CAAC,CAAC,CAAC,CAAC;YAEN,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,aAAa,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;YAErD,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACxE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC;gBAChC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC;gBAC5D,CAAC,CAAC,CAAC,CAAC;YAEN,MAAM,KAAK,GAAG,CACZ,WAAW,GAAG,IAAI;gBAClB,CAAC,CAAC,GAAG,WAAW,CAAC,GAAG,IAAI;gBACxB,aAAa,GAAG,GAAG;gBACnB,SAAS,GAAG,GAAG;gBACf,GAAG,CAAC,gFAAgF;aACrF,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC;gBACV,OAAO,EAAE,EAAE;gBACX,OAAO;gBACP,UAAU,EAAE,OAAO;gBACnB,WAAW;gBACX,KAAK;aACN,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,cAAc;AACd,0EAA0E;AAE1E;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,UAAwB,EAAE;IACnE,MAAM,EACJ,SAAS,GAAG,GAAG,EACf,QAAQ,GAAG,CAAC,EACZ,OAAO,GAAG,IAAI,GACf,GAAG,OAAO,CAAC;IAEZ,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;IACnC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACjB,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,wEAAwE;IACxE,2BAA2B;IAC3B,wEAAwE;IACxE,iEAAiE;IACjE,qEAAqE;IACrE,qDAAqD;IACrD,MAAM,gBAAgB,GAAc,EAAE,CAAC;IAEvC,SAAS,aAAa,CAAC,MAAe,EAAE,KAAa;QACnD,MAAM,QAAQ,GAAG,UAAU,IAAI,MAAM,CAAC,CAAC,CAAE,MAAM,CAAC,QAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK;gBAAE,SAAS;YACnC,MAAM,EAAE,GAAG,KAAgB,CAAC;YAC5B,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YAChD,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,OAAO;gBAAE,SAAS;YAE1D,IAAI,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChC,kEAAkE;gBAClE,aAAa,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gBAC7B,SAAS;YACX,CAAC;YAED,IAAI,kBAAkB,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC;gBAC9B,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC1B,SAAS,CAAC,4CAA4C;YACxD,CAAC;YAED,mCAAmC;YACnC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,aAAa,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,IAAI,EAAE,CAAC;QACT,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,MAAM,EAAE,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;QACf,YAAY,EAAE,CAAC;IACjB,CAAC;IAED,wEAAwE;IACxE,kDAAkD;IAClD,wEAAwE;IACxE,MAAM,aAAa,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/B,MAAM,eAAe,GAAG,aAAa,CAAC,MAAM,CAAC;IAE7C,2CAA2C;IAC3C,IAAI,eAAe,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAClC,MAAM,MAAM,GAAkB,EAAE,CAAC;QACjC,yEAAyE;QACzE,4DAA4D;QAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,CAAC;QACrE,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAE5C,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACxC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YAEtC,IAAI,kBAAkB,GAAG,SAAS,CAAC;YACnC,IAAI,OAAO,EAAE,CAAC;gBACZ,wEAAwE;gBACxE,kBAAkB,GAAG,SAAS,GAAG,GAAG,CAAC;YACvC,CAAC;YAED,oDAAoD;YACpD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC;YAEzD,+CAA+C;YAC/C,MAAM,MAAM,GAAG,MAAM;iBAClB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;iBACzC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAErC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAW,CAAC;YAC3C,IAAI,aAAa,GAAG,CAAC,CAAC;YAEtB,KAAK,MAAM,EAAE,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;oBAAE,SAAS;gBAEhF,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC1E,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;gBACvC,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,GAAG,kBAAkB,CAAC;gBAE3C,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK;oBAAE,SAAS;gBAEhC,qBAAqB;gBACrB,MAAM,SAAS,GAAG,eAAe,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;gBACnE,IAAI,SAAS,GAAG,eAAe;oBAAE,SAAS;gBAE1C,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBAC/B,aAAa,IAAI,CAAC,CAAC,UAAU,CAAC;YAChC,CAAC;YAED,KAAK,MAAM,EAAE,IAAI,eAAe,EAAE,CAAC;gBACjC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;gBACf,YAAY,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;IACpC,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC;IACvC,MAAM,gBAAgB,GAAG,cAAc,GAAG,CAAC;QACzC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,GAAG,YAAY,CAAC,GAAG,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC,CAAC;IAEN,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,YAAY;QACZ,gBAAgB;KACjB,CAAC;AACJ,CAAC"}

package/dist/core/research.d.ts

@@ -0,0 +1,67 @@
+/**
+ * WebPeel Deep Research Agent
+ *
+ * Autonomously searches the web, fetches top sources, filters content with
+ * BM25, optionally follows promising links, and synthesizes a comprehensive
+ * report using an LLM.
+ *
+ * Design principle: orchestrate existing modules (peel, bm25-filter,
+ * llm-extract) — don't reinvent anything.
+ */
+export interface ResearchOptions {
+    /** Research question or topic */
+    query: string;
+    /** Maximum number of sources to consult. Default: 5 */
+    maxSources?: number;
+    /** Maximum depth of link-following. Default: 1 (just search results; 2+ follows links) */
+    maxDepth?: number;
+    /** LLM API key for synthesis */
+    apiKey?: string;
+    /** LLM model for synthesis. Default: gpt-4o-mini */
+    model?: string;
+    /** LLM base URL. Default: https://api.openai.com/v1 */
+    baseUrl?: string;
+    /** Maximum total time in ms. Default: 60000 (1 minute) */
+    timeout?: number;
+    /** Output format: 'report' (markdown synthesis) or 'sources' (raw extracted data). Default: 'report' */
+    outputFormat?: 'report' | 'sources';
+    /** Optional callback for progress updates */
+    onProgress?: (step: ResearchStep) => void;
+}
+export interface ResearchStep {
+    phase: 'searching' | 'fetching' | 'extracting' | 'following' | 'synthesizing';
+    message: string;
+    sourcesFound?: number;
+    sourcesFetched?: number;
+}
+export interface ResearchSource {
+    url: string;
+    title: string;
+    /** Key findings from this source */
+    findings: string;
+    /** Relevance score (0-1) */
+    relevance: number;
+}
+export interface ResearchResult {
+    /** Synthesized research report (markdown) */
+    report: string;
+    /** Sources consulted */
+    sources: ResearchSource[];
+    /** Total sources found vs consulted */
+    totalSourcesFound: number;
+    sourcesConsulted: number;
+    /** Time taken in ms */
+    elapsed: number;
+    /** Tokens used for synthesis */
+    tokensUsed?: {
+        input: number;
+        output: number;
+    };
+    /** Estimated cost in USD */
+    cost?: number;
+}
+/**
+ * Conduct autonomous multi-step web research on a topic.
+ */
+export declare function research(options: ResearchOptions): Promise<ResearchResult>;
+//# sourceMappingURL=research.d.ts.map