webpeel 0.16.0 → 0.17.0

package/dist/core/chunker.js

@@ -0,0 +1,250 @@
+/**
+ * Content chunker for RAG pipelines.
+ * Splits markdown content into overlapping chunks with rich metadata.
+ */
+const DEFAULT_MAX_TOKENS = 512;
+const DEFAULT_OVERLAP = 50;
+const CHARS_PER_TOKEN = 4; // rough approximation
+/**
+ * Split content into RAG-ready chunks with metadata.
+ */
+export function chunkContent(content, options = {}) {
+    const maxTokens = options.maxTokens || DEFAULT_MAX_TOKENS;
+    const overlap = options.overlap || DEFAULT_OVERLAP;
+    const strategy = options.strategy || 'section';
+    const opts = { maxTokens, overlap, strategy };
+    let chunks;
+    switch (strategy) {
+        case 'section':
+            chunks = chunkBySection(content, maxTokens, overlap);
+            break;
+        case 'paragraph':
+            chunks = chunkByParagraph(content, maxTokens, overlap);
+            break;
+        case 'fixed':
+            chunks = chunkByFixed(content, maxTokens, overlap);
+            break;
+        default:
+            chunks = chunkBySection(content, maxTokens, overlap);
+    }
+    return {
+        chunks,
+        totalChunks: chunks.length,
+        originalLength: content.length,
+        strategy,
+        options: opts,
+    };
+}
+/**
+ * Section-based chunking (recommended for RAG).
+ * Splits on markdown headings (## / ### etc.), then splits large sections by paragraph.
+ * Each chunk includes its section heading for context.
+ */
+function chunkBySection(content, maxTokens, overlap) {
+    const maxChars = maxTokens * CHARS_PER_TOKEN;
+    const overlapChars = overlap * CHARS_PER_TOKEN;
+    const chunks = [];
+    // Split content into sections by headings
+    const sections = splitByHeadings(content);
+    let chunkIndex = 0;
+    for (const section of sections) {
+        const { heading, depth, body, startOffset } = section;
+        if (!body.trim())
+            continue;
+        // If section fits in one chunk, use it directly
+        if (body.length <= maxChars) {
+            const text = heading ? `${heading}\n\n${body.trim()}` : body.trim();
+            chunks.push({
+                index: chunkIndex++,
+                text,
+                tokenCount: Math.ceil(text.length / CHARS_PER_TOKEN),
+                wordCount: text.split(/\s+/).filter(Boolean).length,
+                section: heading ? heading.replace(/^#+\s*/, '') : null,
+                sectionDepth: depth,
+                startOffset,
+                endOffset: startOffset + body.length,
+            });
+        }
+        else {
+            // Large section — split by paragraphs with overlap
+            const paragraphs = body.split(/\n\n+/).filter(p => p.trim());
+            let currentText = '';
+            let currentStart = startOffset;
+            for (const para of paragraphs) {
+                const candidate = currentText ? `${currentText}\n\n${para}` : para;
+                if (candidate.length > maxChars && currentText) {
+                    // Emit current chunk
+                    const text = heading ? `${heading}\n\n${currentText.trim()}` : currentText.trim();
+                    chunks.push({
+                        index: chunkIndex++,
+                        text,
+                        tokenCount: Math.ceil(text.length / CHARS_PER_TOKEN),
+                        wordCount: text.split(/\s+/).filter(Boolean).length,
+                        section: heading ? heading.replace(/^#+\s*/, '') : null,
+                        sectionDepth: depth,
+                        startOffset: currentStart,
+                        endOffset: currentStart + currentText.length,
+                    });
+                    // Start new chunk with overlap from end of previous
+                    if (overlapChars > 0 && currentText.length > overlapChars) {
+                        currentText = currentText.slice(-overlapChars) + '\n\n' + para;
+                    }
+                    else {
+                        currentText = para;
+                    }
+                    currentStart = startOffset + body.indexOf(para);
+                }
+                else {
+                    currentText = candidate;
+                }
+            }
+            // Emit remaining
+            if (currentText.trim()) {
+                const text = heading ? `${heading}\n\n${currentText.trim()}` : currentText.trim();
+                chunks.push({
+                    index: chunkIndex++,
+                    text,
+                    tokenCount: Math.ceil(text.length / CHARS_PER_TOKEN),
+                    wordCount: text.split(/\s+/).filter(Boolean).length,
+                    section: heading ? heading.replace(/^#+\s*/, '') : null,
+                    sectionDepth: depth,
+                    startOffset: currentStart,
+                    endOffset: currentStart + currentText.length,
+                });
+            }
+        }
+    }
+    return chunks;
+}
+/**
+ * Paragraph-based chunking.
+ * Groups paragraphs together up to maxTokens, with overlap.
+ */
+function chunkByParagraph(content, maxTokens, overlap) {
+    const maxChars = maxTokens * CHARS_PER_TOKEN;
+    const overlapChars = overlap * CHARS_PER_TOKEN;
+    const chunks = [];
+    const paragraphs = content.split(/\n\n+/).filter(p => p.trim());
+    let currentText = '';
+    let currentStart = 0;
+    let chunkIndex = 0;
+    // Track current section heading
+    let currentHeading = null;
+    let currentDepth = null;
+    for (const para of paragraphs) {
+        // Check if paragraph is a heading
+        const headingMatch = para.match(/^(#{1,6})\s+(.+)/);
+        if (headingMatch) {
+            currentHeading = headingMatch[2];
+            currentDepth = headingMatch[1].length;
+        }
+        const candidate = currentText ? `${currentText}\n\n${para}` : para;
+        if (candidate.length > maxChars && currentText) {
+            chunks.push({
+                index: chunkIndex++,
+                text: currentText.trim(),
+                tokenCount: Math.ceil(currentText.length / CHARS_PER_TOKEN),
+                wordCount: currentText.split(/\s+/).filter(Boolean).length,
+                section: currentHeading,
+                sectionDepth: currentDepth,
+                startOffset: currentStart,
+                endOffset: currentStart + currentText.length,
+            });
+            if (overlapChars > 0 && currentText.length > overlapChars) {
+                currentText = currentText.slice(-overlapChars) + '\n\n' + para;
+            }
+            else {
+                currentText = para;
+            }
+            currentStart = content.indexOf(para, currentStart);
+        }
+        else {
+            currentText = candidate;
+        }
+    }
+    if (currentText.trim()) {
+        chunks.push({
+            index: chunkIndex++,
+            text: currentText.trim(),
+            tokenCount: Math.ceil(currentText.length / CHARS_PER_TOKEN),
+            wordCount: currentText.split(/\s+/).filter(Boolean).length,
+            section: currentHeading,
+            sectionDepth: currentDepth,
+            startOffset: currentStart,
+            endOffset: currentStart + currentText.length,
+        });
+    }
+    return chunks;
+}
+/**
+ * Fixed-size chunking with overlap.
+ * Simple character-based splitting for predictable chunk sizes.
+ */
+function chunkByFixed(content, maxTokens, overlap) {
+    const maxChars = maxTokens * CHARS_PER_TOKEN;
+    const overlapChars = overlap * CHARS_PER_TOKEN;
+    const step = Math.max(maxChars - overlapChars, 100);
+    const chunks = [];
+    let chunkIndex = 0;
+    for (let i = 0; i < content.length; i += step) {
+        const text = content.slice(i, i + maxChars).trim();
+        if (!text)
+            continue;
+        // Try to find section heading within this chunk
+        const headingMatch = text.match(/^(#{1,6})\s+(.+)/m);
+        chunks.push({
+            index: chunkIndex++,
+            text,
+            tokenCount: Math.ceil(text.length / CHARS_PER_TOKEN),
+            wordCount: text.split(/\s+/).filter(Boolean).length,
+            section: headingMatch ? headingMatch[2] : null,
+            sectionDepth: headingMatch ? headingMatch[1].length : null,
+            startOffset: i,
+            endOffset: Math.min(i + maxChars, content.length),
+        });
+    }
+    return chunks;
+}
+/** Split content into sections based on markdown headings */
+function splitByHeadings(content) {
+    const lines = content.split('\n');
+    const sections = [];
+    let currentHeading = null;
+    let currentDepth = null;
+    let currentBody = [];
+    let currentStart = 0;
+    let offset = 0;
+    for (const line of lines) {
+        const headingMatch = line.match(/^(#{1,6})\s+(.+)/);
+        if (headingMatch) {
+            // Save previous section
+            if (currentBody.length > 0 || currentHeading) {
+                sections.push({
+                    heading: currentHeading,
+                    depth: currentDepth,
+                    body: currentBody.join('\n'),
+                    startOffset: currentStart,
+                });
+            }
+            currentHeading = line;
+            currentDepth = headingMatch[1].length;
+            currentBody = [];
+            currentStart = offset;
+        }
+        else {
+            currentBody.push(line);
+        }
+        offset += line.length + 1; // +1 for newline
+    }
+    // Don't forget last section
+    if (currentBody.length > 0 || currentHeading) {
+        sections.push({
+            heading: currentHeading,
+            depth: currentDepth,
+            body: currentBody.join('\n'),
+            startOffset: currentStart,
+        });
+    }
+    return sections;
+}
+//# sourceMappingURL=chunker.js.map

package/dist/core/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/core/chunker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA2CH,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,eAAe,GAAG,EAAE,CAAC;AAC3B,MAAM,eAAe,GAAG,CAAC,CAAC,CAAC,sBAAsB;AAEjD;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,UAAwB,EAAE;IACtE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC;IAE/C,MAAM,IAAI,GAA2B,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAEtE,IAAI,MAAsB,CAAC;IAE3B,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,SAAS;YACZ,MAAM,GAAG,cAAc,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACrD,MAAM;QACR,KAAK,WAAW;YACd,MAAM,GAAG,gBAAgB,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM;QACR,KAAK,OAAO;YACV,MAAM,GAAG,YAAY,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM;QACR;YACE,MAAM,GAAG,cAAc,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACzD,CAAC;IAED,OAAO;QACL,MAAM;QACN,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,cAAc,EAAE,OAAO,CAAC,MAAM;QAC9B,QAAQ;QACR,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,OAAe,EAAE,SAAiB,EAAE,OAAe;IACzE,MAAM,QAAQ,GAAG,SAAS,GAAG,eAAe,CAAC;IAC7C,MAAM,YAAY,GAAG,OAAO,GAAG,eAAe,CAAC;IAC/C,MAAM,MAAM,GAAmB,EAAE,CAAC;IAElC,0CAA0C;IAC1C,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAE1C,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAEtD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAS;QAE3B,gDAAgD;QAChD,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,OAAO,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACpE,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,UAAU,EAAE;gBACnB,IAAI;gBACJ,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC;gBACpD,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM;gBACnD,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;gBACvD,YAAY,EAAE,KAAK;gBACnB,WAAW;gBACX,SAAS,EAAE,WAAW,GAAG,IAAI,CAAC,MAAM;aACrC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,mDAAmD;YACnD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7D,IAAI,WAAW,GAAG,EAAE,CAAC;YACrB,IAAI,YAAY,GAAG,WAAW,CAAC;YAE/B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC9B,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,WAAW,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;gBAEnE,IAAI,SAAS,CAAC,MAAM,GAAG,QAAQ,IAAI,WAAW,EAAE,CAAC;oBAC/C,qBAAqB;oBACrB,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,OAAO,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;oBAClF,MAAM,CAAC,IAAI,CAAC;wBACV,KAAK,EAAE,UAAU,EAAE;wBACnB,IAAI;wBACJ,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC;wBACpD,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM;wBACnD,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;wBACvD,YAAY,EAAE,KAAK;wBACnB,WAAW,EAAE,YAAY;wBACzB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC,MAAM;qBAC7C,CAAC,CAAC;oBAEH,oDAAoD;oBACpD,IAAI,YAAY,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;wBAC1D,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC;oBACjE,CAAC;yBAAM,CAAC;wBACN,WAAW,GAAG,IAAI,CAAC;oBACrB,CAAC;oBACD,YAAY,GAAG,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAClD,CAAC;qBAAM,CAAC;oBACN,WAAW,GAAG,SAAS,CAAC;gBAC1B,CAAC;YACH,CAAC;YAED,iBAAiB;YACjB,IAAI,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;gBACvB,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,OAAO,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;gBAClF,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,UAAU,EAAE;oBACnB,IAAI;oBACJ,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC;oBACpD,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM;oBACnD,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;oBACvD,YAAY,EAAE,KAAK;oBACnB,WAAW,EAAE,YAAY;oBACzB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC,MAAM;iBAC7C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,OAAe,EAAE,SAAiB,EAAE,OAAe;IAC3E,MAAM,QAAQ,GAAG,SAAS,GAAG,eAAe,CAAC;IAC7C,MAAM,YAAY,GAAG,OAAO,GAAG,eAAe,CAAC;IAC/C,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAEhE,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,gCAAgC;IAChC,IAAI,cAAc,GAAkB,IAAI,CAAC;IACzC,IAAI,YAAY,GAAkB,IAAI,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,kCAAkC;QAClC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACpD,IAAI,YAAY,EAAE,CAAC;YACjB,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YACjC,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACxC,CAAC;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,WAAW,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAEnE,IAAI,SAAS,CAAC,MAAM,GAAG,QAAQ,IAAI,WAAW,EAAE,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,UAAU,EAAE;gBACnB,IAAI,EAAE,WAAW,CAAC,IAAI,EAAE;gBACxB,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,eAAe,CAAC;gBAC3D,SAAS,EAAE,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM;gBAC1D,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,YAAY;gBAC1B,WAAW,EAAE,YAAY;gBACzB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC,MAAM;aAC7C,CAAC,CAAC;YAEH,IAAI,YAAY,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;gBAC1D,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC;YACjE,CAAC;iBAAM,CAAC;gBACN,WAAW,GAAG,IAAI,CAAC;YACrB,CAAC;YACD,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QACrD,CAAC;aAAM,CAAC;YACN,WAAW,GAAG,SAAS,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,UAAU,EAAE;YACnB,IAAI,EAAE,WAAW,CAAC,IAAI,EAAE;YACxB,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,eAAe,CAAC;YAC3D,SAAS,EAAE,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM;YAC1D,OAAO,EAAE,cAAc;YACvB,YAAY,EAAE,YAAY;YAC1B,WAAW,EAAE,YAAY;YACzB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC,MAAM;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,OAAe,EAAE,SAAiB,EAAE,OAAe;IACvE,MAAM,QAAQ,GAAG,SAAS,GAAG,eAAe,CAAC;IAC7C,MAAM,YAAY,GAAG,OAAO,GAAG,eAAe,CAAC;IAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,YAAY,EAAE,GAAG,CAAC,CAAC;IACpD,MAAM,MAAM,GAAmB,EAAE,CAAC;IAElC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;QACnD,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,gDAAgD;QAChD,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAErD,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,UAAU,EAAE;YACnB,IAAI;YACJ,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC;YACpD,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM;YACnD,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;YAC9C,YAAY,EAAE,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAC1D,WAAW,EAAE,CAAC;YACd,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC;SAClD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,6DAA6D;AAC7D,SAAS,eAAe,CAAC,OAAe;IACtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,QAAQ,GAA+F,EAAE,CAAC;IAEhH,IAAI,cAAc,GAAkB,IAAI,CAAC;IACzC,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,WAAW,GAAa,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAEpD,IAAI,YAAY,EAAE,CAAC;YACjB,wBAAwB;YACxB,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC;gBAC7C,QAAQ,CAAC,IAAI,CAAC;oBACZ,OAAO,EAAE,cAAc;oBACvB,KAAK,EAAE,YAAY;oBACnB,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;oBAC5B,WAAW,EAAE,YAAY;iBAC1B,CAAC,CAAC;YACL,CAAC;YAED,cAAc,GAAG,IAAI,CAAC;YACtB,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACtC,WAAW,GAAG,EAAE,CAAC;YACjB,YAAY,GAAG,MAAM,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;QAED,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,iBAAiB;IAC9C,CAAC;IAED,4BAA4B;IAC5B,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC;QAC7C,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,cAAc;YACvB,KAAK,EAAE,YAAY;YACnB,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YAC5B,WAAW,EAAE,YAAY;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/core/cloak-fetch.d.ts

@@ -0,0 +1,43 @@
+/**
+ * CloakBrowser fetch — enterprise-grade stealth using patched Chromium.
+ *
+ * CloakBrowser patches Chromium at the C++ source level (not JS injection).
+ * Passes reCAPTCHA v3 (0.9), Cloudflare Turnstile, DataDome, 14/14 tests.
+ *
+ * Requires: npm install cloakbrowser playwright-core
+ * Usage: peel(url, { cloaked: true })
+ */
+export interface CloakFetchOptions {
+    url: string;
+    proxy?: string;
+    userAgent?: string;
+    viewportWidth?: number;
+    viewportHeight?: number;
+    waitMs?: number;
+    waitSelector?: string;
+    waitUntil?: string;
+    timeoutMs?: number;
+    screenshot?: boolean;
+    screenshotFullPage?: boolean;
+    actions?: any[];
+    headers?: Record<string, string>;
+    headed?: boolean;
+}
+export interface CloakFetchResult {
+    html: string;
+    url: string;
+    statusCode?: number;
+    contentType?: string;
+    method: 'cloaked';
+    screenshot?: Buffer;
+    challengeDetected?: boolean;
+}
+/**
+ * Check if CloakBrowser is installed.
+ */
+export declare function isCloakBrowserAvailable(): boolean;
+/**
+ * Fetch a URL using CloakBrowser's patched Chromium.
+ */
+export declare function cloakFetch(options: CloakFetchOptions): Promise<CloakFetchResult>;
+//# sourceMappingURL=cloak-fetch.d.ts.map

package/dist/core/cloak-fetch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloak-fetch.d.ts","sourceRoot":"","sources":["../../src/core/cloak-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,SAAS,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAID;;GAEG;AACH,wBAAgB,uBAAuB,IAAI,OAAO,CAOjD;AAqBD;;GAEG;AACH,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAyGtF"}

package/dist/core/cloak-fetch.js

@@ -0,0 +1,141 @@
+/**
+ * CloakBrowser fetch — enterprise-grade stealth using patched Chromium.
+ *
+ * CloakBrowser patches Chromium at the C++ source level (not JS injection).
+ * Passes reCAPTCHA v3 (0.9), Cloudflare Turnstile, DataDome, 14/14 tests.
+ *
+ * Requires: npm install cloakbrowser playwright-core
+ * Usage: peel(url, { cloaked: true })
+ */
+let cloakModule = null;
+/**
+ * Check if CloakBrowser is installed.
+ */
+export function isCloakBrowserAvailable() {
+    try {
+        require.resolve('cloakbrowser');
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Lazy-load CloakBrowser module.
+ */
+async function getCloakBrowser() {
+    if (cloakModule)
+        return cloakModule;
+    try {
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore — cloakbrowser is an optional peer dependency
+        cloakModule = await import('cloakbrowser');
+        return cloakModule;
+    }
+    catch (e) {
+        throw new Error('CloakBrowser not installed. Run: npm install cloakbrowser playwright-core\n' +
+            'CloakBrowser provides enterprise-grade stealth using a patched Chromium binary.\n' +
+            'Learn more: https://github.com/CloakHQ/cloakbrowser');
+    }
+}
+/**
+ * Fetch a URL using CloakBrowser's patched Chromium.
+ */
+export async function cloakFetch(options) {
+    const cloak = await getCloakBrowser();
+    const launchOptions = {
+        headless: !options.headed,
+    };
+    if (options.proxy) {
+        launchOptions.proxy = options.proxy;
+    }
+    let browser = null;
+    let page = null;
+    try {
+        // Use launchContext if available for richer options, otherwise launch
+        if (cloak.launchContext) {
+            const contextOptions = {};
+            if (options.userAgent)
+                contextOptions.userAgent = options.userAgent;
+            if (options.viewportWidth && options.viewportHeight) {
+                contextOptions.viewport = { width: options.viewportWidth, height: options.viewportHeight };
+            }
+            const context = await cloak.launchContext({
+                ...launchOptions,
+                ...contextOptions,
+            });
+            page = await context.newPage();
+            browser = context.browser();
+        }
+        else {
+            browser = await cloak.launch(launchOptions);
+            const context = await browser.newContext({
+                userAgent: options.userAgent,
+                viewport: options.viewportWidth && options.viewportHeight
+                    ? { width: options.viewportWidth, height: options.viewportHeight }
+                    : undefined,
+            });
+            page = await context.newPage();
+        }
+        // Set custom headers if provided
+        if (options.headers && Object.keys(options.headers).length > 0) {
+            await page.setExtraHTTPHeaders(options.headers);
+        }
+        // Navigate
+        const waitUntil = options.waitUntil || 'domcontentloaded';
+        const timeout = options.timeoutMs || 30000;
+        const response = await page.goto(options.url, {
+            waitUntil,
+            timeout,
+        });
+        // Wait for selector if specified
+        if (options.waitSelector) {
+            await page.waitForSelector(options.waitSelector, { timeout }).catch(() => {
+                if (process.env.DEBUG)
+                    console.debug('[webpeel]', `waitSelector "${options.waitSelector}" timed out`);
+            });
+        }
+        // Additional wait
+        if (options.waitMs && options.waitMs > 0) {
+            await page.waitForTimeout(options.waitMs);
+        }
+        // Execute actions if provided
+        if (options.actions && options.actions.length > 0) {
+            const { executeActions } = await import('./actions.js');
+            await executeActions(page, options.actions);
+        }
+        // Get content
+        const html = await page.content();
+        const statusCode = response?.status();
+        const contentType = response?.headers()['content-type'];
+        const finalUrl = page.url();
+        // Screenshot if requested
+        let screenshotBuffer;
+        if (options.screenshot) {
+            screenshotBuffer = await page.screenshot({
+                fullPage: options.screenshotFullPage || false,
+                type: 'png',
+            });
+        }
+        // Check for challenge pages
+        const { detectChallenge } = await import('./challenge-detection.js');
+        const challengeCheck = detectChallenge(html, statusCode ?? 200);
+        return {
+            html,
+            url: finalUrl,
+            statusCode,
+            contentType,
+            method: 'cloaked',
+            screenshot: screenshotBuffer,
+            challengeDetected: challengeCheck.isChallenge && challengeCheck.confidence >= 0.7,
+        };
+    }
+    finally {
+        // Cleanup
+        if (page)
+            await page.close().catch(() => { });
+        if (browser)
+            await browser.close().catch(() => { });
+    }
+}
+//# sourceMappingURL=cloak-fetch.js.map

package/dist/core/cloak-fetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloak-fetch.js","sourceRoot":"","sources":["../../src/core/cloak-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AA+BH,IAAI,WAAW,GAAQ,IAAI,CAAC;AAE5B;;GAEG;AACH,MAAM,UAAU,uBAAuB;IACrC,IAAI,CAAC;QACH,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,eAAe;IAC5B,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IACpC,IAAI,CAAC;QACH,6DAA6D;QAC7D,2DAA2D;QAC3D,WAAW,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;QAC3C,OAAO,WAAW,CAAC;IACrB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,6EAA6E;YAC7E,mFAAmF;YACnF,qDAAqD,CACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B;IACzD,MAAM,KAAK,GAAG,MAAM,eAAe,EAAE,CAAC;IAEtC,MAAM,aAAa,GAAQ;QACzB,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM;KAC1B,CAAC;IAEF,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,aAAa,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IACtC,CAAC;IAED,IAAI,OAAO,GAAmB,IAAI,CAAC;IACnC,IAAI,IAAI,GAAgB,IAAI,CAAC;IAE7B,IAAI,CAAC;QACH,sEAAsE;QACtE,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;YACxB,MAAM,cAAc,GAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,SAAS;gBAAE,cAAc,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;YACpE,IAAI,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;gBACpD,cAAc,CAAC,QAAQ,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC;YAC7F,CAAC;YAED,MAAM,OAAO,GAAmB,MAAM,KAAK,CAAC,aAAa,CAAC;gBACxD,GAAG,aAAa;gBAChB,GAAG,cAAc;aAClB,CAAC,CAAC;YACH,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YAC/B,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;QAC9B,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,MAAM,OAAQ,CAAC,UAAU,CAAC;gBACxC,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,QAAQ,EAAE,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,cAAc;oBACvD,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,cAAc,EAAE;oBAClE,CAAC,CAAC,SAAS;aACd,CAAC,CAAC;YACH,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACjC,CAAC;QAED,iCAAiC;QACjC,IAAI,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAClD,CAAC;QAED,WAAW;QACX,MAAM,SAAS,GAAI,OAAO,CAAC,SAAiB,IAAI,kBAAkB,CAAC;QACnE,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;QAE3C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE;YAC5C,SAAS;YACT,OAAO;SACR,CAAC,CAAC;QAEH,iCAAiC;QACjC,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;gBACvE,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK;oBAAE,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,iBAAiB,OAAO,CAAC,YAAY,aAAa,CAAC,CAAC;YACxG,CAAC,CAAC,CAAC;QACL,CAAC;QAED,kBAAkB;QAClB,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;QAED,8BAA8B;QAC9B,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;YACxD,MAAM,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9C,CAAC;QAED,cAAc;QACd,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,QAAQ,EAAE,MAAM,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,QAAQ,EAAE,OAAO,EAAE,CAAC,cAAc,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE5B,0BAA0B;QAC1B,IAAI,gBAAoC,CAAC;QACzC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,gBAAgB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC;gBACvC,QAAQ,EAAE,OAAO,CAAC,kBAAkB,IAAI,KAAK;gBAC7C,IAAI,EAAE,KAAK;aACZ,CAAC,CAAC;QACL,CAAC;QAED,4BAA4B;QAC5B,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,0BAA0B,CAAC,CAAC;QACrE,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,EAAE,UAAU,IAAI,GAAG,CAAC,CAAC;QAEhE,OAAO;YACL,IAAI;YACJ,GAAG,EAAE,QAAQ;YACb,UAAU;YACV,WAAW;YACX,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,gBAAgB;YAC5B,iBAAiB,EAAE,cAAc,CAAC,WAAW,IAAI,cAAc,CAAC,UAAU,IAAI,GAAG;SAClF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,UAAU;QACV,IAAI,IAAI;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC7C,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrD,CAAC;AACH,CAAC"}

package/dist/core/crawl-checkpoint.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Crawl checkpoint system for resume capability.
+ * Saves progress to a JSON file so interrupted crawls can continue.
+ */
+export interface CrawlCheckpoint {
+    /** Unique crawl job ID (hash of start URL + options) */
+    jobId: string;
+    /** Starting URL */
+    startUrl: string;
+    /** URLs already crawled (with their results) */
+    completed: Map<string, {
+        status: number;
+        contentLength: number;
+        timestamp: number;
+    }>;
+    /** URLs queued but not yet crawled */
+    pending: string[];
+    /** URLs discovered but not yet queued */
+    discovered: string[];
+    /** Crawl options (serialized) */
+    options: Record<string, any>;
+    /** When crawl started */
+    startedAt: number;
+    /** Last checkpoint time */
+    lastCheckpoint: number;
+    /** Total pages target */
+    maxPages: number;
+}
+/**
+ * Generate a deterministic job ID from URL + options.
+ */
+export declare function generateJobId(url: string, options?: Record<string, any>): string;
+/**
+ * Save a checkpoint to disk.
+ */
+export declare function saveCheckpoint(checkpoint: CrawlCheckpoint): void;
+/**
+ * Load a checkpoint from disk.
+ */
+export declare function loadCheckpoint(jobId: string): CrawlCheckpoint | null;
+/**
+ * Delete a checkpoint (crawl completed or abandoned).
+ */
+export declare function deleteCheckpoint(jobId: string): void;
+/**
+ * List all active checkpoints.
+ */
+export declare function listCheckpoints(): Array<{
+    jobId: string;
+    startUrl: string;
+    completed: number;
+    pending: number;
+    lastCheckpoint: number;
+}>;
+//# sourceMappingURL=crawl-checkpoint.d.ts.map

package/dist/core/crawl-checkpoint.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl-checkpoint.d.ts","sourceRoot":"","sources":["../../src/core/crawl-checkpoint.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,MAAM,WAAW,eAAe;IAC9B,wDAAwD;IACxD,KAAK,EAAE,MAAM,CAAC;IACd,mBAAmB;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,gDAAgD;IAChD,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACrF,sCAAsC;IACtC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,yCAAyC;IACzC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,iCAAiC;IACjC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,yBAAyB;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,2BAA2B;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,yBAAyB;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAID;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,GAAG,MAAM,CASpF;AASD;;GAEG;AACH,wBAAgB,cAAc,CAAC,UAAU,EAAE,eAAe,GAAG,IAAI,CAchE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,eAAe,GAAG,IAAI,CAapE;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAOpD;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,KAAK,CAAC;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CA4BD"}

package/dist/core/crawl-checkpoint.js

@@ -0,0 +1,105 @@
+/**
+ * Crawl checkpoint system for resume capability.
+ * Saves progress to a JSON file so interrupted crawls can continue.
+ */
+import { existsSync, readFileSync, writeFileSync, mkdirSync, unlinkSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { createHash } from 'crypto';
+const CHECKPOINT_DIR = join(process.env.HOME || '/tmp', '.webpeel', 'checkpoints');
+/**
+ * Generate a deterministic job ID from URL + options.
+ */
+export function generateJobId(url, options = {}) {
+    const key = JSON.stringify({
+        url,
+        maxPages: options.maxPages,
+        maxDepth: options.maxDepth,
+        includes: options.includes,
+        excludes: options.excludes,
+    });
+    return createHash('sha256').update(key).digest('hex').slice(0, 16);
+}
+/**
+ * Get the checkpoint file path for a job.
+ */
+function getCheckpointPath(jobId) {
+    return join(CHECKPOINT_DIR, `${jobId}.json`);
+}
+/**
+ * Save a checkpoint to disk.
+ */
+export function saveCheckpoint(checkpoint) {
+    try {
+        mkdirSync(CHECKPOINT_DIR, { recursive: true });
+        const data = {
+            ...checkpoint,
+            completed: Object.fromEntries(checkpoint.completed),
+            lastCheckpoint: Date.now(),
+        };
+        writeFileSync(getCheckpointPath(checkpoint.jobId), JSON.stringify(data, null, 2));
+    }
+    catch (e) {
+        if (process.env.DEBUG) {
+            console.debug('[webpeel]', 'Failed to save checkpoint:', e instanceof Error ? e.message : e);
+        }
+    }
+}
+/**
+ * Load a checkpoint from disk.
+ */
+export function loadCheckpoint(jobId) {
+    const path = getCheckpointPath(jobId);
+    if (!existsSync(path))
+        return null;
+    try {
+        const raw = JSON.parse(readFileSync(path, 'utf-8'));
+        return {
+            ...raw,
+            completed: new Map(Object.entries(raw.completed || {})),
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Delete a checkpoint (crawl completed or abandoned).
+ */
+export function deleteCheckpoint(jobId) {
+    const path = getCheckpointPath(jobId);
+    try {
+        if (existsSync(path)) {
+            unlinkSync(path);
+        }
+    }
+    catch { /* ignore */ }
+}
+/**
+ * List all active checkpoints.
+ */
+export function listCheckpoints() {
+    try {
+        if (!existsSync(CHECKPOINT_DIR))
+            return [];
+        const files = readdirSync(CHECKPOINT_DIR).filter((f) => f.endsWith('.json'));
+        return files.map(f => {
+            try {
+                const raw = JSON.parse(readFileSync(join(CHECKPOINT_DIR, f), 'utf-8'));
+                return {
+                    jobId: raw.jobId,
+                    startUrl: raw.startUrl,
+                    completed: Object.keys(raw.completed || {}).length,
+                    pending: (raw.pending || []).length,
+                    lastCheckpoint: raw.lastCheckpoint,
+                };
+            }
+            catch {
+                return null;
+            }
+        }).filter(Boolean);
+    }
+    catch {
+        return [];
+    }
+}
+//# sourceMappingURL=crawl-checkpoint.js.map

package/dist/core/crawl-checkpoint.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl-checkpoint.js","sourceRoot":"","sources":["../../src/core/crawl-checkpoint.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AACjG,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAuBpC,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC;AAEnF;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,UAA+B,EAAE;IAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC;QACzB,GAAG;QACH,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAC,CAAC;IACH,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACrE,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,IAAI,CAAC,cAAc,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC;AAC/C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,UAA2B;IACxD,IAAI,CAAC;QACH,SAAS,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG;YACX,GAAG,UAAU;YACb,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,SAAS,CAAC;YACnD,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE;SAC3B,CAAC;QACF,aAAa,CAAC,iBAAiB,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACpF,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,4BAA4B,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/F,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IACtC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QACpD,OAAO;YACL,GAAG,GAAG;YACN,SAAS,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;SACxD,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IACtC,IAAI,CAAC;QACH,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,UAAU,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAO7B,IAAI,CAAC;QACH,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC;YAAE,OAAO,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAa,WAAW,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAE/F,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YACnB,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;gBACvE,OAAO;oBACL,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM;oBAClD,OAAO,EAAE,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM;oBACnC,cAAc,EAAE,GAAG,CAAC,cAAc;iBACnC,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAMf,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}

package/dist/core/crawler.d.ts

@@ -4,8 +4,10 @@
  */
 import type { PeelOptions } from '../types.js';
 export interface CrawlOptions extends Omit<PeelOptions, 'format'> {
-    /** Maximum number of pages to crawl (default: 10, max: 100) */
+    /** Maximum number of pages to crawl (default: 10, max: tier-dependent) */
     maxPages?: number;
+    /** Tier for determining the max pages cap (default: 'free') */
+    tier?: string;
     /** Maximum depth to crawl (default: 2, max: 5) */
     maxDepth?: number;
     /** Only crawl URLs from these domains (default: same domain as starting URL) */
@@ -28,6 +30,8 @@ export interface CrawlOptions extends Omit<PeelOptions, 'format'> {
     onProgress?: (status: CrawlProgress) => void;
     /** Per-page callback — receives the full result as soon as a page completes */
     onPage?: (result: CrawlResult) => void;
+    /** Resume an interrupted crawl from its last checkpoint */
+    resume?: boolean;
 }
 export interface CrawlProgress {
     crawled: number;

package/dist/core/crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../src/core/crawler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAkB/C,MAAM,WAAW,YAAa,SAAQ,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC;IAC/D,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gFAAgF;IAChF,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,2DAA2D;IAC3D,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,gFAAgF;IAChF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8DAA8D;IAC9D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC;IACzB,kEAAkE;IAClE,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,oDAAoD;IACpD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,+CAA+C;IAC/C,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,IAAI,CAAC;IAC7C,+EAA+E;IAC/E,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,CAAC;CACxC;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,8BAA8B;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,uBAAuB;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,qCAAqC;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,2CAA2C;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAsFD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,KAAK,CACzB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,WAAW,EAAE,CAAC,CA+NxB"}
+{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../src/core/crawler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAkC/C,MAAM,WAAW,YAAa,SAAQ,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC;IAC/D,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+DAA+D;IAC/D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gFAAgF;IAChF,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,2DAA2D;IAC3D,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,gFAAgF;IAChF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8DAA8D;IAC9D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC;IACzB,kEAAkE;IAClE,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,oDAAoD;IACpD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,+CAA+C;IAC/C,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,IAAI,CAAC;IAC7C,+EAA+E;IAC/E,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,CAAC;IACvC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,8BAA8B;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,uBAAuB;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,qCAAqC;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,2CAA2C;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAsFD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,KAAK,CACzB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,WAAW,EAAE,CAAC,CAsRxB"}