vectra 0.12.2 → 0.12.3

package/lib/TextSplitter.js

@@ -0,0 +1,457 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TextSplitter = void 0;
+const GPT3Tokenizer_1 = require("./GPT3Tokenizer");
+const ALPHANUMERIC_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
+class TextSplitter {
+    constructor(config) {
+        this._config = Object.assign({
+            keepSeparators: false,
+            chunkSize: 400,
+            chunkOverlap: 40,
+        }, config);
+        // Create a default tokenizer if none is provided
+        if (!this._config.tokenizer) {
+            this._config.tokenizer = new GPT3Tokenizer_1.GPT3Tokenizer();
+        }
+        // Use default separators if none are provided
+        if (!this._config.separators || this._config.separators.length === 0) {
+            this._config.separators = this.getSeparators(this._config.docType);
+        }
+        // Validate the config settings
+        if (this._config.chunkSize < 1) {
+            throw new Error("chunkSize must be >= 1");
+        }
+        else if (this._config.chunkOverlap < 0) {
+            throw new Error("chunkOverlap must be >= 0");
+        }
+        else if (this._config.chunkOverlap > this._config.chunkSize) {
+            throw new Error("chunkOverlap must be <= chunkSize");
+        }
+    }
+    split(text) {
+        // Get basic chunks
+        const chunks = this.recursiveSplit(text, this._config.separators, 0);
+        const that = this;
+        function getOverlapTokens(tokens) {
+            if (tokens != undefined) {
+                const len = tokens.length > that._config.chunkOverlap ? that._config.chunkOverlap : tokens.length;
+                return tokens.slice(0, len);
+            }
+            else {
+                return [];
+            }
+        }
+        // Add overlap tokens and text to the start and end of each chunk
+        if (this._config.chunkOverlap > 0) {
+            for (let i = 1; i < chunks.length; i++) {
+                const previousChunk = chunks[i - 1];
+                const chunk = chunks[i];
+                const nextChunk = i < chunks.length - 1 ? chunks[i + 1] : undefined;
+                // Use copies to avoid reversing in place (preserve token order in previous chunks)
+                const prevTokensCopy = previousChunk.tokens.slice();
+                chunk.startOverlap = getOverlapTokens(prevTokensCopy.reverse()).reverse();
+                chunk.endOverlap = getOverlapTokens(nextChunk === null || nextChunk === void 0 ? void 0 : nextChunk.tokens);
+            }
+        }
+        return chunks;
+    }
+    recursiveSplit(text, separators, startPos) {
+        const chunks = [];
+        if (text.length > 0) {
+            // Split text into parts
+            let parts;
+            let separator = '';
+            const nextSeparators = separators.length > 1 ? separators.slice(1) : [];
+            if (separators.length > 0) {
+                // Split by separator
+                separator = separators[0];
+                parts = separator == ' ' ? this.splitBySpaces(text) : text.split(separator);
+            }
+            else {
+                // Cut text in half
+                const half = Math.floor(text.length / 2);
+                parts = [text.substring(0, half), text.substring(half)];
+            }
+            // Iterate over parts
+            for (let i = 0; i < parts.length; i++) {
+                const lastChunk = (i === parts.length - 1);
+                // Get chunk text and endPos
+                let chunk = parts[i];
+                const endPos = (startPos + (chunk.length - 1)) + (lastChunk ? 0 : separator.length);
+                if (this._config.keepSeparators && !lastChunk) {
+                    chunk += separator;
+                }
+                // Keep chunks that contain any non-whitespace; drop whitespace-only
+                if (!/\S/.test(chunk)) {
+                    // drop whitespace-only chunks
+                    startPos = endPos + 1;
+                    continue;
+                }
+                // Optimization to avoid encoding really large chunks
+                if (chunk.length / 6 > this._config.chunkSize) {
+                    // Break the text into smaller chunks
+                    const subChunks = this.recursiveSplit(chunk, nextSeparators, startPos);
+                    chunks.push(...subChunks);
+                }
+                else {
+                    // Encode chunk text
+                    const tokens = this._config.tokenizer.encode(chunk);
+                    if (tokens.length > this._config.chunkSize) {
+                        // Break the text into smaller chunks
+                        const subChunks = this.recursiveSplit(chunk, nextSeparators, startPos);
+                        chunks.push(...subChunks);
+                    }
+                    else {
+                        // Append chunk to output
+                        chunks.push({
+                            text: chunk,
+                            tokens: tokens,
+                            startPos: startPos,
+                            endPos: endPos,
+                            startOverlap: [],
+                            endOverlap: [],
+                        });
+                    }
+                }
+                // Update startPos
+                startPos = endPos + 1;
+            }
+        }
+        return this.combineChunks(chunks);
+    }
+    combineChunks(chunks) {
+        const combinedChunks = [];
+        let currentChunk;
+        let currentLength = 0;
+        // When not keeping separators, we previously inserted a space between merged chunks.
+        // We will still use a space for normal merges, but we will prevent merging punctuation-only
+        // separator chunks (e.g., '---', '***', '====') to preserve them as standalone.
+        const separator = this._config.keepSeparators ? '' : ' ';
+        const isWhitespaceOnly = (t) => !/\S/.test(t);
+        const isPunctuationOnly = (t) => /\S/.test(t) && !/[a-zA-Z0-9]/.test(t);
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            if (!currentChunk) {
+                currentChunk = chunk;
+                currentLength = chunk.tokens.length;
+                continue;
+            }
+            // If either the current or next chunk is punctuation-only (non-whitespace, no alphanumeric),
+            // do not merge; keep them as separate chunks to preserve separators like '---'.
+            if (isPunctuationOnly(currentChunk.text) || isPunctuationOnly(chunk.text)) {
+                combinedChunks.push(currentChunk);
+                currentChunk = chunk;
+                currentLength = chunk.tokens.length;
+                continue;
+            }
+            // Normal merge path constrained by token budget
+            const length = currentChunk.tokens.length + chunk.tokens.length;
+            if (length > this._config.chunkSize) {
+                combinedChunks.push(currentChunk);
+                currentChunk = chunk;
+                currentLength = chunk.tokens.length;
+            }
+            else {
+                // Only insert separator if neither chunk is whitespace-only (defensive)
+                const joiner = (!this._config.keepSeparators && !isWhitespaceOnly(currentChunk.text) && !isWhitespaceOnly(chunk.text)) ? separator : '';
+                currentChunk.text += joiner + chunk.text;
+                currentChunk.endPos = chunk.endPos;
+                currentChunk.tokens.push(...chunk.tokens);
+                currentLength += chunk.tokens.length;
+            }
+        }
+        if (currentChunk) {
+            combinedChunks.push(currentChunk);
+        }
+        return combinedChunks;
+    }
+    splitBySpaces(text) {
+        // Split text by tokens and return parts
+        const parts = [];
+        let tokens = this._config.tokenizer.encode(text);
+        do {
+            if (tokens.length <= this._config.chunkSize) {
+                parts.push(this._config.tokenizer.decode(tokens));
+                break;
+            }
+            else {
+                const span = tokens.splice(0, this._config.chunkSize);
+                parts.push(this._config.tokenizer.decode(span));
+            }
+        } while (true);
+        return parts;
+    }
+    getSeparators(docType) {
+        switch (docType !== null && docType !== void 0 ? docType : '') {
+            case "cpp":
+                return [
+                    "\nclass ",
+                    "\nvoid ",
+                    "\nint ",
+                    "\nfloat ",
+                    "\ndouble ",
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\nswitch ",
+                    "\ncase ",
+                    "\n\n",
+                    "\n",
+                ];
+            case "go":
+                return [
+                    "\nfunc ",
+                    "\nvar ",
+                    "\nconst ",
+                    "\ntype ",
+                    "\nif ",
+                    "\nfor ",
+                    "\nswitch ",
+                    "\ncase ",
+                    "\n\n",
+                    "\n",
+                ];
+            case "java":
+            case "c#":
+            case "csharp":
+            case "cs":
+            case "ts":
+            case "tsx":
+            case "typescript":
+                return [
+                    "// LLM-REGION",
+                    "/* LLM-REGION",
+                    "/** LLM-REGION",
+                    "\nclass ",
+                    "\npublic ",
+                    "\nprotected ",
+                    "\nprivate ",
+                    "\nstatic ",
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\nswitch ",
+                    "\ncase ",
+                    "\n\n",
+                    "\n",
+                    " "
+                ];
+            case "js":
+            case "jsx":
+            case "javascript":
+                return [
+                    "// LLM-REGION",
+                    "/* LLM-REGION",
+                    "/** LLM-REGION",
+                    "\nclass ",
+                    "\nfunction ",
+                    "\nconst ",
+                    "\nlet ",
+                    "\nvar ",
+                    "\nclass ",
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\nswitch ",
+                    "\ncase ",
+                    "\ndefault ",
+                    "\n\n",
+                    "\n",
+                ];
+            case "php":
+                return [
+                    "\nfunction ",
+                    "\nclass ",
+                    "\nif ",
+                    "\nforeach ",
+                    "\nwhile ",
+                    "\ndo ",
+                    "\nswitch ",
+                    "\ncase ",
+                    "\n\n",
+                    "\n",
+                ];
+            case "proto":
+                return [
+                    "\nmessage ",
+                    "\nservice ",
+                    "\nenum ",
+                    "\noption ",
+                    "\nimport ",
+                    "\nsyntax ",
+                    "\n\n",
+                    "\n",
+                ];
+            case "python":
+            case "py":
+                return [
+                    "\nclass ",
+                    "\ndef ",
+                    "\n\tdef ",
+                    "\n\n",
+                    "\n",
+                ];
+            case "rst":
+                return [
+                    "\n===\n",
+                    "\n---\n",
+                    "\n***\n",
+                    "\n.. ",
+                    "\n\n",
+                    "\n",
+                ];
+            case "ruby":
+                return [
+                    "\ndef ",
+                    "\nclass ",
+                    "\nif ",
+                    "\nunless ",
+                    "\nwhile ",
+                    "\nfor ",
+                    "\ndo ",
+                    "\nbegin ",
+                    "\nrescue ",
+                    "\n\n",
+                    "\n",
+                ];
+            case "rust":
+                return [
+                    "\nfn ",
+                    "\nconst ",
+                    "\nlet ",
+                    "\nif ",
+                    "\nwhile ",
+                    "\nfor ",
+                    "\nloop ",
+                    "\nmatch ",
+                    "\nconst ",
+                    "\n\n",
+                    "\n",
+                ];
+            case "scala":
+                return [
+                    "\nclass ",
+                    "\nobject ",
+                    "\ndef ",
+                    "\nval ",
+                    "\nvar ",
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\nmatch ",
+                    "\ncase ",
+                    "\n\n",
+                    "\n",
+                ];
+            case "swift":
+                return [
+                    "\nfunc ",
+                    "\nclass ",
+                    "\nstruct ",
+                    "\nenum ",
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\ndo ",
+                    "\nswitch ",
+                    "\ncase ",
+                    "\n\n",
+                    "\n",
+                ];
+            case "md":
+            case "markdown":
+                return [
+                    "\n## ",
+                    "\n### ",
+                    "\n#### ",
+                    "\n##### ",
+                    "\n###### ",
+                    "```\n\n",
+                    "\n\n***\n\n",
+                    "\n\n---\n\n",
+                    "\n\n___\n\n",
+                    "<table>",
+                    "\n\n",
+                    "\n",
+                ];
+            case "latex":
+                return [
+                    "\n\\chapter{",
+                    "\n\\section{",
+                    "\n\\subsection{",
+                    "\n\\subsubsection{",
+                    "\n\\begin{enumerate}",
+                    "\n\\begin{itemize}",
+                    "\n\\begin{description}",
+                    "\n\\begin{list}",
+                    "\n\\begin{quote}",
+                    "\n\\begin{quotation}",
+                    "\n\\begin{verse}",
+                    "\n\\begin{verbatim}",
+                    "\n\\begin{align}",
+                    "\n\n",
+                    "\n",
+                ];
+            case "html":
+                return [
+                    "<body>",
+                    "<div>",
+                    "<p>",
+                    "<br>",
+                    "<li>",
+                    "<h1>",
+                    "<h2>",
+                    "<h3>",
+                    "<h4>",
+                    "<h5>",
+                    "<h6>",
+                    "<span>",
+                    "<table>",
+                    "<tr>",
+                    "<td>",
+                    "<th>",
+                    "<ul>",
+                    "<ol>",
+                    "<header>",
+                    "<footer>",
+                    "<nav>",
+                    "<head>",
+                    "<style>",
+                    "<script>",
+                    "<meta>",
+                    "<title>",
+                ];
+            case "sol":
+                return [
+                    "\npragma ",
+                    "\nusing ",
+                    "\ncontract ",
+                    "\ninterface ",
+                    "\nlibrary ",
+                    "\nconstructor ",
+                    "\ntype ",
+                    "\nfunction ",
+                    "\nevent ",
+                    "\nmodifier ",
+                    "\nerror ",
+                    "\nstruct ",
+                    "\nenum ",
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\ndo while ",
+                    "\nassembly ",
+                    "\n\n",
+                    "\n",
+                ];
+            default:
+                return [
+                    "\n\n",
+                    "\n",
+                ];
+        }
+    }
+}
+exports.TextSplitter = TextSplitter;
+//# sourceMappingURL=TextSplitter.js.map

package/lib/TextSplitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextSplitter.js","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":";;;AAAA,mDAAgD;AAGhD,MAAM,kBAAkB,GAAG,gEAAgE,CAAC;AAW5F,MAAa,YAAY;IAGvB,YAAmB,MAAoC;QACrD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YAC3B,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,EAAE;SACK,EAAE,MAAM,CAAC,CAAC;QAEjC,iDAAiD;QACjD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,6BAAa,EAAE,CAAC;QAC/C,CAAC;QAED,8CAA8C;QAC9C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrE,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACrE,CAAC;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,IAAY;QACvB,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAErE,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,SAAS,gBAAgB,CAAC,MAAiB;YACzC,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;gBACxB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBAClG,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,iEAAiE;QACjE,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAEpE,mFAAmF;gBACnF,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACpD,KAAK,CAAC,YAAY,GAAG,gBAAgB,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;gBAC1E,KAAK,CAAC,UAAU,GAAG,gBAAgB,CAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,UAAoB,EAAE,QAAgB;QACzE,MAAM,MAAM,GAAgB,EAAE,CAAC;QAE/B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,wBAAwB;YACxB,IAAI,KAAe,CAAC;YACpB,IAAI,SAAS,GAAG,EAAE,CAAC;YACnB,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAExE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,qBAAqB;gBACrB,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC1B,KAAK,GAAG,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC9E,CAAC;iBAAM,CAAC;gBACN,mBAAmB;gBACnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACzC,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1D,CAAC;YAED,qBAAqB;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE3C,4BAA4B;gBAC5B,IAAI,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrB,MAAM,MAAM,GAAG,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBAEpF,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE,CAAC;oBAC9C,KAAK,IAAI,SAAS,CAAC;gBACrB,CAAC;gBAED,oEAAoE;gBACpE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;oBACtB,8BAA8B;oBAC9B,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;oBACtB,SAAS;gBACX,CAAC;gBAED,qDAAqD;gBACrD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;oBAC9C,qCAAqC;oBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;oBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC5B,CAAC;qBAAM,CAAC;oBACN,oBAAoB;oBACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACpD,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;wBAC3C,qCAAqC;wBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;wBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;oBAC5B,CAAC;yBAAM,CAAC;wBACN,yBAAyB;wBACzB,MAAM,CAAC,IAAI,CAAC;4BACV,IAAI,EAAE,KAAK;4BACX,MAAM,EAAE,MAAM;4BACd,QAAQ,EAAE,QAAQ;4BAClB,MAAM,EAAE,MAAM;4BACd,YAAY,EAAE,EAAE;4BAChB,UAAU,EAAE,EAAE;yBACf,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,kBAAkB;gBAClB,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEO,aAAa,CAAC,MAAmB;QACvC,MAAM,cAAc,GAAgB,EAAE,CAAC;QACvC,IAAI,YAAmC,CAAC;QACxC,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,qFAAqF;QACrF,4FAA4F;QAC5F,gFAAgF;QAChF,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAEzD,MAAM,gBAAgB,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,iBAAiB,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAEhF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAExB,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,YAAY,GAAG,KAAK,CAAC;gBACrB,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;gBACpC,SAAS;YACX,CAAC;YAED,6FAA6F;YAC7F,gFAAgF;YAChF,IAAI,iBAAiB,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC1E,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAClC,YAAY,GAAG,KAAK,CAAC;gBACrB,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;gBACpC,SAAS;YACX,CAAC;YAED,gDAAgD;YAChD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YAChE,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;gBACpC,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAClC,YAAY,GAAG,KAAK,CAAC;gBACrB,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACN,wEAAwE;gBACxE,MAAM,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxI,YAAY,CAAC,IAAI,IAAI,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC;gBACzC,YAAY,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;gBACnC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC1C,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YACvC,CAAC;QACH,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YACjB,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACpC,CAAC;QAED,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,aAAa,CAAC,IAAY;QAChC,wCAAwC;QACxC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEjD,GAAG,CAAC;YACF,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC5C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;gBAClD,MAAM;YACR,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBACtD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YAClD,CAAC;QACH,CAAC,QAAQ,IAAI,EAAE;QAEf,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,aAAa,CAAC,OAAgB;QACpC,QAAQ,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE,CAAC;YACtB,KAAK,KAAK;gBACR,OAAO;oBACL,UAAU;oBACV,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,IAAI;gBACP,OAAO;oBACL,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,SAAS;oBACT,OAAO;oBACP,QAAQ;oBACR,WAAW;oBACX,SAAS;oBACT,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,MAAM,CAAC;YACZ,KAAK,IAAI,CAAC;YACV,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACf,OAAO;oBACL,eAAe;oBACf,eAAe;oBACf,gBAAgB;oBAChB,UAAU;oBACV,WAAW;oBACX,cAAc;oBACd,YAAY;oBACZ,WAAW;oBACX,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACJ,CAAC;YACJ,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACf,OAAO;oBACL,eAAe;oBACf,eAAe;oBACf,gBAAgB;oBAChB,UAAU;oBACV,aAAa;oBACb,UAAU;oBACV,QAAQ;oBACR,QAAQ;oBACR,UAAU;oBACV,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,YAAY;oBACZ,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,KAAK;gBACR,OAAO;oBACL,aAAa;oBACb,UAAU;oBACV,OAAO;oBACP,YAAY;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,OAAO;gBACV,OAAO;oBACL,YAAY;oBACZ,YAAY;oBACZ,SAAS;oBACT,WAAW;oBACX,WAAW;oBACX,WAAW;oBACX,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI;gBACP,OAAO;oBACL,UAAU;oBACV,QAAQ;oBACR,UAAU;oBACV,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,KAAK;gBACR,OAAO;oBACL,SAAS;oBACT,SAAS;oBACT,SAAS;oBACT,OAAO;oBACP,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,MAAM;gBACT,OAAO;oBACL,QAAQ;oBACR,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,UAAU;oBACV,QAAQ;oBACR,OAAO;oBACP,UAAU;oBACV,WAAW;oBACX,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,MAAM;gBACT,OAAO;oBACL,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,OAAO;gBACV,OAAO;oBACL,UAAU;oBACV,WAAW;oBACX,QAAQ;oBACR,QAAQ;oBACR,QAAQ;oBACR,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,UAAU;oBACV,SAAS;oBACT,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,OAAO;gBACV,OAAO;oBACL,SAAS;oBACT,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,IAAI,CAAC;YACV,KAAK,UAAU;gBACb,OAAO;oBACL,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,aAAa;oBACb,aAAa;oBACb,aAAa;oBACb,SAAS;oBACT,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,OAAO;gBACV,OAAO;oBACL,cAAc;oBACd,cAAc;oBACd,iBAAiB;oBACjB,oBAAoB;oBACpB,sBAAsB;oBACtB,oBAAoB;oBACpB,wBAAwB;oBACxB,iBAAiB;oBACjB,kBAAkB;oBAClB,sBAAsB;oBACtB,kBAAkB;oBAClB,qBAAqB;oBACrB,kBAAkB;oBAClB,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ,KAAK,MAAM;gBACT,OAAO;oBACL,QAAQ;oBACR,OAAO;oBACP,KAAK;oBACL,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,QAAQ;oBACR,SAAS;oBACT,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,UAAU;oBACV,UAAU;oBACV,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,QAAQ;oBACR,SAAS;iBACV,CAAC;YACJ,KAAK,KAAK;gBACR,OAAO;oBACL,WAAW;oBACX,UAAU;oBACV,aAAa;oBACb,cAAc;oBACd,YAAY;oBACZ,gBAAgB;oBAChB,SAAS;oBACT,aAAa;oBACb,UAAU;oBACV,aAAa;oBACb,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,aAAa;oBACb,aAAa;oBACb,MAAM;oBACN,IAAI;iBACL,CAAC;YACJ;gBACE,OAAO;oBACL,MAAM;oBACN,IAAI;iBACL,CAAC;QACN,CAAC;IACH,CAAC;CACF;AA1dD,oCA0dC"}

package/lib/TextSplitter.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=TextSplitter.spec.d.ts.map

package/lib/TextSplitter.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextSplitter.spec.d.ts","sourceRoot":"","sources":["../src/TextSplitter.spec.ts"],"names":[],"mappings":""}

package/lib/TextSplitter.spec.js

@@ -0,0 +1,109 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const mocha_1 = require("mocha");
+const assert = __importStar(require("node:assert"));
+const TextSplitter_1 = require("./TextSplitter");
+(0, mocha_1.describe)('TextSplitter', () => {
+    const makeSplitter = (opts) => new TextSplitter_1.TextSplitter(Object.assign({ chunkSize: 16, chunkOverlap: 0 }, opts));
+    (0, mocha_1.it)('keeps a leading punctuation-only chunk ("---")', () => {
+        const splitter = makeSplitter({ chunkSize: 3, chunkOverlap: 0 });
+        const chunks = splitter.split('---');
+        assert.deepStrictEqual(chunks.map(c => c.text), ['---']);
+    });
+    (0, mocha_1.it)('keeps punctuation-only separators (---, ***, ====) at start, middle, and end', () => {
+        const splitter = makeSplitter({ chunkSize: 4, chunkOverlap: 0 });
+        const text = ['---', 'Hello world', '***', 'Middle', '===='].join('\n');
+        const chunks = splitter.split(text);
+        assert.ok(chunks.some(c => c.text.includes('---')));
+        assert.ok(chunks.some(c => c.text.includes('***')));
+        assert.ok(chunks.some(c => c.text.includes('====')));
+    });
+    (0, mocha_1.it)('preserves frontmatter delimiters when chunk size is small and overlap is zero', () => {
+        var _a;
+        const splitter = makeSplitter({ chunkSize: 12, chunkOverlap: 0 });
+        const md = [
+            '---',
+            'title: Test',
+            'tags: [a, b]',
+            '---',
+            '# Heading',
+            'Body text goes here.'
+        ].join('\n');
+        const chunks = splitter.split(md);
+        const joined = chunks.map(c => c.text).join('\n');
+        const delimiterCount = ((_a = joined.match(/^---$/gm)) !== null && _a !== void 0 ? _a : []).length;
+        assert.strictEqual(delimiterCount, 2);
+    });
+    (0, mocha_1.it)('keeps trailing punctuation-only chunk', () => {
+        const splitter = makeSplitter({ chunkSize: 4, chunkOverlap: 0 });
+        const chunks = splitter.split('Content\n---');
+        assert.ok(chunks.some(c => c.text.includes('---')));
+    });
+    (0, mocha_1.it)('drops pure whitespace-only chunks', () => {
+        const splitter = makeSplitter({ chunkSize: 10, chunkOverlap: 0 });
+        const chunks1 = splitter.split('   \t  ');
+        const chunks2 = splitter.split('\n\n');
+        const chunks3 = splitter.split(' \n \n  ');
+        assert.strictEqual(chunks1.length, 0);
+        assert.strictEqual(chunks2.length, 0);
+        assert.strictEqual(chunks3.length, 0);
+    });
+    (0, mocha_1.it)('still returns alphanumeric chunks normally', () => {
+        const splitter = makeSplitter({ chunkSize: 5, chunkOverlap: 0 });
+        const chunks = splitter.split('abcde fghij');
+        assert.ok(chunks.length > 0);
+        assert.ok(chunks.map(c => c.text).join(' ').includes('abcde'));
+        assert.ok(chunks.map(c => c.text).join(' ').includes('fghij'));
+    });
+    (0, mocha_1.it)('does not regress with non-zero overlap', () => {
+        const splitter = makeSplitter({ chunkSize: 5, chunkOverlap: 2 });
+        const chunks = splitter.split('---\nabcdef');
+        assert.ok(chunks.some(c => c.text.includes('---')));
+    });
+    (0, mocha_1.it)('handles multiple punctuation-only separators interleaved with content', () => {
+        const splitter = makeSplitter({ chunkSize: 8, chunkOverlap: 0 });
+        const text = ['***', 'A', '---', 'B', '====', 'C'].join('\n');
+        const chunks = splitter.split(text);
+        assert.ok(chunks.some(c => c.text.includes('***')));
+        assert.ok(chunks.some(c => c.text.includes('---')));
+        assert.ok(chunks.some(c => c.text.includes('====')));
+        const joined = chunks.map(c => c.text).join('\n');
+        assert.ok(joined.includes('\nA\n'));
+        assert.ok(joined.includes('\nB\n'));
+        assert.ok(joined.includes('\nC'));
+    });
+});
+//# sourceMappingURL=TextSplitter.spec.js.map

package/lib/TextSplitter.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextSplitter.spec.js","sourceRoot":"","sources":["../src/TextSplitter.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iCAAqC;AACrC,oDAAsC;AACtC,iDAA8C;AAE9C,IAAA,gBAAQ,EAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,MAAM,YAAY,GAAG,CAAC,IAA6D,EAAE,EAAE,CACrF,IAAI,2BAAY,iBAAG,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,IAAK,IAAI,EAAG,CAAC;IAEhE,IAAA,UAAE,EAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,8EAA8E,EAAE,GAAG,EAAE;QACtF,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QACjE,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxE,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEpC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,+EAA+E,EAAE,GAAG,EAAE;;QACvF,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAClE,MAAM,EAAE,GAAG;YACT,KAAK;YACL,aAAa;YACb,cAAc;YACd,KAAK;YACL,WAAW;YACX,sBAAsB;SACvB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAElD,MAAM,cAAc,GAAG,CAAC,MAAA,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,mCAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QAC9D,MAAM,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC9C,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC3C,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC7C,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC7C,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,uEAAuE,EAAE,GAAG,EAAE;QAC/E,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QACjE,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEpC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAErD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/lib/WebFetcher.d.ts

@@ -0,0 +1,15 @@
+import { AxiosRequestConfig } from "axios";
+import { TextFetcher } from './types';
+export interface WebFetcherConfig {
+    headers?: Record<string, string>;
+    requestConfig?: AxiosRequestConfig;
+    htmlToMarkdown: boolean;
+    summarizeHtml: boolean;
+}
+export declare class WebFetcher implements TextFetcher {
+    private readonly _config;
+    constructor(config?: Partial<WebFetcherConfig>);
+    fetch(uri: string, onDocument: (uri: string, text: string, docType?: string) => Promise<boolean>): Promise<boolean>;
+    private htmlToMarkdown;
+}
+//# sourceMappingURL=WebFetcher.d.ts.map

package/lib/WebFetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebFetcher.d.ts","sourceRoot":"","sources":["../src/WebFetcher.ts"],"names":[],"mappings":"AAAA,OAAc,EAAE,kBAAkB,EAAE,MAAM,OAAO,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AA2BtC,MAAM,WAAW,gBAAgB;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;IAChC,aAAa,CAAC,EAAE,kBAAkB,CAAC;IACnC,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;CAC1B;AAED,qBAAa,UAAW,YAAW,WAAW;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;gBAExB,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC;IAOxC,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC;IAyChI,OAAO,CAAC,cAAc;CAmCzB"}