vectra 0.7.6 → 0.10.0

package/lib/TextSplitter.js

@@ -1,543 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TextSplitter = void 0;
-const GPT3Tokenizer_1 = require("./GPT3Tokenizer");
-const ALPHANUMERIC_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
-class TextSplitter {
-    constructor(config) {
-        this._config = Object.assign({
-            keepSeparators: false,
-            chunkSize: 400,
-            chunkOverlap: 40,
-        }, config);
-        // Create a default tokenizer if none is provided
-        if (!this._config.tokenizer) {
-            this._config.tokenizer = new GPT3Tokenizer_1.GPT3Tokenizer();
-        }
-        // Use default separators if none are provided
-        if (!this._config.separators || this._config.separators.length === 0) {
-            this._config.separators = this.getSeparators(this._config.docType);
-        }
-        // Validate the config settings
-        if (this._config.chunkSize < 1) {
-            throw new Error("chunkSize must be >= 1");
-        }
-        else if (this._config.chunkOverlap < 0) {
-            throw new Error("chunkOverlap must be >= 0");
-        }
-        else if (this._config.chunkOverlap > this._config.chunkSize) {
-            throw new Error("chunkOverlap must be <= chunkSize");
-        }
-    }
-    split(text) {
-        // Get basic chunks
-        const chunks = this.recursiveSplit(text, this._config.separators, 0);
-        const that = this;
-        function getOverlapTokens(tokens) {
-            if (tokens != undefined) {
-                const len = tokens.length > that._config.chunkOverlap ? that._config.chunkOverlap : tokens.length;
-                return tokens.slice(0, len);
-            }
-            else {
-                return [];
-            }
-        }
-        // Add overlap tokens and text to the start and end of each chunk
-        if (this._config.chunkOverlap > 0) {
-            for (let i = 1; i < chunks.length; i++) {
-                const previousChunk = chunks[i - 1];
-                const chunk = chunks[i];
-                const nextChunk = i < chunks.length - 1 ? chunks[i + 1] : undefined;
-                chunk.startOverlap = getOverlapTokens(previousChunk.tokens.reverse()).reverse();
-                chunk.endOverlap = getOverlapTokens(nextChunk === null || nextChunk === void 0 ? void 0 : nextChunk.tokens);
-            }
-        }
-        return chunks;
-    }
-    recursiveSplit(text, separators, startPos) {
-        const chunks = [];
-        if (text.length > 0) {
-            // Split text into parts
-            let parts;
-            let separator = '';
-            const nextSeparators = separators.length > 1 ? separators.slice(1) : [];
-            if (separators.length > 0) {
-                // Split by separator
-                separator = separators[0];
-                parts = separator == ' ' ? this.splitBySpaces(text) : text.split(separator);
-            }
-            else {
-                // Cut text in half
-                const half = Math.floor(text.length / 2);
-                parts = [text.substring(0, half), text.substring(half)];
-            }
-            // Iterate over parts
-            for (let i = 0; i < parts.length; i++) {
-                const lastChunk = (i === parts.length - 1);
-                // Get chunk text and endPos
-                let chunk = parts[i];
-                const endPos = (startPos + (chunk.length - 1)) + (lastChunk ? 0 : separator.length);
-                if (this._config.keepSeparators && !lastChunk) {
-                    chunk += separator;
-                }
-                // Ensure chunk contains text
-                if (!this.containsAlphanumeric(chunk)) {
-                    continue;
-                }
-                // Optimization to avoid encoding really large chunks
-                if (chunk.length / 6 > this._config.chunkSize) {
-                    // Break the text into smaller chunks
-                    const subChunks = this.recursiveSplit(chunk, nextSeparators, startPos);
-                    chunks.push(...subChunks);
-                }
-                else {
-                    // Encode chunk text
-                    const tokens = this._config.tokenizer.encode(chunk);
-                    if (tokens.length > this._config.chunkSize) {
-                        // Break the text into smaller chunks
-                        const subChunks = this.recursiveSplit(chunk, nextSeparators, startPos);
-                        chunks.push(...subChunks);
-                    }
-                    else {
-                        // Append chunk to output
-                        chunks.push({
-                            text: chunk,
-                            tokens: tokens,
-                            startPos: startPos,
-                            endPos: endPos,
-                            startOverlap: [],
-                            endOverlap: [],
-                        });
-                    }
-                }
-                // Update startPos
-                startPos = endPos + 1;
-            }
-        }
-        return this.combineChunks(chunks);
-    }
-    combineChunks(chunks) {
-        const combinedChunks = [];
-        let currentChunk;
-        let currentLength = 0;
-        const separator = this._config.keepSeparators ? '' : ' ';
-        for (let i = 0; i < chunks.length; i++) {
-            const chunk = chunks[i];
-            if (currentChunk) {
-                const length = currentChunk.tokens.length + chunk.tokens.length;
-                if (length > this._config.chunkSize) {
-                    combinedChunks.push(currentChunk);
-                    currentChunk = chunk;
-                    currentLength = chunk.tokens.length;
-                }
-                else {
-                    currentChunk.text += separator + chunk.text;
-                    currentChunk.endPos = chunk.endPos;
-                    currentChunk.tokens.push(...chunk.tokens);
-                    currentLength += chunk.tokens.length;
-                }
-            }
-            else {
-                currentChunk = chunk;
-                currentLength = chunk.tokens.length;
-            }
-        }
-        if (currentChunk) {
-            combinedChunks.push(currentChunk);
-        }
-        return combinedChunks;
-    }
-    containsAlphanumeric(text) {
-        for (let i = 0; i < text.length; i++) {
-            if (ALPHANUMERIC_CHARS.includes(text[i])) {
-                return true;
-            }
-        }
-        return false;
-    }
-    splitBySpaces(text) {
-        const parts = [];
-        const words = text.split(' ');
-        if (words.length > 0) {
-            let part = words[0];
-            for (let i = 1; i < words.length; i++) {
-                const nextWord = words[i];
-                if (this._config.tokenizer.encode(part + ' ' + nextWord).length <= this._config.chunkSize) {
-                    part += ' ' + nextWord;
-                }
-                else {
-                    parts.push(part);
-                    part = nextWord;
-                }
-            }
-            parts.push(part);
-        }
-        else {
-            parts.push(text);
-        }
-        return parts;
-    }
-    getSeparators(docType) {
-        switch (docType !== null && docType !== void 0 ? docType : '') {
-            case "cpp":
-                return [
-                    // Split along class definitions
-                    "\nclass ",
-                    // Split along function definitions
-                    "\nvoid ",
-                    "\nint ",
-                    "\nfloat ",
-                    "\ndouble ",
-                    // Split along control flow statements
-                    "\nif ",
-                    "\nfor ",
-                    "\nwhile ",
-                    "\nswitch ",
-                    "\ncase ",
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "go":
-                return [
-                    // Split along function definitions
-                    "\nfunc ",
-                    "\nvar ",
-                    "\nconst ",
-                    "\ntype ",
-                    // Split along control flow statements
-                    "\nif ",
-                    "\nfor ",
-                    "\nswitch ",
-                    "\ncase ",
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "java":
-            case "c#":
-            case "csharp":
-            case "cs":
-            case "ts":
-            case "tsx":
-            case "typescript":
-                return [
-                    // split along regions
-                    "// LLM-REGION",
-                    "/* LLM-REGION",
-                    "/** LLM-REGION",
-                    // Split along class definitions
-                    "\nclass ",
-                    // Split along method definitions
-                    "\npublic ",
-                    "\nprotected ",
-                    "\nprivate ",
-                    "\nstatic ",
-                    // Split along control flow statements
-                    "\nif ",
-                    "\nfor ",
-                    "\nwhile ",
-                    "\nswitch ",
-                    "\ncase ",
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "js":
-            case "jsx":
-            case "javascript":
-                return [
-                    // split along regions
-                    "// LLM-REGION",
-                    "/* LLM-REGION",
-                    "/** LLM-REGION",
-                    // Split along class definitions
-                    "\nclass ",
-                    // Split along function definitions
-                    "\nfunction ",
-                    "\nconst ",
-                    "\nlet ",
-                    "\nvar ",
-                    "\nclass ",
-                    // Split along control flow statements
-                    "\nif ",
-                    "\nfor ",
-                    "\nwhile ",
-                    "\nswitch ",
-                    "\ncase ",
-                    "\ndefault ",
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "php":
-                return [
-                    // Split along function definitions
-                    "\nfunction ",
-                    // Split along class definitions
-                    "\nclass ",
-                    // Split along control flow statements
-                    "\nif ",
-                    "\nforeach ",
-                    "\nwhile ",
-                    "\ndo ",
-                    "\nswitch ",
-                    "\ncase ",
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "proto":
-                return [
-                    // Split along message definitions
-                    "\nmessage ",
-                    // Split along service definitions
-                    "\nservice ",
-                    // Split along enum definitions
-                    "\nenum ",
-                    // Split along option definitions
-                    "\noption ",
-                    // Split along import statements
-                    "\nimport ",
-                    // Split along syntax declarations
-                    "\nsyntax ",
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "python":
-            case "py":
-                return [
-                    // First, try to split along class definitions
-                    "\nclass ",
-                    "\ndef ",
-                    "\n\tdef ",
-                    // Now split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "rst":
-                return [
-                    // Split along section titles
-                    "\n===\n",
-                    "\n---\n",
-                    "\n***\n",
-                    // Split along directive markers
-                    "\n.. ",
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "ruby":
-                return [
-                    // Split along method definitions
-                    "\ndef ",
-                    "\nclass ",
-                    // Split along control flow statements
-                    "\nif ",
-                    "\nunless ",
-                    "\nwhile ",
-                    "\nfor ",
-                    "\ndo ",
-                    "\nbegin ",
-                    "\nrescue ",
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "rust":
-                return [
-                    // Split along function definitions
-                    "\nfn ",
-                    "\nconst ",
-                    "\nlet ",
-                    // Split along control flow statements
-                    "\nif ",
-                    "\nwhile ",
-                    "\nfor ",
-                    "\nloop ",
-                    "\nmatch ",
-                    "\nconst ",
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "scala":
-                return [
-                    // Split along class definitions
-                    "\nclass ",
-                    "\nobject ",
-                    // Split along method definitions
-                    "\ndef ",
-                    "\nval ",
-                    "\nvar ",
-                    // Split along control flow statements
-                    "\nif ",
-                    "\nfor ",
-                    "\nwhile ",
-                    "\nmatch ",
-                    "\ncase ",
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "swift":
-                return [
-                    // Split along function definitions
-                    "\nfunc ",
-                    // Split along class definitions
-                    "\nclass ",
-                    "\nstruct ",
-                    "\nenum ",
-                    // Split along control flow statements
-                    "\nif ",
-                    "\nfor ",
-                    "\nwhile ",
-                    "\ndo ",
-                    "\nswitch ",
-                    "\ncase ",
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "md":
-            case "markdown":
-                return [
-                    // First, try to split along Markdown headings (starting with level 2)
-                    "\n## ",
-                    "\n### ",
-                    "\n#### ",
-                    "\n##### ",
-                    "\n###### ",
-                    // Note the alternative syntax for headings (below) is not handled here
-                    // Heading level 2
-                    // ---------------
-                    // End of code block
-                    "```\n\n",
-                    // Horizontal lines
-                    "\n\n***\n\n",
-                    "\n\n---\n\n",
-                    "\n\n___\n\n",
-                    // Note that this splitter doesn't handle horizontal lines defined
-                    // by *three or more* of ***, ---, or ___, but this is not handled
-                    // Github tables
-                    "<table>",
-                    // "<tr>",
-                    // "<td>",
-                    // "<td ",
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "latex":
-                return [
-                    // First, try to split along Latex sections
-                    "\n\\chapter{",
-                    "\n\\section{",
-                    "\n\\subsection{",
-                    "\n\\subsubsection{",
-                    // Now split by environments
-                    "\n\\begin{enumerate}",
-                    "\n\\begin{itemize}",
-                    "\n\\begin{description}",
-                    "\n\\begin{list}",
-                    "\n\\begin{quote}",
-                    "\n\\begin{quotation}",
-                    "\n\\begin{verse}",
-                    "\n\\begin{verbatim}",
-                    // Now split by math environments
-                    "\n\\begin{align}",
-                    "$$",
-                    "$",
-                    // Now split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            case "html":
-                return [
-                    // First, try to split along HTML tags
-                    "<body>",
-                    "<div>",
-                    "<p>",
-                    "<br>",
-                    "<li>",
-                    "<h1>",
-                    "<h2>",
-                    "<h3>",
-                    "<h4>",
-                    "<h5>",
-                    "<h6>",
-                    "<span>",
-                    "<table>",
-                    "<tr>",
-                    "<td>",
-                    "<th>",
-                    "<ul>",
-                    "<ol>",
-                    "<header>",
-                    "<footer>",
-                    "<nav>",
-                    // Head
-                    "<head>",
-                    "<style>",
-                    "<script>",
-                    "<meta>",
-                    "<title>",
-                    // Normal type of lines
-                    " "
-                ];
-            case "sol":
-                return [
-                    // Split along compiler informations definitions
-                    "\npragma ",
-                    "\nusing ",
-                    // Split along contract definitions
-                    "\ncontract ",
-                    "\ninterface ",
-                    "\nlibrary ",
-                    // Split along method definitions
-                    "\nconstructor ",
-                    "\ntype ",
-                    "\nfunction ",
-                    "\nevent ",
-                    "\nmodifier ",
-                    "\nerror ",
-                    "\nstruct ",
-                    "\nenum ",
-                    // Split along control flow statements
-                    "\nif ",
-                    "\nfor ",
-                    "\nwhile ",
-                    "\ndo while ",
-                    "\nassembly ",
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " "
-                ];
-            default:
-                return [
-                    // Split by the normal type of lines
-                    "\n\n",
-                    "\n",
-                    " ",
-                    "",
-                ];
-        }
-    }
-}
-exports.TextSplitter = TextSplitter;
-//# sourceMappingURL=TextSplitter.js.map

package/lib/TextSplitter.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"TextSplitter.js","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":";;;AAAA,mDAAgD;AAGhD,MAAM,kBAAkB,GAAG,gEAAgE,CAAC;AAW5F,MAAa,YAAY;IAGrB,YAAmB,MAAoC;QACnD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACzB,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,EAAE;SACG,EAAE,MAAM,CAAC,CAAC;QAEjC,iDAAiD;QACjD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACzB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,6BAAa,EAAE,CAAC;SAChD;QAED,8CAA8C;QAC9C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAClE,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;SACtE;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC7C;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;SAChD;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC3D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;SACxD;IACL,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAErE,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,SAAS,gBAAgB,CAAC,MAAiB;YACvC,IAAI,MAAM,IAAI,SAAS,EAAE;gBACrB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBAClG,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aAC/B;iBAAM;gBACH,OAAO,EAAE,CAAC;aACb;QACL,CAAC;QAED,iEAAiE;QACjE,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACpE,KAAK,CAAC,YAAY,GAAG,gBAAgB,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;gBAChF,KAAK,CAAC,UAAU,GAAG,gBAAgB,CAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,CAAC,CAAC;aAC1D;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,UAAoB,EAAE,QAAgB;QACvE,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;YACjB,wBAAwB;YACxB,IAAI,KAAe,CAAC;YACpB,IAAI,SAAS,GAAG,EAAE,CAAC;YACnB,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;gBACvB,qBAAqB;gBACrB,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC1B,KAAK,GAAG,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;aAC/E;iBAAM;gBACH,mBAAmB;gBACnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACzC,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;aAC3D;YAED,qBAAqB;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE3C,4BAA4B;gBAC5B,IAAI,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrB,MAAM,MAAM,GAAG,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACpF,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE;oBAC3C,KAAK,IAAI,SAAS,CAAC;iBACtB;gBAED,6BAA6B;gBAC7B,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE;oBACnC,SAAS;iBACZ;gBAED,qDAAqD;gBACrD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBAC3C,qCAAqC;oBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;oBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;iBAC7B;qBAAM;oBACH,oBAAoB;oBACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACpD,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;wBACxC,qCAAqC;wBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;wBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;qBAC7B;yBAAM;wBACH,yBAAyB;wBACzB,MAAM,CAAC,IAAI,CAAC;4BACR,IAAI,EAAE,KAAK;4BACX,MAAM,EAAE,MAAM;4BACd,QAAQ,EAAE,QAAQ;4BAClB,MAAM,EAAE,MAAM;4BACd,YAAY,EAAE,EAAE;4BAChB,UAAU,EAAE,EAAE;yBACjB,CAAC,CAAC;qBACN;iBAEJ;gBAGD,kBAAkB;gBAClB,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;aACzB;SACJ;QAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,MAAmB;QACrC,MAAM,cAAc,GAAgB,EAAE,CAAC;QACvC,IAAI,YAAiC,CAAC;QACtC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,YAAY,EAAE;gBACd,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;gBAChE,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBACjC,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBAClC,YAAY,GAAG,KAAK,CAAC;oBACrB,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;iBACvC;qBAAM;oBACH,YAAY,CAAC,IAAI,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC;oBAC5C,YAAY,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;oBACnC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;oBAC1C,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;iBACxC;aACJ;iBAAM;gBACH,YAAY,GAAG,KAAK,CAAC;gBACrB,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;aACvC;SACJ;QACD,IAAI,YAAY,EAAE;YACd,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;SACrC;QACD,OAAO,cAAc,CAAC;IAC1B,CAAC;IAEO,oBAAoB,CAAC,IAAY;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,IAAI,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;gBACtC,OAAO,IAAI,CAAC;aACf;SACJ;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,aAAa,CAAC,IAAY;QAC9B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YAClB,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,QAAQ,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBACvF,IAAI,IAAI,GAAG,GAAG,QAAQ,CAAC;iBAC1B;qBAAM;oBACH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACjB,IAAI,GAAG,QAAQ,CAAC;iBACnB;aACJ;YACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACpB;aAAM;YACH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACpB;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,aAAa,CAAC,OAAgB;QAClC,QAAQ,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE;YACnB,KAAK,KAAK;gBACN,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,IAAI;gBACL,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM,CAAC;YACZ,KAAK,IAAI,CAAC;YACV,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,sBAAsB;oBACtB,eAAe;oBACf,eAAe;oBACf,gBAAgB;oBAChB,gCAAgC;oBAChC,UAAU;oBACV,iCAAiC;oBACjC,WAAW;oBACX,cAAc;oBACd,YAAY;oBACZ,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,sBAAsB;oBACtB,eAAe;oBACf,eAAe;oBACf,gBAAgB;oBAChB,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,UAAU;oBACV,QAAQ;oBACR,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,YAAY;oBACZ,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,mCAAmC;oBACnC,aAAa;oBACb,gCAAgC;oBAChC,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,YAAY;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,kCAAkC;oBAClC,YAAY;oBACZ,kCAAkC;oBAClC,YAAY;oBACZ,+BAA+B;oBAC/B,SAAS;oBACT,iCAAiC;oBACjC,WAAW;oBACX,gCAAgC;oBAChC,WAAW;oBACX,kCAAkC;oBAClC,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI;gBACL,OAAO;oBACH,8CAA8C;oBAC9C,UAAU;oBACV,QAAQ;oBACR,UAAU;oBACV,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,6BAA6B;oBAC7B,SAAS;oBACT,SAAS;oBACT,SAAS;oBACT,gCAAgC;oBAChC,OAAO;oBACP,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,iCAAiC;oBACjC,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,WAAW;oBACX,UAAU;oBACV,QAAQ;oBACR,OAAO;oBACP,UAAU;oBACV,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,mCAAmC;oBACnC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,iCAAiC;oBACjC,QAAQ;oBACR,QAAQ;oBACR,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,UAAU;oBACV,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,IAAI,CAAC;YACV,KAAK,UAAU;gBACX,OAAO;oBACH,sEAAsE;oBACtE,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,WAAW;oBACX,uEAAuE;oBACvE,kBAAkB;oBAClB,kBAAkB;oBAClB,oBAAoB;oBACpB,SAAS;oBACT,mBAAmB;oBACnB,aAAa;oBACb,aAAa;oBACb,aAAa;oBACb,kEAAkE;oBAClE,kEAAkE;oBAClE,gBAAgB;oBAChB,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,UAAU;oBACV,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,2CAA2C;oBAC3C,cAAc;oBACd,cAAc;oBACd,iBAAiB;oBACjB,oBAAoB;oBAEpB,4BAA4B;oBAC5B,sBAAsB;oBACtB,oBAAoB;oBACpB,wBAAwB;oBACxB,iBAAiB;oBACjB,kBAAkB;oBAClB,sBAAsB;oBACtB,kBAAkB;oBAClB,qBAAqB;oBAErB,iCAAiC;oBACjC,kBAAkB;oBAClB,IAAI;oBACJ,GAAG;oBAEH,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,sCAAsC;oBACtC,QAAQ;oBACR,OAAO;oBACP,KAAK;oBACL,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,QAAQ;oBACR,SAAS;oBACT,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,UAAU;oBACV,UAAU;oBACV,OAAO;oBACP,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,uBAAuB;oBACvB,GAAG;iBACN,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,gDAAgD;oBAChD,WAAW;oBACX,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,cAAc;oBACd,YAAY;oBACZ,iCAAiC;oBACjC,gBAAgB;oBAChB,SAAS;oBACT,aAAa;oBACb,UAAU;oBACV,aAAa;oBACb,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,aAAa;oBACb,aAAa;oBACb,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN;gBACI,OAAO;oBACH,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;SACT;IACL,CAAC;CACJ;AAviBD,oCAuiBC"}

package/lib/WebFetcher.d.ts

@@ -1,15 +0,0 @@
-import { AxiosRequestConfig } from "axios";
-import { TextFetcher } from './types';
-export interface WebFetcherConfig {
-    headers?: Record<string, string>;
-    requestConfig?: AxiosRequestConfig;
-    htmlToMarkdown: boolean;
-    summarizeHtml: boolean;
-}
-export declare class WebFetcher implements TextFetcher {
-    private readonly _config;
-    constructor(config?: Partial<WebFetcherConfig>);
-    fetch(uri: string, onDocument: (uri: string, text: string, docType?: string) => Promise<boolean>): Promise<boolean>;
-    private htmlToMarkdown;
-}
-//# sourceMappingURL=WebFetcher.d.ts.map

package/lib/WebFetcher.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"WebFetcher.d.ts","sourceRoot":"","sources":["../src/WebFetcher.ts"],"names":[],"mappings":"AAAA,OAAc,EAAE,kBAAkB,EAAE,MAAM,OAAO,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AA2BtC,MAAM,WAAW,gBAAgB;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;IAChC,aAAa,CAAC,EAAE,kBAAkB,CAAC;IACnC,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;CAC1B;AAED,qBAAa,UAAW,YAAW,WAAW;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;gBAExB,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC;IAOxC,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC;IAyChI,OAAO,CAAC,cAAc;CAmCzB"}