vectra 0.1.2 → 0.2.0

package/lib/TextSplitter.js

@@ -0,0 +1,460 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TextSplitter = void 0;
+const GPT3Tokenizer_1 = require("./GPT3Tokenizer");
+class TextSplitter {
+    constructor(config) {
+        this._config = Object.assign({
+            separators: ["\n\n", "\n", " ", ""],
+            keepSeparators: false,
+            chunkSize: 400,
+            chunkOverlap: 40,
+        }, config);
+        // Create a default tokenizer if none is provided
+        if (!this._config.tokenizer) {
+            this._config.tokenizer = new GPT3Tokenizer_1.GPT3Tokenizer();
+        }
+        // Use default separators if none are provided
+        if (!this._config.separators || this._config.separators.length === 0) {
+            this._config.separators = this.getSeparators(this._config.docType);
+        }
+        // Validate the config settings
+        if (this._config.chunkSize < 1) {
+            throw new Error("chunkSize must be >= 1");
+        }
+        else if (this._config.chunkOverlap < 0) {
+            throw new Error("chunkOverlap must be >= 0");
+        }
+        else if (this._config.chunkOverlap > this._config.chunkSize) {
+            throw new Error("chunkOverlap must be <= chunkSize");
+        }
+    }
+    split(text) {
+        // Get basic chunks
+        const chunks = this.recursiveSplit(text, this._config.separators, 0);
+        const that = this;
+        function getOverlapTokens(tokens) {
+            if (tokens != undefined) {
+                const len = tokens.length > that._config.chunkOverlap ? that._config.chunkOverlap : tokens.length;
+                return tokens.slice(tokens.length);
+            }
+            else {
+                return [];
+            }
+        }
+        // Add overlap tokens and text to the start and end of each chunk
+        if (this._config.chunkOverlap > 0) {
+            for (let i = 1; i < chunks.length; i++) {
+                const previousChunk = chunks[i - 1];
+                const chunk = chunks[i];
+                const nextChunk = i < chunks.length - 1 ? chunks[i + 1] : undefined;
+                chunk.startOverlap = getOverlapTokens(previousChunk.tokens.reverse()).reverse();
+                chunk.endOverlap = getOverlapTokens(nextChunk === null || nextChunk === void 0 ? void 0 : nextChunk.tokens);
+            }
+        }
+        return chunks;
+    }
+    recursiveSplit(text, separators, startPos) {
+        const chunks = [];
+        if (text.length > 0 && separators.length > 0) {
+            const separator = separators[0];
+            const nextSeparators = separators.length > 1 ? separators.slice(1) : [];
+            const parts = text.split(separator);
+            for (let i = 0; i < parts.length; i++) {
+                const lastChunk = (i === parts.length - 1);
+                // Get chunk text and endPos
+                let chunk = parts[i];
+                const endPos = (startPos + (chunk.length - 1)) + (lastChunk ? 0 : separator.length);
+                if (this._config.keepSeparators && !lastChunk) {
+                    chunk += separator;
+                }
+                // Encode chunk text
+                const tokens = this._config.tokenizer.encode(chunk);
+                if (tokens.length > this._config.chunkSize) {
+                    // Break the text into smaller chunks
+                    const subChunks = this.recursiveSplit(chunk, nextSeparators, startPos);
+                    chunks.push(...subChunks);
+                }
+                else {
+                    // Append chunk to output
+                    chunks.push({
+                        text: chunk,
+                        tokens: tokens,
+                        startPos: startPos,
+                        endPos: endPos,
+                        startOverlap: [],
+                        endOverlap: [],
+                    });
+                }
+                // Update startPos
+                startPos = endPos + 1;
+            }
+        }
+        return chunks;
+    }
+    getSeparators(docType) {
+        switch (docType !== null && docType !== void 0 ? docType : '') {
+            case "cpp":
+                return [
+                    // Split along class definitions
+                    "\nclass ",
+                    // Split along function definitions
+                    "\nvoid ",
+                    "\nint ",
+                    "\nfloat ",
+                    "\ndouble ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\nswitch ",
+                    "\ncase ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "go":
+                return [
+                    // Split along function definitions
+                    "\nfunc ",
+                    "\nvar ",
+                    "\nconst ",
+                    "\ntype ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nswitch ",
+                    "\ncase ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "java":
+            case "c#":
+            case "csharp":
+            case "cs":
+            case "ts":
+            case "tsx":
+            case "typescript":
+                return [
+                    // Split along class definitions
+                    "\nclass ",
+                    // Split along method definitions
+                    "\npublic ",
+                    "\nprotected ",
+                    "\nprivate ",
+                    "\nstatic ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\nswitch ",
+                    "\ncase ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "js":
+            case "jsx":
+            case "javascript":
+                return [
+                    // Split along class definitions
+                    "\nclass ",
+                    // Split along function definitions
+                    "\nfunction ",
+                    "\nconst ",
+                    "\nlet ",
+                    "\nvar ",
+                    "\nclass ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\nswitch ",
+                    "\ncase ",
+                    "\ndefault ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "php":
+                return [
+                    // Split along function definitions
+                    "\nfunction ",
+                    // Split along class definitions
+                    "\nclass ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nforeach ",
+                    "\nwhile ",
+                    "\ndo ",
+                    "\nswitch ",
+                    "\ncase ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "proto":
+                return [
+                    // Split along message definitions
+                    "\nmessage ",
+                    // Split along service definitions
+                    "\nservice ",
+                    // Split along enum definitions
+                    "\nenum ",
+                    // Split along option definitions
+                    "\noption ",
+                    // Split along import statements
+                    "\nimport ",
+                    // Split along syntax declarations
+                    "\nsyntax ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "python":
+            case "py":
+                return [
+                    // First, try to split along class definitions
+                    "\nclass ",
+                    "\ndef ",
+                    "\n\tdef ",
+                    // Now split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "rst":
+                return [
+                    // Split along section titles
+                    "\n===\n",
+                    "\n---\n",
+                    "\n***\n",
+                    // Split along directive markers
+                    "\n.. ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "ruby":
+                return [
+                    // Split along method definitions
+                    "\ndef ",
+                    "\nclass ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nunless ",
+                    "\nwhile ",
+                    "\nfor ",
+                    "\ndo ",
+                    "\nbegin ",
+                    "\nrescue ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "rust":
+                return [
+                    // Split along function definitions
+                    "\nfn ",
+                    "\nconst ",
+                    "\nlet ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nwhile ",
+                    "\nfor ",
+                    "\nloop ",
+                    "\nmatch ",
+                    "\nconst ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "scala":
+                return [
+                    // Split along class definitions
+                    "\nclass ",
+                    "\nobject ",
+                    // Split along method definitions
+                    "\ndef ",
+                    "\nval ",
+                    "\nvar ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\nmatch ",
+                    "\ncase ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "swift":
+                return [
+                    // Split along function definitions
+                    "\nfunc ",
+                    // Split along class definitions
+                    "\nclass ",
+                    "\nstruct ",
+                    "\nenum ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\ndo ",
+                    "\nswitch ",
+                    "\ncase ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "markdown":
+                return [
+                    // First, try to split along Markdown headings (starting with level 2)
+                    "\n## ",
+                    "\n### ",
+                    "\n#### ",
+                    "\n##### ",
+                    "\n###### ",
+                    // Note the alternative syntax for headings (below) is not handled here
+                    // Heading level 2
+                    // ---------------
+                    // End of code block
+                    "```\n\n",
+                    // Horizontal lines
+                    "\n\n***\n\n",
+                    "\n\n---\n\n",
+                    "\n\n___\n\n",
+                    // Note that this splitter doesn't handle horizontal lines defined
+                    // by *three or more* of ***, ---, or ___, but this is not handled
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "latex":
+                return [
+                    // First, try to split along Latex sections
+                    "\n\\chapter{",
+                    "\n\\section{",
+                    "\n\\subsection{",
+                    "\n\\subsubsection{",
+                    // Now split by environments
+                    "\n\\begin{enumerate}",
+                    "\n\\begin{itemize}",
+                    "\n\\begin{description}",
+                    "\n\\begin{list}",
+                    "\n\\begin{quote}",
+                    "\n\\begin{quotation}",
+                    "\n\\begin{verse}",
+                    "\n\\begin{verbatim}",
+                    // Now split by math environments
+                    "\n\\begin{align}",
+                    "$$",
+                    "$",
+                    // Now split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "html":
+                return [
+                    // First, try to split along HTML tags
+                    "<body>",
+                    "<div>",
+                    "<p>",
+                    "<br>",
+                    "<li>",
+                    "<h1>",
+                    "<h2>",
+                    "<h3>",
+                    "<h4>",
+                    "<h5>",
+                    "<h6>",
+                    "<span>",
+                    "<table>",
+                    "<tr>",
+                    "<td>",
+                    "<th>",
+                    "<ul>",
+                    "<ol>",
+                    "<header>",
+                    "<footer>",
+                    "<nav>",
+                    // Head
+                    "<head>",
+                    "<style>",
+                    "<script>",
+                    "<meta>",
+                    "<title>",
+                    // Normal type of lines
+                    " ",
+                    "",
+                ];
+            case "sol":
+                return [
+                    // Split along compiler informations definitions
+                    "\npragma ",
+                    "\nusing ",
+                    // Split along contract definitions
+                    "\ncontract ",
+                    "\ninterface ",
+                    "\nlibrary ",
+                    // Split along method definitions
+                    "\nconstructor ",
+                    "\ntype ",
+                    "\nfunction ",
+                    "\nevent ",
+                    "\nmodifier ",
+                    "\nerror ",
+                    "\nstruct ",
+                    "\nenum ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\ndo while ",
+                    "\nassembly ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            default:
+                return [
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+        }
+    }
+}
+exports.TextSplitter = TextSplitter;
+//# sourceMappingURL=TextSplitter.js.map

package/lib/TextSplitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextSplitter.js","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":";;;AAAA,mDAAgD;AAYhD,MAAa,YAAY;IAGrB,YAAmB,MAAoC;QACnD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACzB,UAAU,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;YACnC,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,EAAE;SACG,EAAE,MAAM,CAAC,CAAC;QAEjC,iDAAiD;QACjD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACzB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,6BAAa,EAAE,CAAC;SAChD;QAED,8CAA8C;QAC9C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAClE,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;SACtE;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC7C;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;SAChD;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC3D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;SACxD;IACL,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAErE,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,SAAS,gBAAgB,CAAC,MAAiB;YACvC,IAAI,MAAM,IAAI,SAAS,EAAE;gBACrB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBAClG,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aACtC;iBAAM;gBACH,OAAO,EAAE,CAAC;aACb;QACL,CAAC;QAED,iEAAiE;QACjE,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACpE,KAAK,CAAC,YAAY,GAAG,gBAAgB,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;gBAChF,KAAK,CAAC,UAAU,GAAG,gBAAgB,CAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,CAAC,CAAC;aAC1D;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,UAAoB,EAAE,QAAgB;QACvE,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1C,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE3C,4BAA4B;gBAC5B,IAAI,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrB,MAAM,MAAM,GAAG,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACpF,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE;oBAC3C,KAAK,IAAI,SAAS,CAAC;iBACtB;gBAED,oBAAoB;gBACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBACxC,qCAAqC;oBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;oBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;iBAC7B;qBAAM;oBACH,yBAAyB;oBACzB,MAAM,CAAC,IAAI,CAAC;wBACR,IAAI,EAAE,KAAK;wBACX,MAAM,EAAE,MAAM;wBACd,QAAQ,EAAE,QAAQ;wBAClB,MAAM,EAAE,MAAM;wBACd,YAAY,EAAE,EAAE;wBAChB,UAAU,EAAE,EAAE;qBACjB,CAAC,CAAC;iBACN;gBAED,kBAAkB;gBAClB,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;aACzB;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,aAAa,CAAC,OAAgB;QAClC,QAAQ,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE;YACnB,KAAK,KAAK;gBACN,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,IAAI;gBACL,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM,CAAC;YACZ,KAAK,IAAI,CAAC;YACV,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,iCAAiC;oBACjC,WAAW;oBACX,cAAc;oBACd,YAAY;oBACZ,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,UAAU;oBACV,QAAQ;oBACR,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,YAAY;oBACZ,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,mCAAmC;oBACnC,aAAa;oBACb,gCAAgC;oBAChC,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,YAAY;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,kCAAkC;oBAClC,YAAY;oBACZ,kCAAkC;oBAClC,YAAY;oBACZ,+BAA+B;oBAC/B,SAAS;oBACT,iCAAiC;oBACjC,WAAW;oBACX,gCAAgC;oBAChC,WAAW;oBACX,kCAAkC;oBAClC,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI;gBACL,OAAO;oBACH,8CAA8C;oBAC9C,UAAU;oBACV,QAAQ;oBACR,UAAU;oBACV,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,6BAA6B;oBAC7B,SAAS;oBACT,SAAS;oBACT,SAAS;oBACT,gCAAgC;oBAChC,OAAO;oBACP,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,iCAAiC;oBACjC,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,WAAW;oBACX,UAAU;oBACV,QAAQ;oBACR,OAAO;oBACP,UAAU;oBACV,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,mCAAmC;oBACnC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,iCAAiC;oBACjC,QAAQ;oBACR,QAAQ;oBACR,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,UAAU;oBACV,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,UAAU;gBACX,OAAO;oBACH,sEAAsE;oBACtE,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,WAAW;oBACX,uEAAuE;oBACvE,kBAAkB;oBAClB,kBAAkB;oBAClB,oBAAoB;oBACpB,SAAS;oBACT,mBAAmB;oBACnB,aAAa;oBACb,aAAa;oBACb,aAAa;oBACb,kEAAkE;oBAClE,kEAAkE;oBAClE,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,2CAA2C;oBAC3C,cAAc;oBACd,cAAc;oBACd,iBAAiB;oBACjB,oBAAoB;oBAEpB,4BAA4B;oBAC5B,sBAAsB;oBACtB,oBAAoB;oBACpB,wBAAwB;oBACxB,iBAAiB;oBACjB,kBAAkB;oBAClB,sBAAsB;oBACtB,kBAAkB;oBAClB,qBAAqB;oBAErB,iCAAiC;oBACjC,kBAAkB;oBAClB,IAAI;oBACJ,GAAG;oBAEH,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,sCAAsC;oBACtC,QAAQ;oBACR,OAAO;oBACP,KAAK;oBACL,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,QAAQ;oBACR,SAAS;oBACT,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,UAAU;oBACV,UAAU;oBACV,OAAO;oBACP,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,uBAAuB;oBACvB,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,gDAAgD;oBAChD,WAAW;oBACX,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,cAAc;oBACd,YAAY;oBACZ,iCAAiC;oBACjC,gBAAgB;oBAChB,SAAS;oBACT,aAAa;oBACb,UAAU;oBACV,aAAa;oBACb,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,aAAa;oBACb,aAAa;oBACb,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN;gBACI,OAAO;oBACH,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;SACT;IACL,CAAC;CACJ;AAndD,oCAmdC"}

package/lib/WebFetcher.d.ts

@@ -0,0 +1,16 @@
+import { AxiosRequestConfig } from "axios";
+import { TextFetcher } from './types';
+export interface WebFetcherConfig {
+    headers?: Record<string, string>;
+    requestConfig?: AxiosRequestConfig;
+    htmlToText: boolean;
+    summarizeHtml: boolean;
+}
+export declare class WebFetcher implements TextFetcher {
+    private readonly _config;
+    constructor(config?: Partial<WebFetcherConfig>);
+    fetch(uri: string): Promise<string>;
+    private extractText;
+    private fetchPage;
+}
+//# sourceMappingURL=WebFetcher.d.ts.map

package/lib/WebFetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebFetcher.d.ts","sourceRoot":"","sources":["../src/WebFetcher.ts"],"names":[],"mappings":"AAAA,OAAc,EAAE,kBAAkB,EAAE,MAAM,OAAO,CAAC;AAElD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AA2BtC,MAAM,WAAW,gBAAgB;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;IAChC,aAAa,CAAC,EAAE,kBAAkB,CAAC;IACnC,UAAU,EAAE,OAAO,CAAC;IACpB,aAAa,EAAE,OAAO,CAAC;CAC1B;AAED,qBAAa,UAAW,YAAW,WAAW;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;gBAExB,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC;IAOxC,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAShD,OAAO,CAAC,WAAW;YAyCL,SAAS;CA+B1B"}

package/lib/WebFetcher.js

@@ -0,0 +1,144 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebFetcher = void 0;
+const axios_1 = __importDefault(require("axios"));
+const cheerio = __importStar(require("cheerio"));
+const ALLOWED_CONTENT_TYPES = [
+    "text/html",
+    "application/json",
+    "application/xml",
+    "application/javascript",
+    "text/plain",
+];
+const DEFAULT_HEADERS = {
+    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+    "Accept-Encoding": "gzip, deflate",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Alt-Used": "LEAVE-THIS-KEY-SET-BY-TOOL",
+    Connection: "keep-alive",
+    Host: "LEAVE-THIS-KEY-SET-BY-TOOL",
+    Referer: "https://www.google.com/",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "cross-site",
+    "Upgrade-Insecure-Requests": "1",
+    "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/111.0",
+};
+class WebFetcher {
+    constructor(config) {
+        this._config = Object.assign({
+            htmlToText: true,
+            summarizeHtml: false,
+        }, config);
+    }
+    fetch(uri) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { data, contentType } = yield this.fetchPage(uri);
+            if (contentType === "text/html" && this._config.htmlToText) {
+                return this.extractText(data, uri, this._config.summarizeHtml);
+            }
+            else {
+                return data;
+            }
+        });
+    }
+    extractText(html, baseUrl, summarize) {
+        // Parse all elements including <noscript> tags
+        const $ = cheerio.load(html, { scriptingEnabled: true });
+        // If we want a summary, just get use the <body/>
+        let text = '';
+        $(`${summarize ? 'body ' : '*'}:not(style):not(script):not(svg)`).each((i, elem) => {
+            var _a, _b;
+            // Remove any children to avoid duplicate text
+            let content = $(elem).clone().children().remove().end().text().trim();
+            const $el = $(elem);
+            // Print links in markdown format
+            let href = $el.attr("href");
+            if (((_a = $el.prop("tagName")) === null || _a === void 0 ? void 0 : _a.toLowerCase()) === "a" && href) {
+                if (!href.startsWith("http")) {
+                    // Try converting to a relevant link
+                    try {
+                        href = new URL(href, baseUrl).toString();
+                    }
+                    catch (_c) {
+                        // Leave as is
+                    }
+                }
+                // If the link has content, use that as the text
+                const altText = (_b = $el.find("img[alt]").attr("alt")) === null || _b === void 0 ? void 0 : _b.trim();
+                if (altText) {
+                    content += ` ${altText}`;
+                }
+                text += ` [${content}](${href})`;
+            }
+            // otherwise just print the content
+            else if (content !== "") {
+                text += ` ${content}`;
+            }
+        });
+        // Remove newlines
+        return text.trim().replace(/\n+/g, ' ');
+    }
+    fetchPage(baseUrl) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const httpClient = axios_1.default.create({
+                validateStatus: () => true,
+            });
+            // Clone headers to avoid mutating the original
+            const headers = Object.assign({}, DEFAULT_HEADERS, this._config.headers);
+            // get hostname from url
+            const host = new URL(baseUrl).hostname;
+            headers['Host'] = host;
+            headers['Alt-Used'] = host;
+            // Fetch page and check for errors
+            const response = yield httpClient.get(baseUrl, Object.assign({ headers }, this._config.requestConfig));
+            if (response.status >= 400) {
+                throw new Error(`Site returned an HTTP status of ${response.status}`);
+            }
+            // Check for valid content type
+            const contentType = response.headers['content-type'];
+            const contentTypeArray = contentType.split(';');
+            if (!contentTypeArray[0] || !ALLOWED_CONTENT_TYPES.includes(contentTypeArray[0])) {
+                throw new Error(`Site returned an invalid content type of ${contentType}`);
+            }
+            return { data: response.data, contentType: contentTypeArray[0] };
+        });
+    }
+}
+exports.WebFetcher = WebFetcher;
+//# sourceMappingURL=WebFetcher.js.map

package/lib/WebFetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebFetcher.js","sourceRoot":"","sources":["../src/WebFetcher.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kDAAkD;AAClD,iDAAmC;AAInC,MAAM,qBAAqB,GAAG;IAC1B,WAAW;IACX,kBAAkB;IAClB,iBAAiB;IACjB,wBAAwB;IACxB,YAAY;CACf,CAAC;AAGF,MAAM,eAAe,GAAG;IACpB,MAAM,EAAE,uFAAuF;IAC/F,iBAAiB,EAAE,eAAe;IAClC,iBAAiB,EAAE,gBAAgB;IACnC,UAAU,EAAE,4BAA4B;IACxC,UAAU,EAAE,YAAY;IACxB,IAAI,EAAE,4BAA4B;IAClC,OAAO,EAAE,yBAAyB;IAClC,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,YAAY;IAC9B,2BAA2B,EAAE,GAAG;IAChC,YAAY,EAAE,gFAAgF;CACjG,CAAC;AASF,MAAa,UAAU;IAGnB,YAAmB,MAAkC;QACjD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACzB,UAAU,EAAE,IAAI;YAChB,aAAa,EAAE,KAAK;SACH,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IAEY,KAAK,CAAC,GAAW;;YAC1B,MAAM,EAAC,IAAI,EAAE,WAAW,EAAC,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACtD,IAAI,WAAW,KAAK,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE;gBACxD,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;aAClE;iBAAM;gBACH,OAAO,IAAI,CAAC;aACf;QACL,CAAC;KAAA;IAEO,WAAW,CAAC,IAAY,EAAE,OAAe,EAAE,SAAkB;QACjE,+CAA+C;QAC/C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzD,iDAAiD;QACjD,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,kCAAkC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAS,EAAE,EAAE;;YACpF,8CAA8C;YAC9C,IAAI,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACtE,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YAEpB,iCAAiC;YACjC,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5B,IAAI,CAAA,MAAA,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,0CAAE,WAAW,EAAE,MAAK,GAAG,IAAI,IAAI,EAAE;gBACpD,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;oBAC1B,oCAAoC;oBACpC,IAAI;wBACA,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;qBAC5C;oBAAC,WAAM;wBACJ,cAAc;qBACjB;iBACJ;gBAED,gDAAgD;gBAChD,MAAM,OAAO,GAAG,MAAA,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,0CAAE,IAAI,EAAE,CAAC;gBACzD,IAAI,OAAO,EAAE;oBACT,OAAO,IAAI,IAAI,OAAO,EAAE,CAAC;iBAC5B;gBAED,IAAI,IAAI,KAAK,OAAO,KAAK,IAAI,GAAG,CAAC;aACpC;YACD,mCAAmC;iBAC9B,IAAI,OAAO,KAAK,EAAE,EAAE;gBACrB,IAAI,IAAI,IAAI,OAAO,EAAE,CAAC;aACzB;QACL,CAAC,CAAC,CAAC;QAEH,kBAAkB;QAClB,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5C,CAAC;IAEa,SAAS,CAAC,OAAe;;YACnC,MAAM,UAAU,GAAG,eAAK,CAAC,MAAM,CAAC;gBAC5B,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI;aAC7B,CAAC,CAAC;YAEH,+CAA+C;YAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YAExE,wBAAwB;YACxB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC;YACvC,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;YACvB,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;YAE3B,kCAAkC;YAClC,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,OAAO,kBACzC,OAAO,IACJ,IAAI,CAAC,OAAO,CAAC,aAAa,EAC/B,CAAC;YACH,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE;gBACxB,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;aACzE;YAED,+BAA+B;YAC/B,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YACrD,MAAM,gBAAgB,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE;gBAC9E,MAAM,IAAI,KAAK,CAAC,4CAA4C,WAAW,EAAE,CAAC,CAAC;aAC9E;YAED,OAAO,EAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC,CAAC,EAAC,CAAC;QACnE,CAAC;KAAA;CACJ;AA3FD,gCA2FC"}

package/lib/index.d.ts

@@ -1,3 +1,11 @@
+export * from './GPT3Tokenizer';
 export * from './ItemSelector';
 export * from './LocalIndex';
+export * from './LocalDocument';
+export * from './LocalDocumentIndex';
+export * from './LocalDocumentResult';
+export * from './OpenAIEmbeddings';
+export * from './TextSplitter';
+export * from './types';
+export * from './WebFetcher';
 //# sourceMappingURL=index.d.ts.map

package/lib/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,iBAAiB,CAAC;AAChC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC"}