vectra 0.1.1 → 0.2.0

package/lib/OpenAIEmbeddings.js

@@ -0,0 +1,139 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAIEmbeddings = void 0;
+const axios_1 = __importDefault(require("axios"));
+/**
+ * A `PromptCompletionModel` for calling OpenAI and Azure OpenAI hosted models.
+ * @remarks
+ */
+class OpenAIEmbeddings {
+    /**
+     * Creates a new `OpenAIClient` instance.
+     * @param options Options for configuring an `OpenAIClient`.
+     */
+    constructor(options) {
+        this.UserAgent = 'AlphaWave';
+        // Check for azure config
+        if (options.azureApiKey) {
+            this._useAzure = true;
+            this.options = Object.assign({
+                retryPolicy: [2000, 5000],
+                azureApiVersion: '2023-05-15',
+            }, options);
+            // Cleanup and validate endpoint
+            let endpoint = this.options.azureEndpoint.trim();
+            if (endpoint.endsWith('/')) {
+                endpoint = endpoint.substring(0, endpoint.length - 1);
+            }
+            if (!endpoint.toLowerCase().startsWith('https://')) {
+                throw new Error(`Client created with an invalid endpoint of '${endpoint}'. The endpoint must be a valid HTTPS url.`);
+            }
+            this.options.azureEndpoint = endpoint;
+        }
+        else {
+            this._useAzure = false;
+            this.options = Object.assign({
+                retryPolicy: [2000, 5000]
+            }, options);
+        }
+        // Create client
+        this._httpClient = axios_1.default.create({
+            validateStatus: (status) => status < 400 || status == 429
+        });
+    }
+    /**
+     * Creates embeddings for the given inputs using the OpenAI API.
+     * @param model Name of the model to use (or deployment for Azure).
+     * @param inputs Text inputs to create embeddings for.
+     * @returns A `EmbeddingsResponse` with a status and the generated embeddings or a message when an error occurs.
+     */
+    createEmbeddings(inputs) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this.createEmbeddingRequest({
+                input: inputs,
+            });
+            // Process response
+            if (response.status < 300) {
+                return { status: 'success', output: response.data.data.sort((a, b) => a.index - b.index).map((item) => item.embedding) };
+            }
+            else if (response.status == 429) {
+                return { status: 'rate_limited', message: `The embeddings API returned a rate limit error.` };
+            }
+            else {
+                return { status: 'error', message: `The embeddings API returned an error status of ${response.status}: ${response.statusText}` };
+            }
+        });
+    }
+    /**
+     * @private
+     */
+    createEmbeddingRequest(request) {
+        var _a;
+        if (this._useAzure) {
+            const options = this.options;
+            const url = `${options.azureEndpoint}/openai/deployments/${options.azureDeployment}/embeddings?api-version=${options.azureApiVersion}`;
+            return this.post(url, request);
+        }
+        else {
+            const options = this.options;
+            const url = `${(_a = options.endpoint) !== null && _a !== void 0 ? _a : 'https://api.openai.com'}/v1/embeddings`;
+            request.model = options.model;
+            return this.post(url, request);
+        }
+    }
+    /**
+     * @private
+     */
+    post(url, body, retryCount = 0) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Initialize request config
+            const requestConfig = Object.assign({}, this.options.requestConfig);
+            // Initialize request headers
+            if (!requestConfig.headers) {
+                requestConfig.headers = {};
+            }
+            if (!requestConfig.headers['Content-Type']) {
+                requestConfig.headers['Content-Type'] = 'application/json';
+            }
+            if (!requestConfig.headers['User-Agent']) {
+                requestConfig.headers['User-Agent'] = this.UserAgent;
+            }
+            if (this._useAzure) {
+                const options = this.options;
+                requestConfig.headers['api-key'] = options.azureApiKey;
+            }
+            else {
+                const options = this.options;
+                requestConfig.headers['Authorization'] = `Bearer ${options.apiKey}`;
+                if (options.organization) {
+                    requestConfig.headers['OpenAI-Organization'] = options.organization;
+                }
+            }
+            // Send request
+            const response = yield this._httpClient.post(url, body, requestConfig);
+            // Check for rate limit error
+            if (response.status == 429 && Array.isArray(this.options.retryPolicy) && retryCount < this.options.retryPolicy.length) {
+                const delay = this.options.retryPolicy[retryCount];
+                yield new Promise((resolve) => setTimeout(resolve, delay));
+                return this.post(url, body, retryCount + 1);
+            }
+            else {
+                return response;
+            }
+        });
+    }
+}
+exports.OpenAIEmbeddings = OpenAIEmbeddings;
+//# sourceMappingURL=OpenAIEmbeddings.js.map

package/lib/OpenAIEmbeddings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"OpenAIEmbeddings.js","sourceRoot":"","sources":["../src/OpenAIEmbeddings.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,kDAAgF;AA2EhF;;;GAGG;AACH,MAAa,gBAAgB;IAWzB;;;OAGG;IACH,YAAmB,OAA6D;QAX/D,cAAS,GAAG,WAAW,CAAC;QAYrC,yBAAyB;QACzB,IAAK,OAAwC,CAAC,WAAW,EAAE;YACvD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACzB,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;gBACzB,eAAe,EAAE,YAAY;aAChC,EAAE,OAAO,CAAiC,CAAC;YAE5C,gCAAgC;YAChC,IAAI,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACjD,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACxB,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;aACzD;YAED,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CAAC,+CAA+C,QAAQ,4CAA4C,CAAC,CAAC;aACxH;YAED,IAAI,CAAC,OAAO,CAAC,aAAa,GAAG,QAAQ,CAAC;SACzC;aAAM;YACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;YACvB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACzB,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;aAC5B,EAAE,OAAO,CAA4B,CAAC;SAC1C;QAED,gBAAgB;QAChB,IAAI,CAAC,WAAW,GAAG,eAAK,CAAC,MAAM,CAAC;YAC5B,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG;SAC5D,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACU,gBAAgB,CAAC,MAAyB;;YACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC;gBAC/C,KAAK,EAAE,MAAM;aAChB,CAAC,CAAC;YAEH,mBAAmB;YACnB,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE;gBACvB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;aAC5H;iBAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE;gBAC/B,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,iDAAiD,EAAE,CAAA;aAChG;iBAAM;gBACH,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,kDAAkD,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC;aACpI;QACL,CAAC;KAAA;IAED;;OAEG;IACO,sBAAsB,CAAC,OAA+B;;QAC5D,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAuC,CAAC;YAC7D,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,aAAa,uBAAuB,OAAO,CAAC,eAAe,2BAA2B,OAAO,CAAC,eAAgB,EAAE,CAAC;YACxI,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAClC;aAAM;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAkC,CAAC;YACxD,MAAM,GAAG,GAAG,GAAG,MAAA,OAAO,CAAC,QAAQ,mCAAI,wBAAwB,gBAAgB,CAAC;YAC3E,OAAwC,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAChE,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAClC;IACL,CAAC;IAED;;OAEG;IACa,IAAI,CAAQ,GAAW,EAAE,IAAY,EAAE,UAAU,GAAG,CAAC;;YACjE,4BAA4B;YAC5B,MAAM,aAAa,GAAuB,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAExF,6BAA6B;YAC7B,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;gBACxB,aAAa,CAAC,OAAO,GAAG,EAAE,CAAC;aAC9B;YACD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;gBACxC,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,kBAAkB,CAAC;aAC9D;YACD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE;gBACtC,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;aACxD;YACD,IAAI,IAAI,CAAC,SAAS,EAAE;gBAChB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAuC,CAAC;gBAC7D,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC;aAC1D;iBAAM;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAkC,CAAC;gBACxD,aAAa,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpE,IAAI,OAAO,CAAC,YAAY,EAAE;oBACtB,aAAa,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;iBACvE;aACJ;YAED,eAAe;YACf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;YAEvE,6BAA6B;YAC7B,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE;gBACnH,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;gBACnD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC3D,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;aAC/C;iBAAM;gBACH,OAAO,QAAQ,CAAC;aACnB;QACL,CAAC;KAAA;CACJ;AA7HD,4CA6HC"}

package/lib/TextSplitter.d.ts

@@ -0,0 +1,17 @@
+import { TextChunk, Tokenizer } from "./types";
+export interface TextSplitterConfig {
+    separators: string[];
+    keepSeparators: boolean;
+    chunkSize: number;
+    chunkOverlap: number;
+    tokenizer: Tokenizer;
+    docType?: string;
+}
+export declare class TextSplitter {
+    private readonly _config;
+    constructor(config?: Partial<TextSplitterConfig>);
+    split(text: string): TextChunk[];
+    private recursiveSplit;
+    private getSeparators;
+}
+//# sourceMappingURL=TextSplitter.d.ts.map

package/lib/TextSplitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextSplitter.d.ts","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAE1B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IA4BhD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IA4BvC,OAAO,CAAC,cAAc;IA0CtB,OAAO,CAAC,aAAa;CA8WxB"}

package/lib/TextSplitter.js

@@ -0,0 +1,460 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TextSplitter = void 0;
+const GPT3Tokenizer_1 = require("./GPT3Tokenizer");
+class TextSplitter {
+    constructor(config) {
+        this._config = Object.assign({
+            separators: ["\n\n", "\n", " ", ""],
+            keepSeparators: false,
+            chunkSize: 400,
+            chunkOverlap: 40,
+        }, config);
+        // Create a default tokenizer if none is provided
+        if (!this._config.tokenizer) {
+            this._config.tokenizer = new GPT3Tokenizer_1.GPT3Tokenizer();
+        }
+        // Use default separators if none are provided
+        if (!this._config.separators || this._config.separators.length === 0) {
+            this._config.separators = this.getSeparators(this._config.docType);
+        }
+        // Validate the config settings
+        if (this._config.chunkSize < 1) {
+            throw new Error("chunkSize must be >= 1");
+        }
+        else if (this._config.chunkOverlap < 0) {
+            throw new Error("chunkOverlap must be >= 0");
+        }
+        else if (this._config.chunkOverlap > this._config.chunkSize) {
+            throw new Error("chunkOverlap must be <= chunkSize");
+        }
+    }
+    split(text) {
+        // Get basic chunks
+        const chunks = this.recursiveSplit(text, this._config.separators, 0);
+        const that = this;
+        function getOverlapTokens(tokens) {
+            if (tokens != undefined) {
+                const len = tokens.length > that._config.chunkOverlap ? that._config.chunkOverlap : tokens.length;
+                return tokens.slice(tokens.length);
+            }
+            else {
+                return [];
+            }
+        }
+        // Add overlap tokens and text to the start and end of each chunk
+        if (this._config.chunkOverlap > 0) {
+            for (let i = 1; i < chunks.length; i++) {
+                const previousChunk = chunks[i - 1];
+                const chunk = chunks[i];
+                const nextChunk = i < chunks.length - 1 ? chunks[i + 1] : undefined;
+                chunk.startOverlap = getOverlapTokens(previousChunk.tokens.reverse()).reverse();
+                chunk.endOverlap = getOverlapTokens(nextChunk === null || nextChunk === void 0 ? void 0 : nextChunk.tokens);
+            }
+        }
+        return chunks;
+    }
+    recursiveSplit(text, separators, startPos) {
+        const chunks = [];
+        if (text.length > 0 && separators.length > 0) {
+            const separator = separators[0];
+            const nextSeparators = separators.length > 1 ? separators.slice(1) : [];
+            const parts = text.split(separator);
+            for (let i = 0; i < parts.length; i++) {
+                const lastChunk = (i === parts.length - 1);
+                // Get chunk text and endPos
+                let chunk = parts[i];
+                const endPos = (startPos + (chunk.length - 1)) + (lastChunk ? 0 : separator.length);
+                if (this._config.keepSeparators && !lastChunk) {
+                    chunk += separator;
+                }
+                // Encode chunk text
+                const tokens = this._config.tokenizer.encode(chunk);
+                if (tokens.length > this._config.chunkSize) {
+                    // Break the text into smaller chunks
+                    const subChunks = this.recursiveSplit(chunk, nextSeparators, startPos);
+                    chunks.push(...subChunks);
+                }
+                else {
+                    // Append chunk to output
+                    chunks.push({
+                        text: chunk,
+                        tokens: tokens,
+                        startPos: startPos,
+                        endPos: endPos,
+                        startOverlap: [],
+                        endOverlap: [],
+                    });
+                }
+                // Update startPos
+                startPos = endPos + 1;
+            }
+        }
+        return chunks;
+    }
+    getSeparators(docType) {
+        switch (docType !== null && docType !== void 0 ? docType : '') {
+            case "cpp":
+                return [
+                    // Split along class definitions
+                    "\nclass ",
+                    // Split along function definitions
+                    "\nvoid ",
+                    "\nint ",
+                    "\nfloat ",
+                    "\ndouble ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\nswitch ",
+                    "\ncase ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "go":
+                return [
+                    // Split along function definitions
+                    "\nfunc ",
+                    "\nvar ",
+                    "\nconst ",
+                    "\ntype ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nswitch ",
+                    "\ncase ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "java":
+            case "c#":
+            case "csharp":
+            case "cs":
+            case "ts":
+            case "tsx":
+            case "typescript":
+                return [
+                    // Split along class definitions
+                    "\nclass ",
+                    // Split along method definitions
+                    "\npublic ",
+                    "\nprotected ",
+                    "\nprivate ",
+                    "\nstatic ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\nswitch ",
+                    "\ncase ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "js":
+            case "jsx":
+            case "javascript":
+                return [
+                    // Split along class definitions
+                    "\nclass ",
+                    // Split along function definitions
+                    "\nfunction ",
+                    "\nconst ",
+                    "\nlet ",
+                    "\nvar ",
+                    "\nclass ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\nswitch ",
+                    "\ncase ",
+                    "\ndefault ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "php":
+                return [
+                    // Split along function definitions
+                    "\nfunction ",
+                    // Split along class definitions
+                    "\nclass ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nforeach ",
+                    "\nwhile ",
+                    "\ndo ",
+                    "\nswitch ",
+                    "\ncase ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "proto":
+                return [
+                    // Split along message definitions
+                    "\nmessage ",
+                    // Split along service definitions
+                    "\nservice ",
+                    // Split along enum definitions
+                    "\nenum ",
+                    // Split along option definitions
+                    "\noption ",
+                    // Split along import statements
+                    "\nimport ",
+                    // Split along syntax declarations
+                    "\nsyntax ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "python":
+            case "py":
+                return [
+                    // First, try to split along class definitions
+                    "\nclass ",
+                    "\ndef ",
+                    "\n\tdef ",
+                    // Now split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "rst":
+                return [
+                    // Split along section titles
+                    "\n===\n",
+                    "\n---\n",
+                    "\n***\n",
+                    // Split along directive markers
+                    "\n.. ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "ruby":
+                return [
+                    // Split along method definitions
+                    "\ndef ",
+                    "\nclass ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nunless ",
+                    "\nwhile ",
+                    "\nfor ",
+                    "\ndo ",
+                    "\nbegin ",
+                    "\nrescue ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "rust":
+                return [
+                    // Split along function definitions
+                    "\nfn ",
+                    "\nconst ",
+                    "\nlet ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nwhile ",
+                    "\nfor ",
+                    "\nloop ",
+                    "\nmatch ",
+                    "\nconst ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "scala":
+                return [
+                    // Split along class definitions
+                    "\nclass ",
+                    "\nobject ",
+                    // Split along method definitions
+                    "\ndef ",
+                    "\nval ",
+                    "\nvar ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\nmatch ",
+                    "\ncase ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "swift":
+                return [
+                    // Split along function definitions
+                    "\nfunc ",
+                    // Split along class definitions
+                    "\nclass ",
+                    "\nstruct ",
+                    "\nenum ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\ndo ",
+                    "\nswitch ",
+                    "\ncase ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "markdown":
+                return [
+                    // First, try to split along Markdown headings (starting with level 2)
+                    "\n## ",
+                    "\n### ",
+                    "\n#### ",
+                    "\n##### ",
+                    "\n###### ",
+                    // Note the alternative syntax for headings (below) is not handled here
+                    // Heading level 2
+                    // ---------------
+                    // End of code block
+                    "```\n\n",
+                    // Horizontal lines
+                    "\n\n***\n\n",
+                    "\n\n---\n\n",
+                    "\n\n___\n\n",
+                    // Note that this splitter doesn't handle horizontal lines defined
+                    // by *three or more* of ***, ---, or ___, but this is not handled
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "latex":
+                return [
+                    // First, try to split along Latex sections
+                    "\n\\chapter{",
+                    "\n\\section{",
+                    "\n\\subsection{",
+                    "\n\\subsubsection{",
+                    // Now split by environments
+                    "\n\\begin{enumerate}",
+                    "\n\\begin{itemize}",
+                    "\n\\begin{description}",
+                    "\n\\begin{list}",
+                    "\n\\begin{quote}",
+                    "\n\\begin{quotation}",
+                    "\n\\begin{verse}",
+                    "\n\\begin{verbatim}",
+                    // Now split by math environments
+                    "\n\\begin{align}",
+                    "$$",
+                    "$",
+                    // Now split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            case "html":
+                return [
+                    // First, try to split along HTML tags
+                    "<body>",
+                    "<div>",
+                    "<p>",
+                    "<br>",
+                    "<li>",
+                    "<h1>",
+                    "<h2>",
+                    "<h3>",
+                    "<h4>",
+                    "<h5>",
+                    "<h6>",
+                    "<span>",
+                    "<table>",
+                    "<tr>",
+                    "<td>",
+                    "<th>",
+                    "<ul>",
+                    "<ol>",
+                    "<header>",
+                    "<footer>",
+                    "<nav>",
+                    // Head
+                    "<head>",
+                    "<style>",
+                    "<script>",
+                    "<meta>",
+                    "<title>",
+                    // Normal type of lines
+                    " ",
+                    "",
+                ];
+            case "sol":
+                return [
+                    // Split along compiler informations definitions
+                    "\npragma ",
+                    "\nusing ",
+                    // Split along contract definitions
+                    "\ncontract ",
+                    "\ninterface ",
+                    "\nlibrary ",
+                    // Split along method definitions
+                    "\nconstructor ",
+                    "\ntype ",
+                    "\nfunction ",
+                    "\nevent ",
+                    "\nmodifier ",
+                    "\nerror ",
+                    "\nstruct ",
+                    "\nenum ",
+                    // Split along control flow statements
+                    "\nif ",
+                    "\nfor ",
+                    "\nwhile ",
+                    "\ndo while ",
+                    "\nassembly ",
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+            default:
+                return [
+                    // Split by the normal type of lines
+                    "\n\n",
+                    "\n",
+                    " ",
+                    "",
+                ];
+        }
+    }
+}
+exports.TextSplitter = TextSplitter;
+//# sourceMappingURL=TextSplitter.js.map

package/lib/TextSplitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextSplitter.js","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":";;;AAAA,mDAAgD;AAYhD,MAAa,YAAY;IAGrB,YAAmB,MAAoC;QACnD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACzB,UAAU,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;YACnC,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,EAAE;SACG,EAAE,MAAM,CAAC,CAAC;QAEjC,iDAAiD;QACjD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACzB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,6BAAa,EAAE,CAAC;SAChD;QAED,8CAA8C;QAC9C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAClE,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;SACtE;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC7C;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;SAChD;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC3D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;SACxD;IACL,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAErE,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,SAAS,gBAAgB,CAAC,MAAiB;YACvC,IAAI,MAAM,IAAI,SAAS,EAAE;gBACrB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBAClG,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aACtC;iBAAM;gBACH,OAAO,EAAE,CAAC;aACb;QACL,CAAC;QAED,iEAAiE;QACjE,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACpE,KAAK,CAAC,YAAY,GAAG,gBAAgB,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;gBAChF,KAAK,CAAC,UAAU,GAAG,gBAAgB,CAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,CAAC,CAAC;aAC1D;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,UAAoB,EAAE,QAAgB;QACvE,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1C,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE3C,4BAA4B;gBAC5B,IAAI,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrB,MAAM,MAAM,GAAG,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACpF,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE;oBAC3C,KAAK,IAAI,SAAS,CAAC;iBACtB;gBAED,oBAAoB;gBACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBACxC,qCAAqC;oBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;oBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;iBAC7B;qBAAM;oBACH,yBAAyB;oBACzB,MAAM,CAAC,IAAI,CAAC;wBACR,IAAI,EAAE,KAAK;wBACX,MAAM,EAAE,MAAM;wBACd,QAAQ,EAAE,QAAQ;wBAClB,MAAM,EAAE,MAAM;wBACd,YAAY,EAAE,EAAE;wBAChB,UAAU,EAAE,EAAE;qBACjB,CAAC,CAAC;iBACN;gBAED,kBAAkB;gBAClB,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;aACzB;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,aAAa,CAAC,OAAgB;QAClC,QAAQ,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE;YACnB,KAAK,KAAK;gBACN,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,IAAI;gBACL,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM,CAAC;YACZ,KAAK,IAAI,CAAC;YACV,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,iCAAiC;oBACjC,WAAW;oBACX,cAAc;oBACd,YAAY;oBACZ,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,UAAU;oBACV,QAAQ;oBACR,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,YAAY;oBACZ,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,mCAAmC;oBACnC,aAAa;oBACb,gCAAgC;oBAChC,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,YAAY;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,kCAAkC;oBAClC,YAAY;oBACZ,kCAAkC;oBAClC,YAAY;oBACZ,+BAA+B;oBAC/B,SAAS;oBACT,iCAAiC;oBACjC,WAAW;oBACX,gCAAgC;oBAChC,WAAW;oBACX,kCAAkC;oBAClC,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI;gBACL,OAAO;oBACH,8CAA8C;oBAC9C,UAAU;oBACV,QAAQ;oBACR,UAAU;oBACV,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,6BAA6B;oBAC7B,SAAS;oBACT,SAAS;oBACT,SAAS;oBACT,gCAAgC;oBAChC,OAAO;oBACP,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,iCAAiC;oBACjC,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,WAAW;oBACX,UAAU;oBACV,QAAQ;oBACR,OAAO;oBACP,UAAU;oBACV,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,mCAAmC;oBACnC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,iCAAiC;oBACjC,QAAQ;oBACR,QAAQ;oBACR,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,UAAU;oBACV,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,UAAU;gBACX,OAAO;oBACH,sEAAsE;oBACtE,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,WAAW;oBACX,uEAAuE;oBACvE,kBAAkB;oBAClB,kBAAkB;oBAClB,oBAAoB;oBACpB,SAAS;oBACT,mBAAmB;oBACnB,aAAa;oBACb,aAAa;oBACb,aAAa;oBACb,kEAAkE;oBAClE,kEAAkE;oBAClE,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,2CAA2C;oBAC3C,cAAc;oBACd,cAAc;oBACd,iBAAiB;oBACjB,oBAAoB;oBAEpB,4BAA4B;oBAC5B,sBAAsB;oBACtB,oBAAoB;oBACpB,wBAAwB;oBACxB,iBAAiB;oBACjB,kBAAkB;oBAClB,sBAAsB;oBACtB,kBAAkB;oBAClB,qBAAqB;oBAErB,iCAAiC;oBACjC,kBAAkB;oBAClB,IAAI;oBACJ,GAAG;oBAEH,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,sCAAsC;oBACtC,QAAQ;oBACR,OAAO;oBACP,KAAK;oBACL,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,QAAQ;oBACR,SAAS;oBACT,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,UAAU;oBACV,UAAU;oBACV,OAAO;oBACP,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,uBAAuB;oBACvB,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,gDAAgD;oBAChD,WAAW;oBACX,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,cAAc;oBACd,YAAY;oBACZ,iCAAiC;oBACjC,gBAAgB;oBAChB,SAAS;oBACT,aAAa;oBACb,UAAU;oBACV,aAAa;oBACb,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,aAAa;oBACb,aAAa;oBACb,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN;gBACI,OAAO;oBACH,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;SACT;IACL,CAAC;CACJ;AAndD,oCAmdC"}

package/lib/WebFetcher.d.ts

@@ -0,0 +1,16 @@
+import { AxiosRequestConfig } from "axios";
+import { TextFetcher } from './types';
+export interface WebFetcherConfig {
+    headers?: Record<string, string>;
+    requestConfig?: AxiosRequestConfig;
+    htmlToText: boolean;
+    summarizeHtml: boolean;
+}
+export declare class WebFetcher implements TextFetcher {
+    private readonly _config;
+    constructor(config?: Partial<WebFetcherConfig>);
+    fetch(uri: string): Promise<string>;
+    private extractText;
+    private fetchPage;
+}
+//# sourceMappingURL=WebFetcher.d.ts.map

package/lib/WebFetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebFetcher.d.ts","sourceRoot":"","sources":["../src/WebFetcher.ts"],"names":[],"mappings":"AAAA,OAAc,EAAE,kBAAkB,EAAE,MAAM,OAAO,CAAC;AAElD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AA2BtC,MAAM,WAAW,gBAAgB;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;IAChC,aAAa,CAAC,EAAE,kBAAkB,CAAC;IACnC,UAAU,EAAE,OAAO,CAAC;IACpB,aAAa,EAAE,OAAO,CAAC;CAC1B;AAED,qBAAa,UAAW,YAAW,WAAW;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;gBAExB,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC;IAOxC,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAShD,OAAO,CAAC,WAAW;YAyCL,SAAS;CA+B1B"}