vectra 0.9.0 → 0.10.0

package/lib/LocalDocumentResult.js

@@ -1,282 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.LocalDocumentResult = void 0;
-const LocalDocument_1 = require("./LocalDocument");
-/**
- * Represents a search result for a document stored on disk.
- */
-class LocalDocumentResult extends LocalDocument_1.LocalDocument {
-    /**
-     * @private
-     * Internal constructor for `LocalDocumentResult` instances.
-     */
-    constructor(index, id, uri, chunks, tokenizer) {
-        super(index, id, uri);
-        this._chunks = chunks;
-        this._tokenizer = tokenizer;
-        // Compute average score
-        let score = 0;
-        this._chunks.forEach(chunk => score += chunk.score);
-        this._score = score / this._chunks.length;
-    }
-    /**
-     * Returns the chunks of the document that matched the query.
-     */
-    get chunks() {
-        return this._chunks;
-    }
-    /**
-     * Returns the average score of the document result.
-     */
-    get score() {
-        return this._score;
-    }
-    /**
-     * Renders all of the results chunks as spans of text (sections.)
-     * @remarks
-     * The returned sections will be sorted by document order and limited to maxTokens in length.
-     * @param maxTokens Maximum number of tokens per section.
-     * @returns Array of rendered text sections.
-     */
-    renderAllSections(maxTokens) {
-        return __awaiter(this, void 0, void 0, function* () {
-            // Load text from disk
-            const text = yield this.loadText();
-            // Add chunks to a temp array and split any chunks that are longer than maxTokens.
-            const chunks = [];
-            for (let i = 0; i < this._chunks.length; i++) {
-                const chunk = this._chunks[i];
-                const startPos = chunk.item.metadata.startPos;
-                const endPos = chunk.item.metadata.endPos;
-                const chunkText = text.substring(startPos, endPos + 1);
-                const tokens = this._tokenizer.encode(chunkText);
-                let offset = 0;
-                while (offset < tokens.length) {
-                    const chunkLength = Math.min(maxTokens, tokens.length - offset);
-                    chunks.push({
-                        text: this._tokenizer.decode(tokens.slice(offset, offset + chunkLength)),
-                        startPos: startPos + offset,
-                        endPos: startPos + offset + chunkLength - 1,
-                        score: chunk.score,
-                        tokenCount: chunkLength
-                    });
-                    offset += chunkLength;
-                }
-            }
-            // Sort chunks by startPos
-            const sorted = chunks.sort((a, b) => a.startPos - b.startPos);
-            // Generate sections
-            const sections = [];
-            for (let i = 0; i < sorted.length; i++) {
-                const chunk = sorted[i];
-                let section = sections[sections.length - 1];
-                if (!section || section.tokenCount + chunk.tokenCount > maxTokens) {
-                    section = {
-                        chunks: [],
-                        score: 0,
-                        tokenCount: 0
-                    };
-                    sections.push(section);
-                }
-                section.chunks.push(chunk);
-                section.score += chunk.score;
-                section.tokenCount += chunk.tokenCount;
-            }
-            // Normalize section scores
-            sections.forEach(section => section.score /= section.chunks.length);
-            // Return final rendered sections
-            return sections.map(section => {
-                let text = '';
-                section.chunks.forEach(chunk => text += chunk.text);
-                return {
-                    text: text,
-                    tokenCount: section.tokenCount,
-                    score: section.score
-                };
-            });
-        });
-    }
-    /**
-     * Renders the top spans of text (sections) of the document based on the query result.
-     * @remarks
-     * The returned sections will be sorted by relevance and limited to the top `maxSections`.
-     * @param maxTokens Maximum number of tokens per section.
-     * @param maxSections Maximum number of sections to return.
-     * @param overlappingChunks Optional. If true, overlapping chunks of text will be added to each section until the maxTokens is reached.
-     * @returns Array of rendered text sections.
-     */
-    renderSections(maxTokens, maxSections, overlappingChunks = true) {
-        return __awaiter(this, void 0, void 0, function* () {
-            // Load text from disk
-            const text = yield this.loadText();
-            // First check to see if the entire document is shorter than maxTokens
-            const length = yield this.getLength();
-            if (length <= maxTokens) {
-                return [{
-                        text,
-                        tokenCount: length,
-                        score: 1.0
-                    }];
-            }
-            // Otherwise, we need to split the document into sections
-            // - Add each chunk to a temp array and filter out any chunk that's longer then maxTokens.
-            // - Sort the array by startPos to arrange chunks in document order.
-            // - Generate a new array of sections by combining chunks until the maxTokens is reached for each section.
-            // - Generate an aggregate score for each section by averaging the score of each chunk in the section.
-            // - Sort the sections by score and limit to maxSections.
-            // - For each remaining section combine adjacent chunks of text.
-            // - Dynamically add overlapping chunks of text to each section until the maxTokens is reached.
-            const chunks = this._chunks.map(chunk => {
-                const startPos = chunk.item.metadata.startPos;
-                const endPos = chunk.item.metadata.endPos;
-                const chunkText = text.substring(startPos, endPos + 1);
-                return {
-                    text: chunkText,
-                    startPos,
-                    endPos,
-                    score: chunk.score,
-                    tokenCount: this._tokenizer.encode(chunkText).length
-                };
-            }).filter(chunk => chunk.tokenCount <= maxTokens).sort((a, b) => a.startPos - b.startPos);
-            // Check for no chunks
-            if (chunks.length === 0) {
-                // Take the top chunk and return a subset of its text
-                const topChunk = this._chunks[0];
-                const startPos = topChunk.item.metadata.startPos;
-                const endPos = topChunk.item.metadata.endPos;
-                const chunkText = text.substring(startPos, endPos + 1);
-                const tokens = this._tokenizer.encode(chunkText);
-                return [{
-                        text: this._tokenizer.decode(tokens.slice(0, maxTokens)),
-                        tokenCount: maxTokens,
-                        score: topChunk.score
-                    }];
-            }
-            // Generate sections
-            const sections = [];
-            for (let i = 0; i < chunks.length; i++) {
-                const chunk = chunks[i];
-                let section = sections[sections.length - 1];
-                if (!section || section.tokenCount + chunk.tokenCount > maxTokens) {
-                    section = {
-                        chunks: [],
-                        score: 0,
-                        tokenCount: 0
-                    };
-                    sections.push(section);
-                }
-                section.chunks.push(chunk);
-                section.score += chunk.score;
-                section.tokenCount += chunk.tokenCount;
-            }
-            // Normalize section scores
-            sections.forEach(section => section.score /= section.chunks.length);
-            // Sort sections by score and limit to maxSections
-            sections.sort((a, b) => b.score - a.score);
-            if (sections.length > maxSections) {
-                sections.splice(maxSections, sections.length - maxSections);
-            }
-            // Combine adjacent chunks of text
-            sections.forEach(section => {
-                for (let i = 0; i < section.chunks.length - 1; i++) {
-                    const chunk = section.chunks[i];
-                    const nextChunk = section.chunks[i + 1];
-                    if (chunk.endPos + 1 === nextChunk.startPos) {
-                        chunk.text += nextChunk.text;
-                        chunk.endPos = nextChunk.endPos;
-                        chunk.tokenCount += nextChunk.tokenCount;
-                        section.chunks.splice(i + 1, 1);
-                        i--;
-                    }
-                }
-            });
-            // Add overlapping chunks of text to each section until the maxTokens is reached
-            if (overlappingChunks) {
-                const connector = {
-                    text: '\n\n...\n\n',
-                    startPos: -1,
-                    endPos: -1,
-                    score: 0,
-                    tokenCount: this._tokenizer.encode('\n\n...\n\n').length
-                };
-                sections.forEach(section => {
-                    // Insert connectors between chunks
-                    if (section.chunks.length > 1) {
-                        for (let i = 0; i < section.chunks.length - 1; i++) {
-                            section.chunks.splice(i + 1, 0, connector);
-                            section.tokenCount += connector.tokenCount;
-                            i++;
-                        }
-                    }
-                    // Add chunks to beginning and end of the section until maxTokens is reached
-                    let budget = maxTokens - section.tokenCount;
-                    if (budget > 40) {
-                        const sectionStart = section.chunks[0].startPos;
-                        const sectionEnd = section.chunks[section.chunks.length - 1].endPos;
-                        if (sectionStart > 0) {
-                            const beforeTex = text.substring(0, section.chunks[0].startPos);
-                            const beforeTokens = this.encodeBeforeText(beforeTex, Math.ceil(budget / 2));
-                            const beforeBudget = sectionEnd < text.length - 1 ? Math.min(beforeTokens.length, Math.ceil(budget / 2)) : Math.min(beforeTokens.length, budget);
-                            const chunk = {
-                                text: this._tokenizer.decode(beforeTokens.slice(-beforeBudget)),
-                                startPos: sectionStart - beforeBudget,
-                                endPos: sectionStart - 1,
-                                score: 0,
-                                tokenCount: beforeBudget
-                            };
-                            section.chunks.unshift(chunk);
-                            section.tokenCount += chunk.tokenCount;
-                            budget -= chunk.tokenCount;
-                        }
-                        if (sectionEnd < text.length - 1) {
-                            const afterText = text.substring(sectionEnd + 1);
-                            const afterTokens = this.encodeAfterText(afterText, budget);
-                            const afterBudget = Math.min(afterTokens.length, budget);
-                            const chunk = {
-                                text: this._tokenizer.decode(afterTokens.slice(0, afterBudget)),
-                                startPos: sectionEnd + 1,
-                                endPos: sectionEnd + afterBudget,
-                                score: 0,
-                                tokenCount: afterBudget
-                            };
-                            section.chunks.push(chunk);
-                            section.tokenCount += chunk.tokenCount;
-                            budget -= chunk.tokenCount;
-                        }
-                    }
-                });
-            }
-            // Return final rendered sections
-            return sections.map(section => {
-                let text = '';
-                section.chunks.forEach(chunk => text += chunk.text);
-                return {
-                    text: text,
-                    tokenCount: section.tokenCount,
-                    score: section.score
-                };
-            });
-        });
-    }
-    encodeBeforeText(text, budget) {
-        const maxLength = budget * 8;
-        const substr = text.length <= maxLength ? text : text.substring(text.length - maxLength);
-        return this._tokenizer.encode(substr);
-    }
-    encodeAfterText(text, budget) {
-        const maxLength = budget * 8;
-        const substr = text.length <= maxLength ? text : text.substring(0, maxLength);
-        return this._tokenizer.encode(substr);
-    }
-}
-exports.LocalDocumentResult = LocalDocumentResult;
-//# sourceMappingURL=LocalDocumentResult.js.map

package/lib/LocalDocumentResult.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"LocalDocumentResult.js","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mDAAgD;AAIhD;;GAEG;AACH,MAAa,mBAAoB,SAAQ,6BAAa;IAKlD;;;OAGG;IACH,YAAmB,KAAyB,EAAE,EAAU,EAAE,GAAW,EAAE,MAA4C,EAAE,SAAoB;QACrI,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAE5B,wBAAwB;QACxB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;;;;;OAMG;IACU,iBAAiB,CAAC,SAAiB;;YAC5C,sBAAsB;YACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEnC,kFAAkF;YAClF,MAAM,MAAM,GAAoB,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACjD,IAAI,MAAM,GAAG,CAAC,CAAC;gBACf,OAAO,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE;oBAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;oBAChE,MAAM,CAAC,IAAI,CAAC;wBACR,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;wBACxE,QAAQ,EAAE,QAAQ,GAAG,MAAM;wBAC3B,MAAM,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,CAAC;wBAC3C,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,UAAU,EAAE,WAAW;qBAC1B,CAAC,CAAC;oBACH,MAAM,IAAI,WAAW,CAAC;iBACzB;aACJ;YAED,0BAA0B;YAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAE9D,oBAAoB;YACpB,MAAM,QAAQ,GAAc,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,IAAI,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE;oBAC/D,OAAO,GAAG;wBACN,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,CAAC;wBACR,UAAU,EAAE,CAAC;qBAChB,CAAC;oBACF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBAC1B;gBACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC3B,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;gBAC7B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;aAC1C;YAED,2BAA2B;YAC3B,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEpE,iCAAiC;YACjC,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;gBAC1B,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,OAAO;oBACH,IAAI,EAAE,IAAI;oBACV,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;iBACvB,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;IAED;;;;;;;;OAQG;IACU,cAAc,CAAC,SAAiB,EAAE,WAAmB,EAAE,iBAAiB,GAAG,IAAI;;YACxF,sBAAsB;YACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEnC,sEAAsE;YACtE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,IAAI,MAAM,IAAI,SAAS,EAAE;gBACrB,OAAO,CAAC;wBACJ,IAAI;wBACJ,UAAU,EAAE,MAAM;wBAClB,KAAK,EAAE,GAAG;qBACb,CAAC,CAAC;aACN;YAED,yDAAyD;YACzD,0FAA0F;YAC1F,oEAAoE;YACpE,0GAA0G;YAC1G,sGAAsG;YACtG,yDAAyD;YACzD,gEAAgE;YAChE,+FAA+F;YAC/F,MAAM,MAAM,GAAmB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;gBACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,OAAO;oBACH,IAAI,EAAE,SAAS;oBACf,QAAQ;oBACR,MAAM;oBACN,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM;iBACvD,CAAC;YACN,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAE1F,sBAAsB;YACtB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;gBACrB,qDAAqD;gBACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACjD,OAAO,CAAC;wBACJ,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;wBACxD,UAAU,EAAE,SAAS;wBACrB,KAAK,EAAE,QAAQ,CAAC,KAAK;qBACxB,CAAC,CAAC;aACN;YAED,oBAAoB;YACpB,MAAM,QAAQ,GAAc,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,IAAI,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE;oBAC/D,OAAO,GAAG;wBACN,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,CAAC;wBACR,UAAU,EAAE,CAAC;qBAChB,CAAC;oBACF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBAC1B;gBACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC3B,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;gBAC7B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;aAC1C;YAED,2BAA2B;YAC3B,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEpE,kDAAkD;YAClD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAC3C,IAAI,QAAQ,CAAC,MAAM,GAAG,WAAW,EAAE;gBAC/B,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;aAC/D;YAED,kCAAkC;YAClC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;oBAChD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBACxC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,KAAK,SAAS,CAAC,QAAQ,EAAE;wBACzC,KAAK,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC;wBAC7B,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;wBAChC,KAAK,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,CAAC;wBACzC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;wBAChC,CAAC,EAAE,CAAC;qBACP;iBACJ;YACL,CAAC,CAAC,CAAC;YAEH,gFAAgF;YAChF,IAAI,iBAAiB,EAAE;gBACnB,MAAM,SAAS,GAAiB;oBAC5B,IAAI,EAAE,aAAa;oBACnB,QAAQ,EAAE,CAAC,CAAC;oBACZ,MAAM,EAAE,CAAC,CAAC;oBACV,KAAK,EAAE,CAAC;oBACR,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM;iBAC3D,CAAC;gBACF,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;oBACvB,mCAAmC;oBACnC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;wBAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;4BAChD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;4BAC3C,OAAO,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,CAAC;4BAC3C,CAAC,EAAE,CAAC;yBACP;qBACJ;oBAED,4EAA4E;oBAC5E,IAAI,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC;oBAC5C,IAAI,MAAM,GAAG,EAAE,EAAE;wBACb,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;wBAChD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;wBACpE,IAAI,YAAY,GAAG,CAAC,EAAE;4BAClB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;4BAChE,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAC,CAAC,CAAC,CAAC,CAAC;4BAC3E,MAAM,YAAY,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;4BAC/I,MAAM,KAAK,GAAiB;gCACxB,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;gCAC/D,QAAQ,EAAE,YAAY,GAAG,YAAY;gCACrC,MAAM,EAAE,YAAY,GAAG,CAAC;gCACxB,KAAK,EAAE,CAAC;gCACR,UAAU,EAAE,YAAY;6BAC3B,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;4BAC9B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;4BACvC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;yBAC9B;wBAED,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;4BAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;4BACjD,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;4BAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;4BACzD,MAAM,KAAK,GAAiB;gCACxB,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;gCAC/D,QAAQ,EAAE,UAAU,GAAG,CAAC;gCACxB,MAAM,EAAE,UAAU,GAAG,WAAW;gCAChC,KAAK,EAAE,CAAC;gCACR,UAAU,EAAE,WAAW;6BAC1B,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;4BAC3B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;4BACvC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;yBAC9B;qBACJ;gBACL,CAAC,CAAC,CAAC;aACN;YAED,iCAAiC;YACjC,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;gBAC1B,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,OAAO;oBACH,IAAI,EAAE,IAAI;oBACV,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;iBACvB,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;IAEO,gBAAgB,CAAC,IAAY,EAAE,MAAc;QACjD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;QACzF,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAEO,eAAe,CAAC,IAAY,EAAE,MAAc;QAChD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;CAEJ;AA/RD,kDA+RC"}

package/lib/LocalIndex.d.ts

@@ -1,136 +0,0 @@
-import { IndexItem, IndexStats, MetadataFilter, MetadataTypes, QueryResult } from './types';
-export interface CreateIndexConfig {
-    version: number;
-    deleteIfExists?: boolean;
-    metadata_config?: {
-        indexed?: string[];
-    };
-}
-/**
- * Local vector index instance.
- * @remarks
- * This class is used to create, update, and query a local vector index.
- * Each index is a folder on disk containing an index.json file and an optional set of metadata files.
- */
-export declare class LocalIndex<TMetadata extends Record<string, MetadataTypes> = Record<string, MetadataTypes>> {
-    private readonly _folderPath;
-    private readonly _indexName;
-    private _data?;
-    private _update?;
-    /**
-     * Creates a new instance of LocalIndex.
-     * @param folderPath Path to the index folder.
-     * @param indexName Optional name of the index file. Defaults to index.json.
-     */
-    constructor(folderPath: string, indexName?: string);
-    /**
-     * Path to the index folder.
-     */
-    get folderPath(): string;
-    /**
-     * Optional name of the index file.
-     */
-    get indexName(): string;
-    /**
-     * Begins an update to the index.
-     * @remarks
-     * This method loads the index into memory and prepares it for updates.
-     */
-    beginUpdate(): Promise<void>;
-    /**
-     * Cancels an update to the index.
-     * @remarks
-     * This method discards any changes made to the index since the update began.
-     */
-    cancelUpdate(): void;
-    /**
-     * Creates a new index.
-     * @remarks
-     * This method creates a new folder on disk containing an index.json file.
-     * @param config Index configuration.
-     */
-    createIndex(config?: CreateIndexConfig): Promise<void>;
-    /**
-     * Deletes the index.
-     * @remarks
-     * This method deletes the index folder from disk.
-     */
-    deleteIndex(): Promise<void>;
-    /**
-     * Deletes an item from the index.
-     * @param id ID of item to delete.
-     */
-    deleteItem(id: string): Promise<void>;
-    /**
-     * Ends an update to the index.
-     * @remarks
-     * This method saves the index to disk.
-     */
-    endUpdate(): Promise<void>;
-    /**
-     * Loads an index from disk and returns its stats.
-     * @returns Index stats.
-     */
-    getIndexStats(): Promise<IndexStats>;
-    /**
-     * Returns an item from the index given its ID.
-     * @param id ID of the item to retrieve.
-     * @returns Item or undefined if not found.
-     */
-    getItem<TItemMetadata extends TMetadata = TMetadata>(id: string): Promise<IndexItem<TItemMetadata> | undefined>;
-    /**
-     * Adds an item to the index.
-     * @remarks
-     * A new update is started if one is not already in progress. If an item with the same ID
-     * already exists, an error will be thrown.
-     * @param item Item to insert.
-     * @returns Inserted item.
-     */
-    insertItem<TItemMetadata extends TMetadata = TMetadata>(item: Partial<IndexItem<TItemMetadata>>): Promise<IndexItem<TItemMetadata>>;
-    /**
-     * Returns true if the index exists.
-     */
-    isIndexCreated(): Promise<boolean>;
-    /**
-     * Returns all items in the index.
-     * @remarks
-     * This method loads the index into memory and returns all its items. A copy of the items
-     * array is returned so no modifications should be made to the array.
-     * @returns Array of all items in the index.
-     */
-    listItems<TItemMetadata extends TMetadata = TMetadata>(): Promise<IndexItem<TItemMetadata>[]>;
-    /**
-     * Returns all items in the index matching the filter.
-     * @remarks
-     * This method loads the index into memory and returns all its items matching the filter.
-     * @param filter Filter to apply.
-     * @returns Array of items matching the filter.
-     */
-    listItemsByMetadata<TItemMetadata extends TMetadata = TMetadata>(filter: MetadataFilter): Promise<IndexItem<TItemMetadata>[]>;
-    /**
-     * Finds the top k items in the index that are most similar to the vector.
-     * @remarks
-     * This method loads the index into memory and returns the top k items that are most similar.
-     * An optional filter can be applied to the metadata of the items.
-     * @param vector Vector to query against.
-     * @param topK Number of items to return.
-     * @param filter Optional. Filter to apply.
-     * @returns Similar items to the vector that matche the supplied filter.
-     */
-    queryItems<TItemMetadata extends TMetadata = TMetadata>(vector: number[], topK: number, filter?: MetadataFilter): Promise<QueryResult<TItemMetadata>[]>;
-    /**
-     * Adds or replaces an item in the index.
-     * @remarks
-     * A new update is started if one is not already in progress. If an item with the same ID
-     * already exists, it will be replaced.
-     * @param item Item to insert or replace.
-     * @returns Upserted item.
-     */
-    upsertItem<TItemMetadata extends TMetadata = TMetadata>(item: Partial<IndexItem<TItemMetadata>>): Promise<IndexItem<TItemMetadata>>;
-    /**
-     * Ensures that the index has been loaded into memory.
-     */
-    protected loadIndexData(): Promise<void>;
-    private addItemToUpdate;
-}
-//# sourceMappingURL=LocalIndex.d.ts.map

package/lib/LocalIndex.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"LocalIndex.d.ts","sourceRoot":"","sources":["../src/LocalIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE5F,MAAM,WAAW,iBAAiB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE;QACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACL;AAED;;;;;GAKG;AACH,qBAAa,UAAU,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IACjG,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IAEpC,OAAO,CAAC,KAAK,CAAC,CAAY;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAY;IAE5B;;;;OAIG;gBACgB,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;IAKzD;;OAEG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;OAEG;IACH,IAAW,SAAS,IAAI,MAAM,CAE7B;IAED;;;;OAIG;IACU,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IASzC;;;;OAIG;IACI,YAAY,IAAI,IAAI;IAI3B;;;;;OAKG;IACU,WAAW,CAAC,MAAM,GAAE,iBAAgC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BjF;;;;OAIG;IACI,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;;OAGG;IACU,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBlD;;;;OAIG;IACU,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAevC;;;OAGG;IACU,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC;IASjD;;;;OAIG;IACU,OAAO,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,SAAS,CAAC;IAK5H;;;;;;;OAOG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAWhJ;;OAEG;IACU,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAS/C;;;;;;OAMG;IACU,SAAS,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,KAAK,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;IAK1G;;;;;;OAMG;IACU,mBAAmB,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;IAK1I;;;;;;;;;OASG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;IAyCpK;;;;;;;OAOG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAWhJ;;OAEG;cACa,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;YAahC,eAAe;CA8DhC"}