vectra 0.9.0 → 0.10.0

package/lib/LocalDocumentIndex.js

@@ -1,446 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.LocalDocumentIndex = void 0;
-const fs = __importStar(require("fs/promises"));
-const path = __importStar(require("path"));
-const uuid_1 = require("uuid");
-const GPT3Tokenizer_1 = require("./GPT3Tokenizer");
-const LocalIndex_1 = require("./LocalIndex");
-const TextSplitter_1 = require("./TextSplitter");
-const LocalDocumentResult_1 = require("./LocalDocumentResult");
-const LocalDocument_1 = require("./LocalDocument");
-/**
- * Represents a local index of documents stored on disk.
- */
-class LocalDocumentIndex extends LocalIndex_1.LocalIndex {
-    /**
-     * Creates a new `LocalDocumentIndex` instance.
-     * @param config Configuration settings for the document index.
-     */
-    constructor(config) {
-        var _a, _b;
-        super(config.folderPath);
-        this._embeddings = config.embeddings;
-        this._chunkingConfig = Object.assign({
-            keepSeparators: true,
-            chunkSize: 512,
-            chunkOverlap: 0,
-        }, config.chunkingConfig);
-        this._tokenizer = (_b = (_a = config.tokenizer) !== null && _a !== void 0 ? _a : this._chunkingConfig.tokenizer) !== null && _b !== void 0 ? _b : new GPT3Tokenizer_1.GPT3Tokenizer();
-        this._chunkingConfig.tokenizer = this._tokenizer;
-    }
-    /**
-     * Returns the embeddings model used by the index (if configured.)
-     */
-    get embeddings() {
-        return this._embeddings;
-    }
-    /**
-     * Returns the tokenizer used by the index.
-     */
-    get tokenizer() {
-        return this._tokenizer;
-    }
-    /**
-     * Returns true if the document catalog exists.
-     */
-    isCatalogCreated() {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield fs.access(path.join(this.folderPath, 'catalog.json'));
-                return true;
-            }
-            catch (err) {
-                return false;
-            }
-        });
-    }
-    /**
-     * Returns the document ID for the given URI.
-     * @param uri URI of the document to lookup.
-     * @returns Document ID or undefined if not found.
-     */
-    getDocumentId(uri) {
-        var _a;
-        return __awaiter(this, void 0, void 0, function* () {
-            yield this.loadIndexData();
-            return (_a = this._catalog) === null || _a === void 0 ? void 0 : _a.uriToId[uri];
-        });
-    }
-    /**
-     * Returns the document URI for the given ID.
-     * @param documentId ID of the document to lookup.
-     * @returns Document URI or undefined if not found.
-     */
-    getDocumentUri(documentId) {
-        var _a;
-        return __awaiter(this, void 0, void 0, function* () {
-            yield this.loadIndexData();
-            return (_a = this._catalog) === null || _a === void 0 ? void 0 : _a.idToUri[documentId];
-        });
-    }
-    /**
-     * Loads the document catalog from disk and returns its stats.
-     * @returns Catalog stats.
-     */
-    getCatalogStats() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const stats = yield this.getIndexStats();
-            return {
-                version: this._catalog.version,
-                documents: this._catalog.count,
-                chunks: stats.items,
-                metadata_config: stats.metadata_config
-            };
-        });
-    }
-    /**
-     * Deletes a document from the index.
-     * @param uri URI of the document to delete.
-     */
-    deleteDocument(uri) {
-        return __awaiter(this, void 0, void 0, function* () {
-            // Lookup document ID
-            const documentId = yield this.getDocumentId(uri);
-            if (documentId == undefined) {
-                return;
-            }
-            // Delete document chunks from index and remove from catalog
-            yield this.beginUpdate();
-            try {
-                // Get list of chunks for document
-                const chunks = yield this.listItemsByMetadata({ documentId });
-                // Delete chunks
-                for (const chunk of chunks) {
-                    yield this.deleteItem(chunk.id);
-                }
-                // Remove entry from catalog
-                delete this._newCatalog.uriToId[uri];
-                delete this._newCatalog.idToUri[documentId];
-                this._newCatalog.count--;
-                // Commit changes
-                yield this.endUpdate();
-            }
-            catch (err) {
-                // Cancel update and raise error
-                this.cancelUpdate();
-                throw new Error(`Error deleting document "${uri}": ${err.toString()}`);
-            }
-            // Delete text file from disk
-            try {
-                yield fs.unlink(path.join(this.folderPath, `${documentId}.txt`));
-            }
-            catch (err) {
-                throw new Error(`Error removing text file for document "${uri}" from disk: ${err.toString()}`);
-            }
-            // Delete metadata file from disk
-            try {
-                yield fs.unlink(path.join(this.folderPath, `${documentId}.json`));
-            }
-            catch (err) {
-                // Ignore error
-            }
-        });
-    }
-    /**
-     * Adds a document to the catalog.
-     * @remarks
-     * A new update is started if one is not already in progress. If an document with the same uri
-     * already exists, it will be replaced.
-     * @param uri - Document URI
-     * @param text - Document text
-     * @param docType - Optional. Document type
-     * @param metadata - Optional. Document metadata to index
-     * @returns Inserted document
-     */
-    upsertDocument(uri, text, docType, metadata) {
-        return __awaiter(this, void 0, void 0, function* () {
-            // Ensure embeddings configured
-            if (!this._embeddings) {
-                throw new Error(`Embeddings model not configured.`);
-            }
-            // Check for existing document ID
-            let documentId = yield this.getDocumentId(uri);
-            if (documentId != undefined) {
-                // Delete existing document
-                yield this.deleteDocument(uri);
-            }
-            else {
-                // Generate new document ID
-                documentId = (0, uuid_1.v4)();
-            }
-            // Initialize text splitter settings
-            const config = Object.assign({ docType }, this._chunkingConfig);
-            if (config.docType == undefined) {
-                // Populate docType based on extension
-                const pos = uri.lastIndexOf('.');
-                if (pos >= 0) {
-                    const ext = uri.substring(pos + 1).toLowerCase();
-                    config.docType = ext;
-                }
-            }
-            // Split text into chunks
-            const splitter = new TextSplitter_1.TextSplitter(config);
-            const chunks = splitter.split(text);
-            // Break chunks into batches for embedding generation
-            let totalTokens = 0;
-            const chunkBatches = [];
-            let currentBatch = [];
-            for (const chunk of chunks) {
-                totalTokens += chunk.tokens.length;
-                if (totalTokens > this._embeddings.maxTokens) {
-                    chunkBatches.push(currentBatch);
-                    currentBatch = [];
-                    totalTokens = chunk.tokens.length;
-                }
-                currentBatch.push(chunk.text.replace(/\n/g, ' '));
-            }
-            if (currentBatch.length > 0) {
-                chunkBatches.push(currentBatch);
-            }
-            // Generate embeddings for chunks
-            const embeddings = [];
-            for (const batch of chunkBatches) {
-                let response;
-                try {
-                    response = yield this._embeddings.createEmbeddings(batch);
-                }
-                catch (err) {
-                    throw new Error(`Error generating embeddings: ${err.toString()}`);
-                }
-                // Check for error
-                if (response.status != 'success') {
-                    throw new Error(`Error generating embeddings: ${response.message}`);
-                }
-                // Add embeddings to output
-                for (const embedding of response.output) {
-                    embeddings.push(embedding);
-                }
-            }
-            // Add document chunks to index
-            yield this.beginUpdate();
-            try {
-                // Add chunks to index
-                for (let i = 0; i < chunks.length; i++) {
-                    const chunk = chunks[i];
-                    const embedding = embeddings[i];
-                    const chunkMetadata = Object.assign({
-                        documentId,
-                        startPos: chunk.startPos,
-                        endPos: chunk.endPos,
-                    }, metadata);
-                    yield this.insertItem({
-                        id: (0, uuid_1.v4)(),
-                        metadata: chunkMetadata,
-                        vector: embedding,
-                    });
-                }
-                // Save metadata file to disk
-                if (metadata != undefined) {
-                    yield fs.writeFile(path.join(this.folderPath, `${documentId}.json`), JSON.stringify(metadata));
-                }
-                // Save text file to disk
-                yield fs.writeFile(path.join(this.folderPath, `${documentId}.txt`), text);
-                // Add entry to catalog
-                this._newCatalog.uriToId[uri] = documentId;
-                this._newCatalog.idToUri[documentId] = uri;
-                this._newCatalog.count++;
-                // Commit changes
-                yield this.endUpdate();
-            }
-            catch (err) {
-                // Cancel update and raise error
-                this.cancelUpdate();
-                throw new Error(`Error adding document "${uri}": ${err.toString()}`);
-            }
-            // Return document
-            return new LocalDocument_1.LocalDocument(this, documentId, uri);
-        });
-    }
-    /**
-     * Returns all documents in the index.
-     * @remarks
-     * Each document will contain all of the documents indexed chunks.
-     * @returns Array of documents.
-     */
-    listDocuments() {
-        return __awaiter(this, void 0, void 0, function* () {
-            // Sort chunks by document ID
-            const docs = {};
-            const chunks = yield this.listItems();
-            chunks.forEach(chunk => {
-                const metadata = chunk.metadata;
-                if (docs[metadata.documentId] == undefined) {
-                    docs[metadata.documentId] = [];
-                }
-                docs[metadata.documentId].push({ item: chunk, score: 1.0 });
-            });
-            // Create document results
-            const results = [];
-            for (const documentId in docs) {
-                const uri = yield this.getDocumentUri(documentId);
-                const documentResult = new LocalDocumentResult_1.LocalDocumentResult(this, documentId, uri, docs[documentId], this._tokenizer);
-                results.push(documentResult);
-            }
-            return results;
-        });
-    }
-    /**
-     * Queries the index for documents similar to the given query.
-     * @param query Text to query for.
-     * @param options Optional. Query options.
-     * @returns Array of document results.
-     */
-    queryDocuments(query, options) {
-        return __awaiter(this, void 0, void 0, function* () {
-            // Ensure embeddings configured
-            if (!this._embeddings) {
-                throw new Error(`Embeddings model not configured.`);
-            }
-            // Ensure options are defined
-            options = Object.assign({
-                maxDocuments: 10,
-                maxChunks: 50,
-            }, options);
-            // Generate embeddings for query
-            let embeddings;
-            try {
-                embeddings = yield this._embeddings.createEmbeddings(query.replace(/\n/g, ' '));
-            }
-            catch (err) {
-                throw new Error(`Error generating embeddings for query: ${err.toString()}`);
-            }
-            // Check for error
-            if (embeddings.status != 'success') {
-                throw new Error(`Error generating embeddings for query: ${embeddings.message}`);
-            }
-            // Query index for chunks
-            const results = yield this.queryItems(embeddings.output[0], options.maxChunks, options.filter);
-            // Group chunks by document
-            const documentChunks = {};
-            for (const result of results) {
-                const metadata = result.item.metadata;
-                if (documentChunks[metadata.documentId] == undefined) {
-                    documentChunks[metadata.documentId] = [];
-                }
-                documentChunks[metadata.documentId].push(result);
-            }
-            // Create a document result for each document
-            const documentResults = [];
-            for (const documentId in documentChunks) {
-                const chunks = documentChunks[documentId];
-                const uri = yield this.getDocumentUri(documentId);
-                const documentResult = new LocalDocumentResult_1.LocalDocumentResult(this, documentId, uri, chunks, this._tokenizer);
-                documentResults.push(documentResult);
-            }
-            // Sort document results by score and return top results
-            return documentResults.sort((a, b) => b.score - a.score).slice(0, options.maxDocuments);
-        });
-    }
-    // Overrides
-    beginUpdate() {
-        const _super = Object.create(null, {
-            beginUpdate: { get: () => super.beginUpdate }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            yield _super.beginUpdate.call(this);
-            this._newCatalog = Object.assign({}, this._catalog);
-        });
-    }
-    cancelUpdate() {
-        super.cancelUpdate();
-        this._newCatalog = undefined;
-    }
-    createIndex(config) {
-        const _super = Object.create(null, {
-            createIndex: { get: () => super.createIndex }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            yield _super.createIndex.call(this, config);
-            yield this.loadIndexData();
-        });
-    }
-    endUpdate() {
-        const _super = Object.create(null, {
-            endUpdate: { get: () => super.endUpdate }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            yield _super.endUpdate.call(this);
-            try {
-                // Save catalog
-                yield fs.writeFile(path.join(this.folderPath, 'catalog.json'), JSON.stringify(this._newCatalog));
-                this._catalog = this._newCatalog;
-                this._newCatalog = undefined;
-            }
-            catch (err) {
-                throw new Error(`Error saving document catalog: ${err.toString()}`);
-            }
-        });
-    }
-    loadIndexData() {
-        const _super = Object.create(null, {
-            loadIndexData: { get: () => super.loadIndexData }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            yield _super.loadIndexData.call(this);
-            if (this._catalog) {
-                return;
-            }
-            const catalogPath = path.join(this.folderPath, 'catalog.json');
-            if (yield this.isCatalogCreated()) {
-                // Load catalog
-                const buffer = yield fs.readFile(catalogPath);
-                this._catalog = JSON.parse(buffer.toString());
-            }
-            else {
-                try {
-                    // Initialize catalog
-                    this._catalog = {
-                        version: 1,
-                        count: 0,
-                        uriToId: {},
-                        idToUri: {},
-                    };
-                    yield fs.writeFile(catalogPath, JSON.stringify(this._catalog));
-                }
-                catch (err) {
-                    throw new Error(`Error creating document catalog: ${err.toString()}`);
-                }
-            }
-        });
-    }
-}
-exports.LocalDocumentIndex = LocalDocumentIndex;
-//# sourceMappingURL=LocalDocumentIndex.js.map

package/lib/LocalDocumentIndex.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"LocalDocumentIndex.js","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAC7B,+BAA0B;AAC1B,mDAAgD;AAChD,6CAA6D;AAC7D,iDAAkE;AAElE,+DAA4D;AAC5D,mDAAgD;AAmDhD;;GAEG;AACH,MAAa,kBAAmB,SAAQ,uBAAiC;IAOrE;;;OAGG;IACH,YAAmB,MAAgC;;QAC/C,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;YACjC,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,CAAC;SACI,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,MAAA,MAAA,MAAM,CAAC,SAAS,mCAAI,IAAI,CAAC,eAAe,CAAC,SAAS,mCAAI,IAAI,6BAAa,EAAE,CAAC;QAC5F,IAAI,CAAC,eAAe,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACU,gBAAgB;;YACzB,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC;gBAC5D,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAY,EAAE;gBACnB,OAAO,KAAK,CAAC;aAChB;QACL,CAAC;KAAA;IAED;;;;OAIG;IACU,aAAa,CAAC,GAAW;;;YAClC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,MAAA,IAAI,CAAC,QAAQ,0CAAE,OAAO,CAAC,GAAG,CAAC,CAAC;;KACtC;IAED;;;;OAIG;IACU,cAAc,CAAC,UAAkB;;;YAC1C,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,MAAA,IAAI,CAAC,QAAQ,0CAAE,OAAO,CAAC,UAAU,CAAC,CAAC;;KAC7C;IAED;;;OAGG;IACU,eAAe;;YACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAA;YACxC,OAAO;gBACH,OAAO,EAAE,IAAI,CAAC,QAAS,CAAC,OAAO;gBAC/B,SAAS,EAAE,IAAI,CAAC,QAAS,CAAC,KAAK;gBAC/B,MAAM,EAAE,KAAK,CAAC,KAAK;gBACnB,eAAe,EAAE,KAAK,CAAC,eAAe;aACzC,CAAC;QACN,CAAC;KAAA;IAED;;;OAGG;IACU,cAAc,CAAC,GAAW;;YACnC,qBAAqB;YACrB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACjD,IAAI,UAAU,IAAI,SAAS,EAAE;gBACzB,OAAO;aACV;YAED,4DAA4D;YAC5D,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI;gBACA,kCAAkC;gBAClC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;gBAE9D,gBAAgB;gBAChB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;oBACxB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;iBACnC;gBAED,4BAA4B;gBAC5B,OAAO,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACtC,OAAO,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBAC7C,IAAI,CAAC,WAAY,CAAC,KAAK,EAAE,CAAC;gBAE1B,iBAAiB;gBACjB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;aAC1B;YAAC,OAAO,GAAY,EAAE;gBACnB,gCAAgC;gBAChC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACnF;YAED,6BAA6B;YAC7B,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,MAAM,CAAC,CAAC,CAAC;aACpE;YAAC,OAAO,GAAY,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,gBAAiB,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aAC3G;YAED,iCAAiC;YACjC,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,OAAO,CAAC,CAAC,CAAC;aACrE;YAAC,OAAO,GAAY,EAAE;gBACnB,eAAe;aAClB;QACL,CAAC;KAAA;IAED;;;;;;;;;;OAUG;IACU,cAAc,CAAC,GAAW,EAAE,IAAY,EAAE,OAAgB,EAAE,QAAwC;;YAC7G,+BAA+B;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;aACvD;YAED,iCAAiC;YACjC,IAAI,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC/C,IAAI,UAAU,IAAI,SAAS,EAAE;gBACzB,2BAA2B;gBAC3B,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;aAClC;iBAAM;gBACH,2BAA2B;gBAC3B,UAAU,GAAG,IAAA,SAAE,GAAE,CAAC;aACrB;YAED,oCAAoC;YACpC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YAChE,IAAI,MAAM,CAAC,OAAO,IAAI,SAAS,EAAE;gBAC7B,sCAAsC;gBACtC,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACjC,IAAI,GAAG,IAAI,CAAC,EAAE;oBACV,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;oBACjD,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC;iBACxB;aACJ;YAED,yBAAyB;YACzB,MAAM,QAAQ,GAAG,IAAI,2BAAY,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEpC,qDAAqD;YACrD,IAAI,WAAW,GAAG,CAAC,CAAC;YACpB,MAAM,YAAY,GAAe,EAAE,CAAC;YACpC,IAAI,YAAY,GAAa,EAAE,CAAC;YAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;gBACxB,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;gBACnC,IAAI,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE;oBAC1C,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBAChC,YAAY,GAAG,EAAE,CAAC;oBAClB,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;iBACrC;gBACD,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;aACrD;YACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;gBACzB,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;aACnC;YAED,iCAAiC;YACjC,MAAM,UAAU,GAAe,EAAE,CAAC;YAClC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE;gBAC9B,IAAI,QAA4B,CAAC;gBACjC,IAAI;oBACA,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;iBAC7D;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,gCAAiC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBAC9E;gBAED,kBAAkB;gBAClB,IAAI,QAAQ,CAAC,MAAM,IAAI,SAAS,EAAE;oBAC9B,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;iBACvE;gBAED,2BAA2B;gBAC3B,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,MAAO,EAAE;oBACtC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;iBAC9B;aACJ;YAED,+BAA+B;YAC/B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI;gBACA,sBAAsB;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;oBACxB,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM,aAAa,GAA0B,MAAM,CAAC,MAAM,CAAC;wBACvD,UAAU;wBACV,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;qBACvB,EAAE,QAAQ,CAAC,CAAC;oBACb,MAAM,IAAI,CAAC,UAAU,CAAC;wBAClB,EAAE,EAAE,IAAA,SAAE,GAAE;wBACR,QAAQ,EAAE,aAAa;wBACvB,MAAM,EAAE,SAAS;qBACpB,CAAC,CAAC;iBACN;gBAED,6BAA6B;gBAC7B,IAAI,QAAQ,IAAI,SAAS,EAAE;oBACvB,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;iBAClG;gBAED,yBAAyB;gBACzB,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;gBAE1E,uBAAuB;gBACvB,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;gBAC5C,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC;gBAC5C,IAAI,CAAC,WAAY,CAAC,KAAK,EAAE,CAAC;gBAE1B,iBAAiB;gBACjB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;aAC1B;YAAC,OAAO,GAAY,EAAE;gBACnB,gCAAgC;gBAChC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACjF;YAED,kBAAkB;YAClB,OAAO,IAAI,6BAAa,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;QACpD,CAAC;KAAA;IAED;;;;;OAKG;IACU,aAAa;;YACtB,6BAA6B;YAC7B,MAAM,IAAI,GAAoE,EAAE,CAAC;YACjF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;gBACnB,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;gBAChC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,SAAS,EAAE;oBACxC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;iBAClC;gBACD,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YAChE,CAAC,CAAC,CAAC;YAEH,0BAA0B;YAC1B,MAAM,OAAO,GAA0B,EAAE,CAAC;YAC1C,KAAK,MAAM,UAAU,IAAI,IAAI,EAAE;gBAC3B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAW,CAAC;gBAC5D,MAAM,cAAc,GAAG,IAAI,yCAAmB,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;gBACzG,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;aAChC;YAED,OAAO,OAAO,CAAC;QACnB,CAAC;KAAA;IAED;;;;;OAKG;IACU,cAAc,CAAC,KAAa,EAAE,OAA8B;;YACrE,+BAA+B;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;aACvD;YAED,6BAA6B;YAC7B,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACpB,YAAY,EAAE,EAAE;gBAChB,SAAS,EAAE,EAAE;aAChB,EAAE,OAAO,CAAC,CAAC;YAEZ,gCAAgC;YAChC,IAAI,UAA8B,CAAC;YACnC,IAAI;gBACA,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;aACnF;YAAC,OAAO,GAAY,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,0CAA2C,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACxF;YAED,kBAAkB;YAClB,IAAI,UAAU,CAAC,MAAM,IAAI,SAAS,EAAE;gBAChC,MAAM,IAAI,KAAK,CAAC,0CAA0C,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;aACnF;YAED,yBAAyB;YACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,MAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,SAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAEjG,2BAA2B;YAC3B,MAAM,cAAc,GAAoE,EAAE,CAAC;YAC3F,KAAK,MAAM,MAAM,IAAK,OAAO,EAAE;gBAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACtC,IAAI,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,SAAS,EAAE;oBAClD,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;iBAC5C;gBACD,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACpD;YAED,6CAA6C;YAC7C,MAAM,eAAe,GAA0B,EAAE,CAAC;YAClD,KAAK,MAAM,UAAU,IAAI,cAAc,EAAE;gBACrC,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;gBAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAW,CAAC;gBAC5D,MAAM,cAAc,GAAG,IAAI,yCAAmB,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC/F,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;aACxC;YAED,wDAAwD;YACxD,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,YAAa,CAAC,CAAC;QAC7F,CAAC;KAAA;IAED,YAAY;IAEC,WAAW;;;;;YACpB,MAAM,OAAM,WAAW,WAAE,CAAC;YAC1B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxD,CAAC;KAAA;IAEM,YAAY;QACf,KAAK,CAAC,YAAY,EAAE,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;IACjC,CAAC;IAEY,WAAW,CAAC,MAA0B;;;;;YAC/C,MAAM,OAAM,WAAW,YAAC,MAAM,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/B,CAAC;KAAA;IAEY,SAAS;;;;;YAClB,MAAM,OAAM,SAAS,WAAE,CAAC;YAExB,IAAI;gBACA,eAAe;gBACf,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;gBACjG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC;gBACjC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;aAChC;YAAC,OAAM,GAAY,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,kCAAmC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aAChF;QACL,CAAC;KAAA;IAEe,aAAa;;;;;YACzB,MAAM,OAAM,aAAa,WAAE,CAAC;YAE5B,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACf,OAAO;aACV;YAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;YAC/D,IAAI,MAAM,IAAI,CAAC,gBAAgB,EAAE,EAAE;gBAC/B,eAAe;gBACf,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC9C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACjD;iBAAM;gBACH,IAAI;oBACA,qBAAqB;oBACrB,IAAI,CAAC,QAAQ,GAAG;wBACZ,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,CAAC;wBACR,OAAO,EAAE,EAAE;wBACX,OAAO,EAAE,EAAE;qBACd,CAAC;oBACF,MAAM,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;iBAClE;gBAAC,OAAM,GAAY,EAAE;oBAClB,MAAM,IAAI,KAAK,CAAC,oCAAqC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBAClF;aACJ;QACL,CAAC;KAAA;CACJ;AAhZD,gDAgZC"}

package/lib/LocalDocumentResult.d.ts

@@ -1,45 +0,0 @@
-import { LocalDocument } from "./LocalDocument";
-import { LocalDocumentIndex } from "./LocalDocumentIndex";
-import { QueryResult, DocumentChunkMetadata, Tokenizer, DocumentTextSection } from "./types";
-/**
- * Represents a search result for a document stored on disk.
- */
-export declare class LocalDocumentResult extends LocalDocument {
-    private readonly _chunks;
-    private readonly _tokenizer;
-    private readonly _score;
-    /**
-     * @private
-     * Internal constructor for `LocalDocumentResult` instances.
-     */
-    constructor(index: LocalDocumentIndex, id: string, uri: string, chunks: QueryResult<DocumentChunkMetadata>[], tokenizer: Tokenizer);
-    /**
-     * Returns the chunks of the document that matched the query.
-     */
-    get chunks(): QueryResult<DocumentChunkMetadata>[];
-    /**
-     * Returns the average score of the document result.
-     */
-    get score(): number;
-    /**
-     * Renders all of the results chunks as spans of text (sections.)
-     * @remarks
-     * The returned sections will be sorted by document order and limited to maxTokens in length.
-     * @param maxTokens Maximum number of tokens per section.
-     * @returns Array of rendered text sections.
-     */
-    renderAllSections(maxTokens: number): Promise<DocumentTextSection[]>;
-    /**
-     * Renders the top spans of text (sections) of the document based on the query result.
-     * @remarks
-     * The returned sections will be sorted by relevance and limited to the top `maxSections`.
-     * @param maxTokens Maximum number of tokens per section.
-     * @param maxSections Maximum number of sections to return.
-     * @param overlappingChunks Optional. If true, overlapping chunks of text will be added to each section until the maxTokens is reached.
-     * @returns Array of rendered text sections.
-     */
-    renderSections(maxTokens: number, maxSections: number, overlappingChunks?: boolean): Promise<DocumentTextSection[]>;
-    private encodeBeforeText;
-    private encodeAfterText;
-}
-//# sourceMappingURL=LocalDocumentResult.d.ts.map

package/lib/LocalDocumentResult.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"LocalDocumentResult.d.ts","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,qBAAqB,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAE7F;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,aAAa;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAEhC;;;OAGG;gBACgB,KAAK,EAAE,kBAAkB,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,qBAAqB,CAAC,EAAE,EAAE,SAAS,EAAE,SAAS;IAWzI;;OAEG;IACH,IAAW,MAAM,IAAI,WAAW,CAAC,qBAAqB,CAAC,EAAE,CAExD;IAED;;OAEG;IACH,IAAW,KAAK,IAAI,MAAM,CAEzB;IAED;;;;;;OAMG;IACU,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IA8DjF;;;;;;;;OAQG;IACU,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,iBAAiB,UAAO,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAmK7H,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;CAM1B"}