vectra 0.12.2 → 0.12.3

package/lib/LocalDocumentIndex.d.ts

@@ -0,0 +1,132 @@
+import { CreateIndexConfig, LocalIndex } from "./LocalIndex";
+import { TextSplitterConfig } from "./TextSplitter";
+import { MetadataFilter, EmbeddingsModel, Tokenizer, MetadataTypes, DocumentChunkMetadata, DocumentCatalogStats } from "./types";
+import { LocalDocumentResult } from './LocalDocumentResult';
+import { LocalDocument } from './LocalDocument';
+/**
+ * Options for querying documents in the index.
+ */
+export interface DocumentQueryOptions {
+    /**
+     * Optional. Maximum number of documents to return.
+     * @remarks
+     * Default is 10.
+     */
+    maxDocuments?: number;
+    /**
+     * Maximum number of chunks to return per document.
+     * @remarks
+     * Default is 50.
+     */
+    maxChunks?: number;
+    /**
+     * Optional. Filter to apply to the document metadata.
+     */
+    filter?: MetadataFilter;
+    /**
+     * Optional. Turn on bm25 keyword search to perform hybrid search - semantic + keyword
+     */
+    isBm25?: boolean;
+}
+/**
+ * Configuration settings for a local document index.
+ */
+export interface LocalDocumentIndexConfig {
+    /**
+     * Folder path where the index is stored.
+     */
+    folderPath: string;
+    /**
+     * Optional. Embeddings model to use for generating document embeddings.
+     */
+    embeddings?: EmbeddingsModel;
+    /**
+     * Optional. Tokenizer to use for splitting text into tokens.
+     */
+    tokenizer?: Tokenizer;
+    /**
+     * Optional. Configuration settings for splitting text into chunks.
+     */
+    chunkingConfig?: Partial<TextSplitterConfig>;
+}
+/**
+ * Represents a local index of documents stored on disk.
+ */
+export declare class LocalDocumentIndex extends LocalIndex<DocumentChunkMetadata> {
+    private readonly _embeddings?;
+    private readonly _tokenizer;
+    private readonly _chunkingConfig?;
+    private _catalog?;
+    private _newCatalog?;
+    /**
+     * Creates a new `LocalDocumentIndex` instance.
+     * @param config Configuration settings for the document index.
+     */
+    constructor(config: LocalDocumentIndexConfig);
+    /**
+     * Returns the embeddings model used by the index (if configured.)
+     */
+    get embeddings(): EmbeddingsModel | undefined;
+    /**
+     * Returns the tokenizer used by the index.
+     */
+    get tokenizer(): Tokenizer;
+    /**
+     * Returns true if the document catalog exists.
+     */
+    isCatalogCreated(): Promise<boolean>;
+    /**
+     * Returns the document ID for the given URI.
+     * @param uri URI of the document to lookup.
+     * @returns Document ID or undefined if not found.
+     */
+    getDocumentId(uri: string): Promise<string | undefined>;
+    /**
+     * Returns the document URI for the given ID.
+     * @param documentId ID of the document to lookup.
+     * @returns Document URI or undefined if not found.
+     */
+    getDocumentUri(documentId: string): Promise<string | undefined>;
+    /**
+     * Loads the document catalog from disk and returns its stats.
+     * @returns Catalog stats.
+     */
+    getCatalogStats(): Promise<DocumentCatalogStats>;
+    /**
+     * Deletes a document from the index.
+     * @param uri URI of the document to delete.
+     */
+    deleteDocument(uri: string): Promise<void>;
+    /**
+     * Adds a document to the catalog.
+     * @remarks
+     * A new update is started if one is not already in progress. If an document with the same uri
+     * already exists, it will be replaced.
+     * @param uri - Document URI
+     * @param text - Document text
+     * @param docType - Optional. Document type
+     * @param metadata - Optional. Document metadata to index
+     * @returns Inserted document
+     */
+    upsertDocument(uri: string, text: string, docType?: string, metadata?: Record<string, MetadataTypes>): Promise<LocalDocument>;
+    /**
+     * Returns all documents in the index.
+     * @remarks
+     * Each document will contain all of the documents indexed chunks.
+     * @returns Array of documents.
+     */
+    listDocuments(): Promise<LocalDocumentResult[]>;
+    /**
+     * Queries the index for documents similar to the given query.
+     * @param query Text to query for.
+     * @param options Optional. Query options.
+     * @returns Array of document results.
+     */
+    queryDocuments(query: string, options?: DocumentQueryOptions): Promise<LocalDocumentResult[]>;
+    beginUpdate(): Promise<void>;
+    cancelUpdate(): void;
+    createIndex(config?: CreateIndexConfig): Promise<void>;
+    endUpdate(): Promise<void>;
+    protected loadIndexData(): Promise<void>;
+}
+//# sourceMappingURL=LocalDocumentIndex.d.ts.map

package/lib/LocalDocumentIndex.js

@@ -0,0 +1,456 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalDocumentIndex = void 0;
+const fs = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
+const uuid_1 = require("uuid");
+const GPT3Tokenizer_1 = require("./GPT3Tokenizer");
+const LocalIndex_1 = require("./LocalIndex");
+const TextSplitter_1 = require("./TextSplitter");
+const LocalDocumentResult_1 = require("./LocalDocumentResult");
+const LocalDocument_1 = require("./LocalDocument");
+/**
+ * Represents a local index of documents stored on disk.
+ */
+class LocalDocumentIndex extends LocalIndex_1.LocalIndex {
+    /**
+     * Creates a new `LocalDocumentIndex` instance.
+     * @param config Configuration settings for the document index.
+     */
+    constructor(config) {
+        var _a, _b;
+        super(config.folderPath);
+        this._embeddings = config.embeddings;
+        this._chunkingConfig = Object.assign({
+            keepSeparators: true,
+            chunkSize: 512,
+            chunkOverlap: 0,
+        }, config.chunkingConfig);
+        this._tokenizer = (_b = (_a = config.tokenizer) !== null && _a !== void 0 ? _a : this._chunkingConfig.tokenizer) !== null && _b !== void 0 ? _b : new GPT3Tokenizer_1.GPT3Tokenizer();
+        this._chunkingConfig.tokenizer = this._tokenizer;
+    }
+    /**
+     * Returns the embeddings model used by the index (if configured.)
+     */
+    get embeddings() {
+        return this._embeddings;
+    }
+    /**
+     * Returns the tokenizer used by the index.
+     */
+    get tokenizer() {
+        return this._tokenizer;
+    }
+    /**
+     * Returns true if the document catalog exists.
+     */
+    isCatalogCreated() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield fs.access(path.join(this.folderPath, 'catalog.json'));
+                return true;
+            }
+            catch (err) {
+                return false;
+            }
+        });
+    }
+    /**
+     * Returns the document ID for the given URI.
+     * @param uri URI of the document to lookup.
+     * @returns Document ID or undefined if not found.
+     */
+    getDocumentId(uri) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            yield this.loadIndexData();
+            return (_a = this._catalog) === null || _a === void 0 ? void 0 : _a.uriToId[uri];
+        });
+    }
+    /**
+     * Returns the document URI for the given ID.
+     * @param documentId ID of the document to lookup.
+     * @returns Document URI or undefined if not found.
+     */
+    getDocumentUri(documentId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            yield this.loadIndexData();
+            return (_a = this._catalog) === null || _a === void 0 ? void 0 : _a.idToUri[documentId];
+        });
+    }
+    /**
+     * Loads the document catalog from disk and returns its stats.
+     * @returns Catalog stats.
+     */
+    getCatalogStats() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const stats = yield this.getIndexStats();
+            return {
+                version: this._catalog.version,
+                documents: this._catalog.count,
+                chunks: stats.items,
+                metadata_config: stats.metadata_config
+            };
+        });
+    }
+    /**
+     * Deletes a document from the index.
+     * @param uri URI of the document to delete.
+     */
+    deleteDocument(uri) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Lookup document ID
+            const documentId = yield this.getDocumentId(uri);
+            if (documentId == undefined) {
+                return;
+            }
+            // Delete document chunks from index and remove from catalog
+            yield this.beginUpdate();
+            try {
+                // Get list of chunks for document
+                const chunks = yield this.listItemsByMetadata({ documentId });
+                // Delete chunks
+                for (const chunk of chunks) {
+                    yield this.deleteItem(chunk.id);
+                }
+                // Remove entry from catalog
+                delete this._newCatalog.uriToId[uri];
+                delete this._newCatalog.idToUri[documentId];
+                this._newCatalog.count--;
+                // Commit changes
+                yield this.endUpdate();
+            }
+            catch (err) {
+                // Cancel update and raise error
+                this.cancelUpdate();
+                throw new Error(`Error deleting document "${uri}": ${err.toString()}`);
+            }
+            // Delete text file from disk
+            try {
+                yield fs.unlink(path.join(this.folderPath, `${documentId}.txt`));
+            }
+            catch (err) {
+                throw new Error(`Error removing text file for document "${uri}" from disk: ${err.toString()}`);
+            }
+            // Delete metadata file from disk
+            try {
+                yield fs.unlink(path.join(this.folderPath, `${documentId}.json`));
+            }
+            catch (err) {
+                // Ignore error
+            }
+        });
+    }
+    /**
+     * Adds a document to the catalog.
+     * @remarks
+     * A new update is started if one is not already in progress. If an document with the same uri
+     * already exists, it will be replaced.
+     * @param uri - Document URI
+     * @param text - Document text
+     * @param docType - Optional. Document type
+     * @param metadata - Optional. Document metadata to index
+     * @returns Inserted document
+     */
+    upsertDocument(uri, text, docType, metadata) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Ensure embeddings configured
+            if (!this._embeddings) {
+                throw new Error(`Embeddings model not configured.`);
+            }
+            // Check for existing document ID
+            let documentId = yield this.getDocumentId(uri);
+            if (documentId != undefined) {
+                // Delete existing document
+                yield this.deleteDocument(uri);
+            }
+            else {
+                // Generate new document ID
+                documentId = (0, uuid_1.v4)();
+            }
+            // Initialize text splitter settings
+            const config = Object.assign({ docType }, this._chunkingConfig);
+            if (config.docType == undefined) {
+                // Populate docType based on extension
+                const pos = uri.lastIndexOf('.');
+                if (pos >= 0) {
+                    const ext = uri.substring(pos + 1).toLowerCase();
+                    config.docType = ext;
+                }
+            }
+            // Split text into chunks
+            const splitter = new TextSplitter_1.TextSplitter(config);
+            const chunks = splitter.split(text);
+            // Break chunks into batches for embedding generation
+            let totalTokens = 0;
+            const chunkBatches = [];
+            let currentBatch = [];
+            for (const chunk of chunks) {
+                totalTokens += chunk.tokens.length;
+                if (totalTokens > this._embeddings.maxTokens) {
+                    chunkBatches.push(currentBatch);
+                    currentBatch = [];
+                    totalTokens = chunk.tokens.length;
+                }
+                currentBatch.push(chunk.text.replace(/\n/g, ' '));
+            }
+            if (currentBatch.length > 0) {
+                chunkBatches.push(currentBatch);
+            }
+            // Generate embeddings for chunks
+            const embeddings = [];
+            for (const batch of chunkBatches) {
+                let response;
+                try {
+                    response = yield this._embeddings.createEmbeddings(batch);
+                }
+                catch (err) {
+                    throw new Error(`Error generating embeddings: ${err.toString()}`);
+                }
+                // Check for error
+                if (response.status != 'success') {
+                    throw new Error(`Error generating embeddings: ${response.message}`);
+                }
+                // Add embeddings to output
+                for (const embedding of response.output) {
+                    embeddings.push(embedding);
+                }
+            }
+            // Add document chunks to index
+            yield this.beginUpdate();
+            try {
+                // Add chunks to index
+                for (let i = 0; i < chunks.length; i++) {
+                    const chunk = chunks[i];
+                    const embedding = embeddings[i];
+                    const chunkMetadata = Object.assign({
+                        documentId,
+                        startPos: chunk.startPos,
+                        endPos: chunk.endPos,
+                    }, metadata);
+                    yield this.insertItem({
+                        id: (0, uuid_1.v4)(),
+                        metadata: chunkMetadata,
+                        vector: embedding,
+                    });
+                }
+                // Save metadata file to disk
+                if (metadata != undefined) {
+                    yield fs.writeFile(path.join(this.folderPath, `${documentId}.json`), JSON.stringify(metadata));
+                }
+                // Save text file to disk
+                yield fs.writeFile(path.join(this.folderPath, `${documentId}.txt`), text);
+                // Add entry to catalog
+                this._newCatalog.uriToId[uri] = documentId;
+                this._newCatalog.idToUri[documentId] = uri;
+                this._newCatalog.count++;
+                // Commit changes
+                yield this.endUpdate();
+            }
+            catch (err) {
+                // Cancel update and raise error
+                this.cancelUpdate();
+                throw new Error(`Error adding document "${uri}": ${err.toString()}`);
+            }
+            // Return document
+            return new LocalDocument_1.LocalDocument(this, documentId, uri);
+        });
+    }
+    /**
+     * Returns all documents in the index.
+     * @remarks
+     * Each document will contain all of the documents indexed chunks.
+     * @returns Array of documents.
+     */
+    listDocuments() {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Sort chunks by document ID
+            const docs = {};
+            const chunks = yield this.listItems();
+            chunks.forEach(chunk => {
+                const metadata = chunk.metadata;
+                if (docs[metadata.documentId] == undefined) {
+                    docs[metadata.documentId] = [];
+                }
+                docs[metadata.documentId].push({ item: chunk, score: 1.0 });
+            });
+            // Create document results
+            const results = [];
+            for (const documentId in docs) {
+                const uri = yield this.getDocumentUri(documentId);
+                const documentResult = new LocalDocumentResult_1.LocalDocumentResult(this, documentId, uri, docs[documentId], this._tokenizer);
+                results.push(documentResult);
+            }
+            return results;
+        });
+    }
+    /**
+     * Queries the index for documents similar to the given query.
+     * @param query Text to query for.
+     * @param options Optional. Query options.
+     * @returns Array of document results.
+     */
+    queryDocuments(query, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Ensure embeddings configured
+            if (!this._embeddings) {
+                throw new Error(`Embeddings model not configured.`);
+            }
+            // Ensure options are defined
+            options = Object.assign({
+                maxDocuments: 10,
+                maxChunks: 50,
+            }, options);
+            // Generate embeddings for query
+            let embeddings;
+            try {
+                embeddings = yield this._embeddings.createEmbeddings(query.replace(/\n/g, ' '));
+            }
+            catch (err) {
+                throw new Error(`Error generating embeddings for query: ${err.toString()}`);
+            }
+            // Check for error
+            if (embeddings.status != 'success') {
+                throw new Error(`Error generating embeddings for query: ${embeddings.message}`);
+            }
+            // Query index for chunks
+            const results = yield this.queryItems(embeddings.output[0], query, options.maxChunks, options.filter, options.isBm25);
+            // Group chunks by document
+            const documentChunks = {};
+            for (const result of results) {
+                const metadata = result.item.metadata;
+                if (documentChunks[metadata.documentId] == undefined) {
+                    documentChunks[metadata.documentId] = [];
+                }
+                documentChunks[metadata.documentId].push(result);
+            }
+            // Create a document result for each document
+            const documentResults = [];
+            for (const documentId in documentChunks) {
+                const chunks = documentChunks[documentId];
+                const uri = yield this.getDocumentUri(documentId);
+                const documentResult = new LocalDocumentResult_1.LocalDocumentResult(this, documentId, uri, chunks, this._tokenizer);
+                documentResults.push(documentResult);
+            }
+            // Sort document results by score and return top results
+            return documentResults.sort((a, b) => b.score - a.score).slice(0, options.maxDocuments);
+        });
+    }
+    // Overrides
+    beginUpdate() {
+        const _super = Object.create(null, {
+            beginUpdate: { get: () => super.beginUpdate }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.beginUpdate.call(this);
+            this._newCatalog = Object.assign({}, this._catalog);
+        });
+    }
+    cancelUpdate() {
+        super.cancelUpdate();
+        this._newCatalog = undefined;
+    }
+    createIndex(config) {
+        const _super = Object.create(null, {
+            createIndex: { get: () => super.createIndex }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.createIndex.call(this, config);
+            yield this.loadIndexData();
+        });
+    }
+    endUpdate() {
+        const _super = Object.create(null, {
+            endUpdate: { get: () => super.endUpdate }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.endUpdate.call(this);
+            try {
+                // Save catalog
+                yield fs.writeFile(path.join(this.folderPath, 'catalog.json'), JSON.stringify(this._newCatalog));
+                this._catalog = this._newCatalog;
+                this._newCatalog = undefined;
+            }
+            catch (err) {
+                throw new Error(`Error saving document catalog: ${err.toString()}`);
+            }
+        });
+    }
+    loadIndexData() {
+        const _super = Object.create(null, {
+            loadIndexData: { get: () => super.loadIndexData }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.loadIndexData.call(this);
+            if (this._catalog) {
+                return;
+            }
+            const catalogPath = path.join(this.folderPath, 'catalog.json');
+            if (yield this.isCatalogCreated()) {
+                // Load catalog
+                const buffer = yield fs.readFile(catalogPath);
+                this._catalog = JSON.parse(buffer.toString());
+            }
+            else {
+                try {
+                    // Initialize catalog
+                    this._catalog = {
+                        version: 1,
+                        count: 0,
+                        uriToId: {},
+                        idToUri: {},
+                    };
+                    yield fs.writeFile(catalogPath, JSON.stringify(this._catalog));
+                }
+                catch (err) {
+                    throw new Error(`Error creating document catalog: ${err.toString()}`);
+                }
+            }
+        });
+    }
+}
+exports.LocalDocumentIndex = LocalDocumentIndex;
+//# sourceMappingURL=LocalDocumentIndex.js.map

package/lib/LocalDocumentIndex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocumentIndex.js","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAC7B,+BAA0B;AAC1B,mDAAgD;AAChD,6CAA6D;AAC7D,iDAAkE;AAElE,+DAA4D;AAC5D,mDAAgD;AAyDhD;;GAEG;AACH,MAAa,kBAAmB,SAAQ,uBAAiC;IAOrE;;;OAGG;IACH,YAAmB,MAAgC;;QAC/C,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;YACjC,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,CAAC;SACI,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,MAAA,MAAA,MAAM,CAAC,SAAS,mCAAI,IAAI,CAAC,eAAe,CAAC,SAAS,mCAAI,IAAI,6BAAa,EAAE,CAAC;QAC5F,IAAI,CAAC,eAAe,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACU,gBAAgB;;YACzB,IAAI,CAAC;gBACD,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC;gBAC5D,OAAO,IAAI,CAAC;YAChB,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACpB,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;KAAA;IAED;;;;OAIG;IACU,aAAa,CAAC,GAAW;;;YAClC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,MAAA,IAAI,CAAC,QAAQ,0CAAE,OAAO,CAAC,GAAG,CAAC,CAAC;QACvC,CAAC;KAAA;IAED;;;;OAIG;IACU,cAAc,CAAC,UAAkB;;;YAC1C,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,MAAA,IAAI,CAAC,QAAQ,0CAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QAC9C,CAAC;KAAA;IAED;;;OAGG;IACU,eAAe;;YACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAA;YACxC,OAAO;gBACH,OAAO,EAAE,IAAI,CAAC,QAAS,CAAC,OAAO;gBAC/B,SAAS,EAAE,IAAI,CAAC,QAAS,CAAC,KAAK;gBAC/B,MAAM,EAAE,KAAK,CAAC,KAAK;gBACnB,eAAe,EAAE,KAAK,CAAC,eAAe;aACzC,CAAC;QACN,CAAC;KAAA;IAED;;;OAGG;IACU,cAAc,CAAC,GAAW;;YACnC,qBAAqB;YACrB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACjD,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;gBAC1B,OAAO;YACX,CAAC;YAED,4DAA4D;YAC5D,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI,CAAC;gBACD,kCAAkC;gBAClC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;gBAE9D,gBAAgB;gBAChB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBACzB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACpC,CAAC;gBAED,4BAA4B;gBAC5B,OAAO,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACtC,OAAO,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBAC7C,IAAI,CAAC,WAAY,CAAC,KAAK,EAAE,CAAC;gBAE1B,iBAAiB;gBACjB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YAC3B,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACpB,gCAAgC;gBAChC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACpF,CAAC;YAED,6BAA6B;YAC7B,IAAI,CAAC;gBACD,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,MAAM,CAAC,CAAC,CAAC;YACrE,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,gBAAiB,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC5G,CAAC;YAED,iCAAiC;YACjC,IAAI,CAAC;gBACD,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,OAAO,CAAC,CAAC,CAAC;YACtE,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACpB,eAAe;YACnB,CAAC;QACL,CAAC;KAAA;IAED;;;;;;;;;;OAUG;IACU,cAAc,CAAC,GAAW,EAAE,IAAY,EAAE,OAAgB,EAAE,QAAwC;;YAC7G,+BAA+B;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;YACxD,CAAC;YAED,iCAAiC;YACjC,IAAI,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC/C,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;gBAC1B,2BAA2B;gBAC3B,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACJ,2BAA2B;gBAC3B,UAAU,GAAG,IAAA,SAAE,GAAE,CAAC;YACtB,CAAC;YAED,oCAAoC;YACpC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YAChE,IAAI,MAAM,CAAC,OAAO,IAAI,SAAS,EAAE,CAAC;gBAC9B,sCAAsC;gBACtC,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACjC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;oBACX,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;oBACjD,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC;gBACzB,CAAC;YACL,CAAC;YAED,yBAAyB;YACzB,MAAM,QAAQ,GAAG,IAAI,2BAAY,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEpC,qDAAqD;YACrD,IAAI,WAAW,GAAG,CAAC,CAAC;YACpB,MAAM,YAAY,GAAe,EAAE,CAAC;YACpC,IAAI,YAAY,GAAa,EAAE,CAAC;YAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACzB,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;gBACnC,IAAI,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;oBAC3C,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBAChC,YAAY,GAAG,EAAE,CAAC;oBAClB,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;gBACtC,CAAC;gBACD,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACpC,CAAC;YAED,iCAAiC;YACjC,MAAM,UAAU,GAAe,EAAE,CAAC;YAClC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBAC/B,IAAI,QAA4B,CAAC;gBACjC,IAAI,CAAC;oBACD,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;gBAC9D,CAAC;gBAAC,OAAO,GAAY,EAAE,CAAC;oBACpB,MAAM,IAAI,KAAK,CAAC,gCAAiC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBAC/E,CAAC;gBAED,kBAAkB;gBAClB,IAAI,QAAQ,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;oBAC/B,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;gBACxE,CAAC;gBAED,2BAA2B;gBAC3B,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,MAAO,EAAE,CAAC;oBACvC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC/B,CAAC;YACL,CAAC;YAED,+BAA+B;YAC/B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI,CAAC;gBACD,sBAAsB;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACrC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;oBACxB,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM,aAAa,GAA0B,MAAM,CAAC,MAAM,CAAC;wBACvD,UAAU;wBACV,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;qBACvB,EAAE,QAAQ,CAAC,CAAC;oBACb,MAAM,IAAI,CAAC,UAAU,CAAC;wBAClB,EAAE,EAAE,IAAA,SAAE,GAAE;wBACR,QAAQ,EAAE,aAAa;wBACvB,MAAM,EAAE,SAAS;qBACpB,CAAC,CAAC;gBACP,CAAC;gBAED,6BAA6B;gBAC7B,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;oBACxB,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACnG,CAAC;gBAED,yBAAyB;gBACzB,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;gBAE1E,uBAAuB;gBACvB,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;gBAC5C,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC;gBAC5C,IAAI,CAAC,WAAY,CAAC,KAAK,EAAE,CAAC;gBAE1B,iBAAiB;gBACjB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YAC3B,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACpB,gCAAgC;gBAChC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAClF,CAAC;YAED,kBAAkB;YAClB,OAAO,IAAI,6BAAa,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;QACpD,CAAC;KAAA;IAED;;;;;OAKG;IACU,aAAa;;YACtB,6BAA6B;YAC7B,MAAM,IAAI,GAAoE,EAAE,CAAC;YACjF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;gBACnB,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;gBAChC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,SAAS,EAAE,CAAC;oBACzC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;gBACnC,CAAC;gBACD,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YAChE,CAAC,CAAC,CAAC;YAEH,0BAA0B;YAC1B,MAAM,OAAO,GAA0B,EAAE,CAAC;YAC1C,KAAK,MAAM,UAAU,IAAI,IAAI,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAW,CAAC;gBAC5D,MAAM,cAAc,GAAG,IAAI,yCAAmB,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;gBACzG,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACjC,CAAC;YAED,OAAO,OAAO,CAAC;QACnB,CAAC;KAAA;IAED;;;;;OAKG;IACU,cAAc,CAAC,KAAa,EAAE,OAA8B;;YACrE,+BAA+B;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;YACxD,CAAC;YAED,6BAA6B;YAC7B,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACpB,YAAY,EAAE,EAAE;gBAChB,SAAS,EAAE,EAAE;aAChB,EAAE,OAAO,CAAC,CAAC;YAEZ,gCAAgC;YAChC,IAAI,UAA8B,CAAC;YACnC,IAAI,CAAC;gBACD,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACpF,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,0CAA2C,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACzF,CAAC;YAED,kBAAkB;YAClB,IAAI,UAAU,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,0CAA0C,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;YACpF,CAAC;YAED,yBAAyB;YACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,MAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,SAAU,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAExH,2BAA2B;YAC3B,MAAM,cAAc,GAAoE,EAAE,CAAC;YAC3F,KAAK,MAAM,MAAM,IAAK,OAAO,EAAE,CAAC;gBAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACtC,IAAI,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,SAAS,EAAE,CAAC;oBACnD,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;gBAC7C,CAAC;gBACD,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrD,CAAC;YAED,6CAA6C;YAC7C,MAAM,eAAe,GAA0B,EAAE,CAAC;YAClD,KAAK,MAAM,UAAU,IAAI,cAAc,EAAE,CAAC;gBACtC,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;gBAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAW,CAAC;gBAC5D,MAAM,cAAc,GAAG,IAAI,yCAAmB,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC/F,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACzC,CAAC;YAED,wDAAwD;YACxD,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,YAAa,CAAC,CAAC;QAC7F,CAAC;KAAA;IAED,YAAY;IAEC,WAAW;;;;;YACpB,MAAM,OAAM,WAAW,WAAE,CAAC;YAC1B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxD,CAAC;KAAA;IAEM,YAAY;QACf,KAAK,CAAC,YAAY,EAAE,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;IACjC,CAAC;IAEY,WAAW,CAAC,MAA0B;;;;;YAC/C,MAAM,OAAM,WAAW,YAAC,MAAM,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/B,CAAC;KAAA;IAEY,SAAS;;;;;YAClB,MAAM,OAAM,SAAS,WAAE,CAAC;YAExB,IAAI,CAAC;gBACD,eAAe;gBACf,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;gBACjG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC;gBACjC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;YACjC,CAAC;YAAC,OAAM,GAAY,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,kCAAmC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACjF,CAAC;QACL,CAAC;KAAA;IAEe,aAAa;;;;;YACzB,MAAM,OAAM,aAAa,WAAE,CAAC;YAE5B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,OAAO;YACX,CAAC;YAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;YAC/D,IAAI,MAAM,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;gBAChC,eAAe;gBACf,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC9C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;YAClD,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC;oBACD,qBAAqB;oBACrB,IAAI,CAAC,QAAQ,GAAG;wBACZ,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,CAAC;wBACR,OAAO,EAAE,EAAE;wBACX,OAAO,EAAE,EAAE;qBACd,CAAC;oBACF,MAAM,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACnE,CAAC;gBAAC,OAAM,GAAY,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,oCAAqC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBACnF,CAAC;YACL,CAAC;QACL,CAAC;KAAA;CACJ;AAhZD,gDAgZC"}

package/lib/LocalDocumentResult.d.ts

@@ -0,0 +1,45 @@
+import { LocalDocument } from "./LocalDocument";
+import { LocalDocumentIndex } from "./LocalDocumentIndex";
+import { QueryResult, DocumentChunkMetadata, Tokenizer, DocumentTextSection } from "./types";
+/**
+ * Represents a search result for a document stored on disk.
+ */
+export declare class LocalDocumentResult extends LocalDocument {
+    private readonly _chunks;
+    private readonly _tokenizer;
+    private readonly _score;
+    /**
+     * @private
+     * Internal constructor for `LocalDocumentResult` instances.
+     */
+    constructor(index: LocalDocumentIndex, id: string, uri: string, chunks: QueryResult<DocumentChunkMetadata>[], tokenizer: Tokenizer);
+    /**
+     * Returns the chunks of the document that matched the query.
+     */
+    get chunks(): QueryResult<DocumentChunkMetadata>[];
+    /**
+     * Returns the average score of the document result.
+     */
+    get score(): number;
+    /**
+     * Renders all of the results chunks as spans of text (sections.)
+     * @remarks
+     * The returned sections will be sorted by document order and limited to maxTokens in length.
+     * @param maxTokens Maximum number of tokens per section.
+     * @returns Array of rendered text sections.
+     */
+    renderAllSections(maxTokens: number): Promise<DocumentTextSection[]>;
+    /**
+     * Renders the top spans of text (sections) of the document based on the query result.
+     * @remarks
+     * The returned sections will be sorted by relevance and limited to the top `maxSections`.
+     * @param maxTokens Maximum number of tokens per section.
+     * @param maxSections Maximum number of sections to return.
+     * @param overlappingChunks Optional. If true, overlapping chunks of text will be added to each section until the maxTokens is reached.
+     * @returns Array of rendered text sections.
+     */
+    renderSections(maxTokens: number, maxSections: number, overlappingChunks?: boolean): Promise<DocumentTextSection[]>;
+    private encodeBeforeText;
+    private encodeAfterText;
+}
+//# sourceMappingURL=LocalDocumentResult.d.ts.map