vectra 0.1.1 → 0.2.0

package/README.md

@@ -1,9 +1,9 @@
-# vectra
+# Vectra
 Vectra is a local vector database for Node.js with features similar to [Pinecone](https://www.pinecone.io/) or [Qdrant](https://qdrant.tech/) but built using local files. Each Vectra index is a folder on disk. There's an `index.json` file in the folder that contains all the vectors for the index along with any indexed metadata.  When you create an index you can specify which metadata properties to index and only those fields will be stored in the `index.json` file. All of the other metadata for an item will be stored on disk in a separate file keyed by a GUID.
 
 When queryng Vectra you'll be able to use the same subset of [Mongo DB query operators](https://www.mongodb.com/docs/manual/reference/operator/query/) that Pinecone supports and the results will be returned sorted by simularity. Every item in the index will first be filtered by metadata and then ranked for simularity. Even though every item is evaluated its all in memory so it should by nearly instantanious. Likely 1ms - 2ms for even a rather large index. Smaller indexes should be <1ms.
 
-Keep in mind that your entire Vectra index is loaded into memory so it's not well suited for scenarios like long term chat bot memory. Use a really vector DB for that. Vectra is intended to be used in scenarios where you have a small corpus of mostly static data you'd like to include in your prompt. Infinite few shot examples would be a great use case for Vectra or even just a single document you want to ask questions over.
+Keep in mind that your entire Vectra index is loaded into memory so it's not well suited for scenarios like long term chat bot memory. Use a real vector DB for that. Vectra is intended to be used in scenarios where you have a small corpus of mostly static data that you'd like to include in your prompt. Infinite few shot examples would be a great use case for Vectra or even just a single document you want to ask questions over.
 
 Pinecone style namespaces aren't directly supported but you could easily mimic them by creating a separate Vectra index (and folder) for each namespace.
 
@@ -92,4 +92,4 @@ await query('banana');
 [0.8493374123092652] oranges
 [0.8415324469533297] blue
 */
-```
+```

package/bin/vectra.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+var app = require('../lib/vectra-cli.js');
+app.run();

package/lib/GPT3Tokenizer.d.ts

@@ -0,0 +1,9 @@
+import { Tokenizer } from "./types";
+/**
+ * Tokenizer that uses GPT-3's encoder.
+ */
+export declare class GPT3Tokenizer implements Tokenizer {
+    decode(tokens: number[]): string;
+    encode(text: string): number[];
+}
+//# sourceMappingURL=GPT3Tokenizer.d.ts.map

package/lib/GPT3Tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"GPT3Tokenizer.d.ts","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGpC;;GAEG;AACH,qBAAa,aAAc,YAAW,SAAS;IACpC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAIhC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;CAGxC"}

package/lib/GPT3Tokenizer.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GPT3Tokenizer = void 0;
+const gpt_3_encoder_1 = require("gpt-3-encoder");
+/**
+ * Tokenizer that uses GPT-3's encoder.
+ */
+class GPT3Tokenizer {
+    decode(tokens) {
+        return (0, gpt_3_encoder_1.decode)(tokens);
+    }
+    encode(text) {
+        return (0, gpt_3_encoder_1.encode)(text);
+    }
+}
+exports.GPT3Tokenizer = GPT3Tokenizer;
+//# sourceMappingURL=GPT3Tokenizer.js.map

package/lib/GPT3Tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GPT3Tokenizer.js","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":";;;AACA,iDAA+C;AAE/C;;GAEG;AACH,MAAa,aAAa;IACf,MAAM,CAAC,MAAgB;QAC1B,OAAO,IAAA,sBAAM,EAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAA,sBAAM,EAAC,IAAI,CAAC,CAAC;IACxB,CAAC;CACJ;AARD,sCAQC"}

package/lib/ItemSelector.d.ts

@@ -0,0 +1,41 @@
+import { MetadataFilter, MetadataTypes } from './types';
+export declare class ItemSelector {
+    /**
+     * Returns the similarity between two vectors using the cosine similarity.
+     * @param vector1 Vector 1
+     * @param vector2 Vector 2
+     * @returns Similarity between the two vectors
+     */
+    static cosineSimilarity(vector1: number[], vector2: number[]): number;
+    /**
+     * Normalizes a vector.
+     * @remarks
+     * The norm of a vector is the square root of the sum of the squares of the elements.
+     * The LocalIndex pre-normalizes all vectors to improve performance.
+     * @param vector Vector to normalize
+     * @returns Normalized vector
+     */
+    static normalize(vector: number[]): number;
+    /**
+     * Returns the similarity between two vectors using cosine similarity.
+     * @remarks
+     * The LocalIndex pre-normalizes all vectors to improve performance.
+     * This method uses the pre-calculated norms to improve performance.
+     * @param vector1 Vector 1
+     * @param norm1 Norm of vector 1
+     * @param vector2 Vector 2
+     * @param norm2 Norm of vector 2
+     * @returns Similarity between the two vectors
+     */
+    static normalizedCosineSimilarity(vector1: number[], norm1: number, vector2: number[], norm2: number): number;
+    /**
+     * Applies a filter to the metadata of an item.
+     * @param metadata Metadata of the item
+     * @param filter Filter to apply
+     * @returns True if the item matches the filter, false otherwise
+     */
+    static select(metadata: Record<string, MetadataTypes>, filter: MetadataFilter): boolean;
+    private static dotProduct;
+    private static metadataFilter;
+}
+//# sourceMappingURL=ItemSelector.d.ts.map

package/lib/ItemSelector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ItemSelector.d.ts","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExD,qBAAa,YAAY;IACrB;;;;;OAKG;WACW,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE;IAKnE;;;;;;;OAOG;WACW,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE;IAYxC;;;;;;;;;;OAUG;WACW,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM;IAK3G;;;;;OAKG;WACW,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO;IAoC9F,OAAO,CAAC,MAAM,CAAC,UAAU;IAYzB,OAAO,CAAC,MAAM,CAAC,cAAc;CAqDhC"}

package/lib/ItemSelector.js

@@ -0,0 +1,156 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ItemSelector = void 0;
+class ItemSelector {
+    /**
+     * Returns the similarity between two vectors using the cosine similarity.
+     * @param vector1 Vector 1
+     * @param vector2 Vector 2
+     * @returns Similarity between the two vectors
+     */
+    static cosineSimilarity(vector1, vector2) {
+        // Return the quotient of the dot product and the product of the norms
+        return this.dotProduct(vector1, vector2) / (this.normalize(vector1) * this.normalize(vector2));
+    }
+    /**
+     * Normalizes a vector.
+     * @remarks
+     * The norm of a vector is the square root of the sum of the squares of the elements.
+     * The LocalIndex pre-normalizes all vectors to improve performance.
+     * @param vector Vector to normalize
+     * @returns Normalized vector
+     */
+    static normalize(vector) {
+        // Initialize a variable to store the sum of the squares
+        let sum = 0;
+        // Loop through the elements of the array
+        for (let i = 0; i < vector.length; i++) {
+            // Square the element and add it to the sum
+            sum += vector[i] * vector[i];
+        }
+        // Return the square root of the sum
+        return Math.sqrt(sum);
+    }
+    /**
+     * Returns the similarity between two vectors using cosine similarity.
+     * @remarks
+     * The LocalIndex pre-normalizes all vectors to improve performance.
+     * This method uses the pre-calculated norms to improve performance.
+     * @param vector1 Vector 1
+     * @param norm1 Norm of vector 1
+     * @param vector2 Vector 2
+     * @param norm2 Norm of vector 2
+     * @returns Similarity between the two vectors
+     */
+    static normalizedCosineSimilarity(vector1, norm1, vector2, norm2) {
+        // Return the quotient of the dot product and the product of the norms
+        return this.dotProduct(vector1, vector2) / (norm1 * norm2);
+    }
+    /**
+     * Applies a filter to the metadata of an item.
+     * @param metadata Metadata of the item
+     * @param filter Filter to apply
+     * @returns True if the item matches the filter, false otherwise
+     */
+    static select(metadata, filter) {
+        if (filter === undefined || filter === null) {
+            return true;
+        }
+        for (const key in filter) {
+            switch (key) {
+                case '$and':
+                    if (!filter[key].every((f) => this.select(metadata, f))) {
+                        return false;
+                    }
+                    break;
+                case '$or':
+                    if (!filter[key].some((f) => this.select(metadata, f))) {
+                        return false;
+                    }
+                    break;
+                default:
+                    const value = filter[key];
+                    if (value === undefined || value === null) {
+                        return false;
+                    }
+                    else if (typeof value == 'object') {
+                        if (!this.metadataFilter(metadata[key], value)) {
+                            return false;
+                        }
+                    }
+                    else {
+                        if (metadata[key] !== value) {
+                            return false;
+                        }
+                    }
+                    break;
+            }
+        }
+        return true;
+    }
+    static dotProduct(arr1, arr2) {
+        // Initialize a variable to store the sum of the products
+        let sum = 0;
+        // Loop through the elements of the arrays
+        for (let i = 0; i < arr1.length; i++) {
+            // Multiply the corresponding elements and add them to the sum
+            sum += arr1[i] * arr2[i];
+        }
+        // Return the sum
+        return sum;
+    }
+    static metadataFilter(value, filter) {
+        if (value === undefined || value === null) {
+            return false;
+        }
+        for (const key in filter) {
+            switch (key) {
+                case '$eq':
+                    if (value !== filter[key]) {
+                        return false;
+                    }
+                    break;
+                case '$ne':
+                    if (value === filter[key]) {
+                        return false;
+                    }
+                    break;
+                case '$gt':
+                    if (typeof value != 'number' || value <= filter[key]) {
+                        return false;
+                    }
+                    break;
+                case '$gte':
+                    if (typeof value != 'number' || value < filter[key]) {
+                        return false;
+                    }
+                    break;
+                case '$lt':
+                    if (typeof value != 'number' || value >= filter[key]) {
+                        return false;
+                    }
+                    break;
+                case '$lte':
+                    if (typeof value != 'number' || value > filter[key]) {
+                        return false;
+                    }
+                    break;
+                case '$in':
+                    if (typeof value == 'boolean' || !filter[key].includes(value)) {
+                        return false;
+                    }
+                    break;
+                case '$nin':
+                    if (typeof value == 'boolean' || filter[key].includes(value)) {
+                        return false;
+                    }
+                    break;
+                default:
+                    return value === filter[key];
+            }
+        }
+        return true;
+    }
+}
+exports.ItemSelector = ItemSelector;
+//# sourceMappingURL=ItemSelector.js.map

package/lib/ItemSelector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ItemSelector.js","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":";;;AAEA,MAAa,YAAY;IACrB;;;;;OAKG;IACI,MAAM,CAAC,gBAAgB,CAAC,OAAiB,EAAE,OAAiB;QAC/D,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,CAAC;IAED;;;;;;;OAOG;IACI,MAAM,CAAC,SAAS,CAAC,MAAgB;QACpC,wDAAwD;QACxD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,yCAAyC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,2CAA2C;YAC3C,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;SAChC;QACD,oCAAoC;QACpC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,0BAA0B,CAAC,OAAiB,EAAE,KAAa,EAAE,OAAiB,EAAE,KAAa;QACvG,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;IAC/D,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,MAAM,CAAC,QAAuC,EAAE,MAAsB;QAChF,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAE;YACzC,OAAO,IAAI,CAAC;SACf;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,MAAM;oBACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,KAAK,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACtE,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACrE,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;wBACvC,OAAO,KAAK,CAAC;qBAChB;yBAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE;wBACjC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,KAAuB,CAAC,EAAE;4BAC9D,OAAO,KAAK,CAAC;yBAChB;qBACJ;yBAAM;wBACH,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE;4BACzB,OAAO,KAAK,CAAC;yBAChB;qBACJ;oBACD,MAAM;aACb;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,UAAU,CAAC,IAAc,EAAE,IAAc;QACpD,yDAAyD;QACzD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,0CAA0C;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,8DAA8D;YAC9D,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;SAC5B;QACD,iBAAiB;QACjB,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,KAAoB,EAAE,MAAsB;QACtE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;YACvC,OAAO,KAAK,CAAC;SAChB;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAE,EAAE;wBACnD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAE,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAE,EAAE;wBACnD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAE,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,SAAS,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC5D,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,SAAS,IAAI,MAAM,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC3D,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,OAAO,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;aACpC;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ;AA3JD,oCA2JC"}

package/lib/LocalDocument.d.ts

@@ -0,0 +1,16 @@
+import { MetadataTypes } from './types';
+export declare class LocalDocument {
+    private readonly _folderPath;
+    private readonly _id;
+    private readonly _uri;
+    private _metadata;
+    private _text;
+    constructor(folderPath: string, id: string, uri: string);
+    get folderPath(): string;
+    get id(): string;
+    get uri(): string;
+    hasMetadata(): Promise<boolean>;
+    loadMetadata(): Promise<Record<string, MetadataTypes>>;
+    loadText(): Promise<string>;
+}
+//# sourceMappingURL=LocalDocument.d.ts.map

package/lib/LocalDocument.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocument.d.ts","sourceRoot":"","sources":["../src/LocalDocument.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,qBAAa,aAAa;IACtB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,SAAS,CAAyC;IAC1D,OAAO,CAAC,KAAK,CAAmB;gBAEb,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;IAM9D,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED,IAAW,EAAE,IAAI,MAAM,CAEtB;IAED,IAAW,GAAG,IAAI,MAAM,CAEvB;IAEY,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAS/B,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,CAAC;IAmBrD,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;CAY3C"}

package/lib/LocalDocument.js

@@ -0,0 +1,99 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalDocument = void 0;
+const fs = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
+class LocalDocument {
+    constructor(folderPath, id, uri) {
+        this._folderPath = folderPath;
+        this._id = id;
+        this._uri = uri;
+    }
+    get folderPath() {
+        return this._folderPath;
+    }
+    get id() {
+        return this._id;
+    }
+    get uri() {
+        return this._uri;
+    }
+    hasMetadata() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield fs.access(path.join(this.folderPath, `${this.id}.json`));
+                return true;
+            }
+            catch (err) {
+                return false;
+            }
+        });
+    }
+    loadMetadata() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this._metadata == undefined) {
+                let json;
+                try {
+                    json = (yield fs.readFile(path.join(this.folderPath, `${this.id}.json`))).toString();
+                }
+                catch (err) {
+                    throw new Error(`Error reading metadata for document "${this.uri}": ${err.toString()}`);
+                }
+                try {
+                    this._metadata = JSON.parse(json);
+                }
+                catch (err) {
+                    throw new Error(`Error parsing metadata for document "${this.uri}": ${err.toString()}`);
+                }
+            }
+            return this._metadata;
+        });
+    }
+    loadText() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this._text == undefined) {
+                try {
+                    this._text = (yield fs.readFile(path.join(this.folderPath, `${this.id}.txt`))).toString();
+                }
+                catch (err) {
+                    throw new Error(`Error reading text file for document "${this.uri}": ${err.toString()}`);
+                }
+            }
+            return this._text;
+        });
+    }
+}
+exports.LocalDocument = LocalDocument;
+//# sourceMappingURL=LocalDocument.js.map

package/lib/LocalDocument.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocument.js","sourceRoot":"","sources":["../src/LocalDocument.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAG7B,MAAa,aAAa;IAOtB,YAAmB,UAAkB,EAAE,EAAU,EAAE,GAAW;QAC1D,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;IACpB,CAAC;IAED,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAW,EAAE;QACT,OAAO,IAAI,CAAC,GAAG,CAAC;IACpB,CAAC;IAED,IAAW,GAAG;QACV,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAEY,WAAW;;YACpB,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAY,EAAE;gBACnB,OAAO,KAAK,CAAC;aAChB;QACL,CAAC;KAAA;IAEY,YAAY;;YACrB,IAAI,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE;gBAC7B,IAAI,IAAY,CAAC;gBACjB,IAAI;oBACA,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;iBACxF;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACpG;gBAED,IAAI;oBACA,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;iBACrC;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACpG;aACJ;YAED,OAAO,IAAI,CAAC,SAAU,CAAC;QAC3B,CAAC;KAAA;IAEY,QAAQ;;YACjB,IAAI,IAAI,CAAC,KAAK,IAAI,SAAS,EAAE;gBACzB,IAAI;oBACA,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;iBAC7F;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACrG;aACJ;YAED,OAAO,IAAI,CAAC,KAAK,CAAC;QACtB,CAAC;KAAA;CAEJ;AAjED,sCAiEC"}

package/lib/LocalDocumentIndex.d.ts

@@ -0,0 +1,48 @@
+import { CreateIndexConfig, LocalIndex } from "./LocalIndex";
+import { TextSplitterConfig } from "./TextSplitter";
+import { MetadataFilter, EmbeddingsModel, Tokenizer, MetadataTypes, DocumentCatalogStats } from "./types";
+import { LocalDocumentResult } from './LocalDocumentResult';
+import { LocalDocument } from './LocalDocument';
+export interface DocumentQueryOptions {
+    maxDocuments?: number;
+    maxChunks?: number;
+    filter?: MetadataFilter;
+}
+export interface LocalDocumentIndexConfig {
+    folderPath: string;
+    embeddings?: EmbeddingsModel;
+    tokenizer?: Tokenizer;
+    chunkingConfig?: Partial<TextSplitterConfig>;
+}
+export declare class LocalDocumentIndex extends LocalIndex {
+    private readonly _embeddings?;
+    private readonly _tokenizer;
+    private readonly _chunkingConfig?;
+    private _catalog?;
+    private _newCatalog?;
+    constructor(config: LocalDocumentIndexConfig);
+    /**
+     * Returns true if the document catalog exists.
+     */
+    isCatalogCreated(): Promise<boolean>;
+    getDocumentId(uri: string): Promise<string | undefined>;
+    getDocumentUri(documentId: string): Promise<string | undefined>;
+    createIndex(config?: CreateIndexConfig): Promise<void>;
+    deleteDocument(uri: string): Promise<void>;
+    getCatalogStats(): Promise<DocumentCatalogStats>;
+    /**
+     * Adds a document to the catalog.
+     * @remarks
+     * A new update is started if one is not already in progress. If an document with the same uri
+     * already exists, it will be replaced.
+     * @param item Item to insert
+     * @returns Inserted document
+     */
+    upsertDocument(uri: string, text: string, metadata?: Record<string, MetadataTypes>): Promise<LocalDocument>;
+    queryDocuments(query: string, options?: DocumentQueryOptions): Promise<LocalDocumentResult[]>;
+    beginUpdate(): Promise<void>;
+    cancelUpdate(): void;
+    endUpdate(): Promise<void>;
+    protected loadIndexData(): Promise<void>;
+}
+//# sourceMappingURL=LocalDocumentIndex.d.ts.map

package/lib/LocalDocumentIndex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocumentIndex.d.ts","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAgB,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,EAA0D,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAClK,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAIhD,MAAM,WAAW,oBAAoB;IACjC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,cAAc,CAAC;CAC3B;AAED,MAAM,WAAW,wBAAwB;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,eAAe,CAAC;IAC7B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAChD;AAED,qBAAa,kBAAmB,SAAQ,UAAU;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAkB;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAqB;IACtD,OAAO,CAAC,QAAQ,CAAC,CAAkB;IACnC,OAAO,CAAC,WAAW,CAAC,CAAkB;gBAGnB,MAAM,EAAE,wBAAwB;IAYnD;;OAEG;IACU,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC;IASpC,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAKvD,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAK/D,WAAW,CAAC,MAAM,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtD,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8C1C,eAAe,IAAI,OAAO,CAAC,oBAAoB,CAAC;IAU7D;;;;;;;OAOG;IACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IA4G3G,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAqD7F,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAKlC,YAAY,IAAI,IAAI;IAKd,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;cAavB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;CA2BjD"}