vectra 0.10.0 → 0.11.0

package/lib/FileFetcher.d.ts

@@ -0,0 +1,5 @@
+import { TextFetcher } from './types';
+export declare class FileFetcher implements TextFetcher {
+    fetch(uri: string, onDocument: (uri: string, text: string, docType?: string | undefined) => Promise<boolean>): Promise<boolean>;
+}
+//# sourceMappingURL=FileFetcher.d.ts.map

package/lib/FileFetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FileFetcher.d.ts","sourceRoot":"","sources":["../src/FileFetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAItC,qBAAa,WAAY,YAAW,WAAW;IAC9B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC;CAyB/I"}

package/lib/FileFetcher.js

@@ -0,0 +1,69 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FileFetcher = void 0;
+const fs = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
+class FileFetcher {
+    fetch(uri, onDocument) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Does path exist and is it a directory?
+            let isDirectory;
+            try {
+                const stat = yield fs.stat(uri);
+                isDirectory = stat.isDirectory();
+            }
+            catch (_a) {
+                return true;
+            }
+            // If directory, read all files and recurse
+            if (isDirectory) {
+                const files = yield fs.readdir(uri);
+                for (const file of files) {
+                    const filePath = path.join(uri, file);
+                    yield this.fetch(filePath, onDocument);
+                }
+                return true;
+            }
+            else {
+                // Read file and call onDocument
+                const text = yield fs.readFile(uri, 'utf8');
+                const parts = uri.split('.');
+                return yield onDocument(uri, text, parts.length > 0 ? parts[parts.length - 1].toLowerCase() : undefined);
+            }
+        });
+    }
+}
+exports.FileFetcher = FileFetcher;
+//# sourceMappingURL=FileFetcher.js.map

package/lib/FileFetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FileFetcher.js","sourceRoot":"","sources":["../src/FileFetcher.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,gDAAkC;AAClC,2CAA6B;AAE7B,MAAa,WAAW;IACP,KAAK,CAAC,GAAW,EAAE,UAAyF;;YACrH,yCAAyC;YACzC,IAAI,WAAoB,CAAC;YACzB,IAAI;gBACA,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,WAAW,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;aACpC;YAAC,WAAM;gBACJ,OAAO,IAAI,CAAC;aACf;YAED,2CAA2C;YAC3C,IAAI,WAAW,EAAE;gBACb,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;oBACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;oBACtC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;iBAC1C;gBACD,OAAO,IAAI,CAAC;aACf;iBAAM;gBACH,gCAAgC;gBAChC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC7B,OAAO,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;aAC5G;QACL,CAAC;KAAA;CACJ;AA1BD,kCA0BC"}

package/lib/GPT3Tokenizer.d.ts

@@ -0,0 +1,9 @@
+import { Tokenizer } from "./types";
+/**
+ * Tokenizer that uses GPT-3's encoder.
+ */
+export declare class GPT3Tokenizer implements Tokenizer {
+    decode(tokens: number[]): string;
+    encode(text: string): number[];
+}
+//# sourceMappingURL=GPT3Tokenizer.d.ts.map

package/lib/GPT3Tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"GPT3Tokenizer.d.ts","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGpC;;GAEG;AACH,qBAAa,aAAc,YAAW,SAAS;IACpC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAIhC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;CAGxC"}

package/lib/GPT3Tokenizer.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GPT3Tokenizer = void 0;
+const gpt_3_encoder_1 = require("gpt-3-encoder");
+/**
+ * Tokenizer that uses GPT-3's encoder.
+ */
+class GPT3Tokenizer {
+    decode(tokens) {
+        return (0, gpt_3_encoder_1.decode)(tokens);
+    }
+    encode(text) {
+        return (0, gpt_3_encoder_1.encode)(text);
+    }
+}
+exports.GPT3Tokenizer = GPT3Tokenizer;
+//# sourceMappingURL=GPT3Tokenizer.js.map

package/lib/GPT3Tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GPT3Tokenizer.js","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":";;;AACA,iDAA+C;AAE/C;;GAEG;AACH,MAAa,aAAa;IACf,MAAM,CAAC,MAAgB;QAC1B,OAAO,IAAA,sBAAM,EAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAA,sBAAM,EAAC,IAAI,CAAC,CAAC;IACxB,CAAC;CACJ;AARD,sCAQC"}

package/lib/ItemSelector.d.ts

@@ -0,0 +1,41 @@
+import { MetadataFilter, MetadataTypes } from './types';
+export declare class ItemSelector {
+    /**
+     * Returns the similarity between two vectors using the cosine similarity.
+     * @param vector1 Vector 1
+     * @param vector2 Vector 2
+     * @returns Similarity between the two vectors
+     */
+    static cosineSimilarity(vector1: number[], vector2: number[]): number;
+    /**
+     * Normalizes a vector.
+     * @remarks
+     * The norm of a vector is the square root of the sum of the squares of the elements.
+     * The LocalIndex pre-normalizes all vectors to improve performance.
+     * @param vector Vector to normalize
+     * @returns Normalized vector
+     */
+    static normalize(vector: number[]): number;
+    /**
+     * Returns the similarity between two vectors using cosine similarity.
+     * @remarks
+     * The LocalIndex pre-normalizes all vectors to improve performance.
+     * This method uses the pre-calculated norms to improve performance.
+     * @param vector1 Vector 1
+     * @param norm1 Norm of vector 1
+     * @param vector2 Vector 2
+     * @param norm2 Norm of vector 2
+     * @returns Similarity between the two vectors
+     */
+    static normalizedCosineSimilarity(vector1: number[], norm1: number, vector2: number[], norm2: number): number;
+    /**
+     * Applies a filter to the metadata of an item.
+     * @param metadata Metadata of the item
+     * @param filter Filter to apply
+     * @returns True if the item matches the filter, false otherwise
+     */
+    static select(metadata: Record<string, MetadataTypes>, filter: MetadataFilter): boolean;
+    private static dotProduct;
+    private static metadataFilter;
+}
+//# sourceMappingURL=ItemSelector.d.ts.map

package/lib/ItemSelector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ItemSelector.d.ts","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExD,qBAAa,YAAY;IACrB;;;;;OAKG;WACW,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE;IAKnE;;;;;;;OAOG;WACW,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE;IAYxC;;;;;;;;;;OAUG;WACW,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM;IAK3G;;;;;OAKG;WACW,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO;IAoC9F,OAAO,CAAC,MAAM,CAAC,UAAU;IAYzB,OAAO,CAAC,MAAM,CAAC,cAAc;CA+DhC"}

package/lib/ItemSelector.js

@@ -0,0 +1,168 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ItemSelector = void 0;
+class ItemSelector {
+    /**
+     * Returns the similarity between two vectors using the cosine similarity.
+     * @param vector1 Vector 1
+     * @param vector2 Vector 2
+     * @returns Similarity between the two vectors
+     */
+    static cosineSimilarity(vector1, vector2) {
+        // Return the quotient of the dot product and the product of the norms
+        return this.dotProduct(vector1, vector2) / (this.normalize(vector1) * this.normalize(vector2));
+    }
+    /**
+     * Normalizes a vector.
+     * @remarks
+     * The norm of a vector is the square root of the sum of the squares of the elements.
+     * The LocalIndex pre-normalizes all vectors to improve performance.
+     * @param vector Vector to normalize
+     * @returns Normalized vector
+     */
+    static normalize(vector) {
+        // Initialize a variable to store the sum of the squares
+        let sum = 0;
+        // Loop through the elements of the array
+        for (let i = 0; i < vector.length; i++) {
+            // Square the element and add it to the sum
+            sum += vector[i] * vector[i];
+        }
+        // Return the square root of the sum
+        return Math.sqrt(sum);
+    }
+    /**
+     * Returns the similarity between two vectors using cosine similarity.
+     * @remarks
+     * The LocalIndex pre-normalizes all vectors to improve performance.
+     * This method uses the pre-calculated norms to improve performance.
+     * @param vector1 Vector 1
+     * @param norm1 Norm of vector 1
+     * @param vector2 Vector 2
+     * @param norm2 Norm of vector 2
+     * @returns Similarity between the two vectors
+     */
+    static normalizedCosineSimilarity(vector1, norm1, vector2, norm2) {
+        // Return the quotient of the dot product and the product of the norms
+        return this.dotProduct(vector1, vector2) / (norm1 * norm2);
+    }
+    /**
+     * Applies a filter to the metadata of an item.
+     * @param metadata Metadata of the item
+     * @param filter Filter to apply
+     * @returns True if the item matches the filter, false otherwise
+     */
+    static select(metadata, filter) {
+        if (filter === undefined || filter === null) {
+            return true;
+        }
+        for (const key in filter) {
+            switch (key) {
+                case '$and':
+                    if (!filter[key].every((f) => this.select(metadata, f))) {
+                        return false;
+                    }
+                    break;
+                case '$or':
+                    if (!filter[key].some((f) => this.select(metadata, f))) {
+                        return false;
+                    }
+                    break;
+                default:
+                    const value = filter[key];
+                    if (value === undefined || value === null) {
+                        return false;
+                    }
+                    else if (typeof value == 'object') {
+                        if (!this.metadataFilter(metadata[key], value)) {
+                            return false;
+                        }
+                    }
+                    else {
+                        if (metadata[key] !== value) {
+                            return false;
+                        }
+                    }
+                    break;
+            }
+        }
+        return true;
+    }
+    static dotProduct(arr1, arr2) {
+        // Initialize a variable to store the sum of the products
+        let sum = 0;
+        // Loop through the elements of the arrays
+        for (let i = 0; i < arr1.length; i++) {
+            // Multiply the corresponding elements and add them to the sum
+            sum += arr1[i] * arr2[i];
+        }
+        // Return the sum
+        return sum;
+    }
+    static metadataFilter(value, filter) {
+        if (value === undefined || value === null) {
+            return false;
+        }
+        for (const key in filter) {
+            switch (key) {
+                case '$eq':
+                    if (value !== filter[key]) {
+                        return false;
+                    }
+                    break;
+                case '$ne':
+                    if (value === filter[key]) {
+                        return false;
+                    }
+                    break;
+                case '$gt':
+                    if (typeof value != 'number' || value <= filter[key]) {
+                        return false;
+                    }
+                    break;
+                case '$gte':
+                    if (typeof value != 'number' || value < filter[key]) {
+                        return false;
+                    }
+                    break;
+                case '$lt':
+                    if (typeof value != 'number' || value >= filter[key]) {
+                        return false;
+                    }
+                    break;
+                case '$lte':
+                    if (typeof value != 'number' || value > filter[key]) {
+                        return false;
+                    }
+                    break;
+                case '$in':
+                    if (typeof value == 'boolean') {
+                        return false;
+                    }
+                    else if (typeof value == 'string' && !filter[key].includes(value)) {
+                        return false;
+                    }
+                    else if (!filter[key].some(val => typeof val == 'string' && val.includes(value))) {
+                        return false;
+                    }
+                    break;
+                case '$nin':
+                    if (typeof value == 'boolean') {
+                        return false;
+                    }
+                    else if (typeof value == 'string' && filter[key].includes(value)) {
+                        return false;
+                    }
+                    else if (filter[key].some(val => typeof val == 'string' && val.includes(value))) {
+                        return false;
+                    }
+                    break;
+                default:
+                    return value === filter[key];
+            }
+        }
+        return true;
+    }
+}
+exports.ItemSelector = ItemSelector;
+//# sourceMappingURL=ItemSelector.js.map

package/lib/ItemSelector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ItemSelector.js","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":";;;AAEA,MAAa,YAAY;IACrB;;;;;OAKG;IACI,MAAM,CAAC,gBAAgB,CAAC,OAAiB,EAAE,OAAiB;QAC/D,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,CAAC;IAED;;;;;;;OAOG;IACI,MAAM,CAAC,SAAS,CAAC,MAAgB;QACpC,wDAAwD;QACxD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,yCAAyC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,2CAA2C;YAC3C,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;SAChC;QACD,oCAAoC;QACpC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,0BAA0B,CAAC,OAAiB,EAAE,KAAa,EAAE,OAAiB,EAAE,KAAa;QACvG,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;IAC/D,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,MAAM,CAAC,QAAuC,EAAE,MAAsB;QAChF,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAE;YACzC,OAAO,IAAI,CAAC;SACf;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,MAAM;oBACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,KAAK,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACtE,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACrE,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;wBACvC,OAAO,KAAK,CAAC;qBAChB;yBAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE;wBACjC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,KAAuB,CAAC,EAAE;4BAC9D,OAAO,KAAK,CAAC;yBAChB;qBACJ;yBAAM;wBACH,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE;4BACzB,OAAO,KAAK,CAAC;yBAChB;qBACJ;oBACD,MAAM;aACb;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,UAAU,CAAC,IAAc,EAAE,IAAc;QACpD,yDAAyD;QACzD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,0CAA0C;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,8DAA8D;YAC9D,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;SAC5B;QACD,iBAAiB;QACjB,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,KAAoB,EAAE,MAAsB;QACtE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;YACvC,OAAO,KAAK,CAAC;SAChB;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAE,EAAE;wBACnD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAE,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAE,EAAE;wBACnD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAE,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,SAAS,EAAE;wBAC3B,OAAO,KAAK,CAAC;qBAChB;yBAAM,IAAG,OAAO,KAAK,IAAI,QAAQ,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAC;wBAChE,OAAO,KAAK,CAAA;qBACf;yBAAM,IAAG,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,GAAG,IAAI,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAe,CAAC,CAAC,EAAC;wBACzF,OAAO,KAAK,CAAA;qBACf;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,SAAS,EAAE;wBAC3B,OAAO,KAAK,CAAC;qBAChB;yBACI,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,MAAM,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC/D,OAAO,KAAK,CAAC;qBAChB;yBACI,IAAI,MAAM,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,GAAG,IAAI,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAe,CAAC,CAAC,EAAE;wBACxF,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,OAAO,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;aACpC;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ;AArKD,oCAqKC"}

package/lib/LocalDocument.d.ts

@@ -0,0 +1,54 @@
+import { MetadataTypes } from './types';
+import { LocalDocumentIndex } from './LocalDocumentIndex';
+/**
+ * Represents an indexed document stored on disk.
+ */
+export declare class LocalDocument {
+    private readonly _index;
+    private readonly _id;
+    private readonly _uri;
+    private _metadata;
+    private _text;
+    /**
+     * Creates a new `LocalDocument` instance.
+     * @param index Parent index that contains the document.
+     * @param id ID of the document.
+     * @param uri URI of the document.
+     */
+    constructor(index: LocalDocumentIndex, id: string, uri: string);
+    /**
+     * Returns the folder path where the document is stored.
+     */
+    get folderPath(): string;
+    /**
+     * Returns the ID of the document.
+     */
+    get id(): string;
+    /**
+     * Returns the URI of the document.
+     */
+    get uri(): string;
+    /**
+     * Returns the length of the document in tokens.
+     * @remarks
+     * This value will be estimated for documents longer then 40k bytes.
+     * @returns Length of the document in tokens.
+     */
+    getLength(): Promise<number>;
+    /**
+     * Determines if the document has additional metadata storred on disk.
+     * @returns True if the document has metadata; otherwise, false.
+     */
+    hasMetadata(): Promise<boolean>;
+    /**
+     * Loads the metadata for the document from disk.
+     * @returns Metadata for the document.
+     */
+    loadMetadata(): Promise<Record<string, MetadataTypes>>;
+    /**
+     * Loads the text for the document from disk.
+     * @returns Text for the document.
+     */
+    loadText(): Promise<string>;
+}
+//# sourceMappingURL=LocalDocument.d.ts.map

package/lib/LocalDocument.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocument.d.ts","sourceRoot":"","sources":["../src/LocalDocument.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D;;GAEG;AACH,qBAAa,aAAa;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,SAAS,CAAyC;IAC1D,OAAO,CAAC,KAAK,CAAmB;IAEhC;;;;;OAKG;gBACgB,KAAK,EAAE,kBAAkB,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;IAMrE;;OAEG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;OAEG;IACH,IAAW,EAAE,IAAI,MAAM,CAEtB;IAED;;OAEG;IACH,IAAW,GAAG,IAAI,MAAM,CAEvB;IAED;;;;;OAKG;IACU,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;IASzC;;;OAGG;IACU,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAS5C;;;OAGG;IACU,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,CAAC;IAmBlE;;;OAGG;IACU,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;CAW3C"}

package/lib/LocalDocument.js

@@ -0,0 +1,146 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalDocument = void 0;
+const fs = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
+/**
+ * Represents an indexed document stored on disk.
+ */
+class LocalDocument {
+    /**
+     * Creates a new `LocalDocument` instance.
+     * @param index Parent index that contains the document.
+     * @param id ID of the document.
+     * @param uri URI of the document.
+     */
+    constructor(index, id, uri) {
+        this._index = index;
+        this._id = id;
+        this._uri = uri;
+    }
+    /**
+     * Returns the folder path where the document is stored.
+     */
+    get folderPath() {
+        return this._index.folderPath;
+    }
+    /**
+     * Returns the ID of the document.
+     */
+    get id() {
+        return this._id;
+    }
+    /**
+     * Returns the URI of the document.
+     */
+    get uri() {
+        return this._uri;
+    }
+    /**
+     * Returns the length of the document in tokens.
+     * @remarks
+     * This value will be estimated for documents longer then 40k bytes.
+     * @returns Length of the document in tokens.
+     */
+    getLength() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const text = yield this.loadText();
+            if (text.length <= 40000) {
+                return this._index.tokenizer.encode(text).length;
+            }
+            else {
+                return Math.ceil(text.length / 4);
+            }
+        });
+    }
+    /**
+     * Determines if the document has additional metadata storred on disk.
+     * @returns True if the document has metadata; otherwise, false.
+     */
+    hasMetadata() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield fs.access(path.join(this.folderPath, `${this.id}.json`));
+                return true;
+            }
+            catch (err) {
+                return false;
+            }
+        });
+    }
+    /**
+     * Loads the metadata for the document from disk.
+     * @returns Metadata for the document.
+     */
+    loadMetadata() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this._metadata == undefined) {
+                let json;
+                try {
+                    json = (yield fs.readFile(path.join(this.folderPath, `${this.id}.json`))).toString();
+                }
+                catch (err) {
+                    throw new Error(`Error reading metadata for document "${this.uri}": ${err.toString()}`);
+                }
+                try {
+                    this._metadata = JSON.parse(json);
+                }
+                catch (err) {
+                    throw new Error(`Error parsing metadata for document "${this.uri}": ${err.toString()}`);
+                }
+            }
+            return this._metadata;
+        });
+    }
+    /**
+     * Loads the text for the document from disk.
+     * @returns Text for the document.
+     */
+    loadText() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this._text == undefined) {
+                try {
+                    this._text = (yield fs.readFile(path.join(this.folderPath, `${this.id}.txt`))).toString();
+                }
+                catch (err) {
+                    throw new Error(`Error reading text file for document "${this.uri}": ${err.toString()}`);
+                }
+            }
+            return this._text;
+        });
+    }
+}
+exports.LocalDocument = LocalDocument;
+//# sourceMappingURL=LocalDocument.js.map

package/lib/LocalDocument.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalDocument.js","sourceRoot":"","sources":["../src/LocalDocument.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAI7B;;GAEG;AACH,MAAa,aAAa;IAOtB;;;;;OAKG;IACH,YAAmB,KAAyB,EAAE,EAAU,EAAE,GAAW;QACjE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,IAAW,EAAE;QACT,OAAO,IAAI,CAAC,GAAG,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,IAAW,GAAG;QACV,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACU,SAAS;;YAClB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE;gBACtB,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;aACpD;iBAAM;gBACH,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;aACrC;QACL,CAAC;KAAA;IAED;;;OAGG;IACU,WAAW;;YACpB,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAY,EAAE;gBACnB,OAAO,KAAK,CAAC;aAChB;QACL,CAAC;KAAA;IAED;;;OAGG;IACU,YAAY;;YACrB,IAAI,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE;gBAC7B,IAAI,IAAY,CAAC;gBACjB,IAAI;oBACA,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;iBACxF;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACpG;gBAED,IAAI;oBACA,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;iBACrC;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACpG;aACJ;YAED,OAAO,IAAI,CAAC,SAAU,CAAC;QAC3B,CAAC;KAAA;IAED;;;OAGG;IACU,QAAQ;;YACjB,IAAI,IAAI,CAAC,KAAK,IAAI,SAAS,EAAE;gBACzB,IAAI;oBACA,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;iBAC7F;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBACrG;aACJ;YAED,OAAO,IAAI,CAAC,KAAK,CAAC;QACtB,CAAC;KAAA;CACJ;AA1GD,sCA0GC"}