vectra 0.11.0 → 0.12.0

package/lib/LocalIndex.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalIndex.spec.d.ts","sourceRoot":"","sources":["../src/LocalIndex.spec.ts"],"names":[],"mappings":""}

package/lib/LocalIndex.spec.js

@@ -0,0 +1,64 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_assert_1 = __importDefault(require("node:assert"));
+const LocalIndex_1 = require("./LocalIndex");
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+describe('LocalIndex', () => {
+    const testIndexDir = path_1.default.join(__dirname, 'test_index');
+    beforeEach(() => __awaiter(void 0, void 0, void 0, function* () {
+        yield promises_1.default.rm(testIndexDir, { recursive: true, force: true });
+    }));
+    afterEach(() => __awaiter(void 0, void 0, void 0, function* () {
+        yield promises_1.default.rm(testIndexDir, { recursive: true, force: true });
+    }));
+    it('should create a new index', () => __awaiter(void 0, void 0, void 0, function* () {
+        const index = new LocalIndex_1.LocalIndex(testIndexDir);
+        yield index.createIndex();
+        const created = yield index.isIndexCreated();
+        node_assert_1.default.equal(created, true);
+    }));
+    describe('batchInsertItems', () => {
+        const indexItems = [
+            { id: '1', vector: [1, 2, 3] },
+            { id: '2', vector: [2, 3, 4] },
+            { id: '3', vector: [3, 4, 5] }
+        ];
+        it('should insert provided items', () => __awaiter(void 0, void 0, void 0, function* () {
+            const index = new LocalIndex_1.LocalIndex(testIndexDir);
+            yield index.createIndex();
+            const newItems = yield index.batchInsertItems(indexItems);
+            node_assert_1.default.equal(newItems.length, 3);
+            const retrievedItems = yield index.listItems();
+            node_assert_1.default.equal(retrievedItems.length, 3);
+        }));
+        it('on id collision - cancel batch insert & bubble up error', () => __awaiter(void 0, void 0, void 0, function* () {
+            const index = new LocalIndex_1.LocalIndex(testIndexDir);
+            yield index.createIndex();
+            yield index.insertItem({ id: '2', vector: [9, 9, 9] });
+            // ensures insert error is bubbled up to batchIndexItems caller
+            yield node_assert_1.default.rejects(() => __awaiter(void 0, void 0, void 0, function* () {
+                yield index.batchInsertItems(indexItems);
+            }), {
+                name: 'Error',
+                message: 'Item with id 2 already exists'
+            });
+            // ensures no partial update is applied
+            const storedItems = yield index.listItems();
+            node_assert_1.default.equal(storedItems.length, 1);
+        }));
+    });
+});
+//# sourceMappingURL=LocalIndex.spec.js.map

package/lib/LocalIndex.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalIndex.spec.js","sourceRoot":"","sources":["../src/LocalIndex.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,8DAAgC;AAChC,6CAAyC;AAEzC,2DAA4B;AAC5B,gDAAuB;AAEvB,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAExD,UAAU,CAAC,GAAS,EAAE;QACpB,MAAM,kBAAE,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAA,CAAC,CAAC;IAEH,SAAS,CAAC,GAAS,EAAE;QACnB,MAAM,kBAAE,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,GAAS,EAAE;QACzC,MAAM,KAAK,GAAG,IAAI,uBAAU,CAAC,YAAY,CAAC,CAAC;QAC3C,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC;QAC7C,qBAAM,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC9B,CAAC,CAAA,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,MAAM,UAAU,GAAyB;YACvC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE;YAC9B,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE;YAC9B,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE;SAC/B,CAAC;QAEF,EAAE,CAAC,8BAA8B,EAAE,GAAS,EAAE;YAC5C,MAAM,KAAK,GAAG,IAAI,uBAAU,CAAC,YAAY,CAAC,CAAC;YAC3C,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;YAE1B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YAE1D,qBAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAEjC,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;YAC/C,qBAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC,CAAA,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,GAAS,EAAE;YACvE,MAAM,KAAK,GAAG,IAAI,uBAAU,CAAC,YAAY,CAAC,CAAC;YAC3C,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;YAE1B,MAAM,KAAK,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YAEvD,+DAA+D;YAC/D,MAAM,qBAAM,CAAC,OAAO,CAClB,GAAS,EAAE;gBACT,MAAM,KAAK,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YAC3C,CAAC,CAAA,EACD;gBACE,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,+BAA+B;aACzC,CACF,CAAC;YAEF,uCAAuC;YACvC,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;YAC5C,qBAAM,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACtC,CAAC,CAAA,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/package.json

@@ -2,7 +2,7 @@
     "name": "vectra",
     "author": "Steven Ickman",
     "description": "A vector database that uses the local file system for storage.",
-    "version": "0.11.0",
+    "version": "0.12.0",
     "license": "MIT",
     "keywords": [
         "gpt"
@@ -38,6 +38,7 @@
         "openai": "^4.97.0",
         "turndown": "^7.2.0",
         "uuid": "^11.1.0",
+        "wink-bm25-text-search": "^3.1.2",
         "wink-nlp": "^2.3.2",
         "yargs": "^17.7.2"
     },
@@ -50,9 +51,12 @@
         "@types/uuid": "10.0.0",
         "@types/yargs": "17.0.33",
         "mocha": "11.2.2",
+        "npm-run-all": "^4.1.5",
+        "rimraf": "^5.0.1",
         "nyc": "^17.1.0",
         "shx": "^0.4.0",
         "ts-mocha": "11.1.0",
+        "ts-node": "^10.9.1",
         "typescript": "^5.8.3",
         "wink-bm25-text-search": "^3.1.2"
     },
@@ -64,12 +68,13 @@
         "depcheck": "depcheck --config ../../.depcheckrc",
         "lint": "eslint **/src/**/*.{j,t}s{,x} --fix --no-error-on-unmatched-pattern",
         "test": "npm-run-all build test:mocha",
-        "test:mocha": "nyc mocha tests",
+        "test:mocha": "nyc ts-mocha src/**/*.spec.ts --timeout 10000",
         "test:compat": "api-extractor run --verbose"
     },
     "files": [
         "_ts3.4",
         "lib",
         "src"
-    ]
+    ],
+    "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
 }

package/src/LocalIndex.spec.ts

@@ -0,0 +1,66 @@
+import assert from 'node:assert'
+import { LocalIndex } from './LocalIndex'
+import { IndexItem } from './types'
+import fs from 'fs/promises'
+import path from 'path'
+
+describe('LocalIndex', () => {
+  const testIndexDir = path.join(__dirname, 'test_index');
+
+  beforeEach(async () => {
+    await fs.rm(testIndexDir, { recursive: true, force: true });
+  });
+
+  afterEach(async () => {
+    await fs.rm(testIndexDir, { recursive: true, force: true });
+  });
+
+  it('should create a new index', async () => {
+    const index = new LocalIndex(testIndexDir);
+    await index.createIndex();
+    const created = await index.isIndexCreated();
+    assert.equal(created, true);
+  });
+
+  describe('batchInsertItems', () => {
+    const indexItems: Partial<IndexItem>[] = [
+      { id: '1', vector: [1, 2, 3] },
+      { id: '2', vector: [2, 3, 4] },
+      { id: '3', vector: [3, 4, 5] }
+    ];
+
+    it('should insert provided items', async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+
+      const newItems = await index.batchInsertItems(indexItems);
+
+      assert.equal(newItems.length, 3);
+
+      const retrievedItems = await index.listItems();
+      assert.equal(retrievedItems.length, 3);
+    });
+
+    it('on id collision - cancel batch insert & bubble up error', async () => {
+      const index = new LocalIndex(testIndexDir);
+      await index.createIndex();
+
+      await index.insertItem({ id: '2', vector: [9, 9, 9] });
+
+      // ensures insert error is bubbled up to batchIndexItems caller
+      await assert.rejects(
+        async () => {
+          await index.batchInsertItems(indexItems);
+        },
+        {
+          name: 'Error',
+          message: 'Item with id 2 already exists'
+        }
+      );
+
+      // ensures no partial update is applied
+      const storedItems = await index.listItems();
+      assert.equal(storedItems.length, 1);
+    });
+  });
+});

package/src/LocalIndex.ts

@@ -66,7 +66,7 @@ export class LocalIndex<TMetadata extends Record<string,MetadataTypes> = Record<
         }
 
         await this.loadIndexData();
-        this._update = Object.assign({}, this._data);
+        this._update = structuredClone(this._data);
     }
 
     /**
@@ -207,6 +207,32 @@ export class LocalIndex<TMetadata extends Record<string,MetadataTypes> = Record<
         }
     }
 
+    /**
+     * Adds a batch of items to the index.
+     * @remarks
+     * Batch update requires no update to be in progress. This is necessary so that if any one
+     * insert operation fails, the entire update can be safely cancelled. This prevents partial
+     * updates from being applied to the local index.
+     * @param items Items to insert.
+     * @returns Inserted items.
+     */
+    public async batchInsertItems<TItemMetadata extends TMetadata = TMetadata>(items: Partial<IndexItem<TItemMetadata>>[]): Promise<IndexItem[]> {
+        await this.beginUpdate();
+        try {
+            const newItems: IndexItem[] = [];
+            for (const item of items) {
+                const newItem = await this.addItemToUpdate(item, true);
+                newItems.push(newItem);
+            }
+            await this.endUpdate();
+            return newItems;
+        } catch (e) {
+            // cancels this update to prevent partial batch updates. allows error to bubble up.
+            await this.cancelUpdate();
+            throw e;
+        }
+    }
+
     /**
      * Returns true if the index exists.
      */

package/src/global.d.ts

@@ -0,0 +1 @@
+declare const structuredClone: <T>(value: T) => T;

package/lib/FileFetcher.d.ts

@@ -1,5 +0,0 @@
-import { TextFetcher } from './types';
-export declare class FileFetcher implements TextFetcher {
-    fetch(uri: string, onDocument: (uri: string, text: string, docType?: string | undefined) => Promise<boolean>): Promise<boolean>;
-}
-//# sourceMappingURL=FileFetcher.d.ts.map

package/lib/FileFetcher.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"FileFetcher.d.ts","sourceRoot":"","sources":["../src/FileFetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAItC,qBAAa,WAAY,YAAW,WAAW;IAC9B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC;CAyB/I"}

package/lib/FileFetcher.js

@@ -1,69 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.FileFetcher = void 0;
-const fs = __importStar(require("fs/promises"));
-const path = __importStar(require("path"));
-class FileFetcher {
-    fetch(uri, onDocument) {
-        return __awaiter(this, void 0, void 0, function* () {
-            // Does path exist and is it a directory?
-            let isDirectory;
-            try {
-                const stat = yield fs.stat(uri);
-                isDirectory = stat.isDirectory();
-            }
-            catch (_a) {
-                return true;
-            }
-            // If directory, read all files and recurse
-            if (isDirectory) {
-                const files = yield fs.readdir(uri);
-                for (const file of files) {
-                    const filePath = path.join(uri, file);
-                    yield this.fetch(filePath, onDocument);
-                }
-                return true;
-            }
-            else {
-                // Read file and call onDocument
-                const text = yield fs.readFile(uri, 'utf8');
-                const parts = uri.split('.');
-                return yield onDocument(uri, text, parts.length > 0 ? parts[parts.length - 1].toLowerCase() : undefined);
-            }
-        });
-    }
-}
-exports.FileFetcher = FileFetcher;
-//# sourceMappingURL=FileFetcher.js.map

package/lib/FileFetcher.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"FileFetcher.js","sourceRoot":"","sources":["../src/FileFetcher.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,gDAAkC;AAClC,2CAA6B;AAE7B,MAAa,WAAW;IACP,KAAK,CAAC,GAAW,EAAE,UAAyF;;YACrH,yCAAyC;YACzC,IAAI,WAAoB,CAAC;YACzB,IAAI;gBACA,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,WAAW,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;aACpC;YAAC,WAAM;gBACJ,OAAO,IAAI,CAAC;aACf;YAED,2CAA2C;YAC3C,IAAI,WAAW,EAAE;gBACb,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;oBACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;oBACtC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;iBAC1C;gBACD,OAAO,IAAI,CAAC;aACf;iBAAM;gBACH,gCAAgC;gBAChC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC7B,OAAO,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;aAC5G;QACL,CAAC;KAAA;CACJ;AA1BD,kCA0BC"}

package/lib/GPT3Tokenizer.d.ts

@@ -1,9 +0,0 @@
-import { Tokenizer } from "./types";
-/**
- * Tokenizer that uses GPT-3's encoder.
- */
-export declare class GPT3Tokenizer implements Tokenizer {
-    decode(tokens: number[]): string;
-    encode(text: string): number[];
-}
-//# sourceMappingURL=GPT3Tokenizer.d.ts.map

package/lib/GPT3Tokenizer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"GPT3Tokenizer.d.ts","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGpC;;GAEG;AACH,qBAAa,aAAc,YAAW,SAAS;IACpC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAIhC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;CAGxC"}

package/lib/GPT3Tokenizer.js

@@ -1,17 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.GPT3Tokenizer = void 0;
-const gpt_3_encoder_1 = require("gpt-3-encoder");
-/**
- * Tokenizer that uses GPT-3's encoder.
- */
-class GPT3Tokenizer {
-    decode(tokens) {
-        return (0, gpt_3_encoder_1.decode)(tokens);
-    }
-    encode(text) {
-        return (0, gpt_3_encoder_1.encode)(text);
-    }
-}
-exports.GPT3Tokenizer = GPT3Tokenizer;
-//# sourceMappingURL=GPT3Tokenizer.js.map

package/lib/GPT3Tokenizer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"GPT3Tokenizer.js","sourceRoot":"","sources":["../src/GPT3Tokenizer.ts"],"names":[],"mappings":";;;AACA,iDAA+C;AAE/C;;GAEG;AACH,MAAa,aAAa;IACf,MAAM,CAAC,MAAgB;QAC1B,OAAO,IAAA,sBAAM,EAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAA,sBAAM,EAAC,IAAI,CAAC,CAAC;IACxB,CAAC;CACJ;AARD,sCAQC"}

package/lib/ItemSelector.d.ts

@@ -1,41 +0,0 @@
-import { MetadataFilter, MetadataTypes } from './types';
-export declare class ItemSelector {
-    /**
-     * Returns the similarity between two vectors using the cosine similarity.
-     * @param vector1 Vector 1
-     * @param vector2 Vector 2
-     * @returns Similarity between the two vectors
-     */
-    static cosineSimilarity(vector1: number[], vector2: number[]): number;
-    /**
-     * Normalizes a vector.
-     * @remarks
-     * The norm of a vector is the square root of the sum of the squares of the elements.
-     * The LocalIndex pre-normalizes all vectors to improve performance.
-     * @param vector Vector to normalize
-     * @returns Normalized vector
-     */
-    static normalize(vector: number[]): number;
-    /**
-     * Returns the similarity between two vectors using cosine similarity.
-     * @remarks
-     * The LocalIndex pre-normalizes all vectors to improve performance.
-     * This method uses the pre-calculated norms to improve performance.
-     * @param vector1 Vector 1
-     * @param norm1 Norm of vector 1
-     * @param vector2 Vector 2
-     * @param norm2 Norm of vector 2
-     * @returns Similarity between the two vectors
-     */
-    static normalizedCosineSimilarity(vector1: number[], norm1: number, vector2: number[], norm2: number): number;
-    /**
-     * Applies a filter to the metadata of an item.
-     * @param metadata Metadata of the item
-     * @param filter Filter to apply
-     * @returns True if the item matches the filter, false otherwise
-     */
-    static select(metadata: Record<string, MetadataTypes>, filter: MetadataFilter): boolean;
-    private static dotProduct;
-    private static metadataFilter;
-}
-//# sourceMappingURL=ItemSelector.d.ts.map

package/lib/ItemSelector.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"ItemSelector.d.ts","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExD,qBAAa,YAAY;IACrB;;;;;OAKG;WACW,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE;IAKnE;;;;;;;OAOG;WACW,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE;IAYxC;;;;;;;;;;OAUG;WACW,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM;IAK3G;;;;;OAKG;WACW,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO;IAoC9F,OAAO,CAAC,MAAM,CAAC,UAAU;IAYzB,OAAO,CAAC,MAAM,CAAC,cAAc;CA+DhC"}

package/lib/ItemSelector.js

@@ -1,168 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ItemSelector = void 0;
-class ItemSelector {
-    /**
-     * Returns the similarity between two vectors using the cosine similarity.
-     * @param vector1 Vector 1
-     * @param vector2 Vector 2
-     * @returns Similarity between the two vectors
-     */
-    static cosineSimilarity(vector1, vector2) {
-        // Return the quotient of the dot product and the product of the norms
-        return this.dotProduct(vector1, vector2) / (this.normalize(vector1) * this.normalize(vector2));
-    }
-    /**
-     * Normalizes a vector.
-     * @remarks
-     * The norm of a vector is the square root of the sum of the squares of the elements.
-     * The LocalIndex pre-normalizes all vectors to improve performance.
-     * @param vector Vector to normalize
-     * @returns Normalized vector
-     */
-    static normalize(vector) {
-        // Initialize a variable to store the sum of the squares
-        let sum = 0;
-        // Loop through the elements of the array
-        for (let i = 0; i < vector.length; i++) {
-            // Square the element and add it to the sum
-            sum += vector[i] * vector[i];
-        }
-        // Return the square root of the sum
-        return Math.sqrt(sum);
-    }
-    /**
-     * Returns the similarity between two vectors using cosine similarity.
-     * @remarks
-     * The LocalIndex pre-normalizes all vectors to improve performance.
-     * This method uses the pre-calculated norms to improve performance.
-     * @param vector1 Vector 1
-     * @param norm1 Norm of vector 1
-     * @param vector2 Vector 2
-     * @param norm2 Norm of vector 2
-     * @returns Similarity between the two vectors
-     */
-    static normalizedCosineSimilarity(vector1, norm1, vector2, norm2) {
-        // Return the quotient of the dot product and the product of the norms
-        return this.dotProduct(vector1, vector2) / (norm1 * norm2);
-    }
-    /**
-     * Applies a filter to the metadata of an item.
-     * @param metadata Metadata of the item
-     * @param filter Filter to apply
-     * @returns True if the item matches the filter, false otherwise
-     */
-    static select(metadata, filter) {
-        if (filter === undefined || filter === null) {
-            return true;
-        }
-        for (const key in filter) {
-            switch (key) {
-                case '$and':
-                    if (!filter[key].every((f) => this.select(metadata, f))) {
-                        return false;
-                    }
-                    break;
-                case '$or':
-                    if (!filter[key].some((f) => this.select(metadata, f))) {
-                        return false;
-                    }
-                    break;
-                default:
-                    const value = filter[key];
-                    if (value === undefined || value === null) {
-                        return false;
-                    }
-                    else if (typeof value == 'object') {
-                        if (!this.metadataFilter(metadata[key], value)) {
-                            return false;
-                        }
-                    }
-                    else {
-                        if (metadata[key] !== value) {
-                            return false;
-                        }
-                    }
-                    break;
-            }
-        }
-        return true;
-    }
-    static dotProduct(arr1, arr2) {
-        // Initialize a variable to store the sum of the products
-        let sum = 0;
-        // Loop through the elements of the arrays
-        for (let i = 0; i < arr1.length; i++) {
-            // Multiply the corresponding elements and add them to the sum
-            sum += arr1[i] * arr2[i];
-        }
-        // Return the sum
-        return sum;
-    }
-    static metadataFilter(value, filter) {
-        if (value === undefined || value === null) {
-            return false;
-        }
-        for (const key in filter) {
-            switch (key) {
-                case '$eq':
-                    if (value !== filter[key]) {
-                        return false;
-                    }
-                    break;
-                case '$ne':
-                    if (value === filter[key]) {
-                        return false;
-                    }
-                    break;
-                case '$gt':
-                    if (typeof value != 'number' || value <= filter[key]) {
-                        return false;
-                    }
-                    break;
-                case '$gte':
-                    if (typeof value != 'number' || value < filter[key]) {
-                        return false;
-                    }
-                    break;
-                case '$lt':
-                    if (typeof value != 'number' || value >= filter[key]) {
-                        return false;
-                    }
-                    break;
-                case '$lte':
-                    if (typeof value != 'number' || value > filter[key]) {
-                        return false;
-                    }
-                    break;
-                case '$in':
-                    if (typeof value == 'boolean') {
-                        return false;
-                    }
-                    else if (typeof value == 'string' && !filter[key].includes(value)) {
-                        return false;
-                    }
-                    else if (!filter[key].some(val => typeof val == 'string' && val.includes(value))) {
-                        return false;
-                    }
-                    break;
-                case '$nin':
-                    if (typeof value == 'boolean') {
-                        return false;
-                    }
-                    else if (typeof value == 'string' && filter[key].includes(value)) {
-                        return false;
-                    }
-                    else if (filter[key].some(val => typeof val == 'string' && val.includes(value))) {
-                        return false;
-                    }
-                    break;
-                default:
-                    return value === filter[key];
-            }
-        }
-        return true;
-    }
-}
-exports.ItemSelector = ItemSelector;
-//# sourceMappingURL=ItemSelector.js.map

package/lib/ItemSelector.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ItemSelector.js","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":";;;AAEA,MAAa,YAAY;IACrB;;;;;OAKG;IACI,MAAM,CAAC,gBAAgB,CAAC,OAAiB,EAAE,OAAiB;QAC/D,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,CAAC;IAED;;;;;;;OAOG;IACI,MAAM,CAAC,SAAS,CAAC,MAAgB;QACpC,wDAAwD;QACxD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,yCAAyC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,2CAA2C;YAC3C,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;SAChC;QACD,oCAAoC;QACpC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,0BAA0B,CAAC,OAAiB,EAAE,KAAa,EAAE,OAAiB,EAAE,KAAa;QACvG,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;IAC/D,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,MAAM,CAAC,QAAuC,EAAE,MAAsB;QAChF,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAE;YACzC,OAAO,IAAI,CAAC;SACf;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,MAAM;oBACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,KAAK,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACtE,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACrE,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;wBACvC,OAAO,KAAK,CAAC;qBAChB;yBAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE;wBACjC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,KAAuB,CAAC,EAAE;4BAC9D,OAAO,KAAK,CAAC;yBAChB;qBACJ;yBAAM;wBACH,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE;4BACzB,OAAO,KAAK,CAAC;yBAChB;qBACJ;oBACD,MAAM;aACb;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,UAAU,CAAC,IAAc,EAAE,IAAc;QACpD,yDAAyD;QACzD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,0CAA0C;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,8DAA8D;YAC9D,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;SAC5B;QACD,iBAAiB;QACjB,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,KAAoB,EAAE,MAAsB;QACtE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;YACvC,OAAO,KAAK,CAAC;SAChB;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAE,EAAE;wBACnD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAE,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAE,EAAE;wBACnD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAE,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,SAAS,EAAE;wBAC3B,OAAO,KAAK,CAAC;qBAChB;yBAAM,IAAG,OAAO,KAAK,IAAI,QAAQ,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAC;wBAChE,OAAO,KAAK,CAAA;qBACf;yBAAM,IAAG,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,GAAG,IAAI,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAe,CAAC,CAAC,EAAC;wBACzF,OAAO,KAAK,CAAA;qBACf;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,SAAS,EAAE;wBAC3B,OAAO,KAAK,CAAC;qBAChB;yBACI,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,MAAM,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC/D,OAAO,KAAK,CAAC;qBAChB;yBACI,IAAI,MAAM,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,GAAG,IAAI,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAe,CAAC,CAAC,EAAE;wBACxF,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,OAAO,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;aACpC;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ;AArKD,oCAqKC"}

package/lib/LocalDocument.d.ts

@@ -1,54 +0,0 @@
-import { MetadataTypes } from './types';
-import { LocalDocumentIndex } from './LocalDocumentIndex';
-/**
- * Represents an indexed document stored on disk.
- */
-export declare class LocalDocument {
-    private readonly _index;
-    private readonly _id;
-    private readonly _uri;
-    private _metadata;
-    private _text;
-    /**
-     * Creates a new `LocalDocument` instance.
-     * @param index Parent index that contains the document.
-     * @param id ID of the document.
-     * @param uri URI of the document.
-     */
-    constructor(index: LocalDocumentIndex, id: string, uri: string);
-    /**
-     * Returns the folder path where the document is stored.
-     */
-    get folderPath(): string;
-    /**
-     * Returns the ID of the document.
-     */
-    get id(): string;
-    /**
-     * Returns the URI of the document.
-     */
-    get uri(): string;
-    /**
-     * Returns the length of the document in tokens.
-     * @remarks
-     * This value will be estimated for documents longer then 40k bytes.
-     * @returns Length of the document in tokens.
-     */
-    getLength(): Promise<number>;
-    /**
-     * Determines if the document has additional metadata storred on disk.
-     * @returns True if the document has metadata; otherwise, false.
-     */
-    hasMetadata(): Promise<boolean>;
-    /**
-     * Loads the metadata for the document from disk.
-     * @returns Metadata for the document.
-     */
-    loadMetadata(): Promise<Record<string, MetadataTypes>>;
-    /**
-     * Loads the text for the document from disk.
-     * @returns Text for the document.
-     */
-    loadText(): Promise<string>;
-}
-//# sourceMappingURL=LocalDocument.d.ts.map

package/lib/LocalDocument.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"LocalDocument.d.ts","sourceRoot":"","sources":["../src/LocalDocument.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D;;GAEG;AACH,qBAAa,aAAa;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,SAAS,CAAyC;IAC1D,OAAO,CAAC,KAAK,CAAmB;IAEhC;;;;;OAKG;gBACgB,KAAK,EAAE,kBAAkB,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;IAMrE;;OAEG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;OAEG;IACH,IAAW,EAAE,IAAI,MAAM,CAEtB;IAED;;OAEG;IACH,IAAW,GAAG,IAAI,MAAM,CAEvB;IAED;;;;;OAKG;IACU,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;IASzC;;;OAGG;IACU,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAS5C;;;OAGG;IACU,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,CAAC;IAmBlE;;;OAGG;IACU,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;CAW3C"}