vectra 0.12.3 → 0.14.0

package/lib/LocalEmbeddings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalEmbeddings.js","sourceRoot":"","sources":["../src/LocalEmbeddings.ts"],"names":[],"mappings":";;;;;;;;;;;;AAuBA;;;;;;GAMG;AACH,MAAa,eAAe;IAOxB;;;OAGG;IACH,YAAmB,OAAgC;;QAP3C,cAAS,GAAQ,IAAI,CAAC;QACtB,qBAAgB,GAAwB,IAAI,CAAC;QAOjD,IAAI,CAAC,UAAU,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,mCAAI,yBAAyB,CAAC;QAC9D,IAAI,CAAC,SAAS,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,SAAS,mCAAI,GAAG,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACU,gBAAgB,CAAC,MAAyB;;YACnD,IAAI,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;gBACtC,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;gBAC7D,MAAM,MAAM,GAAe,EAAE,CAAC;gBAE9B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE;wBAC7B,OAAO,EAAE,MAAM;wBACf,SAAS,EAAE,IAAI;qBAClB,CAAC,CAAC;oBACH,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAoB,CAAC,CAAC,CAAC;gBACzD,CAAC;gBAED,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;YACzC,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACpB,OAAO;oBACH,MAAM,EAAE,OAAO;oBACf,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC5D,CAAC;YACN,CAAC;QACL,CAAC;KAAA;IAED;;;;OAIG;IACW,WAAW;;YACrB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC,SAAS,CAAC;YAC1B,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACzB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YAChD,CAAC;YAED,OAAO,IAAI,CAAC,gBAAgB,CAAC;QACjC,CAAC;KAAA;IAED;;OAEG;IACW,YAAY;;YACtB,IAAI,YAAiB,CAAC;YACtB,IAAI,CAAC;gBACD,YAAY,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;YACxD,CAAC;YAAC,WAAM,CAAC;gBACL,MAAM,IAAI,KAAK,CACX,0EAA0E;oBAC1E,wDAAwD,CAC3D,CAAC;YACN,CAAC;YAED,IAAI,CAAC,SAAS,GAAG,MAAM,YAAY,CAAC,QAAQ,CACxC,oBAAoB,EACpB,IAAI,CAAC,UAAU,CAClB,CAAC;YAEF,OAAO,IAAI,CAAC,SAAS,CAAC;QAC1B,CAAC;KAAA;CACJ;AAzFD,0CAyFC"}

package/lib/LocalEmbeddings.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=LocalEmbeddings.spec.d.ts.map

package/lib/LocalEmbeddings.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalEmbeddings.spec.d.ts","sourceRoot":"","sources":["../src/LocalEmbeddings.spec.ts"],"names":[],"mappings":""}

package/lib/LocalEmbeddings.spec.js

@@ -0,0 +1,155 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const assert = __importStar(require("assert"));
+const sinon = __importStar(require("sinon"));
+const LocalEmbeddings_1 = require("./LocalEmbeddings");
+describe('LocalEmbeddings', () => {
+    let requireStub;
+    let fakePipeline;
+    let fakePipelineFn;
+    beforeEach(() => {
+        // Create a fake pipeline function that returns tensor-like results
+        fakePipelineFn = sinon.stub();
+        fakePipeline = sinon.stub().resolves(fakePipelineFn);
+        // Stub require to intercept @huggingface/transformers
+        requireStub = sinon.stub(module.constructor.prototype, 'require');
+        requireStub.callThrough(); // Allow all other requires to pass through
+        requireStub.withArgs('@huggingface/transformers').returns({
+            pipeline: fakePipeline,
+        });
+    });
+    afterEach(() => {
+        sinon.restore();
+    });
+    describe('constructor', () => {
+        it('should use default model and maxTokens', () => {
+            const embeddings = new LocalEmbeddings_1.LocalEmbeddings();
+            assert.strictEqual(embeddings.model, 'Xenova/all-MiniLM-L6-v2');
+            assert.strictEqual(embeddings.maxTokens, 256);
+        });
+        it('should accept custom model name', () => {
+            const embeddings = new LocalEmbeddings_1.LocalEmbeddings({ model: 'custom/model' });
+            assert.strictEqual(embeddings.model, 'custom/model');
+        });
+        it('should accept custom maxTokens', () => {
+            const embeddings = new LocalEmbeddings_1.LocalEmbeddings({ maxTokens: 512 });
+            assert.strictEqual(embeddings.maxTokens, 512);
+        });
+    });
+    describe('createEmbeddings', () => {
+        it('should create embeddings for a single string input', () => __awaiter(void 0, void 0, void 0, function* () {
+            const fakeVector = new Float32Array([0.1, 0.2, 0.3]);
+            fakePipelineFn.resolves({ data: fakeVector });
+            const embeddings = new LocalEmbeddings_1.LocalEmbeddings();
+            const result = yield embeddings.createEmbeddings('hello world');
+            assert.strictEqual(result.status, 'success');
+            assert.ok(result.output);
+            assert.strictEqual(result.output.length, 1);
+            assert.deepStrictEqual(result.output[0], [0.10000000149011612, 0.20000000298023224, 0.30000001192092896]);
+        }));
+        it('should create embeddings for an array of inputs', () => __awaiter(void 0, void 0, void 0, function* () {
+            const vec1 = new Float32Array([0.1, 0.2, 0.3]);
+            const vec2 = new Float32Array([0.4, 0.5, 0.6]);
+            fakePipelineFn.onFirstCall().resolves({ data: vec1 });
+            fakePipelineFn.onSecondCall().resolves({ data: vec2 });
+            const embeddings = new LocalEmbeddings_1.LocalEmbeddings();
+            const result = yield embeddings.createEmbeddings(['hello', 'world']);
+            assert.strictEqual(result.status, 'success');
+            assert.ok(result.output);
+            assert.strictEqual(result.output.length, 2);
+        }));
+        it('should call pipeline with mean pooling and normalize', () => __awaiter(void 0, void 0, void 0, function* () {
+            const fakeVector = new Float32Array([0.5, 0.5]);
+            fakePipelineFn.resolves({ data: fakeVector });
+            const embeddings = new LocalEmbeddings_1.LocalEmbeddings();
+            yield embeddings.createEmbeddings('test');
+            assert.ok(fakePipelineFn.calledOnce);
+            const [input, options] = fakePipelineFn.firstCall.args;
+            assert.strictEqual(input, 'test');
+            assert.deepStrictEqual(options, { pooling: 'mean', normalize: true });
+        }));
+        it('should initialize the pipeline with correct model name', () => __awaiter(void 0, void 0, void 0, function* () {
+            const fakeVector = new Float32Array([0.5]);
+            fakePipelineFn.resolves({ data: fakeVector });
+            const embeddings = new LocalEmbeddings_1.LocalEmbeddings({ model: 'my/model' });
+            yield embeddings.createEmbeddings('test');
+            assert.ok(fakePipeline.calledOnce);
+            const [task, model] = fakePipeline.firstCall.args;
+            assert.strictEqual(task, 'feature-extraction');
+            assert.strictEqual(model, 'my/model');
+        }));
+        it('should reuse the pipeline across calls', () => __awaiter(void 0, void 0, void 0, function* () {
+            const fakeVector = new Float32Array([0.5]);
+            fakePipelineFn.resolves({ data: fakeVector });
+            const embeddings = new LocalEmbeddings_1.LocalEmbeddings();
+            yield embeddings.createEmbeddings('first');
+            yield embeddings.createEmbeddings('second');
+            // Pipeline should only be created once
+            assert.ok(fakePipeline.calledOnce);
+            // But the pipeline function should be called twice
+            assert.strictEqual(fakePipelineFn.callCount, 2);
+        }));
+        it('should return error status when pipeline fails', () => __awaiter(void 0, void 0, void 0, function* () {
+            fakePipelineFn.rejects(new Error('Model not found'));
+            const embeddings = new LocalEmbeddings_1.LocalEmbeddings();
+            const result = yield embeddings.createEmbeddings('test');
+            assert.strictEqual(result.status, 'error');
+            assert.ok(result.message);
+            assert.ok(result.message.includes('Model not found'));
+        }));
+    });
+    describe('optional dependency', () => {
+        it('should return error when @huggingface/transformers is not installed', () => __awaiter(void 0, void 0, void 0, function* () {
+            requireStub.withArgs('@huggingface/transformers').throws(new Error('Cannot find module'));
+            const embeddings = new LocalEmbeddings_1.LocalEmbeddings();
+            const result = yield embeddings.createEmbeddings('test');
+            assert.strictEqual(result.status, 'error');
+            assert.ok(result.message);
+            assert.ok(result.message.includes('@huggingface/transformers'));
+            assert.ok(result.message.includes('npm install'));
+        }));
+    });
+});
+//# sourceMappingURL=LocalEmbeddings.spec.js.map

package/lib/LocalEmbeddings.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalEmbeddings.spec.js","sourceRoot":"","sources":["../src/LocalEmbeddings.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AACjC,6CAA+B;AAC/B,uDAAoD;AAEpD,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC7B,IAAI,WAA4B,CAAC;IACjC,IAAI,YAA6B,CAAC;IAClC,IAAI,cAA+B,CAAC;IAEpC,UAAU,CAAC,GAAG,EAAE;QACZ,mEAAmE;QACnE,cAAc,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9B,YAAY,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QAErD,sDAAsD;QACtD,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAClE,WAAW,CAAC,WAAW,EAAE,CAAC,CAAC,2CAA2C;QACtE,WAAW,CAAC,QAAQ,CAAC,2BAA2B,CAAC,CAAC,OAAO,CAAC;YACtD,QAAQ,EAAE,YAAY;SACzB,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACX,KAAK,CAAC,OAAO,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAC9C,MAAM,UAAU,GAAG,IAAI,iCAAe,EAAE,CAAC;YACzC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC;YAChE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACvC,MAAM,UAAU,GAAG,IAAI,iCAAe,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;YAClE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACtC,MAAM,UAAU,GAAG,IAAI,iCAAe,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;YAC3D,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,oDAAoD,EAAE,GAAS,EAAE;YAChE,MAAM,UAAU,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YACrD,cAAc,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAE9C,MAAM,UAAU,GAAG,IAAI,iCAAe,EAAE,CAAC;YACzC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;YAEhE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC7C,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,MAAO,CAAC,CAAC,CAAC,EAAE,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC/G,CAAC,CAAA,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,GAAS,EAAE;YAC7D,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/C,cAAc,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,cAAc,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAEvD,MAAM,UAAU,GAAG,IAAI,iCAAe,EAAE,CAAC;YACzC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YAErE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC7C,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC,CAAA,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,GAAS,EAAE;YAClE,MAAM,UAAU,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YAChD,cAAc,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAE9C,MAAM,UAAU,GAAG,IAAI,iCAAe,EAAE,CAAC;YACzC,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAE1C,MAAM,CAAC,EAAE,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;YACrC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC;YACvD,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAClC,MAAM,CAAC,eAAe,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1E,CAAC,CAAA,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,GAAS,EAAE;YACpE,MAAM,UAAU,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3C,cAAc,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAE9C,MAAM,UAAU,GAAG,IAAI,iCAAe,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;YAC9D,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAE1C,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YACnC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC;YAClD,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;YAC/C,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC1C,CAAC,CAAA,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAS,EAAE;YACpD,MAAM,UAAU,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3C,cAAc,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAE9C,MAAM,UAAU,GAAG,IAAI,iCAAe,EAAE,CAAC;YACzC,MAAM,UAAU,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,UAAU,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAE5C,uCAAuC;YACvC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YACnC,mDAAmD;YACnD,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC,CAAA,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,GAAS,EAAE;YAC5D,cAAc,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAErD,MAAM,UAAU,GAAG,IAAI,iCAAe,EAAE,CAAC;YACzC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAEzD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC3C,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1B,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAQ,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAC3D,CAAC,CAAA,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,qEAAqE,EAAE,GAAS,EAAE;YACjF,WAAW,CAAC,QAAQ,CAAC,2BAA2B,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAE1F,MAAM,UAAU,GAAG,IAAI,iCAAe,EAAE,CAAC;YACzC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAEzD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC3C,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1B,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAQ,CAAC,QAAQ,CAAC,2BAA2B,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;QACvD,CAAC,CAAA,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}

package/lib/LocalIndex.d.ts

@@ -1,4 +1,6 @@
 import { IndexItem, IndexStats, MetadataFilter, MetadataTypes, QueryResult } from './types';
+import { FileStorage } from './storage';
+import { IndexCodec } from './codecs';
 export interface CreateIndexConfig {
     version: number;
     deleteIfExists?: boolean;
@@ -15,23 +17,33 @@ export interface CreateIndexConfig {
 export declare class LocalIndex<TMetadata extends Record<string, MetadataTypes> = Record<string, MetadataTypes>> {
     private readonly _folderPath;
     private readonly _indexName;
+    private readonly _storage;
+    private readonly _codec;
     private _data?;
     private _update?;
     private _bm25Engine;
+    private readonly _bm25Factory;
+    private readonly _docReader;
     /**
      * Creates a new instance of LocalIndex.
      * @param folderPath Path to the index folder.
-     * @param indexName Optional name of the index file. Defaults to index.json.
-     */
-    constructor(folderPath: string, indexName?: string);
-    /**
-     * Path to the index folder.
-     */
+     * @param indexName Optional index file name. Defaults to 'index' + codec.extension.
+     * @param storage Optional file storage instance. Defaults to LocalFileStorage.
+     * @param codec Optional codec for serialization. Defaults to JsonCodec.
+     * @param options Optional constructor options for dependency injection.
+     */
+    constructor(folderPath: string, indexName?: string, storage?: FileStorage, codec?: IndexCodec, options?: {
+        bm25Factory?: () => any;
+        docReader?: (docId: string) => Promise<string>;
+    });
+    /** Path to the index folder. */
     get folderPath(): string;
-    /**
-     * Optional name of the index file.
-     */
+    /** Name of the index file. */
     get indexName(): string;
+    /** Storage provider used to store the index. */
+    get storage(): FileStorage;
+    /** Codec used for serialization. */
+    get codec(): IndexCodec;
     /**
      * Begins an update to the index.
      * @remarks
@@ -98,9 +110,7 @@ export declare class LocalIndex<TMetadata extends Record<string, MetadataTypes>
      * @returns Inserted items.
      */
     batchInsertItems<TItemMetadata extends TMetadata = TMetadata>(items: Partial<IndexItem<TItemMetadata>>[]): Promise<IndexItem[]>;
-    /**
-     * Returns true if the index exists.
-     */
+    /** Returns true if the index exists. */
     isIndexCreated(): Promise<boolean>;
     /**
      * Returns all items in the index.
@@ -124,10 +134,11 @@ export declare class LocalIndex<TMetadata extends Record<string, MetadataTypes>
      * This method loads the index into memory and returns the top k items that are most similar.
      * An optional filter can be applied to the metadata of the items.
      * @param vector Vector to query against.
-     * @param query Query text used for BM25 keyword search.
+     * @param query Query string (used when isBm25=true).
      * @param topK Number of items to return.
      * @param filter Optional. Filter to apply.
-     * @returns Similar items to the vector that matche the supplied filter.
+     * @param isBm25 Optional. If true, append BM25 keyword results to semantic results.
+     * @returns Similar items to the vector that match the supplied filter.
      */
     queryItems<TItemMetadata extends TMetadata = TMetadata>(vector: number[], query: string, topK: number, filter?: MetadataFilter, isBm25?: boolean): Promise<QueryResult<TItemMetadata>[]>;
     /**
@@ -139,12 +150,10 @@ export declare class LocalIndex<TMetadata extends Record<string, MetadataTypes>
      * @returns Upserted item.
      */
     upsertItem<TItemMetadata extends TMetadata = TMetadata>(item: Partial<IndexItem<TItemMetadata>>): Promise<IndexItem<TItemMetadata>>;
-    /**
-     * Ensures that the index has been loaded into memory.
-     */
+    /** Ensures that the index has been loaded into memory. */
     protected loadIndexData(): Promise<void>;
     private addItemToUpdate;
-    private setupbm25;
+    private setupBm25;
     private bm25Search;
 }
 //# sourceMappingURL=LocalIndex.d.ts.map

package/lib/LocalIndex.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"LocalIndex.d.ts","sourceRoot":"","sources":["../src/LocalIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAM5F,MAAM,WAAW,iBAAiB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE;QACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACL;AAED;;;;;GAKG;AACH,qBAAa,UAAU,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IACjG,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IAEpC,OAAO,CAAC,KAAK,CAAC,CAAY;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAY;IAE5B,OAAO,CAAC,WAAW,CAAM;IAEzB;;;;OAIG;gBACgB,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;IAKzD;;OAEG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;OAEG;IACH,IAAW,SAAS,IAAI,MAAM,CAE7B;IAED;;;;OAIG;IACU,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IASzC;;;;OAIG;IACI,YAAY,IAAI,IAAI;IAI3B;;;;;OAKG;IACU,WAAW,CAAC,MAAM,GAAE,iBAAgC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BjF;;;;OAIG;IACI,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;;OAGG;IACU,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBlD;;;;OAIG;IACU,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAevC;;;OAGG;IACU,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC;IASjD;;;;OAIG;IACU,OAAO,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,SAAS,CAAC;IAK5H;;;;;;;OAOG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAWhJ;;;;;;;;OAQG;IACU,gBAAgB,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAiB5I;;OAEG;IACU,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAS/C;;;;;;OAMG;IACU,SAAS,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,KAAK,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;IAK1G;;;;;;OAMG;IACU,mBAAmB,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;IAK1I;;;;;;;;;;OAUG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;IAuErM;;;;;;;OAOG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAWhJ;;OAEG;cACa,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;YAahC,eAAe;YA+Df,SAAS;YAsBT,UAAU;CAQ3B"}
+{"version":3,"file":"LocalIndex.d.ts","sourceRoot":"","sources":["../src/LocalIndex.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAM5F,OAAO,EAAE,WAAW,EAAoB,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAa,MAAM,UAAU,CAAC;AAEjD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE;QAChB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC;CACH;AAED;;;;;GAKG;AACH,qBAAa,UAAU,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC;IACrG,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAwB;IACnD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAc;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,KAAK,CAAC,CAAY;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAY;IAG5B,OAAO,CAAC,WAAW,CAAM;IACzB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAY;IACzC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqC;IAEhE;;;;;;;OAOG;gBAED,UAAU,EAAE,MAAM,EAClB,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,WAAW,EACrB,KAAK,CAAC,EAAE,UAAU,EAClB,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,GAAG,CAAC;QACxB,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;KAChD;IAiBH,gCAAgC;IAChC,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED,8BAA8B;IAC9B,IAAW,SAAS,IAAI,MAAM,CAE7B;IAED,gDAAgD;IAChD,IAAW,OAAO,IAAI,WAAW,CAEhC;IAED,oCAAoC;IACpC,IAAW,KAAK,IAAI,UAAU,CAE7B;IAED;;;;OAIG;IACU,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAQzC;;;;OAIG;IACI,YAAY,IAAI,IAAI;IAI3B;;;;;OAKG;IACU,WAAW,CAAC,MAAM,GAAE,iBAAkC,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BnF;;;;OAIG;IACU,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAKzC;;;OAGG;IACU,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBlD;;;;OAIG;IACU,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAevC;;;OAGG;IACU,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC;IASjD;;;;OAIG;IACU,OAAO,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,SAAS,CAAC;IAK5H;;;;;;;OAOG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAWhJ;;;;;;;;OAQG;IACU,gBAAgB,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAiB5I,wCAAwC;IAC3B,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAI/C;;;;;;OAMG;IACU,SAAS,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,KAAK,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;IAK1G;;;;;;OAMG;IACU,mBAAmB,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;IAK1I;;;;;;;;;;;OAWG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EACjE,MAAM,EAAE,MAAM,EAAE,EAChB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,cAAc,EACvB,MAAM,CAAC,EAAE,OAAO,GACf,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;IA0FxC;;;;;;;OAOG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAWhJ,0DAA0D;cAC1C,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;YAYhC,eAAe;YAyEf,SAAS;YAmBT,UAAU;CAIzB"}

package/lib/LocalIndex.js

@@ -1,37 +1,4 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -46,14 +13,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LocalIndex = void 0;
-const fs = __importStar(require("fs/promises"));
-const path = __importStar(require("path"));
+const pathUtils_1 = require("./utils/pathUtils");
 const uuid_1 = require("uuid");
 const ItemSelector_1 = require("./ItemSelector");
 const LocalDocument_1 = require("./LocalDocument");
 const wink_bm25_text_search_1 = __importDefault(require("wink-bm25-text-search"));
 const wink_nlp_1 = __importDefault(require("wink-nlp"));
 const wink_eng_lite_web_model_1 = __importDefault(require("wink-eng-lite-web-model"));
+const storage_1 = require("./storage");
+const codecs_1 = require("./codecs");
 /**
  * Local vector index instance.
  * @remarks
@@ -64,24 +32,44 @@ class LocalIndex {
     /**
      * Creates a new instance of LocalIndex.
      * @param folderPath Path to the index folder.
-     * @param indexName Optional name of the index file. Defaults to index.json.
+     * @param indexName Optional index file name. Defaults to 'index' + codec.extension.
+     * @param storage Optional file storage instance. Defaults to LocalFileStorage.
+     * @param codec Optional codec for serialization. Defaults to JsonCodec.
+     * @param options Optional constructor options for dependency injection.
      */
-    constructor(folderPath, indexName) {
+    constructor(folderPath, indexName, storage, codec, options) {
+        this._indexName = 'index.json';
         this._folderPath = folderPath;
-        this._indexName = indexName || "index.json";
+        this._codec = codec || new codecs_1.JsonCodec();
+        if (indexName) {
+            this._indexName = indexName;
+        }
+        else {
+            this._indexName = `index${this._codec.extension}`;
+        }
+        this._storage = storage || new storage_1.LocalFileStorage();
+        this._bm25Factory = (options === null || options === void 0 ? void 0 : options.bm25Factory) || (() => (0, wink_bm25_text_search_1.default)());
+        this._docReader = (options === null || options === void 0 ? void 0 : options.docReader) || ((docId) => __awaiter(this, void 0, void 0, function* () {
+            const doc = new LocalDocument_1.LocalDocument(this, docId, '');
+            return yield doc.loadText();
+        }));
     }
-    /**
-     * Path to the index folder.
-     */
+    /** Path to the index folder. */
     get folderPath() {
         return this._folderPath;
     }
-    /**
-     * Optional name of the index file.
-     */
+    /** Name of the index file. */
     get indexName() {
         return this._indexName;
     }
+    /** Storage provider used to store the index. */
+    get storage() {
+        return this._storage;
+    }
+    /** Codec used for serialization. */
+    get codec() {
+        return this._codec;
+    }
     /**
      * Begins an update to the index.
      * @remarks
@@ -124,16 +112,16 @@ class LocalIndex {
             }
             try {
                 // Create folder for index
-                yield fs.mkdir(this._folderPath, { recursive: true });
+                yield this.storage.createFolder(this._folderPath);
                 // Initialize index.json file
                 this._data = {
                     version: config.version,
                     metadata_config: (_a = config.metadata_config) !== null && _a !== void 0 ? _a : {},
                     items: []
                 };
-                yield fs.writeFile(path.join(this._folderPath, this._indexName), JSON.stringify(this._data));
+                yield this.storage.upsertFile(pathUtils_1.pathUtils.join(this._folderPath, this._indexName), this._codec.serializeIndex(this._data));
             }
-            catch (err) {
+            catch (_b) {
                 yield this.deleteIndex();
                 throw new Error('Error creating index');
             }
@@ -145,10 +133,9 @@ class LocalIndex {
      * This method deletes the index folder from disk.
      */
     deleteIndex() {
-        this._data = undefined;
-        return fs.rm(this._folderPath, {
-            recursive: true,
-            maxRetries: 3
+        return __awaiter(this, void 0, void 0, function* () {
+            this._data = undefined;
+            return yield this.storage.deleteFolder(this._folderPath);
         });
     }
     /**
@@ -185,7 +172,7 @@ class LocalIndex {
             }
             try {
                 // Save index
-                yield fs.writeFile(path.join(this._folderPath, this._indexName), JSON.stringify(this._update));
+                yield this.storage.upsertFile(pathUtils_1.pathUtils.join(this._folderPath, this._indexName), this._codec.serializeIndex(this._update));
                 this._data = this._update;
                 this._update = undefined;
             }
@@ -268,18 +255,10 @@ class LocalIndex {
             }
         });
     }
-    /**
-     * Returns true if the index exists.
-     */
+    /** Returns true if the index exists. */
     isIndexCreated() {
         return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield fs.access(path.join(this._folderPath, this.indexName));
-                return true;
-            }
-            catch (err) {
-                return false;
-            }
+            return yield this.storage.pathExists(pathUtils_1.pathUtils.join(this._folderPath, this._indexName));
         });
     }
     /**
@@ -314,13 +293,15 @@ class LocalIndex {
      * This method loads the index into memory and returns the top k items that are most similar.
      * An optional filter can be applied to the metadata of the items.
      * @param vector Vector to query against.
-     * @param query Query text used for BM25 keyword search.
+     * @param query Query string (used when isBm25=true).
      * @param topK Number of items to return.
      * @param filter Optional. Filter to apply.
-     * @returns Similar items to the vector that matche the supplied filter.
+     * @param isBm25 Optional. If true, append BM25 keyword results to semantic results.
+     * @returns Similar items to the vector that match the supplied filter.
      */
     queryItems(vector, query, topK, filter, isBm25) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c, _d;
             yield this.loadIndexData();
             // Filter items
             let items = this._data.items;
@@ -347,37 +328,56 @@ class LocalIndex {
             // Load external metadata
             for (const item of top) {
                 if (item.item.metadataFile) {
-                    const metadataPath = path.join(this._folderPath, item.item.metadataFile);
-                    const metadata = yield fs.readFile(metadataPath);
-                    item.item.metadata = JSON.parse(metadata.toString());
+                    const metadataPath = pathUtils_1.pathUtils.join(this._folderPath, item.item.metadataFile);
+                    const metadataBuffer = yield this.storage.readFile(metadataPath);
+                    item.item.metadata = this._codec.deserializeMetadata(metadataBuffer);
                 }
             }
-            //Peform bm25 search only if enabled. Avoid duplicate chunks, which are already selected during semantic search.
+            // Perform bm25 search only if enabled. Avoid duplicate chunks that are already selected during semantic search.
             if (isBm25) {
                 const itemSet = new Set();
-                for (const item of top)
-                    itemSet.add(item.item.id);
-                this.setupbm25();
-                let currDoc;
-                let currDocTxt;
+                for (const r of top)
+                    itemSet.add(r.item.id);
+                // Set up BM25 engine
+                yield this.setupBm25();
+                // Add docs if we have necessary metadata; guard everything to avoid crashes
                 for (let i = 0; i < items.length; i++) {
                     if (!itemSet.has(items[i].id)) {
                         const item = items[i];
-                        currDoc = new LocalDocument_1.LocalDocument(this, item.metadata.documentId.toString(), '');
-                        currDocTxt = yield currDoc.loadText();
-                        const startPos = item.metadata.startPos;
-                        const endPos = item.metadata.endPos;
-                        const chunkText = currDocTxt.substring(Number(startPos), Number(endPos) + 1);
-                        this._bm25Engine.addDoc({ body: chunkText }, i);
+                        const md = item.metadata || {};
+                        if (md.documentId != undefined && md.startPos != undefined && md.endPos != undefined) {
+                            try {
+                                const currDocTxt = yield this._docReader(String(md.documentId));
+                                const startPos = Number(md.startPos);
+                                const endPos = Number(md.endPos);
+                                const chunkText = currDocTxt.substring(startPos, endPos + 1);
+                                (_b = (_a = this._bm25Engine).addDoc) === null || _b === void 0 ? void 0 : _b.call(_a, { body: chunkText }, i);
+                            }
+                            catch (_e) {
+                                // Ignore load or engine errors for BM25 doc prep
+                            }
+                        }
                     }
                 }
-                this._bm25Engine.consolidate();
-                var results = yield this.bm25Search(query, items, topK);
+                (_d = (_c = this._bm25Engine).consolidate) === null || _d === void 0 ? void 0 : _d.call(_c);
+                const results = yield this.bm25Search(query, items, topK);
                 results.forEach((res) => {
-                    top.push({
-                        item: Object.assign({}, Object.assign(Object.assign({}, items[res[0]]), { metadata: Object.assign(Object.assign({}, items[res[0]].metadata), { isBm25: true }) })),
-                        score: res[1]
-                    });
+                    var _a;
+                    // Support both [index, score] tuples and { item, score } objects
+                    if (Array.isArray(res)) {
+                        const idx = res[0];
+                        const score = res[1];
+                        if (items[idx]) {
+                            top.push({
+                                item: Object.assign({}, Object.assign(Object.assign({}, items[idx]), { metadata: Object.assign(Object.assign({}, items[idx].metadata), { isBm25: true }) })),
+                                score
+                            });
+                        }
+                    }
+                    else if (res && typeof res === 'object' && 'item' in res && 'score' in res) {
+                        const objItem = Object.assign({}, Object.assign(Object.assign({}, (res.item || {})), { metadata: Object.assign(Object.assign({}, (((_a = res.item) === null || _a === void 0 ? void 0 : _a.metadata) || {})), { isBm25: true }) }));
+                        top.push({ item: objItem, score: res.score });
+                    }
                 });
             }
             return top;
@@ -404,9 +404,7 @@ class LocalIndex {
             }
         });
     }
-    /**
-     * Ensures that the index has been loaded into memory.
-     */
+    /** Ensures that the index has been loaded into memory. */
     loadIndexData() {
         return __awaiter(this, void 0, void 0, function* () {
             if (this._data) {
@@ -415,13 +413,13 @@ class LocalIndex {
             if (!(yield this.isIndexCreated())) {
                 throw new Error('Index does not exist');
             }
-            const data = yield fs.readFile(path.join(this._folderPath, this.indexName));
-            this._data = JSON.parse(data.toString());
+            const data = yield this.storage.readFile(pathUtils_1.pathUtils.join(this._folderPath, this.indexName));
+            this._data = this._codec.deserializeIndex(data);
         });
     }
     addItemToUpdate(item, unique) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a;
+            var _a, _b;
             // Ensure vector is provided
             if (!item.vector) {
                 throw new Error('Vector is required');
@@ -437,17 +435,25 @@ class LocalIndex {
             // Check for indexed metadata
             let metadata = {};
             let metadataFile;
-            if (this._update.metadata_config.indexed && this._update.metadata_config.indexed.length > 0 && item.metadata) {
+            const indexedKeys = (_b = this._update.metadata_config.indexed) !== null && _b !== void 0 ? _b : [];
+            if (indexedKeys.length > 0 && item.metadata) {
                 // Copy only indexed metadata
-                for (const key of this._update.metadata_config.indexed) {
-                    if (item.metadata && item.metadata[key]) {
-                        metadata[key] = item.metadata[key];
+                const indexedOnly = {};
+                for (const key of indexedKeys) {
+                    if (Object.prototype.hasOwnProperty.call(item.metadata, key)) {
+                        indexedOnly[key] = item.metadata[key];
                     }
                 }
-                // Save remaining metadata to disk
-                metadataFile = `${(0, uuid_1.v4)()}.json`;
-                const metadataPath = path.join(this._folderPath, metadataFile);
-                yield fs.writeFile(metadataPath, JSON.stringify(item.metadata));
+                // Determine if there are any non-indexed keys
+                const hasNonIndexed = Object.keys(item.metadata).some(k => !indexedKeys.includes(k));
+                // Always store only indexed keys in the index
+                metadata = indexedOnly;
+                // Write full metadata externally only if there are non-indexed keys present
+                if (hasNonIndexed) {
+                    metadataFile = `${(0, uuid_1.v4)()}${this._codec.extension}`;
+                    const metadataPath = pathUtils_1.pathUtils.join(this._folderPath, metadataFile);
+                    yield this.storage.upsertFile(metadataPath, this._codec.serializeMetadata(item.metadata));
+                }
             }
             else if (item.metadata) {
                 metadata = item.metadata;
@@ -482,9 +488,9 @@ class LocalIndex {
             }
         });
     }
-    setupbm25() {
+    setupBm25() {
         return __awaiter(this, void 0, void 0, function* () {
-            this._bm25Engine = (0, wink_bm25_text_search_1.default)();
+            this._bm25Engine = this._bm25Factory();
             const nlp = (0, wink_nlp_1.default)(wink_eng_lite_web_model_1.default);
             const its = nlp.its;
             const prepTask = function (text) {
@@ -502,11 +508,9 @@ class LocalIndex {
             this._bm25Engine.definePrepTasks([prepTask]);
         });
     }
-    bm25Search(searchQuery, items, topK) {
+    bm25Search(searchQuery, _items, topK) {
         return __awaiter(this, void 0, void 0, function* () {
-            var query = searchQuery;
-            // `results` is an array of [ doc-id, score ], sorted by score
-            var results = this._bm25Engine.search(query);
+            const results = this._bm25Engine.search(searchQuery);
             return results.slice(0, topK);
         });
     }