vectra 0.12.2 → 0.14.0

package/lib/codecs/ProtobufCodec.spec.js

@@ -0,0 +1,155 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_assert_1 = __importDefault(require("node:assert"));
+const ProtobufCodec_1 = require("./ProtobufCodec");
+const JsonCodec_1 = require("./JsonCodec");
+describe('ProtobufCodec', () => {
+    const codec = new ProtobufCodec_1.ProtobufCodec();
+    it('has .pb extension', () => {
+        node_assert_1.default.equal(codec.extension, '.pb');
+    });
+    describe('serializeIndex / deserializeIndex', () => {
+        it('round-trips IndexData', () => {
+            const data = {
+                version: 1,
+                metadata_config: { indexed: ['category'] },
+                items: [
+                    { id: 'a', metadata: { category: 'food', score: 42 }, vector: [1.5, 2.5, 3.5], norm: 4.5 },
+                    { id: 'b', metadata: {}, vector: [0, 0, 0], norm: 0, metadataFile: 'ext.pb' },
+                ],
+            };
+            const buf = codec.serializeIndex(data);
+            node_assert_1.default.ok(Buffer.isBuffer(buf));
+            const result = codec.deserializeIndex(buf);
+            node_assert_1.default.equal(result.version, data.version);
+            node_assert_1.default.deepStrictEqual(result.metadata_config, data.metadata_config);
+            node_assert_1.default.equal(result.items.length, 2);
+            node_assert_1.default.equal(result.items[0].id, 'a');
+            node_assert_1.default.deepStrictEqual(result.items[0].metadata, { category: 'food', score: 42 });
+            // Float32 precision: vectors should be close but not exact
+            node_assert_1.default.ok(Math.abs(result.items[0].vector[0] - 1.5) < 0.001);
+            node_assert_1.default.ok(Math.abs(result.items[0].vector[1] - 2.5) < 0.001);
+            node_assert_1.default.ok(Math.abs(result.items[0].vector[2] - 3.5) < 0.001);
+            node_assert_1.default.equal(result.items[0].norm, 4.5); // norm is double, exact
+            node_assert_1.default.equal(result.items[1].metadataFile, 'ext.pb');
+        });
+        it('handles empty items', () => {
+            const data = { version: 2, metadata_config: {}, items: [] };
+            const result = codec.deserializeIndex(codec.serializeIndex(data));
+            node_assert_1.default.equal(result.version, 2);
+            node_assert_1.default.deepStrictEqual(result.items, []);
+        });
+        it('handles boolean metadata values', () => {
+            const data = {
+                version: 1,
+                metadata_config: {},
+                items: [
+                    { id: 'x', metadata: { active: true, deleted: false }, vector: [1], norm: 1 },
+                ],
+            };
+            // Note: false/0/"" are proto3 default values — test the tricky case
+            const result = codec.deserializeIndex(codec.serializeIndex(data));
+            node_assert_1.default.equal(result.items[0].metadata.active, true);
+        });
+        it('handles string metadata', () => {
+            const data = {
+                version: 1,
+                metadata_config: {},
+                items: [
+                    { id: 'x', metadata: { name: 'hello', tag: 'world' }, vector: [1], norm: 1 },
+                ],
+            };
+            const result = codec.deserializeIndex(codec.serializeIndex(data));
+            node_assert_1.default.equal(result.items[0].metadata.name, 'hello');
+            node_assert_1.default.equal(result.items[0].metadata.tag, 'world');
+        });
+    });
+    describe('serializeCatalog / deserializeCatalog', () => {
+        it('round-trips DocumentCatalog', () => {
+            const catalog = {
+                version: 1,
+                count: 2,
+                uriToId: { 'http://a': 'id-a', 'http://b': 'id-b' },
+                idToUri: { 'id-a': 'http://a', 'id-b': 'http://b' },
+            };
+            const buf = codec.serializeCatalog(catalog);
+            node_assert_1.default.ok(Buffer.isBuffer(buf));
+            const result = codec.deserializeCatalog(buf);
+            node_assert_1.default.deepStrictEqual(result, catalog);
+        });
+        it('handles empty catalog', () => {
+            const catalog = { version: 1, count: 0, uriToId: {}, idToUri: {} };
+            const result = codec.deserializeCatalog(codec.serializeCatalog(catalog));
+            node_assert_1.default.deepStrictEqual(result, catalog);
+        });
+    });
+    describe('serializeMetadata / deserializeMetadata', () => {
+        it('round-trips metadata (uses JSON for external files)', () => {
+            const metadata = { name: 'test', score: 42, active: true };
+            const buf = codec.serializeMetadata(metadata);
+            const result = codec.deserializeMetadata(buf);
+            node_assert_1.default.deepStrictEqual(result, metadata);
+        });
+    });
+    describe('cross-codec compatibility', () => {
+        const json = new JsonCodec_1.JsonCodec();
+        const pb = new ProtobufCodec_1.ProtobufCodec();
+        it('data serialized with JSON can be re-serialized with Protobuf and back (logical equivalence)', () => {
+            const data = {
+                version: 1,
+                metadata_config: { indexed: ['cat'] },
+                items: [
+                    { id: 'item1', metadata: { cat: 'a', num: 7 }, vector: [0.1, 0.2, 0.3], norm: 0.374 },
+                ],
+            };
+            // JSON -> Protobuf
+            const jsonBuf = json.serializeIndex(data);
+            const fromJson = json.deserializeIndex(jsonBuf);
+            const pbBuf = pb.serializeIndex(fromJson);
+            const fromPb = pb.deserializeIndex(pbBuf);
+            node_assert_1.default.equal(fromPb.version, data.version);
+            node_assert_1.default.deepStrictEqual(fromPb.metadata_config, data.metadata_config);
+            node_assert_1.default.equal(fromPb.items[0].id, data.items[0].id);
+            node_assert_1.default.deepStrictEqual(fromPb.items[0].metadata, data.items[0].metadata);
+            // Protobuf -> JSON
+            const jsonBuf2 = json.serializeIndex(fromPb);
+            const final = json.deserializeIndex(jsonBuf2);
+            node_assert_1.default.equal(final.items[0].id, 'item1');
+            node_assert_1.default.deepStrictEqual(final.items[0].metadata, { cat: 'a', num: 7 });
+        });
+        it('catalog round-trips across codecs', () => {
+            const catalog = {
+                version: 1,
+                count: 1,
+                uriToId: { 'file://doc.txt': 'uuid-1' },
+                idToUri: { 'uuid-1': 'file://doc.txt' },
+            };
+            const pbBuf = pb.serializeCatalog(catalog);
+            const fromPb = pb.deserializeCatalog(pbBuf);
+            const jsonBuf = json.serializeCatalog(fromPb);
+            const final = json.deserializeCatalog(jsonBuf);
+            node_assert_1.default.deepStrictEqual(final, catalog);
+        });
+    });
+    describe('protobuf produces smaller output than JSON for vector data', () => {
+        it('binary is smaller than JSON for a 1536-dim vector', () => {
+            const vector = Array.from({ length: 1536 }, (_, i) => Math.random() * 2 - 1);
+            const norm = Math.sqrt(vector.reduce((s, v) => s + v * v, 0));
+            const data = {
+                version: 1,
+                metadata_config: {},
+                items: [{ id: 'vec', metadata: {}, vector, norm }],
+            };
+            const jsonSize = new JsonCodec_1.JsonCodec().serializeIndex(data).length;
+            const pbSize = codec.serializeIndex(data).length;
+            // Protobuf should be significantly smaller
+            node_assert_1.default.ok(pbSize < jsonSize, `Expected pb (${pbSize}) < json (${jsonSize})`);
+            // Spec says ~40-50% savings
+            node_assert_1.default.ok(pbSize < jsonSize * 0.7, `Expected at least 30% savings, got ${((1 - pbSize / jsonSize) * 100).toFixed(1)}%`);
+        });
+    });
+});
+//# sourceMappingURL=ProtobufCodec.spec.js.map

package/lib/codecs/ProtobufCodec.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ProtobufCodec.spec.js","sourceRoot":"","sources":["../../src/codecs/ProtobufCodec.spec.ts"],"names":[],"mappings":";;;;;AAAA,8DAAiC;AACjC,mDAAgD;AAChD,2CAAwC;AAIxC,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC3B,MAAM,KAAK,GAAG,IAAI,6BAAa,EAAE,CAAC;IAElC,EAAE,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACzB,qBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC/C,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;YAC7B,MAAM,IAAI,GAAc;gBACpB,OAAO,EAAE,CAAC;gBACV,eAAe,EAAE,EAAE,OAAO,EAAE,CAAC,UAAU,CAAC,EAAE;gBAC1C,KAAK,EAAE;oBACH,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE;oBAC1F,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,YAAY,EAAE,QAAQ,EAAE;iBAChF;aACJ,CAAC;YACF,MAAM,GAAG,GAAG,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YACvC,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC3C,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3C,qBAAM,CAAC,eAAe,CAAC,MAAM,CAAC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YACrE,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACrC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACtC,qBAAM,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;YAClF,2DAA2D;YAC3D,qBAAM,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;YAC7D,qBAAM,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;YAC7D,qBAAM,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;YAC7D,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,wBAAwB;YACjE,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;YAC3B,MAAM,IAAI,GAAc,EAAE,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;YACvE,MAAM,MAAM,GAAG,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;YAClE,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAChC,qBAAM,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACvC,MAAM,IAAI,GAAc;gBACpB,OAAO,EAAE,CAAC;gBACV,eAAe,EAAE,EAAE;gBACnB,KAAK,EAAE;oBACH,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE;iBAChF;aACJ,CAAC;YACF,oEAAoE;YACpE,MAAM,MAAM,GAAG,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;YAClE,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YAC/B,MAAM,IAAI,GAAc;gBACpB,OAAO,EAAE,CAAC;gBACV,eAAe,EAAE,EAAE;gBACnB,KAAK,EAAE;oBACH,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE;iBAC/E;aACJ,CAAC;YACF,MAAM,MAAM,GAAG,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;YAClE,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACrD,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uCAAuC,EAAE,GAAG,EAAE;QACnD,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACnC,MAAM,OAAO,GAAoB;gBAC7B,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE;gBACnD,OAAO,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE;aACtD,CAAC;YACF,MAAM,GAAG,GAAG,KAAK,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC5C,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAC7C,qBAAM,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;YAC7B,MAAM,OAAO,GAAoB,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;YACpF,MAAM,MAAM,GAAG,KAAK,CAAC,kBAAkB,CAAC,KAAK,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;YACzE,qBAAM,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACrD,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC3D,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAC3D,MAAM,GAAG,GAAG,KAAK,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAC9C,qBAAM,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACvC,MAAM,IAAI,GAAG,IAAI,qBAAS,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,6BAAa,EAAE,CAAC;QAE/B,EAAE,CAAC,6FAA6F,EAAE,GAAG,EAAE;YACnG,MAAM,IAAI,GAAc;gBACpB,OAAO,EAAE,CAAC;gBACV,eAAe,EAAE,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE;gBACrC,KAAK,EAAE;oBACH,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE;iBACxF;aACJ,CAAC;YACF,mBAAmB;YACnB,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAChD,MAAM,KAAK,GAAG,EAAE,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAG,EAAE,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAE1C,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3C,qBAAM,CAAC,eAAe,CAAC,MAAM,CAAC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YACrE,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnD,qBAAM,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YAEzE,mBAAmB;YACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAC9C,qBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACzC,qBAAM,CAAC,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YACzC,MAAM,OAAO,GAAoB;gBAC7B,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,EAAE,gBAAgB,EAAE,QAAQ,EAAE;gBACvC,OAAO,EAAE,EAAE,QAAQ,EAAE,gBAAgB,EAAE;aAC1C,CAAC;YACF,MAAM,KAAK,GAAG,EAAE,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,MAAM,GAAG,EAAE,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YAC/C,qBAAM,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACxE,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;YACzD,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7E,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9D,MAAM,IAAI,GAAc;gBACpB,OAAO,EAAE,CAAC;gBACV,eAAe,EAAE,EAAE;gBACnB,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;aACrD,CAAC;YACF,MAAM,QAAQ,GAAG,IAAI,qBAAS,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YAC7D,MAAM,MAAM,GAAG,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YACjD,2CAA2C;YAC3C,qBAAM,CAAC,EAAE,CAAC,MAAM,GAAG,QAAQ,EAAE,gBAAgB,MAAM,aAAa,QAAQ,GAAG,CAAC,CAAC;YAC7E,4BAA4B;YAC5B,qBAAM,CAAC,EAAE,CAAC,MAAM,GAAG,QAAQ,GAAG,GAAG,EAAE,sCAAsC,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC5H,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}

package/lib/codecs/index.d.ts

@@ -0,0 +1,5 @@
+export * from './IndexCodec';
+export * from './JsonCodec';
+export * from './ProtobufCodec';
+export * from './migrateIndex';
+//# sourceMappingURL=index.d.ts.map

package/lib/codecs/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/codecs/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC"}

package/lib/codecs/index.js

@@ -0,0 +1,21 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./IndexCodec"), exports);
+__exportStar(require("./JsonCodec"), exports);
+__exportStar(require("./ProtobufCodec"), exports);
+__exportStar(require("./migrateIndex"), exports);
+//# sourceMappingURL=index.js.map

package/lib/codecs/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/codecs/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,+CAA6B;AAC7B,8CAA4B;AAC5B,kDAAgC;AAChC,iDAA+B"}

package/lib/codecs/migrateIndex.d.ts

@@ -0,0 +1,24 @@
+import { FileStorage } from '../storage';
+import { IndexCodec } from './IndexCodec';
+export type FormatName = 'json' | 'protobuf';
+/**
+ * Detects the codec in use for an existing index folder.
+ * @remarks
+ * Checks for `index.json` and `index.pb` files. If both exist, throws an
+ * error directing the user to re-run migration. If neither exists, throws.
+ * @param folderPath Path to the index folder.
+ * @param storage Storage backend to use.
+ * @returns The detected codec.
+ */
+export declare function detectCodec(folderPath: string, storage: FileStorage): Promise<IndexCodec>;
+export interface MigrateOptions {
+    to: FormatName;
+    storage?: FileStorage;
+}
+/**
+ * Migrates an index folder from one serialization format to another.
+ * @param folderPath Path to the index folder.
+ * @param options Migration options.
+ */
+export declare function migrateIndex(folderPath: string, options: MigrateOptions): Promise<void>;
+//# sourceMappingURL=migrateIndex.d.ts.map

package/lib/codecs/migrateIndex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"migrateIndex.d.ts","sourceRoot":"","sources":["../../src/codecs/migrateIndex.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAoB,MAAM,YAAY,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAI1C,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC;AAM7C;;;;;;;;GAQG;AACH,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAa/F;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,UAAU,CAAC;IACf,OAAO,CAAC,EAAE,WAAW,CAAC;CACzB;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CA8E7F"}

package/lib/codecs/migrateIndex.js

@@ -0,0 +1,119 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectCodec = detectCodec;
+exports.migrateIndex = migrateIndex;
+const pathUtils_1 = require("../utils/pathUtils");
+const storage_1 = require("../storage");
+const JsonCodec_1 = require("./JsonCodec");
+const ProtobufCodec_1 = require("./ProtobufCodec");
+function codecForFormat(format) {
+    return format === 'protobuf' ? new ProtobufCodec_1.ProtobufCodec() : new JsonCodec_1.JsonCodec();
+}
+/**
+ * Detects the codec in use for an existing index folder.
+ * @remarks
+ * Checks for `index.json` and `index.pb` files. If both exist, throws an
+ * error directing the user to re-run migration. If neither exists, throws.
+ * @param folderPath Path to the index folder.
+ * @param storage Storage backend to use.
+ * @returns The detected codec.
+ */
+function detectCodec(folderPath, storage) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const hasJson = yield storage.pathExists(pathUtils_1.pathUtils.join(folderPath, 'index.json'));
+        const hasPb = yield storage.pathExists(pathUtils_1.pathUtils.join(folderPath, 'index.pb'));
+        if (hasJson && hasPb) {
+            throw new Error('Both index.json and index.pb found — the index may be in a partially migrated state. ' +
+                'Run `vectra migrate` or `migrateIndex()` to complete the migration.');
+        }
+        if (hasPb)
+            return new ProtobufCodec_1.ProtobufCodec();
+        if (hasJson)
+            return new JsonCodec_1.JsonCodec();
+        throw new Error('No index file found (expected index.json or index.pb).');
+    });
+}
+/**
+ * Migrates an index folder from one serialization format to another.
+ * @param folderPath Path to the index folder.
+ * @param options Migration options.
+ */
+function migrateIndex(folderPath, options) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const storage = options.storage || new storage_1.LocalFileStorage();
+        const targetCodec = codecForFormat(options.to);
+        // Detect current format
+        const sourceCodec = yield detectCodec(folderPath, storage);
+        if (sourceCodec.extension === targetCodec.extension) {
+            return; // Already in target format
+        }
+        // --- Migrate index file ---
+        const sourceIndexPath = pathUtils_1.pathUtils.join(folderPath, `index${sourceCodec.extension}`);
+        const targetIndexPath = pathUtils_1.pathUtils.join(folderPath, `index${targetCodec.extension}`);
+        const indexBuffer = yield storage.readFile(sourceIndexPath);
+        const indexData = sourceCodec.deserializeIndex(indexBuffer);
+        // Rewrite external metadata file references
+        for (const item of indexData.items) {
+            if (item.metadataFile && item.metadataFile.endsWith(sourceCodec.extension)) {
+                const baseName = item.metadataFile.slice(0, -sourceCodec.extension.length);
+                const oldMetaPath = pathUtils_1.pathUtils.join(folderPath, item.metadataFile);
+                const newMetaFile = `${baseName}${targetCodec.extension}`;
+                const newMetaPath = pathUtils_1.pathUtils.join(folderPath, newMetaFile);
+                // Read, re-serialize, write
+                const metaBuf = yield storage.readFile(oldMetaPath);
+                const metadata = sourceCodec.deserializeMetadata(metaBuf);
+                yield storage.upsertFile(newMetaPath, targetCodec.serializeMetadata(metadata));
+                // Update reference
+                item.metadataFile = newMetaFile;
+            }
+        }
+        // Write new index
+        yield storage.upsertFile(targetIndexPath, targetCodec.serializeIndex(indexData));
+        // --- Migrate catalog if present ---
+        const sourceCatalogPath = pathUtils_1.pathUtils.join(folderPath, `catalog${sourceCodec.extension}`);
+        const targetCatalogPath = pathUtils_1.pathUtils.join(folderPath, `catalog${targetCodec.extension}`);
+        if (yield storage.pathExists(sourceCatalogPath)) {
+            const catalogBuffer = yield storage.readFile(sourceCatalogPath);
+            const catalog = sourceCodec.deserializeCatalog(catalogBuffer);
+            // Migrate per-document metadata files
+            for (const docId of Object.values(catalog.idToUri ? catalog.idToUri : {})) {
+                // Document metadata uses documentId as filename base
+            }
+            // Actually the document IDs are the keys of idToUri
+            for (const docId of Object.keys(catalog.idToUri)) {
+                const oldDocMetaPath = pathUtils_1.pathUtils.join(folderPath, `${docId}${sourceCodec.extension}`);
+                if (yield storage.pathExists(oldDocMetaPath)) {
+                    const docMetaBuf = yield storage.readFile(oldDocMetaPath);
+                    const docMeta = sourceCodec.deserializeMetadata(docMetaBuf);
+                    const newDocMetaPath = pathUtils_1.pathUtils.join(folderPath, `${docId}${targetCodec.extension}`);
+                    yield storage.upsertFile(newDocMetaPath, targetCodec.serializeMetadata(docMeta));
+                    yield storage.deleteFile(oldDocMetaPath);
+                }
+            }
+            yield storage.upsertFile(targetCatalogPath, targetCodec.serializeCatalog(catalog));
+            yield storage.deleteFile(sourceCatalogPath);
+        }
+        // --- Clean up old files (after new files are safely written) ---
+        // Delete old external metadata files
+        for (const item of sourceCodec.deserializeIndex(indexBuffer).items) {
+            if (item.metadataFile) {
+                const oldMetaPath = pathUtils_1.pathUtils.join(folderPath, item.metadataFile);
+                if (yield storage.pathExists(oldMetaPath)) {
+                    yield storage.deleteFile(oldMetaPath);
+                }
+            }
+        }
+        // Delete old index file last
+        yield storage.deleteFile(sourceIndexPath);
+    });
+}
+//# sourceMappingURL=migrateIndex.js.map

package/lib/codecs/migrateIndex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"migrateIndex.js","sourceRoot":"","sources":["../../src/codecs/migrateIndex.ts"],"names":[],"mappings":";;;;;;;;;;;AAqBA,kCAaC;AAYD,oCA8EC;AA5HD,kDAAuD;AACvD,wCAA2D;AAE3D,2CAAwC;AACxC,mDAAgD;AAIhD,SAAS,cAAc,CAAC,MAAkB;IACtC,OAAO,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,6BAAa,EAAE,CAAC,CAAC,CAAC,IAAI,qBAAS,EAAE,CAAC;AACzE,CAAC;AAED;;;;;;;;GAQG;AACH,SAAsB,WAAW,CAAC,UAAkB,EAAE,OAAoB;;QACtE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,qBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC,CAAC;QAC9E,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,qBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;QAE1E,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CACX,uFAAuF;gBACvF,qEAAqE,CACxE,CAAC;QACN,CAAC;QACD,IAAI,KAAK;YAAE,OAAO,IAAI,6BAAa,EAAE,CAAC;QACtC,IAAI,OAAO;YAAE,OAAO,IAAI,qBAAS,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC9E,CAAC;CAAA;AAOD;;;;GAIG;AACH,SAAsB,YAAY,CAAC,UAAkB,EAAE,OAAuB;;QAC1E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,0BAAgB,EAAE,CAAC;QAC1D,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAE/C,wBAAwB;QACxB,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAE3D,IAAI,WAAW,CAAC,SAAS,KAAK,WAAW,CAAC,SAAS,EAAE,CAAC;YAClD,OAAO,CAAC,2BAA2B;QACvC,CAAC;QAED,6BAA6B;QAC7B,MAAM,eAAe,GAAG,qBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,WAAW,CAAC,SAAS,EAAE,CAAC,CAAC;QAC/E,MAAM,eAAe,GAAG,qBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,WAAW,CAAC,SAAS,EAAE,CAAC,CAAC;QAC/E,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,WAAW,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAE5D,4CAA4C;QAC5C,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC3E,MAAM,WAAW,GAAG,qBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC7D,MAAM,WAAW,GAAG,GAAG,QAAQ,GAAG,WAAW,CAAC,SAAS,EAAE,CAAC;gBAC1D,MAAM,WAAW,GAAG,qBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;gBAEvD,4BAA4B;gBAC5B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBACpD,MAAM,QAAQ,GAAG,WAAW,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;gBAC1D,MAAM,OAAO,CAAC,UAAU,CAAC,WAAW,EAAE,WAAW,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAE/E,mBAAmB;gBACnB,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;YACpC,CAAC;QACL,CAAC;QAED,kBAAkB;QAClB,MAAM,OAAO,CAAC,UAAU,CAAC,eAAe,EAAE,WAAW,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC;QAEjF,qCAAqC;QACrC,MAAM,iBAAiB,GAAG,qBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,WAAW,CAAC,SAAS,EAAE,CAAC,CAAC;QACnF,MAAM,iBAAiB,GAAG,qBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,WAAW,CAAC,SAAS,EAAE,CAAC,CAAC;QACnF,IAAI,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAC9C,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;YAChE,MAAM,OAAO,GAAG,WAAW,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;YAE9D,sCAAsC;YACtC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;gBACxE,qDAAqD;YACzD,CAAC;YACD,oDAAoD;YACpD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/C,MAAM,cAAc,GAAG,qBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,KAAK,GAAG,WAAW,CAAC,SAAS,EAAE,CAAC,CAAC;gBACjF,IAAI,MAAM,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;oBAC3C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;oBAC1D,MAAM,OAAO,GAAG,WAAW,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;oBAC5D,MAAM,cAAc,GAAG,qBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,KAAK,GAAG,WAAW,CAAC,SAAS,EAAE,CAAC,CAAC;oBACjF,MAAM,OAAO,CAAC,UAAU,CAAC,cAAc,EAAE,WAAW,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;oBACjF,MAAM,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAC7C,CAAC;YACL,CAAC;YAED,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;YACnF,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;QAChD,CAAC;QAED,kEAAkE;QAClE,qCAAqC;QACrC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;YACjE,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,MAAM,WAAW,GAAG,qBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC7D,IAAI,MAAM,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;oBACxC,MAAM,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;gBAC1C,CAAC;YACL,CAAC;QACL,CAAC;QAED,6BAA6B;QAC7B,MAAM,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;IAC9C,CAAC;CAAA"}

package/lib/codecs/migrateIndex.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=migrateIndex.spec.d.ts.map

package/lib/codecs/migrateIndex.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"migrateIndex.spec.d.ts","sourceRoot":"","sources":["../../src/codecs/migrateIndex.spec.ts"],"names":[],"mappings":""}

package/lib/codecs/migrateIndex.spec.js

@@ -0,0 +1,151 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_assert_1 = __importDefault(require("node:assert"));
+const storage_1 = require("../storage");
+const JsonCodec_1 = require("./JsonCodec");
+const ProtobufCodec_1 = require("./ProtobufCodec");
+const migrateIndex_1 = require("./migrateIndex");
+describe('detectCodec', () => {
+    it('detects JSON format', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        yield storage.createFolder('/idx');
+        yield storage.upsertFile('/idx/index.json', '{}');
+        const codec = yield (0, migrateIndex_1.detectCodec)('/idx', storage);
+        node_assert_1.default.equal(codec.extension, '.json');
+    }));
+    it('detects Protobuf format', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        yield storage.createFolder('/idx');
+        yield storage.upsertFile('/idx/index.pb', Buffer.from([0]));
+        const codec = yield (0, migrateIndex_1.detectCodec)('/idx', storage);
+        node_assert_1.default.equal(codec.extension, '.pb');
+    }));
+    it('throws when both formats exist', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        yield storage.createFolder('/idx');
+        yield storage.upsertFile('/idx/index.json', '{}');
+        yield storage.upsertFile('/idx/index.pb', Buffer.from([0]));
+        yield node_assert_1.default.rejects(() => (0, migrateIndex_1.detectCodec)('/idx', storage), /Both index\.json and index\.pb found/);
+    }));
+    it('throws when no index file exists', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        yield storage.createFolder('/idx');
+        yield node_assert_1.default.rejects(() => (0, migrateIndex_1.detectCodec)('/idx', storage), /No index file found/);
+    }));
+});
+describe('migrateIndex', () => {
+    const json = new JsonCodec_1.JsonCodec();
+    const pb = new ProtobufCodec_1.ProtobufCodec();
+    function makeIndexData() {
+        return {
+            version: 1,
+            metadata_config: { indexed: ['cat'] },
+            items: [
+                { id: 'item1', metadata: { cat: 'food' }, vector: [0.1, 0.2, 0.3], norm: 0.374 },
+                { id: 'item2', metadata: { cat: 'drink' }, vector: [0.4, 0.5, 0.6], norm: 0.877, metadataFile: 'abc.json' },
+            ],
+        };
+    }
+    function makeCatalog() {
+        return {
+            version: 1,
+            count: 1,
+            uriToId: { 'doc.txt': 'doc-id-1' },
+            idToUri: { 'doc-id-1': 'doc.txt' },
+        };
+    }
+    it('migrates JSON -> Protobuf', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        yield storage.createFolder('/idx');
+        const data = makeIndexData();
+        yield storage.upsertFile('/idx/index.json', json.serializeIndex(data));
+        yield storage.upsertFile('/idx/abc.json', json.serializeMetadata({ cat: 'drink', extra: 'data' }));
+        yield (0, migrateIndex_1.migrateIndex)('/idx', { to: 'protobuf', storage });
+        // Old files should be gone
+        node_assert_1.default.equal(yield storage.pathExists('/idx/index.json'), false);
+        node_assert_1.default.equal(yield storage.pathExists('/idx/abc.json'), false);
+        // New files should exist
+        node_assert_1.default.equal(yield storage.pathExists('/idx/index.pb'), true);
+        node_assert_1.default.equal(yield storage.pathExists('/idx/abc.pb'), true);
+        // Data should be intact
+        const result = pb.deserializeIndex(yield storage.readFile('/idx/index.pb'));
+        node_assert_1.default.equal(result.items.length, 2);
+        node_assert_1.default.equal(result.items[0].id, 'item1');
+        node_assert_1.default.equal(result.items[1].metadataFile, 'abc.pb');
+        // External metadata should be readable
+        const meta = pb.deserializeMetadata(yield storage.readFile('/idx/abc.pb'));
+        node_assert_1.default.equal(meta.cat, 'drink');
+        node_assert_1.default.equal(meta.extra, 'data');
+    }));
+    it('migrates Protobuf -> JSON', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        yield storage.createFolder('/idx');
+        const data = makeIndexData();
+        // Adjust metadataFile extension
+        data.items[1].metadataFile = 'abc.pb';
+        yield storage.upsertFile('/idx/index.pb', pb.serializeIndex(data));
+        yield storage.upsertFile('/idx/abc.pb', pb.serializeMetadata({ cat: 'drink', extra: 'data' }));
+        yield (0, migrateIndex_1.migrateIndex)('/idx', { to: 'json', storage });
+        node_assert_1.default.equal(yield storage.pathExists('/idx/index.pb'), false);
+        node_assert_1.default.equal(yield storage.pathExists('/idx/index.json'), true);
+        const result = json.deserializeIndex(yield storage.readFile('/idx/index.json'));
+        node_assert_1.default.equal(result.items.length, 2);
+        node_assert_1.default.equal(result.items[1].metadataFile, 'abc.json');
+    }));
+    it('migrates catalog along with index', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        yield storage.createFolder('/idx');
+        const data = { version: 1, metadata_config: {}, items: [] };
+        const catalog = makeCatalog();
+        yield storage.upsertFile('/idx/index.json', json.serializeIndex(data));
+        yield storage.upsertFile('/idx/catalog.json', json.serializeCatalog(catalog));
+        yield (0, migrateIndex_1.migrateIndex)('/idx', { to: 'protobuf', storage });
+        node_assert_1.default.equal(yield storage.pathExists('/idx/catalog.json'), false);
+        node_assert_1.default.equal(yield storage.pathExists('/idx/catalog.pb'), true);
+        const result = pb.deserializeCatalog(yield storage.readFile('/idx/catalog.pb'));
+        node_assert_1.default.deepStrictEqual(result, catalog);
+    }));
+    it('migrates document metadata files referenced by catalog', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        yield storage.createFolder('/idx');
+        const data = { version: 1, metadata_config: {}, items: [] };
+        const catalog = makeCatalog();
+        yield storage.upsertFile('/idx/index.json', json.serializeIndex(data));
+        yield storage.upsertFile('/idx/catalog.json', json.serializeCatalog(catalog));
+        yield storage.upsertFile('/idx/doc-id-1.json', json.serializeMetadata({ author: 'test' }));
+        yield (0, migrateIndex_1.migrateIndex)('/idx', { to: 'protobuf', storage });
+        node_assert_1.default.equal(yield storage.pathExists('/idx/doc-id-1.json'), false);
+        node_assert_1.default.equal(yield storage.pathExists('/idx/doc-id-1.pb'), true);
+        const meta = pb.deserializeMetadata(yield storage.readFile('/idx/doc-id-1.pb'));
+        node_assert_1.default.equal(meta.author, 'test');
+    }));
+    it('no-op when already in target format', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        yield storage.createFolder('/idx');
+        const data = { version: 1, metadata_config: {}, items: [] };
+        yield storage.upsertFile('/idx/index.json', json.serializeIndex(data));
+        yield (0, migrateIndex_1.migrateIndex)('/idx', { to: 'json', storage });
+        // Should still be there, unchanged
+        node_assert_1.default.equal(yield storage.pathExists('/idx/index.json'), true);
+    }));
+    it('interruption detection: dual format raises error on detectCodec', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        yield storage.createFolder('/idx');
+        yield storage.upsertFile('/idx/index.json', '{}');
+        yield storage.upsertFile('/idx/index.pb', Buffer.from([0]));
+        yield node_assert_1.default.rejects(() => (0, migrateIndex_1.detectCodec)('/idx', storage), /Both index\.json and index\.pb found/);
+    }));
+});
+//# sourceMappingURL=migrateIndex.spec.js.map

package/lib/codecs/migrateIndex.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"migrateIndex.spec.js","sourceRoot":"","sources":["../../src/codecs/migrateIndex.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,8DAAiC;AACjC,wCAAgD;AAChD,2CAAwC;AACxC,mDAAgD;AAChD,iDAA2D;AAI3D,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IACzB,EAAE,CAAC,qBAAqB,EAAE,GAAS,EAAE;QACjC,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;QAClD,MAAM,KAAK,GAAG,MAAM,IAAA,0BAAW,EAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjD,qBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAS,EAAE;QACrC,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,OAAO,CAAC,UAAU,CAAC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,MAAM,IAAA,0BAAW,EAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjD,qBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACzC,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAS,EAAE;QAC5C,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;QAClD,MAAM,OAAO,CAAC,UAAU,CAAC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,qBAAM,CAAC,OAAO,CAChB,GAAG,EAAE,CAAC,IAAA,0BAAW,EAAC,MAAM,EAAE,OAAO,CAAC,EAClC,sCAAsC,CACzC,CAAC;IACN,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAS,EAAE;QAC9C,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,qBAAM,CAAC,OAAO,CAChB,GAAG,EAAE,CAAC,IAAA,0BAAW,EAAC,MAAM,EAAE,OAAO,CAAC,EAClC,qBAAqB,CACxB,CAAC;IACN,CAAC,CAAA,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC1B,MAAM,IAAI,GAAG,IAAI,qBAAS,EAAE,CAAC;IAC7B,MAAM,EAAE,GAAG,IAAI,6BAAa,EAAE,CAAC;IAE/B,SAAS,aAAa;QAClB,OAAO;YACH,OAAO,EAAE,CAAC;YACV,eAAe,EAAE,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE;YACrC,KAAK,EAAE;gBACH,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE;gBAChF,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE;aAC9G;SACJ,CAAC;IACN,CAAC;IAED,SAAS,WAAW;QAChB,OAAO;YACH,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE;YAClC,OAAO,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE;SACrC,CAAC;IACN,CAAC;IAED,EAAE,CAAC,2BAA2B,EAAE,GAAS,EAAE;QACvC,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,aAAa,EAAE,CAAC;QAC7B,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;QACvE,MAAM,OAAO,CAAC,UAAU,CAAC,eAAe,EAAE,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAEnG,MAAM,IAAA,2BAAY,EAAC,MAAM,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;QAExD,2BAA2B;QAC3B,qBAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,KAAK,CAAC,CAAC;QACjE,qBAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,KAAK,CAAC,CAAC;QAE/D,yBAAyB;QACzB,qBAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,IAAI,CAAC,CAAC;QAC9D,qBAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,IAAI,CAAC,CAAC;QAE5D,wBAAwB;QACxB,MAAM,MAAM,GAAG,EAAE,CAAC,gBAAgB,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC;QAC5E,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACrC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC1C,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QAErD,uCAAuC;QACvC,MAAM,IAAI,GAAG,EAAE,CAAC,mBAAmB,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;QAC3E,qBAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAChC,qBAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACrC,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,GAAS,EAAE;QACvC,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,aAAa,EAAE,CAAC;QAC7B,gCAAgC;QAChC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,QAAQ,CAAC;QACtC,MAAM,OAAO,CAAC,UAAU,CAAC,eAAe,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;QACnE,MAAM,OAAO,CAAC,UAAU,CAAC,aAAa,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAE/F,MAAM,IAAA,2BAAY,EAAC,MAAM,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QAEpD,qBAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,KAAK,CAAC,CAAC;QAC/D,qBAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,IAAI,CAAC,CAAC;QAEhE,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAChF,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACrC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAC3D,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAS,EAAE;QAC/C,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,IAAI,GAAc,EAAE,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACvE,MAAM,OAAO,GAAG,WAAW,EAAE,CAAC;QAC9B,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;QACvE,MAAM,OAAO,CAAC,UAAU,CAAC,mBAAmB,EAAE,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;QAE9E,MAAM,IAAA,2BAAY,EAAC,MAAM,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;QAExD,qBAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,KAAK,CAAC,CAAC;QACnE,qBAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,IAAI,CAAC,CAAC;QAEhE,MAAM,MAAM,GAAG,EAAE,CAAC,kBAAkB,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAChF,qBAAM,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAS,EAAE;QACpE,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,IAAI,GAAc,EAAE,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACvE,MAAM,OAAO,GAAG,WAAW,EAAE,CAAC;QAC9B,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;QACvE,MAAM,OAAO,CAAC,UAAU,CAAC,mBAAmB,EAAE,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;QAC9E,MAAM,OAAO,CAAC,UAAU,CAAC,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAE3F,MAAM,IAAA,2BAAY,EAAC,MAAM,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;QAExD,qBAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,EAAE,KAAK,CAAC,CAAC;QACpE,qBAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,IAAI,CAAC,CAAC;QACjE,MAAM,IAAI,GAAG,EAAE,CAAC,mBAAmB,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAChF,qBAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAS,EAAE;QACjD,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,IAAI,GAAc,EAAE,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACvE,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;QAEvE,MAAM,IAAA,2BAAY,EAAC,MAAM,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QAEpD,mCAAmC;QACnC,qBAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,IAAI,CAAC,CAAC;IACpE,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAS,EAAE;QAC7E,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,OAAO,CAAC,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;QAClD,MAAM,OAAO,CAAC,UAAU,CAAC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5D,MAAM,qBAAM,CAAC,OAAO,CAChB,GAAG,EAAE,CAAC,IAAA,0BAAW,EAAC,MAAM,EAAE,OAAO,CAAC,EAClC,sCAAsC,CACzC,CAAC;IACN,CAAC,CAAA,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}

package/lib/codecs/schemas/index.proto

@@ -0,0 +1,34 @@
+syntax = "proto3";
+
+message IndexData {
+  uint32 version = 1;
+  MetadataConfig metadata_config = 2;
+  repeated IndexItem items = 3;
+}
+
+message MetadataConfig {
+  repeated string indexed = 1;
+}
+
+message IndexItem {
+  string id = 1;
+  map<string, MetadataValue> metadata = 2;
+  repeated float vector = 3 [packed = true];
+  double norm = 4;
+  optional string metadata_file = 5;
+}
+
+message MetadataValue {
+  oneof value {
+    string string_value = 1;
+    double number_value = 2;
+    bool bool_value = 3;
+  }
+}
+
+message DocumentCatalog {
+  uint32 version = 1;
+  uint32 count = 2;
+  map<string, string> uri_to_id = 3;
+  map<string, string> id_to_uri = 4;
+}

package/lib/index.d.ts

@@ -0,0 +1,20 @@
+export * from './codecs';
+export * from './storage';
+export * from './FileFetcher';
+export * from './FolderWatcher';
+export * from './GPT3Tokenizer';
+export * from './ItemSelector';
+export * from './LocalEmbeddings';
+export * from './LocalIndex';
+export * from './LocalDocument';
+export * from './LocalDocumentIndex';
+export * from './LocalDocumentResult';
+export * from './TextSplitter';
+export * from './OpenAIEmbeddings';
+export * from './TransformersEmbeddings';
+export * from './TransformersTokenizer';
+export * from './types';
+export * from './WebFetcher';
+export * from './server';
+export * from './utils/pathUtils';
+//# sourceMappingURL=index.d.ts.map

package/lib/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,iBAAiB,CAAC;AAChC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,cAAc,CAAC;AAC7B,cAAc,iBAAiB,CAAC;AAChC,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,oBAAoB,CAAC;AACnC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC;AAC7B,cAAc,UAAU,CAAC;AACzB,cAAc,mBAAmB,CAAC"}