vectra 0.12.3 → 0.14.0

package/lib/browser.js

@@ -0,0 +1,52 @@
+"use strict";
+/**
+ * Browser entry point for Vectra.
+ *
+ * This module exports only browser-compatible components.
+ * Node-specific components (LocalFileStorage, FileFetcher, WebFetcher) are excluded.
+ *
+ * For browser usage:
+ * - Use `VirtualFileStorage` for in-memory storage (ephemeral)
+ * - Use `IndexedDBStorage` for persistent browser storage
+ * - Use `BrowserWebFetcher` for fetching web content
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pathUtils = exports.BrowserWebFetcher = exports.IndexedDBStorage = exports.VirtualFileStorage = exports.FileStorageUtilities = exports.ItemSelector = exports.TransformersTokenizer = exports.TransformersEmbeddings = exports.OpenAIEmbeddings = exports.GPT3Tokenizer = exports.TextSplitter = exports.LocalDocumentResult = exports.LocalDocument = exports.LocalDocumentIndex = exports.LocalIndex = void 0;
+// Core index classes
+var LocalIndex_1 = require("./LocalIndex");
+Object.defineProperty(exports, "LocalIndex", { enumerable: true, get: function () { return LocalIndex_1.LocalIndex; } });
+var LocalDocumentIndex_1 = require("./LocalDocumentIndex");
+Object.defineProperty(exports, "LocalDocumentIndex", { enumerable: true, get: function () { return LocalDocumentIndex_1.LocalDocumentIndex; } });
+// Document classes
+var LocalDocument_1 = require("./LocalDocument");
+Object.defineProperty(exports, "LocalDocument", { enumerable: true, get: function () { return LocalDocument_1.LocalDocument; } });
+var LocalDocumentResult_1 = require("./LocalDocumentResult");
+Object.defineProperty(exports, "LocalDocumentResult", { enumerable: true, get: function () { return LocalDocumentResult_1.LocalDocumentResult; } });
+// Text processing
+var TextSplitter_1 = require("./TextSplitter");
+Object.defineProperty(exports, "TextSplitter", { enumerable: true, get: function () { return TextSplitter_1.TextSplitter; } });
+var GPT3Tokenizer_1 = require("./GPT3Tokenizer");
+Object.defineProperty(exports, "GPT3Tokenizer", { enumerable: true, get: function () { return GPT3Tokenizer_1.GPT3Tokenizer; } });
+// Embeddings
+var OpenAIEmbeddings_1 = require("./OpenAIEmbeddings");
+Object.defineProperty(exports, "OpenAIEmbeddings", { enumerable: true, get: function () { return OpenAIEmbeddings_1.OpenAIEmbeddings; } });
+var TransformersEmbeddings_1 = require("./TransformersEmbeddings");
+Object.defineProperty(exports, "TransformersEmbeddings", { enumerable: true, get: function () { return TransformersEmbeddings_1.TransformersEmbeddings; } });
+var TransformersTokenizer_1 = require("./TransformersTokenizer");
+Object.defineProperty(exports, "TransformersTokenizer", { enumerable: true, get: function () { return TransformersTokenizer_1.TransformersTokenizer; } });
+// Query utilities
+var ItemSelector_1 = require("./ItemSelector");
+Object.defineProperty(exports, "ItemSelector", { enumerable: true, get: function () { return ItemSelector_1.ItemSelector; } });
+var FileStorageUtilities_1 = require("./storage/FileStorageUtilities");
+Object.defineProperty(exports, "FileStorageUtilities", { enumerable: true, get: function () { return FileStorageUtilities_1.FileStorageUtilities; } });
+var VirtualFileStorage_1 = require("./storage/VirtualFileStorage");
+Object.defineProperty(exports, "VirtualFileStorage", { enumerable: true, get: function () { return VirtualFileStorage_1.VirtualFileStorage; } });
+var IndexedDBStorage_1 = require("./storage/IndexedDBStorage");
+Object.defineProperty(exports, "IndexedDBStorage", { enumerable: true, get: function () { return IndexedDBStorage_1.IndexedDBStorage; } });
+// Browser-compatible fetcher
+var BrowserWebFetcher_1 = require("./BrowserWebFetcher");
+Object.defineProperty(exports, "BrowserWebFetcher", { enumerable: true, get: function () { return BrowserWebFetcher_1.BrowserWebFetcher; } });
+// Utilities
+var pathUtils_1 = require("./utils/pathUtils");
+Object.defineProperty(exports, "pathUtils", { enumerable: true, get: function () { return pathUtils_1.pathUtils; } });
+//# sourceMappingURL=browser.js.map

package/lib/browser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser.js","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;AAEH,qBAAqB;AACrB,2CAA6D;AAApD,wGAAA,UAAU,OAAA;AACnB,2DAA0G;AAAjG,wHAAA,kBAAkB,OAAA;AAE3B,mBAAmB;AACnB,iDAAgD;AAAvC,8GAAA,aAAa,OAAA;AACtB,6DAA4D;AAAnD,0HAAA,mBAAmB,OAAA;AAE5B,kBAAkB;AAClB,+CAAkE;AAAzD,4GAAA,YAAY,OAAA;AACrB,iDAAgD;AAAvC,8GAAA,aAAa,OAAA;AAEtB,aAAa;AACb,uDAM4B;AALxB,oHAAA,gBAAgB,OAAA;AAMpB,mEAAiG;AAAxF,gIAAA,sBAAsB,OAAA;AAC/B,iEAAgE;AAAvD,8HAAA,qBAAqB,OAAA;AAE9B,kBAAkB;AAClB,+CAA8C;AAArC,4GAAA,YAAY,OAAA;AAuBrB,uEAAsE;AAA7D,4HAAA,oBAAoB,OAAA;AAE7B,mEAAkE;AAAzD,wHAAA,kBAAkB,OAAA;AAC3B,+DAA8D;AAArD,oHAAA,gBAAgB,OAAA;AAEzB,6BAA6B;AAC7B,yDAAiF;AAAxE,sHAAA,iBAAiB,OAAA;AAE1B,YAAY;AACZ,+CAA8C;AAArC,sGAAA,SAAS,OAAA"}

package/lib/codecs/IndexCodec.d.ts

@@ -0,0 +1,37 @@
+import { IndexData, MetadataTypes } from '../types';
+/**
+ * Catalog structure for document indexes.
+ */
+export interface DocumentCatalog {
+    version: number;
+    count: number;
+    uriToId: {
+        [uri: string]: string;
+    };
+    idToUri: {
+        [id: string]: string;
+    };
+}
+/**
+ * Abstraction for index serialization format.
+ * @remarks
+ * Implementations control how index data, catalogs, and metadata are
+ * serialized to and from Buffers written by the storage backend.
+ */
+export interface IndexCodec {
+    /** File extension used for index files (e.g., '.json', '.pb') */
+    readonly extension: string;
+    /** Serialize IndexData to a Buffer for writing to storage */
+    serializeIndex(data: IndexData): Buffer;
+    /** Deserialize a Buffer read from storage into IndexData */
+    deserializeIndex(buffer: Buffer): IndexData;
+    /** Serialize a DocumentCatalog to a Buffer */
+    serializeCatalog(catalog: DocumentCatalog): Buffer;
+    /** Deserialize a Buffer into a DocumentCatalog */
+    deserializeCatalog(buffer: Buffer): DocumentCatalog;
+    /** Serialize arbitrary metadata to a Buffer */
+    serializeMetadata(metadata: Record<string, MetadataTypes>): Buffer;
+    /** Deserialize a Buffer into metadata */
+    deserializeMetadata(buffer: Buffer): Record<string, MetadataTypes>;
+}
+//# sourceMappingURL=IndexCodec.d.ts.map

package/lib/codecs/IndexCodec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexCodec.d.ts","sourceRoot":"","sources":["../../src/codecs/IndexCodec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEpD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACnC,OAAO,EAAE;QAAE,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;CACrC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACvB,iEAAiE;IACjE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B,6DAA6D;IAC7D,cAAc,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IAExC,4DAA4D;IAC5D,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5C,8CAA8C;IAC9C,gBAAgB,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC;IAEnD,kDAAkD;IAClD,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC;IAEpD,+CAA+C;IAC/C,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,MAAM,CAAC;IAEnE,yCAAyC;IACzC,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;CACtE"}

package/lib/codecs/IndexCodec.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=IndexCodec.js.map

package/lib/codecs/IndexCodec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexCodec.js","sourceRoot":"","sources":["../../src/codecs/IndexCodec.ts"],"names":[],"mappings":""}

package/lib/codecs/JsonCodec.d.ts

@@ -0,0 +1,19 @@
+import { IndexData, MetadataTypes } from '../types';
+import { DocumentCatalog, IndexCodec } from './IndexCodec';
+/**
+ * JSON codec — default serialization format.
+ * @remarks
+ * Wraps the existing JSON.stringify/JSON.parse behavior. This preserves
+ * full backward compatibility with indexes created before the codec
+ * abstraction was introduced.
+ */
+export declare class JsonCodec implements IndexCodec {
+    readonly extension = ".json";
+    serializeIndex(data: IndexData): Buffer;
+    deserializeIndex(buffer: Buffer): IndexData;
+    serializeCatalog(catalog: DocumentCatalog): Buffer;
+    deserializeCatalog(buffer: Buffer): DocumentCatalog;
+    serializeMetadata(metadata: Record<string, MetadataTypes>): Buffer;
+    deserializeMetadata(buffer: Buffer): Record<string, MetadataTypes>;
+}
+//# sourceMappingURL=JsonCodec.d.ts.map

package/lib/codecs/JsonCodec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"JsonCodec.d.ts","sourceRoot":"","sources":["../../src/codecs/JsonCodec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE3D;;;;;;GAMG;AACH,qBAAa,SAAU,YAAW,UAAU;IACxC,QAAQ,CAAC,SAAS,WAAW;IAE7B,cAAc,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM;IAIvC,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS;IAI3C,gBAAgB,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM;IAIlD,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe;IAInD,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,MAAM;IAIlE,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC;CAGrE"}

package/lib/codecs/JsonCodec.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JsonCodec = void 0;
+/**
+ * JSON codec — default serialization format.
+ * @remarks
+ * Wraps the existing JSON.stringify/JSON.parse behavior. This preserves
+ * full backward compatibility with indexes created before the codec
+ * abstraction was introduced.
+ */
+class JsonCodec {
+    constructor() {
+        this.extension = '.json';
+    }
+    serializeIndex(data) {
+        return Buffer.from(JSON.stringify(data), 'utf-8');
+    }
+    deserializeIndex(buffer) {
+        return JSON.parse(buffer.toString('utf-8'));
+    }
+    serializeCatalog(catalog) {
+        return Buffer.from(JSON.stringify(catalog), 'utf-8');
+    }
+    deserializeCatalog(buffer) {
+        return JSON.parse(buffer.toString('utf-8'));
+    }
+    serializeMetadata(metadata) {
+        return Buffer.from(JSON.stringify(metadata), 'utf-8');
+    }
+    deserializeMetadata(buffer) {
+        return JSON.parse(buffer.toString('utf-8'));
+    }
+}
+exports.JsonCodec = JsonCodec;
+//# sourceMappingURL=JsonCodec.js.map

package/lib/codecs/JsonCodec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"JsonCodec.js","sourceRoot":"","sources":["../../src/codecs/JsonCodec.ts"],"names":[],"mappings":";;;AAGA;;;;;;GAMG;AACH,MAAa,SAAS;IAAtB;QACa,cAAS,GAAG,OAAO,CAAC;IAyBjC,CAAC;IAvBG,cAAc,CAAC,IAAe;QAC1B,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;IACtD,CAAC;IAED,gBAAgB,CAAC,MAAc;QAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,gBAAgB,CAAC,OAAwB;QACrC,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;IACzD,CAAC;IAED,kBAAkB,CAAC,MAAc;QAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,iBAAiB,CAAC,QAAuC;QACrD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1D,CAAC;IAED,mBAAmB,CAAC,MAAc;QAC9B,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;CACJ;AA1BD,8BA0BC"}

package/lib/codecs/JsonCodec.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=JsonCodec.spec.d.ts.map

package/lib/codecs/JsonCodec.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"JsonCodec.spec.d.ts","sourceRoot":"","sources":["../../src/codecs/JsonCodec.spec.ts"],"names":[],"mappings":""}

package/lib/codecs/JsonCodec.spec.js

@@ -0,0 +1,66 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_assert_1 = __importDefault(require("node:assert"));
+const JsonCodec_1 = require("./JsonCodec");
+describe('JsonCodec', () => {
+    const codec = new JsonCodec_1.JsonCodec();
+    it('has .json extension', () => {
+        node_assert_1.default.equal(codec.extension, '.json');
+    });
+    describe('serializeIndex / deserializeIndex', () => {
+        it('round-trips IndexData', () => {
+            const data = {
+                version: 1,
+                metadata_config: { indexed: ['category'] },
+                items: [
+                    { id: 'a', metadata: { category: 'food', score: 0.9 }, vector: [1.5, 2.5, 3.5], norm: 4.5 },
+                    { id: 'b', metadata: {}, vector: [0, 0, 0], norm: 0, metadataFile: 'ext.json' },
+                ],
+            };
+            const buf = codec.serializeIndex(data);
+            node_assert_1.default.ok(Buffer.isBuffer(buf));
+            const result = codec.deserializeIndex(buf);
+            node_assert_1.default.deepStrictEqual(result, data);
+        });
+        it('handles empty items', () => {
+            const data = { version: 2, metadata_config: {}, items: [] };
+            const result = codec.deserializeIndex(codec.serializeIndex(data));
+            node_assert_1.default.deepStrictEqual(result, data);
+        });
+    });
+    describe('serializeCatalog / deserializeCatalog', () => {
+        it('round-trips DocumentCatalog', () => {
+            const catalog = {
+                version: 1,
+                count: 2,
+                uriToId: { 'http://a': 'id-a', 'http://b': 'id-b' },
+                idToUri: { 'id-a': 'http://a', 'id-b': 'http://b' },
+            };
+            const buf = codec.serializeCatalog(catalog);
+            node_assert_1.default.ok(Buffer.isBuffer(buf));
+            const result = codec.deserializeCatalog(buf);
+            node_assert_1.default.deepStrictEqual(result, catalog);
+        });
+        it('handles empty catalog', () => {
+            const catalog = { version: 1, count: 0, uriToId: {}, idToUri: {} };
+            const result = codec.deserializeCatalog(codec.serializeCatalog(catalog));
+            node_assert_1.default.deepStrictEqual(result, catalog);
+        });
+    });
+    describe('serializeMetadata / deserializeMetadata', () => {
+        it('round-trips metadata with all types', () => {
+            const metadata = { name: 'test', score: 42, active: true };
+            const buf = codec.serializeMetadata(metadata);
+            const result = codec.deserializeMetadata(buf);
+            node_assert_1.default.deepStrictEqual(result, metadata);
+        });
+        it('handles empty metadata', () => {
+            const result = codec.deserializeMetadata(codec.serializeMetadata({}));
+            node_assert_1.default.deepStrictEqual(result, {});
+        });
+    });
+});
+//# sourceMappingURL=JsonCodec.spec.js.map

package/lib/codecs/JsonCodec.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"JsonCodec.spec.js","sourceRoot":"","sources":["../../src/codecs/JsonCodec.spec.ts"],"names":[],"mappings":";;;;;AAAA,8DAAiC;AACjC,2CAAwC;AAIxC,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACvB,MAAM,KAAK,GAAG,IAAI,qBAAS,EAAE,CAAC;IAE9B,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;QAC3B,qBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC/C,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;YAC7B,MAAM,IAAI,GAAc;gBACpB,OAAO,EAAE,CAAC;gBACV,eAAe,EAAE,EAAE,OAAO,EAAE,CAAC,UAAU,CAAC,EAAE;gBAC1C,KAAK,EAAE;oBACH,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE;oBAC3F,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE;iBAClF;aACJ,CAAC;YACF,MAAM,GAAG,GAAG,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YACvC,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC3C,qBAAM,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;YAC3B,MAAM,IAAI,GAAc,EAAE,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;YACvE,MAAM,MAAM,GAAG,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;YAClE,qBAAM,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uCAAuC,EAAE,GAAG,EAAE;QACnD,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACnC,MAAM,OAAO,GAAoB;gBAC7B,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE;gBACnD,OAAO,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE;aACtD,CAAC;YACF,MAAM,GAAG,GAAG,KAAK,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC5C,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAC7C,qBAAM,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;YAC7B,MAAM,OAAO,GAAoB,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;YACpF,MAAM,MAAM,GAAG,KAAK,CAAC,kBAAkB,CAAC,KAAK,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;YACzE,qBAAM,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACrD,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC3C,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAC3D,MAAM,GAAG,GAAG,KAAK,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAC9C,qBAAM,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;YAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,mBAAmB,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC;YACtE,qBAAM,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}

package/lib/codecs/LocalIndex.protobuf.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=LocalIndex.protobuf.spec.d.ts.map

package/lib/codecs/LocalIndex.protobuf.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalIndex.protobuf.spec.d.ts","sourceRoot":"","sources":["../../src/codecs/LocalIndex.protobuf.spec.ts"],"names":[],"mappings":""}

package/lib/codecs/LocalIndex.protobuf.spec.js

@@ -0,0 +1,108 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_assert_1 = __importDefault(require("node:assert"));
+const LocalIndex_1 = require("../LocalIndex");
+const storage_1 = require("../storage");
+const ProtobufCodec_1 = require("./ProtobufCodec");
+describe('LocalIndex with ProtobufCodec', () => {
+    const codec = new ProtobufCodec_1.ProtobufCodec();
+    it('creates index with .pb extension', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        const index = new LocalIndex_1.LocalIndex('mem://idx', undefined, storage, codec);
+        node_assert_1.default.equal(index.indexName, 'index.pb');
+        yield index.createIndex();
+        node_assert_1.default.equal(yield index.isIndexCreated(), true);
+    }));
+    it('full CRUD operations produce correct results', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        const index = new LocalIndex_1.LocalIndex('mem://idx', undefined, storage, codec);
+        yield index.createIndex();
+        // Insert
+        yield index.insertItem({ id: 'a', vector: [1, 0, 0], metadata: { cat: 'x' } });
+        yield index.insertItem({ id: 'b', vector: [0, 1, 0], metadata: { cat: 'y' } });
+        // List
+        const items = yield index.listItems();
+        node_assert_1.default.equal(items.length, 2);
+        node_assert_1.default.equal(items[0].id, 'a');
+        // Get
+        const item = yield index.getItem('a');
+        node_assert_1.default.ok(item);
+        node_assert_1.default.equal(item.metadata.cat, 'x');
+        // Query
+        const results = yield index.queryItems([1, 0, 0], '', 1);
+        node_assert_1.default.equal(results.length, 1);
+        node_assert_1.default.equal(results[0].item.id, 'a');
+        // Upsert
+        yield index.upsertItem({ id: 'a', vector: [0, 0, 1], metadata: { cat: 'z' } });
+        const updated = yield index.getItem('a');
+        node_assert_1.default.equal(updated === null || updated === void 0 ? void 0 : updated.metadata.cat, 'z');
+        // Delete
+        yield index.deleteItem('b');
+        const remaining = yield index.listItems();
+        node_assert_1.default.equal(remaining.length, 1);
+        node_assert_1.default.equal(remaining[0].id, 'a');
+        // Stats
+        const stats = yield index.getIndexStats();
+        node_assert_1.default.equal(stats.items, 1);
+    }));
+    it('batch insert works', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        const index = new LocalIndex_1.LocalIndex('mem://idx', undefined, storage, codec);
+        yield index.createIndex();
+        yield index.batchInsertItems([
+            { id: '1', vector: [1, 0, 0] },
+            { id: '2', vector: [0, 1, 0] },
+            { id: '3', vector: [0, 0, 1] },
+        ]);
+        const items = yield index.listItems();
+        node_assert_1.default.equal(items.length, 3);
+    }));
+    it('metadata filtering works with protobuf storage', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        const index = new LocalIndex_1.LocalIndex('mem://idx', undefined, storage, codec);
+        yield index.createIndex();
+        yield index.batchInsertItems([
+            { id: '1', vector: [1, 0], metadata: { category: 'food' } },
+            { id: '2', vector: [0, 1], metadata: { category: 'drink' } },
+        ]);
+        const food = yield index.listItemsByMetadata({ category: { $eq: 'food' } });
+        node_assert_1.default.equal(food.length, 1);
+        node_assert_1.default.equal(food[0].id, '1');
+    }));
+    it('external metadata files use .pb extension', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        const index = new LocalIndex_1.LocalIndex('mem://idx', undefined, storage, codec);
+        yield index.createIndex({ version: 1, metadata_config: { indexed: ['keep'] } });
+        yield index.insertItem({ id: 'm1', vector: [1], metadata: { keep: 'x', extra: 'y' } });
+        const items = yield index.listItems();
+        const stored = items.find(i => i.id === 'm1');
+        node_assert_1.default.ok(stored.metadataFile);
+        node_assert_1.default.ok(stored.metadataFile.endsWith('.pb'), `Expected .pb extension, got: ${stored.metadataFile}`);
+    }));
+    it('persists and reloads across instances', () => __awaiter(void 0, void 0, void 0, function* () {
+        const storage = new storage_1.VirtualFileStorage();
+        // First instance: create and populate
+        const idx1 = new LocalIndex_1.LocalIndex('mem://idx', undefined, storage, codec);
+        yield idx1.createIndex();
+        yield idx1.insertItem({ id: 'persist', vector: [1, 2, 3], metadata: { key: 'val' } });
+        // Second instance: should read persisted data
+        const idx2 = new LocalIndex_1.LocalIndex('mem://idx', undefined, storage, codec);
+        const item = yield idx2.getItem('persist');
+        node_assert_1.default.ok(item);
+        node_assert_1.default.equal(item.id, 'persist');
+        node_assert_1.default.equal(item.metadata.key, 'val');
+    }));
+});
+//# sourceMappingURL=LocalIndex.protobuf.spec.js.map

package/lib/codecs/LocalIndex.protobuf.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalIndex.protobuf.spec.js","sourceRoot":"","sources":["../../src/codecs/LocalIndex.protobuf.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,8DAAiC;AACjC,8CAA2C;AAC3C,wCAAgD;AAChD,mDAAgD;AAEhD,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC3C,MAAM,KAAK,GAAG,IAAI,6BAAa,EAAE,CAAC;IAElC,EAAE,CAAC,kCAAkC,EAAE,GAAS,EAAE;QAC9C,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,uBAAU,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QACrE,qBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAC1C,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;QAC1B,qBAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,cAAc,EAAE,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAS,EAAE;QAC1D,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,uBAAU,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QACrE,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;QAE1B,SAAS;QACT,MAAM,KAAK,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC/E,MAAM,KAAK,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAE/E,OAAO;QACP,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,qBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC9B,qBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAE/B,MAAM;QACN,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACtC,qBAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAChB,qBAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAErC,QAAQ;QACR,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QACzD,qBAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChC,qBAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAEtC,SAAS;QACT,MAAM,KAAK,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACzC,qBAAM,CAAC,KAAK,CAAC,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAEzC,SAAS;QACT,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QAC1C,qBAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAClC,qBAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAEnC,QAAQ;QACR,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;QAC1C,qBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACjC,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,oBAAoB,EAAE,GAAS,EAAE;QAChC,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,uBAAU,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QACrE,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;QAE1B,MAAM,KAAK,CAAC,gBAAgB,CAAC;YACzB,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE;YAC9B,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE;YAC9B,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE;SACjC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,qBAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAClC,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAS,EAAE;QAC5D,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,uBAAU,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QACrE,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;QAE1B,MAAM,KAAK,CAAC,gBAAgB,CAAC;YACzB,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE;YAC3D,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE;SAC/D,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,mBAAmB,CAAC,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAC5E,qBAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC7B,qBAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAClC,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAS,EAAE;QACvD,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,uBAAU,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QACrE,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,EAAE,OAAO,EAAE,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QAEhF,MAAM,KAAK,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAEvF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAE,CAAC;QAC/C,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAC/B,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,gCAAgC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;IAC1G,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAS,EAAE;QACnD,MAAM,OAAO,GAAG,IAAI,4BAAkB,EAAE,CAAC;QAEzC,sCAAsC;QACtC,MAAM,IAAI,GAAG,IAAI,uBAAU,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QACpE,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QAEtF,8CAA8C;QAC9C,MAAM,IAAI,GAAG,IAAI,uBAAU,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC3C,qBAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAChB,qBAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QACjC,qBAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC,CAAA,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}

package/lib/codecs/ProtobufCodec.d.ts

@@ -0,0 +1,20 @@
+import { IndexData, MetadataTypes } from '../types';
+import { DocumentCatalog, IndexCodec } from './IndexCodec';
+/**
+ * Protocol Buffers codec — opt-in binary format.
+ * @remarks
+ * Vectors are stored as packed float32 arrays (~50% smaller than JSON).
+ * Norms are stored as float64 to avoid compounding rounding error.
+ * Requires the `protobufjs` package to be installed.
+ */
+export declare class ProtobufCodec implements IndexCodec {
+    readonly extension = ".pb";
+    constructor();
+    serializeIndex(data: IndexData): Buffer;
+    deserializeIndex(buffer: Buffer): IndexData;
+    serializeCatalog(catalog: DocumentCatalog): Buffer;
+    deserializeCatalog(buffer: Buffer): DocumentCatalog;
+    serializeMetadata(metadata: Record<string, MetadataTypes>): Buffer;
+    deserializeMetadata(buffer: Buffer): Record<string, MetadataTypes>;
+}
+//# sourceMappingURL=ProtobufCodec.d.ts.map

package/lib/codecs/ProtobufCodec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ProtobufCodec.d.ts","sourceRoot":"","sources":["../../src/codecs/ProtobufCodec.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAa,aAAa,EAAE,MAAM,UAAU,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AA2F3D;;;;;;GAMG;AACH,qBAAa,aAAc,YAAW,UAAU;IAC5C,QAAQ,CAAC,SAAS,SAAS;;IAO3B,cAAc,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM;IAgBvC,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS;IAmB3C,gBAAgB,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM;IAelD,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe;IAkBnD,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,MAAM;IAalE,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC;CAGrE"}

package/lib/codecs/ProtobufCodec.js

@@ -0,0 +1,225 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProtobufCodec = void 0;
+const path = __importStar(require("path"));
+let protobuf;
+function loadProtobuf() {
+    if (!protobuf) {
+        try {
+            protobuf = require('protobufjs');
+        }
+        catch (_a) {
+            throw new Error("ProtobufCodec requires the 'protobufjs' package. Install it with: npm install protobufjs");
+        }
+    }
+    return protobuf;
+}
+// Cached proto root — loaded once per process
+let cachedRoot;
+function getRoot() {
+    if (!cachedRoot) {
+        const pb = loadProtobuf();
+        cachedRoot = pb.loadSync(path.join(__dirname, 'schemas', 'index.proto'));
+    }
+    return cachedRoot;
+}
+/** Convert MetadataTypes value to protobuf MetadataValue shape */
+function toProtoMetadataValue(value) {
+    if (typeof value === 'string')
+        return { stringValue: value };
+    if (typeof value === 'number')
+        return { numberValue: value };
+    if (typeof value === 'boolean')
+        return { boolValue: value };
+    return { stringValue: String(value) };
+}
+/** Convert protobuf MetadataValue back to MetadataTypes */
+function fromProtoMetadataValue(mv) {
+    if (mv.stringValue !== undefined && mv.stringValue !== '')
+        return mv.stringValue;
+    if (mv.numberValue !== undefined && mv.numberValue !== 0)
+        return mv.numberValue;
+    if (mv.boolValue !== undefined && mv.boolValue !== false)
+        return mv.boolValue;
+    // Disambiguate zero-value fields by checking which oneof field is set
+    if (mv.value === 'boolValue')
+        return false;
+    if (mv.value === 'numberValue')
+        return 0;
+    if (mv.value === 'stringValue')
+        return '';
+    // Fallback — proto3 defaults make this tricky; prefer string empty
+    return '';
+}
+/** Convert JS IndexItem to proto-friendly shape */
+function toProtoItem(item) {
+    const metadata = {};
+    if (item.metadata) {
+        for (const [k, v] of Object.entries(item.metadata)) {
+            metadata[k] = toProtoMetadataValue(v);
+        }
+    }
+    const out = {
+        id: item.id,
+        metadata,
+        vector: item.vector,
+        norm: item.norm,
+    };
+    if (item.metadataFile) {
+        out.metadataFile = item.metadataFile;
+    }
+    return out;
+}
+/** Convert proto IndexItem back to JS IndexItem */
+function fromProtoItem(pi) {
+    const metadata = {};
+    if (pi.metadata) {
+        for (const [k, v] of Object.entries(pi.metadata)) {
+            metadata[k] = fromProtoMetadataValue(v);
+        }
+    }
+    // proto packed float → JS number[]; ensure plain array
+    const vector = pi.vector ? Array.from(pi.vector) : [];
+    const item = {
+        id: pi.id,
+        metadata,
+        vector,
+        norm: pi.norm,
+    };
+    if (pi.metadataFile) {
+        item.metadataFile = pi.metadataFile;
+    }
+    return item;
+}
+/**
+ * Protocol Buffers codec — opt-in binary format.
+ * @remarks
+ * Vectors are stored as packed float32 arrays (~50% smaller than JSON).
+ * Norms are stored as float64 to avoid compounding rounding error.
+ * Requires the `protobufjs` package to be installed.
+ */
+class ProtobufCodec {
+    constructor() {
+        this.extension = '.pb';
+        // Eagerly validate that protobufjs is available
+        loadProtobuf();
+    }
+    serializeIndex(data) {
+        var _a, _b;
+        const root = getRoot();
+        const IndexDataMsg = root.lookupType('IndexData');
+        const payload = {
+            version: data.version,
+            metadataConfig: {
+                indexed: (_b = (_a = data.metadata_config) === null || _a === void 0 ? void 0 : _a.indexed) !== null && _b !== void 0 ? _b : [],
+            },
+            items: data.items.map(toProtoItem),
+        };
+        const err = IndexDataMsg.verify(payload);
+        if (err)
+            throw new Error(`Protobuf verify error: ${err}`);
+        const message = IndexDataMsg.create(payload);
+        return Buffer.from(IndexDataMsg.encode(message).finish());
+    }
+    deserializeIndex(buffer) {
+        var _a, _b;
+        const root = getRoot();
+        const IndexDataMsg = root.lookupType('IndexData');
+        const decoded = IndexDataMsg.decode(new Uint8Array(buffer));
+        const obj = IndexDataMsg.toObject(decoded, {
+            longs: Number,
+            enums: String,
+            defaults: true,
+            oneofs: true,
+        });
+        return {
+            version: obj.version,
+            metadata_config: {
+                indexed: (_b = (_a = obj.metadataConfig) === null || _a === void 0 ? void 0 : _a.indexed) !== null && _b !== void 0 ? _b : [],
+            },
+            items: (obj.items || []).map(fromProtoItem),
+        };
+    }
+    serializeCatalog(catalog) {
+        const root = getRoot();
+        const CatalogMsg = root.lookupType('DocumentCatalog');
+        const payload = {
+            version: catalog.version,
+            count: catalog.count,
+            uriToId: catalog.uriToId,
+            idToUri: catalog.idToUri,
+        };
+        const err = CatalogMsg.verify(payload);
+        if (err)
+            throw new Error(`Protobuf verify error: ${err}`);
+        const message = CatalogMsg.create(payload);
+        return Buffer.from(CatalogMsg.encode(message).finish());
+    }
+    deserializeCatalog(buffer) {
+        const root = getRoot();
+        const CatalogMsg = root.lookupType('DocumentCatalog');
+        const decoded = CatalogMsg.decode(new Uint8Array(buffer));
+        const obj = CatalogMsg.toObject(decoded, {
+            longs: Number,
+            enums: String,
+            defaults: true,
+            oneofs: true,
+        });
+        return {
+            version: obj.version,
+            count: obj.count,
+            uriToId: obj.uriToId || {},
+            idToUri: obj.idToUri || {},
+        };
+    }
+    serializeMetadata(metadata) {
+        // Metadata is a flat key-value map. We encode it as a JSON buffer
+        // wrapped in a simple length-prefixed envelope to stay self-describing
+        // while keeping the main index binary.
+        //
+        // We reuse the IndexItem metadata map encoding for consistency:
+        // encode as { entries: map<string, MetadataValue> }.
+        // However, since there's no standalone proto message for this,
+        // we just use JSON for external metadata files — the size savings
+        // from protobuf on small metadata objects are negligible.
+        return Buffer.from(JSON.stringify(metadata), 'utf-8');
+    }
+    deserializeMetadata(buffer) {
+        return JSON.parse(buffer.toString('utf-8'));
+    }
+}
+exports.ProtobufCodec = ProtobufCodec;
+//# sourceMappingURL=ProtobufCodec.js.map

package/lib/codecs/ProtobufCodec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ProtobufCodec.js","sourceRoot":"","sources":["../../src/codecs/ProtobufCodec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6B;AAI7B,IAAI,QAAiD,CAAC;AAEtD,SAAS,YAAY;IACjB,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,IAAI,CAAC;YACD,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;QACrC,CAAC;QAAC,WAAM,CAAC;YACL,MAAM,IAAI,KAAK,CACX,0FAA0F,CAC7F,CAAC;QACN,CAAC;IACL,CAAC;IACD,OAAO,QAAS,CAAC;AACrB,CAAC;AAED,8CAA8C;AAC9C,IAAI,UAAe,CAAC;AAEpB,SAAS,OAAO;IACZ,IAAI,CAAC,UAAU,EAAE,CAAC;QACd,MAAM,EAAE,GAAG,YAAY,EAAE,CAAC;QAC1B,UAAU,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,UAAU,CAAC;AACtB,CAAC;AAED,kEAAkE;AAClE,SAAS,oBAAoB,CAAC,KAAoB;IAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IAC7D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IAC7D,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AAC1C,CAAC;AAED,2DAA2D;AAC3D,SAAS,sBAAsB,CAAC,EAAO;IACnC,IAAI,EAAE,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,CAAC,WAAW,KAAK,EAAE;QAAE,OAAO,EAAE,CAAC,WAAW,CAAC;IACjF,IAAI,EAAE,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,CAAC,WAAW,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC,WAAW,CAAC;IAChF,IAAI,EAAE,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,CAAC,SAAS,KAAK,KAAK;QAAE,OAAO,EAAE,CAAC,SAAS,CAAC;IAC9E,sEAAsE;IACtE,IAAI,EAAE,CAAC,KAAK,KAAK,WAAW;QAAE,OAAO,KAAK,CAAC;IAC3C,IAAI,EAAE,CAAC,KAAK,KAAK,aAAa;QAAE,OAAO,CAAC,CAAC;IACzC,IAAI,EAAE,CAAC,KAAK,KAAK,aAAa;QAAE,OAAO,EAAE,CAAC;IAC1C,mEAAmE;IACnE,OAAO,EAAE,CAAC;AACd,CAAC;AAED,mDAAmD;AACnD,SAAS,WAAW,CAAC,IAAe;IAChC,MAAM,QAAQ,GAAwB,EAAE,CAAC;IACzC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjD,QAAQ,CAAC,CAAC,CAAC,GAAG,oBAAoB,CAAC,CAAkB,CAAC,CAAC;QAC3D,CAAC;IACL,CAAC;IACD,MAAM,GAAG,GAAwB;QAC7B,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,QAAQ;QACR,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,IAAI,EAAE,IAAI,CAAC,IAAI;KAClB,CAAC;IACF,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,GAAG,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;IACzC,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,mDAAmD;AACnD,SAAS,aAAa,CAAC,EAAO;IAC1B,MAAM,QAAQ,GAAkC,EAAE,CAAC;IACnD,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;QACd,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,QAA+B,CAAC,EAAE,CAAC;YACtE,QAAQ,CAAC,CAAC,CAAC,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC;IACD,uDAAuD;IACvD,MAAM,MAAM,GAAa,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,MAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5E,MAAM,IAAI,GAAc;QACpB,EAAE,EAAE,EAAE,CAAC,EAAE;QACT,QAAQ;QACR,MAAM;QACN,IAAI,EAAE,EAAE,CAAC,IAAI;KAChB,CAAC;IACF,IAAI,EAAE,CAAC,YAAY,EAAE,CAAC;QAClB,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC;IACxC,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,MAAa,aAAa;IAGtB;QAFS,cAAS,GAAG,KAAK,CAAC;QAGvB,gDAAgD;QAChD,YAAY,EAAE,CAAC;IACnB,CAAC;IAED,cAAc,CAAC,IAAe;;QAC1B,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;QACvB,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAClD,MAAM,OAAO,GAAG;YACZ,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,cAAc,EAAE;gBACZ,OAAO,EAAE,MAAA,MAAA,IAAI,CAAC,eAAe,0CAAE,OAAO,mCAAI,EAAE;aAC/C;YACD,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC;SACrC,CAAC;QACF,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7C,OAAO,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,gBAAgB,CAAC,MAAc;;QAC3B,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;QACvB,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAClD,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5D,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,OAAO,EAAE;YACvC,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;QACH,OAAO;YACH,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,eAAe,EAAE;gBACb,OAAO,EAAE,MAAA,MAAA,GAAG,CAAC,cAAc,0CAAE,OAAO,mCAAI,EAAE;aAC7C;YACD,KAAK,EAAE,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC;SAC9C,CAAC;IACN,CAAC;IAED,gBAAgB,CAAC,OAAwB;QACrC,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG;YACZ,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,OAAO,CAAC,OAAO;SAC3B,CAAC;QACF,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,kBAAkB,CAAC,MAAc;QAC7B,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,EAAE;YACrC,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;SACf,CAAC,CAAC;QACH,OAAO;YACH,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE;YAC1B,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE;SAC7B,CAAC;IACN,CAAC;IAED,iBAAiB,CAAC,QAAuC;QACrD,kEAAkE;QAClE,uEAAuE;QACvE,uCAAuC;QACvC,EAAE;QACF,gEAAgE;QAChE,qDAAqD;QACrD,+DAA+D;QAC/D,kEAAkE;QAClE,0DAA0D;QAC1D,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1D,CAAC;IAED,mBAAmB,CAAC,MAAc;QAC9B,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;CACJ;AA5FD,sCA4FC"}