rag-lite-ts 2.1.0 → 2.2.0

package/dist/core/binary-index-format.js

@@ -1,122 +0,0 @@
-/**
- * Binary Index Format Module
- *
- * Provides efficient binary serialization for HNSW vector indices.
- *
- * Format Specification:
- * - Header: 24 bytes (6 × uint32)
- * - Vectors: N × (4 + D × 4) bytes
- * - Little-endian encoding for cross-platform compatibility
- * - 4-byte alignment for Float32Array zero-copy views
- *
- * Performance:
- * - 3.66x smaller than JSON format
- * - 3.5x faster loading
- * - Zero-copy Float32Array views
- */
-import { readFileSync, writeFileSync } from 'fs';
-export class BinaryIndexFormat {
-    /**
-     * Save index data to binary format
-     *
-     * File structure:
-     * - Header (24 bytes): dimensions, maxElements, M, efConstruction, seed, currentSize
-     * - Vectors: For each vector: id (4 bytes) + vector data (dimensions × 4 bytes)
-     *
-     * @param indexPath Path to save the binary index file
-     * @param data Index data to serialize
-     */
-    static async save(indexPath, data) {
-        // Calculate total size
-        const headerSize = 24; // 6 uint32 fields
-        const vectorSize = 4 + (data.dimensions * 4); // id + vector
-        const totalSize = headerSize + (data.currentSize * vectorSize);
-        const buffer = new ArrayBuffer(totalSize);
-        const view = new DataView(buffer);
-        let offset = 0;
-        // Write header (24 bytes, all little-endian)
-        view.setUint32(offset, data.dimensions, true);
-        offset += 4;
-        view.setUint32(offset, data.maxElements, true);
-        offset += 4;
-        view.setUint32(offset, data.M, true);
-        offset += 4;
-        view.setUint32(offset, data.efConstruction, true);
-        offset += 4;
-        view.setUint32(offset, data.seed, true);
-        offset += 4;
-        view.setUint32(offset, data.currentSize, true);
-        offset += 4;
-        // Write vectors
-        for (const item of data.vectors) {
-            // Ensure 4-byte alignment (should always be true with our format)
-            if (offset % 4 !== 0) {
-                throw new Error(`Offset ${offset} is not 4-byte aligned`);
-            }
-            // Write vector ID
-            view.setUint32(offset, item.id, true);
-            offset += 4;
-            // Write vector data
-            for (let i = 0; i < item.vector.length; i++) {
-                view.setFloat32(offset, item.vector[i], true);
-                offset += 4;
-            }
-        }
-        // Write to file
-        writeFileSync(indexPath, Buffer.from(buffer));
-    }
-    /**
-     * Load index data from binary format
-     *
-     * Uses zero-copy Float32Array views for efficient loading.
-     * Copies the views to ensure data persistence after buffer lifecycle.
-     *
-     * @param indexPath Path to the binary index file
-     * @returns Deserialized index data
-     */
-    static async load(indexPath) {
-        const buffer = readFileSync(indexPath);
-        const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
-        let offset = 0;
-        // Read header (24 bytes, all little-endian)
-        const dimensions = view.getUint32(offset, true);
-        offset += 4;
-        const maxElements = view.getUint32(offset, true);
-        offset += 4;
-        const M = view.getUint32(offset, true);
-        offset += 4;
-        const efConstruction = view.getUint32(offset, true);
-        offset += 4;
-        const seed = view.getUint32(offset, true);
-        offset += 4;
-        const currentSize = view.getUint32(offset, true);
-        offset += 4;
-        // Read vectors
-        const vectors = [];
-        for (let i = 0; i < currentSize; i++) {
-            // Ensure 4-byte alignment (should always be true with our format)
-            if (offset % 4 !== 0) {
-                throw new Error(`Offset ${offset} is not 4-byte aligned`);
-            }
-            // Read vector ID
-            const id = view.getUint32(offset, true);
-            offset += 4;
-            // Zero-copy Float32Array view (fast!)
-            const vectorView = new Float32Array(buffer.buffer, buffer.byteOffset + offset, dimensions);
-            // Copy to avoid buffer lifecycle issues
-            const vector = new Float32Array(vectorView);
-            offset += dimensions * 4;
-            vectors.push({ id, vector });
-        }
-        return {
-            dimensions,
-            maxElements,
-            M,
-            efConstruction,
-            seed,
-            currentSize,
-            vectors
-        };
-    }
-}
-//# sourceMappingURL=binary-index-format.js.map