verso-db 0.2.0 → 0.4.0

package/dist/HNSWIndex.js

@@ -11,6 +11,7 @@ export class HNSWIndex {
     static MAX_NODE_ID = HNSWIndex.MISSING_ID_SENTINEL - 1;
     static MAX_ARRAY_BUFFER_BYTES = 0x7FFFFFFF;
     static MAX_NODE_SLOTS = 50_000_000;
+    static MAX_SERIALIZED_LEVEL = 31;
     /**
      * Allocate a Float32Array backed by SharedArrayBuffer when available.
      * This allows workers to read vector data without copying.
@@ -32,6 +33,12 @@ export class HNSWIndex {
     }
     /** Whether flatVectors/flatInt8Vectors use SharedArrayBuffer */
     useSharedMemory = false;
+    // Shared graph SAB state for zero-copy worker communication
+    sharedGraphIndex = null;
+    sharedGraphNeighborData = null;
+    sharedGraphMaxLayerSlots = 0;
+    sharedGraphWriteOffset = 0;
+    sharedMetadata = null;
     M; // Max number of connections per node per level
     M0; // Max number of connections for level 0 (typically M * 2)
     efConstruction; // Size of candidate list during construction
@@ -63,6 +70,7 @@ export class HNSWIndex {
     vectorsAreNormalized = false;
     // Cached distance function to avoid switch overhead
     distanceFn;
+    random;
     // Quantization support for 3-4x faster search with Int8
     scalarQuantizer = null;
     // OPTIMIZATION: Use array instead of Map for int8 vectors too
@@ -92,7 +100,7 @@ export class HNSWIndex {
     constructionMode = false;
     // Adaptive efSearch calibration stats
     calibrationStats = null;
-    constructor(dimension, metric = 'cosine', M = 24, efConstruction = 200) {
+    constructor(dimension, metric = 'cosine', M = 24, efConstruction = 200, random = Math.random) {
         if (dimension <= 0 || !Number.isInteger(dimension)) {
             throw new VectorDBError(`Invalid dimension: must be a positive integer, got ${dimension}`, 'VALIDATION_ERROR');
         }
@@ -102,8 +110,12 @@ export class HNSWIndex {
         if (efConstruction <= 0 || !Number.isInteger(efConstruction)) {
             throw new VectorDBError(`Invalid efConstruction parameter: must be a positive integer, got ${efConstruction}`, 'VALIDATION_ERROR');
         }
+        if (typeof random !== 'function') {
+            throw new VectorDBError('Invalid random source: must be a function returning a number in [0, 1)', 'VALIDATION_ERROR');
+        }
         this.dimension = dimension;
         this.metric = metric;
+        this.random = random;
         this.M = M;
         this.M0 = M * 2;
         this.efConstruction = efConstruction;
@@ -125,6 +137,9 @@ export class HNSWIndex {
         this.visitedArraySize = 10000;
         this.visitedArray = new Uint16Array(this.visitedArraySize);
         this.visitedGeneration = 1;
+        // Pre-allocate batch distance buffers sized for max neighbors at layer 0
+        this.batchNeighborIds = new Uint32Array(this.M0 + 1);
+        this.batchDistances = new Float64Array(this.M0 + 1);
         // Pre-allocate searchLayer heaps - sized for typical ef values
         // Will be resized if needed for larger ef
         this.heapCapacity = Math.max(efConstruction * 2, 500);
@@ -223,9 +238,10 @@ export class HNSWIndex {
             this.nextAutoId = id + 1;
         }
     }
-    // OPTIMIZATION: Reusable arrays for batch distance calculation
-    batchNeighborIds = [];
-    batchDistances = [];
+    // OPTIMIZATION: Pre-allocated typed arrays for batch distance calculation
+    // Sized to M0+1 (max neighbors at layer 0) to eliminate dynamic resizing
+    batchNeighborIds;
+    batchDistances;
     /**
      * OPTIMIZATION: Batch distance calculation for better cache locality
      * Computes distances from query to multiple neighbors at once
@@ -427,8 +443,18 @@ export class HNSWIndex {
         normalizeInPlace(vector);
         return vector;
     }
+    randomFloat() {
+        const value = this.random();
+        if (!Number.isFinite(value))
+            return 0;
+        if (value <= 0)
+            return 0;
+        if (value >= 1)
+            return 1 - Number.EPSILON;
+        return value;
+    }
     selectLevel() {
-        const r = Math.random() || Number.MIN_VALUE;
+        const r = this.randomFloat() || Number.MIN_VALUE;
         const level = Math.floor(-Math.log(r) * this.levelMult);
         return Math.max(0, Math.min(level, this.maxLayers - 1));
     }
@@ -667,10 +693,6 @@ export class HNSWIndex {
                 }
                 // Calculate all distances at once (better cache utilization)
                 if (batchCount > 0) {
-                    // Ensure batch arrays are large enough
-                    if (batchDists.length < batchCount) {
-                        this.batchDistances.length = batchCount;
-                    }
                     this.calculateDistancesBatch(query, batchIds, batchDists, batchCount);
                     // Process batch results
                     for (let i = 0; i < batchCount; i++) {
@@ -774,23 +796,10 @@ export class HNSWIndex {
         if (this.nodes[id]) {
             throw new VectorDBError(`Duplicate node ID ${id}: node already exists`, 'DUPLICATE_VECTOR');
         }
-        // Optimize: only copy when necessary
-        // - Always copy arrays (need Float32Array)
-        // - Copy Float32Array only if we need to normalize (modifies in place)
-        // - Reuse input directly if skipNormalization is set (caller guarantees immutability)
-        let floatVector;
-        if (Array.isArray(vector)) {
-            floatVector = new Float32Array(vector);
-        }
-        else if (this.vectorsAreNormalized && !options?.skipNormalization) {
-            // Need to copy because normalizeVector modifies in place
-            floatVector = new Float32Array(vector);
-        }
-        else {
-            // No normalization needed and input is Float32Array - use directly
-            // Note: caller should not modify this array after passing it
-            floatVector = vector;
-        }
+        // Always take ownership of vector data. Reusing caller-owned
+        // Float32Arrays lets later external mutation diverge node.vector from the
+        // flat vector store and corrupt search behavior.
+        let floatVector = new Float32Array(vector);
         if (floatVector.length !== this.dimension) {
             throw new VectorDBError(`Vector dimension ${floatVector.length} does not match expected ${this.dimension}`, 'DIMENSION_MISMATCH');
         }
@@ -1115,7 +1124,7 @@ export class HNSWIndex {
         const sampleIndices = [];
         const usedIndices = new Set();
         while (sampleIndices.length < sampleSize) {
-            const idx = Math.floor(Math.random() * n);
+            const idx = Math.floor(this.randomFloat() * n);
             if (!usedIndices.has(idx)) {
                 usedIndices.add(idx);
                 sampleIndices.push(idx);
@@ -1297,12 +1306,11 @@ export class HNSWIndex {
         }
         // Pre-normalize all vectors if needed
         const normalizedPoints = points.map(p => {
+            const vector = new Float32Array(p.vector);
             if (this.vectorsAreNormalized && !skipNorm) {
-                const v = new Float32Array(p.vector);
-                normalizeInPlace(v);
-                return { id: p.id, vector: v };
+                normalizeInPlace(vector);
             }
-            return p;
+            return { id: p.id, vector };
         });
         // Phase 1: Sequential seed insertion
         const seedSize = Math.min(Math.max(500, Math.floor(normalizedPoints.length * seedFraction)), normalizedPoints.length);
@@ -1400,17 +1408,27 @@ export class HNSWIndex {
                 }
                 batchCount++;
                 // Send incremental graph update: ALL modified nodes (new + existing with new connections)
-                const graphUpdate = [];
-                for (const nid of modifiedNodeIds) {
-                    const node = this.nodes[nid];
-                    if (node) {
-                        graphUpdate.push({ id: nid, neighbors: node.neighbors });
+                if (this.hasSharedGraph()) {
+                    // Shared graph: write directly to SABs, workers see updates via shared memory
+                    this.updateSharedGraphNodes(modifiedNodeIds);
+                    this.updateSharedMetadata();
+                }
+                else {
+                    // Legacy: send graph data via postMessage
+                    const graphUpdate = [];
+                    for (const nid of modifiedNodeIds) {
+                        const node = this.nodes[nid];
+                        if (node) {
+                            graphUpdate.push({ id: nid, neighbors: node.neighbors });
+                        }
                     }
+                    pool.broadcastGraphUpdate(graphUpdate, this.entryPointId, this.maxLevel);
                 }
-                pool.broadcastGraphUpdate(graphUpdate, this.entryPointId, this.maxLevel);
                 // Full re-sync periodically for accumulated neighbor changes
-                // (bidirectional connections modify existing nodes' neighbor lists)
+                // With shared graph, this defragments dead space in neighbor data.
+                // Without shared graph, this fixes accumulated drift from incremental updates.
                 if (batchCount % resyncInterval === 0 && start + batchSize < normalizedPoints.length) {
+                    this.clearSharedGraph();
                     pool.destroy();
                     await pool.init(this);
                 }
@@ -1572,7 +1590,6 @@ export class HNSWIndex {
     getSharedSearchData() {
         if (this.nodeCount === 0 || this.entryPointId === -1)
             return null;
-        const graphData = this.serializeGraphStructure();
         const nodeLevels = new Uint8Array(this.nodeCount);
         for (let i = 0; i < this.nodeCount; i++) {
             const node = this.nodes[i];
@@ -1606,7 +1623,7 @@ export class HNSWIndex {
                 quantizationParams = this.scalarQuantizer.getParams();
             }
         }
-        return {
+        const baseData = {
             flatVectors,
             flatInt8Vectors,
             dimension: this.dimension,
@@ -1616,14 +1633,154 @@ export class HNSWIndex {
             maxLevel: this.maxLevel,
             M: this.M,
             M0: this.M0,
-            graphData,
             nodeLevels,
             quantizationEnabled: this.quantizationEnabled,
             quantizationParams,
         };
+        // Use shared graph SABs when SharedArrayBuffer is available
+        if (this.useSharedMemory && typeof SharedArrayBuffer !== 'undefined') {
+            const sharedGraph = this.serializeGraphToSharedBuffers();
+            return {
+                ...baseData,
+                graphNeighborData: sharedGraph.graphNeighborData,
+                graphIndex: sharedGraph.graphIndex,
+                maxLayerSlots: sharedGraph.maxLayerSlots,
+                sharedMetadata: sharedGraph.sharedMetadata,
+            };
+        }
+        // Fallback: legacy serialized graph
+        return {
+            ...baseData,
+            graphData: this.serializeGraphStructure(),
+        };
+    }
+    /**
+     * Serialize graph into SAB-backed flat typed arrays for zero-copy worker sharing.
+     * Layout:
+     *   graphIndex[(nodeId * maxLayerSlots + layer) * 2]     = offset into graphNeighborData
+     *   graphIndex[(nodeId * maxLayerSlots + layer) * 2 + 1] = neighbor count
+     *   graphNeighborData[offset..offset+count]              = neighbor IDs
+     *
+     * Pre-allocates extra capacity for growth during parallel build.
+     */
+    serializeGraphToSharedBuffers() {
+        const maxLayerSlots = Math.max(this.maxLevel + 4, 8);
+        // Count total neighbors for sizing
+        let totalNeighbors = 0;
+        for (let i = 0; i < this.nodeCount; i++) {
+            const node = this.nodes[i];
+            if (!node)
+                continue;
+            for (const neighbors of node.neighbors) {
+                totalNeighbors += neighbors?.length ?? 0;
+            }
+        }
+        // Pre-allocate with capacity for ALL nodes (not just current ones).
+        // During parallel build, serializeGraphToSharedBuffers is called after the seed phase
+        // (e.g. 500 nodes), but the full index may grow to flatVectorsCapacity nodes.
+        // Each node has up to M0 layer-0 neighbors + M upper-layer neighbors.
+        // The append-only write pattern creates dead space when nodes are rewritten,
+        // so we use 3x the estimated maximum to accommodate waste.
+        const nodeCapacity = Math.max(this.nodeCount, this.flatVectorsCapacity);
+        const estimatedMaxNeighbors = nodeCapacity * (this.M0 + this.M);
+        const neighborCapacity = Math.max(Math.ceil(totalNeighbors * 3), totalNeighbors + 10000, estimatedMaxNeighbors * 3);
+        const indexSize = nodeCapacity * maxLayerSlots * 2;
+        const graphIndex = new Uint32Array(new SharedArrayBuffer(indexSize * 4));
+        const graphNeighborData = new Uint32Array(new SharedArrayBuffer(neighborCapacity * 4));
+        // Shared metadata: [nodeCount, entryPointId, maxLevel]
+        const sharedMetadata = new Uint32Array(new SharedArrayBuffer(3 * 4));
+        sharedMetadata[0] = this.nodeCount;
+        sharedMetadata[1] = this.entryPointId;
+        sharedMetadata[2] = this.maxLevel;
+        // Serialize current graph
+        let writeOffset = 0;
+        for (let nodeId = 0; nodeId < this.nodeCount; nodeId++) {
+            const node = this.nodes[nodeId];
+            if (!node)
+                continue;
+            for (let l = 0; l < node.neighbors.length && l < maxLayerSlots; l++) {
+                const neighbors = node.neighbors[l] ?? [];
+                const base = (nodeId * maxLayerSlots + l) * 2;
+                graphIndex[base] = writeOffset;
+                graphIndex[base + 1] = neighbors.length;
+                for (let n = 0; n < neighbors.length; n++) {
+                    graphNeighborData[writeOffset + n] = neighbors[n];
+                }
+                writeOffset += neighbors.length;
+            }
+        }
+        // Store references for incremental updates during parallel build
+        this.sharedGraphIndex = graphIndex;
+        this.sharedGraphNeighborData = graphNeighborData;
+        this.sharedGraphMaxLayerSlots = maxLayerSlots;
+        this.sharedGraphWriteOffset = writeOffset;
+        this.sharedMetadata = sharedMetadata;
+        return { graphNeighborData, graphIndex, maxLayerSlots, sharedMetadata };
+    }
+    /**
+     * Update shared graph SABs for specific nodes.
+     * Called during parallel build after each batch to sync graph changes.
+     * Workers see updates immediately via shared memory — no postMessage needed.
+     */
+    updateSharedGraphNodes(nodeIds) {
+        if (!this.sharedGraphIndex || !this.sharedGraphNeighborData)
+            return;
+        const maxLayerSlots = this.sharedGraphMaxLayerSlots;
+        let writeOffset = this.sharedGraphWriteOffset;
+        for (const nodeId of nodeIds) {
+            const node = this.nodes[nodeId];
+            if (!node)
+                continue;
+            for (let l = 0; l < node.neighbors.length && l < maxLayerSlots; l++) {
+                const neighbors = node.neighbors[l] ?? [];
+                const base = (nodeId * maxLayerSlots + l) * 2;
+                // If capacity exhausted, clear shared graph to force fallback to
+                // broadcastGraphUpdate. Workers keep their current SAB data (stale but valid).
+                // The periodic resync in addPointsBulkParallel will create fresh SABs.
+                if (writeOffset + neighbors.length > this.sharedGraphNeighborData.length) {
+                    this.sharedGraphWriteOffset = writeOffset;
+                    this.clearSharedGraph();
+                    return;
+                }
+                this.sharedGraphIndex[base] = writeOffset;
+                this.sharedGraphIndex[base + 1] = neighbors.length;
+                for (let n = 0; n < neighbors.length; n++) {
+                    this.sharedGraphNeighborData[writeOffset + n] = neighbors[n];
+                }
+                writeOffset += neighbors.length;
+            }
+        }
+        this.sharedGraphWriteOffset = writeOffset;
+    }
+    /**
+     * Update shared metadata SAB with current index state.
+     * Workers read these values during search.
+     */
+    updateSharedMetadata() {
+        if (!this.sharedMetadata)
+            return;
+        this.sharedMetadata[0] = this.nodeCount;
+        this.sharedMetadata[1] = this.entryPointId;
+        this.sharedMetadata[2] = this.maxLevel;
+    }
+    /**
+     * Check if shared graph SABs are active (for parallel build optimization).
+     */
+    hasSharedGraph() {
+        return this.sharedGraphIndex !== null;
+    }
+    /**
+     * Clear shared graph references (called when pool is destroyed/re-initialized).
+     */
+    clearSharedGraph() {
+        this.sharedGraphIndex = null;
+        this.sharedGraphNeighborData = null;
+        this.sharedMetadata = null;
+        this.sharedGraphWriteOffset = 0;
     }
     /**
      * Serialize graph structure (neighbor lists) into a compact ArrayBuffer.
+     * Legacy format for non-SAB fallback.
      * Format per node: [numLayers:uint8] [numNeighbors:uint16, neighborId:uint32...] per layer
      */
     serializeGraphStructure() {
@@ -1959,6 +2116,18 @@ export class HNSWIndex {
             const entryPointId = readInt32('entryPointId');
             const nodeCount = readUint32('nodeCount');
             const vectorByteLength = dimension * 4;
+            if (nodeCount > HNSWIndex.MAX_NODE_SLOTS) {
+                throw new VectorDBError(`Corrupt HNSW data: node count ${nodeCount} exceeds maximum supported ${HNSWIndex.MAX_NODE_SLOTS}`, 'CORRUPT_INDEX');
+            }
+            if (maxLevel < -1 || maxLevel > HNSWIndex.MAX_SERIALIZED_LEVEL) {
+                throw new VectorDBError(`Corrupt HNSW data: invalid maxLevel ${maxLevel}`, 'CORRUPT_INDEX');
+            }
+            if (nodeCount === 0 && (entryPointId !== -1 || maxLevel !== -1)) {
+                throw new VectorDBError('Corrupt HNSW data: empty index has non-empty entry point metadata', 'CORRUPT_INDEX');
+            }
+            if (nodeCount > 0 && (entryPointId < 0 || maxLevel < 0)) {
+                throw new VectorDBError('Corrupt HNSW data: populated index has missing entry point metadata', 'CORRUPT_INDEX');
+            }
             // V3+ has vectorDataOffset in header
             let vectorDataOffset = 0;
             if (formatVersion >= 3) {
@@ -1971,6 +2140,41 @@ export class HNSWIndex {
             index.maxLevel = maxLevel;
             index.entryPointId = entryPointId;
             const indexToId = new Array(nodeCount);
+            const seenNodeIds = new Set();
+            let observedMaxLevel = -1;
+            const validateSerializedNode = (id, level, nodeIndex) => {
+                if (id > HNSWIndex.MAX_NODE_ID) {
+                    throw new VectorDBError(`Corrupt HNSW data: node ${nodeIndex} has reserved or unsupported ID ${id}`, 'CORRUPT_INDEX');
+                }
+                if (level > HNSWIndex.MAX_SERIALIZED_LEVEL || level > maxLevel) {
+                    throw new VectorDBError(`Corrupt HNSW data: node ${nodeIndex} has invalid level ${level}`, 'CORRUPT_INDEX');
+                }
+                if (seenNodeIds.has(id)) {
+                    throw new VectorDBError(`Corrupt HNSW data: duplicate node ID ${id}`, 'CORRUPT_INDEX');
+                }
+                seenNodeIds.add(id);
+                if (level > observedMaxLevel)
+                    observedMaxLevel = level;
+            };
+            const validateNeighborList = (nodeId, layer, neighbors) => {
+                const maxConnections = layer === 0 ? M * 2 : M;
+                if (neighbors.length > maxConnections) {
+                    throw new VectorDBError(`Corrupt HNSW data: node ${nodeId} layer ${layer} has ${neighbors.length} neighbors, maximum is ${maxConnections}`, 'CORRUPT_INDEX');
+                }
+                const seenNeighbors = new Set();
+                for (const neighborId of neighbors) {
+                    if (!seenNodeIds.has(neighborId)) {
+                        throw new VectorDBError(`Corrupt HNSW data: node ${nodeId} references missing neighbor ${neighborId}`, 'CORRUPT_INDEX');
+                    }
+                    if (neighborId === nodeId) {
+                        throw new VectorDBError(`Corrupt HNSW data: node ${nodeId} references itself as a neighbor`, 'CORRUPT_INDEX');
+                    }
+                    if (seenNeighbors.has(neighborId)) {
+                        throw new VectorDBError(`Corrupt HNSW data: node ${nodeId} has duplicate neighbor ${neighborId}`, 'CORRUPT_INDEX');
+                    }
+                    seenNeighbors.add(neighborId);
+                }
+            };
             if (formatVersion >= 3) {
                 // V3 format: vectors at end, supports lazy loading
                 const nodeMetadata = [];
@@ -1979,6 +2183,7 @@ export class HNSWIndex {
                 for (let i = 0; i < nodeCount; i++) {
                     const id = readUint32(`node ${i} id`);
                     const level = readUint32(`node ${i} level`);
+                    validateSerializedNode(id, level, i);
                     indexToId[i] = id;
                     nodeMetadata.push({ id, level });
                 }
@@ -2017,6 +2222,7 @@ export class HNSWIndex {
                             }
                             return indexToId[idx];
                         });
+                        validateNeighborList(nodeMetadata[i].id, l, neighbors[l]);
                     }
                     nodeNeighbors.push(neighbors);
                 }
@@ -2081,6 +2287,7 @@ export class HNSWIndex {
                 for (let i = 0; i < nodeCount; i++) {
                     const id = readUint32(`node ${i} id`);
                     const level = readUint32(`node ${i} level`);
+                    validateSerializedNode(id, level, i);
                     indexToId[i] = id;
                     const vector = new Float32Array(dimension);
                     for (let j = 0; j < dimension; j++) {
@@ -2120,6 +2327,7 @@ export class HNSWIndex {
                             }
                             return indexToId[idx];
                         });
+                        validateNeighborList(id, l, neighbors[l]);
                     }
                     const node = { id, level, vector, neighbors };
                     index.setNode(node);
@@ -2130,6 +2338,7 @@ export class HNSWIndex {
                 for (let i = 0; i < nodeCount; i++) {
                     const id = readUint32(`node ${i} id`);
                     const level = readUint32(`node ${i} level`);
+                    validateSerializedNode(id, level, i);
                     indexToId[i] = id;
                     const vector = new Float32Array(dimension);
                     for (let j = 0; j < dimension; j++) {
@@ -2155,9 +2364,20 @@ export class HNSWIndex {
                             }
                             node.neighbors[l][j] = indexToId[neighborIndex];
                         }
+                        validateNeighborList(node.id, l, node.neighbors[l]);
                     }
                 }
             }
+            if (nodeCount > 0 && observedMaxLevel !== maxLevel) {
+                throw new VectorDBError(`Corrupt HNSW data: maxLevel ${maxLevel} does not match highest node level ${observedMaxLevel}`, 'CORRUPT_INDEX');
+            }
+            if (nodeCount > 0 && !seenNodeIds.has(entryPointId)) {
+                throw new VectorDBError(`Corrupt HNSW data: entry point ${entryPointId} is not present in node table`, 'CORRUPT_INDEX');
+            }
+            const entryPoint = nodeCount > 0 ? index.nodes[entryPointId] : undefined;
+            if (entryPoint && entryPoint.level !== maxLevel) {
+                throw new VectorDBError(`Corrupt HNSW data: entry point level ${entryPoint.level} does not match maxLevel ${maxLevel}`, 'CORRUPT_INDEX');
+            }
             return index;
         }
         catch (error) {
@@ -2324,7 +2544,7 @@ export class HNSWIndex {
                 continue;
             const vector = this.getNodeVector(node.id);
             if (vector) {
-                result.set(node.id, vector);
+                result.set(node.id, new Float32Array(vector));
             }
         }
         return result;
@@ -2718,9 +2938,6 @@ export class HNSWIndex {
                     }
                 }
                 if (batchCount > 0) {
-                    if (batchDists.length < batchCount) {
-                        this.batchDistances.length = batchCount;
-                    }
                     this.calculateDistancesBatchInt8(int8Query, batchIds, batchDists, batchCount);
                     for (let i = 0; i < batchCount; i++) {
                         const neighborId = batchIds[i];