sweet-search 2.5.2 → 2.5.4

package/core/vector-store/binary-hnsw-index.js

@@ -27,7 +27,7 @@
  */
 
 import fs from 'fs/promises';
-import { existsSync } from 'fs';
+import { existsSync, statSync } from 'fs';
 import path from 'path';
 import { BINARY_HNSW_CONFIG, DB_PATHS } from '../infrastructure/config/index.js';
 import {
@@ -37,6 +37,7 @@ import {
 } from '../infrastructure/quantization.js';
 import { wasmHammingDistance as hammingDistance } from '../infrastructure/simd-distance.js';
 import { TypedMinHeap, TypedMaxHeap, VisitedList } from './binary-heap.js';
+import { loadBitmap, isSet } from '../infrastructure/tombstone-bitmap-reader.js';
 
 // Current quantization pipeline version. Bump when the encoding pipeline changes
 // (centroid subtraction, rotation, quantization scheme). Indexes built with a
@@ -60,6 +61,7 @@ export class BinaryHNSWIndex {
     this.maxElements = options.maxElements || BINARY_HNSW_CONFIG.maxElements;
 
     this.indexPath = options.indexPath || DB_PATHS.binaryHnswIndex;
+    this.stalePath = options.stalePath || `${this.indexPath}.stale.bin`;
 
     // Storage
     this.vectors = [];           // Array of { id, binary: Uint8Array, metadata }
@@ -81,12 +83,15 @@ export class BinaryHNSWIndex {
     this.centroid = null;       // Float32Array — dataset centroid
     this.signVector = null;     // Float32Array — random ±1 for WHT rotation
     this.useAsymmetric = false; // Enabled after calibration
+    this._staleBitmapCache = null;
+    this._cleanBuild = false;
   }
 
   /** Reset to empty state for a fresh build (skips loading from disk). */
   resetForBuild() {
     this.vectors = [];
     this.idToIndex = new Map();
+    this.int8Vectors.clear();
     this.graph = [];
     this.entryPoint = -1;
     this.maxLevel = 0;
@@ -94,6 +99,12 @@ export class BinaryHNSWIndex {
     this.signVector = null;
     this.useAsymmetric = false;
     this.initialized = true;
+    this._staleBitmapCache = null;
+    this._cleanBuild = true;
+  }
+
+  _stalePathForIndex(indexPath = this.indexPath) {
+    return indexPath === this.indexPath ? this.stalePath : `${indexPath}.stale.bin`;
   }
 
   /**
@@ -422,6 +433,7 @@ export class BinaryHNSWIndex {
     await this.init();
 
     const start = performance.now();
+    const staleBitmap = this._loadStaleBitmap();
 
     if (this.vectors.length === 0) {
       return { results: [], latency_us: 0, k, total: 0 };
@@ -449,12 +461,12 @@ export class BinaryHNSWIndex {
     }
 
     // Adaptive ef: easy queries get a smaller budget, hard queries get more
-    let ef = Math.max(k, this.efSearch);
+    let ef = Math.max(this._oversampleTarget(k, staleBitmap), this.efSearch);
     const greedyDist = hammingDistance(this.vectors[currentNode].binary, queryBinary);
     const maxDist = this.dimension * 8;
     const greedyQuality = 1 - (greedyDist / maxDist);
     if (greedyQuality > 0.85) {
-      ef = Math.max(k, Math.round(ef * 0.6));
+      ef = Math.max(this._oversampleTarget(k, staleBitmap), Math.round(ef * 0.6));
     } else if (greedyQuality < 0.55) {
       ef = Math.round(ef * 1.5);
     }
@@ -463,6 +475,15 @@ export class BinaryHNSWIndex {
     const searchResult = this.searchLayerQuery(currentNode, queryBinary, ef, 0);
     let candidates = searchResult.candidates;
 
+    candidates = candidates.filter(c => !this._isIndexStale(c.idx, staleBitmap));
+    if (candidates.length < k && ef < this.vectors.length) {
+      const retryEf = Math.min(this.vectors.length, ef * 2);
+      if (retryEf > ef) {
+        const retry = this.searchLayerQuery(currentNode, queryBinary, retryEf, 0);
+        candidates = retry.candidates.filter(c => !this._isIndexStale(c.idx, staleBitmap));
+      }
+    }
+
     // Return top k
     const results = candidates.slice(0, k).map(c => ({
       id: this.vectors[c.idx].id,
@@ -478,13 +499,69 @@ export class BinaryHNSWIndex {
       latency_us: Math.round(latency * 1000),
       latency_ms: latency.toFixed(3),
       k,
-      total: this.vectors.length,
+      total: this._liveVectorCount(staleBitmap),
       visitedNodes: searchResult.visitedCount,
       adaptiveEf: ef,
       useAsymmetric: this.useAsymmetric,
     };
   }
 
+  _loadStaleBitmap() {
+    if (!existsSync(this.stalePath)) {
+      this._staleBitmapCache = null;
+      return null;
+    }
+    let stat;
+    try {
+      stat = statSync(this.stalePath, { bigint: true });
+    } catch {
+      this._staleBitmapCache = null;
+      return null;
+    }
+    const statKey = `${stat.mtimeNs}:${stat.ctimeNs}:${stat.size}`;
+
+    if (
+      this._staleBitmapCache
+      && this._staleBitmapCache.statKey === statKey
+    ) {
+      return this._staleBitmapCache.bitmap;
+    }
+
+    try {
+      const bitmap = loadBitmap(this.stalePath);
+      this._staleBitmapCache = { statKey, bitmap };
+      return bitmap;
+    } catch (err) {
+      if (process.env.SWEET_DEBUG) {
+        console.debug(`[BinaryHNSW] ignoring unreadable stale bitmap ${this.stalePath}: ${err.message}`);
+      }
+      this._staleBitmapCache = { statKey, bitmap: null };
+      return null;
+    }
+  }
+
+  _isIndexStale(idx, bitmap) {
+    return bitmap ? isSet(bitmap, idx) : false;
+  }
+
+  _liveVectorCount(bitmap) {
+    if (!bitmap) return this.vectors.length;
+    let live = 0;
+    for (let i = 0; i < this.vectors.length; i++) {
+      if (!this._isIndexStale(i, bitmap)) live++;
+    }
+    return live;
+  }
+
+  _oversampleTarget(k, bitmap) {
+    if (!bitmap) return k;
+    const live = this._liveVectorCount(bitmap);
+    const tombstoned = Math.max(0, this.vectors.length - live);
+    if (tombstoned === 0) return k;
+    const s = Math.max(0, Math.min(tombstoned / Math.max(1, this.vectors.length), 0.5));
+    return Math.min(Math.max(k + 64, Math.ceil(k / Math.max(0.05, 1 - s) * 2)), k * 20);
+  }
+
   /**
    * Greedy search for query vector
    */
@@ -633,16 +710,36 @@ export class BinaryHNSWIndex {
    * Get int8 vector for rescoring
    */
   getInt8Vector(id) {
+    const idx = this.idToIndex.get(id);
+    if (idx === undefined) return undefined;
+    if (this._isIndexStale(idx, this._loadStaleBitmap())) {
+      return undefined;
+    }
     return this.int8Vectors.get(id);
   }
 
   /**
-   * Save index to disk
+   * Save index to disk.
+   *
+   * Publish semantics: every sidecar is written to a sibling
+   * `<path>.tmp.<pid>` then atomically renamed into its canonical
+   * name. Cross-process readers loading the index never observe a
+   * torn `(meta, vectors, graph, int8)` tuple — POSIX `rename` is
+   * atomic per-file, and the publish order below puts the
+   * descriptor (`.meta.json`) LAST so that any reader that observes
+   * the new meta.json is guaranteed to read the matching data
+   * sidecars alongside it.
+   *
+   * The brief window between data renames remains: a reader can
+   * still observe `(NEW vectors, OLD graph)` for a few microseconds.
+   * The size of the rebuilt index makes this window negligible on
+   * the existing single-writer process model; a deeper fix would
+   * publish all sidecars via a single packaged artifact under
+   * versioned manifest paths.
    */
   async save(indexPath = this.indexPath) {
     await fs.mkdir(path.dirname(indexPath), { recursive: true });
 
-    // Save metadata
     const meta = {
       dimension: this.dimension,
       floatDimension: this.floatDimension,
@@ -658,47 +755,82 @@ export class BinaryHNSWIndex {
       savedAt: new Date().toISOString(),
     };
 
-    const metaPath = indexPath.replace('.idx', '.meta.json');
-    await fs.writeFile(metaPath, JSON.stringify(meta, null, 2));
-
-    // Save vectors (binary + metadata)
     const vectorsData = this.vectors.map(v => ({
       id: v.id,
       binary: Array.from(v.binary),
       metadata: v.metadata,
     }));
 
+    const metaPath = indexPath.replace('.idx', '.meta.json');
     const vectorsPath = indexPath.replace('.idx', '.vectors.json');
-    await fs.writeFile(vectorsPath, JSON.stringify(vectorsData));
-
-    // Save graph structure
     const graphPath = indexPath.replace('.idx', '.graph.json');
-    await fs.writeFile(graphPath, JSON.stringify(this.graph));
+    const int8Path = indexPath.replace('.idx', '.int8.json');
+    const calibPath = indexPath.replace('.idx', '.calibration.json');
+    const pidSuffix = `.tmp.${process.pid}`;
 
-    // Save int8 vectors if any
+    // Stage all sidecars to sibling temp paths.
+    await fs.writeFile(metaPath + pidSuffix, JSON.stringify(meta, null, 2));
+    await fs.writeFile(vectorsPath + pidSuffix, JSON.stringify(vectorsData));
+    await fs.writeFile(graphPath + pidSuffix, JSON.stringify(this.graph));
+
+    let stagedInt8 = false;
     if (this.int8Vectors.size > 0) {
+      const liveIds = new Set(this.vectors.map(v => v.id));
       const int8Data = {};
       for (const [id, vec] of this.int8Vectors) {
+        if (!liveIds.has(id)) continue;
         int8Data[id] = Array.from(vec);
       }
-      const int8Path = indexPath.replace('.idx', '.int8.json');
-      await fs.writeFile(int8Path, JSON.stringify(int8Data));
+      if (Object.keys(int8Data).length > 0) {
+        await fs.writeFile(int8Path + pidSuffix, JSON.stringify(int8Data));
+        stagedInt8 = true;
+      }
     }
 
-    // Save asymmetric calibration data (centroid + rotation signs)
+    let stagedCalib = false;
     if (this.useAsymmetric && this.centroid && this.signVector) {
-      const calibPath = indexPath.replace('.idx', '.calibration.json');
-      await fs.writeFile(calibPath, JSON.stringify({
+      await fs.writeFile(calibPath + pidSuffix, JSON.stringify({
         centroid: Array.from(this.centroid),
         signVector: Array.from(this.signVector),
       }));
+      stagedCalib = true;
+    }
+
+    // Atomic publish in safety order: data sidecars first, descriptor
+    // (.meta.json) LAST. Optional sidecars without staged content are
+    // unlinked from the canonical path so the canonical state matches
+    // the staged tuple exactly.
+    await fs.rename(vectorsPath + pidSuffix, vectorsPath);
+    await fs.rename(graphPath + pidSuffix, graphPath);
+    if (stagedInt8) await fs.rename(int8Path + pidSuffix, int8Path);
+    else await fs.rm(int8Path, { force: true });
+    if (stagedCalib) await fs.rename(calibPath + pidSuffix, calibPath);
+    else await fs.rm(calibPath, { force: true });
+    await fs.rename(metaPath + pidSuffix, metaPath);
+
+    if (this._cleanBuild) {
+      await fs.rm(this._stalePathForIndex(indexPath), { force: true });
+      this._staleBitmapCache = null;
+      this._cleanBuild = false;
     }
 
     console.log(`BinaryHNSW: Saved ${this.vectors.length} vectors to ${indexPath} (asymmetric=${this.useAsymmetric})`);
   }
 
   /**
-   * Load index from disk
+   * Load index from disk.
+   *
+   * Torn-publish handling: `save()` publishes data sidecars and
+   * `.meta.json` via separate atomic renames. A reader that opens the
+   * index in the microsecond window between renames can observe a torn
+   * `(meta, vectors)` pair where `meta.entryPoint` references an index
+   * past `vectors.length`. The first `search()` call would then
+   * `TypeError` on `this.vectors[entryPoint].binary`. To keep fresh
+   * readers crash-free we re-read the pair up to three times on
+   * `(vectorCount, entryPoint)` inconsistency — the publish window
+   * closes in microseconds so a brief retry self-heals it. A persistent
+   * mismatch surfaces as an explicit load error rather than a deferred
+   * search crash.
    */
   async load(indexPath = this.indexPath) {
     const metaPath = indexPath.replace('.idx', '.meta.json');
@@ -710,11 +842,12 @@ export class BinaryHNSWIndex {
       throw new Error(`Index metadata not found: ${metaPath}`);
     }
 
-    // Load metadata
-    const meta = JSON.parse(await fs.readFile(metaPath, 'utf-8'));
-
-    // Validate pipeline version — mismatched indexes must be rebuilt
-    const storedVersion = meta.pipelineVersion || 1;
+    // Validate pipeline version first — a stale on-disk artifact from a
+    // previous quantization scheme is a callable-level error, not a
+    // torn-publish race, and the caller needs the specific message to
+    // route to the rebuild path.
+    const initialMeta = JSON.parse(await fs.readFile(metaPath, 'utf-8'));
+    const storedVersion = initialMeta.pipelineVersion || 1;
     if (storedVersion !== PIPELINE_VERSION) {
       throw new Error(
         `Pipeline version mismatch: index=${storedVersion}, current=${PIPELINE_VERSION}. ` +
@@ -722,6 +855,27 @@ export class BinaryHNSWIndex {
       );
     }
 
+    let meta;
+    let vectorsData;
+    let attempt = 0;
+    while (true) {
+      meta = attempt === 0 ? initialMeta : JSON.parse(await fs.readFile(metaPath, 'utf-8'));
+      vectorsData = JSON.parse(await fs.readFile(vectorsPath, 'utf-8'));
+      const consistent = meta.vectorCount === vectorsData.length
+        && (meta.entryPoint === -1 || meta.entryPoint < vectorsData.length);
+      if (consistent) break;
+      if (attempt >= 2) {
+        throw new Error(
+          `BinaryHNSW: persistent meta/vectors inconsistency at ${indexPath} ` +
+          `(meta.vectorCount=${meta.vectorCount}, vectors.length=${vectorsData.length}, ` +
+          `entryPoint=${meta.entryPoint})`
+        );
+      }
+      // Brief delay to let an in-flight save() finish its second rename.
+      await new Promise((resolve) => setTimeout(resolve, 5 * (attempt + 1)));
+      attempt += 1;
+    }
+
     this.dimension = meta.dimension;
     this.floatDimension = meta.floatDimension;
     this.M = meta.M;
@@ -730,9 +884,10 @@ export class BinaryHNSWIndex {
     this.maxLevel = meta.maxLevel;
     this.entryPoint = meta.entryPoint;
     this.useAsymmetric = meta.useAsymmetric || false;
+    this.centroid = null;
+    this.signVector = null;
 
-    // Load vectors
-    const vectorsData = JSON.parse(await fs.readFile(vectorsPath, 'utf-8'));
+    // Vectors already read above as part of the consistency probe.
     this.vectors = vectorsData.map(v => ({
       id: v.id,
       binary: new Uint8Array(v.binary),
@@ -749,9 +904,9 @@ export class BinaryHNSWIndex {
     this.graph = JSON.parse(await fs.readFile(graphPath, 'utf-8'));
 
     // Load int8 vectors if available
+    this.int8Vectors.clear();
     if (existsSync(int8Path)) {
       const int8Data = JSON.parse(await fs.readFile(int8Path, 'utf-8'));
-      this.int8Vectors.clear();
       for (const [id, vec] of Object.entries(int8Data)) {
         this.int8Vectors.set(id, new Int8Array(vec));
       }
@@ -808,6 +963,9 @@ export class BinaryHNSWIndex {
     this.graph = [];
     this.entryPoint = -1;
     this.maxLevel = 0;
+    await fs.rm(this.stalePath, { force: true });
+    this._staleBitmapCache = null;
+    this._cleanBuild = false;
   }
 }
 
@@ -818,7 +976,7 @@ export class BinaryHNSWIndex {
 export async function createBinaryHNSWIndex(options = {}) {
   const index = new BinaryHNSWIndex(options);
 
-  if (options.load !== false && existsSync(options.indexPath || DB_PATHS.binaryHnswIndex)) {
+  if (options.load !== false && binaryHnswArtifactsExist(options.indexPath || DB_PATHS.binaryHnswIndex)) {
     try {
       await index.load(options.indexPath);
     } catch (err) {
@@ -832,6 +990,12 @@ export async function createBinaryHNSWIndex(options = {}) {
   return index;
 }
 
+function binaryHnswArtifactsExist(indexPath) {
+  return existsSync(indexPath.replace('.idx', '.meta.json'))
+    && existsSync(indexPath.replace('.idx', '.vectors.json'))
+    && existsSync(indexPath.replace('.idx', '.graph.json'));
+}
+
 // =============================================================================
 // CLI
 // =============================================================================

package/core/vector-store/float-vector-store.js

@@ -25,7 +25,7 @@
  *   3. Batch dot product for Stage 2.5 candidates
  */
 
-import { readFile, writeFile } from 'fs/promises';
+import { readFile, writeFile, rename } from 'fs/promises';
 import { existsSync } from 'fs';
 import path from 'path';
 import { float32BatchDot } from '../infrastructure/simd-distance.js';
@@ -81,19 +81,29 @@ export class FloatVectorStore {
 
   /**
    * Save to disk: binary file + ID map JSON.
+   *
+   * Both files are written via temp-file + atomic rename so a crash mid-write
+   * never leaves a half-written artifact, and a concurrent reader either sees
+   * the whole previous version or the whole new one. The binary is renamed
+   * first; if a crash interleaves the two renames, the next load sees a new
+   * binary against the old ID map — the count guard in `load()` rejects that
+   * mismatch and Stage 2.5 cleanly falls back to SQLite until re-reconciled.
+   *
    * @param {string} binPath - Path for binary file (e.g., foo.float-vectors.bin)
    */
   async save(binPath) {
     if (!this.loaded) throw new Error('FloatVectorStore: nothing to save');
 
-    // Write binary file
-    await writeFile(binPath, Buffer.from(this.buffer));
-
-    // Write ID map
     const idsPath = binPath.replace(/\.bin$/, '.ids.json');
     const ids = new Array(this.count);
     for (const [id, idx] of this.idToIndex) ids[idx] = id;
-    await writeFile(idsPath, JSON.stringify(ids));
+
+    const binTmp = `${binPath}.tmp.${process.pid}`;
+    const idsTmp = `${idsPath}.tmp.${process.pid}`;
+    await writeFile(binTmp, Buffer.from(this.buffer));
+    await writeFile(idsTmp, JSON.stringify(ids));
+    await rename(binTmp, binPath);
+    await rename(idsTmp, idsPath);
   }
 
   /**
@@ -131,6 +141,17 @@ export class FloatVectorStore {
     // Read ID map
     const idsJson = await readFile(idsPath, 'utf-8');
     const ids = JSON.parse(idsJson);
+
+    // Reject a torn binary/ID-map pair (e.g. a crash between the two atomic
+    // renames in save()). A mismatched count means the index→ID mapping would
+    // be wrong; refuse to load so Stage 2.5 falls back to SQLite instead.
+    if (ids.length !== this.count) {
+      this.buffer = null;
+      this.data = null;
+      this.count = 0;
+      return false;
+    }
+
     this.idToIndex = new Map();
     for (let i = 0; i < ids.length; i++) {
       this.idToIndex.set(ids[i], i);
@@ -140,6 +161,75 @@ export class FloatVectorStore {
     return true;
   }
 
+  /**
+   * Load an existing store, or initialise an empty mutable one at the given
+   * dimension when no store exists on disk yet. Used by the incremental
+   * reconcile path so the first add after an empty baseline produces a real
+   * float store rather than relying on the SQLite fallback.
+   *
+   * When a store loads, its on-disk dimension wins (it is authoritative for
+   * the vectors already stored); `dimension` only seeds a fresh empty store.
+   *
+   * @param {string} binPath
+   * @param {number} dimension
+   * @returns {Promise<boolean>} true if an existing store was loaded, false if initialised empty
+   */
+  async loadOrInit(binPath, dimension) {
+    let loaded = false;
+    try {
+      loaded = await this.load(binPath);
+    } catch {
+      loaded = false; // corrupt/torn store → treat as empty and rebuild from delta
+    }
+    if (loaded) return true;
+
+    this.dimension = dimension;
+    this.count = 0;
+    this.idToIndex = new Map();
+    this.data = new Float32Array(0);
+    this.buffer = null;
+    this.loaded = true;
+    return false;
+  }
+
+  /**
+   * Apply incremental upserts/removes and rebuild the contiguous buffer.
+   *
+   * Writer-side only (the reconcile maintainer). The read hot path
+   * (get/batchScore over the contiguous buffer) is untouched; readers reload
+   * the rebuilt store from disk after the manifest publishes.
+   *
+   * Upserts replace any existing vector for the same ID. Removes drop the ID.
+   * The rebuilt buffer preserves no particular order — IDs are addressed via
+   * the idToIndex map, never by position.
+   *
+   * @param {{upserts?: Array<{id: string, vector: Float32Array|number[]}>, removeIds?: Iterable<string>}} delta
+   * @returns {{count: number}}
+   */
+  applyDelta({ upserts = [], removeIds = [] } = {}) {
+    // Materialise current id → vector. Subarrays view the old buffer; build()
+    // copies their values into a freshly allocated buffer before the old one
+    // is dropped, so the aliasing is safe.
+    const entries = new Map();
+    if (this.loaded && this.data && this.count > 0) {
+      for (const [id, idx] of this.idToIndex) {
+        const offset = idx * this.dimension;
+        entries.set(id, this.data.subarray(offset, offset + this.dimension));
+      }
+    }
+
+    for (const id of removeIds) entries.delete(id);
+    for (const { id, vector } of upserts) {
+      if (!id || !vector) continue;
+      entries.set(id, vector);
+    }
+
+    const list = [];
+    for (const [id, vector] of entries) list.push({ id, vector });
+    this.build(list, this.dimension);
+    return { count: this.count };
+  }
+
   /**
    * Get a float vector by string ID.
    * @param {string} id