sweet-search 2.5.13 → 2.6.0

package/core/vector-store/binary-hnsw-index.js

@@ -27,7 +27,10 @@
  */
 
 import fs from 'fs/promises';
-import { existsSync, statSync } from 'fs';
+import {
+  existsSync, statSync,
+  openSync, readSync, closeSync, fstatSync,
+} from 'fs';
 import path from 'path';
 import { BINARY_HNSW_CONFIG, DB_PATHS } from '../infrastructure/config/index.js';
 import {
@@ -44,6 +47,161 @@ import { loadBitmap, isSet } from '../infrastructure/tombstone-bitmap-reader.js'
 // different version are incompatible and must be rebuilt.
 const PIPELINE_VERSION = 2;
 
+// =============================================================================
+// G9 — PACKED MMAP FORMAT (gated, NO MIGRATION)
+// =============================================================================
+//
+// The default on-disk format is the JSON sidecar tuple
+// (.meta.json / .vectors.json / .graph.json / .int8.json / .calibration.json),
+// written by save() and read by load(). That format is preserved BYTE-FOR-BYTE
+// when SWEET_SEARCH_HNSW_MMAP !== '1' — this whole subsystem is inert by
+// default. Existing on-disk indexes (the 200 bench repos) keep loading via the
+// JSON path regardless of the flag: format is detected by inspecting the .idx
+// file's magic header (`_indexFormatOnDisk`), never by the flag.
+//
+// When SWEET_SEARCH_HNSW_MMAP === '1', NEWLY written indexes are packed into a
+// single flat-binary `.idx` file (magic-prefixed) whose vectors / graph
+// adjacency / int8 sections are laid out so the read/search path can fault in
+// only the pages it touches (lazy fd reads → OS page cache). The packed format
+// preserves the deterministic levels (G1) and deterministic entry point (FIX-B)
+// because those are properties of the in-memory graph/entryPoint that get
+// serialized verbatim — packing does not re-run construction.
+//
+// MMAP_MAGIC is 8 ASCII bytes so a packed file is trivially and cheaply
+// distinguishable from a JSON sidecar's `.idx` (which today is never written
+// with content — only the sidecars carry data) without parsing.
+const MMAP_MAGIC = Buffer.from('SSHNSWP\0', 'ascii'); // 8 bytes
+const MMAP_FORMAT_VERSION = 1;
+
+function hnswMmapEnabled() {
+  return process.env.SWEET_SEARCH_HNSW_MMAP === '1';
+}
+
+/**
+ * Lazy, fd-backed graph adjacency for the packed (mmap) read path.
+ *
+ * Behaves like `graph[level][nodeIndex] -> number[]` (the JSON-loaded shape)
+ * but reads each (level, node) neighbor list from the open file descriptor on
+ * first touch and caches it, so a cold search faults in only the pages it
+ * visits. `?.[idx]` / `.length` / iteration over a returned neighbor list all
+ * work identically to a plain array because each level proxies an array-like
+ * via numeric-index getters that materialize lazily.
+ *
+ * The on-disk graph section layout (all little-endian uint32):
+ *   [levelCount]
+ *   per level: [nodeCount] then nodeCount × [neighborCount, n0, n1, ...]
+ * Per-(level,node) byte offsets are precomputed once into a flat index at load
+ * so a touch is an O(1) seek + a single readSync of just that list.
+ */
+class MmapGraphLevel {
+  constructor(fd, sectionBase, nodeOffsets, nodeCount) {
+    this._fd = fd;
+    this._base = sectionBase;
+    this._nodeOffsets = nodeOffsets; // Int32Array of byte offsets (relative) per node, -1 if empty
+    this._cache = new Array(nodeCount);
+    this.length = nodeCount;
+  }
+
+  _materialize(i) {
+    if (i < 0 || i >= this.length) return undefined;
+    const cached = this._cache[i];
+    if (cached !== undefined) return cached;
+    const rel = this._nodeOffsets[i];
+    if (rel < 0) {
+      this._cache[i] = [];
+      return this._cache[i];
+    }
+    const head = Buffer.allocUnsafe(4);
+    readSync(this._fd, head, 0, 4, this._base + rel);
+    const count = head.readUInt32LE(0);
+    const list = new Array(count);
+    if (count > 0) {
+      const body = Buffer.allocUnsafe(count * 4);
+      readSync(this._fd, body, 0, count * 4, this._base + rel + 4);
+      for (let j = 0; j < count; j += 1) list[j] = body.readUInt32LE(j * 4);
+    }
+    this._cache[i] = list;
+    return list;
+  }
+}
+
+// Numeric-index access (graph[level][idx]) is the hot path; back each level
+// with a Proxy that maps integer keys to lazy materialization and forwards
+// `length`. Optional-chaining (`graph[level]?.[idx]`) and `.length` both work.
+function makeMmapGraphLevelProxy(level) {
+  return new Proxy(level, {
+    get(target, prop) {
+      if (prop === 'length') return target.length;
+      if (typeof prop === 'string') {
+        const n = Number(prop);
+        if (Number.isInteger(n) && n >= 0) return target._materialize(n);
+      }
+      return target[prop];
+    },
+    has(target, prop) {
+      if (prop === 'length') return true;
+      const n = Number(prop);
+      if (Number.isInteger(n) && n >= 0 && n < target.length) return true;
+      return prop in target;
+    },
+  });
+}
+
+/**
+ * Lazy, fd-backed vectors store for the packed read path.
+ *
+ * Behaves like `vectors[idx] -> { id, binary, metadata }`. `id` and `metadata`
+ * are kept fully resident (they live in the small header JSON, parsed once),
+ * but the `binary` Uint8Array is read from the flat vector blob on first touch
+ * so cold ticks hold near-zero binary bytes in heap. Each materialized record
+ * is cached so repeated touches in a search don't re-read.
+ */
+class MmapVectorStore {
+  constructor(fd, blobBase, dimension, ids, metas) {
+    this._fd = fd;
+    this._base = blobBase;
+    this._dim = dimension;
+    this._ids = ids;       // string[]
+    this._metas = metas;   // object[]
+    this._cache = new Array(ids.length);
+    this.length = ids.length;
+  }
+
+  _materialize(i) {
+    if (i < 0 || i >= this.length) return undefined;
+    const cached = this._cache[i];
+    if (cached !== undefined) return cached;
+    const buf = Buffer.allocUnsafe(this._dim);
+    readSync(this._fd, buf, 0, this._dim, this._base + i * this._dim);
+    // Copy into a standalone Uint8Array so callers (hammingDistance) see a
+    // tight, stable view independent of the scratch Buffer.
+    const binary = new Uint8Array(this._dim);
+    binary.set(buf);
+    const rec = { id: this._ids[i], binary, metadata: this._metas[i] };
+    this._cache[i] = rec;
+    return rec;
+  }
+}
+
+function makeMmapVectorProxy(store) {
+  return new Proxy(store, {
+    get(target, prop) {
+      if (prop === 'length') return target.length;
+      if (typeof prop === 'string') {
+        const n = Number(prop);
+        if (Number.isInteger(n) && n >= 0) return target._materialize(n);
+      }
+      return target[prop];
+    },
+    has(target, prop) {
+      if (prop === 'length') return true;
+      const n = Number(prop);
+      if (Number.isInteger(n) && n >= 0 && n < target.length) return true;
+      return prop in target;
+    },
+  });
+}
+
 // =============================================================================
 // BINARY HNSW INDEX CLASS
 // =============================================================================
@@ -85,10 +243,17 @@ export class BinaryHNSWIndex {
     this.useAsymmetric = false; // Enabled after calibration
     this._staleBitmapCache = null;
     this._cleanBuild = false;
+
+    // G9 mmap: set when this instance is backed by an open, packed .idx fd
+    // (loadMmap). `vectors`/`graph` are then lazy proxies and the index is
+    // read/search-only. Null/false on the default JSON path.
+    this._mmapBacked = false;
+    this._mmapFd = null;
   }
 
   /** Reset to empty state for a fresh build (skips loading from disk). */
   resetForBuild() {
+    this._closeMmap();
     this.vectors = [];
     this.idToIndex = new Map();
     this.int8Vectors.clear();
@@ -103,6 +268,15 @@ export class BinaryHNSWIndex {
     this._cleanBuild = true;
   }
 
+  /** Close an open packed-file descriptor if this index is mmap-backed. */
+  _closeMmap() {
+    if (this._mmapFd != null) {
+      try { closeSync(this._mmapFd); } catch { /* best-effort */ }
+    }
+    this._mmapFd = null;
+    this._mmapBacked = false;
+  }
+
   _stalePathForIndex(indexPath = this.indexPath) {
     return indexPath === this.indexPath ? this.stalePath : `${indexPath}.stale.bin`;
   }
@@ -113,9 +287,22 @@ export class BinaryHNSWIndex {
   async init() {
     if (this.initialized) return;
 
-    // Try to load existing index
+    // Try to load an existing index. Detect the on-disk format by inspecting
+    // the artifacts (NOT the flag): a JSON sidecar tuple is identified by its
+    // .meta.json descriptor; a packed (mmap) index is identified by the magic
+    // header on the .idx file itself. Old indexes always take the JSON path
+    // regardless of SWEET_SEARCH_HNSW_MMAP (no migration, ever).
     const metaPath = this.indexPath.replace('.idx', '.meta.json');
-    if (existsSync(metaPath)) {
+    const onDisk = _indexFormatOnDisk(this.indexPath);
+    if (onDisk === 'packed') {
+      try {
+        await this.load(); // load() routes to loadMmap() for packed files
+        this.initialized = true;
+        return;
+      } catch (err) {
+        console.log(`BinaryHNSW: Failed to load packed index, creating new: ${err.message}`);
+      }
+    } else if (existsSync(metaPath)) {
       try {
         await this.load();
         this.initialized = true;
@@ -136,7 +323,15 @@ export class BinaryHNSWIndex {
   }
 
   /**
-   * Calculate random level for new node (exponential distribution)
+   * Calculate random level for new node (exponential distribution).
+   *
+   * Draws from the global, unseeded Math.random() at insert time, so the
+   * level a node receives depends on insertion order / RNG-stream
+   * interleaving (per-file reload-and-insert vs. batched load-once-insert-all
+   * vs. compaction rebuild all consume the stream differently). This is the
+   * default path and is intentionally left unchanged. For an
+   * insertion-order-independent level, see `levelForId(id)` (gated on
+   * SWEET_SEARCH_HNSW_DETERMINISTIC_LEVELS).
    */
   getRandomLevel() {
     const mL = 1 / Math.log(this.M);
@@ -144,6 +339,40 @@ export class BinaryHNSWIndex {
     return Math.min(level, 10); // Cap at 10 levels
   }
 
+  /**
+   * Deterministic level for a node id (insertion-order-independent).
+   *
+   * Pure function of the string id (FNV-1a 32-bit hash → uniform u∈(0,1])
+   * fed through the SAME exponential CDF as `getRandomLevel()`
+   * (`floor(-ln(u) * (1/ln(M)))`, capped at 10). Because the level depends
+   * only on the id (not on Math.random() or insertion order), batched,
+   * per-file, and compaction construction paths all assign the same node the
+   * same level — the prerequisite for a byte-identical graph.
+   *
+   * Used only when SWEET_SEARCH_HNSW_DETERMINISTIC_LEVELS === '1'; the default
+   * path (`getRandomLevel`) is unchanged.
+   *
+   * @param {string} id - node id (coerced to string)
+   * @returns {number} level in [0, 10]
+   */
+  levelForId(id) {
+    // FNV-1a 32-bit over the id's UTF-16 code units. Stable across calls and
+    // across a fresh process (no global state, no RNG).
+    const s = String(id);
+    let h = 0x811c9dc5; // FNV offset basis
+    for (let i = 0; i < s.length; i += 1) {
+      h ^= s.charCodeAt(i);
+      // FNV prime multiply via shifts, kept in unsigned 32-bit space.
+      h = (h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24))) >>> 0;
+    }
+    // Map hash → u ∈ (0,1]. Use (h+1)/2^32 so u is never 0 (avoids -ln(0)=∞)
+    // and never exceeds 1; h ranges over [0, 2^32-1] so (h+1) ∈ [1, 2^32].
+    const u = (h + 1) / 4294967296; // 2^32
+    const mL = 1 / Math.log(this.M);
+    const level = Math.floor(-Math.log(u) * mL);
+    return Math.min(level, 10); // Cap at 10 levels (same as getRandomLevel)
+  }
+
   /**
    * Add a vector to the index
    *
@@ -155,6 +384,19 @@ export class BinaryHNSWIndex {
   async add(id, binaryVector, metadata = {}, int8Vector = null) {
     await this.init();
 
+    // The packed (mmap) layout is a read/search-only representation: `vectors`
+    // and `graph` are lazy fd-backed proxies, not mutable plain arrays. Build
+    // paths must construct on plain arrays and persist via save()/saveMmap().
+    // Mutating a packed-loaded index would silently corrupt the graph, so fail
+    // loudly instead. (This is unreachable on the default path; the reconciler
+    // builds in memory and only the read side mmap-loads.)
+    if (this._mmapBacked) {
+      throw new Error(
+        'BinaryHNSW: cannot add() to an mmap-backed (packed) index — '
+        + 'it is a read/search-only view. Rebuild on a fresh in-memory index.'
+      );
+    }
+
     // Convert float to binary if needed
     let binary;
     if (binaryVector instanceof Uint8Array) {
@@ -185,8 +427,17 @@ export class BinaryHNSWIndex {
       this.int8Vectors.set(id, int8Vector);
     }
 
-    // Add to HNSW graph
-    const level = this.getRandomLevel();
+    // Add to HNSW graph.
+    // Gate: deterministic per-id levels are DEFAULT-ON (disable with
+    // SWEET_SEARCH_HNSW_DETERMINISTIC_LEVELS=0). When on, assign the level from a
+    // stable hash of the id (insertion-order-independent) so all construction
+    // paths (incremental add, end-of-tick batch, compaction rebuild) agree and
+    // the graph is reproducible. Verified recall-neutral on GCSN (+0.01pp MRR)
+    // and byte-identical via the determinism harness. Set the flag to '0' to
+    // restore the legacy random-level (insertion-order-dependent) path.
+    const level = process.env.SWEET_SEARCH_HNSW_DETERMINISTIC_LEVELS !== '0'
+      ? this.levelForId(id)
+      : this.getRandomLevel();
     this.addToGraph(idx, level);
 
     return idx;
@@ -248,6 +499,37 @@ export class BinaryHNSWIndex {
     return selected;
   }
 
+  /**
+   * Whether insertion-order-independent entry-point selection is active.
+   *
+   * Mirrors the gate on level assignment in `add()`. When ON, the entry
+   * point is chosen deterministically (min id at the max level) so batched,
+   * per-file, and compaction construction paths agree on `entryPoint` even
+   * when several nodes share the max level — the last byte-identity gap left
+   * by deterministic levels (E.1 entry-point divergence). DEFAULT-ON (disable
+   * with SWEET_SEARCH_HNSW_DETERMINISTIC_LEVELS=0) → mirrors the level-assignment
+   * gate in `add()`. Set the flag to '0' to restore today's
+   * first-inserted-at-max-level behavior.
+   */
+  _deterministicEntryPoint() {
+    return process.env.SWEET_SEARCH_HNSW_DETERMINISTIC_LEVELS !== '0';
+  }
+
+  /**
+   * Tie-break two graph node indices by their string id (deterministic,
+   * insertion-order-independent). Returns the index whose id sorts first
+   * lexicographically. Lexicographic comparison of the stable string id is
+   * used (not the FNV-1a hash) so the choice is human-auditable and never
+   * subject to hash collisions.
+   */
+  _entryPointWinner(idxA, idxB) {
+    if (idxA === -1) return idxB;
+    if (idxB === -1) return idxA;
+    const idA = String(this.vectors[idxA].id);
+    const idB = String(this.vectors[idxB].id);
+    return idA <= idB ? idxA : idxB;
+  }
+
   /**
    * Add node to HNSW graph.
    * Uses heuristic selection + level-aware M0=2*M.
@@ -310,10 +592,23 @@ export class BinaryHNSWIndex {
       }
     }
 
-    // Update entry point if new level is higher
+    // Update entry point.
+    //
+    // Default path (flag OFF): unchanged — the entry point only moves when a
+    // strictly higher level appears, so the first node inserted at the running
+    // max level keeps it (insertion-order-dependent, today's behavior).
+    //
+    // Deterministic path (flag ON): a strictly higher level still wins, but on
+    // a TIE at the current max level the entry point is refined to the node
+    // with the smallest id. Because `maxLevel` only ever rises during a build
+    // (no deletions here) and equals the max id-assigned level, this converges
+    // to `min(id)` over all nodes at the final max level regardless of
+    // insertion order — closing the E.1 entry-point byte-identity gap.
     if (level > this.maxLevel) {
       this.entryPoint = idx;
       this.maxLevel = level;
+    } else if (level === this.maxLevel && this._deterministicEntryPoint()) {
+      this.entryPoint = this._entryPointWinner(this.entryPoint, idx);
     }
   }
 
@@ -738,6 +1033,17 @@ export class BinaryHNSWIndex {
    * versioned manifest paths.
    */
   async save(indexPath = this.indexPath) {
+    // G9 gate. Default OFF → the JSON-sidecar writer below runs UNCHANGED, so
+    // every existing index keeps its exact byte format and there is no
+    // migration. ONLY when SWEET_SEARCH_HNSW_MMAP === '1' do NEW writes pack
+    // into the mmap-friendly flat-binary .idx (a distinct on-disk format with
+    // its own magic/version). The deterministic levels (G1) and entry point
+    // (FIX-B) live in `this.graph`/`this.entryPoint`, which both writers
+    // serialize verbatim — packing preserves them.
+    if (hnswMmapEnabled()) {
+      return this.saveMmap(indexPath);
+    }
+
     await fs.mkdir(path.dirname(indexPath), { recursive: true });
 
     const meta = {
@@ -833,6 +1139,14 @@ export class BinaryHNSWIndex {
    * search crash.
    */
   async load(indexPath = this.indexPath) {
+    // Route by DETECTED on-disk format, never by the flag: a packed .idx
+    // (magic header) is read via the mmap path even with the flag off, and a
+    // JSON-sidecar index is read via the JSON path even with the flag on. This
+    // is what lets old and new coexist with zero migration.
+    if (_indexFormatOnDisk(indexPath) === 'packed') {
+      return this.loadMmap(indexPath);
+    }
+
     const metaPath = indexPath.replace('.idx', '.meta.json');
     const vectorsPath = indexPath.replace('.idx', '.vectors.json');
     const graphPath = indexPath.replace('.idx', '.graph.json');
@@ -927,13 +1241,338 @@ export class BinaryHNSWIndex {
     console.log(`BinaryHNSW: Loaded ${this.vectors.length} vectors from ${indexPath} (asymmetric=${this.useAsymmetric})`);
   }
 
+  /**
+   * Save the index in the packed, mmap-friendly flat-binary format (G9).
+   *
+   * Activated only via save() under SWEET_SEARCH_HNSW_MMAP === '1'. The whole
+   * index is serialized into a SINGLE `.idx` file (so the read path can fault
+   * in only touched pages) and the legacy JSON sidecars for this path are
+   * removed so a stale JSON descriptor can never shadow the packed file. The
+   * graph + entryPoint are written verbatim, preserving G1/FIX-B determinism.
+   *
+   * On-disk layout (all integers little-endian):
+   *   magic(8) | formatVersion(u32) | headerLen(u32) | header(JSON utf-8) | sections...
+   * The header JSON carries the meta fields, the id/metadata arrays, the
+   * asymmetric calibration (inlined — small), and { offset, len } descriptors
+   * for the three binary sections (vectors, graph, int8), each section offset
+   * being relative to the start of the section region (right after the header).
+   *
+   * Publish is atomic: write to `<path>.tmp.<pid>` then rename over `<path>`.
+   */
+  async saveMmap(indexPath = this.indexPath) {
+    await fs.mkdir(path.dirname(indexPath), { recursive: true });
+
+    const dim = this.dimension;
+    const n = this.vectors.length;
+
+    // --- Section 1: vectors blob (flat n*dim bytes) ---
+    const vectorsBlob = Buffer.allocUnsafe(n * dim);
+    const ids = new Array(n);
+    const metas = new Array(n);
+    for (let i = 0; i < n; i += 1) {
+      const v = this.vectors[i];
+      ids[i] = v.id;
+      metas[i] = v.metadata;
+      vectorsBlob.set(v.binary, i * dim);
+    }
+
+    // --- Section 2: graph blob ---
+    // [levelCount] then per level [nodeCount] then per node [count, n0, n1...]
+    const graphParts = [];
+    const lvlHead = Buffer.allocUnsafe(4);
+    lvlHead.writeUInt32LE(this.graph.length, 0);
+    graphParts.push(lvlHead);
+    for (let l = 0; l < this.graph.length; l += 1) {
+      const level = this.graph[l] || [];
+      const nodeHead = Buffer.allocUnsafe(4);
+      nodeHead.writeUInt32LE(level.length, 0);
+      graphParts.push(nodeHead);
+      for (let i = 0; i < level.length; i += 1) {
+        const neighbors = level[i] || [];
+        const buf = Buffer.allocUnsafe(4 + neighbors.length * 4);
+        buf.writeUInt32LE(neighbors.length, 0);
+        for (let j = 0; j < neighbors.length; j += 1) {
+          buf.writeUInt32LE(neighbors[j], 4 + j * 4);
+        }
+        graphParts.push(buf);
+      }
+    }
+    const graphBlob = Buffer.concat(graphParts);
+
+    // --- Section 3: int8 blob (only live ids, matching JSON save() semantics) ---
+    // [entryCount] then per entry [idIndex(u32), len(u32), bytes...]
+    const liveIds = new Set(ids);
+    const int8Parts = [];
+    const idToIdx = new Map();
+    for (let i = 0; i < n; i += 1) idToIdx.set(ids[i], i);
+    let int8Count = 0;
+    const int8Body = [];
+    for (const [id, vec] of this.int8Vectors) {
+      if (!liveIds.has(id)) continue;
+      const idIndex = idToIdx.get(id);
+      if (idIndex === undefined) continue;
+      const entry = Buffer.allocUnsafe(8 + vec.length);
+      entry.writeUInt32LE(idIndex, 0);
+      entry.writeUInt32LE(vec.length, 4);
+      // Int8Array → bytes (two's complement, identical layout)
+      Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength).copy(entry, 8);
+      int8Body.push(entry);
+      int8Count += 1;
+    }
+    const int8Head = Buffer.allocUnsafe(4);
+    int8Head.writeUInt32LE(int8Count, 0);
+    int8Parts.push(int8Head, ...int8Body);
+    const int8Blob = Buffer.concat(int8Parts);
+
+    // --- Header ---
+    let vOff = 0;
+    const gOff = vOff + vectorsBlob.length;
+    const iOff = gOff + graphBlob.length;
+    const header = {
+      magic: 'SSHNSWP',
+      formatVersion: MMAP_FORMAT_VERSION,
+      dimension: this.dimension,
+      floatDimension: this.floatDimension,
+      M: this.M,
+      efConstruction: this.efConstruction,
+      efSearch: this.efSearch,
+      maxElements: this.maxElements,
+      vectorCount: n,
+      maxLevel: this.maxLevel,
+      entryPoint: this.entryPoint,
+      useAsymmetric: this.useAsymmetric,
+      pipelineVersion: PIPELINE_VERSION,
+      savedAt: new Date().toISOString(),
+      ids,
+      metas,
+      calibration: (this.useAsymmetric && this.centroid && this.signVector)
+        ? { centroid: Array.from(this.centroid), signVector: Array.from(this.signVector) }
+        : null,
+      sections: {
+        vectors: { offset: vOff, len: vectorsBlob.length },
+        graph: { offset: gOff, len: graphBlob.length },
+        int8: { offset: iOff, len: int8Blob.length },
+      },
+    };
+    const headerBuf = Buffer.from(JSON.stringify(header), 'utf-8');
+    const prefix = Buffer.allocUnsafe(MMAP_MAGIC.length + 8);
+    MMAP_MAGIC.copy(prefix, 0);
+    prefix.writeUInt32LE(MMAP_FORMAT_VERSION, MMAP_MAGIC.length);
+    prefix.writeUInt32LE(headerBuf.length, MMAP_MAGIC.length + 4);
+
+    const full = Buffer.concat([prefix, headerBuf, vectorsBlob, graphBlob, int8Blob]);
+
+    const tmpPath = `${indexPath}.tmp.${process.pid}`;
+    await fs.writeFile(tmpPath, full);
+    await fs.rename(tmpPath, indexPath);
+
+    // Retire any legacy JSON sidecars at this path so a stale descriptor can
+    // never shadow the packed file on the next load. This is NOT a migration
+    // of other indexes — it only cleans up the path we just wrote.
+    await Promise.all([
+      fs.rm(indexPath.replace('.idx', '.meta.json'), { force: true }),
+      fs.rm(indexPath.replace('.idx', '.vectors.json'), { force: true }),
+      fs.rm(indexPath.replace('.idx', '.graph.json'), { force: true }),
+      fs.rm(indexPath.replace('.idx', '.int8.json'), { force: true }),
+      fs.rm(indexPath.replace('.idx', '.calibration.json'), { force: true }),
+    ]);
+
+    if (this._cleanBuild) {
+      await fs.rm(this._stalePathForIndex(indexPath), { force: true });
+      this._staleBitmapCache = null;
+      this._cleanBuild = false;
+    }
+
+    console.log(`BinaryHNSW: Saved ${n} vectors to ${indexPath} (packed/mmap, asymmetric=${this.useAsymmetric})`);
+  }
+
+  /**
+   * Load a packed (mmap) index for read/search (G9).
+   *
+   * Keeps the `.idx` file descriptor OPEN and backs `vectors`/`graph` with
+   * lazy fd-readers so a cold tick faults in only the pages a search visits
+   * (touched-pages-only, near-zero heap until traversed). The header (ids,
+   * metadata, calibration, section offsets) is small and parsed eagerly. The
+   * resulting in-memory graph/entryPoint are byte-equivalent to what a
+   * JSON-sidecar build of the same ids would hold, so search results are
+   * identical.
+   */
+  async loadMmap(indexPath = this.indexPath) {
+    this._closeMmap();
+    const fd = openSync(indexPath, 'r');
+    try {
+      const stat = fstatSync(fd);
+      const prefix = Buffer.allocUnsafe(MMAP_MAGIC.length + 8);
+      readSync(fd, prefix, 0, prefix.length, 0);
+      if (!prefix.subarray(0, MMAP_MAGIC.length).equals(MMAP_MAGIC)) {
+        throw new Error(`BinaryHNSW: not a packed index (bad magic): ${indexPath}`);
+      }
+      const fmt = prefix.readUInt32LE(MMAP_MAGIC.length);
+      if (fmt !== MMAP_FORMAT_VERSION) {
+        throw new Error(
+          `BinaryHNSW: packed format version mismatch: file=${fmt}, current=${MMAP_FORMAT_VERSION}.`
+        );
+      }
+      const headerLen = prefix.readUInt32LE(MMAP_MAGIC.length + 4);
+      const headerBuf = Buffer.allocUnsafe(headerLen);
+      readSync(fd, headerBuf, 0, headerLen, prefix.length);
+      const header = JSON.parse(headerBuf.toString('utf-8'));
+
+      const storedVersion = header.pipelineVersion || 1;
+      if (storedVersion !== PIPELINE_VERSION) {
+        throw new Error(
+          `Pipeline version mismatch: index=${storedVersion}, current=${PIPELINE_VERSION}. `
+          + 'Index must be rebuilt (quantization pipeline changed).'
+        );
+      }
+
+      this.dimension = header.dimension;
+      this.floatDimension = header.floatDimension;
+      this.M = header.M;
+      this.efConstruction = header.efConstruction;
+      this.efSearch = header.efSearch;
+      this.maxElements = header.maxElements;
+      this.maxLevel = header.maxLevel;
+      this.entryPoint = header.entryPoint;
+      this.useAsymmetric = header.useAsymmetric || false;
+      this.centroid = null;
+      this.signVector = null;
+      if (this.useAsymmetric && header.calibration) {
+        this.centroid = new Float32Array(header.calibration.centroid);
+        this.signVector = new Float32Array(header.calibration.signVector);
+      }
+
+      const sectionsBase = prefix.length + headerLen;
+      const sec = header.sections;
+      const n = header.vectorCount;
+      const ids = header.ids;
+      const metas = header.metas;
+
+      // Consistency probe mirroring the JSON load() invariant.
+      if (ids.length !== n || (this.entryPoint !== -1 && this.entryPoint >= n)) {
+        throw new Error(
+          `BinaryHNSW: packed index inconsistency at ${indexPath} `
+          + `(vectorCount=${n}, ids=${ids.length}, entryPoint=${this.entryPoint})`
+        );
+      }
+
+      // id → index map (resident; needed by getInt8Vector and searches).
+      this.idToIndex.clear();
+      for (let i = 0; i < n; i += 1) this.idToIndex.set(ids[i], i);
+
+      // Lazy vectors store (binary bytes faulted in per touch).
+      const vectorStore = new MmapVectorStore(
+        fd, sectionsBase + sec.vectors.offset, this.dimension, ids, metas
+      );
+      this.vectors = makeMmapVectorProxy(vectorStore);
+
+      // Build the per-(level,node) offset table for the graph section so each
+      // touch is an O(1) seek. We read the section's structural integers once
+      // (small relative to the neighbor bodies); neighbor LISTS stay on disk
+      // until touched.
+      const graphBase = sectionsBase + sec.graph.offset;
+      const graphLevels = this._buildMmapGraphOffsets(fd, graphBase, sec.graph.len);
+      this.graph = graphLevels;
+
+      // int8 vectors: small Int8Arrays, read eagerly into the resident Map
+      // (getInt8Vector is an O(1) Map lookup on the hot rescore path; keeping
+      // them resident matches the JSON path and avoids per-lookup seeks).
+      this.int8Vectors.clear();
+      if (sec.int8 && sec.int8.len >= 4) {
+        const int8Base = sectionsBase + sec.int8.offset;
+        const head = Buffer.allocUnsafe(4);
+        readSync(fd, head, 0, 4, int8Base);
+        const entryCount = head.readUInt32LE(0);
+        let cursor = int8Base + 4;
+        for (let e = 0; e < entryCount; e += 1) {
+          const eh = Buffer.allocUnsafe(8);
+          readSync(fd, eh, 0, 8, cursor);
+          const idIndex = eh.readUInt32LE(0);
+          const len = eh.readUInt32LE(4);
+          const body = Buffer.allocUnsafe(len);
+          readSync(fd, body, 0, len, cursor + 8);
+          const vec = new Int8Array(len);
+          for (let b = 0; b < len; b += 1) vec[b] = body.readInt8(b);
+          this.int8Vectors.set(ids[idIndex], vec);
+          cursor += 8 + len;
+        }
+      }
+
+      this._mmapFd = fd;
+      this._mmapBacked = true;
+      this._visitedList.ensureCapacity(n);
+      this.initialized = true;
+
+      // Touch the file size to keep the consistency contract auditable.
+      void stat.size;
+      console.log(`BinaryHNSW: Loaded ${n} vectors from ${indexPath} (packed/mmap, asymmetric=${this.useAsymmetric})`);
+    } catch (err) {
+      try { closeSync(fd); } catch { /* best-effort */ }
+      this._mmapFd = null;
+      this._mmapBacked = false;
+      throw err;
+    }
+  }
+
+  /**
+   * Scan the graph section's structural integers once and build a flat
+   * per-(level,node) byte-offset table, returning an array of lazy level
+   * proxies. Only the [levelCount]/[nodeCount]/[neighborCount] integers are
+   * read here; the neighbor id lists themselves remain on disk until a level
+   * proxy materializes a node on touch.
+   */
+  _buildMmapGraphOffsets(fd, base, sectionLen) {
+    const lvlHead = Buffer.allocUnsafe(4);
+    readSync(fd, lvlHead, 0, 4, base);
+    const levelCount = lvlHead.readUInt32LE(0);
+    let cursor = 4; // relative to base
+    const levels = new Array(levelCount);
+    const four = Buffer.allocUnsafe(4);
+    for (let l = 0; l < levelCount; l += 1) {
+      readSync(fd, four, 0, 4, base + cursor);
+      const nodeCount = four.readUInt32LE(0);
+      cursor += 4;
+      const nodeOffsets = new Int32Array(nodeCount);
+      for (let i = 0; i < nodeCount; i += 1) {
+        // Record this node's neighbor-list offset (points at its [count] word).
+        nodeOffsets[i] = cursor;
+        readSync(fd, four, 0, 4, base + cursor);
+        const neighborCount = four.readUInt32LE(0);
+        cursor += 4 + neighborCount * 4;
+      }
+      const level = new MmapGraphLevel(fd, base, nodeOffsets, nodeCount);
+      levels[l] = makeMmapGraphLevelProxy(level);
+    }
+    if (cursor !== sectionLen) {
+      // Structural mismatch → corrupt/truncated section.
+      throw new Error(
+        `BinaryHNSW: packed graph section length mismatch (read ${cursor}, expected ${sectionLen})`
+      );
+    }
+    return levels;
+  }
+
   /**
    * Get index statistics
    */
   getStats() {
-    const graphNodes = this.graph.reduce((sum, level) => sum + level.filter(n => n).length, 0);
-    const graphEdges = this.graph.reduce((sum, level) =>
-      sum + level.reduce((s, neighbors) => s + (neighbors?.length || 0), 0), 0);
+    // On the packed mmap path each `this.graph[l]` is a lazy level proxy, not a
+    // real Array, so Array.prototype.filter/reduce are unavailable and walking
+    // it would also fault in every neighbor list (defeating mmap). Iterate by
+    // index instead, which works for both representations.
+    let graphNodes = 0;
+    let graphEdges = 0;
+    for (let l = 0; l < this.graph.length; l += 1) {
+      const level = this.graph[l];
+      if (!level) continue;
+      for (let i = 0; i < level.length; i += 1) {
+        const neighbors = level[i];
+        if (neighbors) {
+          graphNodes += 1;
+          graphEdges += neighbors.length || 0;
+        }
+      }
+    }
 
     return {
       dimension: this.dimension,
@@ -957,6 +1596,7 @@ export class BinaryHNSWIndex {
    * Clear all data
    */
   async clear() {
+    this._closeMmap();
     this.vectors = [];
     this.idToIndex.clear();
     this.int8Vectors.clear();
@@ -967,6 +1607,14 @@ export class BinaryHNSWIndex {
     this._staleBitmapCache = null;
     this._cleanBuild = false;
   }
+
+  /**
+   * Release the packed (mmap) file descriptor, if any. The OS page cache may
+   * retain touched pages; this only drops the fd. Safe no-op on the JSON path.
+   */
+  close() {
+    this._closeMmap();
+  }
 }
 
 // =============================================================================
@@ -991,11 +1639,44 @@ export async function createBinaryHNSWIndex(options = {}) {
 }
 
 function binaryHnswArtifactsExist(indexPath) {
+  // A packed (mmap) index is a single .idx file; the JSON format is the sidecar
+  // tuple. Either is loadable, so report presence of EITHER. Detection is by
+  // on-disk content (magic header) — never the flag — so old + new coexist.
+  if (_indexFormatOnDisk(indexPath) === 'packed') return true;
   return existsSync(indexPath.replace('.idx', '.meta.json'))
     && existsSync(indexPath.replace('.idx', '.vectors.json'))
     && existsSync(indexPath.replace('.idx', '.graph.json'));
 }
 
+/**
+ * Detect the on-disk format of an HNSW index WITHOUT loading it, by content:
+ *   - 'packed'  : the `.idx` file exists and begins with the MMAP magic header.
+ *   - 'json'    : the JSON-sidecar descriptor (`.meta.json`) exists.
+ *   - 'none'    : neither.
+ * Reads only the leading magic bytes of the `.idx` (cheap), so the JSON path is
+ * chosen for every legacy index regardless of SWEET_SEARCH_HNSW_MMAP — the
+ * guarantee that existing on-disk indexes load unchanged with no migration.
+ */
+function _indexFormatOnDisk(indexPath) {
+  try {
+    if (existsSync(indexPath)) {
+      const st = statSync(indexPath);
+      if (st.isFile() && st.size >= MMAP_MAGIC.length) {
+        const fd = openSync(indexPath, 'r');
+        try {
+          const head = Buffer.allocUnsafe(MMAP_MAGIC.length);
+          readSync(fd, head, 0, MMAP_MAGIC.length, 0);
+          if (head.equals(MMAP_MAGIC)) return 'packed';
+        } finally {
+          closeSync(fd);
+        }
+      }
+    }
+  } catch { /* fall through to JSON detection */ }
+  if (existsSync(indexPath.replace('.idx', '.meta.json'))) return 'json';
+  return 'none';
+}
+
 // =============================================================================
 // CLI
 // =============================================================================
@@ -1085,75 +1766,8 @@ Options:
         console.log(`  Equivalent float: ${(numVectors * floatDim * 4 / 1024 / 1024).toFixed(2)} MB`);
         console.log(`  Compression: ${(floatDim * 4 / binaryDim).toFixed(0)}x`);
 
-      } else if (command === 'compare') {
-        console.log('\n=== Binary vs Float HNSW Comparison ===\n');
-
-        const { HNSWIndex } = await import('./hnsw-index.js');
-
-        const numVectors = 5000;
-        const floatDim = 512;
-
-        console.log(`Testing with ${numVectors} vectors, ${floatDim} dimensions\n`);
-
-        // Generate vectors
-        const vectors = [];
-        for (let i = 0; i < numVectors; i++) {
-          const float = new Array(floatDim).fill(0).map(() => Math.random() * 2 - 1);
-          vectors.push({ id: `vec-${i}`, float });
-        }
-
-        // Binary index
-        const binaryIndex = new BinaryHNSWIndex({ dimension: Math.ceil(floatDim / 8), floatDimension: floatDim });
-        await binaryIndex.init();
-
-        console.log('Building binary index...');
-        let start = performance.now();
-        for (const v of vectors) {
-          await binaryIndex.add(v.id, floatToBinary(v.float));
-        }
-        const binaryBuildTime = performance.now() - start;
-
-        // Float index
-        const floatIndex = new HNSWIndex({ dimension: floatDim });
-        await floatIndex.init();
-
-        console.log('Building float index...');
-        start = performance.now();
-        for (const v of vectors) {
-          await floatIndex.add(v.id, v.float);
-        }
-        const floatBuildTime = performance.now() - start;
-
-        console.log(`\nBuild time: Binary ${binaryBuildTime.toFixed(0)}ms, Float ${floatBuildTime.toFixed(0)}ms`);
-
-        // Search comparison
-        const numQueries = 50;
-        const binaryLatencies = [];
-        const floatLatencies = [];
-
-        for (let i = 0; i < numQueries; i++) {
-          const queryFloat = new Array(floatDim).fill(0).map(() => Math.random() * 2 - 1);
-
-          const binaryResult = await binaryIndex.search(floatToBinary(queryFloat), 10);
-          binaryLatencies.push(binaryResult.latency_us);
-
-          const floatResult = await floatIndex.search(queryFloat, 10);
-          floatLatencies.push(floatResult.latency_us);
-        }
-
-        const binaryP50 = binaryLatencies.sort((a, b) => a - b)[Math.floor(numQueries * 0.5)];
-        const floatP50 = floatLatencies.sort((a, b) => a - b)[Math.floor(numQueries * 0.5)];
-
-        console.log(`\nSearch latency p50: Binary ${binaryP50}μs, Float ${floatP50}μs`);
-        console.log(`Speedup: ${(floatP50 / binaryP50).toFixed(1)}x`);
-
-        const binaryMem = numVectors * Math.ceil(floatDim / 8);
-        const floatMem = numVectors * floatDim * 4;
-        console.log(`\nMemory: Binary ${(binaryMem / 1024).toFixed(0)} KB, Float ${(floatMem / 1024).toFixed(0)} KB`);
-        console.log(`Compression: ${(floatMem / binaryMem).toFixed(0)}x`);
-
       } else {
-        console.log('Unknown command. Use: stats, test, or compare');
+        console.log('Unknown command. Use: stats, test');
       }
     } catch (err) {
       console.error('Error:', err.message);