sweet-search 2.6.10 → 2.6.12

package/core/graph/graph-expansion.js

@@ -249,6 +249,18 @@ const TYPE_BOOST = {
   struct: 1.2,
 };
 
+/**
+ * Fetch int8 vectors for many ids with ONE stale-bitmap snapshot when the
+ * index supports it (BinaryHNSWIndex.getInt8VectorsForIds); falls back to
+ * per-id lookups for index doubles that only expose getInt8Vector.
+ */
+function batchInt8Vectors(hnswIndex, ids) {
+  if (typeof hnswIndex.getInt8VectorsForIds === 'function') {
+    return hnswIndex.getInt8VectorsForIds(ids);
+  }
+  return ids.map((id) => hnswIndex.getInt8Vector(id));
+}
+
 function clampSemanticWeight(value) {
   if (!Number.isFinite(value)) return 0.4;
   return Math.max(0, Math.min(1, value));
@@ -614,21 +626,27 @@ export function expandSecondHop(db, seedIds, expanded, edgeTypes, options = {})
   }
 
   const excluded = new Set([...seedIds, ...expanded.keys()]);
-  const candidates = [];
+  const eligible = [];
   for (const rel of hop2Forward) {
     if (!edgeTypes.has(rel.type) || excluded.has(rel.target_id)) continue;
 
     const hop1Entry = expanded.get(rel.source_id);
     const hop1Score = hop1Entry?.score ?? 1;  // identity: preserves old edgePriority × weight
     const graphScore = hop1Score * (EDGE_PRIORITY[rel.type] || 1) * (rel.weight || 1.0);
+    eligible.push({ rel, graphScore });
+  }
+  // Batch int8 lookup: one stale-bitmap snapshot for all targets instead of
+  // one stat per target.
+  const eligibleInt8 = batchInt8Vectors(hnswIndex, eligible.map((e) => e.rel.target_id));
+  const candidates = eligible.map((e, i) => {
+    const entityInt8 = eligibleInt8[i];
     let normSim = null;
-    const entityInt8 = hnswIndex.getInt8Vector(rel.target_id);
     if (entityInt8) {
       const cosSim = cosineSimilarity(queryInt8, entityInt8);
       normSim = (cosSim + 1) / 2;
     }
-    candidates.push({ rel, graphScore, normSim });
-  }
+    return { rel: e.rel, graphScore: e.graphScore, normSim };
+  });
 
   if (candidates.length === 0) return;
   const normalizedGraphScores = normalizeMinMax(candidates.map(c => c.graphScore));
@@ -729,6 +747,25 @@ export function expandSecondHopAdaptive(db, seedIds, hop1Expanded, edgeTypes, op
   const vectorCache = semanticEnabled ? new Map() : null;
   const scoredCandidates = [];
 
+  // Pre-seed the vector cache with ONE batched int8 fetch (single
+  // stale-bitmap snapshot) so the scoring loop below never does per-target
+  // lookups.
+  if (semanticEnabled) {
+    const uniqueTargets = [];
+    for (const c of rawCandidates) {
+      if (!edgeTypes.has(c.type) || excluded.has(c.target_id)) continue;
+      if (vectorCache.has(c.target_id)) continue;
+      vectorCache.set(c.target_id, undefined);
+      uniqueTargets.push(c.target_id);
+    }
+    if (uniqueTargets.length > 0) {
+      const fetched = batchInt8Vectors(hnswIndex, uniqueTargets);
+      for (let i = 0; i < uniqueTargets.length; i++) {
+        vectorCache.set(uniqueTargets[i], fetched[i]);
+      }
+    }
+  }
+
   for (const c of rawCandidates) {
     if (!edgeTypes.has(c.type) || excluded.has(c.target_id)) continue;
 
@@ -743,6 +780,7 @@ export function expandSecondHopAdaptive(db, seedIds, hop1Expanded, edgeTypes, op
     let normSim = null;
     if (semanticEnabled) {
       if (!vectorCache.has(c.target_id)) {
+        // Miss fallback only — the pre-seed pass below batch-fills the cache.
         vectorCache.set(c.target_id, hnswIndex.getInt8Vector(c.target_id));
       }
       const entityInt8 = vectorCache.get(c.target_id);
@@ -929,12 +967,16 @@ export function rerankExpanded(expandedResults, seedResults, options = {}) {
     }
   } else {
     const normalizedGraphScores = normalizeMinMax(baseScores);
+    // Batch int8 lookup: one stale-bitmap snapshot for the whole result set.
+    const entityInt8s = batchInt8Vectors(
+      hnswIndex,
+      expandedResults.map((er) => er.entity_id || er.id)
+    );
     for (let i = 0; i < expandedResults.length; i++) {
       const er = expandedResults[i];
       const normGraph = normalizedGraphScores[i];
       let rerankScore = normGraph;
-      const entityId = er.entity_id || er.id;
-      const entityInt8 = hnswIndex.getInt8Vector(entityId);
+      const entityInt8 = entityInt8s[i];
       if (entityInt8) {
         const cosSim = cosineSimilarity(queryInt8, entityInt8);
         const normSim = (cosSim + 1) / 2;

package/core/incremental-indexing/application/production-reconciler-helpers.mjs

@@ -60,19 +60,56 @@ export function insertRelationships(db, relationships, liveIdFor, epoch) {
   }
 }
 
-export function markBinaryStale(index, id) {
-  const idx = index.idToIndex.get(id);
-  if (idx == null) return false;
+/**
+ * Batched stale marking. The per-id `markBinaryStale` loads, mutates, and
+ * fsyncs the bitmap file once PER RETIRED ID — O(retires × bitmap size) with
+ * an fsync each. A batch loads the bitmap once (lazily, on the first mark),
+ * applies every mark in memory with the exact per-op semantics (idToIndex /
+ * int8Vectors pruned immediately, reader cache invalidated), and persists
+ * once in flush(). End state on disk is identical to N sequential
+ * markBinaryStale calls.
+ *
+ * Crash window: marks staged between flush()es are lost on crash, which the
+ * reconciler's persist-before-advance gate already handles — an unflushed
+ * tick replays its ops.
+ */
+export function createStaleBatch(index) {
   const stalePath = index.stalePath || `${index.indexPath}.stale.bin`;
   let bitmap = null;
-  try { bitmap = loadBitmap(stalePath); } catch {}
-  bitmap = bitmap ? resizeBitmap(bitmap, Math.max(idx + 1, index.vectors.length, 1)) : createBitmap(Math.max(idx + 1, index.vectors.length, 1));
-  setBit(bitmap, idx);
-  saveBitmap(stalePath, bitmap);
-  index.idToIndex.delete(id);
-  index.int8Vectors.delete(id);
-  index._staleBitmapCache = null;
-  return true;
+  let loaded = false;
+  let dirty = false;
+  return {
+    markStale(id) {
+      const idx = index.idToIndex.get(id);
+      if (idx == null) return false;
+      if (!loaded) {
+        try { bitmap = loadBitmap(stalePath); } catch {}
+        loaded = true;
+      }
+      bitmap = bitmap
+        ? resizeBitmap(bitmap, Math.max(idx + 1, index.vectors.length, 1))
+        : createBitmap(Math.max(idx + 1, index.vectors.length, 1));
+      setBit(bitmap, idx);
+      dirty = true;
+      index.idToIndex.delete(id);
+      index.int8Vectors.delete(id);
+      index._staleBitmapCache = null;
+      return true;
+    },
+    flush() {
+      if (!dirty) return false;
+      saveBitmap(stalePath, bitmap);
+      dirty = false;
+      return true;
+    },
+  };
+}
+
+export function markBinaryStale(index, id) {
+  const batch = createStaleBatch(index);
+  const marked = batch.markStale(id);
+  batch.flush();
+  return marked;
 }
 
 /**

package/core/incremental-indexing/application/production-reconciler.mjs

@@ -13,7 +13,7 @@ import { readManifest, writeManifest } from '../infrastructure/manifest.mjs';
 import { annotateChunksForDelta, snapshotFileRows, diffChunks, applyDiff } from '../infrastructure/vector-delta-writer.mjs';
 import { appendDeltaRecord, FALLBACK_WEIGHTS_ID, fileIdFor, listDeltaSegments } from '../infrastructure/sparse-gram-delta.mjs';
 import { fts5Merge, fts5MergeBudgetPages } from '../infrastructure/sqlite-fts5.mjs';
-import { insertEntity, insertRelationships, markBinaryStale, maintainFloatStore, flushFloatStore } from './production-reconciler-helpers.mjs';
+import { insertEntity, insertRelationships, markBinaryStale, createStaleBatch, maintainFloatStore, flushFloatStore } from './production-reconciler-helpers.mjs';
 import {
   chunkCutoffEnabled,
   computeCutoffSignature,
@@ -1110,9 +1110,12 @@ class ProductionReconcileAdapter {
     if (ctx?.index) {
       const index = ctx.index;
       let append = 0; let tombstone = 0;
+      // One bitmap load + one save+fsync per file's ops instead of per
+      // retired id; marks apply in op order with identical semantics.
+      const staleBatch = createStaleBatch(index);
       for (const op of ops) {
         if (op.retireId) {
-          if (markBinaryStale(index, op.retireId)) tombstone += 1;
+          if (staleBatch.markStale(op.retireId)) tombstone += 1;
           ctx.floatRemoveIds.push(op.retireId);
         }
         if (op.addId && op.embedding) {
@@ -1122,6 +1125,7 @@ class ProductionReconcileAdapter {
           append += 1;
         }
       }
+      staleBatch.flush();
       ctx.tombstone += tombstone;
       // append is committed to ctx.append in finalize after the sorted inserts.
       ctx.append += append;
@@ -1137,9 +1141,13 @@ class ProductionReconcileAdapter {
     let append = 0; let tombstone = 0;
     const floatUpserts = [];
     const floatRemoveIds = [];
+    // One bitmap load + one save+fsync for the whole op list instead of per
+    // retired id. Marks still apply inline in op order (adds never consult
+    // the bitmap), so retire-then-readd sequences behave exactly as before.
+    const staleBatch = createStaleBatch(index);
     for (const op of ops) {
       if (op.retireId) {
-        if (markBinaryStale(index, op.retireId)) tombstone += 1;
+        if (staleBatch.markStale(op.retireId)) tombstone += 1;
         floatRemoveIds.push(op.retireId);
       }
       if (op.addId && op.embedding) {
@@ -1150,6 +1158,7 @@ class ProductionReconcileAdapter {
       }
       if ((append + tombstone) > 0 && (append + tombstone) % 100 === 0) this.progress('production:binary-hnsw-loop');
     }
+    staleBatch.flush();
     await index.save(indexPath);
     this.progress('production:binary-hnsw-saved');
     await maintainFloatStore(indexPath, { upserts: floatUpserts, removeIds: floatRemoveIds, binaryVectorsBefore, dimension: this.modelInfo.hnswDimension });

package/core/infrastructure/hamming-kernel.js

@@ -0,0 +1,293 @@
+/**
+ * Resident-slab Hamming kernel for the binary HNSW hot path.
+ *
+ * The legacy wasmHammingDistance() copies BOTH 64-byte operands into WASM
+ * linear memory on every call — at M=64/ef=400 a query pays thousands of
+ * memcpy+FFI round-trips, construction ~10⁴–10⁵ per insert. This kernel keeps
+ * every indexed vector RESIDENT in its own WASM instance (mirrored from a
+ * contiguous JS slab), so a distance is a single FFI call with zero copies:
+ * measured ~85ns/dist vs ~490ns/dist for the copy-per-call path (M3 Max,
+ * 64-byte vectors). When WASM is unavailable it falls back to an unrolled
+ * SWAR popcount over the JS slab (~190ns/dist vs ~600ns for the byte-LUT).
+ *
+ * The JS slab is the source of truth; per-vector Uint8Array subarray views
+ * keep the existing `vectors[i].binary` contract intact. The WASM mirror is
+ * a private instance of simd-distance.wasm (its exported memory is module-
+ * internal), so the shared instance used by wasmInt8BatchDot & friends keeps
+ * its scratch-at-offset-0 layout untouched.
+ *
+ * All arithmetic is integer Hamming — every path returns bit-exact results.
+ */
+
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+// WASM memory layout (all offsets fixed so memory.grow never moves them):
+//   [0, 256)                    staged query (256B covers 2048-bit vectors)
+//   [256, 256+4*CAP)            batch index scratch (u32 node indices)
+//   [.., +4*CAP)                batch distance output (u32)
+//   [SLAB_BASE, ...)            resident vector slab
+// BATCH_CAP is far above the max graph degree (2*M0 ≈ 256), so a whole
+// neighbor block always fits in one hamming_batch call.
+const QUERY_SLOT_BYTES = 256;
+const BATCH_CAP = 4096;
+const BATCH_IDX_PTR = QUERY_SLOT_BYTES;
+const BATCH_OUT_PTR = BATCH_IDX_PTR + BATCH_CAP * 4;
+const SLAB_BASE = BATCH_OUT_PTR + BATCH_CAP * 4;
+const WASM_PAGE = 65536;
+
+let cachedModule; // WebAssembly.Module | null once resolved
+
+function getWasmModule() {
+  if (cachedModule !== undefined) return cachedModule;
+  try {
+    const here = dirname(fileURLToPath(import.meta.url));
+    const bytes = readFileSync(join(here, 'simd-distance.wasm'));
+    // Sync compile is fine: the module is ~1KB.
+    cachedModule = new WebAssembly.Module(bytes);
+  } catch {
+    cachedModule = null;
+  }
+  return cachedModule;
+}
+
+// SWAR popcount of one u32 (subtract-shift-mask + multiply reduction).
+// Exact for all inputs.
+function popcnt32(x) {
+  x -= (x >>> 1) & 0x55555555;
+  x = (x & 0x33333333) + ((x >>> 2) & 0x33333333);
+  x = (x + (x >>> 4)) & 0x0f0f0f0f;
+  return Math.imul(x, 0x01010101) >>> 24;
+}
+
+export class HammingSlab {
+  /**
+   * @param {number} dimension - vector size in BYTES
+   */
+  constructor(dimension) {
+    this.dim = dimension;
+    this.words = dimension >>> 2;        // full u32 words
+    this.tailBytes = dimension & 3;      // non-multiple-of-4 remainder
+    this.capacity = 0;                   // vectors
+    this.count = 0;
+    this.slab = new Uint8Array(0);       // JS source of truth
+    this.slabU32 = new Uint32Array(0);
+    // Query scratch (JS fallback path): word view + byte view over one buffer
+    this.queryU32 = new Uint32Array(Math.ceil(dimension / 4));
+    this.queryBytes = new Uint8Array(this.queryU32.buffer, 0, dimension);
+
+    // Batch scratch: callers fill batchIdx[0..n) with node indices, call
+    // batchToQuery(n)/batchToVector(t, n), and read distances from the
+    // returned batchOut. One FFI call per block instead of one per neighbor.
+    this.batchIdx = new Uint32Array(BATCH_CAP);
+    this.batchOut = new Uint32Array(BATCH_CAP);
+    this._wasmIdxU32 = null; // cached views over WASM memory (rebuilt on grow)
+    this._wasmOutU32 = null;
+
+    // WASM mirror (optional)
+    this.wasm = null;
+    this.wasmMem = null;
+    const mod = getWasmModule();
+    if (mod) {
+      try {
+        const instance = new WebAssembly.Instance(mod);
+        this.wasm = instance.exports;
+        this.wasmMem = new Uint8Array(this.wasm.memory.buffer);
+      } catch {
+        this.wasm = null;
+      }
+    }
+  }
+
+  _wasmView() {
+    if (this.wasm && this.wasmMem.buffer !== this.wasm.memory.buffer) {
+      this.wasmMem = new Uint8Array(this.wasm.memory.buffer);
+      this._wasmIdxU32 = null;
+      this._wasmOutU32 = null;
+    }
+    return this.wasmMem;
+  }
+
+  /** Grow to hold at least `count` vectors. Existing views stay valid until
+   *  a grow re-points them — callers must use the views returned by set(). */
+  ensure(count) {
+    if (count <= this.capacity) return false;
+    let newCap = Math.max(1024, this.capacity * 2);
+    while (newCap < count) newCap *= 2;
+    const newSlab = new Uint8Array(newCap * this.dim);
+    newSlab.set(this.slab.subarray(0, this.count * this.dim));
+    this.slab = newSlab;
+    this.slabU32 = new Uint32Array(newSlab.buffer, 0, (newCap * this.dim) >>> 2);
+    this.capacity = newCap;
+    if (this.wasm) {
+      const needed = SLAB_BASE + newCap * this.dim;
+      const have = this.wasm.memory.buffer.byteLength;
+      if (needed > have) {
+        try {
+          this.wasm.memory.grow(Math.ceil((needed - have) / WASM_PAGE));
+        } catch {
+          // Growth refused (host memory pressure) — drop the mirror, JS
+          // fallback keeps everything exact.
+          this.wasm = null;
+        }
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Write vector bytes at index (append or overwrite) into the JS slab and
+   * the WASM mirror. Returns the canonical subarray view for storage on the
+   * node record.
+   */
+  set(idx, bytes) {
+    this.ensure(idx + 1);
+    const off = idx * this.dim;
+    const n = Math.min(bytes.length, this.dim);
+    this.slab.set(n === bytes.length ? bytes : bytes.subarray(0, n), off);
+    if (n < this.dim) this.slab.fill(0, off + n, off + this.dim);
+    if (this.wasm) {
+      this._wasmView().set(this.slab.subarray(off, off + this.dim), SLAB_BASE + off);
+    }
+    if (idx >= this.count) this.count = idx + 1;
+    return this.slab.subarray(off, off + this.dim);
+  }
+
+  /** Re-point a node's canonical view after grows (rarely needed by callers
+   *  that persist views long-term across many inserts). */
+  view(idx) {
+    const off = idx * this.dim;
+    return this.slab.subarray(off, off + this.dim);
+  }
+
+  /** Stage the query for distToQuery(). Shorter inputs are zero-padded. */
+  setQuery(bytes) {
+    const n = Math.min(bytes.length, this.dim);
+    this.queryBytes.set(n === bytes.length ? bytes : bytes.subarray(0, n));
+    if (n < this.dim) this.queryBytes.fill(0, n);
+    if (this.wasm) {
+      this._wasmView().set(this.queryBytes, 0);
+    }
+  }
+
+  /** Hamming distance from the staged query to vector idx. */
+  distToQuery(idx) {
+    if (this.wasm) {
+      return this.wasm.hamming_distance(0, SLAB_BASE + idx * this.dim, this.dim);
+    }
+    if (this.tailBytes === 0) {
+      if (this.words === 16) return swar16(this.queryU32, 0, this.slabU32, idx << 4);
+      const s = this.slabU32;
+      const q = this.queryU32;
+      const o = idx * this.words;
+      let c = 0;
+      for (let w = 0; w < this.words; w++) c += popcnt32(q[w] ^ s[o + w]);
+      return c;
+    }
+    // dim not a multiple of 4: per-vector slab offsets are word-unaligned,
+    // so fall back to a byte loop.
+    const off = idx * this.dim;
+    let c = 0;
+    for (let i = 0; i < this.dim; i++) c += popcnt32(this.queryBytes[i] ^ this.slab[off + i]);
+    return c;
+  }
+
+  /** Hamming distance between two resident vectors. */
+  dist(aIdx, bIdx) {
+    if (this.wasm) {
+      return this.wasm.hamming_distance(
+        SLAB_BASE + aIdx * this.dim,
+        SLAB_BASE + bIdx * this.dim,
+        this.dim
+      );
+    }
+    if (this.tailBytes === 0) {
+      if (this.words === 16) return swar16(this.slabU32, aIdx << 4, this.slabU32, bIdx << 4);
+      const s = this.slabU32;
+      const ao = aIdx * this.words;
+      const bo = bIdx * this.words;
+      let c = 0;
+      for (let w = 0; w < this.words; w++) c += popcnt32(s[ao + w] ^ s[bo + w]);
+      return c;
+    }
+    const aOff = aIdx * this.dim;
+    const bOff = bIdx * this.dim;
+    let c = 0;
+    for (let i = 0; i < this.dim; i++) c += popcnt32(this.slab[aOff + i] ^ this.slab[bOff + i]);
+    return c;
+  }
+
+  /**
+   * Batch: distances from the staged query to batchIdx[0..n). Results land
+   * in (and are returned as) batchOut. One FFI call per block on the WASM
+   * path; order-preserving and bit-exact on every path.
+   */
+  batchToQuery(n) {
+    return this._batch(0, n);
+  }
+
+  /** Batch: distances from resident vector targetIdx to batchIdx[0..n). */
+  batchToVector(targetIdx, n) {
+    if (this.wasm) return this._batch(SLAB_BASE + targetIdx * this.dim, n);
+    const idx = this.batchIdx;
+    const out = this.batchOut;
+    for (let i = 0; i < n; i++) out[i] = this.dist(targetIdx, idx[i]);
+    return out;
+  }
+
+  _batch(qPtr, n) {
+    if (this.wasm && this.wasm.hamming_batch && n <= BATCH_CAP) {
+      this._wasmView(); // refresh cached views if memory grew
+      if (this._wasmIdxU32 === null) {
+        this._wasmIdxU32 = new Uint32Array(this.wasm.memory.buffer, BATCH_IDX_PTR, BATCH_CAP);
+        this._wasmOutU32 = new Uint32Array(this.wasm.memory.buffer, BATCH_OUT_PTR, BATCH_CAP);
+      }
+      const wIdx = this._wasmIdxU32;
+      const idx = this.batchIdx;
+      for (let i = 0; i < n; i++) wIdx[i] = idx[i];
+      this.wasm.hamming_batch(qPtr, SLAB_BASE, this.dim, BATCH_IDX_PTR, n, BATCH_OUT_PTR);
+      const wOut = this._wasmOutU32;
+      const out = this.batchOut;
+      for (let i = 0; i < n; i++) out[i] = wOut[i];
+      return out;
+    }
+    const idx = this.batchIdx;
+    const out = this.batchOut;
+    for (let i = 0; i < n; i++) out[i] = this.distToQuery(idx[i]);
+    return out;
+  }
+
+  reset() {
+    this.count = 0;
+    // Keep allocation; contents are overwritten by set() before reads.
+  }
+}
+
+/**
+ * Fully-unrolled 16-word (64-byte / 512-bit) SWAR Hamming with deferred
+ * reduction in ≤7-word groups — 7 words × 8/byte-lane × 4 lanes = 224 < 256,
+ * so the final multiply's top byte can't overflow. Bit-exact.
+ */
+function swar16(A, ao, B, bo) {
+  let x, a = 0, b = 0, c = 0;
+  x = A[ao] ^ B[bo];           x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); a += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 1] ^ B[bo + 1];   x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); a += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 2] ^ B[bo + 2];   x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); a += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 3] ^ B[bo + 3];   x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); a += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 4] ^ B[bo + 4];   x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); a += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 5] ^ B[bo + 5];   x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); a += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 6] ^ B[bo + 6];   x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); a += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 7] ^ B[bo + 7];   x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); b += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 8] ^ B[bo + 8];   x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); b += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 9] ^ B[bo + 9];   x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); b += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 10] ^ B[bo + 10]; x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); b += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 11] ^ B[bo + 11]; x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); b += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 12] ^ B[bo + 12]; x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); b += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 13] ^ B[bo + 13]; x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); b += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 14] ^ B[bo + 14]; x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); c += (x + (x >>> 4)) & 0x0f0f0f0f;
+  x = A[ao + 15] ^ B[bo + 15]; x -= (x >>> 1) & 0x55555555; x = (x & 0x33333333) + ((x >>> 2) & 0x33333333); c += (x + (x >>> 4)) & 0x0f0f0f0f;
+  return (Math.imul(a, 0x01010101) >>> 24)
+       + (Math.imul(b, 0x01010101) >>> 24)
+       + (Math.imul(c, 0x01010101) >>> 24);
+}

package/core/infrastructure/simd-distance.js

@@ -403,9 +403,35 @@ export function wasmMaxSimDequant(queryFlat, docInt8, numQ, numD, dim, min, scal
   return maxsimExports.maxsim_dequant(qPtr, dPtr, numQ, numD, dim, min, scale);
 }
 
+// Query-copy-once session for per-candidate WASM MaxSim loops.
+//
+// The query lives at the fixed offset DATA_OFFSET, so within one query the
+// caller prepares it once and each per-candidate call skips the Q×dim×4-byte
+// memcpy. The query Float32Array's identity is NOT a safe cache key — callers
+// pool and mutate that buffer across queries — hence an explicit session id
+// that the caller must re-obtain per query.
+let _wasmQuerySession = 0;
+let _wasmQueryPrepared = null; // { session, qBytes, memBuffer }
+
+export function wasmMaxSimPrepareQuery(queryFlat, numQ, dim) {
+  if (!maxsimExports) return 0;
+  const qBytes = numQ * dim * 4;
+  if (!maxsimMem || maxsimMem.buffer !== maxsimExports.memory.buffer) {
+    maxsimMem = new Uint8Array(maxsimExports.memory.buffer);
+  }
+  if (qBytes + 1024 > maxsimMem.length) return 0;
+  maxsimMem.set(new Uint8Array(queryFlat.buffer, queryFlat.byteOffset, qBytes), DATA_OFFSET);
+  _wasmQuerySession += 1;
+  _wasmQueryPrepared = { session: _wasmQuerySession, qBytes, memBuffer: maxsimExports.memory.buffer };
+  return _wasmQuerySession;
+}
+
 // Shared layout: copy query + doc + per-token params into WASM memory, return pointers.
 // Returns null if data doesn't fit in WASM memory.
-function _wasmPerTokenLayout(queryFlat, docData, minArray, scaleArray, tokenNorms, numQ, numD, dim, dBytes) {
+// A live `querySession` from wasmMaxSimPrepareQuery (matching qBytes and
+// memory buffer) means the query bytes already sit at DATA_OFFSET, so the
+// query memcpy is skipped.
+function _wasmPerTokenLayout(queryFlat, docData, minArray, scaleArray, tokenNorms, numQ, numD, dim, dBytes, querySession) {
   const qBytes = numQ * dim * 4;
   const paramBytes = numD * 4 * 3;
   if (!maxsimMem || maxsimMem.buffer !== maxsimExports.memory.buffer) {
@@ -419,7 +445,14 @@ function _wasmPerTokenLayout(queryFlat, docData, minArray, scaleArray, tokenNorm
   const scalePtr = minPtr + numD * 4;
   const normPtr = scalePtr + numD * 4;
 
-  maxsimMem.set(new Uint8Array(queryFlat.buffer, queryFlat.byteOffset, qBytes), qPtr);
+  const queryAlreadyPlaced = querySession
+    && _wasmQueryPrepared
+    && _wasmQueryPrepared.session === querySession
+    && _wasmQueryPrepared.qBytes === qBytes
+    && _wasmQueryPrepared.memBuffer === maxsimExports.memory.buffer;
+  if (!queryAlreadyPlaced) {
+    maxsimMem.set(new Uint8Array(queryFlat.buffer, queryFlat.byteOffset, qBytes), qPtr);
+  }
   maxsimMem.set(new Uint8Array(docData.buffer, docData.byteOffset, dBytes), dPtr);
   maxsimMem.set(new Uint8Array(minArray.buffer, minArray.byteOffset, numD * 4), minPtr);
   maxsimMem.set(new Uint8Array(scaleArray.buffer, scaleArray.byteOffset, numD * 4), scalePtr);
@@ -428,16 +461,16 @@ function _wasmPerTokenLayout(queryFlat, docData, minArray, scaleArray, tokenNorm
   return { qPtr, dPtr, minPtr, scalePtr, normPtr };
 }
 
-export function wasmMaxSimDequantPerToken(queryFlat, docInt8, minArray, scaleArray, tokenNorms, numQ, numD, dim) {
+export function wasmMaxSimDequantPerToken(queryFlat, docInt8, minArray, scaleArray, tokenNorms, numQ, numD, dim, querySession) {
   if (!maxsimExports?.maxsim_dequant_pertoken) return null;
-  const ptrs = _wasmPerTokenLayout(queryFlat, docInt8, minArray, scaleArray, tokenNorms, numQ, numD, dim, numD * dim);
+  const ptrs = _wasmPerTokenLayout(queryFlat, docInt8, minArray, scaleArray, tokenNorms, numQ, numD, dim, numD * dim, querySession);
   if (!ptrs) return null;
   return maxsimExports.maxsim_dequant_pertoken(ptrs.qPtr, ptrs.dPtr, ptrs.minPtr, ptrs.scalePtr, ptrs.normPtr, numQ, numD, dim);
 }
 
-export function wasmMaxSimDequant4Bit(queryFlat, docPacked, minArray, scaleArray, tokenNorms, numQ, numD, dim) {
+export function wasmMaxSimDequant4Bit(queryFlat, docPacked, minArray, scaleArray, tokenNorms, numQ, numD, dim, querySession) {
   if (!maxsimExports?.maxsim_dequant_4bit) return null;
-  const ptrs = _wasmPerTokenLayout(queryFlat, docPacked, minArray, scaleArray, tokenNorms, numQ, numD, dim, numD * Math.ceil(dim / 2));
+  const ptrs = _wasmPerTokenLayout(queryFlat, docPacked, minArray, scaleArray, tokenNorms, numQ, numD, dim, numD * Math.ceil(dim / 2), querySession);
   if (!ptrs) return null;
   return maxsimExports.maxsim_dequant_4bit(ptrs.qPtr, ptrs.dPtr, ptrs.minPtr, ptrs.scalePtr, ptrs.normPtr, numQ, numD, dim);
 }

package/core/infrastructure/tombstone-bitmap-reader.js

@@ -109,6 +109,40 @@ export function setBit(bitmap, index) {
   bitmap.payload[byte] |= mask;
 }
 
+/**
+ * Count set bits with index < limitBits (clamped to capacity). Matches an
+ * isSet() loop over [0, limitBits) exactly — used for O(words) live-vector
+ * counting instead of an O(n) per-index probe.
+ */
+export function popcountRange(bitmap, limitBits) {
+  if (!bitmap || limitBits <= 0) return 0;
+  const bits = Math.min(limitBits, bitmap.capacity);
+  const buf = bitmap.payload;
+  const fullBytes = bits >>> 3;
+  let count = 0;
+  let i = 0;
+  // SWAR popcount, 4 bytes at a time
+  for (; i + 4 <= fullBytes; i += 4) {
+    let x = buf.readUInt32LE(i);
+    x -= (x >>> 1) & 0x55555555;
+    x = (x & 0x33333333) + ((x >>> 2) & 0x33333333);
+    x = (x + (x >>> 4)) & 0x0f0f0f0f;
+    count += Math.imul(x, 0x01010101) >>> 24;
+  }
+  for (; i < fullBytes; i++) {
+    let b = buf[i];
+    while (b !== 0) { b &= b - 1; count += 1; }
+  }
+  const tailBits = bits & 7;
+  if (tailBits > 0) {
+    // Bits are LSB-first within a byte (see byteAndMask), so the low mask
+    // selects exactly the indices below `bits`.
+    let b = buf[fullBytes] & ((1 << tailBits) - 1);
+    while (b !== 0) { b &= b - 1; count += 1; }
+  }
+  return count;
+}
+
 export function popcount(bitmap) {
   if (!bitmap) return 0;
   let count = 0;