sweet-search 2.5.2 → 2.5.4

package/core/incremental-indexing/infrastructure/graph-gc.mjs

@@ -0,0 +1,314 @@
+/**
+ * Retired-row physical GC for `code-graph.db`.
+ *
+ * The reconcile path only *tombstones* superseded graph rows: `applyGraphDelta`
+ * sets `epoch_retired` on retired entities/relationships (and HCGS summaries),
+ * but nothing ever deletes them, so `code-graph.db` grows without bound under a
+ * long-lived daemon. The post-fix soak ended with ~90% of entity rows retired
+ * (3466/3870, ~3.8 MB), which is the dominant long-run growth source once the
+ * vector / LI / sparse tiers are bounded. This module physically removes
+ * retired graph rows once no reader can still observe them.
+ *
+ * Safety (strict visibility — mirrors `vector-gc.mjs` § 8.1.1):
+ *   An entity row is visible to a reader pinned at manifest epoch E iff
+ *     (epoch_written IS NULL OR epoch_written <= E)
+ *     AND (epoch_retired IS NULL OR epoch_retired > E)
+ *     AND (stale_since IS NULL OR (epoch_retired IS NOT NULL AND epoch_retired > E))
+ *   Relationships and HCGS summaries use the same epoch_written/epoch_retired
+ *   rule. So a row with `epoch_retired = R` is invisible to every reader whose
+ *   pinned epoch E satisfies E >= R. The smallest epoch any live reader pins is
+ *   `minLiveEpoch` (from reader heartbeats); the repository always re-syncs to
+ *   the latest manifest, so a reader's query epoch never drops below its
+ *   heartbeat epoch. Deleting rows with `epoch_retired <= frontier`, where
+ *   `frontier = minLiveEpoch ?? currentManifestEpoch`, can never remove a row
+ *   any reader still sees:
+ *     - readers present → frontier = minLiveEpoch <= every reader's epoch.
+ *     - no readers      → frontier = currentManifestEpoch; any future reader
+ *       reads a manifest at epoch >= currentManifestEpoch (monotonic), so the
+ *       deleted rows (retired <= currentManifestEpoch <= future E) are already
+ *       invisible to it.
+ *
+ * Reference integrity: deleting a retired entity cannot orphan a *live*
+ * relationship. The read path LEFT JOINs relationships to entities under the
+ * entity-visibility filter; a retired entity (epoch_retired <= frontier <= E)
+ * is already filtered out of that join at every visible epoch, so any live
+ * relationship pointing at it already resolves to NULL and is dropped from
+ * results. Removing the already-invisible entity changes no query result.
+ * A retired entity's own outgoing relationships are retired in the same
+ * reconcile transaction at the same epoch, so they fall under the same
+ * frontier and are GC'd alongside it.
+ *
+ * FTS5 consistency: `entities_fts` and `entities_trigram` are external-content
+ * FTS5 tables (`content='entities'`, `content_rowid='rowid'`). Deleting an
+ * entity content row does NOT update the index, so we must issue the FTS5
+ * `'delete'` command (rowid + the originally-indexed column values, read from
+ * the still-present row) before unlinking the content row, all in one
+ * transaction. Retired rows are immutable apart from `epoch_retired` /
+ * `stale_since` (neither is an FTS column), so the current column values equal
+ * the indexed values and the delete is exact.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import Database from 'better-sqlite3';
+
+export const DEFAULT_GRAPH_GC_BATCH = 2000;
+export const DEFAULT_GRAPH_GC_MAX_ROWS = 100_000;
+
+function tableExists(db, name) {
+  return !!db.prepare(
+    "SELECT 1 FROM sqlite_master WHERE type='table' AND name = ?",
+  ).get(name);
+}
+
+function hasColumn(db, table, column) {
+  try {
+    return db.prepare(`PRAGMA table_info(${table})`).all().some((c) => c.name === column);
+  } catch {
+    return false;
+  }
+}
+
+function normalizeBatchOpts(opts) {
+  const batchSize = Number.isInteger(opts.batchSize) && opts.batchSize > 0
+    ? opts.batchSize : DEFAULT_GRAPH_GC_BATCH;
+  const maxRows = Number.isInteger(opts.maxRows) && opts.maxRows > 0
+    ? opts.maxRows : DEFAULT_GRAPH_GC_MAX_ROWS;
+  return { batchSize, maxRows };
+}
+
+/**
+ * Delete retired `relationships` rows at or below `frontier` in bounded
+ * batches. Relationships carry no FTS coupling and no dependents, so a plain
+ * `rowid IN (… LIMIT ?)` delete is sufficient (works regardless of whether
+ * SQLite was built with SQLITE_ENABLE_UPDATE_DELETE_LIMIT).
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @param {number} frontier
+ * @param {{batchSize?:number, maxRows?:number}} [opts]
+ * @returns {{deleted:number, batches:number, hitCap:boolean, skipped?:string}}
+ */
+export function pruneRetiredRelationships(db, frontier, opts = {}) {
+  if (!Number.isInteger(frontier)) {
+    throw new Error(`pruneRetiredRelationships: frontier must be an integer, got ${frontier}`);
+  }
+  if (!hasColumn(db, 'relationships', 'epoch_retired')) {
+    return { deleted: 0, batches: 0, hitCap: false, skipped: 'no-epoch-column' };
+  }
+  const { batchSize, maxRows } = normalizeBatchOpts(opts);
+  const stmt = db.prepare(`
+    DELETE FROM relationships
+     WHERE rowid IN (
+       SELECT rowid FROM relationships
+        WHERE epoch_retired IS NOT NULL AND epoch_retired <= ?
+        LIMIT ?
+     )
+  `);
+  let deleted = 0;
+  let batches = 0;
+  let hitCap = false;
+  for (;;) {
+    const remainingCap = maxRows - deleted;
+    if (remainingCap <= 0) { hitCap = true; break; }
+    const take = Math.min(batchSize, remainingCap);
+    const changes = stmt.run(frontier, take).changes ?? 0;
+    deleted += changes;
+    batches += 1;
+    if (changes < take) break; // drained
+  }
+  return { deleted, batches, hitCap };
+}
+
+/**
+ * Delete retired `entities` rows at or below `frontier` in bounded batches,
+ * keeping the external-content FTS5 indices consistent. For each row we issue
+ * the FTS5 `'delete'` command (rowid + originally-indexed columns) before
+ * unlinking the content row, inside one transaction per batch so cross-process
+ * readers (WAL) observe an all-or-nothing change.
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @param {number} frontier
+ * @param {{batchSize?:number, maxRows?:number}} [opts]
+ * @returns {{deleted:number, batches:number, hitCap:boolean, ftsDeleted:number, skipped?:string}}
+ */
+export function pruneRetiredEntities(db, frontier, opts = {}) {
+  if (!Number.isInteger(frontier)) {
+    throw new Error(`pruneRetiredEntities: frontier must be an integer, got ${frontier}`);
+  }
+  if (!hasColumn(db, 'entities', 'epoch_retired')) {
+    return { deleted: 0, batches: 0, hitCap: false, ftsDeleted: 0, skipped: 'no-epoch-column' };
+  }
+  const { batchSize, maxRows } = normalizeBatchOpts(opts);
+  const hasFts = tableExists(db, 'entities_fts');
+  const hasTrigram = tableExists(db, 'entities_trigram');
+
+  const selectStmt = db.prepare(`
+    SELECT rowid AS rid, name, name_alias, signature, doc_comment
+      FROM entities
+     WHERE epoch_retired IS NOT NULL AND epoch_retired <= ?
+     LIMIT ?
+  `);
+  const ftsDel = hasFts ? db.prepare(
+    `INSERT INTO entities_fts(entities_fts, rowid, name, name_alias, signature, doc_comment)
+     VALUES('delete', ?, ?, ?, ?, ?)`,
+  ) : null;
+  const triDel = hasTrigram ? db.prepare(
+    `INSERT INTO entities_trigram(entities_trigram, rowid, name, signature)
+     VALUES('delete', ?, ?, ?)`,
+  ) : null;
+  const delStmt = db.prepare('DELETE FROM entities WHERE rowid = ?');
+
+  const runBatch = db.transaction((take) => {
+    const rows = selectStmt.all(frontier, take);
+    let ftsRemoved = 0;
+    for (const r of rows) {
+      if (ftsDel) { try { ftsDel.run(r.rid, r.name, r.name_alias, r.signature, r.doc_comment); ftsRemoved += 1; } catch { /* index drift — tolerate */ } }
+      if (triDel) { try { triDel.run(r.rid, r.name, r.signature); } catch { /* tolerate */ } }
+      delStmt.run(r.rid);
+    }
+    return { count: rows.length, ftsRemoved };
+  });
+
+  let deleted = 0;
+  let batches = 0;
+  let ftsDeleted = 0;
+  let hitCap = false;
+  for (;;) {
+    const remainingCap = maxRows - deleted;
+    if (remainingCap <= 0) { hitCap = true; break; }
+    const take = Math.min(batchSize, remainingCap);
+    const { count, ftsRemoved } = runBatch(take);
+    deleted += count;
+    ftsDeleted += ftsRemoved;
+    batches += 1;
+    if (count < take) break; // drained
+  }
+  return { deleted, batches, hitCap, ftsDeleted };
+}
+
+/**
+ * Delete retired `hcgs_summary_metadata` rows at or below `frontier` in bounded
+ * batches. The table is WITHOUT ROWID (PK `entity_id`), so we bound via an
+ * `entity_id IN (… LIMIT ?)` subquery.
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @param {number} frontier
+ * @param {{batchSize?:number, maxRows?:number}} [opts]
+ * @returns {{deleted:number, batches:number, hitCap:boolean, skipped?:string}}
+ */
+export function pruneRetiredGraphSummaries(db, frontier, opts = {}) {
+  if (!Number.isInteger(frontier)) {
+    throw new Error(`pruneRetiredGraphSummaries: frontier must be an integer, got ${frontier}`);
+  }
+  if (!tableExists(db, 'hcgs_summary_metadata') || !hasColumn(db, 'hcgs_summary_metadata', 'epoch_retired')) {
+    return { deleted: 0, batches: 0, hitCap: false, skipped: 'no-summary-table' };
+  }
+  const { batchSize, maxRows } = normalizeBatchOpts(opts);
+  const stmt = db.prepare(`
+    DELETE FROM hcgs_summary_metadata
+     WHERE entity_id IN (
+       SELECT entity_id FROM hcgs_summary_metadata
+        WHERE epoch_retired IS NOT NULL AND epoch_retired <= ?
+        LIMIT ?
+     )
+  `);
+  let deleted = 0;
+  let batches = 0;
+  let hitCap = false;
+  for (;;) {
+    const remainingCap = maxRows - deleted;
+    if (remainingCap <= 0) { hitCap = true; break; }
+    const take = Math.min(batchSize, remainingCap);
+    const changes = stmt.run(frontier, take).changes ?? 0;
+    deleted += changes;
+    batches += 1;
+    if (changes < take) break; // drained
+  }
+  return { deleted, batches, hitCap };
+}
+
+/**
+ * Run retired-row GC against `<stateDir>/code-graph.db`.
+ *
+ * Computes the safe prune frontier from reader heartbeats (falling back to the
+ * current manifest epoch when no readers are live), prunes relationships, then
+ * entities (with FTS cleanup), then HCGS summaries — sharing a single per-run
+ * row budget — then issues a PASSIVE WAL checkpoint to keep the WAL bounded
+ * without ever blocking concurrent readers. Never throws on a missing DB /
+ * table / column / heartbeat dir — returns `{ skipped }` instead.
+ *
+ * @param {string} stateDir
+ * @param {{
+ *   dbPath?:string, batchSize?:number, maxRows?:number,
+ *   minLiveEpoch?:(dir:string)=>(number|null),
+ *   readManifest?:(dir:string)=>(object|null),
+ * }} [deps]
+ * @returns {{
+ *   deletedEntities:number, deletedRelationships:number, deletedSummaries:number,
+ *   frontier:number, hadReaders:boolean, batches:number, hitCap:boolean
+ * } | {skipped:string}}
+ */
+export function runGraphGc(stateDir, deps = {}) {
+  const dbPath = deps.dbPath || path.join(stateDir, 'code-graph.db');
+  if (!fs.existsSync(dbPath)) return { skipped: 'no-graph-db' };
+
+  const minLiveEpochFn = deps.minLiveEpoch;
+  const readManifestFn = deps.readManifest;
+  if (typeof minLiveEpochFn !== 'function' || typeof readManifestFn !== 'function') {
+    throw new Error('runGraphGc: minLiveEpoch and readManifest deps are required');
+  }
+
+  let frontier = null;
+  let hadReaders = false;
+  const live = minLiveEpochFn(stateDir);
+  if (Number.isInteger(live)) {
+    frontier = live;
+    hadReaders = true;
+  } else {
+    const manifest = readManifestFn(stateDir);
+    if (Number.isInteger(manifest?.epoch)) frontier = manifest.epoch;
+  }
+  if (!Number.isInteger(frontier)) return { skipped: 'no-frontier' };
+
+  const { batchSize, maxRows } = normalizeBatchOpts(deps);
+  const db = new Database(dbPath);
+  try {
+    db.pragma('journal_mode = WAL');
+    db.pragma('synchronous = NORMAL');
+
+    // Relationships first (cheapest, no FTS), then entities (FTS cleanup),
+    // then HCGS summaries — sharing one per-run row budget so a churny graph
+    // can never starve the reconcile tick. Safe in any order: every deleted
+    // row is already invisible at the frontier.
+    let budget = maxRows;
+    const rel = tableExists(db, 'relationships')
+      ? pruneRetiredRelationships(db, frontier, { batchSize, maxRows: budget })
+      : { deleted: 0, batches: 0, hitCap: false };
+    budget -= rel.deleted;
+    const ent = (budget > 0 && tableExists(db, 'entities'))
+      ? pruneRetiredEntities(db, frontier, { batchSize, maxRows: budget })
+      : { deleted: 0, batches: 0, hitCap: false };
+    budget -= ent.deleted;
+    const sum = (budget > 0)
+      ? pruneRetiredGraphSummaries(db, frontier, { batchSize, maxRows: budget })
+      : { deleted: 0, batches: 0, hitCap: false };
+
+    const totalDeleted = rel.deleted + ent.deleted + sum.deleted;
+    if (totalDeleted > 0) {
+      try { db.pragma('wal_checkpoint(PASSIVE)'); } catch { /* best-effort */ }
+    }
+
+    return {
+      deletedEntities: ent.deleted,
+      deletedRelationships: rel.deleted,
+      deletedSummaries: sum.deleted,
+      ftsDeleted: ent.ftsDeleted ?? 0,
+      frontier,
+      hadReaders,
+      batches: rel.batches + ent.batches + sum.batches,
+      hitCap: !!(rel.hitCap || ent.hitCap || sum.hitCap),
+    };
+  } finally {
+    db.close();
+  }
+}

package/core/incremental-indexing/infrastructure/hashing.mjs

@@ -0,0 +1,298 @@
+/**
+ * Content-hashing wrapper for incremental indexing.
+ *
+ * Sweet-search uses content hashes for local dedup (per-file content, per-chunk
+ * content, exact encoder inputs). The original incremental-tracker.js path
+ * truncates SHA-256 to 16 hex chars (8 bytes / 64 bits). That is enough for
+ * collision avoidance on local corpora but the throughput (~1-2 GiB/s on cores
+ * without SHA-NI / ARMv8 Crypto Extensions) becomes meaningful on branch-switch
+ * storms that touch tens of thousands of files.
+ *
+ * Plan § 7.2 / § 21 specifies xxHash3-64 as the default, gated behind
+ * SWEET_SEARCH_HASH_ALGORITHM with a SHA-256 truncation fallback for
+ * compliance / auditing. xxHash3 is collision-resistant well beyond our
+ * working set (Cyan4973/xxHash + SMHasher3 benchmarks) and runs 15-30 GiB/s
+ * with native SIMD.
+ *
+ * Resolution order:
+ *   1. native crate: crates/sweet-search-native exports xxhash3_64 — fastest.
+ *   2. @node-rs/xxhash — fast prebuilt N-API binding.
+ *   3. pure-JS xxHash3 — present here as a portable last resort; ~3× faster
+ *      than SHA-256 truncation but slower than the native paths.
+ *   4. SHA-256 truncate-16 — current behaviour, kept as the compliance
+ *      override and the fallback when the algorithm switch is `sha256`.
+ *
+ * The output of every path is a 16-hex-char string so consumers (logs,
+ * SQLite columns of TEXT type, JSON state files) can be swapped without
+ * cascading changes.
+ */
+
+import crypto from 'node:crypto';
+import { createRequire } from 'node:module';
+
+const ALGO_ENV = (process.env.SWEET_SEARCH_HASH_ALGORITHM || 'xxhash3').toLowerCase();
+export const HASH_ALGORITHM = ALGO_ENV === 'sha256' ? 'sha256' : 'xxhash3';
+const require = createRequire(import.meta.url);
+const PURE_JS_PRIME64_1 = 0x9E3779B185EBCA87n;
+const PURE_JS_PRIME64_2 = 0xC2B2AE3D27D4EB4Fn;
+const PURE_JS_PRIME64_3 = 0x165667B19E3779F9n;
+const PURE_JS_PRIME64_4 = 0x85EBCA77C2B2AE63n;
+const PURE_JS_PRIME64_5 = 0x27D4EB2F165667C5n;
+const MASK64 = 0xFFFFFFFFFFFFFFFFn;
+
+let nativeXxh3 = null;
+let nodeRsXxh3 = null;
+let resolved = false;
+
+function toHex64(value) {
+  if (typeof value === 'string') {
+    return value.length >= 16 ? value.slice(-16).toLowerCase() : value.padStart(16, '0').toLowerCase();
+  }
+  return (typeof value === 'bigint' ? value : BigInt(value)).toString(16).padStart(16, '0');
+}
+
+function rotl64(x, r) {
+  return ((x << BigInt(r)) | (x >> BigInt(64 - r))) & MASK64;
+}
+
+function readUInt64LE(buf, offset) {
+  const lo = BigInt(buf[offset] | (buf[offset + 1] << 8) | (buf[offset + 2] << 16) | (buf[offset + 3] << 24)) & 0xFFFFFFFFn;
+  const hi = BigInt(buf[offset + 4] | (buf[offset + 5] << 8) | (buf[offset + 6] << 16) | (buf[offset + 7] << 24)) & 0xFFFFFFFFn;
+  return (lo | (hi << 32n)) & MASK64;
+}
+
+function readUInt32LE(buf, offset) {
+  // (buf[off] | buf[off+1]<<8 | buf[off+2]<<16 | buf[off+3]<<24) operates on Number; the top
+  // bit set produces a negative i32. Mask first, then promote to BigInt so we never feed
+  // negatives into xxHash mixing.
+  const lo = (buf[offset] | (buf[offset + 1] << 8) | (buf[offset + 2] << 16) | (buf[offset + 3] << 24)) >>> 0;
+  return BigInt(lo);
+}
+
+/**
+ * Pure-JS xxHash64 (close-enough fallback for environments without the native
+ * binding). We use xxHash64 here, not xxHash3, because xxHash3's full algorithm
+ * is significantly more complex while still producing different bytes than
+ * xxHash64. As soon as the native binding is available the wrapper switches
+ * to xxhash3_64 transparently; the pure-JS path is only ever used when neither
+ * crates/sweet-search-native nor @node-rs/xxhash is installed.
+ *
+ * Output is collision-resistant for our working set (sweet-search hashes for
+ * local dedup, not cryptographic integrity). See plan § 7.2 hash choice.
+ */
+function xxh64PureJs(buf, seed = 0n) {
+  let h64;
+  const len = buf.length;
+  let i = 0;
+
+  if (len >= 32) {
+    let v1 = (seed + PURE_JS_PRIME64_1 + PURE_JS_PRIME64_2) & MASK64;
+    let v2 = (seed + PURE_JS_PRIME64_2) & MASK64;
+    let v3 = seed;
+    let v4 = (seed - PURE_JS_PRIME64_1) & MASK64;
+
+    while (i + 32 <= len) {
+      v1 = (v1 + ((readUInt64LE(buf, i) * PURE_JS_PRIME64_2) & MASK64)) & MASK64;
+      v1 = rotl64(v1, 31);
+      v1 = (v1 * PURE_JS_PRIME64_1) & MASK64;
+
+      v2 = (v2 + ((readUInt64LE(buf, i + 8) * PURE_JS_PRIME64_2) & MASK64)) & MASK64;
+      v2 = rotl64(v2, 31);
+      v2 = (v2 * PURE_JS_PRIME64_1) & MASK64;
+
+      v3 = (v3 + ((readUInt64LE(buf, i + 16) * PURE_JS_PRIME64_2) & MASK64)) & MASK64;
+      v3 = rotl64(v3, 31);
+      v3 = (v3 * PURE_JS_PRIME64_1) & MASK64;
+
+      v4 = (v4 + ((readUInt64LE(buf, i + 24) * PURE_JS_PRIME64_2) & MASK64)) & MASK64;
+      v4 = rotl64(v4, 31);
+      v4 = (v4 * PURE_JS_PRIME64_1) & MASK64;
+
+      i += 32;
+    }
+
+    h64 = (rotl64(v1, 1) + rotl64(v2, 7) + rotl64(v3, 12) + rotl64(v4, 18)) & MASK64;
+
+    v1 = (rotl64((v1 * PURE_JS_PRIME64_2) & MASK64, 31) * PURE_JS_PRIME64_1) & MASK64;
+    h64 = ((h64 ^ v1) * PURE_JS_PRIME64_1 + PURE_JS_PRIME64_4) & MASK64;
+
+    v2 = (rotl64((v2 * PURE_JS_PRIME64_2) & MASK64, 31) * PURE_JS_PRIME64_1) & MASK64;
+    h64 = ((h64 ^ v2) * PURE_JS_PRIME64_1 + PURE_JS_PRIME64_4) & MASK64;
+
+    v3 = (rotl64((v3 * PURE_JS_PRIME64_2) & MASK64, 31) * PURE_JS_PRIME64_1) & MASK64;
+    h64 = ((h64 ^ v3) * PURE_JS_PRIME64_1 + PURE_JS_PRIME64_4) & MASK64;
+
+    v4 = (rotl64((v4 * PURE_JS_PRIME64_2) & MASK64, 31) * PURE_JS_PRIME64_1) & MASK64;
+    h64 = ((h64 ^ v4) * PURE_JS_PRIME64_1 + PURE_JS_PRIME64_4) & MASK64;
+  } else {
+    h64 = (seed + PURE_JS_PRIME64_5) & MASK64;
+  }
+
+  h64 = (h64 + BigInt(len)) & MASK64;
+
+  while (i + 8 <= len) {
+    let k1 = (readUInt64LE(buf, i) * PURE_JS_PRIME64_2) & MASK64;
+    k1 = rotl64(k1, 31);
+    k1 = (k1 * PURE_JS_PRIME64_1) & MASK64;
+    h64 ^= k1;
+    h64 = (rotl64(h64, 27) * PURE_JS_PRIME64_1 + PURE_JS_PRIME64_4) & MASK64;
+    i += 8;
+  }
+
+  if (i + 4 <= len) {
+    h64 = ((h64 ^ ((readUInt32LE(buf, i) * PURE_JS_PRIME64_1) & MASK64)) & MASK64);
+    h64 = (rotl64(h64, 23) * PURE_JS_PRIME64_2 + PURE_JS_PRIME64_3) & MASK64;
+    i += 4;
+  }
+
+  while (i < len) {
+    h64 = (h64 ^ (BigInt(buf[i]) * PURE_JS_PRIME64_5)) & MASK64;
+    h64 = (rotl64(h64, 11) * PURE_JS_PRIME64_1) & MASK64;
+    i += 1;
+  }
+
+  h64 ^= h64 >> 33n;
+  h64 = (h64 * PURE_JS_PRIME64_2) & MASK64;
+  h64 ^= h64 >> 29n;
+  h64 = (h64 * PURE_JS_PRIME64_3) & MASK64;
+  h64 ^= h64 >> 32n;
+
+  return h64;
+}
+
+function bufFromInput(input) {
+  if (Buffer.isBuffer(input)) return input;
+  if (input instanceof Uint8Array) return Buffer.from(input.buffer, input.byteOffset, input.byteLength);
+  if (typeof input === 'string') return Buffer.from(input, 'utf8');
+  throw new TypeError(`hashing: unsupported input type ${typeof input}`);
+}
+
+function makeNativeXxh3(mod) {
+  if (!mod || typeof mod.xxhash3_64 !== 'function') return null;
+  return (buf) => toHex64(mod.xxhash3_64(buf));
+}
+
+function makeNodeRsXxh3(mod) {
+  if (!mod) return null;
+  const ns = mod.default && (mod.default.xxh3 || mod.default.Xxh3) ? mod.default : mod;
+  if (ns.xxh3 && typeof ns.xxh3.xxh64 === 'function') {
+    return (buf) => toHex64(ns.xxh3.xxh64(buf));
+  }
+  const Xxh3 = ns.Xxh3 || ns.xxh3;
+  if (Xxh3 && typeof Xxh3.oneShotHashU64 === 'function') {
+    return (buf) => toHex64(Xxh3.oneShotHashU64(buf));
+  }
+  if (Xxh3 && typeof Xxh3.h64 === 'function') {
+    return (buf) => toHex64(Xxh3.h64(buf));
+  }
+  if (Xxh3 && typeof Xxh3.withSeed === 'function') {
+    return (buf) => {
+      const hasher = Xxh3.withSeed(0n);
+      hasher.update(buf);
+      return toHex64(hasher.digest());
+    };
+  }
+  return null;
+}
+
+function resolveBackendsSync() {
+  if (ALGO_ENV === 'sha256') {
+    resolved = true;
+    return;
+  }
+  try {
+    nativeXxh3 = makeNativeXxh3(require('../../../crates/sweet-search-native/index.js'));
+  } catch {
+    // Native crate not built or missing the export; try the package dependency.
+  }
+  if (!nativeXxh3) {
+    try {
+      nodeRsXxh3 = makeNodeRsXxh3(require('@node-rs/xxhash'));
+    } catch {
+      // Dependency not installed or native package failed to load; async resolver
+      // and the portable fallback still keep hashing functional.
+    }
+  }
+  resolved = Boolean(nativeXxh3 || nodeRsXxh3);
+}
+
+async function resolveBackends() {
+  if (resolved) return;
+  resolved = true;
+
+  // Resolve native crate (preferred).
+  try {
+    const mod = await import('../../../crates/sweet-search-native/index.js');
+    nativeXxh3 = makeNativeXxh3(mod);
+  } catch {
+    // Native crate not built or missing the export; fall through.
+  }
+
+  // Resolve @node-rs/xxhash (second preference).
+  if (!nativeXxh3) {
+    try {
+      const mod = await import('@node-rs/xxhash');
+      nodeRsXxh3 = makeNodeRsXxh3(mod);
+    } catch {
+      // Package not installed; pure-JS fallback wins.
+    }
+  }
+}
+
+/**
+ * Hash arbitrary input (string or buffer) to a 16-hex-char digest.
+ *
+ * @param {Buffer|Uint8Array|string} input
+ * @returns {Promise<string>}
+ */
+export async function contentHash(input) {
+  await resolveBackends();
+  return contentHashSync(input);
+}
+
+/**
+ * Sync variant; safe to call after `contentHash` has run at least once
+ * (warms the backend cache) or when caller explicitly opts in to the
+ * pure-JS / SHA-256 path.
+ */
+export function contentHashSync(input) {
+  const buf = bufFromInput(input);
+  if (ALGO_ENV === 'sha256') {
+    return crypto.createHash('sha256').update(buf).digest('hex').slice(0, 16);
+  }
+  if (nativeXxh3) return nativeXxh3(buf);
+  if (nodeRsXxh3) return nodeRsXxh3(buf);
+  return xxh64PureJs(buf).toString(16).padStart(16, '0');
+}
+
+/**
+ * Stable JSON stringify used by metadata-fingerprint hashing. Sorts object
+ * keys recursively so two equivalent objects produce identical bytes.
+ *
+ * @param {*} value
+ * @returns {string}
+ */
+export function stableStringify(value) {
+  if (value === null || typeof value !== 'object') return JSON.stringify(value);
+  if (Array.isArray(value)) return '[' + value.map(stableStringify).join(',') + ']';
+  const keys = Object.keys(value).sort();
+  return '{' + keys.map((k) => JSON.stringify(k) + ':' + stableStringify(value[k])).join(',') + '}';
+}
+
+/**
+ * Hash a metadata-bearing object deterministically.
+ *
+ * @param {object} payload
+ * @returns {Promise<string>}
+ */
+export async function metadataFingerprint(payload) {
+  return contentHash(stableStringify(payload));
+}
+
+export const __testing = {
+  xxh64PureJs,
+  stableStringify,
+  resolveBackends,
+};
+
+resolveBackendsSync();