vectlite 0.9.3 → 0.11.0

package/README.md

@@ -68,7 +68,7 @@ db.close()
 ### Data Management
 
 - **Physical collections** -- `vectlite.openStore()` manages a directory of independent databases
-- **Bulk ingestion** -- `bulkIngest()` with deferred index rebuilds for fast imports
+- **Bulk ingestion** -- `bulkIngest()` with Rayon-parallel HNSW build, coalesced WAL fsync, and tunable `m` / `efConstruction` / `efSearch` / tombstone rebuild threshold
 - **Listing & filtered counts** -- `list()` and `count({ namespace, filter })` without a vector query
 - **Delete by filter** -- `deleteByFilter()` for bulk deletion by metadata filter
 - **Partial metadata updates** -- `updateMetadata()` merges a patch without re-writing the vector or rebuilding indexes
@@ -345,6 +345,37 @@ await db.compactAsync()
 const count = await db.bulkIngestAsync(records, { batchSize: 5000 })
 ```
 
+### Tuning the HNSW index
+
+`bulkIngest()` and `bulkIngestAsync()` accept optional HNSW parameters that
+control the recall/latency trade-off and trigger Rayon-backed parallel graph
+construction once the dataset crosses `parallelInsertThreshold` (default 256):
+
+```js
+// Higher recall, slightly slower build/search
+db.bulkIngest(records, {
+  batchSize: 5000,
+  m: 32,              // max bidirectional links per node (default 16)
+  efConstruction: 400, // build-time search width (default 200)
+  efSearch: 200,       // query-time search width (default: auto)
+})
+
+// Faster build/search, lower recall
+db.bulkIngest(records, { m: 8, efConstruction: 100, efSearch: 40 })
+```
+
+The same parameters can be changed at any time without re-ingesting:
+
+```js
+db.setIndexConfig({ m: 32, efConstruction: 400 }) // rebuilds the ANN graph
+db.setEfSearch(200)                               // query-time only, no rebuild
+console.log(db.indexConfig())
+// { m: 32, ef_construction: 400, ef_search: 200, parallel_insert_threshold: 256 }
+```
+
+Use higher `m` / `efConstruction` / `efSearch` to push Recall@10 toward `1.0`;
+use lower values when latency or memory matter more than recall.
+
 ### OpenTelemetry Integration
 
 vectlite ships with optional OpenTelemetry tracing. When enabled, every search
@@ -398,7 +429,15 @@ before re-throwing.
 | `db.insert(id, vector, metadata, options)` | Insert a record (throws on duplicate id) |
 | `db.upsertMany(records, { namespace })` | Upsert a batch of records |
 | `db.insertMany(records, { namespace })` | Insert a batch |
-| `db.bulkIngest(records, { namespace, batchSize })` | Fastest bulk import with batched WAL writes |
+| `db.bulkIngest(records, { namespace, batchSize, m, efConstruction, efSearch, parallelInsertThreshold, tombstoneRebuildPct })` | Fastest bulk import with coalesced WAL fsync and Rayon-parallel HNSW build |
+| `db.setIndexConfig({ m, efConstruction, efSearch, parallelInsertThreshold, tombstoneRebuildPct })` | Update HNSW parameters; rebuilds the ANN graph if `m`/`efConstruction` changed |
+| `db.setEfSearch(efSearch)` | Adjust query-time HNSW search width without rebuilding |
+| `db.indexConfig()` | Return the current HNSW configuration |
+| `db.setWalSyncMode(mode, n)` | Configure WAL fsync cadence: `'per_op'`, `'every_n'`, or `'on_flush'` |
+| `db.walSyncMode()` | Return the current WAL sync mode |
+| `db.tombstoneStats()` | Return live and tombstoned HNSW node counts |
+| `db.prepareForScan()` | Materialise the contiguous vector arena |
+| `db.vectorArenaLen()` | Return the vector arena size or `null` |
 | `db.delete(id, { namespace })` | Delete a single record |
 | `db.deleteMany(ids, { namespace })` | Delete multiple records by id |
 | `db.deleteByFilter(filter, { namespace })` | Delete all records matching a filter |
@@ -462,7 +501,7 @@ before re-throwing.
 | `db.searchWithStatsAsync(query, options)` | Non-blocking search with stats (returns Promise) |
 | `db.flushAsync()` | Non-blocking flush/compact (returns Promise) |
 | `db.compactAsync()` | Non-blocking compact (returns Promise) |
-| `db.bulkIngestAsync(records, options)` | Non-blocking bulk import (returns Promise) |
+| `db.bulkIngestAsync(records, options)` | Non-blocking bulk import (returns Promise); accepts the same HNSW tuning options as `bulkIngest` |
 
 ## Filter Operators
 

package/index.d.ts

@@ -110,6 +110,50 @@ export interface ListCursorResult {
 export interface BulkIngestOptions {
   namespace?: string | null
   batchSize?: number
+  /** Max bidirectional links per HNSW node (default 16). */
+  m?: number | null
+  /** Build-time search width (default 200). Higher = better recall, slower build. */
+  efConstruction?: number | null
+  /** Query-time search width. `null` = auto (derived from k). */
+  efSearch?: number | null
+  /** Minimum dataset size to engage Rayon-parallel HNSW insertion (default 256). */
+  parallelInsertThreshold?: number | null
+  /**
+   * Percentage (0..=100) of dead nodes at which `compact()` triggers an
+   * HNSW rebuild. Default 30. Set to 100 to disable automatic rebuild.
+   */
+  tombstoneRebuildPct?: number | null
+}
+
+export interface IndexConfig {
+  m: number
+  ef_construction: number
+  ef_search: number | null
+  parallel_insert_threshold: number
+  tombstone_rebuild_pct: number
+}
+
+export interface SetIndexConfigOptions {
+  m?: number | null
+  efConstruction?: number | null
+  efSearch?: number | null
+  parallelInsertThreshold?: number | null
+  tombstoneRebuildPct?: number | null
+}
+
+/** WAL fsync mode. See `Database.setWalSyncMode`. */
+export type WalSyncMode = 'per_op' | 'every_n' | 'on_flush'
+
+export type WalSyncModeInfo =
+  | { mode: 'per_op' }
+  | { mode: 'every_n'; n: number }
+  | { mode: 'on_flush' }
+
+export interface TombstoneStats {
+  /** Live (non-tombstoned) records across all HNSW graphs. */
+  live: number
+  /** Dead (tombstoned) records still in the graphs, awaiting compact(). */
+  dead: number
 }
 
 export interface SearchOptions {
@@ -212,6 +256,37 @@ export class Database {
   insertMany(records: Record[], options?: { namespace?: string | null }): number
   upsertMany(records: Record[], options?: { namespace?: string | null }): number
   bulkIngest(records: Record[], options?: BulkIngestOptions): number
+  /** Get the current HNSW configuration. */
+  indexConfig(): IndexConfig
+  /** Adjust query-time `ef_search` only (no rebuild). `null` reverts to auto. */
+  setEfSearch(efSearch: number | null): void
+  /** Update HNSW parameters; rebuilds the ANN graph if `m`/`efConstruction` changed. */
+  setIndexConfig(config: SetIndexConfigOptions): void
+  /**
+   * Configure WAL durability.
+   *
+   * - `"per_op"`   (default): fsync after every insert. Strongest durability.
+   * - `"every_n"`  : fsync once every `n` inserts (pass `n` as 2nd arg).
+   * - `"on_flush"` : only fsync at `flush()` / `compact()` / `close()`.
+   *
+   * On macOS APFS, `"on_flush"` typically multiplies ingestion throughput
+   * by 5–10× at the cost of losing un-flushed writes on a crash.
+   */
+  setWalSyncMode(mode: WalSyncMode, n?: number | null): void
+  /** Return the current WAL sync mode. */
+  walSyncMode(): WalSyncModeInfo
+  /** Total live and tombstoned record counts across every HNSW graph. */
+  tombstoneStats(): TombstoneStats
+  /**
+   * Materialise the contiguous-vector arena up front for cache- and
+   * SIMD-friendly scans. Normally built lazily on first use.
+   */
+  prepareForScan(): void
+  /**
+   * Number of vectors in the contiguous arena, or `null` if it hasn't
+   * been materialised yet in this session.
+   */
+  vectorArenaLen(): number | null
   get(id: string, options?: { namespace?: string | null }): Record | null
   delete(id: string, options?: { namespace?: string | null }): boolean
   deleteMany(ids: string[], options?: { namespace?: string | null }): number

package/index.js

@@ -415,10 +415,89 @@ class Database {
 
   bulkIngest(records, options = {}) {
     return wrapError(() =>
-      this._native.bulkIngest(encode(records), options.namespace ?? null, options.batchSize ?? 10_000),
+      this._native.bulkIngest(
+        encode(records),
+        options.namespace ?? null,
+        options.batchSize ?? 10_000,
+        options.m ?? null,
+        options.efConstruction ?? null,
+        options.efSearch ?? null,
+        options.parallelInsertThreshold ?? null,
+        options.tombstoneRebuildPct ?? null,
+      ),
+    )
+  }
+
+  indexConfig() {
+    return wrapError(() => decode(this._native.indexConfig()))
+  }
+
+  setEfSearch(efSearch) {
+    return wrapError(() => this._native.setEfSearch(efSearch ?? null))
+  }
+
+  setIndexConfig(config = {}) {
+    return wrapError(() =>
+      this._native.setIndexConfig(
+        config.m ?? null,
+        config.efConstruction ?? null,
+        config.efSearch ?? null,
+        config.parallelInsertThreshold ?? null,
+        config.tombstoneRebuildPct ?? null,
+      ),
     )
   }
 
+  /**
+   * Configure WAL durability. Valid modes are:
+   *   - "per_op"  : fsync after every insert (default, strongest durability)
+   *   - "every_n" : fsync once every `n` inserts — pass `n` as second arg
+   *   - "on_flush": fsync only at flush() / compact() / close()
+   * On macOS APFS, "on_flush" typically multiplies ingestion throughput by
+   * 5–10× at the cost of losing un-flushed writes on a crash.
+   */
+  setWalSyncMode(mode, n = null) {
+    return wrapError(() => this._native.setWalSyncMode(mode, n))
+  }
+
+  /**
+   * Return the current WAL sync mode. Shape:
+   *   { mode: "per_op" } | { mode: "every_n", n: number } | { mode: "on_flush" }
+   */
+  walSyncMode() {
+    return wrapError(() => decode(this._native.walSyncMode()))
+  }
+
+  /**
+   * Return `{ live, dead }` summed across every HNSW graph (global +
+   * namespaced). Use to monitor when a compact() will rebuild the graph
+   * for recall.
+   */
+  tombstoneStats() {
+    return wrapError(() => {
+      const [live, dead] = this._native.tombstoneStats()
+      return { live, dead }
+    })
+  }
+
+  /**
+   * Materialise the contiguous-vector arena up front. The arena mirrors
+   * every record's default dense vector into a single flat Float32 buffer
+   * for cache- and SIMD-friendly brute-force / rescoring scans. Built
+   * lazily on first use otherwise.
+   */
+  prepareForScan() {
+    return wrapError(() => this._native.prepareForScan())
+  }
+
+  /**
+   * Number of vectors in the contiguous arena, or `null` if it has not
+   * been materialised yet in this session.
+   */
+  vectorArenaLen() {
+    return wrapError(() => this._native.vectorArenaLen())
+  }
+
   get(id, options = {}) {
     return wrapError(() => decode(this._native.get(id, options.namespace ?? null)))
   }
@@ -585,7 +664,16 @@ class Database {
 
   bulkIngestAsync(records, options = {}) {
     return wrapAsync(
-      this._native.bulkIngestAsync(encode(records), options.namespace ?? null, options.batchSize ?? 10_000),
+      this._native.bulkIngestAsync(
+        encode(records),
+        options.namespace ?? null,
+        options.batchSize ?? 10_000,
+        options.m ?? null,
+        options.efConstruction ?? null,
+        options.efSearch ?? null,
+        options.parallelInsertThreshold ?? null,
+        options.tombstoneRebuildPct ?? null,
+      ),
     )
   }
 }

package/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectlite-node"
-version = "0.9.3"
+version = "0.11.0"
 edition = "2024"
 license = "MIT"
 description = "Node.js bindings for vectlite."

package/native/src/lib.rs

@@ -12,10 +12,10 @@ use vectlite::quantization::{
     default_product_num_sub_vectors,
 };
 use vectlite::{
-    Database as CoreDatabase, DistanceMetric, FusionStrategy, HybridSearchOptions, Metadata,
-    MetadataFilter, MetadataValue, MultiVectorSearchOptions, MultiVectors, NamedVectors,
+    Database as CoreDatabase, DistanceMetric, FusionStrategy, HybridSearchOptions, IndexConfig,
+    Metadata, MetadataFilter, MetadataValue, MultiVectorSearchOptions, MultiVectors, NamedVectors,
     PayloadIndexType, Record, SearchOutcome, SearchResult, SparseVector, Store as CoreStore,
-    WriteOperation,
+    WalSyncMode, WriteOperation,
 };
 
 #[napi(js_name = "NativeDatabase")]
@@ -417,13 +417,140 @@ impl NativeDatabase {
         records_json: String,
         namespace: Option<String>,
         batch_size: u32,
+        m: Option<u32>,
+        ef_construction: Option<u32>,
+        ef_search: Option<u32>,
+        parallel_insert_threshold: Option<u32>,
+        tombstone_rebuild_pct: Option<u32>,
     ) -> Result<u32> {
         let records = parse_record_batch_json(&records_json, namespace.as_deref())?;
         let mut database = self.write_open()?;
-        database
-            .bulk_ingest(records, batch_size as usize)
-            .map(|count| count as u32)
-            .map_err(to_napi_error)
+        let tuning = merge_index_config(
+            m,
+            ef_construction,
+            ef_search,
+            parallel_insert_threshold,
+            tombstone_rebuild_pct,
+        );
+        let count = if let Some(cfg) = tuning {
+            let merged = apply_index_overrides(database.index_config(), cfg);
+            database.bulk_ingest_with_config(records, batch_size as usize, Some(merged))
+        } else {
+            database.bulk_ingest(records, batch_size as usize)
+        };
+        count.map(|n| n as u32).map_err(to_napi_error)
+    }
+
+    #[napi(js_name = "indexConfig")]
+    pub fn index_config(&self) -> Result<String> {
+        let cfg = self.read()?.index_config();
+        let value = json!({
+            "m": cfg.m as u32,
+            "ef_construction": cfg.ef_construction as u32,
+            "ef_search": cfg.ef_search.map(|v| v as u32),
+            "parallel_insert_threshold": cfg.parallel_insert_threshold as u32,
+            "tombstone_rebuild_pct": cfg.tombstone_rebuild_pct as u32,
+        });
+        stringify_value(value)
+    }
+
+    #[napi(js_name = "setEfSearch")]
+    pub fn set_ef_search(&self, ef_search: Option<u32>) -> Result<()> {
+        let ef = ef_search.map(|v| v as usize);
+        self.write_open()?.set_ef_search(ef).map_err(to_napi_error)
+    }
+
+    #[napi(js_name = "setIndexConfig")]
+    pub fn set_index_config(
+        &self,
+        m: Option<u32>,
+        ef_construction: Option<u32>,
+        ef_search: Option<u32>,
+        parallel_insert_threshold: Option<u32>,
+        tombstone_rebuild_pct: Option<u32>,
+    ) -> Result<()> {
+        let mut database = self.write_open()?;
+        let overrides = merge_index_config(
+            m,
+            ef_construction,
+            ef_search,
+            parallel_insert_threshold,
+            tombstone_rebuild_pct,
+        )
+        .ok_or_else(|| err("setIndexConfig requires at least one field"))?;
+        let merged = apply_index_overrides(database.index_config(), overrides);
+        database.set_index_config(merged).map_err(to_napi_error)
+    }
+
+    /// Configure WAL durability. `mode` is one of: `"per_op"` (the default,
+    /// fsync after every insert), `"every_n"` (fsync once every `n` inserts
+    /// — provide `n`), `"on_flush"` (only fsync at flush / compact / close).
+    ///
+    /// Relaxing this knob is the single biggest ingestion throughput lever
+    /// on macOS APFS: `on_flush` typically multiplies throughput by 5–10×
+    /// at the cost of losing un-flushed writes on a crash.
+    #[napi(js_name = "setWalSyncMode")]
+    pub fn set_wal_sync_mode(&self, mode: String, n: Option<u32>) -> Result<()> {
+        let parsed = match mode.to_ascii_lowercase().as_str() {
+            "per_op" | "perop" => WalSyncMode::PerOp,
+            "every_n" | "everyn" => {
+                let n = n.ok_or_else(|| {
+                    err("setWalSyncMode(\"every_n\", ...) requires the second `n` argument")
+                })?;
+                WalSyncMode::EveryN(n as usize)
+            }
+            "on_flush" | "onflush" => WalSyncMode::OnFlush,
+            other => {
+                return Err(err(format!(
+                    "unknown WAL sync mode '{other}' (expected 'per_op', 'every_n', or 'on_flush')"
+                )));
+            }
+        };
+        let mut database = self.write_open()?;
+        database.set_wal_sync_mode(parsed).map_err(to_napi_error)
+    }
+
+    /// Return the current WAL sync mode as a JSON string: either
+    /// `{"mode":"per_op"}`, `{"mode":"every_n","n":64}`, or
+    /// `{"mode":"on_flush"}`.
+    #[napi(js_name = "walSyncMode")]
+    pub fn wal_sync_mode(&self) -> Result<String> {
+        let database = self.read()?;
+        let value = match database.wal_sync_mode() {
+            WalSyncMode::PerOp => json!({"mode": "per_op"}),
+            WalSyncMode::EveryN(n) => json!({"mode": "every_n", "n": n as u32}),
+            WalSyncMode::OnFlush => json!({"mode": "on_flush"}),
+        };
+        stringify_value(value)
+    }
+
+    /// Return `[live, dead]` summed across every HNSW graph (global +
+    /// namespaced). Useful for monitoring when to `compact()`.
+    #[napi(js_name = "tombstoneStats")]
+    pub fn tombstone_stats(&self) -> Result<Vec<u32>> {
+        let database = self.read()?;
+        let (live, dead) = database.tombstone_stats();
+        Ok(vec![live as u32, dead as u32])
+    }
+
+    /// Materialise the contiguous-vector arena. Mirrors every record's
+    /// default dense vector into a single flat `Float32Array`-shaped
+    /// buffer for cache- and SIMD-friendly brute-force / rescoring scans.
+    /// Normally built lazily; call this before a heavy scan to pay the
+    /// build cost up front. Cheap when already fresh.
+    #[napi(js_name = "prepareForScan")]
+    pub fn prepare_for_scan(&self) -> Result<()> {
+        let mut database = self.write_open()?;
+        database.prepare_for_scan();
+        Ok(())
+    }
+
+    /// Number of vectors in the contiguous arena, or `null` if it has
+    /// not been materialised yet in this session.
+    #[napi(js_name = "vectorArenaLen")]
+    pub fn vector_arena_len(&self) -> Result<Option<u32>> {
+        let database = self.read()?;
+        Ok(database.vector_arena_len().map(|n| n as u32))
     }
 
     #[napi]
@@ -878,6 +1005,7 @@ pub struct BulkIngestTask {
     db: Arc<RwLock<CoreDatabase>>,
     records: Vec<Record>,
     batch_size: usize,
+    tuning: Option<IndexConfigPatch>,
 }
 
 impl napi::Task for BulkIngestTask {
@@ -890,10 +1018,13 @@ impl napi::Task for BulkIngestTask {
             .db
             .write()
             .map_err(|e| err(format!("lock poisoned: {e}")))?;
-        database
-            .bulk_ingest(records, self.batch_size)
-            .map(|count| count as u32)
-            .map_err(to_napi_error)
+        let res = if let Some(cfg) = self.tuning.clone() {
+            let merged = apply_index_overrides(database.index_config(), cfg);
+            database.bulk_ingest_with_config(records, self.batch_size, Some(merged))
+        } else {
+            database.bulk_ingest(records, self.batch_size)
+        };
+        res.map(|count| count as u32).map_err(to_napi_error)
     }
 
     fn resolve(&mut self, _env: napi::Env, output: Self::Output) -> Result<Self::JsValue> {
@@ -959,12 +1090,25 @@ impl NativeDatabase {
         records_json: String,
         namespace: Option<String>,
         batch_size: u32,
+        m: Option<u32>,
+        ef_construction: Option<u32>,
+        ef_search: Option<u32>,
+        parallel_insert_threshold: Option<u32>,
+        tombstone_rebuild_pct: Option<u32>,
     ) -> Result<AsyncTask<BulkIngestTask>> {
         let records = parse_record_batch_json(&records_json, namespace.as_deref())?;
+        let tuning = merge_index_config(
+            m,
+            ef_construction,
+            ef_search,
+            parallel_insert_threshold,
+            tombstone_rebuild_pct,
+        );
         Ok(AsyncTask::new(BulkIngestTask {
             db: self.inner.clone(),
             records,
             batch_size: batch_size as usize,
+            tuning,
         }))
     }
 }
@@ -1962,6 +2106,60 @@ fn value_to_usize(value: &Value, label: &str) -> Result<usize> {
         .ok_or_else(|| err(format!("{label} must be an unsigned integer")))
 }
 
+#[derive(Clone, Copy)]
+struct IndexConfigPatch {
+    m: Option<usize>,
+    ef_construction: Option<usize>,
+    ef_search: Option<usize>,
+    parallel_insert_threshold: Option<usize>,
+    tombstone_rebuild_pct: Option<u8>,
+}
+
+/// Pack the five optional HNSW tuning fields into a patch. Returns `None`
+/// when every field is `None`; explicit zeroes are preserved so core
+/// validation can reject invalid build/search widths instead of treating
+/// them as "not provided".
+fn merge_index_config(
+    m: Option<u32>,
+    ef_construction: Option<u32>,
+    ef_search: Option<u32>,
+    parallel_insert_threshold: Option<u32>,
+    tombstone_rebuild_pct: Option<u32>,
+) -> Option<IndexConfigPatch> {
+    if m.is_none()
+        && ef_construction.is_none()
+        && ef_search.is_none()
+        && parallel_insert_threshold.is_none()
+        && tombstone_rebuild_pct.is_none()
+    {
+        return None;
+    }
+    Some(IndexConfigPatch {
+        m: m.map(|v| v as usize),
+        ef_construction: ef_construction.map(|v| v as usize),
+        ef_search: ef_search.map(|v| v as usize),
+        parallel_insert_threshold: parallel_insert_threshold.map(|v| v as usize),
+        tombstone_rebuild_pct: tombstone_rebuild_pct.map(|v| if v > 100 { 101 } else { v as u8 }),
+    })
+}
+
+/// Merge a tuning patch into the current `IndexConfig`. Omitted fields inherit
+/// from `current`; `ef_search = None` in the patch means "no change" because
+/// callers use `setEfSearch(null)` to reset query-time tuning to auto.
+fn apply_index_overrides(current: IndexConfig, patch: IndexConfigPatch) -> IndexConfig {
+    IndexConfig {
+        m: patch.m.unwrap_or(current.m),
+        ef_construction: patch.ef_construction.unwrap_or(current.ef_construction),
+        ef_search: patch.ef_search.or(current.ef_search),
+        parallel_insert_threshold: patch
+            .parallel_insert_threshold
+            .unwrap_or(current.parallel_insert_threshold),
+        tombstone_rebuild_pct: patch
+            .tombstone_rebuild_pct
+            .unwrap_or(current.tombstone_rebuild_pct),
+    }
+}
+
 fn err(message: impl Into<String>) -> NapiError {
     NapiError::from_reason(message.into())
 }

package/native/vectlite-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectlite-core"
-version = "0.9.3"
+version = "0.11.0"
 edition = "2024"
 license = "MIT"
 description = "Core storage engine for vectlite."