vectlite 0.10.0 → 0.11.0

package/README.md

@@ -68,7 +68,7 @@ db.close()
 ### Data Management
 
 - **Physical collections** -- `vectlite.openStore()` manages a directory of independent databases
-- **Bulk ingestion** -- `bulkIngest()` with Rayon-parallel HNSW build, coalesced WAL fsync, and tunable `m` / `efConstruction` / `efSearch`
+- **Bulk ingestion** -- `bulkIngest()` with Rayon-parallel HNSW build, coalesced WAL fsync, and tunable `m` / `efConstruction` / `efSearch` / tombstone rebuild threshold
 - **Listing & filtered counts** -- `list()` and `count({ namespace, filter })` without a vector query
 - **Delete by filter** -- `deleteByFilter()` for bulk deletion by metadata filter
 - **Partial metadata updates** -- `updateMetadata()` merges a patch without re-writing the vector or rebuilding indexes
@@ -429,10 +429,15 @@ before re-throwing.
 | `db.insert(id, vector, metadata, options)` | Insert a record (throws on duplicate id) |
 | `db.upsertMany(records, { namespace })` | Upsert a batch of records |
 | `db.insertMany(records, { namespace })` | Insert a batch |
-| `db.bulkIngest(records, { namespace, batchSize, m, efConstruction, efSearch, parallelInsertThreshold })` | Fastest bulk import with coalesced WAL fsync and Rayon-parallel HNSW build |
-| `db.setIndexConfig({ m, efConstruction, efSearch, parallelInsertThreshold })` | Update HNSW parameters; rebuilds the ANN graph if `m`/`efConstruction` changed |
+| `db.bulkIngest(records, { namespace, batchSize, m, efConstruction, efSearch, parallelInsertThreshold, tombstoneRebuildPct })` | Fastest bulk import with coalesced WAL fsync and Rayon-parallel HNSW build |
+| `db.setIndexConfig({ m, efConstruction, efSearch, parallelInsertThreshold, tombstoneRebuildPct })` | Update HNSW parameters; rebuilds the ANN graph if `m`/`efConstruction` changed |
 | `db.setEfSearch(efSearch)` | Adjust query-time HNSW search width without rebuilding |
 | `db.indexConfig()` | Return the current HNSW configuration |
+| `db.setWalSyncMode(mode, n)` | Configure WAL fsync cadence: `'per_op'`, `'every_n'`, or `'on_flush'` |
+| `db.walSyncMode()` | Return the current WAL sync mode |
+| `db.tombstoneStats()` | Return live and tombstoned HNSW node counts |
+| `db.prepareForScan()` | Materialise the contiguous vector arena |
+| `db.vectorArenaLen()` | Return the vector arena size or `null` |
 | `db.delete(id, { namespace })` | Delete a single record |
 | `db.deleteMany(ids, { namespace })` | Delete multiple records by id |
 | `db.deleteByFilter(filter, { namespace })` | Delete all records matching a filter |

package/index.d.ts

@@ -118,6 +118,11 @@ export interface BulkIngestOptions {
   efSearch?: number | null
   /** Minimum dataset size to engage Rayon-parallel HNSW insertion (default 256). */
   parallelInsertThreshold?: number | null
+  /**
+   * Percentage (0..=100) of dead nodes at which `compact()` triggers an
+   * HNSW rebuild. Default 30. Set to 100 to disable automatic rebuild.
+   */
+  tombstoneRebuildPct?: number | null
 }
 
 export interface IndexConfig {
@@ -125,6 +130,7 @@ export interface IndexConfig {
   ef_construction: number
   ef_search: number | null
   parallel_insert_threshold: number
+  tombstone_rebuild_pct: number
 }
 
 export interface SetIndexConfigOptions {
@@ -132,6 +138,22 @@ export interface SetIndexConfigOptions {
   efConstruction?: number | null
   efSearch?: number | null
   parallelInsertThreshold?: number | null
+  tombstoneRebuildPct?: number | null
+}
+
+/** WAL fsync mode. See `Database.setWalSyncMode`. */
+export type WalSyncMode = 'per_op' | 'every_n' | 'on_flush'
+
+export type WalSyncModeInfo =
+  | { mode: 'per_op' }
+  | { mode: 'every_n'; n: number }
+  | { mode: 'on_flush' }
+
+export interface TombstoneStats {
+  /** Live (non-tombstoned) records across all HNSW graphs. */
+  live: number
+  /** Dead (tombstoned) records still in the graphs, awaiting compact(). */
+  dead: number
 }
 
 export interface SearchOptions {
@@ -240,6 +262,31 @@ export class Database {
   setEfSearch(efSearch: number | null): void
   /** Update HNSW parameters; rebuilds the ANN graph if `m`/`efConstruction` changed. */
   setIndexConfig(config: SetIndexConfigOptions): void
+  /**
+   * Configure WAL durability.
+   *
+   * - `"per_op"`   (default): fsync after every insert. Strongest durability.
+   * - `"every_n"`  : fsync once every `n` inserts (pass `n` as 2nd arg).
+   * - `"on_flush"` : only fsync at `flush()` / `compact()` / `close()`.
+   *
+   * On macOS APFS, `"on_flush"` typically multiplies ingestion throughput
+   * by 5–10× at the cost of losing un-flushed writes on a crash.
+   */
+  setWalSyncMode(mode: WalSyncMode, n?: number | null): void
+  /** Return the current WAL sync mode. */
+  walSyncMode(): WalSyncModeInfo
+  /** Total live and tombstoned record counts across every HNSW graph. */
+  tombstoneStats(): TombstoneStats
+  /**
+   * Materialise the contiguous-vector arena up front for cache- and
+   * SIMD-friendly scans. Normally built lazily on first use.
+   */
+  prepareForScan(): void
+  /**
+   * Number of vectors in the contiguous arena, or `null` if it hasn't
+   * been materialised yet in this session.
+   */
+  vectorArenaLen(): number | null
   get(id: string, options?: { namespace?: string | null }): Record | null
   delete(id: string, options?: { namespace?: string | null }): boolean
   deleteMany(ids: string[], options?: { namespace?: string | null }): number

package/index.js

@@ -423,6 +423,7 @@ class Database {
         options.efConstruction ?? null,
         options.efSearch ?? null,
         options.parallelInsertThreshold ?? null,
+        options.tombstoneRebuildPct ?? null,
       ),
     )
   }
@@ -442,10 +443,61 @@ class Database {
         config.efConstruction ?? null,
         config.efSearch ?? null,
         config.parallelInsertThreshold ?? null,
+        config.tombstoneRebuildPct ?? null,
       ),
     )
   }
 
+  /**
+   * Configure WAL durability. Valid modes are:
+   *   - "per_op"  : fsync after every insert (default, strongest durability)
+   *   - "every_n" : fsync once every `n` inserts — pass `n` as second arg
+   *   - "on_flush": fsync only at flush() / compact() / close()
+   * On macOS APFS, "on_flush" typically multiplies ingestion throughput by
+   * 5–10× at the cost of losing un-flushed writes on a crash.
+   */
+  setWalSyncMode(mode, n = null) {
+    return wrapError(() => this._native.setWalSyncMode(mode, n))
+  }
+
+  /**
+   * Return the current WAL sync mode. Shape:
+   *   { mode: "per_op" } | { mode: "every_n", n: number } | { mode: "on_flush" }
+   */
+  walSyncMode() {
+    return wrapError(() => decode(this._native.walSyncMode()))
+  }
+
+  /**
+   * Return `{ live, dead }` summed across every HNSW graph (global +
+   * namespaced). Use to monitor when a compact() will rebuild the graph
+   * for recall.
+   */
+  tombstoneStats() {
+    return wrapError(() => {
+      const [live, dead] = this._native.tombstoneStats()
+      return { live, dead }
+    })
+  }
+
+  /**
+   * Materialise the contiguous-vector arena up front. The arena mirrors
+   * every record's default dense vector into a single flat Float32 buffer
+   * for cache- and SIMD-friendly brute-force / rescoring scans. Built
+   * lazily on first use otherwise.
+   */
+  prepareForScan() {
+    return wrapError(() => this._native.prepareForScan())
+  }
+
+  /**
+   * Number of vectors in the contiguous arena, or `null` if it has not
+   * been materialised yet in this session.
+   */
+  vectorArenaLen() {
+    return wrapError(() => this._native.vectorArenaLen())
+  }
+
   get(id, options = {}) {
     return wrapError(() => decode(this._native.get(id, options.namespace ?? null)))
   }
@@ -620,6 +672,7 @@ class Database {
         options.efConstruction ?? null,
         options.efSearch ?? null,
         options.parallelInsertThreshold ?? null,
+        options.tombstoneRebuildPct ?? null,
       ),
     )
   }

package/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectlite-node"
-version = "0.10.0"
+version = "0.11.0"
 edition = "2024"
 license = "MIT"
 description = "Node.js bindings for vectlite."

package/native/src/lib.rs

@@ -15,7 +15,7 @@ use vectlite::{
     Database as CoreDatabase, DistanceMetric, FusionStrategy, HybridSearchOptions, IndexConfig,
     Metadata, MetadataFilter, MetadataValue, MultiVectorSearchOptions, MultiVectors, NamedVectors,
     PayloadIndexType, Record, SearchOutcome, SearchResult, SparseVector, Store as CoreStore,
-    WriteOperation,
+    WalSyncMode, WriteOperation,
 };
 
 #[napi(js_name = "NativeDatabase")]
@@ -421,10 +421,17 @@ impl NativeDatabase {
         ef_construction: Option<u32>,
         ef_search: Option<u32>,
         parallel_insert_threshold: Option<u32>,
+        tombstone_rebuild_pct: Option<u32>,
     ) -> Result<u32> {
         let records = parse_record_batch_json(&records_json, namespace.as_deref())?;
         let mut database = self.write_open()?;
-        let tuning = merge_index_config(m, ef_construction, ef_search, parallel_insert_threshold);
+        let tuning = merge_index_config(
+            m,
+            ef_construction,
+            ef_search,
+            parallel_insert_threshold,
+            tombstone_rebuild_pct,
+        );
         let count = if let Some(cfg) = tuning {
             let merged = apply_index_overrides(database.index_config(), cfg);
             database.bulk_ingest_with_config(records, batch_size as usize, Some(merged))
@@ -442,6 +449,7 @@ impl NativeDatabase {
             "ef_construction": cfg.ef_construction as u32,
             "ef_search": cfg.ef_search.map(|v| v as u32),
             "parallel_insert_threshold": cfg.parallel_insert_threshold as u32,
+            "tombstone_rebuild_pct": cfg.tombstone_rebuild_pct as u32,
         });
         stringify_value(value)
     }
@@ -459,15 +467,92 @@ impl NativeDatabase {
         ef_construction: Option<u32>,
         ef_search: Option<u32>,
         parallel_insert_threshold: Option<u32>,
+        tombstone_rebuild_pct: Option<u32>,
     ) -> Result<()> {
         let mut database = self.write_open()?;
-        let overrides =
-            merge_index_config(m, ef_construction, ef_search, parallel_insert_threshold)
-                .ok_or_else(|| err("setIndexConfig requires at least one field"))?;
+        let overrides = merge_index_config(
+            m,
+            ef_construction,
+            ef_search,
+            parallel_insert_threshold,
+            tombstone_rebuild_pct,
+        )
+        .ok_or_else(|| err("setIndexConfig requires at least one field"))?;
         let merged = apply_index_overrides(database.index_config(), overrides);
         database.set_index_config(merged).map_err(to_napi_error)
     }
 
+    /// Configure WAL durability. `mode` is one of: `"per_op"` (the default,
+    /// fsync after every insert), `"every_n"` (fsync once every `n` inserts
+    /// — provide `n`), `"on_flush"` (only fsync at flush / compact / close).
+    ///
+    /// Relaxing this knob is the single biggest ingestion throughput lever
+    /// on macOS APFS: `on_flush` typically multiplies throughput by 5–10×
+    /// at the cost of losing un-flushed writes on a crash.
+    #[napi(js_name = "setWalSyncMode")]
+    pub fn set_wal_sync_mode(&self, mode: String, n: Option<u32>) -> Result<()> {
+        let parsed = match mode.to_ascii_lowercase().as_str() {
+            "per_op" | "perop" => WalSyncMode::PerOp,
+            "every_n" | "everyn" => {
+                let n = n.ok_or_else(|| {
+                    err("setWalSyncMode(\"every_n\", ...) requires the second `n` argument")
+                })?;
+                WalSyncMode::EveryN(n as usize)
+            }
+            "on_flush" | "onflush" => WalSyncMode::OnFlush,
+            other => {
+                return Err(err(format!(
+                    "unknown WAL sync mode '{other}' (expected 'per_op', 'every_n', or 'on_flush')"
+                )));
+            }
+        };
+        let mut database = self.write_open()?;
+        database.set_wal_sync_mode(parsed).map_err(to_napi_error)
+    }
+
+    /// Return the current WAL sync mode as a JSON string: either
+    /// `{"mode":"per_op"}`, `{"mode":"every_n","n":64}`, or
+    /// `{"mode":"on_flush"}`.
+    #[napi(js_name = "walSyncMode")]
+    pub fn wal_sync_mode(&self) -> Result<String> {
+        let database = self.read()?;
+        let value = match database.wal_sync_mode() {
+            WalSyncMode::PerOp => json!({"mode": "per_op"}),
+            WalSyncMode::EveryN(n) => json!({"mode": "every_n", "n": n as u32}),
+            WalSyncMode::OnFlush => json!({"mode": "on_flush"}),
+        };
+        stringify_value(value)
+    }
+
+    /// Return `[live, dead]` summed across every HNSW graph (global +
+    /// namespaced). Useful for monitoring when to `compact()`.
+    #[napi(js_name = "tombstoneStats")]
+    pub fn tombstone_stats(&self) -> Result<Vec<u32>> {
+        let database = self.read()?;
+        let (live, dead) = database.tombstone_stats();
+        Ok(vec![live as u32, dead as u32])
+    }
+
+    /// Materialise the contiguous-vector arena. Mirrors every record's
+    /// default dense vector into a single flat `Float32Array`-shaped
+    /// buffer for cache- and SIMD-friendly brute-force / rescoring scans.
+    /// Normally built lazily; call this before a heavy scan to pay the
+    /// build cost up front. Cheap when already fresh.
+    #[napi(js_name = "prepareForScan")]
+    pub fn prepare_for_scan(&self) -> Result<()> {
+        let mut database = self.write_open()?;
+        database.prepare_for_scan();
+        Ok(())
+    }
+
+    /// Number of vectors in the contiguous arena, or `null` if it has
+    /// not been materialised yet in this session.
+    #[napi(js_name = "vectorArenaLen")]
+    pub fn vector_arena_len(&self) -> Result<Option<u32>> {
+        let database = self.read()?;
+        Ok(database.vector_arena_len().map(|n| n as u32))
+    }
+
     #[napi]
     pub fn get(&self, id: String, namespace: Option<String>) -> Result<Option<String>> {
         let record = {
@@ -1009,9 +1094,16 @@ impl NativeDatabase {
         ef_construction: Option<u32>,
         ef_search: Option<u32>,
         parallel_insert_threshold: Option<u32>,
+        tombstone_rebuild_pct: Option<u32>,
     ) -> Result<AsyncTask<BulkIngestTask>> {
         let records = parse_record_batch_json(&records_json, namespace.as_deref())?;
-        let tuning = merge_index_config(m, ef_construction, ef_search, parallel_insert_threshold);
+        let tuning = merge_index_config(
+            m,
+            ef_construction,
+            ef_search,
+            parallel_insert_threshold,
+            tombstone_rebuild_pct,
+        );
         Ok(AsyncTask::new(BulkIngestTask {
             db: self.inner.clone(),
             records,
@@ -2020,9 +2112,10 @@ struct IndexConfigPatch {
     ef_construction: Option<usize>,
     ef_search: Option<usize>,
     parallel_insert_threshold: Option<usize>,
+    tombstone_rebuild_pct: Option<u8>,
 }
 
-/// Pack the four optional HNSW tuning fields into a patch. Returns `None`
+/// Pack the five optional HNSW tuning fields into a patch. Returns `None`
 /// when every field is `None`; explicit zeroes are preserved so core
 /// validation can reject invalid build/search widths instead of treating
 /// them as "not provided".
@@ -2031,11 +2124,13 @@ fn merge_index_config(
     ef_construction: Option<u32>,
     ef_search: Option<u32>,
     parallel_insert_threshold: Option<u32>,
+    tombstone_rebuild_pct: Option<u32>,
 ) -> Option<IndexConfigPatch> {
     if m.is_none()
         && ef_construction.is_none()
         && ef_search.is_none()
         && parallel_insert_threshold.is_none()
+        && tombstone_rebuild_pct.is_none()
     {
         return None;
     }
@@ -2044,6 +2139,7 @@ fn merge_index_config(
         ef_construction: ef_construction.map(|v| v as usize),
         ef_search: ef_search.map(|v| v as usize),
         parallel_insert_threshold: parallel_insert_threshold.map(|v| v as usize),
+        tombstone_rebuild_pct: tombstone_rebuild_pct.map(|v| if v > 100 { 101 } else { v as u8 }),
     })
 }
 
@@ -2058,6 +2154,9 @@ fn apply_index_overrides(current: IndexConfig, patch: IndexConfigPatch) -> Index
         parallel_insert_threshold: patch
             .parallel_insert_threshold
             .unwrap_or(current.parallel_insert_threshold),
+        tombstone_rebuild_pct: patch
+            .tombstone_rebuild_pct
+            .unwrap_or(current.tombstone_rebuild_pct),
     }
 }
 

package/native/vectlite-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectlite-core"
-version = "0.10.0"
+version = "0.11.0"
 edition = "2024"
 license = "MIT"
 description = "Core storage engine for vectlite."