vectlite 0.9.3 → 0.10.0

package/README.md

@@ -68,7 +68,7 @@ db.close()
 ### Data Management
 
 - **Physical collections** -- `vectlite.openStore()` manages a directory of independent databases
-- **Bulk ingestion** -- `bulkIngest()` with deferred index rebuilds for fast imports
+- **Bulk ingestion** -- `bulkIngest()` with Rayon-parallel HNSW build, coalesced WAL fsync, and tunable `m` / `efConstruction` / `efSearch`
 - **Listing & filtered counts** -- `list()` and `count({ namespace, filter })` without a vector query
 - **Delete by filter** -- `deleteByFilter()` for bulk deletion by metadata filter
 - **Partial metadata updates** -- `updateMetadata()` merges a patch without re-writing the vector or rebuilding indexes
@@ -345,6 +345,37 @@ await db.compactAsync()
 const count = await db.bulkIngestAsync(records, { batchSize: 5000 })
 ```
 
+### Tuning the HNSW index
+
+`bulkIngest()` and `bulkIngestAsync()` accept optional HNSW parameters that
+control the recall/latency trade-off and trigger Rayon-backed parallel graph
+construction once the dataset crosses `parallelInsertThreshold` (default 256):
+
+```js
+// Higher recall, slightly slower build/search
+db.bulkIngest(records, {
+  batchSize: 5000,
+  m: 32,              // max bidirectional links per node (default 16)
+  efConstruction: 400, // build-time search width (default 200)
+  efSearch: 200,       // query-time search width (default: auto)
+})
+
+// Faster build/search, lower recall
+db.bulkIngest(records, { m: 8, efConstruction: 100, efSearch: 40 })
+```
+
+The same parameters can be changed at any time without re-ingesting:
+
+```js
+db.setIndexConfig({ m: 32, efConstruction: 400 }) // rebuilds the ANN graph
+db.setEfSearch(200)                               // query-time only, no rebuild
+console.log(db.indexConfig())
+// { m: 32, ef_construction: 400, ef_search: 200, parallel_insert_threshold: 256 }
+```
+
+Use higher `m` / `efConstruction` / `efSearch` to push Recall@10 toward `1.0`;
+use lower values when latency or memory matter more than recall.
+
 ### OpenTelemetry Integration
 
 vectlite ships with optional OpenTelemetry tracing. When enabled, every search
@@ -398,7 +429,10 @@ before re-throwing.
 | `db.insert(id, vector, metadata, options)` | Insert a record (throws on duplicate id) |
 | `db.upsertMany(records, { namespace })` | Upsert a batch of records |
 | `db.insertMany(records, { namespace })` | Insert a batch |
-| `db.bulkIngest(records, { namespace, batchSize })` | Fastest bulk import with batched WAL writes |
+| `db.bulkIngest(records, { namespace, batchSize, m, efConstruction, efSearch, parallelInsertThreshold })` | Fastest bulk import with coalesced WAL fsync and Rayon-parallel HNSW build |
+| `db.setIndexConfig({ m, efConstruction, efSearch, parallelInsertThreshold })` | Update HNSW parameters; rebuilds the ANN graph if `m`/`efConstruction` changed |
+| `db.setEfSearch(efSearch)` | Adjust query-time HNSW search width without rebuilding |
+| `db.indexConfig()` | Return the current HNSW configuration |
 | `db.delete(id, { namespace })` | Delete a single record |
 | `db.deleteMany(ids, { namespace })` | Delete multiple records by id |
 | `db.deleteByFilter(filter, { namespace })` | Delete all records matching a filter |
@@ -462,7 +496,7 @@ before re-throwing.
 | `db.searchWithStatsAsync(query, options)` | Non-blocking search with stats (returns Promise) |
 | `db.flushAsync()` | Non-blocking flush/compact (returns Promise) |
 | `db.compactAsync()` | Non-blocking compact (returns Promise) |
-| `db.bulkIngestAsync(records, options)` | Non-blocking bulk import (returns Promise) |
+| `db.bulkIngestAsync(records, options)` | Non-blocking bulk import (returns Promise); accepts the same HNSW tuning options as `bulkIngest` |
 
 ## Filter Operators
 

package/index.d.ts

@@ -110,6 +110,28 @@ export interface ListCursorResult {
 export interface BulkIngestOptions {
   namespace?: string | null
   batchSize?: number
+  /** Max bidirectional links per HNSW node (default 16). */
+  m?: number | null
+  /** Build-time search width (default 200). Higher = better recall, slower build. */
+  efConstruction?: number | null
+  /** Query-time search width. `null` = auto (derived from k). */
+  efSearch?: number | null
+  /** Minimum dataset size to engage Rayon-parallel HNSW insertion (default 256). */
+  parallelInsertThreshold?: number | null
+}
+
+export interface IndexConfig {
+  m: number
+  ef_construction: number
+  ef_search: number | null
+  parallel_insert_threshold: number
+}
+
+export interface SetIndexConfigOptions {
+  m?: number | null
+  efConstruction?: number | null
+  efSearch?: number | null
+  parallelInsertThreshold?: number | null
 }
 
 export interface SearchOptions {
@@ -212,6 +234,12 @@ export class Database {
   insertMany(records: Record[], options?: { namespace?: string | null }): number
   upsertMany(records: Record[], options?: { namespace?: string | null }): number
   bulkIngest(records: Record[], options?: BulkIngestOptions): number
+  /** Get the current HNSW configuration. */
+  indexConfig(): IndexConfig
+  /** Adjust query-time `ef_search` only (no rebuild). `null` reverts to auto. */
+  setEfSearch(efSearch: number | null): void
+  /** Update HNSW parameters; rebuilds the ANN graph if `m`/`efConstruction` changed. */
+  setIndexConfig(config: SetIndexConfigOptions): void
   get(id: string, options?: { namespace?: string | null }): Record | null
   delete(id: string, options?: { namespace?: string | null }): boolean
   deleteMany(ids: string[], options?: { namespace?: string | null }): number

package/index.js

@@ -415,7 +415,34 @@ class Database {
 
   bulkIngest(records, options = {}) {
     return wrapError(() =>
-      this._native.bulkIngest(encode(records), options.namespace ?? null, options.batchSize ?? 10_000),
+      this._native.bulkIngest(
+        encode(records),
+        options.namespace ?? null,
+        options.batchSize ?? 10_000,
+        options.m ?? null,
+        options.efConstruction ?? null,
+        options.efSearch ?? null,
+        options.parallelInsertThreshold ?? null,
+      ),
+    )
+  }
+
+  indexConfig() {
+    return wrapError(() => decode(this._native.indexConfig()))
+  }
+
+  setEfSearch(efSearch) {
+    return wrapError(() => this._native.setEfSearch(efSearch ?? null))
+  }
+
+  setIndexConfig(config = {}) {
+    return wrapError(() =>
+      this._native.setIndexConfig(
+        config.m ?? null,
+        config.efConstruction ?? null,
+        config.efSearch ?? null,
+        config.parallelInsertThreshold ?? null,
+      ),
     )
   }
 
@@ -585,7 +612,15 @@ class Database {
 
   bulkIngestAsync(records, options = {}) {
     return wrapAsync(
-      this._native.bulkIngestAsync(encode(records), options.namespace ?? null, options.batchSize ?? 10_000),
+      this._native.bulkIngestAsync(
+        encode(records),
+        options.namespace ?? null,
+        options.batchSize ?? 10_000,
+        options.m ?? null,
+        options.efConstruction ?? null,
+        options.efSearch ?? null,
+        options.parallelInsertThreshold ?? null,
+      ),
     )
   }
 }

package/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectlite-node"
-version = "0.9.3"
+version = "0.10.0"
 edition = "2024"
 license = "MIT"
 description = "Node.js bindings for vectlite."

package/native/src/lib.rs

@@ -12,8 +12,8 @@ use vectlite::quantization::{
     default_product_num_sub_vectors,
 };
 use vectlite::{
-    Database as CoreDatabase, DistanceMetric, FusionStrategy, HybridSearchOptions, Metadata,
-    MetadataFilter, MetadataValue, MultiVectorSearchOptions, MultiVectors, NamedVectors,
+    Database as CoreDatabase, DistanceMetric, FusionStrategy, HybridSearchOptions, IndexConfig,
+    Metadata, MetadataFilter, MetadataValue, MultiVectorSearchOptions, MultiVectors, NamedVectors,
     PayloadIndexType, Record, SearchOutcome, SearchResult, SparseVector, Store as CoreStore,
     WriteOperation,
 };
@@ -417,13 +417,55 @@ impl NativeDatabase {
         records_json: String,
         namespace: Option<String>,
         batch_size: u32,
+        m: Option<u32>,
+        ef_construction: Option<u32>,
+        ef_search: Option<u32>,
+        parallel_insert_threshold: Option<u32>,
     ) -> Result<u32> {
         let records = parse_record_batch_json(&records_json, namespace.as_deref())?;
         let mut database = self.write_open()?;
-        database
-            .bulk_ingest(records, batch_size as usize)
-            .map(|count| count as u32)
-            .map_err(to_napi_error)
+        let tuning = merge_index_config(m, ef_construction, ef_search, parallel_insert_threshold);
+        let count = if let Some(cfg) = tuning {
+            let merged = apply_index_overrides(database.index_config(), cfg);
+            database.bulk_ingest_with_config(records, batch_size as usize, Some(merged))
+        } else {
+            database.bulk_ingest(records, batch_size as usize)
+        };
+        count.map(|n| n as u32).map_err(to_napi_error)
+    }
+
+    #[napi(js_name = "indexConfig")]
+    pub fn index_config(&self) -> Result<String> {
+        let cfg = self.read()?.index_config();
+        let value = json!({
+            "m": cfg.m as u32,
+            "ef_construction": cfg.ef_construction as u32,
+            "ef_search": cfg.ef_search.map(|v| v as u32),
+            "parallel_insert_threshold": cfg.parallel_insert_threshold as u32,
+        });
+        stringify_value(value)
+    }
+
+    #[napi(js_name = "setEfSearch")]
+    pub fn set_ef_search(&self, ef_search: Option<u32>) -> Result<()> {
+        let ef = ef_search.map(|v| v as usize);
+        self.write_open()?.set_ef_search(ef).map_err(to_napi_error)
+    }
+
+    #[napi(js_name = "setIndexConfig")]
+    pub fn set_index_config(
+        &self,
+        m: Option<u32>,
+        ef_construction: Option<u32>,
+        ef_search: Option<u32>,
+        parallel_insert_threshold: Option<u32>,
+    ) -> Result<()> {
+        let mut database = self.write_open()?;
+        let overrides =
+            merge_index_config(m, ef_construction, ef_search, parallel_insert_threshold)
+                .ok_or_else(|| err("setIndexConfig requires at least one field"))?;
+        let merged = apply_index_overrides(database.index_config(), overrides);
+        database.set_index_config(merged).map_err(to_napi_error)
     }
 
     #[napi]
@@ -878,6 +920,7 @@ pub struct BulkIngestTask {
     db: Arc<RwLock<CoreDatabase>>,
     records: Vec<Record>,
     batch_size: usize,
+    tuning: Option<IndexConfigPatch>,
 }
 
 impl napi::Task for BulkIngestTask {
@@ -890,10 +933,13 @@ impl napi::Task for BulkIngestTask {
             .db
             .write()
             .map_err(|e| err(format!("lock poisoned: {e}")))?;
-        database
-            .bulk_ingest(records, self.batch_size)
-            .map(|count| count as u32)
-            .map_err(to_napi_error)
+        let res = if let Some(cfg) = self.tuning.clone() {
+            let merged = apply_index_overrides(database.index_config(), cfg);
+            database.bulk_ingest_with_config(records, self.batch_size, Some(merged))
+        } else {
+            database.bulk_ingest(records, self.batch_size)
+        };
+        res.map(|count| count as u32).map_err(to_napi_error)
     }
 
     fn resolve(&mut self, _env: napi::Env, output: Self::Output) -> Result<Self::JsValue> {
@@ -959,12 +1005,18 @@ impl NativeDatabase {
         records_json: String,
         namespace: Option<String>,
         batch_size: u32,
+        m: Option<u32>,
+        ef_construction: Option<u32>,
+        ef_search: Option<u32>,
+        parallel_insert_threshold: Option<u32>,
     ) -> Result<AsyncTask<BulkIngestTask>> {
         let records = parse_record_batch_json(&records_json, namespace.as_deref())?;
+        let tuning = merge_index_config(m, ef_construction, ef_search, parallel_insert_threshold);
         Ok(AsyncTask::new(BulkIngestTask {
             db: self.inner.clone(),
             records,
             batch_size: batch_size as usize,
+            tuning,
         }))
     }
 }
@@ -1962,6 +2014,53 @@ fn value_to_usize(value: &Value, label: &str) -> Result<usize> {
         .ok_or_else(|| err(format!("{label} must be an unsigned integer")))
 }
 
+#[derive(Clone, Copy)]
+struct IndexConfigPatch {
+    m: Option<usize>,
+    ef_construction: Option<usize>,
+    ef_search: Option<usize>,
+    parallel_insert_threshold: Option<usize>,
+}
+
+/// Pack the four optional HNSW tuning fields into a patch. Returns `None`
+/// when every field is `None`; explicit zeroes are preserved so core
+/// validation can reject invalid build/search widths instead of treating
+/// them as "not provided".
+fn merge_index_config(
+    m: Option<u32>,
+    ef_construction: Option<u32>,
+    ef_search: Option<u32>,
+    parallel_insert_threshold: Option<u32>,
+) -> Option<IndexConfigPatch> {
+    if m.is_none()
+        && ef_construction.is_none()
+        && ef_search.is_none()
+        && parallel_insert_threshold.is_none()
+    {
+        return None;
+    }
+    Some(IndexConfigPatch {
+        m: m.map(|v| v as usize),
+        ef_construction: ef_construction.map(|v| v as usize),
+        ef_search: ef_search.map(|v| v as usize),
+        parallel_insert_threshold: parallel_insert_threshold.map(|v| v as usize),
+    })
+}
+
+/// Merge a tuning patch into the current `IndexConfig`. Omitted fields inherit
+/// from `current`; `ef_search = None` in the patch means "no change" because
+/// callers use `setEfSearch(null)` to reset query-time tuning to auto.
+fn apply_index_overrides(current: IndexConfig, patch: IndexConfigPatch) -> IndexConfig {
+    IndexConfig {
+        m: patch.m.unwrap_or(current.m),
+        ef_construction: patch.ef_construction.unwrap_or(current.ef_construction),
+        ef_search: patch.ef_search.or(current.ef_search),
+        parallel_insert_threshold: patch
+            .parallel_insert_threshold
+            .unwrap_or(current.parallel_insert_threshold),
+    }
+}
+
 fn err(message: impl Into<String>) -> NapiError {
     NapiError::from_reason(message.into())
 }

package/native/vectlite-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectlite-core"
-version = "0.9.3"
+version = "0.10.0"
 edition = "2024"
 license = "MIT"
 description = "Core storage engine for vectlite."

package/native/vectlite-core/src/lib.rs

@@ -35,6 +35,10 @@ const ANN_OVERSAMPLE: usize = 8;
 const ANN_MIN_CANDIDATES: usize = 64;
 const ANN_M: usize = 16;
 const ANN_EF_CONSTRUCTION: usize = 200;
+/// Threshold above which HNSW construction uses parallel batch insert
+/// (Rayon-based). Below this, sequential insert is cheaper because of
+/// thread setup overhead.
+const ANN_PARALLEL_INSERT_THRESHOLD: usize = 256;
 const BM25_K1: f32 = 1.2;
 const BM25_B: f32 = 0.75;
 
@@ -754,6 +758,87 @@ pub struct SearchOptions {
     pub truncate_dim: Option<usize>,
 }
 
+/// HNSW tuning parameters. Exposed so callers can trade off recall, latency,
+/// memory and build time.
+///
+/// Defaults mirror VectLite's historical built-in values (`m = 16`,
+/// `ef_construction = 200`). `ef_search = None` means VectLite picks an
+/// `ef_search` derived from `top_k * ANN_OVERSAMPLE`.
+///
+/// Reference: Malkov & Yashunin, *Efficient and robust approximate nearest
+/// neighbor search using Hierarchical Navigable Small World graphs*.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct IndexConfig {
+    /// Max number of bidirectional links per node. Higher = better recall,
+    /// more memory, slower build. Typical range: 8..64.
+    pub m: usize,
+    /// Width of the search during graph construction. Higher = better recall,
+    /// slower build. Typical range: 64..800.
+    pub ef_construction: usize,
+    /// Width of the search at query time. None = auto (derived from top_k).
+    /// Higher = better recall, slower search.
+    pub ef_search: Option<usize>,
+    /// Use parallel (Rayon-backed) HNSW insertion when the dataset has at
+    /// least this many vectors. Defaults to `ANN_PARALLEL_INSERT_THRESHOLD`.
+    /// Set very high to disable parallel insert.
+    pub parallel_insert_threshold: usize,
+}
+
+impl Default for IndexConfig {
+    fn default() -> Self {
+        Self {
+            m: ANN_M,
+            ef_construction: ANN_EF_CONSTRUCTION,
+            ef_search: None,
+            parallel_insert_threshold: ANN_PARALLEL_INSERT_THRESHOLD,
+        }
+    }
+}
+
+impl IndexConfig {
+    /// A preset tuned for higher recall at the cost of build/search time.
+    /// Useful for benchmark comparisons where recall@10 must approach 1.0.
+    pub fn high_recall() -> Self {
+        Self {
+            m: 32,
+            ef_construction: 400,
+            ef_search: Some(200),
+            parallel_insert_threshold: ANN_PARALLEL_INSERT_THRESHOLD,
+        }
+    }
+
+    /// A preset tuned for fast build & low latency, lower recall.
+    pub fn fast() -> Self {
+        Self {
+            m: 8,
+            ef_construction: 100,
+            ef_search: Some(40),
+            parallel_insert_threshold: ANN_PARALLEL_INSERT_THRESHOLD,
+        }
+    }
+
+    fn validate(&self) -> Result<()> {
+        if self.m == 0 {
+            return Err(VectLiteError::InvalidFormat(
+                "IndexConfig.m must be >= 1".to_owned(),
+            ));
+        }
+        if self.ef_construction == 0 {
+            return Err(VectLiteError::InvalidFormat(
+                "IndexConfig.ef_construction must be >= 1".to_owned(),
+            ));
+        }
+        if let Some(ef) = self.ef_search {
+            if ef == 0 {
+                return Err(VectLiteError::InvalidFormat(
+                    "IndexConfig.ef_search must be >= 1 when set".to_owned(),
+                ));
+            }
+        }
+        Ok(())
+    }
+}
+
 impl Default for SearchOptions {
     fn default() -> Self {
         Self {
@@ -1230,6 +1315,10 @@ pub struct Database {
     payload_index_defs: BTreeMap<String, PayloadIndexType>,
     /// Live payload indexes, populated from records.
     payload_indexes: BTreeMap<String, PayloadIndexData>,
+    /// HNSW tuning parameters. Not persisted to disk: this is a per-session
+    /// knob so callers can change recall/latency tradeoffs without migrating
+    /// data files. A subsequent `set_index_config` triggers a rebuild.
+    index_config: IndexConfig,
 }
 
 #[derive(Default)]
@@ -1327,6 +1416,7 @@ impl Database {
             multi_vector_quantized_keys: BTreeMap::new(),
             payload_index_defs: BTreeMap::new(),
             payload_indexes: BTreeMap::new(),
+            index_config: IndexConfig::default(),
         };
 
         database.flush()?;
@@ -2487,11 +2577,41 @@ impl Database {
     /// `batch_size`, but the ANN index and sparse index are only rebuilt once
     /// at the very end, making this much faster than `upsert_many` for large
     /// imports.
+    ///
+    /// Performance notes:
+    /// - The WAL is written without a per-batch `fsync` (each batch goes
+    ///   through `BufWriter` and is appended to the open file). A single
+    ///   `sync_all` is issued at the end. This avoids the per-batch fsync
+    ///   tax that dominates ingestion latency on macOS and modern SSDs.
+    /// - The final ANN rebuild uses parallel HNSW insertion (Rayon) when
+    ///   the dataset is large enough (see
+    ///   `IndexConfig.parallel_insert_threshold`).
     pub fn bulk_ingest<I>(&mut self, records: I, batch_size: usize) -> Result<usize>
+    where
+        I: IntoIterator<Item = Record>,
+    {
+        self.bulk_ingest_with_config(records, batch_size, None)
+    }
+
+    /// Bulk-ingest with an override for the HNSW index configuration. The
+    /// override is applied for the rebuild step at the end, so the resulting
+    /// graph uses the requested `m` / `ef_construction`. The new config is
+    /// also stored on the database (so subsequent searches use the
+    /// corresponding `ef_search`).
+    pub fn bulk_ingest_with_config<I>(
+        &mut self,
+        records: I,
+        batch_size: usize,
+        config: Option<IndexConfig>,
+    ) -> Result<usize>
     where
         I: IntoIterator<Item = Record>,
     {
         self.check_writable()?;
+        if let Some(cfg) = config {
+            cfg.validate()?;
+            self.index_config = cfg;
+        }
         let batch_size = batch_size.max(1);
         let mut total = 0_usize;
         let mut batch = Vec::with_capacity(batch_size);
@@ -2502,7 +2622,8 @@ impl Database {
 
             if batch.len() >= batch_size {
                 total += batch.len();
-                self.append_wal_batch(&batch)?;
+                // Coalesced WAL writes: append without per-batch fsync.
+                self.append_wal_batch_unsynced(&batch)?;
                 self.apply_ops_in_memory(batch);
                 batch = Vec::with_capacity(batch_size);
             }
@@ -2510,11 +2631,16 @@ impl Database {
 
         if !batch.is_empty() {
             total += batch.len();
-            self.append_wal_batch(&batch)?;
+            self.append_wal_batch_unsynced(&batch)?;
             self.apply_ops_in_memory(batch);
         }
 
         if total > 0 {
+            // Single fsync at the very end to make all batches durable in
+            // one shot. This is the major ingestion optimisation: instead
+            // of paying fsync per batch (every `batch_size` records) we pay
+            // it once for the whole bulk_ingest call.
+            self.sync_wal()?;
             self.rebuild_sparse_index();
             self.rebuild_ann();
             self.ann_loaded_from_disk = false;
@@ -2526,6 +2652,42 @@ impl Database {
         Ok(total)
     }
 
+    /// Replace the HNSW tuning parameters and rebuild the ANN index.
+    /// Use this to trade off recall vs latency without re-ingesting data.
+    pub fn set_index_config(&mut self, config: IndexConfig) -> Result<()> {
+        self.check_writable()?;
+        config.validate()?;
+        let changed_build_params = self.index_config.m != config.m
+            || self.index_config.ef_construction != config.ef_construction;
+        self.index_config = config;
+        if changed_build_params {
+            // m / ef_construction affect graph structure → full rebuild.
+            self.rebuild_ann();
+            self.ann_loaded_from_disk = false;
+            self.persist_ann_to_disk()?;
+        }
+        Ok(())
+    }
+
+    /// Return the current HNSW tuning parameters.
+    pub fn index_config(&self) -> IndexConfig {
+        self.index_config
+    }
+
+    /// Convenience: update only the query-time `ef_search` without rebuilding
+    /// the index. Higher = better recall, slower search.
+    pub fn set_ef_search(&mut self, ef_search: Option<usize>) -> Result<()> {
+        if let Some(ef) = ef_search {
+            if ef == 0 {
+                return Err(VectLiteError::InvalidFormat(
+                    "ef_search must be >= 1".to_owned(),
+                ));
+            }
+        }
+        self.index_config.ef_search = ef_search;
+        Ok(())
+    }
+
     pub fn compact(&mut self) -> Result<()> {
         self.check_writable()?;
         self.compact_inner()
@@ -3498,6 +3660,17 @@ impl Database {
     }
 
     fn append_wal_batch(&self, ops: &[WalOp]) -> Result<()> {
+        self.append_wal_batch_inner(ops, true)
+    }
+
+    /// Append a WAL batch without issuing an fsync. The caller is responsible
+    /// for issuing `sync_wal` later (typically once at the end of a bulk
+    /// ingest). This is the hot path for `bulk_ingest`.
+    fn append_wal_batch_unsynced(&self, ops: &[WalOp]) -> Result<()> {
+        self.append_wal_batch_inner(ops, false)
+    }
+
+    fn append_wal_batch_inner(&self, ops: &[WalOp], sync: bool) -> Result<()> {
         if let Some(parent) = self.wal_path.parent() {
             if !parent.as_os_str().is_empty() {
                 fs::create_dir_all(parent)?;
@@ -3522,6 +3695,21 @@ impl Database {
 
         write_u32(&mut file, u32_from_usize(buffer.len())?)?;
         file.write_all(&buffer)?;
+        if sync {
+            file.sync_all()?;
+        }
+        Ok(())
+    }
+
+    /// Force a durability fence on the WAL file. Opens the file in append
+    /// mode and calls `sync_all`, which makes all previous unsynced writes
+    /// durable in one shot. This is used by `bulk_ingest` to amortise fsync
+    /// cost across many batches.
+    fn sync_wal(&self) -> Result<()> {
+        if !self.wal_path.exists() {
+            return Ok(());
+        }
+        let file = OpenOptions::new().append(true).open(&self.wal_path)?;
         file.sync_all()?;
         Ok(())
     }
@@ -3696,6 +3884,7 @@ impl Database {
             multi_vector_quantized_keys: BTreeMap::new(),
             payload_index_defs: BTreeMap::new(),
             payload_indexes: BTreeMap::new(),
+            index_config: IndexConfig::default(),
         })
     }
 
@@ -3854,13 +4043,14 @@ impl Database {
             }
         }
 
+        let cfg = self.index_config;
         self.ann.global = global_by_vector
             .into_iter()
             .filter_map(|(vector_name, records)| {
                 if records.len() < ANN_MIN_POINTS {
                     None
                 } else {
-                    Some((vector_name, build_ann_index(records, self.metric)))
+                    Some((vector_name, build_ann_index(records, self.metric, &cfg)))
                 }
             })
             .collect();
@@ -3874,7 +4064,7 @@ impl Database {
                         if records.len() < ANN_MIN_POINTS {
                             None
                         } else {
-                            Some((vector_name, build_ann_index(records, self.metric)))
+                            Some((vector_name, build_ann_index(records, self.metric, &cfg)))
                         }
                     })
                     .collect::<BTreeMap<_, _>>();
@@ -4205,7 +4395,14 @@ impl Database {
             return None;
         }
 
-        let ef_search = candidate_count.max(ANN_EF_CONSTRUCTION);
+        // ef_search controls recall vs latency at query time. When the user
+        // explicitly sets `IndexConfig.ef_search`, honour it directly.
+        // Otherwise default to max(candidate_count, ef_construction) which is
+        // a conservative high-recall heuristic.
+        let ef_search = match self.index_config.ef_search {
+            Some(ef) => ef.max(candidate_count),
+            None => candidate_count.max(self.index_config.ef_construction),
+        };
         let neighbours = index.hnsw.search(query, candidate_count, ef_search);
         Some(
             neighbours
@@ -4475,23 +4672,41 @@ fn score_dense_prefix(
     metric.score(&left[..dimension], &right[..dimension])
 }
 
-fn build_ann_index(records: Vec<(RecordKey, &Vec<f32>)>, metric: DistanceMetric) -> AnnIndex {
+fn build_ann_index(
+    records: Vec<(RecordKey, &Vec<f32>)>,
+    metric: DistanceMetric,
+    config: &IndexConfig,
+) -> AnnIndex {
     let max_layer = compute_hnsw_layers(records.len());
     let count = records.len();
+    let use_parallel = count >= config.parallel_insert_threshold;
 
     macro_rules! build_hnsw {
         ($dist_type:ty, $dist_val:expr, $variant:ident) => {{
             let mut hnsw = Hnsw::<f32, $dist_type>::new(
-                ANN_M,
-                count,
+                config.m,
+                count.max(1),
                 max_layer,
-                ANN_EF_CONSTRUCTION,
+                config.ef_construction,
                 $dist_val,
             );
             let mut keys = Vec::with_capacity(count);
-            for (origin_id, (key, vector)) in records.into_iter().enumerate() {
-                hnsw.insert((vector.as_slice(), origin_id));
-                keys.push(key);
+            if use_parallel {
+                // hnsw_rs's `parallel_insert` takes `&[(&Vec<T>, usize)]`
+                // (the API is built around owned-Vec borrows) and uses Rayon
+                // internally so the dominant cost (distance calculations
+                // during graph neighbour selection) is multi-threaded.
+                let mut batch: Vec<(&Vec<f32>, usize)> = Vec::with_capacity(count);
+                for (origin_id, (key, vector)) in records.into_iter().enumerate() {
+                    batch.push((vector, origin_id));
+                    keys.push(key);
+                }
+                hnsw.parallel_insert(&batch);
+            } else {
+                for (origin_id, (key, vector)) in records.into_iter().enumerate() {
+                    hnsw.insert((vector.as_slice(), origin_id));
+                    keys.push(key);
+                }
             }
             hnsw.set_searching_mode(true);
             AnnIndex {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectlite",
-  "version": "0.9.3",
+  "version": "0.10.0",
   "description": "Embedded vector store for local-first AI applications.",
   "main": "index.js",
   "types": "index.d.ts",