vectlite 0.1.8 → 0.1.12

package/native/vectlite-core/src/lib.rs

@@ -1,3 +1,5 @@
+pub mod quantization;
+
 use std::collections::{BTreeMap, BTreeSet};
 use std::error::Error as StdError;
 use std::fmt;
@@ -9,6 +11,8 @@ use std::time::Instant;
 use fs2::FileExt;
 use hnsw_rs::prelude::*;
 
+use quantization::{QuantizationConfig, QuantizedIndex};
+
 const MAGIC: &[u8; 4] = b"VDB1";
 const VERSION: u16 = 4;
 const WAL_MAGIC: &[u8; 4] = b"VWL1";
@@ -81,7 +85,9 @@ impl fmt::Display for VectLiteError {
             Self::DimensionMismatch { expected, found } => {
                 write!(
                     f,
-                    "vector dimension mismatch: expected {expected}, found {found}"
+                    "vector dimension mismatch: expected {expected}, found {found}. \
+                     If you changed embedding models, delete the existing .vdb file \
+                     or use a different path to create a new database with dimension {found}"
                 )
             }
             Self::DuplicateId { namespace, id } => {
@@ -685,6 +691,8 @@ impl Store {
         let _ = fs::remove_file(&wal);
         let manifest = ann_manifest_path(&path);
         let _ = fs::remove_file(&manifest);
+        let quant = quantization_params_path(&path);
+        let _ = fs::remove_file(&quant);
         // Remove any .hnsw.* sidecar files
         if let Some(parent) = path.parent() {
             if let Some(stem) = path.file_name().and_then(|n| n.to_str()) {
@@ -736,6 +744,12 @@ pub struct Database {
     /// Holds the lock file open for the lifetime of the database.
     /// Dropping this releases the advisory lock.
     _lock_file: Option<File>,
+    /// Optional quantized index for accelerated search.
+    quantized: Option<QuantizedIndex>,
+    /// Configuration used to build the quantized index (persisted).
+    quantization_config: Option<QuantizationConfig>,
+    /// Ordered keys mapping quantized index positions to record keys.
+    quantized_keys: Vec<RecordKey>,
 }
 
 #[derive(Default)]
@@ -786,6 +800,9 @@ impl Database {
             ann_loaded_from_disk: false,
             read_only: false,
             _lock_file: Some(lock),
+            quantized: None,
+            quantization_config: None,
+            quantized_keys: Vec::new(),
         };
 
         database.flush()?;
@@ -805,6 +822,28 @@ impl Database {
         if !database.ann_loaded_from_disk {
             database.rebuild_ann();
         }
+        database.try_load_quantization();
+        Ok(database)
+    }
+
+    /// Open an existing database with a lock timeout in seconds.
+    /// If the lock cannot be acquired within the timeout, returns
+    /// `VectLiteError::LockContention`.
+    pub fn open_with_timeout(path: impl AsRef<Path>, timeout_secs: f64) -> Result<Self> {
+        let path = path.as_ref().to_path_buf();
+        let timeout = timeout_duration(timeout_secs, "lock_timeout")?;
+        let lock = acquire_exclusive_lock_with_timeout(&path, Some(timeout))?;
+        let mut file = File::open(&path)?;
+        let mut database = Self::read_from(&path, &mut file)?;
+        database._lock_file = Some(lock);
+        database.read_only = false;
+        database.replay_wal()?;
+        database.rebuild_sparse_index();
+        database.ann_loaded_from_disk = database.try_load_ann_from_disk();
+        if !database.ann_loaded_from_disk {
+            database.rebuild_ann();
+        }
+        database.try_load_quantization();
         Ok(database)
     }
 
@@ -812,8 +851,20 @@ impl Database {
     /// so multiple readers can coexist. All write operations will return
     /// `VectLiteError::ReadOnly`.
     pub fn open_read_only(path: impl AsRef<Path>) -> Result<Self> {
+        Self::open_read_only_with_timeout(path, None)
+    }
+
+    /// Open an existing database in read-only mode, optionally waiting for a
+    /// shared lock to become available.
+    pub fn open_read_only_with_timeout(
+        path: impl AsRef<Path>,
+        timeout_secs: Option<f64>,
+    ) -> Result<Self> {
         let path = path.as_ref().to_path_buf();
-        let lock = acquire_shared_lock(&path)?;
+        let timeout = timeout_secs
+            .map(|seconds| timeout_duration(seconds, "lock_timeout"))
+            .transpose()?;
+        let lock = acquire_shared_lock_with_timeout(&path, timeout)?;
         let mut file = File::open(&path)?;
         let mut database = Self::read_from(&path, &mut file)?;
         database._lock_file = Some(lock);
@@ -824,6 +875,7 @@ impl Database {
         if !database.ann_loaded_from_disk {
             database.rebuild_ann();
         }
+        database.try_load_quantization();
         Ok(database)
     }
 
@@ -831,7 +883,46 @@ impl Database {
         self.read_only
     }
 
+    /// Returns true if the database has been closed.
+    pub fn is_closed(&self) -> bool {
+        self._lock_file.is_none() && self.records.is_empty() && self.dimension == 0
+    }
+
+    /// Close the database: flush WAL (if writable), release the file lock,
+    /// and clear all in-memory state.  After calling this, any further
+    /// operation will return an error.
+    pub fn close(&mut self) -> Result<()> {
+        if self.is_closed() {
+            return Ok(());
+        }
+        // Flush WAL to main file if writable
+        if !self.read_only {
+            self.compact_inner()?;
+        }
+        // Release the lock by dropping the file handle
+        self._lock_file = None;
+        // Clear in-memory state
+        self.records.clear();
+        self.ann = AnnCatalog::default();
+        self.sparse_index = SparseIndex::default();
+        self.quantized = None;
+        self.quantization_config = None;
+        self.quantized_keys.clear();
+        self.dimension = 0;
+        Ok(())
+    }
+
+    fn check_open(&self) -> Result<()> {
+        if self.is_closed() {
+            return Err(VectLiteError::InvalidFormat(
+                "database is closed".to_owned(),
+            ));
+        }
+        Ok(())
+    }
+
     fn check_writable(&self) -> Result<()> {
+        self.check_open()?;
         if self.read_only {
             return Err(VectLiteError::ReadOnly);
         }
@@ -869,6 +960,95 @@ impl Database {
         self.records.is_empty()
     }
 
+    /// Count records, optionally filtered by namespace and/or metadata filter.
+    pub fn count_filtered(
+        &self,
+        namespace: Option<&str>,
+        filter: Option<&MetadataFilter>,
+    ) -> usize {
+        self.records
+            .iter()
+            .filter(|((ns, _), record)| {
+                if let Some(target_ns) = namespace {
+                    if ns != target_ns {
+                        return false;
+                    }
+                }
+                if let Some(filter) = filter {
+                    if !filter.matches(&record.metadata) {
+                        return false;
+                    }
+                }
+                true
+            })
+            .count()
+    }
+
+    /// List records by namespace and/or metadata filter without requiring a
+    /// vector query.  Returns records ordered by (namespace, id).
+    pub fn list(
+        &self,
+        namespace: Option<&str>,
+        filter: Option<&MetadataFilter>,
+        limit: usize,
+        offset: usize,
+    ) -> Vec<&Record> {
+        self.records
+            .iter()
+            .filter(|((ns, _), record)| {
+                if let Some(target_ns) = namespace {
+                    if ns != target_ns {
+                        return false;
+                    }
+                }
+                if let Some(filter) = filter {
+                    if !filter.matches(&record.metadata) {
+                        return false;
+                    }
+                }
+                true
+            })
+            .skip(offset)
+            .take(if limit == 0 { usize::MAX } else { limit })
+            .map(|(_, record)| record)
+            .collect()
+    }
+
+    /// Delete all records matching a filter, optionally within a namespace.
+    /// Returns the number of records deleted.
+    pub fn delete_by_filter(
+        &mut self,
+        namespace: Option<&str>,
+        filter: &MetadataFilter,
+    ) -> Result<usize> {
+        self.check_writable()?;
+        let keys_to_delete: Vec<(String, String)> = self
+            .records
+            .iter()
+            .filter(|((ns, _), record)| {
+                if let Some(target_ns) = namespace {
+                    if ns != target_ns {
+                        return false;
+                    }
+                }
+                filter.matches(&record.metadata)
+            })
+            .map(|(key, _)| key.clone())
+            .collect();
+
+        let count = keys_to_delete.len();
+        if count == 0 {
+            return Ok(0);
+        }
+
+        let ops: Vec<WalOp> = keys_to_delete
+            .into_iter()
+            .map(|(namespace, id)| WalOp::Delete { namespace, id })
+            .collect();
+        self.apply_wal_batch(ops)?;
+        Ok(count)
+    }
+
     pub fn insert(
         &mut self,
         id: impl Into<String>,
@@ -1026,7 +1206,12 @@ impl Database {
             return Ok(0);
         }
 
-        self.apply_wal_batch(records.into_iter().map(WalOp::Upsert).collect())?;
+        self.apply_wal_batch_deferred(records.into_iter().map(WalOp::Upsert).collect())?;
+        self.rebuild_sparse_index();
+        self.rebuild_ann();
+        self.ann_loaded_from_disk = false;
+        self.persist_ann_to_disk()?;
+        self.rebuild_quantized_index();
         Ok(count)
     }
 
@@ -1048,7 +1233,12 @@ impl Database {
             return Ok(0);
         }
 
-        self.apply_wal_batch(records.into_iter().map(WalOp::Upsert).collect())?;
+        self.apply_wal_batch_deferred(records.into_iter().map(WalOp::Upsert).collect())?;
+        self.rebuild_sparse_index();
+        self.rebuild_ann();
+        self.ann_loaded_from_disk = false;
+        self.persist_ann_to_disk()?;
+        self.rebuild_quantized_index();
         Ok(count)
     }
 
@@ -1239,7 +1429,16 @@ impl Database {
                 WriteOperation::Delete { namespace, id } => Ok(WalOp::Delete { namespace, id }),
             })
             .collect::<Result<Vec<_>>>()?;
-        self.apply_wal_batch(ops)
+        if ops.is_empty() {
+            return Ok(());
+        }
+        self.apply_wal_batch_deferred(ops)?;
+        self.rebuild_sparse_index();
+        self.rebuild_ann();
+        self.ann_loaded_from_disk = false;
+        self.persist_ann_to_disk()?;
+        self.rebuild_quantized_index();
+        Ok(())
     }
 
     fn hybrid_search_internal(
@@ -1249,6 +1448,7 @@ impl Database {
         options: HybridSearchOptions,
         namespace: Option<&str>,
     ) -> Result<SearchOutcome> {
+        self.check_open()?;
         if let Some(query) = dense_query {
             self.validate_vector(query)?;
         }
@@ -1288,8 +1488,22 @@ impl Database {
         let vector_name = options.vector_name.as_deref();
 
         let dense_start = Instant::now();
-        let ann_candidates = dense_query
-            .and_then(|query| self.ann_candidate_keys(namespace, vector_name, query, fetch_k));
+        // Use quantized index for candidate selection if available (2-stage pipeline).
+        // The quantized index operates on the default vector only and globally (not per-namespace).
+        let quantized_candidates =
+            if vector_name.is_none() || vector_name == Some(DEFAULT_VECTOR_NAME) {
+                dense_query.and_then(|query| self.quantized_candidate_keys(query, fetch_k))
+            } else {
+                None
+            };
+        let ann_candidates = if quantized_candidates.is_some() {
+            // Skip HNSW if quantized index provided candidates
+            None
+        } else {
+            dense_query
+                .and_then(|query| self.ann_candidate_keys(namespace, vector_name, query, fetch_k))
+        };
+        let effective_dense_candidates = quantized_candidates.or(ann_candidates);
         let dense_us = dense_start.elapsed().as_micros() as u64;
 
         let sparse_start = Instant::now();
@@ -1298,17 +1512,19 @@ impl Database {
             .unwrap_or_default();
         let sparse_us = sparse_start.elapsed().as_micros() as u64;
 
-        let candidate_keys = if dense_query.is_some() && ann_candidates.is_none() {
+        let candidate_keys = if dense_query.is_none() {
+            Some(sparse_candidates.clone())
+        } else if dense_query.is_some() && effective_dense_candidates.is_none() {
             None
         } else {
             merge_candidate_keys(
-                ann_candidates.as_deref(),
+                effective_dense_candidates.as_deref(),
                 Some(sparse_candidates.as_slice()),
             )
         };
         let mut stats = SearchStats {
-            used_ann: ann_candidates.is_some(),
-            ann_candidate_count: ann_candidates.as_ref().map_or(0, Vec::len),
+            used_ann: effective_dense_candidates.is_some(),
+            ann_candidate_count: effective_dense_candidates.as_ref().map_or(0, Vec::len),
             fetch_k,
             sparse_candidate_count: sparse_candidates.len(),
             ann_loaded_from_disk: self.ann_loaded_from_disk,
@@ -1326,7 +1542,7 @@ impl Database {
         );
         stats.considered_count = results.len();
 
-        if ann_candidates.is_some() && results.len() < fetch_k {
+        if effective_dense_candidates.is_some() && results.len() < fetch_k {
             stats.exact_fallback = true;
             results = self.collect_results(dense_query, sparse_query, &options, namespace, None);
             stats.considered_count = results.len();
@@ -1418,6 +1634,7 @@ impl Database {
             self.rebuild_ann();
             self.ann_loaded_from_disk = false;
             self.persist_ann_to_disk()?;
+            self.rebuild_quantized_index();
         }
 
         Ok(total)
@@ -1428,6 +1645,142 @@ impl Database {
         self.compact_inner()
     }
 
+    // -----------------------------------------------------------------------
+    // Quantization API
+    // -----------------------------------------------------------------------
+
+    /// Enable quantization on this database. Trains the quantizer on all current
+    /// vectors and persists the configuration. Subsequent searches will use the
+    /// quantized index for fast candidate selection followed by exact rescoring.
+    pub fn enable_quantization(&mut self, config: QuantizationConfig) -> Result<()> {
+        self.check_writable()?;
+        if self.records.is_empty() {
+            return Err(VectLiteError::InvalidFormat(
+                "cannot enable quantization on an empty database".to_owned(),
+            ));
+        }
+        self.quantization_config = Some(config);
+        self.rebuild_quantized_index();
+        self.persist_quantization_params()?;
+        Ok(())
+    }
+
+    /// Disable quantization and remove persisted parameters.
+    pub fn disable_quantization(&mut self) -> Result<()> {
+        self.check_writable()?;
+        self.quantized = None;
+        self.quantization_config = None;
+        self.quantized_keys.clear();
+        // Remove the sidecar file
+        let params_path = quantization_params_path(&self.path);
+        if params_path.exists() {
+            fs::remove_file(&params_path)?;
+        }
+        Ok(())
+    }
+
+    /// Returns true if quantization is enabled.
+    pub fn is_quantized(&self) -> bool {
+        self.quantized.is_some()
+    }
+
+    /// Returns the quantization configuration if enabled.
+    pub fn quantization_config(&self) -> Option<&QuantizationConfig> {
+        self.quantization_config.as_ref()
+    }
+
+    /// Rebuild the quantized index from current records.
+    fn rebuild_quantized_index(&mut self) {
+        let config = match &self.quantization_config {
+            Some(config) => config.clone(),
+            None => return,
+        };
+
+        if self.records.is_empty() {
+            self.quantized = None;
+            self.quantized_keys.clear();
+            return;
+        }
+
+        let mut keys = Vec::with_capacity(self.records.len());
+        let mut vectors: Vec<Vec<f32>> = Vec::with_capacity(self.records.len());
+
+        for (key, record) in &self.records {
+            keys.push(key.clone());
+            vectors.push(record.vector.clone());
+        }
+
+        let refs: Vec<&[f32]> = vectors.iter().map(Vec::as_slice).collect();
+        let index = QuantizedIndex::build(&refs, self.dimension, &config);
+
+        self.quantized = Some(index);
+        self.quantized_keys = keys;
+    }
+
+    /// Persist quantization parameters to a sidecar file.
+    fn persist_quantization_params(&self) -> Result<()> {
+        let params_path = quantization_params_path(&self.path);
+        if let Some(index) = &self.quantized {
+            let mut file = File::create(&params_path)?;
+            index.write_params(&mut file).map_err(VectLiteError::Io)?;
+            file.sync_all()?;
+        } else {
+            if params_path.exists() {
+                fs::remove_file(&params_path)?;
+            }
+        }
+        Ok(())
+    }
+
+    /// Try to load quantization parameters from disk and rebuild codes.
+    fn try_load_quantization(&mut self) -> bool {
+        let params_path = quantization_params_path(&self.path);
+        if !params_path.exists() {
+            return false;
+        }
+
+        let file = match File::open(&params_path) {
+            Ok(f) => f,
+            Err(_) => return false,
+        };
+        let mut reader = BufReader::new(file);
+        let mut index = match QuantizedIndex::read_params(&mut reader) {
+            Ok(idx) => idx,
+            Err(_) => return false,
+        };
+
+        // Rebuild codes from current records
+        let mut keys = Vec::with_capacity(self.records.len());
+        let mut vectors: Vec<Vec<f32>> = Vec::with_capacity(self.records.len());
+        for (key, record) in &self.records {
+            keys.push(key.clone());
+            vectors.push(record.vector.clone());
+        }
+        let refs: Vec<&[f32]> = vectors.iter().map(Vec::as_slice).collect();
+        index.rebuild_codes(&refs);
+
+        self.quantization_config = Some(index.config());
+        self.quantized = Some(index);
+        self.quantized_keys = keys;
+        true
+    }
+
+    /// Use the quantized index to get candidate record keys for rescoring.
+    fn quantized_candidate_keys(&self, query: &[f32], top_k: usize) -> Option<Vec<RecordKey>> {
+        let index = self.quantized.as_ref()?;
+        if index.count() == 0 {
+            return None;
+        }
+
+        let candidate_indices = index.search_candidates(query, top_k);
+        Some(
+            candidate_indices
+                .into_iter()
+                .filter_map(|idx| self.quantized_keys.get(idx).cloned())
+                .collect(),
+        )
+    }
+
     fn compact_inner(&mut self) -> Result<()> {
         if let Some(parent) = self.path.parent() {
             if !parent.as_os_str().is_empty() {
@@ -1459,6 +1812,7 @@ impl Database {
     /// self-contained `.vdb` file (WAL is folded in). The current database is
     /// not modified. Works in both read-only and read-write mode.
     pub fn snapshot(&self, dest: impl AsRef<Path>) -> Result<()> {
+        self.check_open()?;
         let dest = dest.as_ref();
         if let Some(parent) = dest.parent() {
             if !parent.as_os_str().is_empty() {
@@ -1479,6 +1833,7 @@ impl Database {
     /// including the `.vdb` file and ANN sidecar files. The backup is
     /// compacted (WAL folded in). Works in both read-only and read-write mode.
     pub fn backup(&self, dest: impl AsRef<Path>) -> Result<()> {
+        self.check_open()?;
         let dest = dest.as_ref();
         fs::create_dir_all(dest)?;
 
@@ -1562,12 +1917,43 @@ impl Database {
             return Ok(());
         }
 
+        let has_sparse = ops.iter().any(|op| match op {
+            WalOp::Upsert(record) => {
+                !record.sparse.is_empty()
+                    || self
+                        .records
+                        .get(&(record.namespace.clone(), record.id.clone()))
+                        .map_or(false, |r| !r.sparse.is_empty())
+            }
+            WalOp::Delete { namespace, id } => self
+                .records
+                .get(&(namespace.clone(), id.clone()))
+                .map_or(false, |r| !r.sparse.is_empty()),
+        });
+
         self.append_wal_batch(&ops)?;
         self.apply_ops_in_memory(ops);
-        self.rebuild_sparse_index();
+
+        if has_sparse {
+            self.rebuild_sparse_index();
+        }
         self.rebuild_ann();
         self.ann_loaded_from_disk = false;
         self.persist_ann_to_disk()?;
+        self.rebuild_quantized_index();
+        Ok(())
+    }
+
+    /// Write ops to WAL and apply in memory, but defer index rebuilds.
+    /// The caller is responsible for calling `rebuild_sparse_index()`,
+    /// `rebuild_ann()`, and `persist_ann_to_disk()` after all batches are done.
+    fn apply_wal_batch_deferred(&mut self, ops: Vec<WalOp>) -> Result<()> {
+        if ops.is_empty() {
+            return Ok(());
+        }
+
+        self.append_wal_batch(&ops)?;
+        self.apply_ops_in_memory(ops);
         Ok(())
     }
 
@@ -1754,6 +2140,9 @@ impl Database {
             ann_loaded_from_disk: false,
             read_only: false,
             _lock_file: None,
+            quantized: None,
+            quantization_config: None,
+            quantized_keys: Vec::new(),
         })
     }
 
@@ -2409,6 +2798,15 @@ fn candidate_count(top_k: usize, total: usize) -> usize {
         .min(total)
 }
 
+fn timeout_duration(timeout_secs: f64, label: &str) -> Result<std::time::Duration> {
+    if !timeout_secs.is_finite() || timeout_secs < 0.0 {
+        return Err(VectLiteError::InvalidFormat(format!(
+            "{label} must be a finite, non-negative number of seconds"
+        )));
+    }
+    Ok(std::time::Duration::from_secs_f64(timeout_secs))
+}
+
 fn wal_path(path: &Path) -> PathBuf {
     let mut wal = path.as_os_str().to_os_string();
     wal.push(".wal");
@@ -2421,7 +2819,20 @@ fn lock_path(path: &Path) -> PathBuf {
     PathBuf::from(lock)
 }
 
+fn quantization_params_path(path: &Path) -> PathBuf {
+    let mut p = path.as_os_str().to_os_string();
+    p.push(".quant");
+    PathBuf::from(p)
+}
+
 fn acquire_exclusive_lock(path: &Path) -> Result<File> {
+    acquire_exclusive_lock_with_timeout(path, None)
+}
+
+fn acquire_exclusive_lock_with_timeout(
+    path: &Path,
+    timeout: Option<std::time::Duration>,
+) -> Result<File> {
     if let Some(parent) = path.parent() {
         if !parent.as_os_str().is_empty() && !parent.exists() {
             fs::create_dir_all(parent)?;
@@ -2433,16 +2844,43 @@ fn acquire_exclusive_lock(path: &Path) -> Result<File> {
         .read(true)
         .write(true)
         .open(lock_path(path))?;
-    file.try_lock_exclusive().map_err(|err| {
-        VectLiteError::LockContention(format!(
-            "could not acquire exclusive lock on '{}': {err}",
-            path.display()
-        ))
-    })?;
+
+    match timeout {
+        None => {
+            file.try_lock_exclusive().map_err(|err| {
+                VectLiteError::LockContention(format!(
+                    "could not acquire exclusive lock on '{}': {err}",
+                    path.display()
+                ))
+            })?;
+        }
+        Some(duration) => {
+            let start = Instant::now();
+            let interval = std::time::Duration::from_millis(50);
+            loop {
+                match file.try_lock_exclusive() {
+                    Ok(()) => break,
+                    Err(err) => {
+                        if start.elapsed() >= duration {
+                            return Err(VectLiteError::LockContention(format!(
+                                "could not acquire exclusive lock on '{}' after {:.1}s: {err}",
+                                path.display(),
+                                duration.as_secs_f64()
+                            )));
+                        }
+                        std::thread::sleep(interval);
+                    }
+                }
+            }
+        }
+    }
     Ok(file)
 }
 
-fn acquire_shared_lock(path: &Path) -> Result<File> {
+fn acquire_shared_lock_with_timeout(
+    path: &Path,
+    timeout: Option<std::time::Duration>,
+) -> Result<File> {
     let lock_file = lock_path(path);
     if !lock_file.exists() {
         // Lock file may not exist yet for read-only opens on existing dbs
@@ -2458,12 +2896,36 @@ fn acquire_shared_lock(path: &Path) -> Result<File> {
         .read(true)
         .write(true)
         .open(&lock_file)?;
-    file.try_lock_shared().map_err(|err| {
-        VectLiteError::LockContention(format!(
-            "could not acquire shared lock on '{}': {err}",
-            path.display()
-        ))
-    })?;
+
+    match timeout {
+        None => {
+            file.try_lock_shared().map_err(|err| {
+                VectLiteError::LockContention(format!(
+                    "could not acquire shared lock on '{}': {err}",
+                    path.display()
+                ))
+            })?;
+        }
+        Some(duration) => {
+            let start = Instant::now();
+            let interval = std::time::Duration::from_millis(50);
+            loop {
+                match file.try_lock_shared() {
+                    Ok(()) => break,
+                    Err(err) => {
+                        if start.elapsed() >= duration {
+                            return Err(VectLiteError::LockContention(format!(
+                                "could not acquire shared lock on '{}' after {:.1}s: {err}",
+                                path.display(),
+                                duration.as_secs_f64()
+                            )));
+                        }
+                        std::thread::sleep(interval);
+                    }
+                }
+            }
+        }
+    }
     Ok(file)
 }
 
@@ -3088,7 +3550,7 @@ fn usize_from_u64(value: u64) -> Result<usize> {
 mod tests {
     use super::{
         Database, HybridSearchOptions, Metadata, MetadataFilter, MetadataValue, NamedVectors,
-        Record, SearchOptions, SparseVector,
+        Record, SearchOptions, SparseVector, VectLiteError,
     };
     use std::path::{Path, PathBuf};
     use std::time::{SystemTime, UNIX_EPOCH};
@@ -3343,6 +3805,67 @@ mod tests {
         cleanup(&path);
     }
 
+    #[test]
+    fn upsert_without_sparse_rebuilds_sparse_index() {
+        let path = temp_file("sparse-upsert-clear");
+        let mut database = Database::create(&path, 2).expect("create database");
+
+        let mut sparse_auth = SparseVector::new();
+        sparse_auth.insert("auth".to_owned(), 1.0);
+
+        database
+            .upsert_with_sparse_in_namespace(
+                "docs",
+                "doc1",
+                vec![1.0, 0.0],
+                sparse_auth,
+                Metadata::new(),
+            )
+            .expect("insert sparse doc");
+
+        let mut query_sparse = SparseVector::new();
+        query_sparse.insert("auth".to_owned(), 1.0);
+
+        let initial_outcome = database
+            .hybrid_search_in_namespace_with_stats(
+                "docs",
+                None,
+                Some(&query_sparse),
+                HybridSearchOptions {
+                    top_k: 10,
+                    filter: None,
+                    dense_weight: 0.0,
+                    sparse_weight: 1.0,
+                    ..HybridSearchOptions::default()
+                },
+            )
+            .expect("initial sparse search");
+        assert_eq!(initial_outcome.stats.sparse_candidate_count, 1);
+
+        database
+            .upsert_in_namespace("docs", "doc1", vec![1.0, 0.0], Metadata::new())
+            .expect("replace doc without sparse terms");
+
+        let updated_outcome = database
+            .hybrid_search_in_namespace_with_stats(
+                "docs",
+                None,
+                Some(&query_sparse),
+                HybridSearchOptions {
+                    top_k: 10,
+                    filter: None,
+                    dense_weight: 0.0,
+                    sparse_weight: 1.0,
+                    ..HybridSearchOptions::default()
+                },
+            )
+            .expect("sparse search after clearing sparse terms");
+        assert_eq!(updated_outcome.stats.sparse_candidate_count, 0);
+        assert!(updated_outcome.results.is_empty());
+
+        cleanup(&path);
+    }
+
     #[test]
     fn named_vectors_roundtrip_and_search() {
         let path = temp_file("named-vectors");
@@ -3528,6 +4051,69 @@ mod tests {
         cleanup(&path);
     }
 
+    #[test]
+    fn closed_database_rejects_result_based_operations() {
+        let path = temp_file("closed-db");
+        let snapshot = temp_file("closed-db-snapshot");
+        let mut database = Database::create(&path, 2).expect("create database");
+        database
+            .insert("doc1", vec![1.0, 0.0], Metadata::new())
+            .expect("insert doc1");
+        database.close().expect("close database");
+
+        let search_err = database
+            .search(
+                &[1.0, 0.0],
+                SearchOptions {
+                    top_k: 1,
+                    filter: None,
+                },
+            )
+            .expect_err("search on closed database should fail");
+        assert!(matches!(
+            search_err,
+            VectLiteError::InvalidFormat(message) if message.contains("database is closed")
+        ));
+
+        let snapshot_err = database
+            .snapshot(&snapshot)
+            .expect_err("snapshot on closed database should fail");
+        assert!(matches!(
+            snapshot_err,
+            VectLiteError::InvalidFormat(message) if message.contains("database is closed")
+        ));
+
+        cleanup(&path);
+        cleanup(&snapshot);
+    }
+
+    #[test]
+    fn lock_timeout_must_be_non_negative_and_finite() {
+        let path = temp_file("timeout-validation");
+        let database = Database::create(&path, 2).expect("create database");
+
+        let negative_err = match Database::open_with_timeout(&path, -1.0) {
+            Ok(_) => panic!("negative lock timeout should fail"),
+            Err(err) => err,
+        };
+        assert!(matches!(
+            negative_err,
+            VectLiteError::InvalidFormat(message) if message.contains("lock_timeout")
+        ));
+
+        let nan_err = match Database::open_with_timeout(&path, f64::NAN) {
+            Ok(_) => panic!("NaN lock timeout should fail"),
+            Err(err) => err,
+        };
+        assert!(matches!(
+            nan_err,
+            VectLiteError::InvalidFormat(message) if message.contains("lock_timeout")
+        ));
+
+        drop(database);
+        cleanup(&path);
+    }
+
     fn temp_file(name: &str) -> PathBuf {
         let nanos = SystemTime::now()
             .duration_since(UNIX_EPOCH)
@@ -3541,5 +4127,234 @@ mod tests {
 
     fn cleanup(path: &Path) {
         let _ = std::fs::remove_file(path);
+        // Also clean up sidecar files
+        let mut quant = path.as_os_str().to_os_string();
+        quant.push(".quant");
+        let _ = std::fs::remove_file(PathBuf::from(&quant));
+        let mut wal = path.as_os_str().to_os_string();
+        wal.push(".wal");
+        let _ = std::fs::remove_file(PathBuf::from(&wal));
+        let mut lock = path.as_os_str().to_os_string();
+        lock.push(".lock");
+        let _ = std::fs::remove_file(PathBuf::from(&lock));
+    }
+
+    // -----------------------------------------------------------------------
+    // Quantization integration tests
+    // -----------------------------------------------------------------------
+
+    #[test]
+    fn scalar_quantization_enables_search_and_persists() {
+        use super::quantization::{QuantizationConfig, ScalarQuantizationConfig};
+
+        let path = temp_file("quant-scalar");
+        let dim = 32;
+
+        {
+            let mut db = Database::create(&path, dim).expect("create");
+            // Insert enough records for meaningful search
+            for i in 0..50 {
+                let mut v = vec![0.0_f32; dim];
+                v[i % dim] = 1.0;
+                v[(i + 1) % dim] = 0.5;
+                db.upsert(format!("doc{i}"), v, Metadata::new())
+                    .expect("upsert");
+            }
+
+            // Enable scalar quantization
+            db.enable_quantization(QuantizationConfig::Scalar(ScalarQuantizationConfig {
+                rescore_multiplier: 5,
+            }))
+            .expect("enable quant");
+
+            assert!(db.is_quantized());
+
+            // Search should work with quantization
+            let query = {
+                let mut q = vec![0.0_f32; dim];
+                q[0] = 1.0;
+                q
+            };
+            let results = db
+                .search(
+                    &query,
+                    SearchOptions {
+                        top_k: 5,
+                        filter: None,
+                    },
+                )
+                .expect("search");
+            assert!(!results.is_empty());
+            // The most similar vector (doc0 has [1,0.5,0,...]) should be first
+            assert_eq!(results[0].id, "doc0");
+        }
+
+        // Reopen and verify quantization persists
+        {
+            let db = Database::open(&path).expect("reopen");
+            assert!(db.is_quantized());
+            assert!(matches!(
+                db.quantization_config(),
+                Some(QuantizationConfig::Scalar(_))
+            ));
+
+            let query = {
+                let mut q = vec![0.0_f32; dim];
+                q[0] = 1.0;
+                q
+            };
+            let results = db
+                .search(
+                    &query,
+                    SearchOptions {
+                        top_k: 5,
+                        filter: None,
+                    },
+                )
+                .expect("search after reopen");
+            assert!(!results.is_empty());
+            assert_eq!(results[0].id, "doc0");
+        }
+
+        cleanup(&path);
+    }
+
+    #[test]
+    fn binary_quantization_enables_search() {
+        use super::quantization::{BinaryQuantizationConfig, QuantizationConfig};
+
+        let path = temp_file("quant-binary");
+        let dim = 64;
+
+        let mut db = Database::create(&path, dim).expect("create");
+        for i in 0..100 {
+            let mut v = vec![0.0_f32; dim];
+            // Set some positive dimensions for the binary representation
+            for j in 0..dim {
+                v[j] = if (i + j) % 3 == 0 { 1.0 } else { -1.0 };
+            }
+            db.upsert(format!("doc{i}"), v, Metadata::new())
+                .expect("upsert");
+        }
+
+        db.enable_quantization(QuantizationConfig::Binary(BinaryQuantizationConfig {
+            rescore_multiplier: 10,
+        }))
+        .expect("enable quant");
+
+        assert!(db.is_quantized());
+
+        // Search: query matches doc0's pattern
+        let query: Vec<f32> = (0..dim)
+            .map(|j| if j % 3 == 0 { 1.0 } else { -1.0 })
+            .collect();
+        let results = db
+            .search(
+                &query,
+                SearchOptions {
+                    top_k: 5,
+                    filter: None,
+                },
+            )
+            .expect("search");
+        assert!(!results.is_empty());
+        // doc0 should be the best match (identical pattern)
+        assert_eq!(results[0].id, "doc0");
+
+        cleanup(&path);
+    }
+
+    #[test]
+    fn product_quantization_enables_search() {
+        use super::quantization::{ProductQuantizationConfig, QuantizationConfig};
+
+        let path = temp_file("quant-pq");
+        let dim = 32;
+
+        let mut db = Database::create(&path, dim).expect("create");
+        for i in 0..100 {
+            let v: Vec<f32> = (0..dim)
+                .map(|j| ((i * 7 + j * 13) % 100) as f32 / 100.0)
+                .collect();
+            db.upsert(format!("doc{i}"), v, Metadata::new())
+                .expect("upsert");
+        }
+
+        db.enable_quantization(QuantizationConfig::Product(ProductQuantizationConfig {
+            num_sub_vectors: 4,
+            num_centroids: 16,
+            training_iterations: 5,
+            rescore_multiplier: 10,
+        }))
+        .expect("enable quant");
+
+        assert!(db.is_quantized());
+
+        // Search with the same vector as doc0
+        let query: Vec<f32> = (0..dim).map(|j| (j * 13 % 100) as f32 / 100.0).collect();
+        let results = db
+            .search(
+                &query,
+                SearchOptions {
+                    top_k: 5,
+                    filter: None,
+                },
+            )
+            .expect("search");
+        assert!(!results.is_empty());
+        assert_eq!(results[0].id, "doc0");
+
+        cleanup(&path);
+    }
+
+    #[test]
+    fn disable_quantization_removes_sidecar() {
+        use super::quantization::{QuantizationConfig, ScalarQuantizationConfig};
+
+        let path = temp_file("quant-disable");
+        let dim = 8;
+
+        let mut db = Database::create(&path, dim).expect("create");
+        for i in 0..10 {
+            let v: Vec<f32> = (0..dim).map(|j| (i + j) as f32).collect();
+            db.upsert(format!("doc{i}"), v, Metadata::new())
+                .expect("upsert");
+        }
+
+        db.enable_quantization(QuantizationConfig::Scalar(
+            ScalarQuantizationConfig::default(),
+        ))
+        .expect("enable");
+        assert!(db.is_quantized());
+
+        // Verify sidecar exists
+        let quant_path = {
+            let mut p = path.as_os_str().to_os_string();
+            p.push(".quant");
+            PathBuf::from(p)
+        };
+        assert!(quant_path.exists());
+
+        db.disable_quantization().expect("disable");
+        assert!(!db.is_quantized());
+        assert!(!quant_path.exists());
+
+        cleanup(&path);
+    }
+
+    #[test]
+    fn quantization_empty_database_returns_error() {
+        use super::quantization::{QuantizationConfig, ScalarQuantizationConfig};
+
+        let path = temp_file("quant-empty");
+        let mut db = Database::create(&path, 4).expect("create");
+
+        let result = db.enable_quantization(QuantizationConfig::Scalar(
+            ScalarQuantizationConfig::default(),
+        ));
+        assert!(result.is_err());
+        assert!(!db.is_quantized());
+
+        cleanup(&path);
     }
 }