hyperstreamdb 0.4.1__tar.gz → 0.5.1__tar.gz

{hyperstreamdb-0.4.1 → hyperstreamdb-0.5.1}/CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+---
+
+## [0.5.0] - 2026-06-03
+
 ### Added
 - Comprehensive unit test suites for Spark and Trino connectors
 - Thread-local GPU context for concurrent query safety
@@ -14,11 +18,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Query configuration API
 - Metrics instrumentation for observability
 - Release profiles (`release` and `release-lto`) in Cargo.toml
+- `WriteAheadLog::append_fire_and_forget` — non-blocking WAL append that hands off batches to the WAL worker without blocking on `fdatasync`, eliminating ~800 ms write latency per call
 
 ### Changed
 - CUDA is now an optional feature (no longer required for CI builds)
 - FFI bounds hardened with panic safety guarantees
 - Public APIs documented with rustdoc
+- **`Table(index_all=False)` is now the default** (previously `True`). Automatic HNSW/BM25 index building on commit is now opt-in. This eliminates a silent 15–18 s background build that previously fired on every `commit()` for any table containing a vector column. To restore the old behaviour: `Table(uri, index_all=True)` or `table.index_all = True`.
+- **`Table(autocommit=False)` is now the default** (previously `True`). Writes accumulate in an in-memory buffer and must be explicitly committed with `table.commit()`. This eliminates unexpected auto-flush overhead during ingestion loops.
+- `write_async` no longer auto-detects columns named `"embedding"` as implicit HNSW index targets. Only columns explicitly registered via `add_index()` or `set_index_columns()` are indexed.
+- `manifest_manager.load_latest_full` replaced with `load_latest` in the `flush_async` hot path, eliminating two unnecessary full manifest scans per commit.
+- Primary key uniqueness check upgraded from O(N²) to O(N) using `HashSet`.
 
 ### Fixed
 - Rustdoc warnings across the codebase
@@ -282,7 +292,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
-[Unreleased]: https://github.com/rla3rd/hyperstreamdb/compare/v0.4.0...HEAD
+[Unreleased]: https://github.com/rla3rd/hyperstreamdb/compare/v0.5.0...HEAD
+[0.5.0]: https://github.com/rla3rd/hyperstreamdb/compare/v0.4.1...v0.5.0
 [0.4.0]: https://github.com/rla3rd/hyperstreamdb/compare/v0.3.3...v0.4.0
 [0.3.3]: https://github.com/rla3rd/hyperstreamdb/compare/v0.3.2...v0.3.3
 [0.3.2]: https://github.com/rla3rd/hyperstreamdb/compare/v0.3.1...v0.3.2

{hyperstreamdb-0.4.1 → hyperstreamdb-0.5.1}/Cargo.lock

@@ -3392,7 +3392,7 @@ dependencies = [
 
 [[package]]
 name = "hyperstreamdb"
-version = "0.4.1"
+version = "0.5.1"
 dependencies = [
  "ahash",
  "anyhow",

{hyperstreamdb-0.4.1 → hyperstreamdb-0.5.1}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hyperstreamdb"
-version = "0.4.1"
+version = "0.5.1"
 edition = "2021"
 license = "MIT AND Apache-2.0"
 description = "HyperStreamDB - Serverless Index-Streaming Database with Overlay Indexing and Vector Search"

{hyperstreamdb-0.4.1 → hyperstreamdb-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperstreamdb
-Version: 0.4.1
+Version: 0.5.1
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Rust

{hyperstreamdb-0.4.1 → hyperstreamdb-0.5.1}/python/hyperstreamdb/__init__.py

@@ -181,9 +181,31 @@ class Query:
 
 class Table:
     """
-    Enhanced HyperStreamDB Table with auto-vectorization and embedding registry support.
+    HyperStreamDB Table — Apache Iceberg/Parquet-compatible columnar vector store.
+
+    **Default behaviour (v0.4.1+)**
+
+    - ``index_all = False`` — Vector indexes are *not* built automatically.
+      Call ``table.index_all = True`` or ``table.add_index(column, 'hnsw')``
+      to enable indexing for a specific session or column.
+    - ``autocommit = False`` — Writes accumulate in an in-memory buffer.
+      Call ``table.commit()`` (or ``await table.commit_async()``) to persist
+      data to Parquet and advance the Iceberg snapshot.
+
+    These defaults exist for performance: automatic indexing previously caused
+    silent 15-18 s HNSW build latency on every ``commit()`` for tables with
+    vector columns, even when the user had not requested an index.
+
+    Args:
+        uri:         Table location (``file:///path`` or cloud URI).
+        inner_table: Internal — do not pass directly.
+        device:      Optional compute device for GPU-accelerated index builds.
+        index_all:   Enable automatic indexing of all compatible columns.
+                     Defaults to ``False``. Set ``True`` to restore legacy behaviour.
+        primary_key: Column name (or list) to use as primary key.
+        explain:     If ``True``, return query plans instead of results.
     """
-    def __init__(self, uri: str, inner_table: Optional[_RustTable] = None, device: Optional[Any] = None, index_all: bool = True, primary_key: Optional[str] = None, explain: bool = False):
+    def __init__(self, uri: str, inner_table: Optional[_RustTable] = None, device: Optional[Any] = None, index_all: bool = False, primary_key: Optional[str] = None, explain: bool = False):
         uri = _resolve_uri(uri)
         self.explain = explain
         if inner_table:
@@ -522,15 +544,15 @@ class Table:
         """
         return self.to_arrow(filter, vector_filter, columns, device=device, **kwargs)
 
-    def vector_search(self, column: str, query: List[float], k: int = 10, filter: Optional[str] = None, device: Optional[Any] = None, **kwargs):
+    def vector_search(self, column: str, query: List[float], k: int = 10, filter: Optional[str] = None, columns: Optional[List[str]] = None, device: Optional[Any] = None, **kwargs):
         """Backward compatibility alias for to_pandas with vector filter."""
         vf = {"column": column, "query": query, "k": k}
         vf.update(kwargs)
-        return self.to_pandas(filter=filter, vector_filter=vf, device=device)
+        return self.to_pandas(filter=filter, vector_filter=vf, columns=columns, device=device)
 
-    def search(self, column: str, query: List[float], k: int = 10, filter: Optional[str] = None, device: Optional[Any] = None, **kwargs):
+    def search(self, column: str, query: List[float], k: int = 10, filter: Optional[str] = None, columns: Optional[List[str]] = None, device: Optional[Any] = None, **kwargs):
         """Alias for vector_search."""
-        return self.vector_search(column, query, k, filter, device, **kwargs)
+        return self.vector_search(column, query, k, filter, columns, device, **kwargs)
 
     def filter(self, expr: Optional[str] = None, vector_filter: Optional[Union[Dict[str, Any], List[float]]] = None, **kwargs) -> 'Query':
         """
@@ -569,7 +591,16 @@ class Table:
 
     @property
     def index_all(self):
-        """Whether to index all compatible columns by default."""
+        """
+        Whether to build HNSW/BM25 indexes for all compatible columns on commit.
+
+        Defaults to ``False`` (opt-in). Setting this to ``True`` triggers
+        background index builds after every ``commit()`` call — useful when
+        you want fast ANN search but be aware of the additional commit latency
+        (~15 s per 100 K rows with 768-dim vectors on CPU).
+
+        For selective indexing, prefer ``table.add_index(column, 'hnsw')``.
+        """
         return self._inner.get_index_all()
 
     @index_all.setter

{hyperstreamdb-0.4.1 → hyperstreamdb-0.5.1}/src/core/index/distance.rs

@@ -52,8 +52,66 @@ fn l2_distance_squared_portable(a: &[f32], b: &[f32]) -> f32 {
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 #[target_feature(enable = "avx2,fma")]
 unsafe fn l2_distance_squared_avx2(a: &[f32], b: &[f32]) -> f32 {
-    // LLVM will now generate aggressive AVX2/FMA instructions here
-    l2_distance_squared_portable(a, b)
+    #[cfg(target_arch = "x86")]
+    use std::arch::x86::*;
+    #[cfg(target_arch = "x86_64")]
+    use std::arch::x86_64::*;
+
+    let n = a.len();
+    let mut i = 0;
+    
+    let mut sum0 = _mm256_setzero_ps();
+    let mut sum1 = _mm256_setzero_ps();
+    let mut sum2 = _mm256_setzero_ps();
+    let mut sum3 = _mm256_setzero_ps();
+
+    while i + 31 < n {
+        let a0 = _mm256_loadu_ps(a.as_ptr().add(i));
+        let b0 = _mm256_loadu_ps(b.as_ptr().add(i));
+        let diff0 = _mm256_sub_ps(a0, b0);
+        sum0 = _mm256_fmadd_ps(diff0, diff0, sum0);
+
+        let a1 = _mm256_loadu_ps(a.as_ptr().add(i + 8));
+        let b1 = _mm256_loadu_ps(b.as_ptr().add(i + 8));
+        let diff1 = _mm256_sub_ps(a1, b1);
+        sum1 = _mm256_fmadd_ps(diff1, diff1, sum1);
+
+        let a2 = _mm256_loadu_ps(a.as_ptr().add(i + 16));
+        let b2 = _mm256_loadu_ps(b.as_ptr().add(i + 16));
+        let diff2 = _mm256_sub_ps(a2, b2);
+        sum2 = _mm256_fmadd_ps(diff2, diff2, sum2);
+
+        let a3 = _mm256_loadu_ps(a.as_ptr().add(i + 24));
+        let b3 = _mm256_loadu_ps(b.as_ptr().add(i + 24));
+        let diff3 = _mm256_sub_ps(a3, b3);
+        sum3 = _mm256_fmadd_ps(diff3, diff3, sum3);
+
+        i += 32;
+    }
+
+    sum0 = _mm256_add_ps(sum0, sum1);
+    sum2 = _mm256_add_ps(sum2, sum3);
+    sum0 = _mm256_add_ps(sum0, sum2);
+
+    while i + 7 < n {
+        let a0 = _mm256_loadu_ps(a.as_ptr().add(i));
+        let b0 = _mm256_loadu_ps(b.as_ptr().add(i));
+        let diff0 = _mm256_sub_ps(a0, b0);
+        sum0 = _mm256_fmadd_ps(diff0, diff0, sum0);
+        i += 8;
+    }
+
+    let mut sum_array = [0.0f32; 8];
+    _mm256_storeu_ps(sum_array.as_mut_ptr(), sum0);
+    let mut total = sum_array.iter().sum::<f32>();
+
+    while i < n {
+        let diff = a[i] - b[i];
+        total += diff * diff;
+        i += 1;
+    }
+
+    total
 }
 
 #[inline(always)]
@@ -76,6 +134,17 @@ pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
 
 #[inline(always)]
 pub fn dot_product(a: &[f32], b: &[f32]) -> f32 {
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    {
+        if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") {
+            return unsafe { dot_product_avx2(a, b) };
+        }
+    }
+    dot_product_portable(a, b)
+}
+
+#[inline(always)]
+fn dot_product_portable(a: &[f32], b: &[f32]) -> f32 {
     let n = a.len();
     assert_eq!(n, b.len(), "Vectors must have the same length");
 
@@ -100,6 +169,65 @@ pub fn dot_product(a: &[f32], b: &[f32]) -> f32 {
     sum
 }
 
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[target_feature(enable = "avx2,fma")]
+unsafe fn dot_product_avx2(a: &[f32], b: &[f32]) -> f32 {
+    #[cfg(target_arch = "x86")]
+    use std::arch::x86::*;
+    #[cfg(target_arch = "x86_64")]
+    use std::arch::x86_64::*;
+
+    let n = a.len();
+    let mut i = 0;
+    
+    let mut sum0 = _mm256_setzero_ps();
+    let mut sum1 = _mm256_setzero_ps();
+    let mut sum2 = _mm256_setzero_ps();
+    let mut sum3 = _mm256_setzero_ps();
+
+    while i + 31 < n {
+        let a0 = _mm256_loadu_ps(a.as_ptr().add(i));
+        let b0 = _mm256_loadu_ps(b.as_ptr().add(i));
+        sum0 = _mm256_fmadd_ps(a0, b0, sum0);
+
+        let a1 = _mm256_loadu_ps(a.as_ptr().add(i + 8));
+        let b1 = _mm256_loadu_ps(b.as_ptr().add(i + 8));
+        sum1 = _mm256_fmadd_ps(a1, b1, sum1);
+
+        let a2 = _mm256_loadu_ps(a.as_ptr().add(i + 16));
+        let b2 = _mm256_loadu_ps(b.as_ptr().add(i + 16));
+        sum2 = _mm256_fmadd_ps(a2, b2, sum2);
+
+        let a3 = _mm256_loadu_ps(a.as_ptr().add(i + 24));
+        let b3 = _mm256_loadu_ps(b.as_ptr().add(i + 24));
+        sum3 = _mm256_fmadd_ps(a3, b3, sum3);
+
+        i += 32;
+    }
+
+    sum0 = _mm256_add_ps(sum0, sum1);
+    sum2 = _mm256_add_ps(sum2, sum3);
+    sum0 = _mm256_add_ps(sum0, sum2);
+
+    while i + 7 < n {
+        let a0 = _mm256_loadu_ps(a.as_ptr().add(i));
+        let b0 = _mm256_loadu_ps(b.as_ptr().add(i));
+        sum0 = _mm256_fmadd_ps(a0, b0, sum0);
+        i += 8;
+    }
+
+    let mut sum_array = [0.0f32; 8];
+    _mm256_storeu_ps(sum_array.as_mut_ptr(), sum0);
+    let mut total = sum_array.iter().sum::<f32>();
+
+    while i < n {
+        total += a[i] * b[i];
+        i += 1;
+    }
+
+    total
+}
+
 /// Vectorized batch L2 distance calculation
 pub fn l2_distance_batch<V: AsRef<[f32]> + Sync>(query: &[f32], vectors: &[V]) -> Vec<f32> {
     use rayon::prelude::*;
@@ -217,9 +345,53 @@ pub fn jaccard_distance(a: &[f32], b: &[f32]) -> f32 {
 
 /// Bit-optimized Hamming distance for packed binary vectors (e.g. 1 bit per element)
 pub fn hamming_distance_packed(a: &[u8], b: &[u8]) -> u32 {
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    {
+        if is_x86_feature_detected!("popcnt") {
+            return unsafe { hamming_distance_packed_popcnt(a, b) };
+        }
+    }
+    hamming_distance_packed_portable(a, b)
+}
+
+#[inline(always)]
+fn hamming_distance_packed_portable(a: &[u8], b: &[u8]) -> u32 {
     a.iter().zip(b.iter()).map(|(&x, &y)| (x ^ y).count_ones()).sum()
 }
 
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[target_feature(enable = "popcnt")]
+unsafe fn hamming_distance_packed_popcnt(a: &[u8], b: &[u8]) -> u32 {
+    #[cfg(target_arch = "x86")]
+    use std::arch::x86::*;
+    #[cfg(target_arch = "x86_64")]
+    use std::arch::x86_64::*;
+
+    let n = a.len();
+    let mut i = 0;
+    let mut sum = 0;
+
+    let a_ptr = a.as_ptr();
+    let b_ptr = b.as_ptr();
+
+    #[cfg(target_arch = "x86_64")]
+    {
+        while i + 7 < n {
+            let a_val = std::ptr::read_unaligned(a_ptr.add(i) as *const u64);
+            let b_val = std::ptr::read_unaligned(b_ptr.add(i) as *const u64);
+            sum += _popcnt64((a_val ^ b_val) as i64) as u32;
+            i += 8;
+        }
+    }
+
+    while i < n {
+        sum += (a[i] ^ b[i]).count_ones();
+        i += 1;
+    }
+
+    sum
+}
+
 /// Sparse dot product: intersection of two sorted index/value pairs
 pub fn sparse_dot_product(
     a_indices: &[u32], a_values: &[f32],
@@ -283,6 +455,17 @@ pub fn sparse_l2_distance_squared(
 /// Optimized L2 distance for quantized u8 vectors
 #[inline(always)]
 pub fn l2_distance_u8(a: &[u8], b: &[u8]) -> f32 {
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    {
+        if is_x86_feature_detected!("avx2") {
+            return unsafe { l2_distance_u8_avx2(a, b) };
+        }
+    }
+    l2_distance_u8_portable(a, b)
+}
+
+#[inline(always)]
+fn l2_distance_u8_portable(a: &[u8], b: &[u8]) -> f32 {
     let mut sum = 0;
     let chunks_a = a.chunks_exact(16);
     let chunks_b = b.chunks_exact(16);
@@ -303,6 +486,55 @@ pub fn l2_distance_u8(a: &[u8], b: &[u8]) -> f32 {
     sum as f32
 }
 
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[target_feature(enable = "avx2")]
+unsafe fn l2_distance_u8_avx2(a: &[u8], b: &[u8]) -> f32 {
+    #[cfg(target_arch = "x86")]
+    use std::arch::x86::*;
+    #[cfg(target_arch = "x86_64")]
+    use std::arch::x86_64::*;
+
+    let n = a.len();
+    let mut i = 0;
+    let mut sum_vec = _mm256_setzero_si256();
+
+    while i + 31 < n {
+        let a_vec = _mm256_loadu_si256(a.as_ptr().add(i) as *const __m256i);
+        let b_vec = _mm256_loadu_si256(b.as_ptr().add(i) as *const __m256i);
+
+        let zero = _mm256_setzero_si256();
+        
+        let a_lo = _mm256_unpacklo_epi8(a_vec, zero);
+        let a_hi = _mm256_unpackhi_epi8(a_vec, zero);
+        
+        let b_lo = _mm256_unpacklo_epi8(b_vec, zero);
+        let b_hi = _mm256_unpackhi_epi8(b_vec, zero);
+
+        let diff_lo = _mm256_sub_epi16(a_lo, b_lo);
+        let diff_hi = _mm256_sub_epi16(a_hi, b_hi);
+
+        let sq_lo = _mm256_madd_epi16(diff_lo, diff_lo);
+        let sq_hi = _mm256_madd_epi16(diff_hi, diff_hi);
+
+        sum_vec = _mm256_add_epi32(sum_vec, sq_lo);
+        sum_vec = _mm256_add_epi32(sum_vec, sq_hi);
+
+        i += 32;
+    }
+
+    let mut sum_array = [0i32; 8];
+    _mm256_storeu_si256(sum_array.as_mut_ptr() as *mut __m256i, sum_vec);
+    let mut total = sum_array.iter().sum::<i32>();
+
+    while i < n {
+        let diff = (a[i] as i32) - (b[i] as i32);
+        total += diff * diff;
+        i += 1;
+    }
+
+    total as f32
+}
+
 /// Asymmetric Distance Calculation (ADC) for quantized vectors.
 /// Calculates L2 distance between a float32 query and a quantized u8 vector.
 #[inline(always)]

{hyperstreamdb-0.4.1 → hyperstreamdb-0.5.1}/src/core/index/hnsw_ivf.rs

@@ -88,6 +88,7 @@ pub enum HnswGraph {
     TurboQuant8(Hnsw<u8, crate::core::index::distance::DistL2u8>),
     TurboQuant4(Hnsw<u8, crate::core::index::distance::DistL2u4>),
     SparseDot(Hnsw<crate::core::index::SparseVector, DistSparseDot>),
+    Pq(Hnsw<u8, crate::core::index::pq::DistPqSdc>),
 }
 
 impl HnswGraph {
@@ -103,6 +104,7 @@ impl HnswGraph {
             (HnswGraph::BinaryHamming(h), VectorValue::Binary(q)) => h.search(q, k, ef, filter),
             (HnswGraph::TurboQuant8(h), VectorValue::Binary(q)) => h.search(q, k, ef, filter),
             (HnswGraph::TurboQuant4(h), VectorValue::Binary(q)) => h.search(q, k, ef, filter),
+            (HnswGraph::Pq(h), VectorValue::Binary(q)) => h.search(q, k, ef, filter),
             (HnswGraph::SparseDot(h), VectorValue::Sparse(q)) => h.search(std::slice::from_ref(q), k, ef, filter),
             
             // Casting paths
@@ -128,6 +130,7 @@ impl HnswGraph {
             (HnswGraph::BinaryHamming(h), VectorValue::Binary(v)) => h.insert_slice((&v, local_id)),
             (HnswGraph::TurboQuant8(h), VectorValue::Binary(v)) => h.insert_slice((&v, local_id)),
             (HnswGraph::TurboQuant4(h), VectorValue::Binary(v)) => h.insert_slice((&v, local_id)),
+            (HnswGraph::Pq(h), VectorValue::Binary(v)) => h.insert_slice((&v, local_id)),
             (HnswGraph::SparseDot(h), VectorValue::Sparse(v)) => h.insert_slice((&vec![v], local_id)),
             
             (HnswGraph::L2(h), VectorValue::Float16(v)) => h.insert_slice((&v, local_id)),
@@ -167,6 +170,7 @@ impl HnswGraph {
             HnswGraph::BinaryHamming(h) => h.file_dump(&path_string).map(|_| ()).map_err(|e| Box::new(std::io::Error::other(e)) as Box<dyn std::error::Error>),
             HnswGraph::TurboQuant8(h) => h.file_dump(&path_string).map(|_| ()).map_err(|e| Box::new(std::io::Error::other(e)) as Box<dyn std::error::Error>),
             HnswGraph::TurboQuant4(h) => h.file_dump(&path_string).map(|_| ()).map_err(|e| Box::new(std::io::Error::other(e)) as Box<dyn std::error::Error>),
+            HnswGraph::Pq(h) => h.file_dump(&path_string).map(|_| ()).map_err(|e| Box::new(std::io::Error::other(e)) as Box<dyn std::error::Error>),
             HnswGraph::SparseDot(h) => h.file_dump(&path_string).map(|_| ()).map_err(|e| Box::new(std::io::Error::other(e)) as Box<dyn std::error::Error>),
         }
     }
@@ -285,10 +289,18 @@ impl HnswIvfIndex {
                 
                 // Determine HNSW node type and distance metric
                 let hnsw = if let Some(q) = quantizer.as_ref() {
-                    if q.bits() == 4 {
-                        HnswGraph::TurboQuant4(Hnsw::new(hnsw_m, vecs.len(), max_layers, ef_construction, crate::core::index::distance::DistL2u4))
-                    } else {
-                        HnswGraph::TurboQuant8(Hnsw::new(hnsw_m, vecs.len(), max_layers, ef_construction, crate::core::index::distance::DistL2u8))
+                    match q {
+                        QuantizerImpl::TurboQuant(tq) => {
+                            if tq.bits() == 4 {
+                                HnswGraph::TurboQuant4(Hnsw::new(hnsw_m, vecs.len(), max_layers, ef_construction, crate::core::index::distance::DistL2u4))
+                            } else {
+                                HnswGraph::TurboQuant8(Hnsw::new(hnsw_m, vecs.len(), max_layers, ef_construction, crate::core::index::distance::DistL2u8))
+                            }
+                        },
+                        QuantizerImpl::Pq(pq) => {
+                            let dist = crate::core::index::pq::DistPqSdc { pq: std::sync::Arc::new(pq.clone()) };
+                            HnswGraph::Pq(Hnsw::new(hnsw_m, vecs.len(), max_layers, ef_construction, dist))
+                        }
                     }
                 } else {
                     match metric {
@@ -333,6 +345,7 @@ impl HnswIvfIndex {
 
     /// Search for k nearest neighbors
     pub fn search(&self, query: &VectorValue, k: usize, n_probe: usize, filter: Option<&roaring::RoaringBitmap>) -> Result<Vec<(usize, f32)>> {
+        let t_start = std::time::Instant::now();
         let query_f32 = match query {
             VectorValue::Float32(v) => v,
             VectorValue::Float16(v) => v,
@@ -342,10 +355,10 @@ impl HnswIvfIndex {
         };
 
         // Pre-quantize query if needed for faster graph traversal (SDC)
-        let effective_query = if let Some(QuantizerImpl::TurboQuant(ref q)) = self.quantizer {
-            VectorValue::Binary(q.encode(query_f32))
-        } else {
-            query.clone()
+        let effective_query = match self.quantizer {
+            Some(QuantizerImpl::TurboQuant(ref q)) => VectorValue::Binary(q.encode(query_f32)),
+            Some(QuantizerImpl::Pq(ref q)) => VectorValue::Binary(q.encode(query_f32)),
+            _ => query.clone()
         };
 
         // Step 1: Find n_probe nearest clusters (coarse search)
@@ -368,10 +381,14 @@ impl HnswIvfIndex {
         cluster_distances.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal));
         
         let clusters_to_search: Vec<usize> = cluster_distances.iter().take(n_probe).map(|(i, _)| *i).collect();
+        let t_coarse = t_start.elapsed();
+        let t_fine_start = std::time::Instant::now();
 
-        // Step 2: Search HNSW graphs in selected clusters in parallel (fine search)
+        // Step 2: Search HNSW graphs in selected clusters sequentially (fine search)
+        // Avoid into_par_iter() here to prevent Rayon thread pool contention,
+        // since queries are already parallelized across segments in query.rs
         let mut candidates: Vec<(usize, f32)> = clusters_to_search
-            .into_par_iter()
+            .into_iter()
             .flat_map(|cluster_id| {
                 if let Some((hnsw, row_id_mapping)) = self.cluster_graphs.get(&cluster_id) {
                     let cluster_k = if filter.is_some() { k * 4 } else { k * 2 };
@@ -416,9 +433,11 @@ impl HnswIvfIndex {
             .collect();
 
         // Step 3: Merge and return top-k
-        candidates.par_sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());
+        candidates.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());
         candidates.dedup_by_key(|x| x.0);  // Remove duplicates
         candidates.truncate(k);
+        let t_fine = t_fine_start.elapsed();
+        println!("Search Profile -> Coarse: {:?}, Fine: {:?}", t_coarse, t_fine);
         Ok(candidates)
     }
 
@@ -745,7 +764,12 @@ impl HnswIvfIndex {
                         }
                     } else if kv.key == "quantizer_config" {
                         if let Some(ref val) = kv.value {
-                            loaded_quantizer = serde_json::from_str::<QuantizerImpl>(val).ok();
+                            if let Some(mut q) = serde_json::from_str::<QuantizerImpl>(val).ok() {
+                                if let QuantizerImpl::Pq(ref mut pq) = q {
+                                    pq.init_lut();
+                                }
+                                loaded_quantizer = Some(q);
+                            }
                         }
                     }
                 }
@@ -805,13 +829,22 @@ impl HnswIvfIndex {
                     // For now we assume L2 if not specified or derive from path?
                     // Better: HnswIvfIndex should save metric in centroids parquet metadata.
                     
-                    let hnsw = if let Some(QuantizerImpl::TurboQuant(q)) = q_inner {
-                        if q.bits == 4 {
-                            HnswGraph::TurboQuant4(crate::core::index::hnsw_rs::hnswio::load_hnsw_with_dist(&mut graph_reader, &description, crate::core::index::distance::DistL2u4, &mut data_reader)
-                                .map_err(|e| anyhow::anyhow!("HNSW load failed: {}", e))?)
-                        } else {
-                            HnswGraph::TurboQuant8(crate::core::index::hnsw_rs::hnswio::load_hnsw_with_dist(&mut graph_reader, &description, crate::core::index::distance::DistL2u8, &mut data_reader)
-                                .map_err(|e| anyhow::anyhow!("HNSW load failed: {}", e))?)
+                    let hnsw = if let Some(q_impl) = q_inner {
+                        match q_impl {
+                            QuantizerImpl::TurboQuant(q) => {
+                                if q.bits() == 4 {
+                                    HnswGraph::TurboQuant4(crate::core::index::hnsw_rs::hnswio::load_hnsw_with_dist(&mut graph_reader, &description, crate::core::index::distance::DistL2u4, &mut data_reader)
+                                        .map_err(|e| anyhow::anyhow!("HNSW load failed: {}", e))?)
+                                } else {
+                                    HnswGraph::TurboQuant8(crate::core::index::hnsw_rs::hnswio::load_hnsw_with_dist(&mut graph_reader, &description, crate::core::index::distance::DistL2u8, &mut data_reader)
+                                        .map_err(|e| anyhow::anyhow!("HNSW load failed: {}", e))?)
+                                }
+                            },
+                            QuantizerImpl::Pq(pq) => {
+                                let dist = crate::core::index::pq::DistPqSdc { pq: std::sync::Arc::new(pq.clone()) };
+                                HnswGraph::Pq(crate::core::index::hnsw_rs::hnswio::load_hnsw_with_dist(&mut graph_reader, &description, dist, &mut data_reader)
+                                    .map_err(|e| anyhow::anyhow!("HNSW load failed: {}", e))?)
+                            }
                         }
                     } else {
                         match metric {