hyperstreamdb 0.5.0__tar.gz → 0.5.1__tar.gz

{hyperstreamdb-0.5.0 → hyperstreamdb-0.5.1}/Cargo.lock

@@ -3392,7 +3392,7 @@ dependencies = [
 
 [[package]]
 name = "hyperstreamdb"
-version = "0.5.0"
+version = "0.5.1"
 dependencies = [
  "ahash",
  "anyhow",

{hyperstreamdb-0.5.0 → hyperstreamdb-0.5.1}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hyperstreamdb"
-version = "0.5.0"
+version = "0.5.1"
 edition = "2021"
 license = "MIT AND Apache-2.0"
 description = "HyperStreamDB - Serverless Index-Streaming Database with Overlay Indexing and Vector Search"

{hyperstreamdb-0.5.0 → hyperstreamdb-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperstreamdb
-Version: 0.5.0
+Version: 0.5.1
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Rust

{hyperstreamdb-0.5.0 → hyperstreamdb-0.5.1}/python/hyperstreamdb/__init__.py

@@ -544,15 +544,15 @@ class Table:
         """
         return self.to_arrow(filter, vector_filter, columns, device=device, **kwargs)
 
-    def vector_search(self, column: str, query: List[float], k: int = 10, filter: Optional[str] = None, device: Optional[Any] = None, **kwargs):
+    def vector_search(self, column: str, query: List[float], k: int = 10, filter: Optional[str] = None, columns: Optional[List[str]] = None, device: Optional[Any] = None, **kwargs):
         """Backward compatibility alias for to_pandas with vector filter."""
         vf = {"column": column, "query": query, "k": k}
         vf.update(kwargs)
-        return self.to_pandas(filter=filter, vector_filter=vf, device=device)
+        return self.to_pandas(filter=filter, vector_filter=vf, columns=columns, device=device)
 
-    def search(self, column: str, query: List[float], k: int = 10, filter: Optional[str] = None, device: Optional[Any] = None, **kwargs):
+    def search(self, column: str, query: List[float], k: int = 10, filter: Optional[str] = None, columns: Optional[List[str]] = None, device: Optional[Any] = None, **kwargs):
         """Alias for vector_search."""
-        return self.vector_search(column, query, k, filter, device, **kwargs)
+        return self.vector_search(column, query, k, filter, columns, device, **kwargs)
 
     def filter(self, expr: Optional[str] = None, vector_filter: Optional[Union[Dict[str, Any], List[float]]] = None, **kwargs) -> 'Query':
         """

{hyperstreamdb-0.5.0 → hyperstreamdb-0.5.1}/src/core/index/hnsw_ivf.rs

@@ -345,6 +345,7 @@ impl HnswIvfIndex {
 
     /// Search for k nearest neighbors
     pub fn search(&self, query: &VectorValue, k: usize, n_probe: usize, filter: Option<&roaring::RoaringBitmap>) -> Result<Vec<(usize, f32)>> {
+        let t_start = std::time::Instant::now();
         let query_f32 = match query {
             VectorValue::Float32(v) => v,
             VectorValue::Float16(v) => v,
@@ -380,10 +381,14 @@ impl HnswIvfIndex {
         cluster_distances.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal));
         
         let clusters_to_search: Vec<usize> = cluster_distances.iter().take(n_probe).map(|(i, _)| *i).collect();
+        let t_coarse = t_start.elapsed();
+        let t_fine_start = std::time::Instant::now();
 
-        // Step 2: Search HNSW graphs in selected clusters in parallel (fine search)
+        // Step 2: Search HNSW graphs in selected clusters sequentially (fine search)
+        // Avoid into_par_iter() here to prevent Rayon thread pool contention,
+        // since queries are already parallelized across segments in query.rs
         let mut candidates: Vec<(usize, f32)> = clusters_to_search
-            .into_par_iter()
+            .into_iter()
             .flat_map(|cluster_id| {
                 if let Some((hnsw, row_id_mapping)) = self.cluster_graphs.get(&cluster_id) {
                     let cluster_k = if filter.is_some() { k * 4 } else { k * 2 };
@@ -428,9 +433,11 @@ impl HnswIvfIndex {
             .collect();
 
         // Step 3: Merge and return top-k
-        candidates.par_sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());
+        candidates.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());
         candidates.dedup_by_key(|x| x.0);  // Remove duplicates
         candidates.truncate(k);
+        let t_fine = t_fine_start.elapsed();
+        println!("Search Profile -> Coarse: {:?}, Fine: {:?}", t_coarse, t_fine);
         Ok(candidates)
     }
 

{hyperstreamdb-0.5.0 → hyperstreamdb-0.5.1}/src/core/segment.rs

@@ -279,7 +279,7 @@ impl HybridSegmentWriter {
             .set_compression(parquet::basic::Compression::UNCOMPRESSED)
             .set_dictionary_enabled(false)
             .set_statistics_enabled(parquet::file::properties::EnabledStatistics::None)
-            .set_data_page_size_limit(1024 * 1024); // 1MB pages for better random access
+            .set_data_page_size_limit(8192); // 8KB pages for highly granular random access
         
         // Enable Bloom Filters for Primary Keys if defined
         for pk in &self.primary_key {