hyperstreamdb 0.2.5__tar.gz → 0.2.6__tar.gz

{hyperstreamdb-0.2.5 → hyperstreamdb-0.2.6}/Cargo.lock

@@ -3375,7 +3375,7 @@ dependencies = [
 
 [[package]]
 name = "hyperstreamdb"
-version = "0.2.5"
+version = "0.2.6"
 dependencies = [
  "ahash 0.8.12",
  "anyhow",

{hyperstreamdb-0.2.5 → hyperstreamdb-0.2.6}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hyperstreamdb"
-version = "0.2.5"
+version = "0.2.6"
 edition = "2021"
 license = "MIT AND Apache-2.0"
 description = "HyperStreamDB - Serverless Index-Streaming Database with Overlay Indexing and Vector Search"

{hyperstreamdb-0.2.5 → hyperstreamdb-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperstreamdb
-Version: 0.2.5
+Version: 0.2.6
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Rust

{hyperstreamdb-0.2.5 → hyperstreamdb-0.2.6}/src/core/cache.rs

@@ -59,7 +59,22 @@ impl DiskCache {
             }
 
             let b = self.store.get(&object_store::path::Path::from(path)).await?.bytes().await?;
-            let _ = std::fs::write(&cache_path, &b);
+            
+            // Atomic write: write to unique temp file then rename
+            let thread_id = format!("{:?}", std::thread::current().id());
+            let temp_name = format!("{}.{}.{}.tmp", hash, std::process::id(), thread_id);
+            let temp_path = cache_dir.join(temp_name);
+            
+            if let Ok(mut f) = std::fs::File::create(&temp_path) {
+                use std::io::Write;
+                if f.write_all(&b).is_ok() {
+                    // rename is atomic on POSIX
+                    let _ = std::fs::rename(&temp_path, &cache_path);
+                } else {
+                    let _ = std::fs::remove_file(&temp_path);
+                }
+            }
+
             Ok(b)
         } else {
             let res = self.store.get(&object_store::path::Path::from(path)).await

{hyperstreamdb-0.2.5 → hyperstreamdb-0.2.6}/src/core/index/gpu.rs

@@ -424,6 +424,25 @@ impl ComputeContext {
     }
 
     pub fn auto_detect() -> Self {
+        {
+            let read = GLOBAL_GPU_CONTEXT.read().unwrap();
+            if let Some(ctx) = &*read {
+                return ctx.clone();
+            }
+        }
+
+        let mut write = GLOBAL_GPU_CONTEXT.write().unwrap();
+        // Check again after acquiring lock
+        if let Some(ctx) = &*write {
+            return ctx.clone();
+        }
+
+        let ctx = Self::do_auto_detect();
+        *write = Some(ctx.clone());
+        ctx
+    }
+
+    fn do_auto_detect() -> Self {
         #[cfg(feature = "cuda")]
         if let Ok(b) = CudaBackend::new(0) { return Self { backend: ComputeBackend::Cuda, device_id: 0, implementation: Some(Arc::new(b)) }; }
         #[cfg(all(target_os = "macos", feature = "mps"))]

{hyperstreamdb-0.2.5 → hyperstreamdb-0.2.6}/src/core/index/hnsw_ivf.rs

@@ -645,7 +645,7 @@ impl HnswIvfIndex {
 
         let disk_cache = DiskCache::new(store.clone());
 
-        let root_path = if base_path.contains("://") {
+        let mut root_path = if base_path.contains("://") {
              if let Ok(url) = url::Url::parse(base_path) {
                  url.path().trim_start_matches('/').to_string()
              } else {
@@ -655,6 +655,13 @@ impl HnswIvfIndex {
             base_path.to_string()
         };
 
+        // Robustness: If the path points to a specific cluster shard or graph file, strip it to get the base
+        if let Some(idx) = root_path.find(".cluster_") {
+            root_path = root_path[..idx].to_string();
+        } else if let Some(idx) = root_path.find(".hnsw.graph") {
+            root_path = root_path[..idx].to_string();
+        }
+
         let centroids_path = format!("{}.centroids.parquet", root_path);
         let centroids_bytes = disk_cache.get_bytes(&centroids_path).await?;
         
@@ -814,7 +821,14 @@ impl HnswIvfIndex {
         use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
         use std::fs::File;
 
-        let centroids_path = format!("{}.centroids.parquet", base_path);
+        let mut base_path_str = base_path.to_string();
+        if let Some(idx) = base_path_str.find(".cluster_") {
+            base_path_str = base_path_str[..idx].to_string();
+        } else if let Some(idx) = base_path_str.find(".hnsw.graph") {
+            base_path_str = base_path_str[..idx].to_string();
+        }
+
+        let centroids_path = format!("{}.centroids.parquet", base_path_str);
         let file = File::open(&centroids_path)?;
         let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
         let reader = builder.build()?;
@@ -991,4 +1005,25 @@ mod tests {
             assert!(filter.contains(*id as u32), "Result ID {} was not in the filter!", id);
         }
     }
+
+    #[test]
+    fn test_hnsw_ivf_path_robustness() {
+        let base_path_str = "/tmp/test_robust";
+        
+        let paths = vec![
+            format!("{}.cluster_0.hnsw.graph", base_path_str),
+            format!("{}.cluster_99.hnsw.graph", base_path_str),
+            format!("{}.hnsw.graph", base_path_str),
+        ];
+
+        for p in paths {
+            let mut root = p.clone();
+            if let Some(idx) = root.find(".cluster_") {
+                root = root[..idx].to_string();
+            } else if let Some(idx) = root.find(".hnsw.graph") {
+                root = root[..idx].to_string();
+            }
+            assert_eq!(root, base_path_str, "Failed to strip suffix from {}", p);
+        }
+    }
 }

{hyperstreamdb-0.2.5 → hyperstreamdb-0.2.6}/src/core/index/hnsw_rs/flatten.rs

@@ -205,17 +205,15 @@ fn test_dump_reload_graph_flatten() {
     let neighborhood_before_dump = FlatNeighborhood::from(&hnsw);
     let nbg_2_before = neighborhood_before_dump.get_neighbours(2).unwrap();
     println!("voisins du point 2 {:?}", nbg_2_before);
-    // dump in a file. Must take care of name as tests runs in // !!!
-    let fname = String::from("dumpreloadtestflat");
-    let _res = hnsw.file_dump(&fname);
-    // This will dump in 2 files named dumpreloadtest.hnsw.graph and dumpreloadtest.hnsw.data
-    //
+    // dump in a file. Use tempdir for parallel safety.
+    let temp_dir = tempfile::tempdir().unwrap();
+    let fname = temp_dir.path().join("dumpreloadtestflat");
+    let fname_str = fname.to_str().unwrap().to_string();
+    let _res = hnsw.file_dump(&fname_str);
+
     // reload
     log::debug!("\n\n  hnsw reload");
-    // we will need a procedural macro to get from distance name to its instanciation. 
-    // from now on we test with DistL1
-    let graphfname = String::from("dumpreloadtestflat.hnsw.graph");
-    let graphpath = PathBuf::from(graphfname);
+    let graphpath = PathBuf::from(format!("{}.hnsw.graph", fname_str));
     let graphfileres = OpenOptions::new().read(true).open(&graphpath);
     if graphfileres.is_err() {
         println!("test_dump_reload: could not open file {:?}", graphpath.as_os_str());
@@ -223,8 +221,7 @@ fn test_dump_reload_graph_flatten() {
     }
     let graphfile = graphfileres.unwrap();
     //  
-    let datafname = String::from("dumpreloadtestflat.hnsw.data");
-    let datapath = PathBuf::from(datafname);
+    let datapath = PathBuf::from(format!("{}.hnsw.data", fname_str));
     let datafileres = OpenOptions::new().read(true).open(&datapath);
     if datafileres.is_err() {
         println!("test_dump_reload : could not open file {:?}", datapath.as_os_str());

{hyperstreamdb-0.2.5 → hyperstreamdb-0.2.6}/src/core/index/hnsw_rs/hnswio.rs

@@ -783,19 +783,15 @@ fn test_dump_reload_1() {
     for i in 0..data.len() {
         hnsw.insert((&data[i], i));
     }
-    // some loggin info
-    hnsw.dump_layer_info();
-    // dump in a file.  Must take care of name as tests runs in // !!!
-    let fname = String::from("dumpreloadtest1");
-    let _res = hnsw.file_dump(&fname);
-    // This will dump in 2 files named dumpreloadtest.hnsw.graph and dumpreloadtest.hnsw.data
-    //
+    // dump in a file. Use tempdir for parallel safety.
+    let temp_dir = tempfile::tempdir().unwrap();
+    let fname = temp_dir.path().join("dumpreloadtest1");
+    let fname_str = fname.to_str().unwrap().to_string();
+    let _res = hnsw.file_dump(&fname_str);
+
     // reload
     log::debug!("\n\n  hnsw reload");
-    // we will need a procedural macro to get from distance name to its instanciation. 
-    // from now on we test with DistL1
-    let graphfname = String::from("dumpreloadtest1.hnsw.graph");
-    let graphpath = PathBuf::from(graphfname);
+    let graphpath = PathBuf::from(format!("{}.hnsw.graph", fname_str));
     let graphfileres = OpenOptions::new().read(true).open(&graphpath);
     if graphfileres.is_err() {
         println!("test_dump_reload: could not open file {:?}", graphpath.as_os_str());
@@ -803,8 +799,7 @@ fn test_dump_reload_1() {
     }
     let graphfile = graphfileres.unwrap();
     //  
-    let datafname = String::from("dumpreloadtest1.hnsw.data");
-    let datapath = PathBuf::from(datafname);
+    let datapath = PathBuf::from(format!("{}.hnsw.data", fname_str));
     let datafileres = OpenOptions::new().read(true).open(&datapath);
     if datafileres.is_err() {
         println!("test_dump_reload : could not open file {:?}", datapath.as_os_str());
@@ -852,19 +847,15 @@ fn test_dump_reload_myfn() {
     for i in 0..data.len() {
         hnsw.insert((&data[i], i));
     }
-    // some loggin info
-    hnsw.dump_layer_info();
-    // dump in a file.  Must take care of name as tests runs in // !!!
-    let fname = String::from("dumpreloadtest_myfn");
-    let _res = hnsw.file_dump(&fname);
-    // This will dump in 2 files named dumpreloadtest.hnsw.graph and dumpreloadtest.hnsw.data
-    //
+    // dump in a file. Use tempdir for parallel safety.
+    let temp_dir = tempfile::tempdir().unwrap();
+    let fname = temp_dir.path().join("dumpreloadtest_myfn");
+    let fname_str = fname.to_str().unwrap().to_string();
+    let _res = hnsw.file_dump(&fname_str);
+
     // reload
     log::debug!("\n\n  hnsw reload");
-    // we will need a procedural macro to get from distance name to its instanciation. 
-    // from now on we test with DistL1
-    let graphfname = String::from("dumpreloadtest_myfn.hnsw.graph");
-    let graphpath = PathBuf::from(graphfname);
+    let graphpath = PathBuf::from(format!("{}.hnsw.graph", fname_str));
     let graphfileres = OpenOptions::new().read(true).open(&graphpath);
     if graphfileres.is_err() {
         println!("test_dump_reload: could not open file {:?}", graphpath.as_os_str());
@@ -872,8 +863,7 @@ fn test_dump_reload_myfn() {
     }
     let graphfile = graphfileres.unwrap();
     //  
-    let datafname = String::from("dumpreloadtest_myfn.hnsw.data");
-    let datapath = PathBuf::from(datafname);
+    let datapath = PathBuf::from(format!("{}.hnsw.data", fname_str));
     let datafileres = OpenOptions::new().read(true).open(&datapath);
     if datafileres.is_err() {
         println!("test_dump_reload : could not open file {:?}", datapath.as_os_str());
@@ -918,19 +908,15 @@ fn test_dump_reload_graph_only() {
     for i in 0..data.len() {
         hnsw.insert((&data[i], i));
     }
-    // some loggin info
-    hnsw.dump_layer_info();
-    // dump in a file. Must take care of name as tests runs in // !!!
-    let fname = String::from("dumpreloadtestgraph");
-    let _res = hnsw.file_dump(&fname);
-    // This will dump in 2 files named dumpreloadtest.hnsw.graph and dumpreloadtest.hnsw.data
-    //
+    // dump in a file. Use tempdir for parallel safety.
+    let temp_dir = tempfile::tempdir().unwrap();
+    let fname = temp_dir.path().join("dumpreloadtestgraph");
+    let fname_str = fname.to_str().unwrap().to_string();
+    let _res = hnsw.file_dump(&fname_str);
+
     // reload
     log::debug!("\n\n  hnsw reload");
-    // we will need a procedural macro to get from distance name to its instanciation. 
-    // from now on we test with DistL1
-    let graphfname = String::from("dumpreloadtestgraph.hnsw.graph");
-    let graphpath = PathBuf::from(graphfname);
+    let graphpath = PathBuf::from(format!("{}.hnsw.graph", fname_str));
     let graphfileres = OpenOptions::new().read(true).open(&graphpath);
     if graphfileres.is_err() {
         println!("test_dump_reload: could not open file {:?}", graphpath.as_os_str());
@@ -938,8 +924,7 @@ fn test_dump_reload_graph_only() {
     }
     let graphfile = graphfileres.unwrap();
     //  
-    let datafname = String::from("dumpreloadtestgraph.hnsw.data");
-    let datapath = PathBuf::from(datafname);
+    let datapath = PathBuf::from(format!("{}.hnsw.data", fname_str));
     let datafileres = OpenOptions::new().read(true).open(&datapath);
     if datafileres.is_err() {
         println!("test_dump_reload : could not open file {:?}", datapath.as_os_str());

{hyperstreamdb-0.2.5 → hyperstreamdb-0.2.6}/src/core/index/memory.rs

@@ -125,13 +125,16 @@ impl InMemoryVectorIndex {
             let values = fsl.values().as_any().downcast_ref::<Float32Array>()
                 .context("Expected Float32Array values in FixedSizeListArray")?;
             
-            // Bulk extend for high performance
-            self.vectors.extend_from_slice(values.values());
+            // Respect slicing: only copy the range of the value array that belongs to this FSL slice
+            let start_offset = fsl.offset() * self.dim;
+            let len = fsl.len() * self.dim;
+            let slice = &values.values()[start_offset..start_offset + len];
+            
+            self.vectors.extend_from_slice(slice);
             self.count += fsl.len();
         } else if let Some(list) = col.as_any().downcast_ref::<ListArray>() {
             for i in 0..list.len() {
                 if list.is_null(i) {
-                     // Fill with zeros to maintain alignment or handle nulls
                      self.vectors.extend(std::iter::repeat_n(0.0, self.dim));
                 } else {
                     let vector_array = list.value(i);
@@ -244,4 +247,39 @@ mod tests {
         assert_eq!(results[0].0, 0); // v1 is closest
         assert_eq!(results[1].0, 1); // v2 is second closest
     }
+
+    #[test]
+    fn test_memory_index_sliced_array() {
+        let dim = 4;
+        let mut index = InMemoryVectorIndex::new(dim);
+        
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("vec", DataType::FixedSizeList(
+                Arc::new(Field::new("item", DataType::Float32, true)),
+                dim as i32
+            ), true),
+        ]));
+
+        let v1 = vec![Some(1.0), Some(0.0), Some(0.0), Some(0.0)];
+        let v2 = vec![Some(0.0), Some(1.0), Some(0.0), Some(0.0)];
+        let v3 = vec![Some(0.0), Some(0.0), Some(1.0), Some(0.0)];
+
+        let array = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+            vec![Some(v1), Some(v2.clone()), Some(v3)],
+            dim as i32
+        );
+
+        // Slice the array to only include v2
+        let sliced_array = array.slice(1, 1);
+        let batch = RecordBatch::try_new(schema, vec![Arc::new(sliced_array)]).unwrap();
+        index.insert_batch(&batch, "vec", 0).unwrap();
+
+        assert_eq!(index.count, 1);
+        assert_eq!(index.vectors.len(), dim);
+        
+        let query = crate::core::index::VectorValue::Float32(v2.into_iter().map(|x| x.unwrap()).collect());
+        let results = index.search(&query, 1, None);
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0].1, 0.0); // Exact match
+    }
 }

{hyperstreamdb-0.2.5 → hyperstreamdb-0.2.6}/src/core/segment.rs

@@ -105,22 +105,23 @@ impl HybridSegmentWriter {
                     offset: None,
                     length: None,
                 });
-            } else if filename.contains(".hnsw.") {
-                  // Vector Index
-                  let parts: Vec<&str> = filename.split('.').collect();
-                  let col = parts.get(1).map(|s| s.to_string());
-                  
-                  if (filename.ends_with(".hnsw.graph") || filename.ends_with(".centroids.parquet"))
-                      && !index_files.iter().any(|idx| idx.column_name == col && idx.index_type == "vector") {
-                          index_files.push(crate::core::manifest::IndexFile {
-                            file_path: filename.clone(),
-                            index_type: "vector".to_string(),
-                            column_name: col,
-                            blob_type: None,
-                            offset: None,
-                            length: None,
-                          });
-                      }
+            } else if filename.contains(".hnsw.") || filename.ends_with(".centroids.parquet") {
+                   // Vector Index
+                   let col_opt = filename.split('.').nth(1).map(|s| s.to_string());
+
+                   if let Some(col) = col_opt {
+                       if !index_files.iter().any(|idx| idx.column_name.as_ref() == Some(&col) && idx.index_type == "vector") {
+                           let base_path = format!("{}.{}", self.config.segment_id, col);
+                           index_files.push(crate::core::manifest::IndexFile {
+                               file_path: base_path,
+                               index_type: "vector".to_string(),
+                               column_name: Some(col),
+                               blob_type: None,
+                               offset: None,
+                               length: None,
+                           });
+                       }
+                   }
             }
         }
         

{hyperstreamdb-0.2.5 → hyperstreamdb-0.2.6}/src/telemetry/tracing.rs

@@ -15,15 +15,28 @@ impl<S: Subscriber> Layer<S> for ReloadableFilter {
     fn enabled(&self, metadata: &Metadata<'_>, ctx: Context<'_, S>) -> bool {
         self.filter.read().unwrap().enabled(metadata, ctx)
     }
-
-    fn on_event(&self, event: &tracing::Event<'_>, ctx: Context<'_, S>) {
-        self.filter.read().unwrap().on_event(event, ctx);
-    }
 }
 
 static SHARED_FILTER: once_cell::sync::OnceCell<Arc<RwLock<EnvFilter>>> = once_cell::sync::OnceCell::new();
+static INIT: std::sync::Once = std::sync::Once::new();
 
 pub fn init_tracing(service_name: &str) -> Result<(), Box<dyn std::error::Error>> {
+    let mut init_err: Option<Box<dyn std::error::Error>> = None;
+    
+    INIT.call_once(|| {
+        if let Err(e) = do_init_tracing(service_name) {
+            init_err = Some(e);
+        }
+    });
+
+    if let Some(e) = init_err {
+        return Err(e);
+    }
+    
+    Ok(())
+}
+
+fn do_init_tracing(service_name: &str) -> Result<(), Box<dyn std::error::Error>> {
     global::set_text_map_propagator(TraceContextPropagator::new());
 
     let base_filter = EnvFilter::from_default_env();