hyperstreamdb 0.2.4__tar.gz → 0.2.6__tar.gz

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/Cargo.lock

@@ -3375,7 +3375,7 @@ dependencies = [
 
 [[package]]
 name = "hyperstreamdb"
-version = "0.2.4"
+version = "0.2.6"
 dependencies = [
  "ahash 0.8.12",
  "anyhow",
@@ -7141,6 +7141,16 @@ dependencies = [
  "web-time",
 ]
 
+[[package]]
+name = "tracing-serde"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1"
+dependencies = [
+ "serde",
+ "tracing-core",
+]
+
 [[package]]
 name = "tracing-subscriber"
 version = "0.3.23"
@@ -7151,12 +7161,15 @@ dependencies = [
  "nu-ansi-term",
  "once_cell",
  "regex-automata",
+ "serde",
+ "serde_json",
  "sharded-slab",
  "smallvec",
  "thread_local",
  "tracing",
  "tracing-core",
  "tracing-log",
+ "tracing-serde",
 ]
 
 [[package]]

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hyperstreamdb"
-version = "0.2.4"
+version = "0.2.6"
 edition = "2021"
 license = "MIT AND Apache-2.0"
 description = "HyperStreamDB - Serverless Index-Streaming Database with Overlay Indexing and Vector Search"
@@ -102,7 +102,7 @@ anyhow = "1.0"
 
 # logging
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
 url = "2.5"
 uuid = { version = "1.10", features = ["v4", "fast-rng", "macro-diagnostics"] }
 chrono = "0.4"

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperstreamdb
-Version: 0.2.4
+Version: 0.2.6
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Rust

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/src/core/cache.rs

@@ -59,7 +59,22 @@ impl DiskCache {
             }
 
             let b = self.store.get(&object_store::path::Path::from(path)).await?.bytes().await?;
-            let _ = std::fs::write(&cache_path, &b);
+            
+            // Atomic write: write to unique temp file then rename
+            let thread_id = format!("{:?}", std::thread::current().id());
+            let temp_name = format!("{}.{}.{}.tmp", hash, std::process::id(), thread_id);
+            let temp_path = cache_dir.join(temp_name);
+            
+            if let Ok(mut f) = std::fs::File::create(&temp_path) {
+                use std::io::Write;
+                if f.write_all(&b).is_ok() {
+                    // rename is atomic on POSIX
+                    let _ = std::fs::rename(&temp_path, &cache_path);
+                } else {
+                    let _ = std::fs::remove_file(&temp_path);
+                }
+            }
+
             Ok(b)
         } else {
             let res = self.store.get(&object_store::path::Path::from(path)).await

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/src/core/compaction.rs

@@ -510,7 +510,7 @@ mod tests {
         // - Manifests v1, v2, v3
         // - The 3 original data files (since they are ONLY in v1-v3)
         let deleted = table.vacuum_async(1).await?;
-        println!("Vacuum deleted {} files", deleted);
+        tracing::info!("Vacuum deleted {} files", deleted);
         
         // We expect at least 3 data files + 3 manifest files = 6 files deleted
         assert!(deleted >= 6);

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/src/core/ffi.rs

@@ -108,7 +108,7 @@ pub extern "system" fn Java_com_hyperstreamdb_trino_HyperStreamDBPageSource_open
             Box::into_raw(Box::new(session)) as jlong
         },
         Err(e) => {
-            eprintln!("FFI Error opening session: {}", e);
+            tracing::error!("FFI Error opening session: {}", e);
             0
         }
     }
@@ -196,7 +196,7 @@ pub extern "system" fn Java_com_hyperstreamdb_trino_HyperStreamDBSplitManager_ge
             }
         },
         Err(e) => {
-             eprintln!("FFI Error creating table: {}", e);
+             tracing::error!("FFI Error creating table: {}", e);
              "[]".to_string()
         }
     };
@@ -234,7 +234,7 @@ pub extern "system" fn Java_com_hyperstreamdb_spark_HyperStreamScanBuilder_listD
             }
         },
         Err(e) => {
-             eprintln!("FFI Error creating table: {}", e);
+             tracing::error!("FFI Error creating table: {}", e);
              "[]".to_string()
         }
     };
@@ -267,7 +267,7 @@ fn open_session_helper(mut env: JNIEnv, path: JString) -> jlong {
     match HyperStreamSession::new(&path_str) {
         Ok(session) => Box::into_raw(Box::new(session)) as jlong,
         Err(e) => {
-            eprintln!("FFI Error opening session: {}", e);
+            tracing::error!("FFI Error opening session: {}", e);
             0
         }
     }

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/src/core/iceberg.rs

@@ -1240,19 +1240,24 @@ impl IcebergWriter {
                  s if s.starts_with("truncate[") => r#"["null", "string"]"#,
                  "identity" => {
                      let source_id = field.source_ids.first().copied().or(field.source_id);
-                     if let Some(id) = source_id {
-                         if let Some(source_field) = schema.fields.iter().find(|f| f.id == id) {
-                             match source_field.type_str.as_str() {
-                                 "Int32" | "int" => r#"["null", "int"]"#,
-                                 "Int64" | "long" => r#"["null", "long"]"#,
-                                 "Float32" | "float" => r#"["null", "float"]"#,
-                                 "Float64" | "double" => r#"["null", "double"]"#,
-                                 "Boolean" | "bool" | "boolean" => r#"["null", "boolean"]"#,
-                                 "string" | "utf8" | "utf-8" | "String" | "Utf8" => r#"["null", "string"]"#,
-                                 _ => r#"["null", "string"]"#,
-                             }
+                     // Robustness: Prioritize Name-based resolution, fallback to ID (Name-First strategy)
+                     let resolved_field = schema.fields.iter().find(|sf| sf.name == field.name).cloned().or_else(|| {
+                         if let Some(id) = source_id {
+                             schema.fields.iter().find(|f| f.id == id).cloned()
                          } else {
-                             r#"["null", "string"]"#
+                             None
+                         }
+                     });
+
+                     if let Some(f) = resolved_field {
+                         match f.type_str.as_str() {
+                             "Int32" | "int" => r#"["null", "int"]"#,
+                             "Int64" | "long" => r#"["null", "long"]"#,
+                             "Float32" | "float" => r#"["null", "float"]"#,
+                             "Float64" | "double" => r#"["null", "double"]"#,
+                             "Boolean" | "bool" | "boolean" => r#"["null", "boolean"]"#,
+                             "string" | "utf8" | "utf-8" | "String" | "Utf8" => r#"["null", "string"]"#,
+                             _ => r#"["null", "string"]"#,
                          }
                      } else {
                          r#"["null", "string"]"#

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/src/core/index/gpu.rs

@@ -424,6 +424,25 @@ impl ComputeContext {
     }
 
     pub fn auto_detect() -> Self {
+        {
+            let read = GLOBAL_GPU_CONTEXT.read().unwrap();
+            if let Some(ctx) = &*read {
+                return ctx.clone();
+            }
+        }
+
+        let mut write = GLOBAL_GPU_CONTEXT.write().unwrap();
+        // Check again after acquiring lock
+        if let Some(ctx) = &*write {
+            return ctx.clone();
+        }
+
+        let ctx = Self::do_auto_detect();
+        *write = Some(ctx.clone());
+        ctx
+    }
+
+    fn do_auto_detect() -> Self {
         #[cfg(feature = "cuda")]
         if let Ok(b) = CudaBackend::new(0) { return Self { backend: ComputeBackend::Cuda, device_id: 0, implementation: Some(Arc::new(b)) }; }
         #[cfg(all(target_os = "macos", feature = "mps"))]

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/src/core/index/hnsw_ivf.rs

@@ -645,7 +645,7 @@ impl HnswIvfIndex {
 
         let disk_cache = DiskCache::new(store.clone());
 
-        let root_path = if base_path.contains("://") {
+        let mut root_path = if base_path.contains("://") {
              if let Ok(url) = url::Url::parse(base_path) {
                  url.path().trim_start_matches('/').to_string()
              } else {
@@ -655,6 +655,13 @@ impl HnswIvfIndex {
             base_path.to_string()
         };
 
+        // Robustness: If the path points to a specific cluster shard or graph file, strip it to get the base
+        if let Some(idx) = root_path.find(".cluster_") {
+            root_path = root_path[..idx].to_string();
+        } else if let Some(idx) = root_path.find(".hnsw.graph") {
+            root_path = root_path[..idx].to_string();
+        }
+
         let centroids_path = format!("{}.centroids.parquet", root_path);
         let centroids_bytes = disk_cache.get_bytes(&centroids_path).await?;
         
@@ -814,7 +821,14 @@ impl HnswIvfIndex {
         use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
         use std::fs::File;
 
-        let centroids_path = format!("{}.centroids.parquet", base_path);
+        let mut base_path_str = base_path.to_string();
+        if let Some(idx) = base_path_str.find(".cluster_") {
+            base_path_str = base_path_str[..idx].to_string();
+        } else if let Some(idx) = base_path_str.find(".hnsw.graph") {
+            base_path_str = base_path_str[..idx].to_string();
+        }
+
+        let centroids_path = format!("{}.centroids.parquet", base_path_str);
         let file = File::open(&centroids_path)?;
         let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
         let reader = builder.build()?;
@@ -991,4 +1005,25 @@ mod tests {
             assert!(filter.contains(*id as u32), "Result ID {} was not in the filter!", id);
         }
     }
+
+    #[test]
+    fn test_hnsw_ivf_path_robustness() {
+        let base_path_str = "/tmp/test_robust";
+        
+        let paths = vec![
+            format!("{}.cluster_0.hnsw.graph", base_path_str),
+            format!("{}.cluster_99.hnsw.graph", base_path_str),
+            format!("{}.hnsw.graph", base_path_str),
+        ];
+
+        for p in paths {
+            let mut root = p.clone();
+            if let Some(idx) = root.find(".cluster_") {
+                root = root[..idx].to_string();
+            } else if let Some(idx) = root.find(".hnsw.graph") {
+                root = root[..idx].to_string();
+            }
+            assert_eq!(root, base_path_str, "Failed to strip suffix from {}", p);
+        }
+    }
 }

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/src/core/index/hnsw_rs/flatten.rs

@@ -205,17 +205,15 @@ fn test_dump_reload_graph_flatten() {
     let neighborhood_before_dump = FlatNeighborhood::from(&hnsw);
     let nbg_2_before = neighborhood_before_dump.get_neighbours(2).unwrap();
     println!("voisins du point 2 {:?}", nbg_2_before);
-    // dump in a file. Must take care of name as tests runs in // !!!
-    let fname = String::from("dumpreloadtestflat");
-    let _res = hnsw.file_dump(&fname);
-    // This will dump in 2 files named dumpreloadtest.hnsw.graph and dumpreloadtest.hnsw.data
-    //
+    // dump in a file. Use tempdir for parallel safety.
+    let temp_dir = tempfile::tempdir().unwrap();
+    let fname = temp_dir.path().join("dumpreloadtestflat");
+    let fname_str = fname.to_str().unwrap().to_string();
+    let _res = hnsw.file_dump(&fname_str);
+
     // reload
     log::debug!("\n\n  hnsw reload");
-    // we will need a procedural macro to get from distance name to its instanciation. 
-    // from now on we test with DistL1
-    let graphfname = String::from("dumpreloadtestflat.hnsw.graph");
-    let graphpath = PathBuf::from(graphfname);
+    let graphpath = PathBuf::from(format!("{}.hnsw.graph", fname_str));
     let graphfileres = OpenOptions::new().read(true).open(&graphpath);
     if graphfileres.is_err() {
         println!("test_dump_reload: could not open file {:?}", graphpath.as_os_str());
@@ -223,8 +221,7 @@ fn test_dump_reload_graph_flatten() {
     }
     let graphfile = graphfileres.unwrap();
     //  
-    let datafname = String::from("dumpreloadtestflat.hnsw.data");
-    let datapath = PathBuf::from(datafname);
+    let datapath = PathBuf::from(format!("{}.hnsw.data", fname_str));
     let datafileres = OpenOptions::new().read(true).open(&datapath);
     if datafileres.is_err() {
         println!("test_dump_reload : could not open file {:?}", datapath.as_os_str());

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/src/core/index/hnsw_rs/hnswio.rs

@@ -783,19 +783,15 @@ fn test_dump_reload_1() {
     for i in 0..data.len() {
         hnsw.insert((&data[i], i));
     }
-    // some loggin info
-    hnsw.dump_layer_info();
-    // dump in a file.  Must take care of name as tests runs in // !!!
-    let fname = String::from("dumpreloadtest1");
-    let _res = hnsw.file_dump(&fname);
-    // This will dump in 2 files named dumpreloadtest.hnsw.graph and dumpreloadtest.hnsw.data
-    //
+    // dump in a file. Use tempdir for parallel safety.
+    let temp_dir = tempfile::tempdir().unwrap();
+    let fname = temp_dir.path().join("dumpreloadtest1");
+    let fname_str = fname.to_str().unwrap().to_string();
+    let _res = hnsw.file_dump(&fname_str);
+
     // reload
     log::debug!("\n\n  hnsw reload");
-    // we will need a procedural macro to get from distance name to its instanciation. 
-    // from now on we test with DistL1
-    let graphfname = String::from("dumpreloadtest1.hnsw.graph");
-    let graphpath = PathBuf::from(graphfname);
+    let graphpath = PathBuf::from(format!("{}.hnsw.graph", fname_str));
     let graphfileres = OpenOptions::new().read(true).open(&graphpath);
     if graphfileres.is_err() {
         println!("test_dump_reload: could not open file {:?}", graphpath.as_os_str());
@@ -803,8 +799,7 @@ fn test_dump_reload_1() {
     }
     let graphfile = graphfileres.unwrap();
     //  
-    let datafname = String::from("dumpreloadtest1.hnsw.data");
-    let datapath = PathBuf::from(datafname);
+    let datapath = PathBuf::from(format!("{}.hnsw.data", fname_str));
     let datafileres = OpenOptions::new().read(true).open(&datapath);
     if datafileres.is_err() {
         println!("test_dump_reload : could not open file {:?}", datapath.as_os_str());
@@ -852,19 +847,15 @@ fn test_dump_reload_myfn() {
     for i in 0..data.len() {
         hnsw.insert((&data[i], i));
     }
-    // some loggin info
-    hnsw.dump_layer_info();
-    // dump in a file.  Must take care of name as tests runs in // !!!
-    let fname = String::from("dumpreloadtest_myfn");
-    let _res = hnsw.file_dump(&fname);
-    // This will dump in 2 files named dumpreloadtest.hnsw.graph and dumpreloadtest.hnsw.data
-    //
+    // dump in a file. Use tempdir for parallel safety.
+    let temp_dir = tempfile::tempdir().unwrap();
+    let fname = temp_dir.path().join("dumpreloadtest_myfn");
+    let fname_str = fname.to_str().unwrap().to_string();
+    let _res = hnsw.file_dump(&fname_str);
+
     // reload
     log::debug!("\n\n  hnsw reload");
-    // we will need a procedural macro to get from distance name to its instanciation. 
-    // from now on we test with DistL1
-    let graphfname = String::from("dumpreloadtest_myfn.hnsw.graph");
-    let graphpath = PathBuf::from(graphfname);
+    let graphpath = PathBuf::from(format!("{}.hnsw.graph", fname_str));
     let graphfileres = OpenOptions::new().read(true).open(&graphpath);
     if graphfileres.is_err() {
         println!("test_dump_reload: could not open file {:?}", graphpath.as_os_str());
@@ -872,8 +863,7 @@ fn test_dump_reload_myfn() {
     }
     let graphfile = graphfileres.unwrap();
     //  
-    let datafname = String::from("dumpreloadtest_myfn.hnsw.data");
-    let datapath = PathBuf::from(datafname);
+    let datapath = PathBuf::from(format!("{}.hnsw.data", fname_str));
     let datafileres = OpenOptions::new().read(true).open(&datapath);
     if datafileres.is_err() {
         println!("test_dump_reload : could not open file {:?}", datapath.as_os_str());
@@ -918,19 +908,15 @@ fn test_dump_reload_graph_only() {
     for i in 0..data.len() {
         hnsw.insert((&data[i], i));
     }
-    // some loggin info
-    hnsw.dump_layer_info();
-    // dump in a file. Must take care of name as tests runs in // !!!
-    let fname = String::from("dumpreloadtestgraph");
-    let _res = hnsw.file_dump(&fname);
-    // This will dump in 2 files named dumpreloadtest.hnsw.graph and dumpreloadtest.hnsw.data
-    //
+    // dump in a file. Use tempdir for parallel safety.
+    let temp_dir = tempfile::tempdir().unwrap();
+    let fname = temp_dir.path().join("dumpreloadtestgraph");
+    let fname_str = fname.to_str().unwrap().to_string();
+    let _res = hnsw.file_dump(&fname_str);
+
     // reload
     log::debug!("\n\n  hnsw reload");
-    // we will need a procedural macro to get from distance name to its instanciation. 
-    // from now on we test with DistL1
-    let graphfname = String::from("dumpreloadtestgraph.hnsw.graph");
-    let graphpath = PathBuf::from(graphfname);
+    let graphpath = PathBuf::from(format!("{}.hnsw.graph", fname_str));
     let graphfileres = OpenOptions::new().read(true).open(&graphpath);
     if graphfileres.is_err() {
         println!("test_dump_reload: could not open file {:?}", graphpath.as_os_str());
@@ -938,8 +924,7 @@ fn test_dump_reload_graph_only() {
     }
     let graphfile = graphfileres.unwrap();
     //  
-    let datafname = String::from("dumpreloadtestgraph.hnsw.data");
-    let datapath = PathBuf::from(datafname);
+    let datapath = PathBuf::from(format!("{}.hnsw.data", fname_str));
     let datafileres = OpenOptions::new().read(true).open(&datapath);
     if datafileres.is_err() {
         println!("test_dump_reload : could not open file {:?}", datapath.as_os_str());

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/src/core/index/memory.rs

@@ -125,13 +125,16 @@ impl InMemoryVectorIndex {
             let values = fsl.values().as_any().downcast_ref::<Float32Array>()
                 .context("Expected Float32Array values in FixedSizeListArray")?;
             
-            // Bulk extend for high performance
-            self.vectors.extend_from_slice(values.values());
+            // Respect slicing: only copy the range of the value array that belongs to this FSL slice
+            let start_offset = fsl.offset() * self.dim;
+            let len = fsl.len() * self.dim;
+            let slice = &values.values()[start_offset..start_offset + len];
+            
+            self.vectors.extend_from_slice(slice);
             self.count += fsl.len();
         } else if let Some(list) = col.as_any().downcast_ref::<ListArray>() {
             for i in 0..list.len() {
                 if list.is_null(i) {
-                     // Fill with zeros to maintain alignment or handle nulls
                      self.vectors.extend(std::iter::repeat_n(0.0, self.dim));
                 } else {
                     let vector_array = list.value(i);
@@ -244,4 +247,39 @@ mod tests {
         assert_eq!(results[0].0, 0); // v1 is closest
         assert_eq!(results[1].0, 1); // v2 is second closest
     }
+
+    #[test]
+    fn test_memory_index_sliced_array() {
+        let dim = 4;
+        let mut index = InMemoryVectorIndex::new(dim);
+        
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("vec", DataType::FixedSizeList(
+                Arc::new(Field::new("item", DataType::Float32, true)),
+                dim as i32
+            ), true),
+        ]));
+
+        let v1 = vec![Some(1.0), Some(0.0), Some(0.0), Some(0.0)];
+        let v2 = vec![Some(0.0), Some(1.0), Some(0.0), Some(0.0)];
+        let v3 = vec![Some(0.0), Some(0.0), Some(1.0), Some(0.0)];
+
+        let array = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+            vec![Some(v1), Some(v2.clone()), Some(v3)],
+            dim as i32
+        );
+
+        // Slice the array to only include v2
+        let sliced_array = array.slice(1, 1);
+        let batch = RecordBatch::try_new(schema, vec![Arc::new(sliced_array)]).unwrap();
+        index.insert_batch(&batch, "vec", 0).unwrap();
+
+        assert_eq!(index.count, 1);
+        assert_eq!(index.vectors.len(), dim);
+        
+        let query = crate::core::index::VectorValue::Float32(v2.into_iter().map(|x| x.unwrap()).collect());
+        let results = index.search(&query, 1, None);
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0].1, 0.0); // Exact match
+    }
 }

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/src/core/manifest.rs

@@ -1628,15 +1628,19 @@ impl PartitionSpec {
         for i in 0..batch.num_rows() {
             let mut key = Vec::with_capacity(self.fields.len());
             for field in &self.fields {
-                // Determine source columns
+                                // Determine source columns
                 let source_ids = field.get_source_ids();
                 let mut cols = Vec::new();
                 
-                // Try finding by name first
+                // Prioritize finding by Name (most intuitive for users)
+                let mut found = false;
                 if let Ok(idx) = batch.schema().index_of(&field.name) {
                     cols.push(batch.column(idx));
-                } else {
-                    // Search by Iceberg IDs
+                    found = true;
+                }
+
+                // Fallback to Iceberg IDs only if name lookup failed
+                if !found {
                     for id in &source_ids {
                         let idx = batch.schema().fields().iter().position(|f| {
                             f.metadata().get("iceberg.id")

{hyperstreamdb-0.2.4 → hyperstreamdb-0.2.6}/src/core/merge.rs

@@ -126,7 +126,7 @@ impl MergePlanner {
     ) -> Result<Vec<(Option<SegmentId>, SegmentId)>> 
     where F: Fn(&SegmentId, &str) -> Result<Option<RoaringBitmap>>
     {
-        println!("Executing Merge on {} candidates", candidate_segments.len());
+        tracing::info!("Executing Merge on {} candidates", candidate_segments.len());
         
         let mut updates_by_segment: HashMap<SegmentId, Vec<usize>> = HashMap::new();
         let mut unmatched_rows: Vec<usize> = Vec::new();
@@ -198,7 +198,7 @@ impl MergePlanner {
         
         // 2. Process Updates (Copy-on-Write)
         for (seg_id, source_row_indices) in updates_by_segment {
-            println!("Updating Segment {} with {} rows", seg_id, source_row_indices.len());
+            tracing::info!("Updating Segment {} with {} rows", seg_id, source_row_indices.len());
             
             // A. Read Original Segment
             // Use empty base path for reader configuration because store is already rooted at base_path
@@ -276,7 +276,7 @@ impl MergePlanner {
         
         // 3. Process Inserts (Unmatched)
         if !unmatched_rows.is_empty() {
-            println!("Inserting {} new rows", unmatched_rows.len());
+            tracing::info!("Inserting {} new rows", unmatched_rows.len());
             let indices_arr = arrow::array::UInt32Array::from(unmatched_rows.iter().map(|&x| x as u32).collect::<Vec<u32>>());
             let inserts_batch = arrow::compute::take_record_batch(source_batch, &indices_arr)?;