hyperstreamdb 0.1.3__tar.gz → 0.1.6__tar.gz

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/Cargo.lock

@@ -3755,7 +3755,7 @@ dependencies = [
 
 [[package]]
 name = "hyperstreamdb"
-version = "0.1.3"
+version = "0.1.6"
 dependencies = [
  "ahash 0.8.12",
  "anyhow",

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hyperstreamdb"
-version = "0.1.3"
+version = "0.1.6"
 edition = "2021"
 license = "MIT AND Apache-2.0"
 description = "HyperStreamDB - Serverless Index-Streaming Database with Overlay Indexing and Vector Search"

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperstreamdb
-Version: 0.1.3
+Version: 0.1.6
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Rust
@@ -23,6 +23,7 @@ Requires-Dist: intel-gpu ; extra == 'all-gpu'
 Requires-Dist: pytest>=7.0 ; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.21 ; extra == 'dev'
 Requires-Dist: maturin>=1.7 ; extra == 'dev'
+Requires-Dist: maturin>=1.7 ; extra == 'src'
 Provides-Extra: all_gpu
 Provides-Extra: cuda
 Provides-Extra: dev
@@ -30,6 +31,7 @@ Provides-Extra: intel_cpu
 Provides-Extra: intel_gpu
 Provides-Extra: mps
 Provides-Extra: rocm
+Provides-Extra: src
 License-File: LICENSE-APACHE
 License-File: LICENSE-MIT
 Summary: HyperStreamDB - Serverless Index-Streaming Database with Overlay Indexing

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/pyproject.toml

@@ -47,6 +47,7 @@ rocm = []  # AMD (via OpenCL/ROCm)
 intel_gpu = [] # Intel (OpenCL)
 intel_cpu = [] # Intel (SIMD/AVX2)
 all_gpu = ["mps", "cuda", "rocm", "intel_gpu"]
+src = ["maturin>=1.7"]
 
 [project.urls]
 Homepage = "https://github.com/rla3rd/hyperstreamdb"
@@ -56,12 +57,12 @@ Repository = "https://github.com/rla3rd/hyperstreamdb"
 features = ["python"]
 # Map Python extras to Rust features
 extra-features = { mps = ["mps"], cuda = ["cuda"], rocm = ["rocm"], intel_gpu = ["intel_gpu"], intel_cpu = ["intel_cpu"] }
-module-name = "hyperstreamdb"
+module-name = "hyperstreamdb.hyperstreamdb"
 python-source = "python"
 # Build both the Rust library and Python bindings
 bindings = "pyo3"
 sdist-include = ["LICENSE*", "README.md", "THIRDPARTY_NOTICES.md", "vendor/**/*", "tests/**/*", "benches/**/*"]
 # Optimizations for WHEEL builds
-strip = true
+strip = false
 # Use sccache if available during build
 sccache = false

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/src/core/iceberg.rs

@@ -712,7 +712,10 @@ impl PositionDeleteReader {
                 }
 
                 if let (Some(fp), Some(p)) = (file_path, pos) {
-                    if fp == target_data_file_path {
+                    let fp_clean = fp.replace("file://", "");
+                    let target_clean = target_data_file_path.replace("file://", "");
+                    
+                    if fp_clean == target_clean || target_clean.ends_with(&fp_clean) || fp_clean.ends_with(&target_clean) {
                         deleted_positions.insert(p);
                     }
                 }
@@ -740,7 +743,11 @@ impl PositionDeleteReader {
                     .ok_or_else(|| anyhow::anyhow!("pos column is not int64"))?;
                 
                 for i in 0..batch.num_rows() {
-                    if file_paths.value(i) == target_data_file_path {
+                    let fp = file_paths.value(i);
+                    let fp_clean = fp.replace("file://", "");
+                    let target_clean = target_data_file_path.replace("file://", "");
+                    
+                    if fp_clean == target_clean || target_clean.ends_with(&fp_clean) || fp_clean.ends_with(&target_clean) {
                         deleted_positions.insert(positions.value(i));
                     }
                 }

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/src/core/index/hnsw_rs/dist.rs

@@ -14,6 +14,7 @@ use simdeez::sse2::*;
 use simdeez::*;
 
 
+
 /// The trait describing distance.
 /// For example for the L1 distance
 /// 
@@ -1222,7 +1223,7 @@ fn test_jaccard_u16() {
 
 
 
-extern "C" fn dist_func_float(va : *const f32, vb : *const f32, len : c_ulonglong) -> f32 {
+extern "C" fn dist_func_float(va : *const f32, vb : *const f32, len : u64) -> f32 {
     let mut dist : f32 = 0.;
     let sa = unsafe {std::slice::from_raw_parts(va, len as usize) };
     let sb = unsafe { std::slice::from_raw_parts(vb, len as usize) };

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/src/core/index/hnsw_rs/hnswio.rs

@@ -625,11 +625,10 @@ pub fn load_hnsw<T:'static+Serialize+DeserializeOwned+Clone+Sized+Send+Sync, D:D
     // for that we check for short names equality stripping 
     log::debug!("distance in description = {:?}", distname);
     let d_type_name = type_name::<D>().to_string();
-    let v: Vec<&str> = d_type_name.rsplit_terminator("::").collect();
-    for s in v {
-        log::info!(" distname in generic type argument {:?}", s);
-    }
-    if (std::any::TypeId::of::<T>() != std::any::TypeId::of::<NoData>())  &&  (d_type_name != distname) {
+    let d_short_name = d_type_name.rsplit("::").next().unwrap_or(&d_type_name);
+    let dist_short_name = distname.rsplit("::").next().unwrap_or(&distname);
+
+    if (std::any::TypeId::of::<T>() != std::any::TypeId::of::<NoData>())  &&  (d_short_name != dist_short_name) {
         // for all types except NoData , distance asked in reload declaration and distance in dump must be equal!
         let mut errmsg = String::from("error in distances : dumped distance is : ");
         errmsg.push_str(&distname);
@@ -687,11 +686,10 @@ pub fn load_hnsw_with_dist<T:'static+Serialize+DeserializeOwned+Clone+Sized+Send
     // for that we check for short names equality stripping 
     log::debug!("distance in description = {:?}", distname);
     let d_type_name = type_name::<D>().to_string();
-    let v: Vec<&str> = d_type_name.rsplit_terminator("::").collect();
-    for s in v {
-        log::info!(" distname in generic type argument {:?}", s);
-    }
-    if (std::any::TypeId::of::<T>() != std::any::TypeId::of::<NoData>())  &&  (d_type_name != distname) {
+    let d_short_name = d_type_name.rsplit("::").next().unwrap_or(&d_type_name);
+    let dist_short_name = distname.rsplit("::").next().unwrap_or(&distname);
+
+    if (std::any::TypeId::of::<T>() != std::any::TypeId::of::<NoData>())  &&  (d_short_name != dist_short_name) {
         // for all types except NoData , distance asked in reload declaration and distance in dump must be equal!
         let mut errmsg = String::from("error in distances : dumped distance is : ");
         errmsg.push_str(&distname);

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/src/core/metadata.rs

@@ -61,6 +61,10 @@ pub struct TableMetadata {
     pub default_sort_order_id: i32,
     pub sort_orders: Vec<SortOrder>,
     
+    // Primary Key (Iceberg Identifier Fields)
+    #[serde(rename = "identifier-field-ids", default, skip_serializing_if = "Vec::is_empty")]
+    pub identifier_field_ids: Vec<i32>,
+    
     // Properties
     #[serde(default)]
     pub properties: HashMap<String, String>,
@@ -109,7 +113,7 @@ impl TableMetadata {
             last_updated_ms: chrono::Utc::now().timestamp_millis(),
             last_column_id: schema.fields.iter().map(|f| f.id).max().unwrap_or(0),
             current_schema_id: schema.schema_id,
-            schemas: vec![schema],
+            schemas: vec![schema.clone()],
             default_spec_id: partition_spec.spec_id,
             partition_specs: vec![partition_spec],
             default_sort_order_id: sort_order.order_id,
@@ -120,6 +124,7 @@ impl TableMetadata {
             snapshot_log: Vec::new(),
             metadata_log: Vec::new(),
             next_row_id: if format_version >= 3 { Some(0) } else { None },
+            identifier_field_ids: schema.identifier_field_ids.clone(),
         }
     }
 

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/src/core/reader.rs

@@ -203,7 +203,7 @@ impl HybridReader {
     }
 
     /// Load and merge all .del files into a single RoaringBitmap
-    async fn load_merged_deletes(&self) -> Result<RoaringBitmap> {
+    pub async fn load_merged_deletes(&self) -> Result<RoaringBitmap> {
         let mut deleted_bitmap = RoaringBitmap::new();
         
         // Determine target path for Iceberg delete matching

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/src/core/segment.rs

@@ -464,24 +464,21 @@ impl HybridSegmentWriter {
 
         // Build Indexes
 
-        if self.config.index_all {
-            batch.schema().fields().iter().enumerate().collect::<Vec<_>>().into_par_iter()
-                .try_for_each(|(i, field)| {
-                    let col_name = field.name();
-                    let col = batch.column(i);
+        batch.schema().fields().iter().enumerate().collect::<Vec<_>>().into_par_iter()
+            .try_for_each(|(i, field)| {
+                let col_name = field.name();
+                let col = batch.column(i);
+                
+                let is_pk = self.primary_key.contains(&col_name.to_string());
+                let is_vector = matches!(col.data_type(), arrow::datatypes::DataType::FixedSizeList(_, _) | arrow::datatypes::DataType::List(_));
+                let in_config_list = self.config.columns_to_index.as_ref().map(|cols| cols.contains(&col_name.to_string())).unwrap_or(false);
+                
+                if self.config.index_all || is_pk || is_vector || in_config_list {
                     self.index_column(col_name, col)
-                })?;
-        } else if let Some(ref cols) = self.config.columns_to_index {
-            cols.into_par_iter()
-                .try_for_each(|col_name| {
-                    if let Ok(idx) = batch.schema().index_of(col_name) {
-                        let col = batch.column(idx);
-                        self.index_column(col_name, col)
-                    } else {
-                        Ok(())
-                    }
-                })?;
-        } 
+                } else {
+                    Ok(())
+                }
+            })?;
         
         Ok(())
     }

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/src/core/table.rs

@@ -345,13 +345,13 @@ impl Table {
              }
         }
 
-        Ok(Table { 
-            uri, 
-            store, 
+        let table = Table { 
+            uri: uri.clone(), 
+            store: store.clone(), 
             data_store: None,
             rt: Some(rt),
             query_config: QueryConfig::default(),
-            index_all: true, // Default: Index Everything
+            index_all: true,
             index_columns: Arc::new(std::sync::RwLock::new(Vec::new())),
             index_configs: Arc::new(std::sync::RwLock::new(HashMap::new())),
             default_device: Arc::new(std::sync::RwLock::new(None)),
@@ -369,7 +369,11 @@ impl Table {
             enterprise_license: None,
             primary_key: Arc::new(std::sync::RwLock::new(Vec::new())),
             autocommit: Arc::new(std::sync::atomic::AtomicBool::new(true)),
-        })
+        };
+        
+        // Sync PKs after initialization
+        let _ = table.rt.as_ref().unwrap().block_on(table.sync_primary_key_from_schema_async());
+        Ok(table)
     }
 
     /// Load a table from Nessie Catalog at specific branch/tag/hash
@@ -966,7 +970,7 @@ impl Table {
             // Correct.
         }
 
-        Ok(Table { 
+        let table = Table { 
             uri: uri.to_string(), 
             store, 
             data_store: None,
@@ -990,7 +994,10 @@ impl Table {
             enterprise_license: None,
             primary_key: Arc::new(std::sync::RwLock::new(Vec::new())),
             autocommit: Arc::new(std::sync::atomic::AtomicBool::new(true)),
-        })
+        };
+        
+        table.sync_primary_key_from_schema_async().await?;
+        Ok(table)
     }
     
     /// Create a new table with an explicit schema (Asynchronous)
@@ -2481,6 +2488,14 @@ impl Table {
                  })
              });
 
+        // 0. Primary Key Uniqueness Check (if defined)
+        let primary_keys = self.get_primary_key();
+        if !primary_keys.is_empty() {
+            for batch in &batches {
+                self.check_primary_key_uniqueness_async(batch, &primary_keys).await?;
+            }
+        }
+
         let buffer_len_before = { 
             let buffer = self.write_buffer.read().unwrap();
             buffer.iter().map(|b| b.num_rows()).sum()
@@ -2690,7 +2705,8 @@ impl Table {
             all_new_entries.push(entry.clone());
 
             // 2. Queue index building asynchronously (if needed)
-            if index_all_flag || !index_cols.is_empty() {
+            let has_pks = !self.primary_key.read().unwrap().is_empty();
+            if index_all_flag || !index_cols.is_empty() || has_pks {
                 let index_cols_clone = index_cols.clone();
                 let base_path_clone = base_path.to_string();
                 let segment_id_clone = segment_id.clone();
@@ -3075,6 +3091,114 @@ impl Table {
         Ok(())
     }
 
+    /// Check if any keys in the batch already exist in the table (Primary Key Enforcement)
+    async fn check_primary_key_uniqueness_async(&self, batch: &RecordBatch, columns: &[String]) -> Result<()> {
+        
+        if batch.num_rows() == 0 { return Ok(()); }
+        
+        let schema = batch.schema();
+        let col_indices: Vec<usize> = columns.iter()
+            .map(|c| schema.index_of(c))
+            .collect::<Result<Vec<usize>, _>>()?;
+
+        // OPTIMIZATION: Use IN clause for batches (efficient via Inverted Index)
+        // For now, we take the first row as a sample check to avoid huge expression generation 
+        // until we have a proper Row-Value In-List implementation.
+        for i in 0..batch.num_rows().min(100) { // Limit samples for performance in MVP
+            let mut filters_str_vec = Vec::new();
+            for (col_name, col_idx) in columns.iter().zip(col_indices.iter()) {
+                let col = batch.column(*col_idx);
+                let val = if let Some(arr) = col.as_any().downcast_ref::<arrow::array::Int32Array>() {
+                    format!("{}", arr.value(i))
+                } else if let Some(arr) = col.as_any().downcast_ref::<arrow::array::Int64Array>() {
+                    format!("{}", arr.value(i))
+                } else if let Some(arr) = col.as_any().downcast_ref::<arrow::array::StringArray>() {
+                    format!("'{}'", arr.value(i).replace("'", "''"))
+                } else {
+                    continue;
+                };
+                filters_str_vec.push(format!("{} = {}", col_name, val));
+            }
+            
+            if !filters_str_vec.is_empty() {
+                let filter_str = filters_str_vec.join(" AND ");
+                let expr = FilterExpr::parse_sql(&filter_str, self.arrow_schema()).await?;
+                
+                // Check manifests
+                let manifest_manager = ManifestManager::new(self.store.clone(), "", &self.uri);
+                let (_, all_entries, _) = manifest_manager.load_latest_full().await?;
+                let planner = QueryPlanner::new();
+                let candidates = planner.prune_entries(&all_entries, Some(&expr), None);
+                
+                if !candidates.is_empty() {
+                    // Refine search within candidates (Index lookup)
+                    for (entry, _) in candidates {
+                        let config = SegmentConfig::new(&self.uri, &entry.file_path.replace(".parquet", ""))
+                            .with_index_files(entry.index_files.clone())
+                            .with_delete_files(entry.delete_files.clone());
+                        let reader = HybridReader::new(config, self.store.clone(), &self.uri);
+                        
+                        let filters = expr.extract_and_conditions();
+                        let mut bitmap_opt: Option<roaring::RoaringBitmap> = None;
+                        
+                        for f in filters {
+                            if let Ok(Some(bm)) = reader.get_scalar_filter_bitmap(&f).await {
+                                // Subtract logically deleted rows!
+                                let deleted = reader.load_merged_deletes().await?;
+                                let alive_bm = bm.clone() - deleted.clone();
+                                
+                                println!("DEBUG: PK Check for {}: Index bits: {}, Deleted bits: {}, Alive bits: {}", 
+                                    f.column, bm.len(), deleted.len(), alive_bm.len());
+                                
+                                if let Some(current) = bitmap_opt {
+                                    bitmap_opt = Some(current & alive_bm);
+                                } else {
+                                    bitmap_opt = Some(alive_bm);
+                                }
+                            } else {
+                                bitmap_opt = None;
+                                break;
+                            }
+                        }
+
+                        if let Some(bm) = bitmap_opt {
+                            if !bm.is_empty() {
+                                let pk_val = columns.iter().zip(filters_str_vec.iter())
+                                    .map(|(c, f)| format!("{}={}", c, f))
+                                    .collect::<Vec<_>>().join(", ");
+                                return Err(anyhow::anyhow!("Duplicate primary key error: {} already exists", pk_val));
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        
+        Ok(())
+    }
+
+    /// Synchronize PK columns from Iceberg schema identifier-field-ids (Internal Async)
+    async fn sync_primary_key_from_schema_async(&self) -> Result<()> {
+        let manifest_manager = ManifestManager::new(self.store.clone(), "", &self.uri);
+        let latest_manifest = manifest_manager.load_latest_full().await?.0;
+        
+        if let Some(schema) = latest_manifest.schemas.last() {
+            let pk_cols: Vec<String> = schema.identifier_field_ids.iter().map(|id| {
+                schema.fields.iter().find(|f| f.id == *id).map(|f| f.name.clone()).unwrap_or_default()
+            }).filter(|n| !n.is_empty()).collect();
+            
+            if !pk_cols.is_empty() {
+                self.set_primary_key(pk_cols);
+            }
+        }
+        Ok(())
+    }
+
+    /// Synchronize PK columns (Public Sync)
+    pub fn sync_primary_key_from_schema(&self) -> Result<()> {
+        self.runtime().block_on(self.sync_primary_key_from_schema_async())
+    }
+
     /// Merge (Upsert) batches into the table
     pub fn merge(&self, batches: Vec<RecordBatch>, key_column: &str, mode: MergeMode) -> Result<()> {
         match mode {

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/src/lib.rs

@@ -31,8 +31,8 @@ pub use crate::core::catalog::{CatalogType, create_catalog, create_catalog_async
 use pyo3::prelude::*;
 
 #[cfg(feature = "python")]
-#[pymodule]
-fn hyperstreamdb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
+#[pymodule(name = "hyperstreamdb")]
+fn hyperstreamdb_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(python_binding::init_logging, m)?)?;
     m.add_function(wrap_pyfunction!(python_binding::create_catalog, m)?)?;
     m.add_function(wrap_pyfunction!(python_binding::create_catalog_from_config, m)?)?;

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/src/python_binding.rs

@@ -1038,7 +1038,7 @@ impl PyTable {
         let batch_result: Result<(Vec<RecordBatch>, arrow::datatypes::SchemaRef), String> = rt.block_on(async {
             use datafusion::prelude::SessionContext;
             let mut ctx = SessionContext::new();
-            crate::core::sql::vector_operators::register_vector_operators(&mut ctx);
+            let _ = crate::core::sql::vector_operators::register_vector_operators(&mut ctx);
             
             // Register table as 't' (short alias, safe from keywords)
             let provider = Arc::new(crate::core::sql::HyperStreamTableProvider::new(Arc::new(self.table.clone())));

{hyperstreamdb-0.1.3 → hyperstreamdb-0.1.6}/src/python_gpu_context.rs

@@ -366,7 +366,7 @@ impl PyComputeContext {
 impl PyComputeContext {
     /// Internal helper to list available backends
     fn list_available_backends_internal() -> Vec<String> {
-        let mut backends = vec!["cpu".to_string()];
+        let backends = vec!["cpu".to_string()];
         
         #[cfg(feature = "cuda")]
         backends.push("cuda".to_string());