hyperstreamdb 0.3.1__tar.gz → 0.3.2__tar.gz

{hyperstreamdb-0.3.1 → hyperstreamdb-0.3.2}/Cargo.lock

@@ -3375,7 +3375,7 @@ dependencies = [
 
 [[package]]
 name = "hyperstreamdb"
-version = "0.3.1"
+version = "0.3.2"
 dependencies = [
  "ahash 0.8.12",
  "anyhow",

{hyperstreamdb-0.3.1 → hyperstreamdb-0.3.2}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hyperstreamdb"
-version = "0.3.1"
+version = "0.3.2"
 edition = "2021"
 license = "MIT AND Apache-2.0"
 description = "HyperStreamDB - Serverless Index-Streaming Database with Overlay Indexing and Vector Search"

{hyperstreamdb-0.3.1 → hyperstreamdb-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperstreamdb
-Version: 0.3.1
+Version: 0.3.2
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Rust

{hyperstreamdb-0.3.1 → hyperstreamdb-0.3.2}/src/core/cache.rs

@@ -125,8 +125,22 @@ pub static INDEX_CACHE: Lazy<Cache<String, Arc<RoaringBitmap>>> = Lazy::new(|| {
 });
 
 pub static BYTE_CACHE: Lazy<Cache<String, Arc<Vec<u8>>>> = Lazy::new(|| {
+    let cache_gb: u64 = std::env::var("HYPERSTREAM_CACHE_GB")
+        .unwrap_or_else(|_| "2".to_string())
+        .parse()
+        .unwrap_or(2);
+    
+    // Allocate 10% of global cache to Byte Cache (max 512MB)
+    let limit_bytes = (cache_gb * 1024 * 1024 * 1024 / 10).min(512 * 1024 * 1024);
+    let max_kb = limit_bytes / 1024;
+
+    tracing::info!("Initializing Byte Cache with {} MB limit", limit_bytes / (1024 * 1024));
+
     Cache::builder()
-        .max_capacity(1000) // 1000 small files (manifests, etc)
+        .weigher(|_key, value: &Arc<Vec<u8>>| -> u32 {
+            (value.len() / 1024) as u32
+        })
+        .max_capacity(max_kb)
         .time_to_idle(Duration::from_secs(60 * 30)) // 30 mins
         .build()
 });
@@ -163,9 +177,24 @@ pub static HNSW_IVF_CACHE: Lazy<Cache<String, Arc<HnswIvfIndex>>> = Lazy::new(||
 });
 
 pub static INVERTED_INDEX_CACHE: Lazy<Cache<String, Arc<Vec<RecordBatch>>>> = Lazy::new(|| {
+    let cache_gb: u64 = std::env::var("HYPERSTREAM_CACHE_GB")
+        .unwrap_or_else(|_| "2".to_string())
+        .parse()
+        .unwrap_or(2);
+    
+    // Allocate 25% of global cache to Inverted Index Cache
+    let limit_bytes = cache_gb * 1024 * 1024 * 1024 / 4;
+    let max_kb = limit_bytes / 1024;
+
+    tracing::info!("Initializing Inverted Index Cache with {} MB limit", limit_bytes / (1024 * 1024));
+
     Cache::builder()
-        .max_capacity(1000) // Cache 1000 decoded inverted index files
-        .time_to_idle(Duration::from_secs(60 * 5)) 
+        .weigher(|_key, value: &Arc<Vec<RecordBatch>>| -> u32 {
+            let bytes: usize = value.iter().map(|b| b.get_array_memory_size()).sum();
+            (bytes / 1024) as u32
+        })
+        .max_capacity(max_kb)
+        .time_to_idle(Duration::from_secs(60 * 15)) 
         .build()
 });
 

{hyperstreamdb-0.3.1 → hyperstreamdb-0.3.2}/src/core/reader.rs

@@ -1635,11 +1635,10 @@ impl HybridReader {
         };
         
         // Load HNSW-IVF index
-        let hnsw_ivf = if idx_info.blob_type.is_some() {
-             HnswIvfIndex::load_puffin_async(self.store.clone(), &idx_path_str).await?
-        } else {
-             HnswIvfIndex::load_async_with_cache_key(self.store.clone(), &idx_path_str, &cache_key).await?
-        };
+        // NOTE: blob_type records the *algorithm* (e.g. "hnsw_tq8"), not the storage format.
+        // The writer always uses the multi-file layout (.centroids.parquet, .cluster_N.hnsw.*),
+        // so we always load via load_async_with_cache_key regardless of blob_type.
+        let hnsw_ivf = HnswIvfIndex::load_async_with_cache_key(self.store.clone(), &idx_path_str, &cache_key).await?;
         
         // Search with HNSW-IVF
         let query_clone = query.clone();

{hyperstreamdb-0.3.1 → hyperstreamdb-0.3.2}/src/core/segment.rs

@@ -1551,8 +1551,8 @@ mod tests {
         assert!(std::path::Path::new(&format!("{}.id.inv.parquet", base)).exists(), "Inverted index for id should exist");
         
         // Vector Index (embedding) - HNSW-IVF saves centroids and cluster graphs
-        assert!(std::path::Path::new(&format!("{}.embedding.centroids.parquet", base)).exists(), "Vector index centroids should exist");
-        assert!(std::path::Path::new(&format!("{}.embedding.cluster_0.hnsw.graph", base)).exists(), "Vector index graph should exist");
+        assert!(std::path::Path::new(&format!("{}.embedding.tq8.centroids.parquet", base)).exists(), "Vector index centroids should exist");
+        assert!(std::path::Path::new(&format!("{}.embedding.tq8.cluster_0.hnsw.graph", base)).exists(), "Vector index graph should exist");
 
         Ok(())
     }

{hyperstreamdb-0.3.1 → hyperstreamdb-0.3.2}/src/core/table/builder.rs

@@ -24,18 +24,37 @@ use super::Table;
 /// rebuilds the in-memory vector index from recovered data.
 /// Returns (aligned_buffer, optional_memory_index, promoted_schema).
 pub(crate) fn recover_wal_state(
-    recovered_batches: Vec<RecordBatch>,
+    recovered_stream: Box<dyn Iterator<Item = Result<RecordBatch>>>,
     mut schema_val: SchemaRef,
 ) -> (Vec<RecordBatch>, Option<InMemoryVectorIndex>, SchemaRef) {
-    if recovered_batches.is_empty() {
+    let mut aligned_buffer = Vec::new();
+    let mut total_rows = 0;
+    
+    // 1. First pass: Collect batches and merge schema
+    let mut batches = Vec::new();
+    for batch_res in recovered_stream {
+        match batch_res {
+            Ok(batch) => {
+                // Safely merge schema
+                match arrow::datatypes::Schema::try_merge(vec![schema_val.as_ref().clone(), batch.schema().as_ref().clone()]) {
+                    Ok(s) => schema_val = Arc::new(s),
+                    Err(e) => tracing::warn!("Failed to merge WAL batch schema: {}", e),
+                }
+                batches.push(batch);
+            }
+            Err(e) => tracing::error!("WAL Replay Error: {}", e),
+        }
+    }
+
+    if batches.is_empty() {
         return (Vec::new(), None, schema_val);
     }
 
-    tracing::info!("Recovering {} batches from WAL...", recovered_batches.len());
+    tracing::info!("Recovering {} batches from WAL...", batches.len());
 
     // Use first batch schema if current schema is empty
     if schema_val.fields().is_empty() {
-        if let Some(first) = recovered_batches.first() {
+        if let Some(first) = batches.first() {
             schema_val = first.schema();
         }
     }
@@ -43,7 +62,7 @@ pub(crate) fn recover_wal_state(
     // Safely attempt to merge all WAL schemas to capture any column additions
     // or type evolutions instead of fragile field count comparisons.
     let mut merged_schema = schema_val.as_ref().clone();
-    for batch in &recovered_batches {
+    for batch in &batches {
         match arrow::datatypes::Schema::try_merge(vec![merged_schema.clone(), batch.schema().as_ref().clone()]) {
             Ok(s) => merged_schema = s,
             Err(e) => tracing::warn!("Failed to merge WAL batch schema: {}", e),
@@ -52,8 +71,8 @@ pub(crate) fn recover_wal_state(
     let schema_val = std::sync::Arc::new(merged_schema);
 
     // Align all recovered batches to the widest schema
-    let aligned_buffer: Vec<RecordBatch> = recovered_batches.into_iter().map(|b| {
-        if b.schema() != schema_val {
+    for b in batches {
+        let aligned = if b.schema() != schema_val {
             let mut cols = Vec::with_capacity(schema_val.fields().len());
             for field in schema_val.fields() {
                 let col = if let Some(c) = b.column_by_name(field.name()) {
@@ -66,8 +85,9 @@ pub(crate) fn recover_wal_state(
             RecordBatch::try_new(schema_val.clone(), cols).unwrap_or(b)
         } else {
             b
-        }
-    }).collect();
+        };
+        aligned_buffer.push(aligned);
+    }
 
     // Rebuild in-memory vector index from recovered data.
     // Look for an "embedding" column (the most common convention), supporting
@@ -94,10 +114,9 @@ pub(crate) fn recover_wal_state(
 
                 if let Some(d) = dim {
                     let mut idx = InMemoryVectorIndex::new(d);
-                    let mut offset = 0;
                     for batch in &aligned_buffer {
-                        let _ = idx.insert_batch(batch, col_name, offset);
-                        offset += batch.num_rows();
+                        let _ = idx.insert_batch(batch, col_name, total_rows);
+                        total_rows += batch.num_rows();
                     }
                     mem_index = Some(idx);
                 }
@@ -231,13 +250,15 @@ impl TableBuilder {
         let _ = wal.spawn_worker();
         
         // Replay WAL (Recovery)
-        let (recovered_batches, recovered_paths) = wal.replay().unwrap_or_else(|e| {
+        let recovered_stream = wal.replay_stream().unwrap_or_else(|e| {
             tracing::warn!("WAL Recovery Warning: {}" , e);
-            (Vec::new(), Vec::new())
+            Box::new(std::iter::empty())
         });
         
+        let (_, recovered_paths) = wal.replay().unwrap_or_else(|_| (vec![], vec![])); // For paths cleanup only
+
         let (initial_buffer, initial_mem_index, schema_val) = recover_wal_state(
-            recovered_batches, schema_val,
+            recovered_stream, schema_val,
         );
 
         let table = Table { 

{hyperstreamdb-0.3.1 → hyperstreamdb-0.3.2}/src/core/table/read.rs

@@ -18,6 +18,7 @@ use crate::SegmentConfig;
 
 use super::Table;
 use crate::core::search::{HybridSearchCoordinator, KeywordSearchParams, ScoredResult};
+use futures::stream::BoxStream;
 
 impl Table {
 
@@ -45,6 +46,21 @@ impl Table {
         self.read_with_config_async(filter_str, vector_filter, columns, self.query_config.clone()).await
     }
 
+    pub async fn read_stream_async(&self, filter_str: Option<&str>, vector_filter: Option<VectorSearchParams>, columns: Option<&[&str]>) -> Result<BoxStream<'static, Result<RecordBatch>>> {
+        self.read_with_config_stream_async(filter_str, vector_filter, columns, self.query_config.clone()).await
+    }
+
+    pub async fn read_with_config_stream_async(&self, filter_str: Option<&str>, vector_filter: Option<VectorSearchParams>, columns: Option<&[&str]>, config: QueryConfig) -> Result<BoxStream<'static, Result<RecordBatch>>> {
+        let expr = match filter_str {
+            Some(f) => {
+                let schema = self.arrow_schema();
+                Some(FilterExpr::parse_sql(f, schema).await?)
+            }
+            _ => None,
+        };
+        self.read_expr_stream_async(expr, vector_filter, columns, config, filter_str).await
+    }
+
     pub fn query(&self) -> TableQuery<'_> {
         TableQuery::new(self)
     }
@@ -277,6 +293,19 @@ impl Table {
         config: QueryConfig,
         filter_str: Option<&str>,
     ) -> Result<Vec<RecordBatch>> {
+        let stream = self.read_expr_stream_async(expr, vector_filter, columns, config, filter_str).await?;
+        let results: Vec<Result<RecordBatch>> = stream.collect().await;
+        results.into_iter().collect()
+    }
+
+    pub async fn read_expr_stream_async(
+        &self,
+        expr: Option<FilterExpr>,
+        vector_filter: Option<VectorSearchParams>,
+        columns: Option<&[&str]>,
+        config: QueryConfig,
+        filter_str: Option<&str>,
+    ) -> Result<BoxStream<'static, Result<RecordBatch>>> {
         use futures::StreamExt;
 
         let manifest_manager = ManifestManager::new(self.store.clone(), "", &self.uri);
@@ -375,7 +404,9 @@ impl Table {
 
                  // Convert ScoredResults back to RecordBatches by fetching from Parquet
                  // This is a simplified version of the final row-fetcher
-                 return self.fetch_results_by_id(scored_results, columns).await;
+                                   let results = self.fetch_results_by_id(scored_results, columns).await?;
+                  return Ok(futures::stream::iter(results.into_iter().map(Ok)).boxed());
+
              }
         }
 
@@ -472,7 +503,7 @@ impl Table {
                   }
               }
 
-              return Ok(results);
+              return Ok(futures::stream::iter(results.into_iter().map(Ok)).boxed());
         }
 
         // Extract Iceberg schema from the already-loaded manifest to avoid
@@ -490,35 +521,52 @@ impl Table {
         let concurrency = config.max_parallel_readers.unwrap_or_else(|| {
             std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4)
         });
+
+        struct ReadCtx {
+            table: Table,
+            expr: Option<Arc<FilterExpr>>,
+            schema: Option<Arc<crate::core::manifest::Schema>>,
+            gpu: Option<crate::core::index::gpu::ComputeContext>,
+            columns: Option<Vec<String>>,
+            version: u64,
+        }
+        let read_ctx = Arc::new(ReadCtx {
+            table: self.clone(),
+            expr: expr_arc.clone(),
+            schema: iceberg_schema_arc,
+            gpu: current_gpu_context,
+            columns: columns.map(|c| c.iter().map(|s| s.to_string()).collect()),
+            version: version as u64,
+        });
+
         let stream = futures::stream::iter(entries_to_read)
-            .map(|entry| {
-                let expr_clone = expr_arc.clone();
-                let schema_clone = iceberg_schema_arc.clone();
-                let ctx = current_gpu_context.clone();
-                async move {
-                    if let Some(c) = ctx {
-                        crate::core::index::gpu::set_global_gpu_context(Some(c));
+            .map({
+                let read_ctx = read_ctx.clone();
+                move |entry| {
+                    let ctx = read_ctx.clone();
+                    async move {
+                        if let Some(c) = ctx.gpu.clone() {
+                            crate::core::index::gpu::set_global_gpu_context(Some(c));
+                        }
+                        let cols_refs: Option<Vec<&str>> = ctx.columns.as_ref().map(|v| v.iter().map(|s| s.as_str()).collect());
+                        ctx.table.read_segment_expr(
+                            &entry, ctx.expr.as_deref(), ctx.version, cols_refs.as_deref(),
+                            ctx.schema.as_deref(),
+                        ).await
                     }
-                    self.read_segment_expr(
-                        &entry, expr_clone.as_deref(), version, columns,
-                        schema_clone.as_deref(),
-                    ).await
                 }
             })
             .buffer_unordered(concurrency);
 
-        let results: Vec<Result<Vec<RecordBatch>>> = stream.collect().await;
-        let mut all_batches = Vec::new();
-        for (i, res) in results.into_iter().enumerate() {
+        let results_stream = stream.flat_map(|res| {
             match res {
-                Ok(b_vec) => {
-                     all_batches.extend(b_vec);
-                },
-                Err(e) => tracing::error!("Error reading batch {}: {}", i, e),
+                Ok(b_vec) => futures::stream::iter(b_vec.into_iter().map(Ok)).boxed(),
+                Err(e) => futures::stream::once(async move { Err(e) }).boxed(),
             }
-        }
+        });
 
         // --- Read from In-Memory Write Buffer ---
+        let mut mem_batches = Vec::new();
         {
             let buffer = self.write_buffer.read().unwrap();
             if !buffer.is_empty() {
@@ -536,7 +584,7 @@ impl Table {
 
                          if let Ok(filtered) = planner.filter_expr(&batch_to_scan, e) {
                              if filtered.num_rows() > 0 {
-                                 all_batches.push(filtered);
+                                 mem_batches.push(Ok(filtered));
                              }
                          }
                     }
@@ -547,18 +595,19 @@ impl Table {
                                 .filter_map(|name| batch.schema().index_of(name).ok())
                                 .collect();
                              if let Ok(projected) = batch.project(&indices) {
-                                 all_batches.push(projected);
+                                 mem_batches.push(Ok(projected));
                              }
                          } else {
-                             all_batches.push(batch.clone());
+                             mem_batches.push(Ok(batch.clone()));
                          }
                      }
                 }
             }
         }
         
+        let mem_stream = futures::stream::iter(mem_batches);
 
-        Ok(all_batches)
+        Ok(results_stream.chain(mem_stream).boxed())
     }
 
     pub async fn read_filter_async(

{hyperstreamdb-0.3.1 → hyperstreamdb-0.3.2}/src/core/wal.rs

@@ -175,15 +175,13 @@ impl WriteAheadLog {
         }
     }
 
-    /// Replay all log files in the WAL directory and return all batches.
-    /// This should be called on startup.
-    pub fn replay(&self) -> Result<(Vec<RecordBatch>, Vec<String>)> {
+    /// Replay all log files in the WAL directory and return an iterator of batches.
+    /// This should be used on startup for memory-efficient recovery.
+    pub fn replay_stream(&self) -> Result<Box<dyn Iterator<Item = Result<RecordBatch>>>> {
         if !self.dir.exists() {
-            return Ok((Vec::new(), Vec::new()));
+            return Ok(Box::new(std::iter::empty()));
         }
 
-        let mut all_batches = Vec::new();
-
         // 1. List all .arrow files in the directory
         let entries = std::fs::read_dir(&self.dir)?;
         let mut wal_files = Vec::new();
@@ -196,10 +194,10 @@ impl WriteAheadLog {
             }
         }
 
-        // Sort for deterministic replay (optional but good)
+        // Sort for deterministic replay
         wal_files.sort();
 
-        let mut replayed_paths = Vec::new();
+        let mut all_iterators = Vec::new();
 
         for path in wal_files {
             let file = File::open(&path)?;
@@ -208,27 +206,35 @@ impl WriteAheadLog {
             }
             
             let reader = BufReader::new(file);
-            let ipc_reader = StreamReader::try_new(reader, None);
-            
-            match ipc_reader {
-                Ok(reader) => {
-                    let mut count = 0;
-                    for batch in reader {
-                        all_batches.push(batch?);
-                        count += 1;
-                    }
-                    if count > 0 {
-                        println!("WAL: Replayed {} batches from {:?}", count, path);
-                        replayed_paths.push(path.to_str().unwrap().to_string());
-                    }
-                },
-                Err(e) => {
-                    println!("WAL Recovery Warning: Could not read log {:?}: {}", path, e);
+            let ipc_reader = StreamReader::try_new(reader, None)?;
+            all_iterators.push(ipc_reader);
+        }
+
+        Ok(Box::new(all_iterators.into_iter().flatten().map(|res| res.map_err(anyhow::Error::from))))
+    }
+
+    /// Replay all log files in the WAL directory and return all batches.
+    /// Legacy method, consider using replay_stream for large logs.
+    pub fn replay(&self) -> Result<(Vec<RecordBatch>, Vec<String>)> {
+        let stream = self.replay_stream()?;
+        let mut batches = Vec::new();
+        for b in stream {
+            batches.push(b?);
+        }
+        
+        // Return paths for cleanup (simplified for now)
+        let mut paths = Vec::new();
+        if self.dir.exists() {
+            for entry in std::fs::read_dir(&self.dir)? {
+                let entry = entry?;
+                let path = entry.path();
+                if path.is_file() && path.extension().and_then(|s| s.to_str()) == Some("arrow") {
+                    paths.push(path.to_str().unwrap().to_string());
                 }
             }
         }
 
-        Ok((all_batches, replayed_paths))
+        Ok((batches, paths))
     }
 
     /// Initialize the writer with a schema.