hyperstreamdb 0.2.6__tar.gz → 0.3.1__tar.gz

hyperstreamdb-0.3.1/.ipynb_checkpoints/Untitled-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

{hyperstreamdb-0.2.6 → hyperstreamdb-0.3.1}/Cargo.lock

@@ -3375,7 +3375,7 @@ dependencies = [
 
 [[package]]
 name = "hyperstreamdb"
-version = "0.2.6"
+version = "0.3.1"
 dependencies = [
  "ahash 0.8.12",
  "anyhow",
@@ -3397,6 +3397,7 @@ dependencies = [
  "cpu-time",
  "criterion",
  "cudarc",
+ "dashmap",
  "datafusion",
  "datafusion-expr-common",
  "datafusion-functions",

{hyperstreamdb-0.2.6 → hyperstreamdb-0.3.1}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hyperstreamdb"
-version = "0.2.6"
+version = "0.3.1"
 edition = "2021"
 license = "MIT AND Apache-2.0"
 description = "HyperStreamDB - Serverless Index-Streaming Database with Overlay Indexing and Vector Search"
@@ -84,6 +84,7 @@ pilota = "0.11"
 # Hardware Acceleration (moved to bottom of file)
 # indexing
 roaring = "0.10.2" # For scalar bitmaps
+dashmap = "6.0"      # Concurrent HashMaps
 # hnsw_rs removed and internalized to src/core/index/hnsw_rs
 bincode = "1.3"
 cpu-time = "1.0"

{hyperstreamdb-0.2.6 → hyperstreamdb-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperstreamdb
-Version: 0.2.6
+Version: 0.3.1
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Rust
@@ -62,12 +62,8 @@ A production-ready indexed data lake format that combines the transactional guar
 | **Time Travel** | ✅ Yes | ✅ Yes |
 | **Scalar Indexes** | ❌ No | ✅ RoaringBitmap |
 | **Boolean Indexes** | ❌ No | ✅ Native Boolean |
-| **Vector Search** | ❌ No | ✅ HNSW |
-| **pgvector SQL** | ❌ No | ✅ Full Compatibility |
-| **GPU Acceleration** | ❌ No | ✅ CUDA/ROCm/XPU/Metal |
-| **Torch Alignment** | ❌ No | ✅ ROCm-as-CUDA |
-| **Python Vector API** | ❌ No | ✅ NumPy-compatible |
-| **Fluent Query API** | ❌ No | ✅ Method Chaining |
+| **TurboQuant** | ❌ No | ✅ TQ8 & TQ4 (8-bit/4-bit) |
+| **Fluent Indexing API** | ❌ No | ✅ Method Chaining |
 | **Hybrid Queries** | ❌ No | ✅ Scalar + Vector |
 | **Native SQL** | ❌ No | ✅ DataFusion |
 | **Index-Optimized Joins** | ❌ No | ✅ Index Nested Loop |
@@ -204,10 +200,12 @@ table = hdb.Table("s3://bucket/my-table")
 import pandas as pd
 df = pd.DataFrame({
     "id": [1, 2, 3],
-    "text": ["hello", "world", "test"],
     "embedding": [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]
 })
-table.write_pandas(df)
+table.insert(df) # Convenient alias for write_pandas
+
+# Create high-performance vector index (TQ8 - 4x compression)
+table.add_index("embedding", "hnsw_tq8")
 
 # Query with filters (uses indexes!) - Fluent API
 results = table.query().filter("id > 1").execute()
@@ -300,6 +298,30 @@ async fn main() -> anyhow::Result<()> {
 - **Performance**: Same underlying optimized execution as traditional APIs
 - **Interoperable**: Mix with SQL queries and traditional `to_pandas()` calls
 - **GPU Acceleration**: Automatic GPU context propagation for vector operations
+- **TurboQuant Optimized**: Seamless integration with 8-bit/4-bit quantization
+
+### TurboQuant Quantization (TQ8 / TQ4)
+
+HyperStreamDB features **TurboQuant**, an optimized quantization engine that reduces vector storage costs while maintaining high search accuracy:
+
+- **TQ8 (8-bit)**: 4x compression vs. float32. Near-lossless accuracy (typically >99% recall retention). Ideal for general-purpose RAG.
+- **TQ4 (4-bit)**: 8x compression vs. float32. Maximum efficiency for massive datasets where storage cost is the primary bottleneck.
+
+```python
+# Use enterprise defaults (HNSW-TQ8)
+table.add_index("embedding", "hnsw_tq8")
+
+# High-compression mode
+table.add_index("embedding", "hnsw_tq4")
+
+# Custom HNSW-PQ configuration
+table.add_index("embedding", {
+    "type": "hnsw_pq",
+    "complexity": 32,
+    "quality": 300,
+    "compression": 32 # PQ subspaces
+})
+```
 
 ### Python Vector Distance API with GPU Acceleration
 

{hyperstreamdb-0.2.6 → hyperstreamdb-0.3.1}/README.md

@@ -17,12 +17,8 @@ A production-ready indexed data lake format that combines the transactional guar
 | **Time Travel** | ✅ Yes | ✅ Yes |
 | **Scalar Indexes** | ❌ No | ✅ RoaringBitmap |
 | **Boolean Indexes** | ❌ No | ✅ Native Boolean |
-| **Vector Search** | ❌ No | ✅ HNSW |
-| **pgvector SQL** | ❌ No | ✅ Full Compatibility |
-| **GPU Acceleration** | ❌ No | ✅ CUDA/ROCm/XPU/Metal |
-| **Torch Alignment** | ❌ No | ✅ ROCm-as-CUDA |
-| **Python Vector API** | ❌ No | ✅ NumPy-compatible |
-| **Fluent Query API** | ❌ No | ✅ Method Chaining |
+| **TurboQuant** | ❌ No | ✅ TQ8 & TQ4 (8-bit/4-bit) |
+| **Fluent Indexing API** | ❌ No | ✅ Method Chaining |
 | **Hybrid Queries** | ❌ No | ✅ Scalar + Vector |
 | **Native SQL** | ❌ No | ✅ DataFusion |
 | **Index-Optimized Joins** | ❌ No | ✅ Index Nested Loop |
@@ -159,10 +155,12 @@ table = hdb.Table("s3://bucket/my-table")
 import pandas as pd
 df = pd.DataFrame({
     "id": [1, 2, 3],
-    "text": ["hello", "world", "test"],
     "embedding": [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]
 })
-table.write_pandas(df)
+table.insert(df) # Convenient alias for write_pandas
+
+# Create high-performance vector index (TQ8 - 4x compression)
+table.add_index("embedding", "hnsw_tq8")
 
 # Query with filters (uses indexes!) - Fluent API
 results = table.query().filter("id > 1").execute()
@@ -255,6 +253,30 @@ async fn main() -> anyhow::Result<()> {
 - **Performance**: Same underlying optimized execution as traditional APIs
 - **Interoperable**: Mix with SQL queries and traditional `to_pandas()` calls
 - **GPU Acceleration**: Automatic GPU context propagation for vector operations
+- **TurboQuant Optimized**: Seamless integration with 8-bit/4-bit quantization
+
+### TurboQuant Quantization (TQ8 / TQ4)
+
+HyperStreamDB features **TurboQuant**, an optimized quantization engine that reduces vector storage costs while maintaining high search accuracy:
+
+- **TQ8 (8-bit)**: 4x compression vs. float32. Near-lossless accuracy (typically >99% recall retention). Ideal for general-purpose RAG.
+- **TQ4 (4-bit)**: 8x compression vs. float32. Maximum efficiency for massive datasets where storage cost is the primary bottleneck.
+
+```python
+# Use enterprise defaults (HNSW-TQ8)
+table.add_index("embedding", "hnsw_tq8")
+
+# High-compression mode
+table.add_index("embedding", "hnsw_tq4")
+
+# Custom HNSW-PQ configuration
+table.add_index("embedding", {
+    "type": "hnsw_pq",
+    "complexity": 32,
+    "quality": 300,
+    "compression": 32 # PQ subspaces
+})
+```
 
 ### Python Vector Distance API with GPU Acceleration
 

hyperstreamdb-0.3.1/Untitled.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c2ee4517-3acc-4aaf-aace-60dfb489e44d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

{hyperstreamdb-0.2.6 → hyperstreamdb-0.3.1}/benches/performance.rs

@@ -42,7 +42,7 @@ fn bench_query_indexed(c: &mut Criterion) {
         .with_columns_to_index(vec!["id".to_string()]);
     let writer = HybridSegmentWriter::new(writer_config);
     writer.write_batch(&batch).unwrap();
-    writer.build_indexes(&batch).unwrap();
+    writer.build_indexes(&batch, 0).unwrap();
     let entry = writer.to_manifest_entry();
     
     // For Reader: Use relative path logic since store is rooted at tmp_dir
@@ -81,7 +81,7 @@ fn bench_vector_search(c: &mut Criterion) {
     
     let writer = HybridSegmentWriter::new(config);
     writer.write_batch(&batch).unwrap();
-    writer.build_indexes(&batch).unwrap();
+    writer.build_indexes(&batch, 0).unwrap();
     
     // Generate a random query vector
     let mut rng = rand::thread_rng();
@@ -128,7 +128,7 @@ fn bench_hybrid_search(c: &mut Criterion) {
     
     let writer = HybridSegmentWriter::new(config);
     writer.write_batch(&batch).unwrap();
-    writer.build_indexes(&batch).unwrap();
+    writer.build_indexes(&batch, 0).unwrap();
     
     let mut rng = rand::thread_rng();
     let query_vec: Vec<f32> = (0..vec_dim).map(|_| rng.gen()).collect();
@@ -192,7 +192,7 @@ fn bench_high_selectivity(c: &mut Criterion) {
         .with_columns_to_index(vec!["id".to_string()]);
     let writer = HybridSegmentWriter::new(config);
     writer.write_batch(&batch).unwrap();
-    writer.build_indexes(&batch).unwrap();
+    writer.build_indexes(&batch, 0).unwrap();
     let entry = writer.to_manifest_entry();
     
     // For Reader: Use relative path logic

hyperstreamdb-0.3.1/benchmark_results/multi_filter_vector_20260409_231713.json

@@ -0,0 +1,14 @@
+[
+  {
+    "name": "multi_filter_vector",
+    "throughput": null,
+    "latency_p50_ms": 19.483089447021484,
+    "latency_p95_ms": 22.400259971618652,
+    "latency_p99_ms": 22.65956401824951,
+    "latency_mean_ms": 20.386934280395508,
+    "latency_min_ms": 18.953323364257812,
+    "latency_max_ms": 22.724390029907227,
+    "elapsed_sec": 0.06162667274475098,
+    "memory_delta_mb": 0.21875
+  }
+]

hyperstreamdb-0.3.1/benchmark_results/multi_filter_vector_20260409_231713.md

@@ -0,0 +1,7 @@
+## Benchmark Results
+
+**Hardware & OS:** Linux 6.8.0-106-generic | x86_64 (8 threads) | 63 GB RAM
+
+| name | throughput | latency_p50_ms | latency_p95_ms | latency_p99_ms | latency_mean_ms | latency_min_ms | latency_max_ms | elapsed_sec | memory_delta_mb |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| multi_filter_vector | None | 19.48 | 22.40 | 22.66 | 20.39 | 18.95 | 22.72 | 0.06 | 0.22 |

{hyperstreamdb-0.2.6 → hyperstreamdb-0.3.1}/docs/PYTHON_VECTOR_API.md

@@ -4,6 +4,9 @@
 
 HyperStreamDB provides a comprehensive Python API for vector distance computations with GPU acceleration support across multiple hardware backends. This API allows you to compute distances between vectors directly from Python without writing SQL queries, with optional GPU acceleration for high-performance batch operations.
 
+> [!NOTE]
+> This guide covers **standalone** distance functions for CPU/GPU. For persistent vector storage and search with TurboQuant indexing (TQ8/TQ4), please see [Vector Configuration](VECTOR_CONFIGURATION.md).
+
 ## Supported Distance Metrics
 
 The API supports six distance metrics:

{hyperstreamdb-0.2.6 → hyperstreamdb-0.3.1}/docs/VECTOR_CONFIGURATION.md

@@ -234,32 +234,37 @@ ORDER BY distance
 LIMIT 10;
 ```
 
-### Index Build Parameters
-
-Index build parameters are set during index creation:
+Index build parameters are set during index creation using the fluent `add_index` method:
 
 ```python
 import hyperstreamdb as hdb
 
 table = hdb.Table("s3://bucket/my-table")
 
-# Create HNSW index with custom parameters
-table.create_index(
+# TurboQuant 8-bit quantization (Recommended Default)
+# 4x compression, near-lossless accuracy
+table.add_index("embedding", "hnsw_tq8")
+
+# TurboQuant 4-bit quantization
+# 8x compression, maximum efficiency
+table.add_index("embedding", "hnsw_tq4")
+
+# Custom HNSW parameters
+table.add_index(
     column="embedding",
-    index_type="hnsw",
-    params={
-        "m": 16,              # Number of connections per layer
-        "ef_construction": 200, # Build-time beam width
+    index_config={
+        "type": "hnsw",
+        "complexity": 16, # Max connections per node (formerly 'm')
+        "quality": 200,    # Construction beam width (formerly 'ef_construction')
     }
 )
 
-# Create IVF index with custom parameters
-table.create_index(
+# Product Quantization (PQ)
+table.add_index(
     column="embedding",
-    index_type="ivf",
-    params={
-        "n_lists": 1000,      # Number of clusters
-        "n_probes": 10,       # Default search probes
+    index_config={
+        "type": "hnsw_pq",
+        "compression": 32 # PQ subspaces (formerly 'subspaces')
     }
 )
 ```

{hyperstreamdb-0.2.6 → hyperstreamdb-0.3.1}/docs/api_reference.md

@@ -224,13 +224,22 @@ See [GPU Setup Guide](GPU_SETUP_GUIDE.md) for installation and configuration.
 ```python
 import hyperstreamdb as hdb
 
-# Configure HNSW index parameters
 table = hdb.Table("s3://bucket/table")
-table.create_vector_index(
+
+# TurboQuant 8-bit (4x compression)
+table.add_index("embedding", "hnsw_tq8")
+
+# TurboQuant 4-bit (8x compression)
+table.add_index("embedding", "hnsw_tq4")
+
+# Custom HNSW configuration
+table.add_index(
     column="embedding",
-    metric="cosine",
-    m=16,              # Number of connections per layer
-    ef_construction=200 # Size of dynamic candidate list
+    index_config={
+        "type": "hnsw",
+        "complexity": 16, # connections per node
+        "quality": 200     # construction search width
+    }
 )
 ```
 

{hyperstreamdb-0.2.6 → hyperstreamdb-0.3.1}/python/hyperstreamdb/__init__.py

@@ -96,12 +96,20 @@ except ImportError:
 from typing import List, Optional, Union, Dict, Any
 import os
 
+class IndexType:
+    """
+    HyperStreamDB Indexing Algorithms.
+    """
+    HNSW = "hnsw"
+    BM25 = "bm25"
+    BLOOM = "bloom"
+    BITMAP = "bitmap"
+    INVERTED = "inverted"
+
 def _resolve_uri(uri: str) -> str:
-    """Resolve a URI to an absolute path if it's a local relative path."""
-    if "://" not in uri and not uri.startswith("/"):
-        return os.path.abspath(uri)
+    if not uri.startswith(("s3://", "file://", "az://", "gs://", "http://", "https://")):
+        return f"file://{os.abspath(uri)}" if hasattr(os, "abspath") else uri
     return uri
-
 class Query:
     """
     Fluent Query interface for HyperStreamDB.
@@ -251,6 +259,10 @@ class Table:
                 pass
             raise TypeError(f"Unsupported data type for write: {type(data)}")
 
+    def insert(self, data: Any, device: Optional[Any] = None):
+        """Alias for write() for compatibility with common vector DB APIs."""
+        return self.write(data, device=device)
+
     def write_pandas(self, df: pd.DataFrame, device: Optional[Any] = None):
         """High-level Pandas ingestion with auto-vectorization."""
         return self._write_pandas(df, device=device)
@@ -583,24 +595,39 @@ class Table:
     def set_index_config(self, column: str, enabled: bool = True, tokenizer: Optional[str] = None, device: Optional[str] = None):
         """
         Set indexing configuration for a specific column.
+        (Legacy compatibility wrapper)
+        """
+        if not enabled:
+            return self.drop_index(column)
+            
+        config = {"type": "hnsw"}
+        if tokenizer: config["tokenizer"] = tokenizer
+        if device: config["build_device"] = device
+        return self.add_index(column, config)
+
+    def set_index_columns(self, config: Dict[str, Union[str, List[Union[str, Dict[str, Any]]], Dict[str, Any]]]):
+        """
+        Update indexing specifications for multiple columns at once.
+        Supports both simple strings and advanced configuration dictionaries.
         
-        Args:
-            column: Name of the column to configure.
-            enabled: Whether to enable indexing for this column (default: True).
-            tokenizer: Tokenizer name from the registry ('identity', 'whitespace', 'standard').
-            device: Compute device ('cpu', 'cuda', 'mps') if specific processing is needed.
+        Example:
+            table.set_index_columns({
+                "embedding": IndexType.HNSW,
+                "content": ["hnsw", "bm25"],
+                "category": "bitmap"
+            })
         """
-        self._inner.set_index_config(column, enabled, tokenizer, device)
+        return self._inner.set_index_columns(config)
 
-    def add_index(self, column: str, tokenizer: Optional[str] = None, device: Optional[str] = None):
+    def add_index(self, column: str, algorithm: Union[str, Dict[str, Any]] = "hnsw"):
         """
-        Add an index to a column (convenience wrapper).
+        Add an indexing strategy to a column.
         """
-        return self._inner.add_index(column, tokenizer, device)
+        return self._inner.add_index(column, algorithm)
 
     def drop_index(self, column: str):
         """
-        Drop an index from a column (convenience wrapper).
+        Remove all indexing strategies from a column.
         """
         return self._inner.drop_index(column)
 

hyperstreamdb-0.3.1/split_table.py

@@ -0,0 +1,19 @@
+import re
+import os
+
+with open("src/core/table/mod.rs", "r") as f:
+    mod_lines = f.readlines()
+
+def extract_between(start_str, end_str):
+    start_idx = -1
+    end_idx = -1
+    for i, line in enumerate(mod_lines):
+        if start_str in line and start_idx == -1:
+            # check if it's the right indentation (4 spaces)
+            if line.startswith("    pub ") or line.startswith("    async fn ") or line.startswith("    fn "):
+                start_idx = i
+        if end_str in line and start_idx != -1:
+            # find the end of that block
+            pass # this is hard in python.
+
+# Let's just use rustc --pretty !!

{hyperstreamdb-0.2.6 → hyperstreamdb-0.3.1}/src/core/cache.rs

@@ -11,6 +11,7 @@ use crate::core::index::hnsw_rs::hnsw::Hnsw;
 use crate::core::index::hnsw_rs::dist::DistL2;
 use arrow::record_batch::RecordBatch;
 use parquet::file::metadata::ParquetMetaData;
+use parquet::bloom_filter::Sbbf;
 use std::path::PathBuf;
 use object_store::ObjectStore;
 use anyhow::Result;
@@ -175,6 +176,13 @@ pub static PARQUET_META_CACHE: Lazy<Cache<String, (Arc<ParquetMetaData>, usize)>
         .build()
 });
 
+pub static BLOOM_FILTER_CACHE: Lazy<Cache<String, Arc<Sbbf>>> = Lazy::new(|| {
+    Cache::builder()
+        .max_capacity(2048) // Roughly 250MB if each is 128KB
+        .time_to_idle(Duration::from_secs(60 * 30))
+        .build()
+});
+
 /// Doris-inspired Block Cache for decoded RecordBatches.
 /// Bypasses Parquet decoding/decompression for frequently accessed blocks.
 pub static BLOCK_CACHE: Lazy<Cache<String, Arc<RecordBatch>>> = Lazy::new(|| {

{hyperstreamdb-0.2.6 → hyperstreamdb-0.3.1}/src/core/index/distance.rs

@@ -274,3 +274,91 @@ pub fn sparse_l2_distance_squared(
     
     sum
 }
+
+/// Optimized L2 distance for quantized u8 vectors
+#[inline(always)]
+pub fn l2_distance_u8(a: &[u8], b: &[u8]) -> f32 {
+    let mut sum = 0;
+    for (&x, &y) in a.iter().zip(b.iter()) {
+        let diff = (x as i32) - (y as i32);
+        sum += diff * diff;
+    }
+    sum as f32
+}
+
+/// Asymmetric Distance Calculation (ADC) for quantized vectors.
+/// Calculates L2 distance between a float32 query and a quantized u8 vector.
+#[inline(always)]
+pub fn l2_distance_adc(query: &[f32], encoded: &[u8], offset: f32, scale: f32) -> f32 {
+    let mut sum = 0.0;
+    let inv_scale = 1.0 / scale;
+    
+    // Unrolled for performance
+    let chunks_q = query.chunks_exact(8);
+    let chunks_e = encoded.chunks_exact(8);
+    let rem_q = chunks_q.remainder();
+    let rem_e = chunks_e.remainder();
+
+    for (q_chunk, e_chunk) in chunks_q.zip(chunks_e) {
+        for i in 0..8 {
+            let decoded = (e_chunk[i] as f32 * inv_scale) + offset;
+            let diff = q_chunk[i] - decoded;
+            sum += diff * diff;
+        }
+    }
+
+    for (q, e) in rem_q.iter().zip(rem_e.iter()) {
+        let decoded = (*e as f32 * inv_scale) + offset;
+        let diff = *q - decoded;
+        sum += diff * diff;
+    }
+    sum
+}
+
+/// Optimized L2 distance for packed 4-bit quantized vectors (u4)
+#[inline(always)]
+pub fn l2_distance_u4(a: &[u8], b: &[u8]) -> f32 {
+    let mut sum = 0;
+    for (&x, &y) in a.iter().zip(b.iter()) {
+        // Low nibbles
+        let diff_low = ((x & 0x0F) as i32) - ((y & 0x0F) as i32);
+        sum += diff_low * diff_low;
+        
+        // High nibbles
+        let diff_high = (((x >> 4) & 0x0F) as i32) - (((y >> 4) & 0x0F) as i32);
+        sum += diff_high * diff_high;
+    }
+    sum as f32
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct DistL2u8;
+
+impl DistL2u8 {
+    #[inline(always)]
+    pub fn distance(&self, a: &[u8], b: &[u8]) -> f32 {
+        l2_distance_u8(a, b)
+    }
+}
+
+impl super::hnsw_rs::dist::Distance<u8> for DistL2u8 {
+    fn eval(&self, va: &[u8], vb: &[u8]) -> f32 {
+        self.distance(va, vb)
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct DistL2u4;
+
+impl DistL2u4 {
+    #[inline(always)]
+    pub fn distance(&self, a: &[u8], b: &[u8]) -> f32 {
+        l2_distance_u4(a, b)
+    }
+}
+
+impl super::hnsw_rs::dist::Distance<u8> for DistL2u4 {
+    fn eval(&self, va: &[u8], vb: &[u8]) -> f32 {
+        self.distance(va, vb)
+    }
+}