hyperstreamdb 0.1.2__tar.gz → 0.1.5__tar.gz

{hyperstreamdb-0.1.2 → hyperstreamdb-0.1.5}/.gitignore

@@ -98,3 +98,8 @@ python/hyperstreamdb/*.pyd
 python/hyperstreamdb/*.dylib
 python/hyperstreamdb/*.dSYM/
 groq_api_key.txt
+
+# Local Vector DBs
+rag_db/
+news_db/
+

hyperstreamdb-0.1.5/.instructions.md

@@ -0,0 +1,67 @@
+# HyperStreamDB Development Guidelines
+
+**Foundation:** These guidelines are rooted in [The Pragmatic Programmer](https://pragprog.com/) principles. See `/memories/STEERING.md` for the full foundational principles that guide all projects.
+
+## Core HyperStreamDB Principles
+
+### Code Quality
+- **Always attempt to fix warnings if possible** - Don't ignore compiler warnings; resolve them during implementation (Aligns with: Don't Live with Broken Windows #4)
+- Use proper error handling instead of silencing warnings
+- Maintain clean build output (zero clippy warnings)
+- Type safety is paramount
+- Refactor early and often as code is written
+
+### Versioning & Releases
+- Version numbers are injected from git tags during build
+- `Cargo.toml` and `pyproject.toml` use dynamic versioning from `build.rs`
+- Tag format: `v0.X.Y` (e.g., `v0.1.2`)
+- All releases should be tagged and pushed with comprehensive release notes
+
+### Licensing & Attribution
+- Apache 2.0 / MIT dual license
+- Properly attribute optimizations borrowed from iceberg-rust or other Apache-licensed projects
+- Maintain THIRDPARTY_NOTICES.md with all third-party code usage
+- Include inline code comments for borrowed patterns
+
+### API Design
+- Use native Rust names (PascalCase for types: `Cosine`, `L2`, `InnerProduct`)
+- Python API uses lowercase for user-facing functions: `metric="cosine"`
+- Support both primary parameter names and backward-compatible aliases
+- Document all vector distance functions and index parameters
+
+### Testing & Validation
+- Unit tests required for new optimizer rules and query optimizations (Aligns with: Design to Test #20, Test Your Software #21)
+- Explain() output should show correct index usage and query strategy
+- Test suite must pass before any release tagging (Aligns with: Make Quality a Requirements Issue #7)
+- Performance benchmarks are part of testing, not optional extras
+- Integration tests for query planning changes
+- Run full test suite before tagging releases
+- Verify explain() output shows correct index detection
+
+### Packaging
+- Exclude build artifacts, virtual environments from Cargo.toml (Aligns with: Make It Easy to Reuse #11)
+- Ensure PyPI builds succeed without external dependencies
+- Test `maturin develop --release` before release tagging
+- Dynamic versioning from git tags (build.rs injection)
+
+## Current Focus Areas
+
+### Hybrid Query Optimization
+- Support multiple vector columns with ordering
+- Compound primary key detection for automatic hybrid search
+- Vector search always runs after scalar filters (push-down)
+- Proper explain() output showing both scalar and vector execution
+- Use tracer bullets: Implement end-to-end before perfecting details (Aligns with: Use Tracer Bullets #14)
+
+### Iceberg-Rust Optimizations
+- LIMIT pushdown to file scanning layer
+- Row group skipping based on column statistics
+- Manifest metadata caching for performance
+- Predicate pushdown improvements
+- Properly attribute all borrowed code patterns (Aligns with: Communicate! #9, Make It Easy to Reuse #11)
+
+### Observability & Debugging
+- Structured tracing via `tracing` crate (no println!) (Aligns with: Program Close to Problem Domain #16)
+- Proper log levels (debug, info, warn, error)
+- Explain query plans for user visibility (Aligns with: Use Tracer Bullets #14)
+- Document query strategy through explain() output

{hyperstreamdb-0.1.2 → hyperstreamdb-0.1.5}/Cargo.lock

@@ -116,7 +116,7 @@ version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
 dependencies = [
- "windows-sys 0.61.2",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -127,7 +127,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
 dependencies = [
  "anstyle",
  "once_cell_polyfill",
- "windows-sys 0.61.2",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -2779,7 +2779,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -3548,26 +3548,6 @@ dependencies = [
  "digest",
 ]
 
-[[package]]
-name = "hnsw_rs"
-version = "0.1.19"
-dependencies = [
- "bincode",
- "cpu-time",
- "env_logger",
- "hashbrown 0.9.1",
- "lazy_static",
- "log",
- "num-traits",
- "num_cpus",
- "parking_lot",
- "rand 0.8.5",
- "rayon",
- "roaring",
- "serde",
- "skiplist",
-]
-
 [[package]]
 name = "home"
 version = "0.5.12"
@@ -3775,7 +3755,7 @@ dependencies = [
 
 [[package]]
 name = "hyperstreamdb"
-version = "0.1.2"
+version = "0.1.5"
 dependencies = [
  "ahash 0.8.12",
  "anyhow",
@@ -3790,28 +3770,34 @@ dependencies = [
  "aws-sdk-glue",
  "axum 0.7.9",
  "base64 0.22.1",
+ "bincode",
  "bytes",
  "candle-core",
  "candle-nn",
  "candle-transformers",
  "chrono",
  "clap",
+ "cpu-time",
  "criterion",
  "cust",
  "datafusion",
  "datafusion-expr-common",
  "datafusion-functions-aggregate-common",
  "dirs",
+ "env_logger",
  "faststr",
  "futures",
+ "hashbrown 0.9.1",
  "hive_metastore",
- "hnsw_rs",
  "hyperloglogplus",
  "jni",
  "lazy_static",
+ "log",
  "metal",
  "moka",
  "murmur3",
+ "num-traits",
+ "num_cpus",
  "numpy",
  "object_store",
  "once_cell",
@@ -3819,6 +3805,7 @@ dependencies = [
  "opentelemetry",
  "opentelemetry-otlp",
  "opentelemetry_sdk",
+ "parking_lot",
  "parquet",
  "pilota",
  "prometheus",
@@ -3835,6 +3822,7 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2",
+ "skiplist",
  "smartcore",
  "sqlx",
  "tempfile",
@@ -4074,7 +4062,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
 dependencies = [
  "hermit-abi",
  "libc",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -4741,7 +4729,7 @@ version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
 dependencies = [
- "windows-sys 0.61.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -6258,7 +6246,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -6704,7 +6692,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
 dependencies = [
  "libc",
- "windows-sys 0.61.2",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -7182,7 +7170,7 @@ dependencies = [
  "getrandom 0.4.2",
  "once_cell",
  "rustix",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -8223,7 +8211,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.61.2",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]

{hyperstreamdb-0.1.2 → hyperstreamdb-0.1.5}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hyperstreamdb"
-version = "0.1.2"
+version = "0.1.5"
 edition = "2021"
 license = "MIT AND Apache-2.0"
 description = "HyperStreamDB - Serverless Index-Streaming Database with Overlay Indexing and Vector Search"
@@ -9,17 +9,28 @@ homepage = "https://github.com/rla3rd/hyperstreamdb"
 readme = "README.md"
 keywords = ["database", "vector-search", "iceberg", "parquet", "indexing"]
 categories = ["database-implementations", "data-structures", "concurrency"]
+exclude = [
+    ".venv",
+    "venv",
+    "target",
+    ".git",
+    ".github",
+    "*.db",
+    "test_*",
+    "rag_db",
+    "news_db",
+    "demo",
+    "docker",
+    "benchmarks",
+    "examples"
+]
 
 [lib]
 name = "hyperstreamdb"
 crate-type = ["cdylib", "rlib"]
 
-[[test]]
-name = "test_connector_ffi"
-required-features = ["java"]
-
 [features]
-default = ["intel_gpu", "candle"]
+default = ["candle"]
 candle = []
 cuda = ["dep:cust", "intel_gpu"]
 rocm = ["dep:opencl3", "intel_gpu"] # Fallback to OpenCL for now
@@ -28,6 +39,10 @@ intel_gpu = ["dep:opencl3"]
 enterprise = []
 python = ["dep:pyo3", "pyo3/extension-module", "dep:numpy", "dep:pythonize"]
 java = ["dep:jni"]
+simdeez_f = []
+stdsimd = []
+intel_cpu = ["simdeez_f", "stdsimd"]
+simd = ["intel_cpu"]
 
 [dependencies]
 # core columnar memory format
@@ -71,7 +86,16 @@ opencl3 = { version = "0.9", optional = true } # Intel / OpenCL
 
 # indexing
 roaring = "0.10.2" # For scalar bitmaps
-hnsw_rs = { version = "0.1.19", path = "vendor/hnsw_rs" } # For vector indexing
+# hnsw_rs removed and internalized to src/core/index/hnsw_rs
+bincode = "1.3"
+cpu-time = "1.0"
+env_logger = "0.10"
+log = "0.4"
+hashbrown = "0.9"
+num-traits = "0.2"
+num_cpus = "1.16"
+parking_lot = "0.12"
+skiplist = "0.3"
 # lance removed due to missing protoc
 
 # error handling
@@ -148,7 +172,7 @@ optional = true
 criterion = { version = "0.5", features = ["async_tokio"] }
 rand = "0.8"
 rand_distr = "0.4"
-hnsw_rs = { version = "0.1.19", path = "vendor/hnsw_rs" }
+# hnsw_rs removed and internalized
 tempfile = "3.10"
 proptest = "1.4"
 
@@ -167,3 +191,4 @@ harness = false
 [[bench]]
 name = "bench_table"
 harness = false
+