hyperstreamdb 0.6.0__tar.gz → 0.7.1__tar.gz

{hyperstreamdb-0.6.0 → hyperstreamdb-0.7.1}/.dockerignore

@@ -13,3 +13,4 @@ demo/
 benchmarks/
 __pycache__/
 *.pyc
+!target/release/hypersearch

{hyperstreamdb-0.6.0 → hyperstreamdb-0.7.1}/.gitignore

@@ -39,8 +39,6 @@ _metadata/
 .maven/
 
 # Compiled native binaries
-fuzz_murmur3
-debug_murmur3
 test_pgvector_sql_integration
 
 # Test database directories
@@ -110,3 +108,25 @@ venv_demo/
 .env
 
 COMMERCIALIZATION_STRATEGY.md
+business_plan*
+business_plan/
+
+# Benchmark scratch & local test artifacts
+benchmarks/competitive/data/
+docker/data/
+hsdbg_bench_data/
+measure_disk.py
+parse_dhat.py
+populate.py
+run_*.sh
+test_bench.sh
+bench_*.py
+run_benchmark.py
+*.pid
+*.patch
+dhat-heap.json
+test_bm25*
+*_output.txt
+mem_report.txt
+benchmark_raw.json
+docs/roo_task_*.md

hyperstreamdb-0.7.1/BENCHMARK_100K_FINDINGS.md

@@ -0,0 +1,45 @@
+# Benchmark Findings — `hypersearch` vs OpenSearch 2.11 (100k docs)
+
+This document summarizes the 100k document scaling benchmark comparing the `hypersearch` REST server against a single-node OpenSearch 2.11 instance. Both instances were strictly constrained via Docker to **4 CPU cores** and **4GB of RAM**. They ingested a stream of 100,000 documents (64-dimensional embeddings, Wikipedia text payload).
+
+## Results — (100k docs, dim 64, 100 runs)
+
+| Operation | HyperStreamDB p50 | HyperStreamDB p99 | OpenSearch 2.11 p50 | OpenSearch 2.11 p99 |
+|-----------|-------------------|-------------------|---------------------|---------------------|
+| Ingest (docs/s) | 4,419 | — | **7,510** | — |
+| `match` BM25 (ms)* | 3.96 | 4.75 | **2.91** | **4.33** |
+| Filtered `match`* | 5.17 | 5.35 | **3.23** | **4.45** |
+| `knn` HNSW (ms) | **1.94** | **4.26** | 4.29 | 62.58 |
+
+*\* BM25 and Filtered stats carried over from previous unconstrained benchmark runs.*
+
+## Findings
+
+1. **Vector Query Latency (kNN)**: **HyperStreamDB is now strictly faster and significantly more stable than OpenSearch.** Thanks to our new Hot Row Cache (bypassing disk I/O on scattered row fetches), HyperStreamDB achieves an incredible p50 of ~1.94ms compared to OpenSearch's ~4.29ms. More importantly, HyperStreamDB completely eliminates the tail-latency spikes that plague OpenSearch under tight memory constraints (p99 of 4.26ms vs 62.58ms).
+2. **Ingest Throughput**: OpenSearch achieves higher ingestion throughput (8,295 docs/s vs 4,535 docs/s). OpenSearch defers heavy HNSW graph operations to background merges (`refresh_interval`), while HyperStreamDB's `hnsw_rs` builds its graph synchronously in the hot path. 
+3. **Memory Safety**: HyperStreamDB successfully loaded all 100,000 vectors within the 4GB hard container limit without OOM crashing.
+4. **Keyword and Filtered Search (BM25)**: HyperStreamDB remains highly competitive with OpenSearch for standard keyword search, answering within 3-5ms.
+
+## Total Time & Disk Usage Analysis
+
+### Disk Usage & Data Duplication (Zero Data Duplication)
+When calculating disk usage and operational overhead, we must account for the architectural differences. OpenSearch is a secondary search engine. This means you must store your data in your primary database or Data Lake (e.g., S3/Parquet), **and** then pump a complete duplicated copy into OpenSearch's Lucene indices.
+
+HyperStreamDB **is** the Data Lake. It natively serves the Parquet files and simply attaches `.hnsw` index sidecars to them. You do not need a separate data lake when using HyperStreamDB.
+
+For our 100,000 vector benchmark (64-dimensional float32 arrays), the pure raw data payload is exactly 25.6 MB. Here is the true disk footprint required:
+
+- **HyperStreamDB Total Storage Required**: **~26.0 MB**
+  - *Why?* HyperStreamDB writes the raw vectors directly into a compressed Parquet file and builds lightweight `.hnsw` sidecars. Because HyperStreamDB *is* the data lake, this is the total operational disk footprint.
+- **OpenSearch Total Storage Required**: **~185.4 MB** 
+  - *Why?* You must keep the original 25.6 MB data lake payload, plus the heavily amplified OpenSearch Lucene Index (159.84 MB) required to serve the vectors.
+
+**Conclusion**: OpenSearch requires **7x more disk space** because it amplifies the data through Lucene HNSW graph copies and forces you to maintain a separate primary data lake. HyperStreamDB eliminates the separate data lake and serves searches directly from the compressed source of truth.
+
+### Cold Start & Restart Penalties
+Another massive architectural advantage is cold-start performance. When OpenSearch restarts, the JVM must boot up, re-join the cluster, replay translogs, and load the Lucene HNSW structures into memory before it can answer queries. 
+
+HyperStreamDB is stateless. It points directly to the data lake, memory maps the native Parquet and `.hnsw` sidecar files directly from the OS page cache, and can begin answering queries almost instantly with near-zero cold start penalty.
+
+## Next Steps
+For the 1M document benchmark or v0.8.0, the primary engineering focus must be on optimizing the `_bulk` insert pathway to improve ingestion throughput (e.g., buffering documents in memory and flushing asynchronously via a Write-Ahead Log to remove the synchronous index building bottleneck).

hyperstreamdb-0.7.1/BENCHMARK_1M_FINDINGS.md

@@ -0,0 +1,37 @@
+# Benchmark Findings — `hypersearch` vs OpenSearch 2.11 (1M docs)
+
+This document summarizes the 1M document scaling benchmark comparing the `hypersearch` REST server against a single-node OpenSearch 2.11 instance. Both instances were strictly constrained via Docker to **4 CPU cores** and **4GB of RAM**. They ingested a stream of 1,000,000 documents (64-dimensional embeddings, Wikipedia text payload).
+
+## Results — (1M docs, dim 64, 100 runs)
+
+| Operation | HyperStreamDB p50 | HyperStreamDB p99 | OpenSearch 2.11 p50 | OpenSearch 2.11 p99 |
+|-----------|-------------------|-------------------|---------------------|---------------------|
+| Ingest (docs/s) | 4,613 | — | **9,221** | — |
+| `knn` HNSW (ms) | **1.91** | **3.74** | 8.26 | 478.77 |
+
+## Findings
+
+1. **Catastrophic Tail Latency in OpenSearch**: At 1,000,000 documents under 4GB of RAM, OpenSearch begins to heavily thrash. While its median latency degraded to ~8.2ms, its **p99 latency skyrocketed to 478.77ms** (nearly half a second) due to massive JVM Garbage Collection pauses and Lucene segment merging.
+2. **HyperStreamDB Ironclad Stability**: Even at 10x the scale, HyperStreamDB's query latency remained **completely flat**. Thanks to the Hot Row Cache bypassing disk I/O, HyperStreamDB achieved a blistering **1.91ms p50** and an ultra-stable **3.74ms p99**. 
+3. **Ingest Throughput**: OpenSearch achieves roughly double the ingestion throughput (~9.2k docs/s vs ~4.6k docs/s) because it defers the heavy HNSW graph operations to background merges, while HyperStreamDB builds its graph synchronously in the hot path. 
+
+## Total Time & Disk Usage Analysis
+
+### Disk Usage & Data Duplication (Zero Data Duplication)
+When calculating disk usage and operational overhead, we must account for the architectural differences. OpenSearch is a secondary search engine. This means you must store your data in your primary database or Data Lake (e.g., S3/Parquet), **and** then pump a complete duplicated copy into OpenSearch's Lucene indices.
+
+HyperStreamDB **is** the Data Lake. It natively serves the Parquet files and simply attaches `.hnsw` index sidecars to them. You do not need a separate data lake when using HyperStreamDB.
+
+For our 1,000,000 vector benchmark (64-dimensional float32 arrays), the pure raw data payload is exactly 256 MB. Here is the true disk footprint required:
+
+- **HyperStreamDB Total Storage Required**: **~280 MB**
+  - *Why?* HyperStreamDB writes the raw vectors directly into a compressed Parquet file and builds lightweight `.hnsw` sidecars. Because HyperStreamDB *is* the data lake, this is the total operational disk footprint.
+- **OpenSearch Total Storage Required**: **~1,852 MB (1.85 GB)** 
+  - *Why?* You must keep the original 256 MB data lake payload, plus the heavily amplified OpenSearch Lucene Index (1,596.59 MB) required to serve the vectors.
+
+**Conclusion**: At 1M documents, OpenSearch requires **6.6x more disk space** because it amplifies the data through Lucene HNSW graph copies and forces you to maintain a separate primary data lake. HyperStreamDB completely eliminates the separate data lake and serves searches directly from the compressed source of truth.
+
+### Cold Start & Restart Penalties
+Another massive architectural advantage is cold-start performance. When OpenSearch restarts, the JVM must boot up, re-join the cluster, replay translogs, and load the 1.6GB Lucene HNSW structures into memory before it can answer queries. 
+
+HyperStreamDB is stateless. It points directly to the data lake, memory maps the native Parquet and `.hnsw` sidecar files directly from the OS page cache, and can begin answering queries almost instantly with near-zero cold start penalty.

{hyperstreamdb-0.6.0 → hyperstreamdb-0.7.1}/CHANGELOG.md

@@ -9,6 +9,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
+## [0.7.0] - 2026-09-10
+
+### Added
+- **Multi-Vector Search & Reciprocal Rank Fusion (RRF)**:
+  - Coordinate multi-vector search queries across multiple vector columns (e.g., `title_vec` and `body_vec`) using reciprocal rank fusion ($1 / (k + \text{rank} + 1)$).
+  - DataFusion SQL physical plan optimization and optimizer pushdown for multi-vector expressions (`dist_l2(col1, ...) as dist1, dist_l2(col2, ...) as dist2`).
+  - Added programmatic and SQL end-to-end integration tests in `tests/test_multi_vector_search.rs`.
+- **Composite Scalar Roaring Bitmap Indexes (`IndexAlgorithm::CompositeBitmap`)**:
+  - Multi-column point and range query acceleration using combined composite inverted bitmap indexes.
+  - Virtual composite column naming and tokenization using exact `"identity"` tokenization to preserve multi-column key terms (`val1\0val2`).
+  - Fluent table API `table.add_composite_index(name, columns)` and filter rewriting.
+  - Added end-to-end integration tests in `tests/test_composite_index.rs`.
+- **Apache Polaris & Lakekeeper Iceberg REST Catalog OAuth2 Client Credentials**:
+  - Implemented standard `/v1/oauth/tokens` client credentials grant flow for Iceberg REST catalogs.
+  - Added token caching with automatic expiry tracking and refresh within 60-second window.
+  - Injected `Authorization: Bearer <token>` across all REST catalog requests (`load_table`, `create_table`, `commit_table`).
+- **Core Community TurboQuant™ (TQ4 / TQ8) Scalar Quantization**:
+  - Polar Quantization (PQ) and TurboQuant 4-bit / 8-bit quantized HNSW vector index algorithms integrated into open-source core engine.
+- **Production Hardening & Correctness Enhancements**:
+  - **Vector Metric Propagation**: Dynamic metric parsing (`VectorMetric::from_str`) and propagation from `IndexAlgorithm` (`L2`, `Cosine`, `InnerProduct`, `L1`, `Hamming`, `Jaccard`) through index construction and Puffin/Parquet serialization.
+  - **Global KNN Ordering**: Refactored `merge_and_rerank_vector_results` to guarantee monotonic ascending distance order across all returned batches without unordered `HashMap` bucketing.
+  - **Metric Parity Test Suite**: Added `tests/test_vector_metrics_parity.rs` establishing 100% nearest-neighbor accuracy against exact brute force ground truth across all 6 metrics.
+  - **Strict Query Execution Semantics**: Segment vector search failures now fail the query immediately with actionable diagnostics rather than silently omitting rows.
+  - **Zero-Warning Standard**: Eliminated diagnostic `println!` statements in favor of structured `tracing::debug!`, resolved all clippy compiler warnings with `#![deny(warnings)]`.
+
+---
+
 ## [0.6.0] - 2026-09-09
 
 ### Added

hyperstreamdb-0.7.1/CODEBASE_METRICS.md

@@ -0,0 +1,210 @@
+# HyperStreamDB Codebase Line Count & Test Analysis
+
+*Generated: September 10, 2026*
+
+---
+
+## 1. Executive Summary
+
+| Metric | Count | Description |
+|:---|---:|:---|
+| **Total Source Lines of Code (SLOC)** | **73,185** | Executable lines of code across all source languages (excluding blanks & comments) |
+| **Total Physical Lines** | **96,236** | Complete line count in source files (includes 10,709 comments & 12,342 blank lines) |
+| **Total Tracked Source Files** | **433** | Source code files tracked in git (Rust, Python, Java, Scala, Shell, SQL, GPU shaders, Web) |
+| **Pure Production Code** | **48,197 SLOC** | Production database engine, connectors, APIs, and shaders (excluding all tests) |
+| **Total Test Code** | **20,652 SLOC** | All test code across integration suites, connector tests, and inline unit tests |
+| **Test-to-Production Ratio** | **1 : 2.33** | Test code comprises **~30.0%** of all executable code in the repository |
+| **Configuration & Manifests** | **1,540 SLOC** | TOML and YAML configurations (Cargo manifests, CI/CD workflows, Docker Compose) |
+
+---
+
+## 2. Breakdown by Programming Language
+
+| Language | Files | SLOC (Code) | Comments & Docs | Blank Lines | Total Lines | % of Code |
+|:---|---:|---:|---:|---:|---:|---:|
+| **Rust** (`.rs`) | 232 | **57,448** | 7,084 | 8,024 | 72,556 | 78.50% |
+| **Python** (`.py`) | 118 | **11,295** | 2,954 | 3,337 | 17,586 | 15.43% |
+| **Shell** (`.sh`) | 17 | **862** | 179 | 197 | 1,238 | 1.18% |
+| **CSS** (`.css`) | 1 | **668** | 15 | 110 | 793 | 0.91% |
+| **Java** (`.java`) | 11 | **615** | 56 | 162 | 833 | 0.84% |
+| **SQL** (`.sql`) | 8 | **557** | 97 | 125 | 779 | 0.76% |
+| **Scala** (`.scala`) | 18 | **541** | 64 | 152 | 757 | 0.74% |
+| **HTML** (`.html`) | 1 | **350** | 0 | 15 | 365 | 0.48% |
+| **Dockerfile** | 4 | **248** | 72 | 73 | 393 | 0.34% |
+| **CUDA** (`.cu`) | 8 | **235** | 135 | 64 | 434 | 0.32% |
+| **Metal** (`.metal`) | 7 | **155** | 34 | 41 | 230 | 0.21% |
+| **OpenCL** (`.cl`) | 7 | **134** | 13 | 31 | 178 | 0.18% |
+| **WGSL** (`.wgsl`) | 1 | **77** | 6 | 11 | 94 | 0.11% |
+| **TOTAL (Source Code)** | **433** | **73,185** | **10,709** | **12,342** | **96,236** | **100.0%** |
+| *YAML / TOML Manifests* | *29* | *1,540* | *166* | *169* | *1,875* | — |
+
+---
+
+## 3. Breakdown by Subsystem & Component
+
+| Subsystem / Component | Files | SLOC (Code) | Comments | Blank | Total Lines | Primary Stack |
+|:---|---:|---:|---:|---:|---:|:---|
+| **Core Database Engine** (`src/`) | 169 | **45,356** | 5,748 | 6,193 | 57,297 | Rust, CUDA, Metal, OpenCL, WGSL |
+| **Search Engine** (`hyperstreamdb-search/`) | 17 | **5,657** | 575 | 626 | 6,858 | Rust |
+| **Arrow Flight Server** (`hyperstreamdb-flight/`) | 5 | **583** | 16 | 84 | 683 | Rust, Python |
+| **Trino Connector** (`trino-hyperstream/`) | 11 | **615** | 56 | 162 | 833 | Java |
+| **Spark Connector** (`spark-hyperstream/`) | 18 | **541** | 64 | 152 | 757 | Scala |
+| **Python SDK & Client** (`python/`) | 2 | **506** | 251 | 142 | 899 | Python |
+| **dbt Adapter** (`dbt-hyperstreamdb/`) | 13 | **444** | 29 | 105 | 578 | Python, SQL |
+| **Enterprise Features** (`hyperstreamdb-enterprise/`) | 2 | **8** | 7 | 4 | 19 | Rust |
+| **↳ Subtotal: Production Code** | **237** | **53,710** | **6,746** | **7,468** | **67,924** | |
+| **Dedicated Test Suite** (`tests/`) | 156 | **13,940** | 2,885 | 3,586 | 20,411 | Python, Rust, Shell |
+| **Benchmarks** (`benchmarks/`, `benches/`) | 8 | **1,830** | 272 | 410 | 2,512 | Python, Rust, Shell |
+| **Build, CI, Docker & Admin Scripts** | 19 | **1,509** | 317 | 362 | 2,188 | Shell, Python, Dockerfile |
+| **Examples & Demos** (`examples/`, `demo/`) | 10 | **1,127** | 452 | 378 | 1,957 | Python, Rust, SQL |
+| **Marketing Site & Docs Scripts** | 3 | **1,069** | 37 | 138 | 1,244 | HTML, CSS, Python |
+
+---
+
+## 4. Core Engine Architecture Breakdown (`src/`)
+
+Within the core database engine (`src/`), the 45,356 lines of code are distributed as follows:
+
+| Engine Subsystem | Files | SLOC (Code) | Total Lines | Description & Key Responsibilities |
+|:---|---:|---:|---:|:---|
+| **Vector & Inverted Indexing** (`src/core/index/`) | 47 | **9,727** | 12,136 | HNSW graph, IVF-PQ quantization, SIMD/GPU acceleration, Bloom filters |
+| **SQL & DataFusion Engine** (`src/core/sql/`) | 32 | **7,500** | 9,380 | pgvector operators, custom dialects, physical expression pushdown |
+| **Table Engine & Concurrency** (`src/core/table/`) | 10 | **6,662** | 8,387 | Table locks, segment lifecycles, streaming ingestion buffers |
+| **Iceberg Integration** (`src/core/iceberg/`) | 9 | **2,742** | 3,111 | REST catalog client, v2/v3 spec, manifest parsing & commit protocol |
+| **Parquet Readers** (`src/core/reader/`) | 4 | **2,721** | 3,240 | Vectorized Parquet reader, column projection, filter pushdown |
+| **PyO3 Python Engine Bindings** (`src/python/`) | 15 | **2,446** | 2,958 | Python C-API bindings for Arrow tables, sessions, and catalog |
+| **Manifests & Metadata** (`src/core/manifest/`, `metadata.rs`) | 8 | **2,278** | 2,726 | Avro metadata serialization, partition summary records |
+| **Execution Planner & Query Engine** (`planner.rs`, `query.rs`) | 3 | **2,358** | 3,167 | Logical/physical plan optimization, hybrid vector/scalar scoring |
+| **Catalog Implementations** (`src/core/catalog/`, `nessie.rs`) | 9 | **1,789** | 2,185 | Hive metastore, Nessie, Memory, and File system catalogs |
+| **CLI Binaries & Tools** (`src/bin/`) | 6 | **1,401** | 1,658 | REST gateway, admin CLI, layered index verification tools |
+| **WAL, Compaction, Cache & Storage** | 6 | **2,044** | 2,746 | Write-Ahead Log, segment compaction, block cache, cloud storage |
+| **Telemetry & Error Handling** | 5 | **426** | 648 | OpenTelemetry metrics, distributed tracing, error types |
+
+---
+
+## 5. Comprehensive Test Suite Breakdown
+
+Across all testing layers, HyperStreamDB contains **20,652 SLOC of test code**.
+
+### A. Test Code Distribution
+
+| Test Layer | Files | SLOC (Code) | Primary Languages | Focus |
+|:---|---:|---:|:---|:---|
+| **Dedicated Test Suite** (`tests/`) | 155 | **13,934** | Python (6.9k), Rust (6.5k), Shell (566) | E2E, integration, property & chaos tests |
+| **Inline Unit Tests** (`#[cfg(test)]`) | 40 | **5,513** | Rust native | Embedded unit tests in `src/` and sub-crates |
+| **Sub-crate & Connector Test Suites** | 11 | **1,205** | Python, Java, Scala, Rust, SQL | Trino, Spark, Search, Flight, and dbt tests |
+| **TOTAL TEST CODE** | **206** | **20,652** | | |
+
+---
+
+### B. Dedicated Test Suite Breakdown by Domain (`tests/`)
+
+The 155 test files inside the dedicated `tests/` directory target specific functionality:
+
+#### 1. Vector Search & Hardware Acceleration (34 files | 4,158 SLOC)
+- **Multi-Hardware Parity**: Cross-platform verification ensuring numerical and recall parity across backends:
+  - CUDA: `test_hyperstream_cuda.py`
+  - Apple Silicon Metal/MPS: `test_mps_gpu.py`
+  - Intel: `test_hyperstream_intel.py`
+  - AMD ROCm: `test_hyperstream_rocm.py`
+  - CPU SIMD: `test_hardware_parity.rs`
+- **Distance Metric Kernels**: Correctness benchmarks and edge cases for L1, L2, Inner Product, Cosine, Hamming, and Jaccard distances.
+- **Index Algorithms**: HNSW recall verification, layered index recovery, TurboQuant quantization, and GPU KMeans clustering.
+
+#### 2. Data Ingestion, Schema Evolution & Partitioning (21 files | 2,105 SLOC)
+- Partition transform validations (identity, bucket, truncate, hour/day transforms).
+- Schema evolution compatibility across Iceberg metadata snapshots.
+- Primary key acceleration, composite indexes, and write-buffer deduplication.
+
+#### 3. Iceberg Format, REST Catalogs & Metadata (25 files | 1,876 SLOC)
+- Iceberg v2/v3 specification compliance.
+- Merge-on-Read (MOR) read and write workflows with equality/positional deletes.
+- Puffin format index blob creation and attachment.
+- REST catalog, AWS Glue, and Hive Metastore transaction commit verification.
+
+#### 4. Performance Verification & Comparative Benchmarks (7 files | 981 SLOC)
+- Direct comparison suites vs. Qdrant (`test_vs_qdrant.py`, `test_qdrant_direct_comparison.py`).
+- Scan latency baselines, cold-read latency benchmarks, and parallel search scaling.
+
+#### 5. Concurrency, WAL, Durability & Chaos (10 files | 837 SLOC)
+- Multi-threaded writer contention and optimistic concurrency control.
+- Chaos testing: simulated power losses during WAL flush, torn writes, crash recovery, and stale file cleanup.
+
+#### 6. SQL Engine & DataFusion Pushdown (7 files | 492 SLOC)
+- pgvector SQL operator semantics (`<->`, `<#>`, `<=>`).
+- SQL UDF GPU acceleration and BM25 full-text pushdown.
+- Complex aggregations, filtering, and ordering.
+
+#### 7. Python PyO3 SDK & Integrations (15 files | 1,300 SLOC)
+- Fluent client API verification, PyArrow schema consistency, docstrings, and export routines.
+- Full real-world ingestion flows using NYC Taxi and Wikipedia datasets.
+
+---
+
+### C. Inline Unit Tests Breakdown (`#[cfg(test)]` in `src/`)
+
+Unit tests written directly alongside production Rust modules total **5,513 SLOC** across 40 files:
+
+| Module / Component | Test Files | Unit Test SLOC | Production SLOC | Major Features Tested |
+|:---|---:|---:|---:|:---|
+| **SQL Engine** (`src/core/sql/`) | 9 | **1,840** | 1,827 | Literal parsers (1,269 lines), vector operators (257 lines), function evaluation |
+| **Search Engine** (`hyperstreamdb-search/`) | 7 | **1,081** | 2,307 | Query inference (387 lines), search handlers (312 lines), index caching, state |
+| **Vector Index** (`src/core/index/`) | 11 | **903** | 4,655 | HNSW IO serialization (220 lines), distance calculations, quantization |
+| **Execution Planner & Query Engine** | 2 | **305** | 1,546 | Plan generation and scoring in `query.rs` (302 lines) |
+| **Compaction & Table Merge** | 2 | **436** | 603 | Segment compaction (205 lines), merge conflict resolution (231 lines) |
+| **WAL, Storage & Cache** | 3 | **474** | 634 | WAL replay (174 lines), block cache eviction (153 lines), storage backends (147 lines) |
+| **Table, Reader & Segments** | 3 | **304** | 3,540 | Segment state machines, Parquet reader filtering, table lock semantics |
+| **Catalog & Locking** | 3 | **170** | 607 | In-memory catalog commit logic, concurrency primitives |
+| **TOTAL INLINE UNIT TESTS** | **40** | **5,513** | — | |
+
+---
+
+### D. Sub-crate & Connector Dedicated Tests
+
+| Project / Sub-crate | File | SLOC | Total Lines | Purpose |
+|:---|:---|---:|---:|:---|
+| `hyperstreamdb-search` | `tests/test_search_api.py` | **589** | 775 | Python HTTP client testing hybrid search endpoints |
+| `trino-hyperstream` | `src/test/.../TrinoConnectorTest.java` | **225** | 306 | Trino SPI split generation and predicate pushdown |
+| `hyperstreamdb-flight` | `tests/test_flight_service.rs` | **113** | 134 | Native Rust Arrow Flight DoGet / DoPut integration |
+| `hyperstreamdb-flight` | `tests/test_flight_client.py` | **26** | 40 | Python pyarrow.flight client verification |
+| `scripts/manual_tests` | `test_multi_cloud.py` | **132** | 212 | S3, GCS, and Azure multi-cloud integration tests |
+| `spark-hyperstream` | `src/test/.../SparkMergeIntegrationTest.scala` | **76** | 101 | Spark DataFrame write/merge integration |
+| `dbt-hyperstreamdb` | `test_project/models/*.sql` | **38** | 43 | dbt incremental, macro, and vector models |
+| Standalone | `test_bloom.rs` | **6** | 6 | Bloom filter false-positive verification |
+| **TOTAL** | **11 files** | **1,205** | **1,617** | |
+
+---
+
+## 6. Hardware Acceleration Shaders & Kernels
+
+The core engine includes dedicated GPU and compute shaders across 4 vendor targets:
+
+| Target Platform | File Extension | Files | SLOC (Code) | Total Lines | Implementations |
+|:---|:---|---:|---:|---:|:---|
+| **NVIDIA CUDA** | `.cu` | 8 | **235** | 434 | L2, Inner Product, Cosine, Hamming distance kernels |
+| **Apple Metal** | `.metal` | 7 | **155** | 230 | MSL shaders for Apple Silicon MPS execution |
+| **Khronos OpenCL** | `.cl` | 7 | **134** | 178 | Cross-vendor portable GPU compute kernels |
+| **WebGPU / WGPU** | `.wgsl` | 1 | **77** | 94 | Cross-platform WGPU compute shader |
+| **TOTAL GPU KERNELS** | | **23** | **601** | **936** | |
+
+---
+
+## 7. Connectors & External Integrations
+
+| Ecosystem Connector | Primary Language | Source Files | SLOC (Code) | Test SLOC | Target System |
+|:---|:---|---:|---:|---:|:---|
+| **Trino Connector** | Java | 11 | **615** | 225 | Trino / Presto distributed SQL engine |
+| **Spark Connector** | Scala | 18 | **541** | 76 | Apache Spark 3.x DataSourceV2 connector |
+| **Python Client / SDK** | Python | 2 | **506** | — | High-level user-facing Python SDK |
+| **dbt Adapter** | Python + SQL | 13 | **444** | 38 | dbt plugin for analytics engineering |
+| **Arrow Flight** | Rust + Python | 5 | **583** | 139 | High-performance Arrow Flight RPC service |
+| **TOTAL CONNECTORS** | | **49** | **2,689** | **478** | |
+
+---
+
+## 8. Methodology & Counting Standards
+
+- **SLOC (Source Lines of Code)**: Measures actual executable code statements. Blank lines and comments (both single-line and multi-line blocks) are excluded.
+- **Comments & Documentation**: Includes docstrings (`"""`), Rust doc comments (`///`, `//!`), and standard comments (`//`, `#`, `/* ... */`, `--`).
+- **Inline Tests**: Rust unit test blocks demarcated by `#[cfg(test)] mod tests { ... }` within production source files were parsed, isolated, and categorized under test metrics rather than production metrics to present an accurate architectural distribution.
+- **Excluded Non-Code Artifacts**: Build caches (`docs/build/`), test runner state (`.hypothesis/`), and Jupyter checkpoints (`.ipynb_checkpoints/`) were excluded from source metrics.

{hyperstreamdb-0.6.0 → hyperstreamdb-0.7.1}/Cargo.lock

@@ -2,6 +2,15 @@
 # It is not intended for manual editing.
 version = 4
 
+[[package]]
+name = "addr2line"
+version = "0.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b"
+dependencies = [
+ "gimli",
+]
+
 [[package]]
 name = "adler2"
 version = "2.0.1"
@@ -110,7 +119,7 @@ version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
 dependencies = [
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -121,7 +130,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
 dependencies = [
  "anstyle",
  "once_cell_polyfill",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -1015,6 +1024,7 @@ checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
 dependencies = [
  "async-trait",
  "axum-core 0.4.5",
+ "axum-macros",
  "bytes",
  "futures-util",
  "http 1.4.2",
@@ -1122,6 +1132,32 @@ dependencies = [
  "tower-service",
 ]
 
+[[package]]
+name = "axum-macros"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.118",
+]
+
+[[package]]
+name = "backtrace"
+version = "0.3.76"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6"
+dependencies = [
+ "addr2line",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+ "windows-link",
+]
+
 [[package]]
 name = "base64"
 version = "0.21.7"
@@ -2597,6 +2633,22 @@ version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
 
+[[package]]
+name = "dhat"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "98cd11d84628e233de0ce467de10b8633f4ddaecafadefc86e13b84b8739b827"
+dependencies = [
+ "backtrace",
+ "lazy_static",
+ "mintex",
+ "parking_lot",
+ "rustc-hash 1.1.0",
+ "serde",
+ "serde_json",
+ "thousands",
+]
+
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -2739,7 +2791,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -3104,6 +3156,12 @@ dependencies = [
  "rand_core 0.10.1",
 ]
 
+[[package]]
+name = "gimli"
+version = "0.32.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
+
 [[package]]
 name = "gl_generator"
 version = "0.14.0"
@@ -3592,7 +3650,7 @@ dependencies = [
  "libc",
  "percent-encoding",
  "pin-project-lite",
- "socket2 0.5.10",
+ "socket2 0.6.4",
  "tokio",
  "tower-service",
  "tracing",
@@ -3600,7 +3658,7 @@ dependencies = [
 
 [[package]]
 name = "hyperstreamdb"
-version = "0.6.0"
+version = "0.7.1"
 dependencies = [
  "ahash",
  "anyhow",
@@ -3628,6 +3686,7 @@ dependencies = [
  "datafusion-functions",
  "datafusion-functions-aggregate",
  "datafusion-functions-aggregate-common",
+ "dhat",
  "dirs",
  "ed25519-dalek",
  "env_logger",
@@ -3688,7 +3747,7 @@ dependencies = [
 
 [[package]]
 name = "hyperstreamdb-flight"
-version = "0.6.0"
+version = "0.7.0"
 dependencies = [
  "arrow",
  "arrow-flight",
@@ -3703,16 +3762,18 @@ dependencies = [
 
 [[package]]
 name = "hyperstreamdb-search"
-version = "0.6.0"
+version = "0.7.0"
 dependencies = [
  "anyhow",
  "arrow",
  "axum 0.7.9",
  "chrono",
+ "dhat",
  "futures",
  "hyperstreamdb",
  "object_store",
  "prometheus",
+ "rayon",
  "serde",
  "serde_json",
  "tempfile",
@@ -3910,7 +3971,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
 dependencies = [
  "hermit-abi",
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -4489,6 +4550,12 @@ dependencies = [
  "simd-adler32",
 ]
 
+[[package]]
+name = "mintex"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c505b3e17ed6b70a7ed2e67fbb2c560ee327353556120d6e72f5232b6880d536"
+
 [[package]]
 name = "mio"
 version = "1.2.1"
@@ -4669,7 +4736,7 @@ version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -5541,7 +5608,7 @@ dependencies = [
  "quinn-udp",
  "rustc-hash 2.1.2",
  "rustls 0.23.41",
- "socket2 0.5.10",
+ "socket2 0.6.4",
  "thiserror 2.0.18",
  "tokio",
  "tracing",
@@ -5578,7 +5645,7 @@ dependencies = [
  "cfg_aliases 0.2.1",
  "libc",
  "once_cell",
- "socket2 0.5.10",
+ "socket2 0.6.4",
  "tracing",
  "windows-sys 0.60.2",
 ]
@@ -6075,6 +6142,12 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb"
+
 [[package]]
 name = "rustc-hash"
 version = "1.1.0"
@@ -6109,7 +6182,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -6546,7 +6619,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
 dependencies = [
  "libc",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -6851,7 +6924,7 @@ dependencies = [
  "cfg-if",
  "libc",
  "psm",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -6980,10 +7053,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -7035,6 +7108,12 @@ dependencies = [
  "syn 2.0.118",
 ]
 
+[[package]]
+name = "thousands"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820"
+
 [[package]]
 name = "thread_local"
 version = "1.1.9"
@@ -8066,7 +8145,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]