pyvectorhound 1.4.0__tar.gz → 1.5.0__tar.gz

{pyvectorhound-1.4.0 → pyvectorhound-1.5.0}/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 All notable changes to PyVectorHound are documented in this file.
 
+## [1.5.0]
+
+### Added
+
+- **int8 scalar quantization** (`src/quantization.rs`) — per-vector min/max
+  linear quantization to `i8`, with `quantize`/`dequantize` (and batch
+  variants) exposed via PyO3. 4x memory reduction vs `f32` for large local
+  vector indices, with measured reconstruction error (0.0039 max on a
+  384-dim `[-1,1]` test vector) within the theoretical 8-bit bound
+  (`range/255`). SIMD/GPU acceleration is a separate, larger effort and
+  intentionally out of scope here.
+
 ## [1.4.0]
 
 ### Added

{pyvectorhound-1.4.0 → pyvectorhound-1.5.0}/Cargo.lock

@@ -218,7 +218,7 @@ dependencies = [
 
 [[package]]
 name = "pyvectorhound"
-version = "1.4.0"
+version = "1.5.0"
 dependencies = [
  "numpy",
  "pyo3",

{pyvectorhound-1.4.0 → pyvectorhound-1.5.0}/Cargo.toml

@@ -1,11 +1,10 @@
 [package]
 name = "pyvectorhound"
-version = "1.4.0"
+version = "1.5.0"
 edition = "2021"
 authors = ["Georgi Mammen Mullassery <mullassery@gmail.com>"]
 license = "Proprietary"
 description = "Vector search diagnostics engine for RAG/LLM systems"
-readme = "README.md"
 
 [dependencies]
 pyo3 = { version = "0.23", features = ["extension-module"] }

{pyvectorhound-1.4.0 → pyvectorhound-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.3
 Name: pyvectorhound
-Version: 1.4.0
+Version: 1.5.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -18,46 +18,47 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Text Processing
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
-Requires-Dist: numpy>=1.21,<3
-Requires-Dist: requests>=2.28,<3
-Requires-Dist: python-frontmatter>=1.0.0,<1.2.0
-Requires-Dist: pydantic>=2.0,<3
+Requires-Dist: numpy >=1.21, <3
+Requires-Dist: requests >=2.28, <3
+Requires-Dist: python-frontmatter >=1.0.0, <1.2.0
+Requires-Dist: pydantic >=2.0, <3
+Requires-Dist: qdrant-client >=1.7, <3 ; extra == 'qdrant'
+Requires-Dist: chromadb >=0.4, <1 ; extra == 'chroma'
+Requires-Dist: pymilvus >=2.3, <3 ; extra == 'milvus'
+Requires-Dist: weaviate-client >=3.24, <4 ; extra == 'weaviate'
+Requires-Dist: psycopg2-binary >=2.9, <3 ; extra == 'postgres'
+Requires-Dist: psycopg2-binary >=2.9, <3 ; extra == 'pgvector'
+Requires-Dist: pytest >=7.0, <9 ; extra == 'dev'
+Requires-Dist: pytest-cov >=4.0, <7 ; extra == 'dev'
+Requires-Dist: mypy >=1.0, <2 ; extra == 'dev'
+Requires-Dist: black >=23.0, <26 ; extra == 'dev'
+Requires-Dist: ruff >=0.1, <1 ; extra == 'dev'
+Requires-Dist: maturin >=1.7, <2 ; extra == 'dev'
 Requires-Dist: pyvectorhound[qdrant,chroma,milvus,weaviate,postgres,pgvector,dev] ; extra == 'all'
-Requires-Dist: chromadb>=0.4,<1 ; extra == 'chroma'
-Requires-Dist: pytest>=7.0,<9 ; extra == 'dev'
-Requires-Dist: pytest-cov>=4.0,<7 ; extra == 'dev'
-Requires-Dist: mypy>=1.0,<2 ; extra == 'dev'
-Requires-Dist: black>=23.0,<26 ; extra == 'dev'
-Requires-Dist: ruff>=0.1,<1 ; extra == 'dev'
-Requires-Dist: maturin>=1.7,<2 ; extra == 'dev'
-Requires-Dist: pymilvus>=2.3,<3 ; extra == 'milvus'
-Requires-Dist: psycopg2-binary>=2.9,<3 ; extra == 'pgvector'
-Requires-Dist: psycopg2-binary>=2.9,<3 ; extra == 'postgres'
-Requires-Dist: qdrant-client>=1.7,<3 ; extra == 'qdrant'
-Requires-Dist: weaviate-client>=3.24,<4 ; extra == 'weaviate'
-Provides-Extra: all
+Provides-Extra: qdrant
 Provides-Extra: chroma
-Provides-Extra: dev
 Provides-Extra: milvus
-Provides-Extra: pgvector
-Provides-Extra: postgres
-Provides-Extra: qdrant
 Provides-Extra: weaviate
+Provides-Extra: postgres
+Provides-Extra: pgvector
+Provides-Extra: dev
+Provides-Extra: all
 License-File: LICENSE
 Summary: Diagnostic tool for vector search failures in RAG and LLM systems. Evaluate embedding quality, analyze vector search rank correlation, benchmark BM25 vs semantic search, profile reranker performance. Identify root cause of retrieval failures.
 Keywords: rag,retrieval-augmented-generation,vector-search,embedding,semantic-search,llm,large-language-models,information-retrieval,search-ranking,evaluation-metrics,bm25,reranking,debugging,diagnostics,nlp,natural-language-processing,similarity-search,faiss,elasticsearch,pinecone,milvus,ai-debugging
+Author: Georgi Mammen Mullassery <mullassery@gmail.com>
 Author-email: Georgi Mammen Mullassery <mullassery@gmail.com>
 Maintainer-email: Georgi Mammen Mullassery <mullassery@gmail.com>
 License: Proprietary
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
-Project-URL: Bug Tracker, https://github.com/Mullassery/Pyvectorhound/issues
-Project-URL: Changelog, https://github.com/Mullassery/Pyvectorhound/releases
-Project-URL: Discussions, https://github.com/Mullassery/Pyvectorhound/discussions
-Project-URL: Documentation, https://github.com/Mullassery/Pyvectorhound#readme
 Project-URL: Homepage, https://github.com/Mullassery/Pyvectorhound
 Project-URL: Repository, https://github.com/Mullassery/Pyvectorhound
+Project-URL: Documentation, https://github.com/Mullassery/Pyvectorhound#readme
+Project-URL: Bug Tracker, https://github.com/Mullassery/Pyvectorhound/issues
+Project-URL: Changelog, https://github.com/Mullassery/Pyvectorhound/releases
 Project-URL: Source Code, https://github.com/Mullassery/Pyvectorhound/tree/main
+Project-URL: Discussions, https://github.com/Mullassery/Pyvectorhound/discussions
 
 # PyVectorHound
 
@@ -131,6 +132,50 @@ native `_core` extension but not yet exposed as a Python-callable function —
 if you need it from Python today, treat it as in-progress internal
 infrastructure rather than a public API.
 
+### New: int8 scalar quantization (Rust core)
+
+`src/quantization.rs` adds per-vector int8 scalar quantization for shrinking
+the memory footprint of large local vector indices. Each `f32` vector is
+mapped linearly to the `i8` range (-128..=127) using that vector's own
+min/max, alongside a `(scale, offset)` pair needed to dequantize it back to
+an approximate `f32` vector. This is scalar quantization, not product
+quantization, and it isn't SIMD/GPU accelerated — both are out of scope for
+this feature.
+
+**Tradeoff:** storing `i8` instead of `f32` is a **4x memory reduction**
+for the vector data (1 byte/dim vs 4 bytes/dim), at the cost of bounded
+reconstruction error — max absolute error per element is at most
+`(max - min) / 255` for the vector being quantized (half a quantization
+step in practice). For a typical normalized embedding in `[-1, 1]`, that's
+a worst-case error bound of about `0.0078` per dimension; measured max
+error on a random 384-dim `[-1, 1]` vector in the test suite was `~0.0039`.
+Constant vectors (including all-zero) round-trip exactly.
+
+Exposed to Python via `pyvectorhound._core`:
+
+```python
+from pyvectorhound import _core
+
+# Single vector
+result = _core.py_quantize_vector([0.12, -0.87, 0.5, 0.0])
+# {"values": [i8, ...], "scale": float, "offset": float, "min": float, "max": float}
+restored = _core.py_dequantize_vector(result["values"], result["scale"], result["offset"])
+
+# Batch (one scale/offset per vector)
+batch = _core.py_quantize_batch([[0.1, 0.2], [-1.0, 1.0]])
+# {"values": [[i8, ...], ...], "params": [{"scale":..., "offset":..., "min":..., "max":...}, ...]}
+restored_batch = _core.py_dequantize_batch(
+    batch["values"],
+    [p["scale"] for p in batch["params"]],
+    [p["offset"] for p in batch["params"]],
+)
+```
+
+This is currently a low-level building block (quantize/dequantize
+primitives) rather than integrated into `Hound`'s storage/retrieval path —
+if you want quantized on-disk storage for your own index today, call these
+functions directly.
+
 ### Not yet real (known limitations)
 
 Being upfront about what's still a stub, rather than leaving it to look
@@ -146,16 +191,16 @@ finished:
   has a historical data store to compute a real trend from. Use
   `Hound.track_metric()` + `Hound.get_trend_report()` (backed by the real,
   tested `TrendAnalyzer`) instead.
-- As of v1.3.1, PyPI only carries a macOS arm64 / CPython 3.11 wheel and no
-  source distribution. On any other interpreter or OS, `pip install
-  pyvectorhound` does **not** build the current version from source — it
-  silently falls back to the last release that does have an sdist (currently
-  **1.3.0**, three releases behind), with no warning that you got an old
-  version. If you need the current release outside macOS arm64/CPython 3.11,
-  install straight from the repo instead, which does build the latest source
-  correctly (needs a Rust toolchain — `maturin`/`pip` handle the build, but
-  `cargo` must be available):
+- As of v1.4.0, PyPI carries a macOS arm64 / CPython 3.9 wheel plus a source
+  distribution (`sdist`). The wheel is still single-platform/single-ABI (no
+  `abi3` build yet — see below), but the sdist means `pip install
+  pyvectorhound` on any other interpreter or OS now builds the current
+  version from source instead of silently falling back to an old release,
+  as long as a Rust toolchain is available locally. If the sdist build
+  fails for you, install straight from the repo instead:
   `pip install git+https://github.com/Mullassery/PyVectorHound.git`.
+  A proper multi-platform wheel matrix (built in CI, `abi3` so one wheel
+  covers multiple CPython versions) is still open work.
 - GitHub Actions CI (the badge above) is currently red on every job across
   recent pushes to `main` — not because of failing tests, but because
   `.github/workflows/ci.yml`'s `dtolnay/rust-toolchain@v1` step is missing

{pyvectorhound-1.4.0 → pyvectorhound-1.5.0}/README.md

@@ -70,6 +70,50 @@ native `_core` extension but not yet exposed as a Python-callable function —
 if you need it from Python today, treat it as in-progress internal
 infrastructure rather than a public API.
 
+### New: int8 scalar quantization (Rust core)
+
+`src/quantization.rs` adds per-vector int8 scalar quantization for shrinking
+the memory footprint of large local vector indices. Each `f32` vector is
+mapped linearly to the `i8` range (-128..=127) using that vector's own
+min/max, alongside a `(scale, offset)` pair needed to dequantize it back to
+an approximate `f32` vector. This is scalar quantization, not product
+quantization, and it isn't SIMD/GPU accelerated — both are out of scope for
+this feature.
+
+**Tradeoff:** storing `i8` instead of `f32` is a **4x memory reduction**
+for the vector data (1 byte/dim vs 4 bytes/dim), at the cost of bounded
+reconstruction error — max absolute error per element is at most
+`(max - min) / 255` for the vector being quantized (half a quantization
+step in practice). For a typical normalized embedding in `[-1, 1]`, that's
+a worst-case error bound of about `0.0078` per dimension; measured max
+error on a random 384-dim `[-1, 1]` vector in the test suite was `~0.0039`.
+Constant vectors (including all-zero) round-trip exactly.
+
+Exposed to Python via `pyvectorhound._core`:
+
+```python
+from pyvectorhound import _core
+
+# Single vector
+result = _core.py_quantize_vector([0.12, -0.87, 0.5, 0.0])
+# {"values": [i8, ...], "scale": float, "offset": float, "min": float, "max": float}
+restored = _core.py_dequantize_vector(result["values"], result["scale"], result["offset"])
+
+# Batch (one scale/offset per vector)
+batch = _core.py_quantize_batch([[0.1, 0.2], [-1.0, 1.0]])
+# {"values": [[i8, ...], ...], "params": [{"scale":..., "offset":..., "min":..., "max":...}, ...]}
+restored_batch = _core.py_dequantize_batch(
+    batch["values"],
+    [p["scale"] for p in batch["params"]],
+    [p["offset"] for p in batch["params"]],
+)
+```
+
+This is currently a low-level building block (quantize/dequantize
+primitives) rather than integrated into `Hound`'s storage/retrieval path —
+if you want quantized on-disk storage for your own index today, call these
+functions directly.
+
 ### Not yet real (known limitations)
 
 Being upfront about what's still a stub, rather than leaving it to look
@@ -85,16 +129,16 @@ finished:
   has a historical data store to compute a real trend from. Use
   `Hound.track_metric()` + `Hound.get_trend_report()` (backed by the real,
   tested `TrendAnalyzer`) instead.
-- As of v1.3.1, PyPI only carries a macOS arm64 / CPython 3.11 wheel and no
-  source distribution. On any other interpreter or OS, `pip install
-  pyvectorhound` does **not** build the current version from source — it
-  silently falls back to the last release that does have an sdist (currently
-  **1.3.0**, three releases behind), with no warning that you got an old
-  version. If you need the current release outside macOS arm64/CPython 3.11,
-  install straight from the repo instead, which does build the latest source
-  correctly (needs a Rust toolchain — `maturin`/`pip` handle the build, but
-  `cargo` must be available):
+- As of v1.4.0, PyPI carries a macOS arm64 / CPython 3.9 wheel plus a source
+  distribution (`sdist`). The wheel is still single-platform/single-ABI (no
+  `abi3` build yet — see below), but the sdist means `pip install
+  pyvectorhound` on any other interpreter or OS now builds the current
+  version from source instead of silently falling back to an old release,
+  as long as a Rust toolchain is available locally. If the sdist build
+  fails for you, install straight from the repo instead:
   `pip install git+https://github.com/Mullassery/PyVectorHound.git`.
+  A proper multi-platform wheel matrix (built in CI, `abi3` so one wheel
+  covers multiple CPython versions) is still open work.
 - GitHub Actions CI (the badge above) is currently red on every job across
   recent pushes to `main` — not because of failing tests, but because
   `.github/workflows/ci.yml`'s `dtolnay/rust-toolchain@v1` step is missing

{pyvectorhound-1.4.0 → pyvectorhound-1.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "pyvectorhound"
-version = "1.4.0"
+version = "1.5.0"
 description = "Diagnostic tool for vector search failures in RAG and LLM systems. Evaluate embedding quality, analyze vector search rank correlation, benchmark BM25 vs semantic search, profile reranker performance. Identify root cause of retrieval failures."
 readme = "README.md"
 requires-python = ">=3.8"
@@ -93,6 +93,7 @@ Discussions = "https://github.com/Mullassery/Pyvectorhound/discussions"
 [tool.maturin]
 module-name = "pyvectorhound._core"
 features = ["pyo3/extension-module"]
+include = ["LICENSE"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

pyvectorhound-1.5.0/src/lib.rs

@@ -0,0 +1,196 @@
+// PyHound Core - Rust implementation of retrieval diagnostics
+//
+// High-performance embedding quality metrics, pipeline analysis, and diagnostics.
+
+mod metrics;
+mod quantization;
+mod retrieval_ranking;
+
+use metrics::{
+    compute_coverage, compute_distinctiveness, compute_isotropy, compute_retrieval_metrics,
+    detect_drift,
+};
+use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyList};
+use pyo3::Bound;
+use quantization::{dequantize, dequantize_batch, quantize, quantize_batch, QuantizationParams};
+
+/// Build a Python dict from quantization params: {"scale", "offset", "min", "max"}.
+fn params_to_dict<'py>(
+    py: Python<'py>,
+    params: &QuantizationParams,
+) -> PyResult<Bound<'py, PyDict>> {
+    let dict = PyDict::new(py);
+    dict.set_item("scale", params.scale)?;
+    dict.set_item("offset", params.offset)?;
+    dict.set_item("min", params.min)?;
+    dict.set_item("max", params.max)?;
+    Ok(dict)
+}
+
+/// Compute isotropy of embeddings (vector space utilization)
+#[pyfunction]
+fn py_compute_isotropy(embeddings: Vec<Vec<f32>>) -> PyResult<f32> {
+    Ok(compute_isotropy(&embeddings))
+}
+
+/// Compute coverage of embeddings (diversity of semantic space)
+#[pyfunction]
+fn py_compute_coverage(embeddings: Vec<Vec<f32>>) -> PyResult<f32> {
+    Ok(compute_coverage(&embeddings))
+}
+
+/// Compute distinctiveness of embeddings (semantic separation)
+#[pyfunction]
+fn py_compute_distinctiveness(embeddings: Vec<Vec<f32>>) -> PyResult<f32> {
+    Ok(compute_distinctiveness(&embeddings))
+}
+
+/// Detect drift in embedding quality
+#[pyfunction]
+fn py_detect_drift(
+    baseline_embeddings: Vec<Vec<f32>>,
+    current_embeddings: Vec<Vec<f32>>,
+) -> PyResult<f32> {
+    Ok(detect_drift(&baseline_embeddings, &current_embeddings))
+}
+
+/// Compute retrieval metrics (precision, recall, F1, MRR, NDCG)
+#[pyfunction]
+fn py_compute_retrieval_metrics(
+    py: Python,
+    retrieved: Vec<usize>,
+    relevant: Vec<usize>,
+    top_k: usize,
+) -> PyResult<PyObject> {
+    let metrics = compute_retrieval_metrics(&retrieved, &relevant, top_k);
+
+    let dict = PyDict::new(py);
+    dict.set_item("precision", metrics.precision)?;
+    dict.set_item("recall", metrics.recall)?;
+    dict.set_item("f1_score", metrics.f1_score)?;
+    dict.set_item("mrr", metrics.mrr)?;
+    dict.set_item("ndcg", metrics.ndcg)?;
+
+    Ok(dict.into())
+}
+
+/// Compute embedding quality score (composite metric)
+#[pyfunction]
+fn py_compute_quality_score(embeddings: Vec<Vec<f32>>) -> PyResult<f32> {
+    if embeddings.is_empty() {
+        return Ok(0.0);
+    }
+
+    let isotropy = compute_isotropy(&embeddings);
+    let coverage = compute_coverage(&embeddings);
+    let distinctiveness = compute_distinctiveness(&embeddings);
+
+    // Composite score: weighted average
+    let quality = (isotropy * 0.4 + coverage * 0.3 + distinctiveness * 0.3).clamp(0.0, 1.0);
+
+    Ok(quality)
+}
+
+/// Quantize a single embedding vector to int8 for storage.
+///
+/// Returns a dict: `{"values": List[int], "scale": float, "offset": float,
+/// "min": float, "max": float}`. `values` holds signed 8-bit integers
+/// (-128..=127); `scale`/`offset` are required to dequantize.
+#[pyfunction]
+fn py_quantize_vector(py: Python, values: Vec<f32>) -> PyResult<PyObject> {
+    let (quantized, params) = quantize(&values);
+
+    let dict = PyDict::new(py);
+    dict.set_item("values", quantized)?;
+    dict.set_item("scale", params.scale)?;
+    dict.set_item("offset", params.offset)?;
+    dict.set_item("min", params.min)?;
+    dict.set_item("max", params.max)?;
+
+    Ok(dict.into())
+}
+
+/// Dequantize an int8 vector (produced by `py_quantize_vector`) back to f32.
+#[pyfunction]
+fn py_dequantize_vector(values: Vec<i8>, scale: f32, offset: f32) -> PyResult<Vec<f32>> {
+    let params = QuantizationParams {
+        scale,
+        offset,
+        min: 0.0,
+        max: 0.0,
+    };
+    Ok(dequantize(&values, &params))
+}
+
+/// Quantize a batch of embedding vectors to int8, one set of quantization
+/// parameters per vector (per-vector min/max scalar quantization).
+///
+/// Returns a dict: `{"values": List[List[int]], "params": List[dict]}`
+/// where each entry of `params` has the same shape as
+/// `py_quantize_vector`'s return dict.
+#[pyfunction]
+fn py_quantize_batch(py: Python, vectors: Vec<Vec<f32>>) -> PyResult<PyObject> {
+    let (values, params) = quantize_batch(&vectors);
+
+    let values_list = PyList::empty(py);
+    for v in &values {
+        values_list.append(v)?;
+    }
+
+    let params_list = PyList::empty(py);
+    for p in &params {
+        params_list.append(params_to_dict(py, p)?)?;
+    }
+
+    let dict = PyDict::new(py);
+    dict.set_item("values", values_list)?;
+    dict.set_item("params", params_list)?;
+
+    Ok(dict.into())
+}
+
+/// Dequantize a batch of int8 vectors (produced by `py_quantize_batch`)
+/// back to f32, given parallel `scales` and `offsets` lists.
+#[pyfunction]
+fn py_dequantize_batch(
+    values: Vec<Vec<i8>>,
+    scales: Vec<f32>,
+    offsets: Vec<f32>,
+) -> PyResult<Vec<Vec<f32>>> {
+    if values.len() != scales.len() || values.len() != offsets.len() {
+        return Err(pyo3::exceptions::PyValueError::new_err(
+            "values, scales, and offsets must have the same length",
+        ));
+    }
+
+    let params: Vec<QuantizationParams> = scales
+        .into_iter()
+        .zip(offsets)
+        .map(|(scale, offset)| QuantizationParams {
+            scale,
+            offset,
+            min: 0.0,
+            max: 0.0,
+        })
+        .collect();
+
+    Ok(dequantize_batch(&values, &params))
+}
+
+/// PyHound Python module
+#[pymodule]
+fn _core(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
+    m.add_function(wrap_pyfunction!(py_compute_isotropy, m)?)?;
+    m.add_function(wrap_pyfunction!(py_compute_coverage, m)?)?;
+    m.add_function(wrap_pyfunction!(py_compute_distinctiveness, m)?)?;
+    m.add_function(wrap_pyfunction!(py_detect_drift, m)?)?;
+    m.add_function(wrap_pyfunction!(py_compute_retrieval_metrics, m)?)?;
+    m.add_function(wrap_pyfunction!(py_compute_quality_score, m)?)?;
+    m.add_function(wrap_pyfunction!(py_quantize_vector, m)?)?;
+    m.add_function(wrap_pyfunction!(py_dequantize_vector, m)?)?;
+    m.add_function(wrap_pyfunction!(py_quantize_batch, m)?)?;
+    m.add_function(wrap_pyfunction!(py_dequantize_batch, m)?)?;
+
+    Ok(())
+}

pyvectorhound-1.5.0/src/quantization.rs

@@ -0,0 +1,300 @@
+//! Int8 scalar quantization for vector storage.
+//!
+//! Large local vector indices stored as `Vec<f32>` cost 4 bytes per
+//! dimension. Scalar quantization maps each vector's float components
+//! linearly into the `i8` range (-128..=127), storing them as 1 byte per
+//! dimension plus a small per-vector `(scale, offset)` pair needed to
+//! reconstruct approximate float values. This gives a ~4x memory reduction
+//! for the vector data itself, at the cost of bounded reconstruction error.
+//!
+//! This is *scalar* quantization (per-vector min/max), not product
+//! quantization, and it is not SIMD/GPU accelerated — both are explicitly
+//! out of scope here.
+
+use serde::{Deserialize, Serialize};
+
+/// Parameters needed to dequantize an `i8` vector back to `f32`.
+///
+/// Reconstruction: `f32_value = i8_value as f32 * scale + offset`.
+#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
+pub struct QuantizationParams {
+    /// Multiplier applied to the raw i8 value.
+    pub scale: f32,
+    /// Additive offset applied after scaling.
+    pub offset: f32,
+    /// Original minimum value of the vector (kept for introspection).
+    pub min: f32,
+    /// Original maximum value of the vector (kept for introspection).
+    pub max: f32,
+}
+
+/// Quantize a single `f32` vector into `i8` values plus the parameters
+/// needed to dequantize it.
+///
+/// Uses per-vector min/max scalar quantization: the vector's min and max
+/// are mapped to the full `i8` range (-128..=127), and every other value is
+/// linearly interpolated between them. This is standard for embedding
+/// vectors, where per-vector dynamic range is usually similar across a
+/// batch but not identical.
+///
+/// An empty input returns an empty output with a degenerate
+/// (scale = 0, offset = 0) parameter set.
+///
+/// A constant vector (min == max, including all-zero) maps every value to
+/// `0i8` and reconstructs exactly via `offset` (scale = 0), so round-trip
+/// error is zero in that case.
+pub fn quantize(values: &[f32]) -> (Vec<i8>, QuantizationParams) {
+    if values.is_empty() {
+        return (
+            Vec::new(),
+            QuantizationParams {
+                scale: 0.0,
+                offset: 0.0,
+                min: 0.0,
+                max: 0.0,
+            },
+        );
+    }
+
+    let mut min = values[0];
+    let mut max = values[0];
+    for &v in values.iter() {
+        if v < min {
+            min = v;
+        }
+        if v > max {
+            max = v;
+        }
+    }
+
+    // Degenerate case: every value identical. Store the value itself as
+    // `offset`, use a zero scale, and quantize everything to 0 so
+    // dequantization is exact.
+    if min == max {
+        let params = QuantizationParams {
+            scale: 0.0,
+            offset: min,
+            min,
+            max,
+        };
+        return (vec![0i8; values.len()], params);
+    }
+
+    // Map [min, max] -> [-128, 127] (255 steps).
+    let scale = (max - min) / 255.0;
+    let offset = min + scale * 128.0; // value that maps to raw i8 = 0
+
+    let quantized: Vec<i8> = values
+        .iter()
+        .map(|&v| {
+            let raw = ((v - min) / scale).round() - 128.0;
+            raw.clamp(-128.0, 127.0) as i8
+        })
+        .collect();
+
+    let params = QuantizationParams {
+        scale,
+        offset,
+        min,
+        max,
+    };
+
+    (quantized, params)
+}
+
+/// Dequantize `i8` values back into approximate `f32` values using the
+/// scale/offset produced by [`quantize`].
+pub fn dequantize(values: &[i8], params: &QuantizationParams) -> Vec<f32> {
+    values
+        .iter()
+        .map(|&q| q as f32 * params.scale + params.offset)
+        .collect()
+}
+
+/// Quantize a batch of vectors independently (one `QuantizationParams` per
+/// vector). This is the typical entry point for shrinking a whole local
+/// vector index.
+pub fn quantize_batch(vectors: &[Vec<f32>]) -> (Vec<Vec<i8>>, Vec<QuantizationParams>) {
+    let mut out_values = Vec::with_capacity(vectors.len());
+    let mut out_params = Vec::with_capacity(vectors.len());
+    for v in vectors {
+        let (q, p) = quantize(v);
+        out_values.push(q);
+        out_params.push(p);
+    }
+    (out_values, out_params)
+}
+
+/// Dequantize a batch of vectors produced by [`quantize_batch`].
+pub fn dequantize_batch(values: &[Vec<i8>], params: &[QuantizationParams]) -> Vec<Vec<f32>> {
+    values
+        .iter()
+        .zip(params.iter())
+        .map(|(v, p)| dequantize(v, p))
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Deterministic pseudo-random f32 generator (xorshift) so tests don't
+    /// need an external `rand` dependency.
+    fn pseudo_random_floats(seed: u64, n: usize, lo: f32, hi: f32) -> Vec<f32> {
+        let mut state = seed.wrapping_add(0x9E3779B97F4A7C15);
+        (0..n)
+            .map(|_| {
+                // xorshift64
+                state ^= state << 13;
+                state ^= state >> 7;
+                state ^= state << 17;
+                let unit = (state % 1_000_000) as f32 / 1_000_000.0; // [0, 1)
+                lo + unit * (hi - lo)
+            })
+            .collect()
+    }
+
+    fn max_abs_error(original: &[f32], reconstructed: &[f32]) -> f32 {
+        original
+            .iter()
+            .zip(reconstructed.iter())
+            .map(|(a, b)| (a - b).abs())
+            .fold(0.0f32, f32::max)
+    }
+
+    #[test]
+    fn test_round_trip_unit_range() {
+        // Typical normalized-embedding range [-1, 1].
+        let values = pseudo_random_floats(42, 384, -1.0, 1.0);
+        let (quantized, params) = quantize(&values);
+        assert_eq!(quantized.len(), values.len());
+
+        let reconstructed = dequantize(&quantized, &params);
+        let err = max_abs_error(&values, &reconstructed);
+
+        // Expected bound for 8-bit quantization: half a quantization step,
+        // i.e. range / 255 (using <= with a small epsilon for float
+        // rounding slack).
+        let expected_bound = (params.max - params.min) / 255.0;
+        assert!(
+            err <= expected_bound + 1e-4,
+            "max abs error {err} exceeded expected bound {expected_bound}"
+        );
+    }
+
+    #[test]
+    fn test_round_trip_zero_to_one_range() {
+        // Typical unnormalized embedding range [0, 1].
+        let values = pseudo_random_floats(7, 768, 0.0, 1.0);
+        let (quantized, params) = quantize(&values);
+        let reconstructed = dequantize(&quantized, &params);
+        let err = max_abs_error(&values, &reconstructed);
+
+        let expected_bound = (params.max - params.min) / 255.0;
+        assert!(
+            err <= expected_bound + 1e-4,
+            "max abs error {err} exceeded expected bound {expected_bound}"
+        );
+    }
+
+    #[test]
+    fn test_round_trip_wide_range() {
+        // Wider range to make sure scale/offset math generalizes beyond
+        // [-1, 1] / [0, 1].
+        let values = pseudo_random_floats(1234, 512, -50.0, 75.0);
+        let (quantized, params) = quantize(&values);
+        let reconstructed = dequantize(&quantized, &params);
+        let err = max_abs_error(&values, &reconstructed);
+
+        let expected_bound = (params.max - params.min) / 255.0;
+        assert!(
+            err <= expected_bound + 1e-4,
+            "max abs error {err} exceeded expected bound {expected_bound}"
+        );
+    }
+
+    #[test]
+    fn test_all_zero_vector_is_exact() {
+        let values = vec![0.0f32; 128];
+        let (quantized, params) = quantize(&values);
+        assert!(quantized.iter().all(|&q| q == 0));
+
+        let reconstructed = dequantize(&quantized, &params);
+        let err = max_abs_error(&values, &reconstructed);
+        assert_eq!(err, 0.0, "all-zero vector should round-trip exactly");
+    }
+
+    #[test]
+    fn test_single_value_constant_vector_is_exact() {
+        let values = vec![3.5f32; 64];
+        let (quantized, params) = quantize(&values);
+        let reconstructed = dequantize(&quantized, &params);
+        let err = max_abs_error(&values, &reconstructed);
+        assert_eq!(err, 0.0, "constant vector should round-trip exactly");
+        assert!(reconstructed.iter().all(|&v| v == 3.5));
+    }
+
+    #[test]
+    fn test_single_element_vector() {
+        let values = vec![-0.234f32];
+        let (quantized, params) = quantize(&values);
+        assert_eq!(quantized.len(), 1);
+        let reconstructed = dequantize(&quantized, &params);
+        // A single element is trivially min == max -> exact round trip.
+        assert_eq!(reconstructed[0], values[0]);
+    }
+
+    #[test]
+    fn test_empty_vector() {
+        let values: Vec<f32> = Vec::new();
+        let (quantized, params) = quantize(&values);
+        assert!(quantized.is_empty());
+        let reconstructed = dequantize(&quantized, &params);
+        assert!(reconstructed.is_empty());
+    }
+
+    #[test]
+    fn test_extremes_map_to_i8_bounds() {
+        let values = vec![-1.0f32, 0.0, 0.5, 1.0];
+        let (quantized, _params) = quantize(&values);
+        assert_eq!(*quantized.first().unwrap(), -128);
+        assert_eq!(*quantized.last().unwrap(), 127);
+    }
+
+    #[test]
+    fn test_memory_footprint_is_4x_smaller() {
+        // i8 storage is 1 byte per dim vs 4 bytes for f32 -- this is the
+        // whole point of scalar quantization for large local indices.
+        let values = pseudo_random_floats(99, 1536, -1.0, 1.0);
+        let (quantized, _params) = quantize(&values);
+
+        let f32_bytes = values.len() * std::mem::size_of::<f32>();
+        let i8_bytes = quantized.len() * std::mem::size_of::<i8>();
+
+        assert_eq!(f32_bytes, i8_bytes * 4);
+    }
+
+    #[test]
+    fn test_quantize_batch_round_trip() {
+        let vectors = vec![
+            pseudo_random_floats(1, 256, -1.0, 1.0),
+            pseudo_random_floats(2, 256, 0.0, 1.0),
+            vec![0.0; 256],
+        ];
+        let (quantized, params) = quantize_batch(&vectors);
+        let reconstructed = dequantize_batch(&quantized, &params);
+
+        for (orig, recon) in vectors.iter().zip(reconstructed.iter()) {
+            let err = max_abs_error(orig, recon);
+            let bound = {
+                let min = orig.iter().cloned().fold(f32::INFINITY, f32::min);
+                let max = orig.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
+                (max - min) / 255.0
+            };
+            assert!(
+                err <= bound + 1e-4,
+                "batch round-trip error {err} exceeded bound {bound}"
+            );
+        }
+    }
+}

pyvectorhound-1.4.0/src/lib.rs

@@ -1,91 +0,0 @@
-// PyHound Core - Rust implementation of retrieval diagnostics
-//
-// High-performance embedding quality metrics, pipeline analysis, and diagnostics.
-
-mod metrics;
-mod retrieval_ranking;
-
-use metrics::{
-    compute_coverage, compute_distinctiveness, compute_isotropy, compute_retrieval_metrics,
-    detect_drift,
-};
-use pyo3::prelude::*;
-use pyo3::types::PyDict;
-use pyo3::Bound;
-
-/// Compute isotropy of embeddings (vector space utilization)
-#[pyfunction]
-fn py_compute_isotropy(embeddings: Vec<Vec<f32>>) -> PyResult<f32> {
-    Ok(compute_isotropy(&embeddings))
-}
-
-/// Compute coverage of embeddings (diversity of semantic space)
-#[pyfunction]
-fn py_compute_coverage(embeddings: Vec<Vec<f32>>) -> PyResult<f32> {
-    Ok(compute_coverage(&embeddings))
-}
-
-/// Compute distinctiveness of embeddings (semantic separation)
-#[pyfunction]
-fn py_compute_distinctiveness(embeddings: Vec<Vec<f32>>) -> PyResult<f32> {
-    Ok(compute_distinctiveness(&embeddings))
-}
-
-/// Detect drift in embedding quality
-#[pyfunction]
-fn py_detect_drift(
-    baseline_embeddings: Vec<Vec<f32>>,
-    current_embeddings: Vec<Vec<f32>>,
-) -> PyResult<f32> {
-    Ok(detect_drift(&baseline_embeddings, &current_embeddings))
-}
-
-/// Compute retrieval metrics (precision, recall, F1, MRR, NDCG)
-#[pyfunction]
-fn py_compute_retrieval_metrics(
-    py: Python,
-    retrieved: Vec<usize>,
-    relevant: Vec<usize>,
-    top_k: usize,
-) -> PyResult<PyObject> {
-    let metrics = compute_retrieval_metrics(&retrieved, &relevant, top_k);
-
-    let dict = PyDict::new(py);
-    dict.set_item("precision", metrics.precision)?;
-    dict.set_item("recall", metrics.recall)?;
-    dict.set_item("f1_score", metrics.f1_score)?;
-    dict.set_item("mrr", metrics.mrr)?;
-    dict.set_item("ndcg", metrics.ndcg)?;
-
-    Ok(dict.into())
-}
-
-/// Compute embedding quality score (composite metric)
-#[pyfunction]
-fn py_compute_quality_score(embeddings: Vec<Vec<f32>>) -> PyResult<f32> {
-    if embeddings.is_empty() {
-        return Ok(0.0);
-    }
-
-    let isotropy = compute_isotropy(&embeddings);
-    let coverage = compute_coverage(&embeddings);
-    let distinctiveness = compute_distinctiveness(&embeddings);
-
-    // Composite score: weighted average
-    let quality = (isotropy * 0.4 + coverage * 0.3 + distinctiveness * 0.3).clamp(0.0, 1.0);
-
-    Ok(quality)
-}
-
-/// PyHound Python module
-#[pymodule]
-fn _core(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
-    m.add_function(wrap_pyfunction!(py_compute_isotropy, m)?)?;
-    m.add_function(wrap_pyfunction!(py_compute_coverage, m)?)?;
-    m.add_function(wrap_pyfunction!(py_compute_distinctiveness, m)?)?;
-    m.add_function(wrap_pyfunction!(py_detect_drift, m)?)?;
-    m.add_function(wrap_pyfunction!(py_compute_retrieval_metrics, m)?)?;
-    m.add_function(wrap_pyfunction!(py_compute_quality_score, m)?)?;
-
-    Ok(())
-}