zeusdb-vector-database 0.2.0__tar.gz → 0.2.1__tar.gz

{zeusdb_vector_database-0.2.0 → zeusdb_vector_database-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zeusdb-vector-database
-Version: 0.2.0
+Version: 0.2.1
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Requires-Dist: numpy>=2.2.6,<3.0.0
@@ -594,12 +594,13 @@ To enable PQ, pass a `quantization_config` dictionary to the `.create()` index m
 | `bits` | `int` | Bits per quantized code (controls centroids per subvector) | 1-8 | `8` |
 | `training_size` | `int` | Minimum vectors needed for stable k-means clustering | ≥ 1000 | 1000 |
 | `max_training_vectors` | `int` | Maximum vectors used during training (optional limit) | ≥ training_size | `None` |
+| `storage_mode` | `str` | Storage strategy: "quantized_only" (memory optimized) or "quantized_with_raw" (keep raw vectors for exact reconstruction) | "quantized_only", "quantized_with_raw" | `"quantized_only"` |
 
 
 <br/>
 
 
-### 🔧 Usage Example
+### 🔧 Usage Example 1
 
 ```python
 from zeusdb_vector_database import VectorDatabase
@@ -665,6 +666,36 @@ Results
 {'id': 'doc_8148', 'score': 0.5139288306236267, 'metadata': {'category': 'tech', 'year': 2026}}, 
 {'id': 'doc_7822', 'score': 0.5151920914649963, 'metadata': {'category': 'tech', 'year': 2026}}, 
 ]
+```
+<br />
+
+### 🔧 Usage Example 2 - with explicit storage mode
+
+```python
+from zeusdb_vector_database import VectorDatabase
+import numpy as np
+
+# Create index with product quantization
+vdb = VectorDatabase()
+
+# Configure quantization for memory efficiency
+quantization_config = {
+    'type': 'pq',                  # `pq` for Product Quantization
+    'subvectors': 8,               # Divide 1536-dim vectors into 8 subvectors of 192 dims each
+    'bits': 8,                     # 256 centroids per subvector (2^8)
+    'training_size': 10000,        # Train when 10k vectors are collected
+    'max_training_vectors': 50000,  # Use max 50k vectors for training
+    'storage_mode': 'quantized_only'  # Explicitly set storage mode to only keep quantized values
+}
+
+# Create index with quantization
+# This will automatically handle training when enough vectors are added
+index = vdb.create(
+    index_type="hnsw",
+    dim=3072,                                  # OpenAI `text-embedding-3-large` dimension
+    quantization_config=quantization_config    # Add the compression configuration
+)
+
 ```
 
 <br />
@@ -677,7 +708,8 @@ quantization_config = {
     'type': 'pq',
     'subvectors': 8,      # Balanced: moderate compression, good accuracy
     'bits': 8,            # 256 centroids per subvector (high precision)
-    'training_size': 10000  # Or higher for large datasets
+    'training_size': 10000,  # Or higher for large datasets
+    'storage_mode': 'quantized_only'  # Default, memory efficient
 }
 # Achieves ~16x–32x compression with strong recall for most applications
 ```
@@ -689,7 +721,8 @@ quantization_config = {
     'type': 'pq',
     'subvectors': 16,      # More subvectors = better compression
     'bits': 6,             # Fewer bits = less memory per centroid
-    'training_size': 20000
+    'training_size': 20000,
+    'storage_mode': 'quantized_only'
 }
 # Achieves ~32x compression ratio
 ```
@@ -701,6 +734,7 @@ quantization_config = {
     'subvectors': 4,       # Fewer subvectors = better accuracy
     'bits': 8,             # More bits = more precise quantization
     'training_size': 50000 # More training data = better centroids
+    'storage_mode': 'quantized_with_raw'  # Keep raw vectors for exact recall
 }
 # Achieves ~4x compression ratio with minimal accuracy loss
 ```
@@ -714,6 +748,10 @@ quantization_config = {
 
 Quantization is ideal for production deployments with large vector datasets (100k+ vectors) where memory efficiency is critical.
 
+`"quantized_only"` is recommended for most use cases and maximizes memory savings.
+
+`"quantized_with_raw"` keeps both quantized and raw vectors for exact reconstruction, but uses more memory.
+
 
 <br/>
 

{zeusdb_vector_database-0.2.0 → zeusdb_vector_database-0.2.1}/README.md

@@ -575,12 +575,13 @@ To enable PQ, pass a `quantization_config` dictionary to the `.create()` index m
 | `bits` | `int` | Bits per quantized code (controls centroids per subvector) | 1-8 | `8` |
 | `training_size` | `int` | Minimum vectors needed for stable k-means clustering | ≥ 1000 | 1000 |
 | `max_training_vectors` | `int` | Maximum vectors used during training (optional limit) | ≥ training_size | `None` |
+| `storage_mode` | `str` | Storage strategy: "quantized_only" (memory optimized) or "quantized_with_raw" (keep raw vectors for exact reconstruction) | "quantized_only", "quantized_with_raw" | `"quantized_only"` |
 
 
 <br/>
 
 
-### 🔧 Usage Example
+### 🔧 Usage Example 1
 
 ```python
 from zeusdb_vector_database import VectorDatabase
@@ -646,6 +647,36 @@ Results
 {'id': 'doc_8148', 'score': 0.5139288306236267, 'metadata': {'category': 'tech', 'year': 2026}}, 
 {'id': 'doc_7822', 'score': 0.5151920914649963, 'metadata': {'category': 'tech', 'year': 2026}}, 
 ]
+```
+<br />
+
+### 🔧 Usage Example 2 - with explicit storage mode
+
+```python
+from zeusdb_vector_database import VectorDatabase
+import numpy as np
+
+# Create index with product quantization
+vdb = VectorDatabase()
+
+# Configure quantization for memory efficiency
+quantization_config = {
+    'type': 'pq',                  # `pq` for Product Quantization
+    'subvectors': 8,               # Divide 1536-dim vectors into 8 subvectors of 192 dims each
+    'bits': 8,                     # 256 centroids per subvector (2^8)
+    'training_size': 10000,        # Train when 10k vectors are collected
+    'max_training_vectors': 50000,  # Use max 50k vectors for training
+    'storage_mode': 'quantized_only'  # Explicitly set storage mode to only keep quantized values
+}
+
+# Create index with quantization
+# This will automatically handle training when enough vectors are added
+index = vdb.create(
+    index_type="hnsw",
+    dim=3072,                                  # OpenAI `text-embedding-3-large` dimension
+    quantization_config=quantization_config    # Add the compression configuration
+)
+
 ```
 
 <br />
@@ -658,7 +689,8 @@ quantization_config = {
     'type': 'pq',
     'subvectors': 8,      # Balanced: moderate compression, good accuracy
     'bits': 8,            # 256 centroids per subvector (high precision)
-    'training_size': 10000  # Or higher for large datasets
+    'training_size': 10000,  # Or higher for large datasets
+    'storage_mode': 'quantized_only'  # Default, memory efficient
 }
 # Achieves ~16x–32x compression with strong recall for most applications
 ```
@@ -670,7 +702,8 @@ quantization_config = {
     'type': 'pq',
     'subvectors': 16,      # More subvectors = better compression
     'bits': 6,             # Fewer bits = less memory per centroid
-    'training_size': 20000
+    'training_size': 20000,
+    'storage_mode': 'quantized_only'
 }
 # Achieves ~32x compression ratio
 ```
@@ -682,6 +715,7 @@ quantization_config = {
     'subvectors': 4,       # Fewer subvectors = better accuracy
     'bits': 8,             # More bits = more precise quantization
     'training_size': 50000 # More training data = better centroids
+    'storage_mode': 'quantized_with_raw'  # Keep raw vectors for exact recall
 }
 # Achieves ~4x compression ratio with minimal accuracy loss
 ```
@@ -695,6 +729,10 @@ quantization_config = {
 
 Quantization is ideal for production deployments with large vector datasets (100k+ vectors) where memory efficiency is critical.
 
+`"quantized_only"` is recommended for most use cases and maximizes memory savings.
+
+`"quantized_with_raw"` keeps both quantized and raw vectors for exact reconstruction, but uses more memory.
+
 
 <br/>
 

{zeusdb_vector_database-0.2.0 → zeusdb_vector_database-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "zeusdb-vector-database"
-version = "0.2.0"
+version = "0.2.1"
 description = "Blazing-fast vector DB with real-time similarity search and metadata filtering."
 readme = "README.md"
 authors = [

{zeusdb_vector_database-0.2.0 → zeusdb_vector_database-0.2.1}/src/zeusdb_vector_database/__init__.py

@@ -1,7 +1,7 @@
 """
 ZeusDB Vector Database Module
 """
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 from .vector_database import VectorDatabase # imports the VectorDatabase class from the vector_database.py file
 

{zeusdb_vector_database-0.2.0 → zeusdb_vector_database-0.2.1}/src/zeusdb_vector_database/vector_database.py

@@ -56,7 +56,8 @@ class VectorDatabase:
                     'subvectors': 8,           # Number of subvectors (must divide dim evenly, default: 8)
                     'bits': 8,                 # Bits per subvector (1-8, controls centroids, default: 8)
                     'training_size': None,     # Auto-calculated based on subvectors & bits (or specify manually)
-                    'max_training_vectors': None  # Optional limit on training vectors used
+                    'max_training_vectors': None,  # Optional limit on training vectors used
+                    'storage_mode': 'quantized_only' # Storage mode for quantized vectors (or 'quantized_with_raw')  
                 }
 
             Note: Quantization reduces memory usage (typically 4-32x compression) but may 
@@ -88,7 +89,8 @@ class VectorDatabase:
                 'type': 'pq',
                 'subvectors': 16,         # More subvectors = better compression
                 'bits': 6,                # Fewer bits = less memory per centroid
-                'training_size': 75000    # Override auto-calculation
+                'training_size': 75000,    # Override auto-calculation
+                'storage_mode': 'quantized_only'  # Only store quantized vectors
             }
             index = vdb.create(
                 index_type="hnsw",
@@ -126,11 +128,12 @@ class VectorDatabase:
         
         try:
             # Always pass quantization_config parameter
-            clean_config = None
             if quantization_config is not None:
-                # Clean quantization_config before passing to Rust (remove internal keys)
-                clean_config = {k: v for k, v in quantization_config.items() if not k.startswith('_')}
-            
+                # Remove keys with None values and internal keys
+                clean_config = {k: v for k, v in quantization_config.items() if not k.startswith('_') and v is not None}
+            else:
+                clean_config = None
+
             return constructor(quantization_config=clean_config, **kwargs)
         except Exception as e:
             raise RuntimeError(f"Failed to create {index_type.upper()} index: {e}") from e
@@ -172,7 +175,7 @@ class VectorDatabase:
         if dim % subvectors != 0:
             raise ValueError(
                 f"subvectors ({subvectors}) must divide dimension ({dim}) evenly. "
-                f"Consider using subvectors: {self._suggest_subvector_divisors(dim)}"
+                f"Consider using subvectors: {', '.join(map(str, self._suggest_subvector_divisors(dim)))}"
             )
         
         if subvectors > dim:
@@ -206,9 +209,38 @@ class VectorDatabase:
                 )
             validated_config['max_training_vectors'] = max_training_vectors
         
+        # Validate storage mode
+        storage_mode = str(validated_config.get('storage_mode', 'quantized_only')).lower()
+        valid_modes = {'quantized_only', 'quantized_with_raw'}
+        if storage_mode not in valid_modes:
+            raise ValueError(
+                f"Invalid storage_mode: '{storage_mode}'. Supported modes: {', '.join(sorted(valid_modes))}"
+            )
+        
+        validated_config['storage_mode'] = storage_mode
+
         # Calculate and warn about memory usage
         self._check_memory_usage(validated_config, dim)
+
+        # Add helpful warnings about storage mode
+        if storage_mode == 'quantized_with_raw':
+            import warnings
+            compression_ratio = validated_config.get('__memory_info__', {}).get('compression_ratio', 1.0)
+            warnings.warn(
+                f"storage_mode='quantized_with_raw' will use ~{compression_ratio:.1f}x more memory "
+                f"than 'quantized_only' but enables exact vector reconstruction.",
+                UserWarning,
+                stacklevel=2
+            )
         
+        # Final safety check: ensure all expected keys are present
+        # This is a final defensive programming - all the keys should already be set above, but added just in case
+        validated_config.setdefault('type', 'pq')
+        validated_config.setdefault('subvectors', 8)
+        validated_config.setdefault('bits', 8)
+        validated_config.setdefault('max_training_vectors', None)
+        validated_config.setdefault('storage_mode', 'quantized_only')
+
         return validated_config
 
     def _calculate_smart_training_size(self, subvectors: int, bits: int) -> int:
@@ -236,13 +268,14 @@ class VectorDatabase:
         
         return min(max(statistical_minimum, reasonable_minimum), reasonable_maximum)
 
-    def _suggest_subvector_divisors(self, dim: int) -> str:
-        """Suggest valid subvector counts that divide the dimension evenly."""
-        divisors = []
-        for i in range(1, min(33, dim + 1)):  # Common subvector counts up to 32
-            if dim % i == 0:
-                divisors.append(str(i))
-        return ', '.join(divisors[:8])  # Show first 8 suggestions
+    
+    def _suggest_subvector_divisors(self, dim: int) -> list[int]:
+        """Return valid subvector counts that divide the dimension evenly (up to 32)."""
+        return [i for i in range(1, min(33, dim + 1)) if dim % i == 0]
+    
+
+
+
 
     def _check_memory_usage(self, config: Dict[str, Any], dim: int) -> None:
         """

{zeusdb_vector_database-0.2.0 → zeusdb_vector_database-0.2.1}/vdb-core/Cargo.lock

@@ -105,6 +105,26 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "bincode"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
+dependencies = [
+ "bincode_derive",
+ "serde",
+ "unty",
+]
+
+[[package]]
+name = "bincode_derive"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
+dependencies = [
+ "virtue",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -282,7 +302,7 @@ checksum = "b53dc5b9b07424143d016ba843c9b510f424e239118697f5d5d582f2d437df41"
 dependencies = [
  "anndists",
  "anyhow",
- "bincode",
+ "bincode 1.3.3",
  "cfg-if",
  "cpu-time",
  "env_logger",
@@ -728,9 +748,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.16"
+version = "0.5.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7251471db004e509f4e75a62cca9435365b5ec7bcdff530d612ac7c87c44a792"
+checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77"
 dependencies = [
  "bitflags 2.9.1",
 ]
@@ -892,12 +912,24 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
 
+[[package]]
+name = "unty"
+version = "0.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
+
 [[package]]
 name = "utf8parse"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
+[[package]]
+name = "virtue"
+version = "0.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
+
 [[package]]
 name = "walkdir"
 version = "2.5.0"
@@ -1124,8 +1156,9 @@ dependencies = [
 
 [[package]]
 name = "zeusdb-vector-database"
-version = "0.2.0"
+version = "0.2.1"
 dependencies = [
+ "bincode 2.0.1",
  "hnsw_rs",
  "numpy",
  "pyo3",

{zeusdb_vector_database-0.2.0 → zeusdb_vector_database-0.2.1}/vdb-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "zeusdb-vector-database"
-version = "0.2.0"
+version = "0.2.1"
 edition = "2021"
 resolver = "2" # <-- Avoid compiling unnecessary features from dependencies.
 
@@ -17,6 +17,7 @@ serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 rayon = "1.10"
 rand = "0.9.1"
+bincode = "2.0.1"
 
 [profile.release]
 lto = true # <-- Enable Link-Time Optimization

{zeusdb_vector_database-0.2.0 → zeusdb_vector_database-0.2.1}/vdb-core/src/hnsw_index.rs

@@ -6,6 +6,7 @@ use std::sync::{Mutex, RwLock, Arc};
 use hnsw_rs::prelude::{Hnsw, DistCosine, DistL2, DistL1, Distance};
 use serde_json::Value;
 use rayon::prelude::*;
+use serde::{Serialize, Deserialize};
 
 // Import PQ module
 use crate::pq::PQ;
@@ -24,15 +25,54 @@ macro_rules! debug_log {
 }
 
 
-// Quantization configuration structure
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum StorageMode {
+    #[serde(rename = "quantized_only")]
+    QuantizedOnly,
+
+    #[serde(rename = "quantized_with_raw")]
+    QuantizedWithRaw,
+}
+
+impl StorageMode {
+    pub fn from_string(s: &str) -> Result<Self, String> {
+        match s {
+            "quantized_only" => Ok(StorageMode::QuantizedOnly),
+            "quantized_with_raw" => Ok(StorageMode::QuantizedWithRaw),
+            _ => Err(format!(
+                "Invalid storage_mode: '{}'. Supported: quantized_only, quantized_with_raw",
+                s
+            ))
+        }
+    }
+
+    pub fn to_string(&self) -> &'static str {
+        match self {
+            StorageMode::QuantizedOnly => "quantized_only",
+            StorageMode::QuantizedWithRaw => "quantized_with_raw",
+        }
+    }
+}
+
+impl Default for StorageMode {
+    fn default() -> Self {
+        StorageMode::QuantizedOnly
+    }
+}
+
+
+// Updated QuantizationConfig structure
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct QuantizationConfig {
     pub subvectors: usize,
     pub bits: usize,
     pub training_size: usize,
     pub max_training_vectors: Option<usize>,
+    pub storage_mode: StorageMode,
 }
 
+
+
 /// Custom distance function for Product Quantization using ADC
 #[derive(Clone)]
 pub struct DistPQ {
@@ -435,6 +475,16 @@ impl HNSWIndex {
             let max_training_vectors = config.get_item("max_training_vectors")?
                 .map(|v| v.extract::<usize>())
                 .transpose()?;
+
+            // Extract storage_mode
+            let storage_mode_str = config.get_item("storage_mode")?
+                .map(|v| v.extract::<String>())
+                .transpose()?
+                .unwrap_or_else(|| "quantized_only".to_string());
+
+            let storage_mode = StorageMode::from_string(&storage_mode_str)
+                .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e))?;
+
             
             // Validate PQ parameters
             if dim % subvectors != 0 {
@@ -460,6 +510,7 @@ impl HNSWIndex {
                 bits,
                 training_size,
                 max_training_vectors,
+                storage_mode,
             };
             
             // Create PQ instance
@@ -941,7 +992,57 @@ impl HNSWIndex {
 
 
 
-    /// Get records by ID(s) with PQ reconstruction support
+    // /// Get records by ID(s) with PQ reconstruction support
+    // #[pyo3(signature = (input, return_vector = true))]
+    // pub fn get_records(&self, py: Python<'_>, input: &Bound<PyAny>, return_vector: bool) -> PyResult<Vec<Py<PyDict>>> {
+    //     let ids: Vec<String> = if let Ok(id_str) = input.extract::<String>() {
+    //         vec![id_str]
+    //     } else if let Ok(id_list) = input.extract::<Vec<String>>() {
+    //         id_list
+    //     } else {
+    //         return Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(
+    //             "Expected a string or a list of strings for ID(s)",
+    //         ));
+    //     };
+
+    //     let mut records = Vec::with_capacity(ids.len());
+        
+    //     // Use read locks for concurrent access
+    //     let vectors = self.vectors.read().unwrap();
+    //     let pq_codes = self.pq_codes.read().unwrap();
+    //     let vector_metadata = self.vector_metadata.read().unwrap();
+
+    //     for id in ids {
+    //         if let Some(vector) = vectors.get(&id) {
+    //             let metadata = vector_metadata.get(&id).cloned().unwrap_or_default();
+
+    //             let dict = PyDict::new(py);
+    //             dict.set_item("id", id.clone())?;
+    //             dict.set_item("metadata", self.value_map_to_python(&metadata, py)?)?;
+
+    //             if return_vector {
+    //                 // Try raw vector first, then PQ reconstruction
+    //                 let vector_data = if !vector.is_empty() {
+    //                     vector.clone()
+    //                 } else if let (Some(pq), Some(codes)) = (&self.pq, pq_codes.get(&id)) {
+    //                     pq.reconstruct(codes).unwrap_or_else(|_| vector.clone())
+    //                 } else {
+    //                     vector.clone()
+    //                 };
+                    
+    //                 dict.set_item("vector", vector_data)?;
+    //             }
+
+    //             records.push(dict.into());
+    //         }
+    //     }
+
+    //     Ok(records)
+    // }
+
+
+
+    /// Get records by ID(s) with PQ reconstruction support and storage mode awareness
     #[pyo3(signature = (input, return_vector = true))]
     pub fn get_records(&self, py: Python<'_>, input: &Bound<PyAny>, return_vector: bool) -> PyResult<Vec<Py<PyDict>>> {
         let ids: Vec<String> = if let Ok(id_str) = input.extract::<String>() {
@@ -955,14 +1056,17 @@ impl HNSWIndex {
         };
 
         let mut records = Vec::with_capacity(ids.len());
-        
+
         // Use read locks for concurrent access
         let vectors = self.vectors.read().unwrap();
         let pq_codes = self.pq_codes.read().unwrap();
         let vector_metadata = self.vector_metadata.read().unwrap();
 
         for id in ids {
-            if let Some(vector) = vectors.get(&id) {
+            // Check if this ID exists in either storage
+            let exists = vectors.contains_key(&id) || pq_codes.contains_key(&id);
+
+            if exists {
                 let metadata = vector_metadata.get(&id).cloned().unwrap_or_default();
 
                 let dict = PyDict::new(py);
@@ -970,16 +1074,27 @@ impl HNSWIndex {
                 dict.set_item("metadata", self.value_map_to_python(&metadata, py)?)?;
 
                 if return_vector {
-                    // Try raw vector first, then PQ reconstruction
-                    let vector_data = if !vector.is_empty() {
-                        vector.clone()
+                    // Priority: raw vector > PQ reconstruction
+                    let vector_data = if let Some(raw_vector) = vectors.get(&id) {
+                        // Case 1: Raw vector available (QuantizedWithRaw mode or non-quantized)
+                        Some(raw_vector.clone())
                     } else if let (Some(pq), Some(codes)) = (&self.pq, pq_codes.get(&id)) {
-                        pq.reconstruct(codes).unwrap_or_else(|_| vector.clone())
+                        // Case 2: Only quantized codes available (QuantizedOnly mode)
+                        match pq.reconstruct(codes) {
+                            Ok(reconstructed) => Some(reconstructed),
+                            Err(e) => {
+                                eprintln!("Warning: Failed to reconstruct vector for ID {}: {}", id, e);
+                                None
+                            }
+                        }
                     } else {
-                        vector.clone()
+                        // Case 3: No vector data available
+                        None
                     };
-                    
-                    dict.set_item("vector", vector_data)?;
+
+                    if let Some(vec) = vector_data {
+                        dict.set_item("vector", vec)?;
+                    }
                 }
 
                 records.push(dict.into());
@@ -992,7 +1107,75 @@ impl HNSWIndex {
 
 
 
-    /// Enhanced get_stats with training info
+
+
+
+
+
+
+
+
+    // /// Enhanced get_stats with training info
+    // pub fn get_stats(&self) -> HashMap<String, String> {
+    //     let mut stats = HashMap::new();
+
+    //     let vectors = self.vectors.read().unwrap();
+    //     let pq_codes = self.pq_codes.read().unwrap();
+    //     let vector_count = *self.vector_count.lock().unwrap();
+    //     let training_ids = self.training_ids.read().unwrap();
+
+    //     // Basic stats
+    //     stats.insert("total_vectors".to_string(), vector_count.to_string());
+    //     stats.insert("dimension".to_string(), self.dim.to_string());
+    //     stats.insert("expected_size".to_string(), self.expected_size.to_string());
+    //     stats.insert("space".to_string(), self.space.clone());
+    //     stats.insert("index_type".to_string(), "HNSW".to_string());
+
+    //     stats.insert("m".to_string(), self.m.to_string());
+    //     stats.insert("ef_construction".to_string(), self.ef_construction.to_string());
+    //     stats.insert("thread_safety".to_string(), "RwLock+Mutex".to_string());
+
+    //     // Storage breakdown
+    //     stats.insert("raw_vectors_stored".to_string(), vectors.len().to_string());
+    //     stats.insert("quantized_codes_stored".to_string(), pq_codes.len().to_string());
+
+    //     // Training info
+    //     if let Some(config) = &self.quantization_config {
+    //         stats.insert("quantization_type".to_string(), "pq".to_string());
+    //         stats.insert("quantization_training_size".to_string(), config.training_size.to_string());
+
+    //         let collected_count = training_ids.len();
+    //         let progress = self.get_training_progress();
+    //         stats.insert("training_progress".to_string(), 
+    //             format!("{}/{} ({:.1}%)", collected_count, config.training_size, progress));
+            
+    //         let vectors_needed = self.training_vectors_needed();
+    //         stats.insert("training_vectors_needed".to_string(), vectors_needed.to_string());
+    //         stats.insert("training_threshold_reached".to_string(), 
+    //             self.training_threshold_reached.load(Ordering::Acquire).to_string());
+            
+    //         if let Some(pq) = &self.pq {
+    //             let is_trained = pq.is_trained();
+    //             stats.insert("quantization_trained".to_string(), is_trained.to_string());
+    //             stats.insert("quantization_active".to_string(), self.is_quantized().to_string());
+
+    //             if is_trained {
+    //                 let compression_ratio = (pq.dim * 4) as f64 / pq.subvectors as f64;
+    //                 stats.insert("quantization_compression_ratio".to_string(), format!("{:.1}x", compression_ratio));
+    //             }
+    //         }
+    //     } else {
+    //         stats.insert("quantization_type".to_string(), "none".to_string());
+    //     }
+
+    //     stats.insert("storage_mode".to_string(), self.get_storage_mode());
+
+    //     stats
+    // }
+
+
+
+    /// Enhanced get_stats with storage mode information
     pub fn get_stats(&self) -> HashMap<String, String> {
         let mut stats = HashMap::new();
 
@@ -1021,16 +1204,37 @@ impl HNSWIndex {
             stats.insert("quantization_type".to_string(), "pq".to_string());
             stats.insert("quantization_training_size".to_string(), config.training_size.to_string());
 
+            // Storage mode information
+            stats.insert("storage_mode".to_string(), config.storage_mode.to_string().to_string());
+
+            // Calculate actual memory usage based on storage mode
+            let raw_memory_mb = (vectors.len() * self.dim * 4) as f64 / (1024.0 * 1024.0);
+            let quantized_memory_mb = (pq_codes.len() * config.subvectors) as f64 / (1024.0 * 1024.0);
+
+            stats.insert("raw_vectors_memory_mb".to_string(), format!("{:.2}", raw_memory_mb));
+            stats.insert("quantized_codes_memory_mb".to_string(), format!("{:.2}", quantized_memory_mb));
+
+            match config.storage_mode {
+                StorageMode::QuantizedOnly => {
+                    stats.insert("storage_strategy".to_string(), "memory_optimized".to_string());
+                    stats.insert("memory_savings".to_string(), "maximum".to_string());
+                }
+                StorageMode::QuantizedWithRaw => {
+                    stats.insert("storage_strategy".to_string(), "quality_optimized".to_string());
+                    stats.insert("memory_savings".to_string(), "raw_vectors_kept".to_string());
+                }
+            }
+
             let collected_count = training_ids.len();
             let progress = self.get_training_progress();
-            stats.insert("training_progress".to_string(), 
+            stats.insert("training_progress".to_string(),
                 format!("{}/{} ({:.1}%)", collected_count, config.training_size, progress));
             
             let vectors_needed = self.training_vectors_needed();
             stats.insert("training_vectors_needed".to_string(), vectors_needed.to_string());
-            stats.insert("training_threshold_reached".to_string(), 
+            stats.insert("training_threshold_reached".to_string(),
                 self.training_threshold_reached.load(Ordering::Acquire).to_string());
-            
+
             if let Some(pq) = &self.pq {
                 let is_trained = pq.is_trained();
                 stats.insert("quantization_trained".to_string(), is_trained.to_string());
@@ -1043,12 +1247,68 @@ impl HNSWIndex {
             }
         } else {
             stats.insert("quantization_type".to_string(), "none".to_string());
+            stats.insert("storage_mode".to_string(), "raw_only".to_string());
         }
 
-        stats.insert("storage_mode".to_string(), self.get_storage_mode());
+        stats.insert("storage_mode_description".to_string(), self.get_storage_mode());
 
         stats
     }
+            
+
+        
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -1413,7 +1673,61 @@ impl HNSWIndex {
         Ok(())
     }
 
-    /// Path C: Quantized storage (trained and active)
+    // /// Path C: Quantized storage (trained and active)
+    // fn add_quantized_vector(
+    //     &mut self,
+    //     id: String,
+    //     vector: Vec<f32>,  // Already processed
+    //     metadata: HashMap<String, Value>
+    // ) -> PyResult<()> {
+    //     let internal_id = self.get_next_id();
+
+    //     // Store metadata
+    //     {
+    //         let mut vector_metadata = self.vector_metadata.write().unwrap();
+    //         vector_metadata.insert(id.clone(), metadata);
+    //     }
+
+    //     // Update ID mappings
+    //     {
+    //         let mut id_map = self.id_map.write().unwrap();
+    //         let mut rev_map = self.rev_map.write().unwrap();
+
+    //         id_map.insert(id.clone(), internal_id);
+    //         rev_map.insert(internal_id, id.clone());
+    //     }
+
+    //     // Quantize the vector
+    //     let pq = self.pq.as_ref().unwrap();
+    //     let codes = pq.quantize(&vector).map_err(|e| {
+    //         PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(
+    //             format!("Failed to quantize vector: {}", e)
+    //         )
+    //     })?;
+
+    //     // Store quantized codes
+    //     {
+    //         let mut pq_codes = self.pq_codes.write().unwrap();
+    //         pq_codes.insert(id.clone(), codes.clone());
+    //     }
+
+    //     // Store raw vector for exact reconstruction (persistence-ready)
+    //     {
+    //         let mut vectors = self.vectors.write().unwrap();
+    //         vectors.insert(id, vector.clone());
+    //     }
+
+    //     // Insert codes into quantized HNSW
+    //     {
+    //         let mut hnsw_guard = self.hnsw.lock().unwrap();
+    //         hnsw_guard.insert_pq_codes(&codes, internal_id);
+    //     }
+
+    //     Ok(())
+    // }
+
+
+    /// Path C: Quantized storage with configurable raw vector retention
     fn add_quantized_vector(
         &mut self,
         id: String,
@@ -1445,16 +1759,19 @@ impl HNSWIndex {
             )
         })?;
 
-        // Store quantized codes
+        // Store quantized codes (always)
         {
             let mut pq_codes = self.pq_codes.write().unwrap();
             pq_codes.insert(id.clone(), codes.clone());
         }
 
-        // Store raw vector for exact reconstruction (persistence-ready)
-        {
-            let mut vectors = self.vectors.write().unwrap();
-            vectors.insert(id, vector.clone());
+        // Store raw vector only if configured to keep them
+        if let Some(config) = &self.quantization_config {
+            if config.storage_mode == StorageMode::QuantizedWithRaw {
+                let mut vectors = self.vectors.write().unwrap();
+                vectors.insert(id.clone(), vector.clone());
+            }
+            // If QuantizedOnly mode, we don't store raw vectors (saves memory)
         }
 
         // Insert codes into quantized HNSW
@@ -1466,6 +1783,41 @@ impl HNSWIndex {
         Ok(())
     }
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
     /// TRAINING TRIGGER: Uses threshold flag for race condition safety
     fn maybe_trigger_training(&mut self) -> Result<(), String> {
         // Check atomic flag first (fast path)