zeusdb-vector-database 0.2.0__cp310-cp310-musllinux_1_2_armv7l.whl → 0.2.1__cp310-cp310-musllinux_1_2_armv7l.whl

zeusdb_vector_database/__init__.py

@@ -1,7 +1,7 @@
 """
 ZeusDB Vector Database Module
 """
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 from .vector_database import VectorDatabase # imports the VectorDatabase class from the vector_database.py file
 

zeusdb_vector_database/vector_database.py

@@ -56,7 +56,8 @@ class VectorDatabase:
                     'subvectors': 8,           # Number of subvectors (must divide dim evenly, default: 8)
                     'bits': 8,                 # Bits per subvector (1-8, controls centroids, default: 8)
                     'training_size': None,     # Auto-calculated based on subvectors & bits (or specify manually)
-                    'max_training_vectors': None  # Optional limit on training vectors used
+                    'max_training_vectors': None,  # Optional limit on training vectors used
+                    'storage_mode': 'quantized_only' # Storage mode for quantized vectors (or 'quantized_with_raw')  
                 }
 
             Note: Quantization reduces memory usage (typically 4-32x compression) but may 
@@ -88,7 +89,8 @@ class VectorDatabase:
                 'type': 'pq',
                 'subvectors': 16,         # More subvectors = better compression
                 'bits': 6,                # Fewer bits = less memory per centroid
-                'training_size': 75000    # Override auto-calculation
+                'training_size': 75000,    # Override auto-calculation
+                'storage_mode': 'quantized_only'  # Only store quantized vectors
             }
             index = vdb.create(
                 index_type="hnsw",
@@ -126,11 +128,12 @@ class VectorDatabase:
         
         try:
             # Always pass quantization_config parameter
-            clean_config = None
             if quantization_config is not None:
-                # Clean quantization_config before passing to Rust (remove internal keys)
-                clean_config = {k: v for k, v in quantization_config.items() if not k.startswith('_')}
-            
+                # Remove keys with None values and internal keys
+                clean_config = {k: v for k, v in quantization_config.items() if not k.startswith('_') and v is not None}
+            else:
+                clean_config = None
+
             return constructor(quantization_config=clean_config, **kwargs)
         except Exception as e:
             raise RuntimeError(f"Failed to create {index_type.upper()} index: {e}") from e
@@ -172,7 +175,7 @@ class VectorDatabase:
         if dim % subvectors != 0:
             raise ValueError(
                 f"subvectors ({subvectors}) must divide dimension ({dim}) evenly. "
-                f"Consider using subvectors: {self._suggest_subvector_divisors(dim)}"
+                f"Consider using subvectors: {', '.join(map(str, self._suggest_subvector_divisors(dim)))}"
             )
         
         if subvectors > dim:
@@ -206,9 +209,38 @@ class VectorDatabase:
                 )
             validated_config['max_training_vectors'] = max_training_vectors
         
+        # Validate storage mode
+        storage_mode = str(validated_config.get('storage_mode', 'quantized_only')).lower()
+        valid_modes = {'quantized_only', 'quantized_with_raw'}
+        if storage_mode not in valid_modes:
+            raise ValueError(
+                f"Invalid storage_mode: '{storage_mode}'. Supported modes: {', '.join(sorted(valid_modes))}"
+            )
+        
+        validated_config['storage_mode'] = storage_mode
+
         # Calculate and warn about memory usage
         self._check_memory_usage(validated_config, dim)
+
+        # Add helpful warnings about storage mode
+        if storage_mode == 'quantized_with_raw':
+            import warnings
+            compression_ratio = validated_config.get('__memory_info__', {}).get('compression_ratio', 1.0)
+            warnings.warn(
+                f"storage_mode='quantized_with_raw' will use ~{compression_ratio:.1f}x more memory "
+                f"than 'quantized_only' but enables exact vector reconstruction.",
+                UserWarning,
+                stacklevel=2
+            )
         
+        # Final safety check: ensure all expected keys are present
+        # This is a final defensive programming - all the keys should already be set above, but added just in case
+        validated_config.setdefault('type', 'pq')
+        validated_config.setdefault('subvectors', 8)
+        validated_config.setdefault('bits', 8)
+        validated_config.setdefault('max_training_vectors', None)
+        validated_config.setdefault('storage_mode', 'quantized_only')
+
         return validated_config
 
     def _calculate_smart_training_size(self, subvectors: int, bits: int) -> int:
@@ -236,13 +268,14 @@ class VectorDatabase:
         
         return min(max(statistical_minimum, reasonable_minimum), reasonable_maximum)
 
-    def _suggest_subvector_divisors(self, dim: int) -> str:
-        """Suggest valid subvector counts that divide the dimension evenly."""
-        divisors = []
-        for i in range(1, min(33, dim + 1)):  # Common subvector counts up to 32
-            if dim % i == 0:
-                divisors.append(str(i))
-        return ', '.join(divisors[:8])  # Show first 8 suggestions
+    
+    def _suggest_subvector_divisors(self, dim: int) -> list[int]:
+        """Return valid subvector counts that divide the dimension evenly (up to 32)."""
+        return [i for i in range(1, min(33, dim + 1)) if dim % i == 0]
+    
+
+
+
 
     def _check_memory_usage(self, config: Dict[str, Any], dim: int) -> None:
         """

{zeusdb_vector_database-0.2.0.dist-info → zeusdb_vector_database-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zeusdb-vector-database
-Version: 0.2.0
+Version: 0.2.1
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Requires-Dist: numpy>=2.2.6,<3.0.0
@@ -594,12 +594,13 @@ To enable PQ, pass a `quantization_config` dictionary to the `.create()` index m
 | `bits` | `int` | Bits per quantized code (controls centroids per subvector) | 1-8 | `8` |
 | `training_size` | `int` | Minimum vectors needed for stable k-means clustering | ≥ 1000 | 1000 |
 | `max_training_vectors` | `int` | Maximum vectors used during training (optional limit) | ≥ training_size | `None` |
+| `storage_mode` | `str` | Storage strategy: "quantized_only" (memory optimized) or "quantized_with_raw" (keep raw vectors for exact reconstruction) | "quantized_only", "quantized_with_raw" | `"quantized_only"` |
 
 
 <br/>
 
 
-### 🔧 Usage Example
+### 🔧 Usage Example 1
 
 ```python
 from zeusdb_vector_database import VectorDatabase
@@ -665,6 +666,36 @@ Results
 {'id': 'doc_8148', 'score': 0.5139288306236267, 'metadata': {'category': 'tech', 'year': 2026}}, 
 {'id': 'doc_7822', 'score': 0.5151920914649963, 'metadata': {'category': 'tech', 'year': 2026}}, 
 ]
+```
+<br />
+
+### 🔧 Usage Example 2 - with explicit storage mode
+
+```python
+from zeusdb_vector_database import VectorDatabase
+import numpy as np
+
+# Create index with product quantization
+vdb = VectorDatabase()
+
+# Configure quantization for memory efficiency
+quantization_config = {
+    'type': 'pq',                  # `pq` for Product Quantization
+    'subvectors': 8,               # Divide 1536-dim vectors into 8 subvectors of 192 dims each
+    'bits': 8,                     # 256 centroids per subvector (2^8)
+    'training_size': 10000,        # Train when 10k vectors are collected
+    'max_training_vectors': 50000,  # Use max 50k vectors for training
+    'storage_mode': 'quantized_only'  # Explicitly set storage mode to only keep quantized values
+}
+
+# Create index with quantization
+# This will automatically handle training when enough vectors are added
+index = vdb.create(
+    index_type="hnsw",
+    dim=3072,                                  # OpenAI `text-embedding-3-large` dimension
+    quantization_config=quantization_config    # Add the compression configuration
+)
+
 ```
 
 <br />
@@ -677,7 +708,8 @@ quantization_config = {
     'type': 'pq',
     'subvectors': 8,      # Balanced: moderate compression, good accuracy
     'bits': 8,            # 256 centroids per subvector (high precision)
-    'training_size': 10000  # Or higher for large datasets
+    'training_size': 10000,  # Or higher for large datasets
+    'storage_mode': 'quantized_only'  # Default, memory efficient
 }
 # Achieves ~16x–32x compression with strong recall for most applications
 ```
@@ -689,7 +721,8 @@ quantization_config = {
     'type': 'pq',
     'subvectors': 16,      # More subvectors = better compression
     'bits': 6,             # Fewer bits = less memory per centroid
-    'training_size': 20000
+    'training_size': 20000,
+    'storage_mode': 'quantized_only'
 }
 # Achieves ~32x compression ratio
 ```
@@ -701,6 +734,7 @@ quantization_config = {
     'subvectors': 4,       # Fewer subvectors = better accuracy
     'bits': 8,             # More bits = more precise quantization
     'training_size': 50000 # More training data = better centroids
+    'storage_mode': 'quantized_with_raw'  # Keep raw vectors for exact recall
 }
 # Achieves ~4x compression ratio with minimal accuracy loss
 ```
@@ -714,6 +748,10 @@ quantization_config = {
 
 Quantization is ideal for production deployments with large vector datasets (100k+ vectors) where memory efficiency is critical.
 
+`"quantized_only"` is recommended for most use cases and maximizes memory savings.
+
+`"quantized_with_raw"` keeps both quantized and raw vectors for exact reconstruction, but uses more memory.
+
 
 <br/>
 

zeusdb_vector_database-0.2.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+zeusdb_vector_database-0.2.1.dist-info/METADATA,sha256=OmbOu55sPIUBfAUtO2Wz1UksOrOQPw7MhFvhH1IO4VE,31415
+zeusdb_vector_database-0.2.1.dist-info/WHEEL,sha256=7AJc9MU64jshfU-eRqQ46d3BaJaQ19p_pWwYMfVfR1o,107
+zeusdb_vector_database-0.2.1.dist-info/licenses/LICENSE,sha256=82Hi3E_KqpDOBk00HrY6fGiErqL3QJquGQ6dUu9wJzE,11336
+zeusdb_vector_database-0.2.1.dist-info/licenses/NOTICE,sha256=GDGZ9V3p4Uvaj-1RT9Pbeczps-rSeZz8q8wSxb_Q13o,971
+zeusdb_vector_database.libs/libgcc_s-5b5488a6.so.1,sha256=HGKUsVmTeNAxEdSy7Ua5Vh_I9FN3RCbPWzvZ7H_TrwE,2749061
+zeusdb_vector_database/__init__.py,sha256=ywLk8n8oB_zUCZCuFZAfAIBc7vh3sQ3EJA1YEo7oVQw,202
+zeusdb_vector_database/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+zeusdb_vector_database/vector_database.py,sha256=x7Log7dtdsldHCc9ogRTzi7reEqJ1e-5HG1ABo1Py14,15102
+zeusdb_vector_database/zeusdb_vector_database.cpython-310-arm-linux-gnueabihf.so,sha256=IYwKWxxm7SdiWUdvxBuloaD_gp_Rh3ZMSvZc-CX2O6A,4563125
+zeusdb_vector_database-0.2.1.dist-info/RECORD,,

zeusdb_vector_database-0.2.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-zeusdb_vector_database-0.2.0.dist-info/METADATA,sha256=lWikG9QZDEEMRYY1zh45XBiTL7A8Mh4FTJ_cVN7TNfE,29736
-zeusdb_vector_database-0.2.0.dist-info/WHEEL,sha256=7AJc9MU64jshfU-eRqQ46d3BaJaQ19p_pWwYMfVfR1o,107
-zeusdb_vector_database-0.2.0.dist-info/licenses/LICENSE,sha256=82Hi3E_KqpDOBk00HrY6fGiErqL3QJquGQ6dUu9wJzE,11336
-zeusdb_vector_database-0.2.0.dist-info/licenses/NOTICE,sha256=GDGZ9V3p4Uvaj-1RT9Pbeczps-rSeZz8q8wSxb_Q13o,971
-zeusdb_vector_database.libs/libgcc_s-5b5488a6.so.1,sha256=HGKUsVmTeNAxEdSy7Ua5Vh_I9FN3RCbPWzvZ7H_TrwE,2749061
-zeusdb_vector_database/__init__.py,sha256=w2XfHqGQVShv1zDZfvLY-sSw4zzV1kdAFLjSCdGWlkU,202
-zeusdb_vector_database/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-zeusdb_vector_database/vector_database.py,sha256=n1fiDvzpyyvLhOOaLfQSge9p4uhm8YWjX6i8m-rYoRI,13581
-zeusdb_vector_database/zeusdb_vector_database.cpython-310-arm-linux-gnueabihf.so,sha256=PAzVo6Tlg7z3WbtXbXQBOOeTj2lzO8BBuPsJwt5bq_E,4559029
-zeusdb_vector_database-0.2.0.dist-info/RECORD,,