nidx-binding 6.6.1.post456__tar.gz → 6.7.0.post460__tar.gz

{nidx_binding-6.6.1.post456 → nidx_binding-6.7.0.post460}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.6.1.post456
+Version: 6.7.0.post460
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.6.1.post456 → nidx_binding-6.7.0.post460}/nidx_protos/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.6.1.post456"
+version = "6.7.0.post460"
 license = "AGPL-3.0-or-later"
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [{ name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.6.1.post456 → nidx_binding-6.7.0.post460}/nidx_vector/src/config.rs

@@ -27,7 +27,8 @@ use crate::VectorErr;
 use crate::vector_types::*;
 
 pub mod flags {
-    pub const DATA_STORE_V2: &str = "data_store_v2";
+    // pub const DATA_STORE_V2: &str = "data_store_v2";
+    pub const FORCE_DATA_STORE_V1: &str = "force_data_store_v1"; // For testing of v1+v2 merges
 }
 
 #[derive(Default, Debug, Clone, Copy, Serialize, Deserialize)]

{nidx_binding-6.6.1.post456 → nidx_binding-6.7.0.post460}/nidx_vector/src/segment/tests.rs

@@ -143,6 +143,9 @@ fn single_graph() {
 
 #[test]
 fn data_merge() -> anyhow::Result<()> {
+    let mut v1_config = CONFIG.clone();
+    v1_config.flags.push(flags::FORCE_DATA_STORE_V1.to_string());
+
     let key0 = "9cb39c75f8d9498d8f82d92b173011f5/f/field/0-100".to_string();
     let vector0 = create_query();
     let elems0 = vec![Elem::new(key0.clone(), vector0.clone(), vec![], None)];
@@ -151,30 +154,30 @@ fn data_merge() -> anyhow::Result<()> {
     let elems1 = vec![Elem::new(key1.clone(), vector1.clone(), vec![], None)];
 
     let dp0_path = tempdir()?;
-    let dp0 = segment::create(dp0_path.path(), elems0, &CONFIG, HashSet::new()).unwrap();
+    let dp0 = segment::create(dp0_path.path(), elems0, &v1_config, HashSet::new()).unwrap();
     assert!(dp0.data_store.as_any().downcast_ref::<DataStoreV1>().is_some());
 
     let dp1_path = tempdir()?;
-    let dp1 = segment::create(dp1_path.path(), elems1, &CONFIG, HashSet::new()).unwrap();
+    let dp1 = segment::create(dp1_path.path(), elems1, &v1_config, HashSet::new()).unwrap();
     assert!(dp1.data_store.as_any().downcast_ref::<DataStoreV1>().is_some());
 
     let work = &[&dp1, &dp0];
 
     let dp_path = tempdir()?;
-    let dp = segment::merge(dp_path.path(), work, &CONFIG).unwrap();
+    let dp = segment::merge(dp_path.path(), work, &v1_config).unwrap();
     assert!(dp.data_store.as_any().downcast_ref::<DataStoreV1>().is_some());
 
     let formula = Formula::new();
-    let result: Vec<_> = dp.search(&vector1, &formula, true, 1, &CONFIG, -1.0).collect();
+    let result: Vec<_> = dp.search(&vector1, &formula, true, 1, &v1_config, -1.0).collect();
     assert_eq!(result.len(), 1);
     assert!(result[0].score() >= 0.9);
     assert!(dp.get_paragraph(result[0].paragraph()).id() == key1);
-    let result: Vec<_> = dp.search(&vector0, &formula, true, 1, &CONFIG, -1.0).collect();
+    let result: Vec<_> = dp.search(&vector0, &formula, true, 1, &v1_config, -1.0).collect();
     assert_eq!(result.len(), 1);
     assert!(result[0].score() >= 0.9);
     assert!(dp.get_paragraph(result[0].paragraph()).id() == key0);
-    let mut dp0 = segment::open(dp0.metadata, &CONFIG).unwrap();
-    let mut dp1 = segment::open(dp1.metadata, &CONFIG).unwrap();
+    let mut dp0 = segment::open(dp0.metadata, &v1_config).unwrap();
+    let mut dp1 = segment::open(dp1.metadata, &v1_config).unwrap();
     dp0.apply_deletion(&key0);
     dp0.apply_deletion(&key1);
     dp1.apply_deletion(&key0);
@@ -182,7 +185,7 @@ fn data_merge() -> anyhow::Result<()> {
     let work = &[&dp1, &dp0];
 
     let dp_path = tempdir()?;
-    let dp = segment::merge(dp_path.path(), work, &CONFIG).unwrap();
+    let dp = segment::merge(dp_path.path(), work, &v1_config).unwrap();
 
     assert_eq!(dp.metadata.records, 0);
 
@@ -198,34 +201,31 @@ fn data_merge_v2() -> anyhow::Result<()> {
     let vector1 = create_query();
     let elems1 = vec![Elem::new(key1.clone(), vector1.clone(), vec![], None)];
 
-    let mut v2_config = CONFIG.clone();
-    v2_config.flags.push(flags::DATA_STORE_V2.to_string());
-
     let dp0_path = tempdir()?;
-    let dp0 = segment::create(dp0_path.path(), elems0, &v2_config, HashSet::new()).unwrap();
+    let dp0 = segment::create(dp0_path.path(), elems0, &CONFIG, HashSet::new()).unwrap();
     assert!(dp0.data_store.as_any().downcast_ref::<DataStoreV2>().is_some());
 
     let dp1_path = tempdir()?;
-    let dp1 = segment::create(dp1_path.path(), elems1, &v2_config, HashSet::new()).unwrap();
+    let dp1 = segment::create(dp1_path.path(), elems1, &CONFIG, HashSet::new()).unwrap();
     assert!(dp1.data_store.as_any().downcast_ref::<DataStoreV2>().is_some());
 
     let work = &[&dp1, &dp0];
 
     let dp_path = tempdir()?;
-    let dp = segment::merge(dp_path.path(), work, &v2_config).unwrap();
+    let dp = segment::merge(dp_path.path(), work, &CONFIG).unwrap();
     assert!(dp.data_store.as_any().downcast_ref::<DataStoreV2>().is_some());
 
     let formula = Formula::new();
-    let result: Vec<_> = dp.search(&vector1, &formula, true, 1, &v2_config, -1.0).collect();
+    let result: Vec<_> = dp.search(&vector1, &formula, true, 1, &CONFIG, -1.0).collect();
     assert_eq!(result.len(), 1);
     assert!(result[0].score() >= 0.9);
     assert!(dp.get_paragraph(result[0].paragraph()).id() == key1);
-    let result: Vec<_> = dp.search(&vector0, &formula, true, 1, &v2_config, -1.0).collect();
+    let result: Vec<_> = dp.search(&vector0, &formula, true, 1, &CONFIG, -1.0).collect();
     assert_eq!(result.len(), 1);
     assert!(result[0].score() >= 0.9);
     assert!(dp.get_paragraph(result[0].paragraph()).id() == key0);
-    let mut dp0 = segment::open(dp0.metadata, &v2_config).unwrap();
-    let mut dp1 = segment::open(dp1.metadata, &v2_config).unwrap();
+    let mut dp0 = segment::open(dp0.metadata, &CONFIG).unwrap();
+    let mut dp1 = segment::open(dp1.metadata, &CONFIG).unwrap();
     dp0.apply_deletion(&key0);
     dp0.apply_deletion(&key1);
     dp1.apply_deletion(&key0);
@@ -233,7 +233,7 @@ fn data_merge_v2() -> anyhow::Result<()> {
     let work = &[&dp1, &dp0];
 
     let dp_path = tempdir()?;
-    let dp = segment::merge(dp_path.path(), work, &v2_config).unwrap();
+    let dp = segment::merge(dp_path.path(), work, &CONFIG).unwrap();
 
     assert_eq!(dp.metadata.records, 0);
 
@@ -249,34 +249,34 @@ fn data_merge_mixed() -> anyhow::Result<()> {
     let vector1 = create_query();
     let elems1 = vec![Elem::new(key1.clone(), vector1.clone(), vec![], None)];
 
-    let mut v2_config = CONFIG.clone();
-    v2_config.flags.push(flags::DATA_STORE_V2.to_string());
+    let mut v1_config = CONFIG.clone();
+    v1_config.flags.push(flags::FORCE_DATA_STORE_V1.to_string());
 
     let dp0_path = tempdir()?;
-    let dp0 = segment::create(dp0_path.path(), elems0, &CONFIG, HashSet::new()).unwrap();
+    let dp0 = segment::create(dp0_path.path(), elems0, &v1_config, HashSet::new()).unwrap();
     assert!(dp0.data_store.as_any().downcast_ref::<DataStoreV1>().is_some());
 
     let dp1_path = tempdir()?;
-    let dp1 = segment::create(dp1_path.path(), elems1, &v2_config, HashSet::new()).unwrap();
+    let dp1 = segment::create(dp1_path.path(), elems1, &CONFIG, HashSet::new()).unwrap();
     assert!(dp1.data_store.as_any().downcast_ref::<DataStoreV2>().is_some());
 
     let work = &[&dp1, &dp0];
 
     let dp_path = tempdir()?;
-    let dp = segment::merge(dp_path.path(), work, &v2_config).unwrap();
+    let dp = segment::merge(dp_path.path(), work, &CONFIG).unwrap();
     assert!(dp.data_store.as_any().downcast_ref::<DataStoreV2>().is_some());
 
     let formula = Formula::new();
-    let result: Vec<_> = dp.search(&vector1, &formula, true, 1, &v2_config, -1.0).collect();
+    let result: Vec<_> = dp.search(&vector1, &formula, true, 1, &CONFIG, -1.0).collect();
     assert_eq!(result.len(), 1);
     assert!(result[0].score() >= 0.9);
     assert!(dp.get_paragraph(result[0].paragraph()).id() == key1);
-    let result: Vec<_> = dp.search(&vector0, &formula, true, 1, &v2_config, -1.0).collect();
+    let result: Vec<_> = dp.search(&vector0, &formula, true, 1, &CONFIG, -1.0).collect();
     assert_eq!(result.len(), 1);
     assert!(result[0].score() >= 0.9);
     assert!(dp.get_paragraph(result[0].paragraph()).id() == key0);
-    let mut dp0 = segment::open(dp0.metadata, &v2_config).unwrap();
-    let mut dp1 = segment::open(dp1.metadata, &v2_config).unwrap();
+    let mut dp0 = segment::open(dp0.metadata, &CONFIG).unwrap();
+    let mut dp1 = segment::open(dp1.metadata, &CONFIG).unwrap();
     dp0.apply_deletion(&key0);
     dp0.apply_deletion(&key1);
     dp1.apply_deletion(&key0);
@@ -284,7 +284,7 @@ fn data_merge_mixed() -> anyhow::Result<()> {
     let work = &[&dp1, &dp0];
 
     let dp_path = tempdir()?;
-    let dp = segment::merge(dp_path.path(), work, &v2_config).unwrap();
+    let dp = segment::merge(dp_path.path(), work, &CONFIG).unwrap();
 
     assert_eq!(dp.metadata.records, 0);
 

{nidx_binding-6.6.1.post456 → nidx_binding-6.7.0.post460}/nidx_vector/src/segment.rs

@@ -99,15 +99,7 @@ pub fn merge(segment_path: &Path, operants: &[&OpenSegment], config: &VectorConf
     }
 
     // Creating the node store
-    if config.flags.contains(&flags::DATA_STORE_V2.to_string()) {
-        let node_producers = operants
-            .iter()
-            .map(|dp| (dp.alive_paragraphs(), dp.data_store.as_ref()))
-            .collect();
-        DataStoreV2::merge(segment_path, node_producers, &config.vector_type)?;
-        let data_store = DataStoreV2::open(segment_path, &config.vector_type, OpenReason::Create)?;
-        merge_indexes(segment_path, data_store, operants, config)
-    } else {
+    if config.flags.contains(&flags::FORCE_DATA_STORE_V1.to_string()) {
         // V1 can only merge from V1
         let mut node_producers = Vec::new();
         for dp in &operants {
@@ -123,6 +115,14 @@ pub fn merge(segment_path: &Path, operants: &[&OpenSegment], config: &VectorConf
         DataStoreV1::merge(segment_path, node_producers.as_mut_slice(), config)?;
         let data_store = DataStoreV1::open(segment_path, &config.vector_type, OpenReason::Create)?;
         merge_indexes(segment_path, data_store, operants, config)
+    } else {
+        let node_producers = operants
+            .iter()
+            .map(|dp| (dp.alive_paragraphs(), dp.data_store.as_ref()))
+            .collect();
+        DataStoreV2::merge(segment_path, node_producers, &config.vector_type)?;
+        let data_store = DataStoreV2::open(segment_path, &config.vector_type, OpenReason::Create)?;
+        merge_indexes(segment_path, data_store, operants, config)
     }
 }
 
@@ -215,15 +215,7 @@ pub fn create(path: &Path, elems: Vec<Elem>, config: &VectorConfig, tags: HashSe
     // Serializing nodes on disk
     // Nodes are stored on disk and mmaped.
     // Then trigger the rest of the creation (indexes)
-    if config.flags.contains(&flags::DATA_STORE_V2.to_string()) {
-        DataStoreV2::create(path, elems, &config.vector_type)?;
-        create_indexes(
-            path,
-            DataStoreV2::open(path, &config.vector_type, OpenReason::Create)?,
-            config,
-            tags,
-        )
-    } else {
+    if config.flags.contains(&flags::FORCE_DATA_STORE_V1.to_string()) {
         // Double check vector cardinality
         if elems.iter().any(|e| e.vectors.len() != 1) {
             return Err(crate::VectorErr::InvalidConfiguration(
@@ -237,6 +229,14 @@ pub fn create(path: &Path, elems: Vec<Elem>, config: &VectorConfig, tags: HashSe
             config,
             tags,
         )
+    } else {
+        DataStoreV2::create(path, elems, &config.vector_type)?;
+        create_indexes(
+            path,
+            DataStoreV2::open(path, &config.vector_type, OpenReason::Create)?,
+            config,
+            tags,
+        )
     }
 }
 

{nidx_binding-6.6.1.post456 → nidx_binding-6.7.0.post460}/nidx_vector/tests/test_basic_search.rs

@@ -51,9 +51,9 @@ fn test_basic_search(
     use nidx_vector::{VectorIndexer, VectorSearchRequest, VectorSearcher};
 
     let flags = if data_store_v2 {
-        vec![flags::DATA_STORE_V2.to_string()]
-    } else {
         vec![]
+    } else {
+        vec![flags::FORCE_DATA_STORE_V1.to_string()]
     };
 
     let config = VectorConfig {

{nidx_binding-6.6.1.post456 → nidx_binding-6.7.0.post460}/nidx_vector/tests/test_maxsim.rs

@@ -19,7 +19,7 @@
 //
 
 use nidx_protos::VectorSentence;
-use nidx_vector::config::{Similarity, VectorCardinality, VectorConfig, VectorType, flags};
+use nidx_vector::config::{Similarity, VectorCardinality, VectorConfig, VectorType};
 use tempfile::tempdir;
 
 mod common;
@@ -52,7 +52,7 @@ fn test_maxsim() -> anyhow::Result<()> {
         vector_type: VectorType::DenseF32 { dimension: 5 },
         normalize_vectors: false,
         vector_cardinality: VectorCardinality::Multi,
-        flags: vec![flags::DATA_STORE_V2.to_string()],
+        flags: vec![],
     };
 
     // Creates a resource with some orthogonal vectors, to test search

{nidx_binding-6.6.1.post456 → nidx_binding-6.7.0.post460}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.6.1.post456"
+version = "6.7.0.post460"
 
 [project.urls]
 Homepage = "https://nuclia.com"