nidx-binding 6.9.7.post690__tar.gz → 6.9.7.post701__tar.gz

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.9.7.post690
+Version: 6.9.7.post701
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/nidx_protos/pyproject.toml

@@ -3,14 +3,14 @@ requires = [
     "pdm-backend",
     # Pin grpio-tools so `make protos` compile with the version we want
     # To use grpc stubs, you need a newer dependency than what was used to build this module
-    "grpcio-tools>=1.71.0,==1.71.0,<1.72.0",
+    "grpcio-tools==1.76.0",
     "mypy-protobuf==3.6.0",
 ]
 build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.9.7.post690"
+version = "6.9.7.post701"
 license = "AGPL-3.0-or-later"
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [{ name = "Nuclia", email = "nucliadb@nuclia.com" }]
@@ -24,6 +24,12 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 requires-python = ">=3.10, <4"
+dependencies = [
+    "protobuf>=6.31.1,<7.0.0"
+]
+
+[project.optional-dependencies]
+grpc = ["grpcio>=1.76.0"]
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/nidx_vector/src/config.rs

@@ -106,6 +106,8 @@ pub struct VectorConfig {
     pub vector_cardinality: VectorCardinality,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub flags: Vec<String>,
+    #[serde(default)]
+    pub disable_indexes: bool,
 }
 
 impl VectorConfig {
@@ -144,6 +146,7 @@ impl TryFrom<VectorIndexConfig> for VectorConfig {
             vector_type,
             flags: vec![],
             vector_cardinality: VectorCardinality::Single,
+            disable_indexes: false,
         })
     }
 }

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/nidx_vector/src/data_store/v1/store.rs

@@ -219,6 +219,7 @@ mod tests {
         normalize_vectors: false,
         flags: vec![],
         vector_cardinality: VectorCardinality::Single,
+        disable_indexes: false,
     };
 
     #[test]

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/nidx_vector/src/searcher.rs

@@ -416,6 +416,7 @@ mod tests {
             vector_type: VectorType::DenseF32 { dimension: 3 },
             flags: vec![],
             vector_cardinality: VectorCardinality::Single,
+            disable_indexes: false,
         };
         let raw_sentences = [
             (
@@ -511,6 +512,7 @@ mod tests {
             vector_type: VectorType::DenseF32 { dimension: 3 },
             flags: vec![],
             vector_cardinality: VectorCardinality::Single,
+            disable_indexes: false,
         };
         let raw_sentences = [
             (
@@ -625,6 +627,7 @@ mod tests {
             vector_type: VectorType::DenseF32 { dimension: 3 },
             flags: vec![],
             vector_cardinality: VectorCardinality::Single,
+            disable_indexes: false,
         };
         let raw_sentences = [
             (

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/nidx_vector/src/segment/tests.rs

@@ -35,6 +35,7 @@ const CONFIG: VectorConfig = VectorConfig {
     vector_type: crate::config::VectorType::DenseF32 { dimension: 128 },
     flags: vec![],
     vector_cardinality: VectorCardinality::Single,
+    disable_indexes: false,
 };
 
 fn create_query() -> Vec<f32> {

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/nidx_vector/src/segment.rs

@@ -56,7 +56,7 @@ pub fn open(metadata: VectorSegmentMetadata, config: &VectorConfig) -> VectorR<O
         let data_store = DataStoreV2::open(path, &config.vector_type, OpenReason::Search { prewarm })?;
         // Build the index at runtime if they do not exist. This can
         // be removed once we have migrated all existing indexes
-        if !InvertedIndexes::exists(path) {
+        if !InvertedIndexes::exists(path) && !config.disable_indexes {
             build_indexes(path, &data_store)?;
         }
         Box::new(data_store)
@@ -64,7 +64,15 @@ pub fn open(metadata: VectorSegmentMetadata, config: &VectorConfig) -> VectorR<O
 
     let index = open_disk_hnsw(path, prewarm)?;
 
-    let inverted_indexes = InvertedIndexes::open(path, metadata.records, inverted_index::OpenOptions { prewarm })?;
+    let inverted_indexes = if !config.disable_indexes {
+        Some(InvertedIndexes::open(
+            path,
+            metadata.records,
+            inverted_index::OpenOptions { prewarm },
+        )?)
+    } else {
+        None
+    };
     let alive_bitset = FilterBitSet::new(metadata.records, true);
 
     Ok(OpenSegment {
@@ -160,12 +168,16 @@ fn merge_indexes<DS: DataStore + 'static>(
         },
     };
 
-    build_indexes(segment_path, &data_store)?;
-    let inverted_indexes = InvertedIndexes::open(
-        segment_path,
-        merged_vectors_count as usize,
-        inverted_index::OpenOptions { prewarm: false },
-    )?;
+    let inverted_indexes = if !config.disable_indexes {
+        build_indexes(segment_path, &data_store)?;
+        Some(InvertedIndexes::open(
+            segment_path,
+            merged_vectors_count as usize,
+            inverted_index::OpenOptions { prewarm: false },
+        )?)
+    } else {
+        None
+    };
     let alive_bitset = FilterBitSet::new(metadata.records, true);
 
     Ok(OpenSegment {
@@ -250,12 +262,15 @@ fn create_indexes<DS: DataStore + 'static>(
         index_metadata: VectorSegmentMeta { tags },
     };
 
-    build_indexes(path, &data_store)?;
-    let inverted_indexes = InvertedIndexes::open(
-        path,
-        vector_count as usize,
-        inverted_index::OpenOptions { prewarm: false },
-    )?;
+    let inverted_indexes = if !config.disable_indexes {
+        Some(InvertedIndexes::open(
+            path,
+            vector_count as usize,
+            inverted_index::OpenOptions { prewarm: false },
+        )?)
+    } else {
+        None
+    };
     let alive_bitset = FilterBitSet::new(metadata.records, true);
 
     Ok(OpenSegment {
@@ -396,7 +411,7 @@ pub struct OpenSegment {
     metadata: VectorSegmentMetadata,
     data_store: Box<dyn DataStore>,
     index: Box<dyn DiskHnsw>,
-    inverted_indexes: InvertedIndexes,
+    inverted_indexes: Option<InvertedIndexes>,
     alive_bitset: FilterBitSet,
 }
 
@@ -408,7 +423,10 @@ impl AsRef<OpenSegment> for OpenSegment {
 
 impl OpenSegment {
     pub fn apply_deletion(&mut self, key: &str) {
-        if let Some(deleted_ids) = self.inverted_indexes.ids_for_deletion_key(key) {
+        let Some(inverted_indexes) = &self.inverted_indexes else {
+            return;
+        };
+        if let Some(deleted_ids) = inverted_indexes.ids_for_deletion_key(key) {
             for id in deleted_ids {
                 self.alive_bitset.remove(id);
             }
@@ -428,7 +446,9 @@ impl OpenSegment {
     }
 
     pub fn space_usage(&self) -> usize {
-        self.data_store.size_bytes() + self.index.size() + self.inverted_indexes.space_usage()
+        self.data_store.size_bytes()
+            + self.index.size()
+            + self.inverted_indexes.as_ref().map_or(0, InvertedIndexes::space_usage)
     }
 
     pub fn get_paragraph(&self, id: ParagraphAddr) -> ParagraphRef<'_> {
@@ -478,7 +498,7 @@ impl OpenSegment {
         };
         let retriever = Retriever::new(data_store, config, min_score);
 
-        let mut filter_bitset = self.inverted_indexes.filter(filter);
+        let mut filter_bitset = self.inverted_indexes.as_ref().and_then(|ii| ii.filter(filter));
         if let Some(ref mut bitset) = filter_bitset {
             bitset.intersect_with(&self.alive_bitset);
         }
@@ -710,6 +730,7 @@ mod test {
             normalize_vectors: false,
             flags: vec![],
             vector_cardinality: VectorCardinality::Single,
+            disable_indexes: false,
         };
         let mut rng = SmallRng::seed_from_u64(1234567890);
         let temp_dir = tempdir()?;
@@ -756,6 +777,7 @@ mod test {
             normalize_vectors: false,
             flags: vec![],
             vector_cardinality: VectorCardinality::Single,
+            disable_indexes: false,
         };
         let mut rng = SmallRng::seed_from_u64(1234567890);
 
@@ -834,6 +856,7 @@ mod test {
             normalize_vectors: false,
             flags: vec![],
             vector_cardinality: VectorCardinality::Single,
+            disable_indexes: false,
         };
 
         // Create a segment

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/nidx_vector/tests/test_basic_search.rs

@@ -62,6 +62,7 @@ fn test_basic_search(
         normalize_vectors: false,
         flags,
         vector_cardinality: VectorCardinality::Single,
+        disable_indexes: false,
     };
 
     // Creates a resource with some orthogonal vectors, to test search
@@ -163,6 +164,7 @@ fn test_deletions() -> anyhow::Result<()> {
         normalize_vectors: false,
         flags: vec![],
         vector_cardinality: VectorCardinality::Single,
+        disable_indexes: false,
     };
 
     // Creates a couple of resources
@@ -240,6 +242,7 @@ fn test_filtered_search() -> anyhow::Result<()> {
         normalize_vectors: false,
         flags: vec![],
         vector_cardinality: VectorCardinality::Single,
+        disable_indexes: false,
     };
 
     // Create 4 resources

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/nidx_vector/tests/test_hidden.rs

@@ -37,6 +37,7 @@ fn test_hidden_search() -> anyhow::Result<()> {
         vector_type: nidx_vector::config::VectorType::DenseF32 { dimension: 4 },
         flags: vec![],
         vector_cardinality: VectorCardinality::Single,
+        disable_indexes: false,
     };
 
     // Create two resources, one hidden and one not

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/nidx_vector/tests/test_maxsim.rs

@@ -53,6 +53,7 @@ fn test_maxsim() -> anyhow::Result<()> {
         normalize_vectors: false,
         vector_cardinality: VectorCardinality::Multi,
         flags: vec![],
+        disable_indexes: false,
     };
 
     // Creates a resource with some orthogonal vectors, to test search

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.9.7.post690"
+version = "6.9.7.post701"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/src/import_export.rs

@@ -282,6 +282,7 @@ mod tests {
         vector_type: VectorType::DenseF32 { dimension: 3 },
         flags: vec![],
         vector_cardinality: VectorCardinality::Single,
+        disable_indexes: false,
     };
 
     #[sqlx::test]

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/src/indexer.rs

@@ -435,6 +435,7 @@ mod tests {
         vector_type: VectorType::DenseF32 { dimension: 3 },
         flags: vec![],
         vector_cardinality: VectorCardinality::Single,
+        disable_indexes: false,
     };
 
     #[sqlx::test]

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/src/metadata/merge_job.rs

@@ -134,6 +134,7 @@ mod tests {
         vector_type: nidx_vector::config::VectorType::DenseF32 { dimension: 3 },
         flags: vec![],
         vector_cardinality: VectorCardinality::Single,
+        disable_indexes: false,
     };
 
     #[sqlx::test]

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/src/metadata.rs

@@ -106,6 +106,7 @@ mod tests {
         vector_type: VectorType::DenseF32 { dimension: 3 },
         flags: vec![],
         vector_cardinality: VectorCardinality::Single,
+        disable_indexes: false,
     };
 
     #[sqlx::test(migrations = false)]

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/src/scheduler/merge_task.rs

@@ -162,6 +162,7 @@ mod tests {
             vector_type: nidx_vector::config::VectorType::DenseF32 { dimension: 3 },
             flags: vec![],
             vector_cardinality: VectorCardinality::Single,
+            disable_indexes: false,
         };
 
         fn merge_scheduler() -> MergeScheduler {

{nidx_binding-6.9.7.post690 → nidx_binding-6.9.7.post701}/src/searcher/sync.rs

@@ -614,6 +614,7 @@ mod tests {
         vector_type: nidx_vector::config::VectorType::DenseF32 { dimension: 3 },
         flags: vec![],
         vector_cardinality: VectorCardinality::Single,
+        disable_indexes: false,
     };
 
     #[sqlx::test]

nidx_binding-6.9.7.post690/tests/test_date_range_search.rs → nidx_binding-6.9.7.post701/tests/integration/date_range_search.rs

@@ -18,11 +18,9 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 
-mod common;
-
 use std::collections::HashMap;
 
-use common::services::NidxFixture;
+use crate::common::services::NidxFixture;
 use nidx_protos::filter_expression::date_range_filter::DateField;
 use nidx_protos::filter_expression::{DateRangeFilter, Expr, FilterExpressionList};
 use nidx_protos::prost_types::Timestamp;

nidx_binding-6.9.7.post701/tests/integration/mod.rs

@@ -0,0 +1,32 @@
+// Copyright (C) 2021 Bosutech XXI S.L.
+//
+// nucliadb is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at info@nuclia.com.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+//
+
+mod date_range_search;
+mod search_filtering;
+mod search_relations;
+mod search_sorting;
+mod searcher_cluster;
+mod security_search;
+mod shards;
+mod shards_api;
+mod suggest;
+mod synced_searcher;
+mod vector_normalization;
+mod vectorsets;

nidx_binding-6.9.7.post690/tests/test_search_filtering.rs → nidx_binding-6.9.7.post701/tests/integration/search_filtering.rs

@@ -18,9 +18,7 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 
-mod common;
-
-use common::services::NidxFixture;
+use crate::common::services::NidxFixture;
 use nidx_protos::filter_expression::{Expr, FieldFilter, FilterExpressionList, KeywordFilter, ResourceFilter};
 use nidx_protos::prost_types::Timestamp;
 use nidx_protos::{FilterExpression, nodereader, noderesources};

nidx_binding-6.9.7.post690/tests/test_search_relations.rs → nidx_binding-6.9.7.post701/tests/integration/search_relations.rs

@@ -18,12 +18,10 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 
-mod common;
-
 use std::collections::HashMap;
 use std::time::SystemTime;
 
-use common::services::NidxFixture;
+use crate::common::services::NidxFixture;
 use nidx_protos::VectorIndexConfig;
 use nidx_protos::filter_expression::{Expr, ResourceFilter};
 use nidx_protos::graph_query::node::{FuzzyMatch, MatchKind, MatchLocation};

nidx_binding-6.9.7.post690/tests/test_search_sorting.rs → nidx_binding-6.9.7.post701/tests/integration/search_sorting.rs

@@ -18,12 +18,10 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 
-mod common;
-
 use std::collections::HashMap;
 use std::time::SystemTime;
 
-use common::services::NidxFixture;
+use crate::common::services::NidxFixture;
 use nidx_protos::nodewriter::VectorIndexConfig;
 use nidx_protos::prost_types::Timestamp;
 use nidx_protos::resource::ResourceStatus;

nidx_binding-6.9.7.post690/tests/test_searcher_cluster.rs → nidx_binding-6.9.7.post701/tests/integration/searcher_cluster.rs

@@ -18,9 +18,7 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 
-mod common;
-
-use common::services::NidxFixture;
+use crate::common::services::NidxFixture;
 use nidx::grpc_server::GrpcServer;
 use nidx::searcher::{ListNodes, SyncedSearcher, grpc::SearchServer, shard_selector::ShardSelector};
 use nidx_protos::SearchRequest;

nidx_binding-6.9.7.post690/tests/test_security_search.rs → nidx_binding-6.9.7.post701/tests/integration/security_search.rs

@@ -18,12 +18,10 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 
-mod common;
-
 use std::collections::HashMap;
 use std::time::SystemTime;
 
-use common::services::NidxFixture;
+use crate::common::services::NidxFixture;
 use nidx_protos::nodewriter::VectorIndexConfig;
 use nidx_protos::prost_types::Timestamp;
 use nidx_protos::relation::RelationType;

nidx_binding-6.9.7.post690/tests/test_shards.rs → nidx_binding-6.9.7.post701/tests/integration/shards.rs

@@ -18,11 +18,9 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 
-mod common;
-
 use std::collections::HashMap;
 
-use common::services::NidxFixture;
+use crate::common::services::NidxFixture;
 use nidx_protos::{
     EmptyQuery, GetShardRequest, NewShardRequest, ShardId, VectorIndexConfig, nidx::nidx_api_client::NidxApiClient,
 };

nidx_binding-6.9.7.post690/tests/test_shards_api.rs → nidx_binding-6.9.7.post701/tests/integration/shards_api.rs

@@ -18,8 +18,6 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 
-mod common;
-
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 
@@ -37,7 +35,7 @@ use nidx::{NidxMetadata, metadata::Shard};
 use nidx_tests::*;
 use nidx_vector::config::{VectorCardinality, VectorConfig};
 
-use common::metadata::{count_deletions, count_indexes, count_merge_jobs, count_segments, count_shards};
+use crate::common::metadata::{count_deletions, count_indexes, count_merge_jobs, count_segments, count_shards};
 
 const VECTOR_CONFIG: VectorConfig = VectorConfig {
     similarity: nidx_vector::config::Similarity::Cosine,
@@ -45,6 +43,7 @@ const VECTOR_CONFIG: VectorConfig = VectorConfig {
     vector_type: nidx_vector::config::VectorType::DenseF32 { dimension: 3 },
     flags: vec![],
     vector_cardinality: VectorCardinality::Single,
+    disable_indexes: false,
 };
 
 #[sqlx::test]

nidx_binding-6.9.7.post690/tests/test_suggest.rs → nidx_binding-6.9.7.post701/tests/integration/suggest.rs

@@ -18,9 +18,7 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 
-mod common;
-
-use common::services::NidxFixture;
+use crate::common::services::NidxFixture;
 use itertools::Itertools;
 use nidx_protos::{
     FilterExpression, NewShardRequest, SuggestFeatures, SuggestRequest, SuggestResponse, VectorIndexConfig,

nidx_binding-6.9.7.post690/tests/test_synced_searcher.rs → nidx_binding-6.9.7.post701/tests/integration/synced_searcher.rs

@@ -43,6 +43,7 @@ const VECTOR_CONFIG: VectorConfig = VectorConfig {
     vector_type: nidx_vector::config::VectorType::DenseF32 { dimension: 3 },
     flags: vec![],
     vector_cardinality: VectorCardinality::Single,
+    disable_indexes: false,
 };
 
 #[sqlx::test]

nidx_binding-6.9.7.post690/tests/test_vector_normalization.rs → nidx_binding-6.9.7.post701/tests/integration/vector_normalization.rs

@@ -18,9 +18,7 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 
-mod common;
-
-use common::services::NidxFixture;
+use crate::common::services::NidxFixture;
 use sqlx::PgPool;
 use std::collections::HashMap;
 use std::time::SystemTime;

nidx_binding-6.9.7.post690/tests/test_vectorsets.rs → nidx_binding-6.9.7.post701/tests/integration/vectorsets.rs

@@ -18,9 +18,7 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 
-mod common;
-
-use common::services::NidxFixture;
+use crate::common::services::NidxFixture;
 use nidx_protos::{
     NewShardRequest, NewVectorSetRequest, ShardId, VectorIndexConfig, VectorSetId, VectorSimilarity, op_status::Status,
 };

nidx_binding-6.9.7.post701/tests/test_integration.rs

@@ -0,0 +1,27 @@
+// Copyright (C) 2021 Bosutech XXI S.L.
+//
+// nucliadb is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at info@nuclia.com.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+//
+
+mod common;
+
+// We put all tests in submodules so we can keep granularity
+// but we import them into a single module here so they compile
+// into a single binary, which is significantly faster since linking
+// is a bottleneck.
+mod integration;