nidx-binding 6.3.5.post243__tar.gz → 6.3.5.post268__tar.gz

{nidx_binding-6.3.5.post243 → nidx_binding-6.3.5.post268}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.5.post243
+Version: 6.3.5.post268
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.5.post243 → nidx_binding-6.3.5.post268}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.5.post243"
+version = "6.3.5.post268"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.5.post243 → nidx_binding-6.3.5.post268}/nidx_text/src/lib.rs

@@ -37,7 +37,7 @@ use prefilter::PreFilterRequest;
 use reader::TextReaderService;
 use resource_indexer::index_document;
 use schema::TextSchema;
-
+use schema::encode_field_id_bytes;
 use serde::{Deserialize, Serialize};
 use tantivy::indexer::merge_indices;
 use tantivy::schema::Field;
@@ -46,7 +46,8 @@ use tantivy::{
     directory::MmapDirectory,
     query::{Query, TermSetQuery},
 };
-use tracing::instrument;
+use tracing::*;
+use uuid::Uuid;
 
 pub use request_types::DocumentSearchRequest;
 
@@ -74,12 +75,38 @@ fn default_version() -> u64 {
 
 pub struct TextIndexer;
 
-pub struct TextDeletionQueryBuilder(Field);
+pub struct TextDeletionQueryBuilder {
+    resource_uuid: Field,
+    encoded_field_id_bytes: Field,
+}
 impl DeletionQueryBuilder for TextDeletionQueryBuilder {
     fn query<'a>(&self, keys: impl Iterator<Item = &'a String>) -> Box<dyn Query> {
-        Box::new(TermSetQuery::new(
-            keys.map(|k| Term::from_field_bytes(self.0, k.as_bytes())),
-        ))
+        Box::new(TermSetQuery::new(keys.filter_map(|k| {
+            // Our keys can be resource ids or encoded field ids (e.g: uuid/t/text)
+            // match the corresponding tantivy field
+            let is_field = k.len() > 32;
+            if is_field {
+                let Ok(uuid) = Uuid::parse_str(&k[0..32]) else {
+                    error!(?k, "Invalid deletion key in nidx_text");
+                    return None;
+                };
+                let field_id = &k[33..];
+                Some(Term::from_field_bytes(
+                    self.encoded_field_id_bytes,
+                    &encode_field_id_bytes(uuid, field_id)[..],
+                ))
+            } else {
+                Some(Term::from_field_bytes(self.resource_uuid, k.as_bytes()))
+            }
+        })))
+    }
+}
+impl TextDeletionQueryBuilder {
+    fn new(schema: &TextSchema) -> Self {
+        TextDeletionQueryBuilder {
+            resource_uuid: schema.schema.get_field("uuid").unwrap(),
+            encoded_field_id_bytes: schema.schema.get_field("encoded_field_id_bytes").unwrap(),
+        }
     }
 }
 
@@ -100,6 +127,8 @@ impl TextIndexer {
 
     pub fn deletions_for_resource(&self, resource: &nidx_protos::Resource) -> Vec<String> {
         vec![resource.resource.as_ref().unwrap().uuid.clone()]
+        // TODO: Uncomment in another PR
+        // resource.texts_to_delete.clone()
     }
 
     #[instrument(name = "text::merge", skip_all)]
@@ -109,8 +138,9 @@ impl TextIndexer {
         config: TextConfig,
         open_index: impl OpenIndexMetadata<TantivyMeta>,
     ) -> anyhow::Result<TantivySegmentMetadata> {
-        let schema = TextSchema::new(config.version).schema;
-        let query_builder = TextDeletionQueryBuilder(schema.get_field("uuid").unwrap());
+        let text_schema = TextSchema::new(config.version);
+        let query_builder = TextDeletionQueryBuilder::new(&text_schema);
+        let schema = text_schema.clone().schema;
         let index = open_index_with_deletions(schema, open_index, query_builder)?;
 
         let output_index = merge_indices(&[index], MmapDirectory::open(work_dir)?)?;
@@ -134,8 +164,11 @@ impl TextSearcher {
     #[instrument(name = "text::open", skip_all)]
     pub fn open(config: TextConfig, open_index: impl OpenIndexMetadata<TantivyMeta>) -> anyhow::Result<Self> {
         let schema = TextSchema::new(config.version);
-        let index =
-            open_index_with_deletions(schema.schema.clone(), open_index, TextDeletionQueryBuilder(schema.uuid))?;
+        let index = open_index_with_deletions(
+            schema.schema.clone(),
+            open_index,
+            TextDeletionQueryBuilder::new(&schema),
+        )?;
 
         Ok(Self {
             reader: TextReaderService {

{nidx_binding-6.3.5.post243 → nidx_binding-6.3.5.post268}/nidx_text/src/reader.rs

@@ -35,14 +35,12 @@ use nidx_protos::{
 };
 use nidx_types::prefilter::{FieldId, PrefilterResult};
 use tantivy::collector::{Collector, Count, FacetCollector, FacetCounts, SegmentCollector, TopDocs};
-use tantivy::columnar::{BytesColumn, Column};
-use tantivy::fastfield::FacetReader;
+use tantivy::columnar::Column;
 use tantivy::query::{AllQuery, BooleanQuery, Query, QueryParser, TermQuery};
 use tantivy::schema::Value;
 use tantivy::schema::*;
 use tantivy::{DocAddress, Index, IndexReader, Searcher};
 use tracing::*;
-use uuid::Uuid;
 
 fn facet_count(facet: &str, facets_count: &FacetCounts) -> Vec<FacetResult> {
     facets_count
@@ -90,78 +88,6 @@ impl Debug for TextReaderService {
     }
 }
 
-struct FieldUuidSegmentCollector {
-    uuid_reader: BytesColumn,
-    field_reader: FacetReader,
-    results: Vec<FieldId>,
-}
-
-impl SegmentCollector for FieldUuidSegmentCollector {
-    type Fruit = Vec<FieldId>;
-
-    fn collect(&mut self, doc: tantivy::DocId, _score: tantivy::Score) {
-        let uuid_ord = self.uuid_reader.term_ords(doc).next().unwrap();
-        let mut uuid_bytes = Vec::new();
-        self.uuid_reader.ord_to_bytes(uuid_ord, &mut uuid_bytes).unwrap();
-
-        let mut facet_ords = self.field_reader.facet_ords(doc);
-        let mut facet = Facet::root();
-        self.field_reader
-            .facet_from_ord(facet_ords.next().unwrap(), &mut facet)
-            .expect("field facet not found");
-
-        if let Ok(resource_id) = Uuid::parse_str(std::str::from_utf8(&uuid_bytes).unwrap()) {
-            self.results.push(FieldId {
-                resource_id,
-                field_id: facet.to_path_string(),
-            });
-        }
-    }
-
-    fn harvest(self) -> Self::Fruit {
-        self.results
-    }
-}
-
-struct FieldUuidCollector;
-
-impl Collector for FieldUuidCollector {
-    type Fruit = Vec<FieldId>;
-
-    type Child = FieldUuidSegmentCollector;
-
-    fn for_segment(
-        &self,
-        _segment_local_id: tantivy::SegmentOrdinal,
-        segment: &tantivy::SegmentReader,
-    ) -> tantivy::Result<Self::Child> {
-        let uuid_reader = segment.fast_fields().bytes("uuid")?.unwrap();
-        let field_reader = segment.facet_reader("field")?;
-        Ok(FieldUuidSegmentCollector {
-            uuid_reader,
-            field_reader,
-            results: vec![],
-        })
-    }
-
-    fn requires_scoring(&self) -> bool {
-        false
-    }
-
-    fn merge_fruits(
-        &self,
-        segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
-    ) -> tantivy::Result<Self::Fruit> {
-        Ok(segment_fruits
-            .into_iter()
-            .reduce(|mut a, mut b| {
-                a.append(&mut b);
-                a
-            })
-            .unwrap_or_default())
-    }
-}
-
 struct FieldUuidSegmentCollectorV2 {
     encoded_field_id_reader: Column,
     results: Vec<FieldId>,
@@ -186,9 +112,9 @@ impl SegmentCollector for FieldUuidSegmentCollectorV2 {
     }
 }
 
-struct FieldUuidCollectorV2;
+struct FieldUuidCollector;
 
-impl Collector for FieldUuidCollectorV2 {
+impl Collector for FieldUuidCollector {
     type Fruit = Vec<FieldId>;
 
     type Child = FieldUuidSegmentCollectorV2;
@@ -263,11 +189,8 @@ impl TextReaderService {
 
         let prefilter_query: Box<dyn Query> = Box::new(BooleanQuery::intersection(subqueries));
         let searcher = self.reader.searcher();
-        let docs_fulfilled = if schema.encoded_field_id.is_some() {
-            searcher.search(&prefilter_query, &FieldUuidCollectorV2)?
-        } else {
-            searcher.search(&prefilter_query, &FieldUuidCollector)?
-        };
+        let docs_fulfilled = searcher.search(&prefilter_query, &FieldUuidCollector)?;
+
         // If none of the fields match the pre-filter, thats all the query planner needs to know.
         if docs_fulfilled.is_empty() {
             return Ok(PrefilterResult::None);

{nidx_binding-6.3.5.post243 → nidx_binding-6.3.5.post268}/nidx_text/src/resource_indexer.rs

@@ -80,16 +80,11 @@ pub fn index_document(
         field_doc.add_facet(schema.field, facet_field);
         field_doc.add_text(schema.text, &text_info.text);
 
-        if let Some(encoded_field_id) = schema.encoded_field_id {
-            for d in encode_field_id(resource_uuid, &format!("/{field}")) {
-                field_doc.add_u64(encoded_field_id, d);
-            }
-        }
-
-        if let Some(encoded_field_id_bytes) = schema.encoded_field_id_bytes {
-            let encoded = encode_field_id_bytes(resource_uuid, field);
-            field_doc.add_bytes(encoded_field_id_bytes, encoded.as_slice());
+        for d in encode_field_id(resource_uuid, &format!("/{field}")) {
+            field_doc.add_u64(schema.encoded_field_id, d);
         }
+        let encoded = encode_field_id_bytes(resource_uuid, field);
+        field_doc.add_bytes(schema.encoded_field_id_bytes, encoded.as_slice());
 
         for label in text_info.labels.iter() {
             let facet = Facet::from(label.as_str());

{nidx_binding-6.3.5.post243 → nidx_binding-6.3.5.post268}/nidx_text/src/schema.rs

@@ -45,8 +45,8 @@ pub struct TextSchema {
     pub groups_public: Field,
     pub groups_with_access: Field,
 
-    pub encoded_field_id: Option<Field>,
-    pub encoded_field_id_bytes: Option<Field>,
+    pub encoded_field_id: Field,
+    pub encoded_field_id_bytes: Field,
 }
 
 pub fn timestamp_to_datetime_utc(timestamp: &nidx_protos::prost_types::Timestamp) -> DateTime {
@@ -54,7 +54,7 @@ pub fn timestamp_to_datetime_utc(timestamp: &nidx_protos::prost_types::Timestamp
 }
 
 impl TextSchema {
-    pub fn new(version: u64) -> Self {
+    pub fn new(_version: u64) -> Self {
         let mut sb = Schema::builder();
         let num_options: NumericOptions = NumericOptions::default().set_indexed().set_fast();
 
@@ -81,30 +81,12 @@ impl TextSchema {
         let groups_public = sb.add_u64_field("groups_public", num_options);
         let groups_with_access = sb.add_facet_field("groups_with_access", facet_options);
 
-        let encoded_field_id = if version >= 4 {
-            // v4: Field ID encoded as array of u64 for faster retrieval during prefilter
-            // Using a bytes field is slow due to tantivy's implementation being slow with many unique values.
-            // A better implementation is tracked in https://github.com/quickwit-oss/tantivy/issues/2090
-            Some(sb.add_u64_field("encoded_field_id", FAST))
-        } else if version == 3 {
-            // v3: Field ID encoded as array of u64 for faster retrieval during prefilter
-            // Using a bytes field is slow due to tantivy's implementation being slow with many unique values.
-            // A better implementation is tracked in https://github.com/quickwit-oss/tantivy/issues/2090
-            // The INDEXED flag is unnecessary here, and this is backwards compatible code until we remove it in v4.
-            Some(sb.add_u64_field("encoded_field_id", FAST | INDEXED))
-        } else if version == 2 {
-            // v2 did not have the field indexed
-            Some(sb.add_u64_field("encoded_field_id", FAST))
-        } else {
-            None
-        };
-
-        let encoded_field_id_bytes = if version >= 4 {
-            // v4: Field ID encoded as array of u8 for faster deletions
-            Some(sb.add_bytes_field("encoded_field_id_bytes", INDEXED))
-        } else {
-            None
-        };
+        // v4: Field ID encoded as array of u64 for faster retrieval during prefilter
+        // Using a bytes field is slow due to tantivy's implementation being slow with many unique values.
+        // A better implementation is tracked in https://github.com/quickwit-oss/tantivy/issues/2090
+        let encoded_field_id = sb.add_u64_field("encoded_field_id", FAST);
+        // v4: Field ID encoded as array of u8 for faster deletions
+        let encoded_field_id_bytes = sb.add_bytes_field("encoded_field_id_bytes", INDEXED);
 
         let schema = sb.build();
 

nidx_binding-6.3.5.post268/nidx_text/tests/test_deletions.rs

@@ -0,0 +1,89 @@
+// Copyright (C) 2021 Bosutech XXI S.L.
+//
+// nucliadb is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at info@nuclia.com.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+//
+
+mod common;
+use common::TestOpener;
+use nidx_protos::TextInformation;
+use nidx_tests::minimal_resource;
+use nidx_text::{DocumentSearchRequest, TextConfig, TextIndexer, TextSearcher};
+use tempfile::tempdir;
+
+#[test]
+fn test_texts_deletions() -> anyhow::Result<()> {
+    // Create a resource with a couple of fields
+    let mut resource = minimal_resource("shard".to_string());
+    resource.texts.insert(
+        "a/title".to_string(),
+        TextInformation {
+            text: "The little prince".to_string(),
+            ..Default::default()
+        },
+    );
+    resource.texts.insert(
+        "a/summary".to_string(),
+        TextInformation {
+            text: "A story about a little prince".to_string(),
+            ..Default::default()
+        },
+    );
+    let rid = resource.resource.clone().unwrap().uuid.clone();
+    let segment_dir = tempdir()?;
+    let meta1 = TextIndexer
+        .index_resource(segment_dir.path(), TextConfig::default(), &resource)?
+        .unwrap();
+
+    // Search to validate that both fields are indexed
+    let searcher = TextSearcher::open(
+        TextConfig::default(),
+        TestOpener::new(vec![(meta1.clone(), 1i64.into())], vec![]),
+    )?;
+    let result = searcher.search(&DocumentSearchRequest {
+        result_per_page: 20,
+        ..Default::default()
+    })?;
+    assert_eq!(result.results.len(), 2);
+
+    // Search with resource deletion
+    let searcher = TextSearcher::open(
+        TextConfig::default(),
+        TestOpener::new(vec![(meta1.clone(), 1i64.into())], vec![(rid.clone(), 2i64.into())]),
+    )?;
+    let result = searcher.search(&DocumentSearchRequest {
+        result_per_page: 20,
+        ..Default::default()
+    })?;
+    assert_eq!(result.results.len(), 0);
+
+    // Search with field deletion
+    let searcher = TextSearcher::open(
+        TextConfig::default(),
+        TestOpener::new(
+            vec![(meta1.clone(), 1i64.into())],
+            vec![(format!("{rid}/a/title"), 2i64.into())],
+        ),
+    )?;
+    let result = searcher.search(&DocumentSearchRequest {
+        result_per_page: 20,
+        ..Default::default()
+    })?;
+    assert_eq!(result.results.len(), 1);
+
+    Ok(())
+}

{nidx_binding-6.3.5.post243 → nidx_binding-6.3.5.post268}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.5.post243"
+version = "6.3.5.post268"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.3.5.post243 → nidx_binding-6.3.5.post268}/src/import_export.rs

@@ -306,12 +306,15 @@ mod tests {
         );
 
         let storage_source = Arc::new(object_store::memory::InMemory::new());
+        let mut resource = little_prince(kbid.to_string(), None);
+        // Add some random deletions to test the import
+        resource.paragraphs_to_delete.push("1".to_string());
         index_resource(
             &meta_source,
             storage_source.clone(),
             &tempfile::env::temp_dir(),
             &shard.id.to_string(),
-            little_prince(shard.id.to_string(), None),
+            resource.clone(),
             123i64.into(),
         )
         .await?;
@@ -323,12 +326,10 @@ mod tests {
                 .await?,
             3
         );
-        assert_eq!(
-            sqlx::query_scalar!("SELECT COUNT(*) AS \"cnt!\" FROM deletions")
-                .fetch_one(&meta_source.pool)
-                .await?,
-            1
-        );
+        let deletions_count = sqlx::query_scalar!("SELECT COUNT(*) AS \"cnt!\" FROM deletions")
+            .fetch_one(&meta_source.pool)
+            .await?;
+        assert!(deletions_count >= 1, "Expected at least one deletion");
 
         // Create destination DB
         let config_dest = Postgres::test_context(&TestArgs::new("test_export_and_import_dest")).await?;
@@ -346,12 +347,10 @@ mod tests {
         let shard_dest = Shard::get(&meta_dest.pool, shard.id).await?;
         assert_eq!(shard_dest.kbid, shard.kbid);
         assert_eq!(Index::for_shard(&meta_dest.pool, shard.id).await?.len(), 3);
-        assert_eq!(
-            sqlx::query_scalar!("SELECT COUNT(*) AS \"cnt!\" FROM deletions")
-                .fetch_one(&meta_dest.pool)
-                .await?,
-            1
-        );
+        let after_deletions_count = sqlx::query_scalar!("SELECT COUNT(*) AS \"cnt!\" FROM deletions")
+            .fetch_one(&meta_dest.pool)
+            .await?;
+        assert_eq!(after_deletions_count, deletions_count);
 
         let all_segments = Segment::in_indexes(&meta_dest.pool, &index_ids).await?;
         assert_eq!(all_segments.len(), 3);