nidx-binding 6.3.5.post215__tar.gz → 6.3.5.post220__tar.gz

{nidx_binding-6.3.5.post215 → nidx_binding-6.3.5.post220}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.5.post215
+Version: 6.3.5.post220
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.5.post215 → nidx_binding-6.3.5.post220}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.5.post215"
+version = "6.3.5.post220"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.5.post215 → nidx_binding-6.3.5.post220}/nidx_text/src/lib.rs

@@ -61,7 +61,7 @@ impl Default for TextConfig {
         Self {
             // This is the default version when creating a new index.
             // Should typically be set to the latest supported version
-            version: 3,
+            version: 4,
         }
     }
 }

{nidx_binding-6.3.5.post215 → nidx_binding-6.3.5.post220}/nidx_text/src/reader.rs

@@ -35,14 +35,12 @@ use nidx_protos::{
 };
 use nidx_types::prefilter::{FieldId, PrefilterResult};
 use tantivy::collector::{Collector, Count, FacetCollector, FacetCounts, SegmentCollector, TopDocs};
-use tantivy::columnar::{BytesColumn, Column};
-use tantivy::fastfield::FacetReader;
+use tantivy::columnar::Column;
 use tantivy::query::{AllQuery, BooleanQuery, Query, QueryParser, TermQuery};
 use tantivy::schema::Value;
 use tantivy::schema::*;
 use tantivy::{DocAddress, Index, IndexReader, Searcher};
 use tracing::*;
-use uuid::Uuid;
 
 fn facet_count(facet: &str, facets_count: &FacetCounts) -> Vec<FacetResult> {
     facets_count
@@ -91,83 +89,11 @@ impl Debug for TextReaderService {
 }
 
 struct FieldUuidSegmentCollector {
-    uuid_reader: BytesColumn,
-    field_reader: FacetReader,
-    results: Vec<FieldId>,
-}
-
-impl SegmentCollector for FieldUuidSegmentCollector {
-    type Fruit = Vec<FieldId>;
-
-    fn collect(&mut self, doc: tantivy::DocId, _score: tantivy::Score) {
-        let uuid_ord = self.uuid_reader.term_ords(doc).next().unwrap();
-        let mut uuid_bytes = Vec::new();
-        self.uuid_reader.ord_to_bytes(uuid_ord, &mut uuid_bytes).unwrap();
-
-        let mut facet_ords = self.field_reader.facet_ords(doc);
-        let mut facet = Facet::root();
-        self.field_reader
-            .facet_from_ord(facet_ords.next().unwrap(), &mut facet)
-            .expect("field facet not found");
-
-        if let Ok(resource_id) = Uuid::parse_str(std::str::from_utf8(&uuid_bytes).unwrap()) {
-            self.results.push(FieldId {
-                resource_id,
-                field_id: facet.to_path_string(),
-            });
-        }
-    }
-
-    fn harvest(self) -> Self::Fruit {
-        self.results
-    }
-}
-
-struct FieldUuidCollector;
-
-impl Collector for FieldUuidCollector {
-    type Fruit = Vec<FieldId>;
-
-    type Child = FieldUuidSegmentCollector;
-
-    fn for_segment(
-        &self,
-        _segment_local_id: tantivy::SegmentOrdinal,
-        segment: &tantivy::SegmentReader,
-    ) -> tantivy::Result<Self::Child> {
-        let uuid_reader = segment.fast_fields().bytes("uuid")?.unwrap();
-        let field_reader = segment.facet_reader("field")?;
-        Ok(FieldUuidSegmentCollector {
-            uuid_reader,
-            field_reader,
-            results: vec![],
-        })
-    }
-
-    fn requires_scoring(&self) -> bool {
-        false
-    }
-
-    fn merge_fruits(
-        &self,
-        segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
-    ) -> tantivy::Result<Self::Fruit> {
-        Ok(segment_fruits
-            .into_iter()
-            .reduce(|mut a, mut b| {
-                a.append(&mut b);
-                a
-            })
-            .unwrap_or_default())
-    }
-}
-
-struct FieldUuidSegmentCollectorV2 {
     encoded_field_id_reader: Column,
     results: Vec<FieldId>,
 }
 
-impl SegmentCollector for FieldUuidSegmentCollectorV2 {
+impl SegmentCollector for FieldUuidSegmentCollector {
     type Fruit = Vec<FieldId>;
 
     fn collect(&mut self, doc: tantivy::DocId, _score: tantivy::Score) {
@@ -186,12 +112,12 @@ impl SegmentCollector for FieldUuidSegmentCollectorV2 {
     }
 }
 
-struct FieldUuidCollectorV2;
+struct FieldUuidCollector;
 
-impl Collector for FieldUuidCollectorV2 {
+impl Collector for FieldUuidCollector {
     type Fruit = Vec<FieldId>;
 
-    type Child = FieldUuidSegmentCollectorV2;
+    type Child = FieldUuidSegmentCollector;
 
     fn for_segment(
         &self,
@@ -199,7 +125,7 @@ impl Collector for FieldUuidCollectorV2 {
         segment: &tantivy::SegmentReader,
     ) -> tantivy::Result<Self::Child> {
         let encoded_field_id_reader = segment.fast_fields().u64("encoded_field_id")?;
-        Ok(FieldUuidSegmentCollectorV2 {
+        Ok(FieldUuidSegmentCollector {
             encoded_field_id_reader,
             results: vec![],
         })
@@ -263,12 +189,7 @@ impl TextReaderService {
 
         let prefilter_query: Box<dyn Query> = Box::new(BooleanQuery::intersection(subqueries));
         let searcher = self.reader.searcher();
-        let docs_fulfilled = if schema.encoded_field_id.is_some() {
-            searcher.search(&prefilter_query, &FieldUuidCollectorV2)?
-        } else {
-            searcher.search(&prefilter_query, &FieldUuidCollector)?
-        };
-
+        let docs_fulfilled = searcher.search(&prefilter_query, &FieldUuidCollector)?;
         // If none of the fields match the pre-filter, thats all the query planner needs to know.
         if docs_fulfilled.is_empty() {
             return Ok(PrefilterResult::None);

{nidx_binding-6.3.5.post215 → nidx_binding-6.3.5.post220}/nidx_text/src/resource_indexer.rs

@@ -22,7 +22,7 @@ use anyhow::anyhow;
 use nidx_tantivy::TantivyIndexer;
 use tantivy::{doc, schema::Facet};
 
-use crate::schema::{TextSchema, encode_field_id, timestamp_to_datetime_utc};
+use crate::schema::{TextSchema, encode_field_id, encode_field_id_bytes, timestamp_to_datetime_utc};
 
 pub fn index_document(
     writer: &mut TantivyIndexer,
@@ -80,10 +80,13 @@ pub fn index_document(
         field_doc.add_facet(schema.field, facet_field);
         field_doc.add_text(schema.text, &text_info.text);
 
-        if let Some(encoded_field_id) = schema.encoded_field_id {
-            for d in encode_field_id(resource_uuid, &format!("/{field}")) {
-                field_doc.add_u64(encoded_field_id, d);
-            }
+        for d in encode_field_id(resource_uuid, &format!("/{field}")) {
+            field_doc.add_u64(schema.encoded_field_id, d);
+        }
+
+        if let Some(encoded_field_id_bytes) = schema.encoded_field_id_bytes {
+            let encoded = encode_field_id_bytes(resource_uuid, field);
+            field_doc.add_bytes(encoded_field_id_bytes, encoded.as_slice());
         }
 
         for label in text_info.labels.iter() {

{nidx_binding-6.3.5.post215 → nidx_binding-6.3.5.post220}/nidx_text/src/schema.rs

@@ -45,7 +45,8 @@ pub struct TextSchema {
     pub groups_public: Field,
     pub groups_with_access: Field,
 
-    pub encoded_field_id: Option<Field>,
+    pub encoded_field_id: Field,
+    pub encoded_field_id_bytes: Option<Field>,
 }
 
 pub fn timestamp_to_datetime_utc(timestamp: &nidx_protos::prost_types::Timestamp) -> DateTime {
@@ -80,14 +81,22 @@ impl TextSchema {
         let groups_public = sb.add_u64_field("groups_public", num_options);
         let groups_with_access = sb.add_facet_field("groups_with_access", facet_options);
 
-        let encoded_field_id = if version >= 3 {
-            // v3: We index the encoded_field_id to allow optimal field-level deletions
-            Some(sb.add_u64_field("encoded_field_id", FAST | INDEXED))
-        } else if version == 2 {
-            // v2: Field ID encoded as array of u64 for faster retrieval during prefilter
+        let encoded_field_id = if version >= 4 {
+            // v4: Field ID encoded as array of u64 for faster retrieval during prefilter
             // Using a bytes field is slow due to tantivy's implementation being slow with many unique values.
             // A better implementation is tracked in https://github.com/quickwit-oss/tantivy/issues/2090
-            Some(sb.add_u64_field("encoded_field_id", FAST))
+            sb.add_u64_field("encoded_field_id", FAST)
+        } else {
+            // v3: Field ID encoded as array of u64 for faster retrieval during prefilter
+            // Using a bytes field is slow due to tantivy's implementation being slow with many unique values.
+            // A better implementation is tracked in https://github.com/quickwit-oss/tantivy/issues/2090
+            // The INDEXED flag is unnecessary here, and this is backwards compatible code until we remove it in v4.
+            sb.add_u64_field("encoded_field_id", FAST | INDEXED)
+        };
+
+        let encoded_field_id_bytes = if version >= 4 {
+            // v4: Field ID encoded as array of u8 for faster deletions
+            Some(sb.add_bytes_field("encoded_field_id_bytes", INDEXED))
         } else {
             None
         };
@@ -106,6 +115,7 @@ impl TextSchema {
             groups_public,
             groups_with_access,
             encoded_field_id,
+            encoded_field_id_bytes,
         }
     }
 }
@@ -147,9 +157,28 @@ pub fn decode_field_id(data: &[u64]) -> (uuid::Uuid, String) {
     (rid, String::from_utf8(ubytes).unwrap())
 }
 
+/// Encodes a resource and field id as a series of u8
+/// This is stored in the indexed field `encoded_field_id_bytes`
+pub fn encode_field_id_bytes(rid: uuid::Uuid, fid: &str) -> Vec<u8> {
+    let mut out = Vec::with_capacity(16 + fid.len());
+    out.extend_from_slice(rid.as_bytes());
+    out.extend_from_slice(fid.as_bytes());
+    out
+}
+
 #[cfg(test)]
 mod tests {
-    use super::{decode_field_id, encode_field_id};
+    use super::{decode_field_id, encode_field_id, encode_field_id_bytes};
+
+    /// Decodes a resource and field id from a series of u8
+    pub fn decode_field_id_bytes(data: &[u8]) -> (uuid::Uuid, String) {
+        if data.len() < 16 {
+            panic!("Data is too short to contain a valid UUID");
+        }
+        let rid = uuid::Uuid::from_slice(&data[..16]).unwrap();
+        let fid = String::from_utf8(data[16..].to_vec()).unwrap();
+        (rid, fid)
+    }
 
     #[test]
     fn test_encode_decode_field_id() {
@@ -172,4 +201,26 @@ mod tests {
             assert_eq!(decoded_fid, t.1);
         }
     }
+
+    #[test]
+    fn test_encode_decode_field_id_bytes() {
+        // Test with different lengths to ensure it works when crossing block length
+        let testcases = [
+            ("03ce0c2cdcc0060069fb8f69c73fa8e2", "a/title"),
+            ("03ce0c2cdcc0060069fb8f69c73fa8e2", "a/title1"),
+            ("03ce0c2cdcc0060069fb8f69c73fa8e2", "a/title12"),
+            ("03ce0c2cdcc0060069fb8f69c73fa8e2", "a/title123"),
+            ("03ce0c2cdcc0060069fb8f69c73fa8e2", "a/title1234"),
+            ("03ce0c2cdcc0060069fb8f69c73fa8e2", "a/title12345"),
+            ("03ce0c2cdcc0060069fb8f69c73fa8e2", "a/title123456"),
+            ("03ce0c2cdcc0060069fb8f69c73fa8e2", "a/title1234567"),
+        ];
+        for t in testcases {
+            let rid = uuid::Uuid::parse_str(t.0).unwrap();
+            let data = encode_field_id_bytes(rid, t.1);
+            let (decoded_rid, decoded_fid) = decode_field_id_bytes(&data);
+            assert_eq!(decoded_rid.simple().to_string().as_str(), t.0);
+            assert_eq!(decoded_fid, t.1);
+        }
+    }
 }

{nidx_binding-6.3.5.post215 → nidx_binding-6.3.5.post220}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.5.post215"
+version = "6.3.5.post220"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.3.5.post215 → nidx_binding-6.3.5.post220}/src/metadata/index.rs

@@ -345,12 +345,12 @@ mod tests {
     async fn test_text_config(pool: sqlx::PgPool) -> anyhow::Result<()> {
         let meta = NidxMetadata::new_with_pool(pool).await.unwrap();
 
-        // Default version for new indexes is 3
+        // Default version for new indexes is 4
         let shard = Shard::create(&meta.pool, Uuid::new_v4()).await.unwrap();
         let index = Index::create(&meta.pool, shard.id, "multilingual", IndexConfig::new_text())
             .await
             .unwrap();
-        assert_eq!(index.config::<TextConfig>()?.version, 3);
+        assert_eq!(index.config::<TextConfig>()?.version, 4);
 
         // Default version if DB is empty is 1
         sqlx::query("UPDATE indexes SET configuration = NULL")