nidx-binding 6.3.5.post220__tar.gz → 6.3.5.post224__tar.gz

{nidx_binding-6.3.5.post220 → nidx_binding-6.3.5.post224}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.5.post220
+Version: 6.3.5.post224
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.5.post220 → nidx_binding-6.3.5.post224}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.5.post220"
+version = "6.3.5.post224"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.5.post220 → nidx_binding-6.3.5.post224}/nidx_text/src/reader.rs

@@ -35,12 +35,14 @@ use nidx_protos::{
 };
 use nidx_types::prefilter::{FieldId, PrefilterResult};
 use tantivy::collector::{Collector, Count, FacetCollector, FacetCounts, SegmentCollector, TopDocs};
-use tantivy::columnar::Column;
+use tantivy::columnar::{BytesColumn, Column};
+use tantivy::fastfield::FacetReader;
 use tantivy::query::{AllQuery, BooleanQuery, Query, QueryParser, TermQuery};
 use tantivy::schema::Value;
 use tantivy::schema::*;
 use tantivy::{DocAddress, Index, IndexReader, Searcher};
 use tracing::*;
+use uuid::Uuid;
 
 fn facet_count(facet: &str, facets_count: &FacetCounts) -> Vec<FacetResult> {
     facets_count
@@ -89,13 +91,85 @@ impl Debug for TextReaderService {
 }
 
 struct FieldUuidSegmentCollector {
-    encoded_field_id_reader: Column,
+    uuid_reader: BytesColumn,
+    field_reader: FacetReader,
     results: Vec<FieldId>,
 }
 
 impl SegmentCollector for FieldUuidSegmentCollector {
     type Fruit = Vec<FieldId>;
 
+    fn collect(&mut self, doc: tantivy::DocId, _score: tantivy::Score) {
+        let uuid_ord = self.uuid_reader.term_ords(doc).next().unwrap();
+        let mut uuid_bytes = Vec::new();
+        self.uuid_reader.ord_to_bytes(uuid_ord, &mut uuid_bytes).unwrap();
+
+        let mut facet_ords = self.field_reader.facet_ords(doc);
+        let mut facet = Facet::root();
+        self.field_reader
+            .facet_from_ord(facet_ords.next().unwrap(), &mut facet)
+            .expect("field facet not found");
+
+        if let Ok(resource_id) = Uuid::parse_str(std::str::from_utf8(&uuid_bytes).unwrap()) {
+            self.results.push(FieldId {
+                resource_id,
+                field_id: facet.to_path_string(),
+            });
+        }
+    }
+
+    fn harvest(self) -> Self::Fruit {
+        self.results
+    }
+}
+
+struct FieldUuidCollector;
+
+impl Collector for FieldUuidCollector {
+    type Fruit = Vec<FieldId>;
+
+    type Child = FieldUuidSegmentCollector;
+
+    fn for_segment(
+        &self,
+        _segment_local_id: tantivy::SegmentOrdinal,
+        segment: &tantivy::SegmentReader,
+    ) -> tantivy::Result<Self::Child> {
+        let uuid_reader = segment.fast_fields().bytes("uuid")?.unwrap();
+        let field_reader = segment.facet_reader("field")?;
+        Ok(FieldUuidSegmentCollector {
+            uuid_reader,
+            field_reader,
+            results: vec![],
+        })
+    }
+
+    fn requires_scoring(&self) -> bool {
+        false
+    }
+
+    fn merge_fruits(
+        &self,
+        segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+    ) -> tantivy::Result<Self::Fruit> {
+        Ok(segment_fruits
+            .into_iter()
+            .reduce(|mut a, mut b| {
+                a.append(&mut b);
+                a
+            })
+            .unwrap_or_default())
+    }
+}
+
+struct FieldUuidSegmentCollectorV2 {
+    encoded_field_id_reader: Column,
+    results: Vec<FieldId>,
+}
+
+impl SegmentCollector for FieldUuidSegmentCollectorV2 {
+    type Fruit = Vec<FieldId>;
+
     fn collect(&mut self, doc: tantivy::DocId, _score: tantivy::Score) {
         let mut data = Vec::new();
         self.encoded_field_id_reader.fill_vals(doc, &mut data);
@@ -112,12 +186,12 @@ impl SegmentCollector for FieldUuidSegmentCollector {
     }
 }
 
-struct FieldUuidCollector;
+struct FieldUuidCollectorV2;
 
-impl Collector for FieldUuidCollector {
+impl Collector for FieldUuidCollectorV2 {
     type Fruit = Vec<FieldId>;
 
-    type Child = FieldUuidSegmentCollector;
+    type Child = FieldUuidSegmentCollectorV2;
 
     fn for_segment(
         &self,
@@ -125,7 +199,7 @@ impl Collector for FieldUuidCollector {
         segment: &tantivy::SegmentReader,
     ) -> tantivy::Result<Self::Child> {
         let encoded_field_id_reader = segment.fast_fields().u64("encoded_field_id")?;
-        Ok(FieldUuidSegmentCollector {
+        Ok(FieldUuidSegmentCollectorV2 {
             encoded_field_id_reader,
             results: vec![],
         })
@@ -189,7 +263,11 @@ impl TextReaderService {
 
         let prefilter_query: Box<dyn Query> = Box::new(BooleanQuery::intersection(subqueries));
         let searcher = self.reader.searcher();
-        let docs_fulfilled = searcher.search(&prefilter_query, &FieldUuidCollector)?;
+        let docs_fulfilled = if schema.encoded_field_id.is_some() {
+            searcher.search(&prefilter_query, &FieldUuidCollectorV2)?
+        } else {
+            searcher.search(&prefilter_query, &FieldUuidCollector)?
+        };
         // If none of the fields match the pre-filter, thats all the query planner needs to know.
         if docs_fulfilled.is_empty() {
             return Ok(PrefilterResult::None);

{nidx_binding-6.3.5.post220 → nidx_binding-6.3.5.post224}/nidx_text/src/resource_indexer.rs

@@ -80,8 +80,10 @@ pub fn index_document(
         field_doc.add_facet(schema.field, facet_field);
         field_doc.add_text(schema.text, &text_info.text);
 
-        for d in encode_field_id(resource_uuid, &format!("/{field}")) {
-            field_doc.add_u64(schema.encoded_field_id, d);
+        if let Some(encoded_field_id) = schema.encoded_field_id {
+            for d in encode_field_id(resource_uuid, &format!("/{field}")) {
+                field_doc.add_u64(encoded_field_id, d);
+            }
         }
 
         if let Some(encoded_field_id_bytes) = schema.encoded_field_id_bytes {

{nidx_binding-6.3.5.post220 → nidx_binding-6.3.5.post224}/nidx_text/src/schema.rs

@@ -45,7 +45,7 @@ pub struct TextSchema {
     pub groups_public: Field,
     pub groups_with_access: Field,
 
-    pub encoded_field_id: Field,
+    pub encoded_field_id: Option<Field>,
     pub encoded_field_id_bytes: Option<Field>,
 }
 
@@ -85,13 +85,18 @@ impl TextSchema {
             // v4: Field ID encoded as array of u64 for faster retrieval during prefilter
             // Using a bytes field is slow due to tantivy's implementation being slow with many unique values.
             // A better implementation is tracked in https://github.com/quickwit-oss/tantivy/issues/2090
-            sb.add_u64_field("encoded_field_id", FAST)
-        } else {
+            Some(sb.add_u64_field("encoded_field_id", FAST))
+        } else if version == 3 {
             // v3: Field ID encoded as array of u64 for faster retrieval during prefilter
             // Using a bytes field is slow due to tantivy's implementation being slow with many unique values.
             // A better implementation is tracked in https://github.com/quickwit-oss/tantivy/issues/2090
             // The INDEXED flag is unnecessary here, and this is backwards compatible code until we remove it in v4.
-            sb.add_u64_field("encoded_field_id", FAST | INDEXED)
+            Some(sb.add_u64_field("encoded_field_id", FAST | INDEXED))
+        } else if version == 2 {
+            // v2 did not have the field indexed
+            Some(sb.add_u64_field("encoded_field_id", FAST))
+        } else {
+            None
         };
 
         let encoded_field_id_bytes = if version >= 4 {

{nidx_binding-6.3.5.post220 → nidx_binding-6.3.5.post224}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.5.post220"
+version = "6.3.5.post224"
 
 [project.urls]
 Homepage = "https://nuclia.com"