nidx-binding 6.9.4.post651__tar.gz → 6.9.5.post653__tar.gz

{nidx_binding-6.9.4.post651 → nidx_binding-6.9.5.post653}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.9.4.post651
+Version: 6.9.5.post653
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.9.4.post651 → nidx_binding-6.9.5.post653}/nidx_paragraph/src/reader.rs

@@ -138,30 +138,6 @@ impl ParagraphReaderService {
             debug!("Fallback fuzzy query took {}µs", time_tracker.checkpoint().as_micros());
         }
 
-        let total = response.results.len() as f32;
-        let mut some_below_min_score: bool = false;
-        let response_results = std::mem::take(&mut response.results);
-
-        for (i, mut r) in response_results.into_iter().enumerate() {
-            match &mut r.score {
-                None => continue,
-                Some(sc) if sc.bm25 < request.min_score => {
-                    // We can break here because the results are sorted by score
-                    some_below_min_score = true;
-                    break;
-                }
-                Some(sc) => {
-                    sc.booster = total - (i as f32);
-                    response.results.push(r);
-                }
-            }
-        }
-
-        if some_below_min_score {
-            // We set next_page to false so that the client stops asking for more results
-            response.next_page = false;
-        }
-
         debug!("Result processing took {}µs", time_tracker.checkpoint().as_micros());
         debug!("Paragraph search took {}µs", time_tracker.elapsed().as_micros());
 
@@ -298,6 +274,7 @@ impl Searcher<'_> {
             let extra_result = self.results + 1;
             match self.request.order.clone() {
                 Some(order) => {
+                    let sort_field = order.sort_by();
                     let custom_collector = self.custom_order_collector(order, extra_result);
                     let collector = &(Count, custom_collector);
                     let (total, top_docs) = searcher.search(&query, collector)?;
@@ -311,6 +288,7 @@ impl Searcher<'_> {
                         query: self.text,
                         results_per_page: self.results as i32,
                         searcher,
+                        sort_field,
                     }))
                 }
                 None => {
@@ -336,6 +314,7 @@ impl Searcher<'_> {
 
             match self.request.order.clone() {
                 Some(order) => {
+                    let sort_field = order.sort_by();
                     let custom_collector = self.custom_order_collector(order, extra_result);
                     let collector = &(Count, facet_collector, custom_collector);
                     let (total, facets_count, top_docs) = searcher.search(&query, collector)?;
@@ -349,6 +328,7 @@ impl Searcher<'_> {
                         query: self.text,
                         results_per_page: self.results as i32,
                         searcher,
+                        sort_field,
                     }))
                 }
                 None => {

{nidx_binding-6.9.4.post651 → nidx_binding-6.9.5.post653}/nidx_paragraph/src/schema.rs

@@ -63,6 +63,13 @@ pub fn timestamp_to_datetime_utc(timestamp: &nidx_protos::prost_types::Timestamp
     DateTime::from_timestamp_secs(timestamp.seconds)
 }
 
+pub fn datetime_utc_to_timestamp(tantivy: &DateTime) -> nidx_protos::prost_types::Timestamp {
+    nidx_protos::prost_types::Timestamp {
+        seconds: tantivy.into_timestamp_secs(),
+        nanos: 0,
+    }
+}
+
 impl ParagraphSchema {
     pub fn new() -> ParagraphSchema {
         ParagraphSchema::default()

{nidx_binding-6.9.4.post651 → nidx_binding-6.9.5.post653}/nidx_paragraph/src/search_response.rs

@@ -20,6 +20,8 @@
 use std::collections::HashMap;
 
 use lazy_static::lazy_static;
+use nidx_protos::order_by::OrderField;
+use nidx_protos::paragraph_result::SortValue;
 use nidx_protos::{FacetResult, FacetResults, ParagraphResult, ParagraphSearchResponse, ResultScore};
 use nidx_tantivy::utils::decode_facet;
 use tantivy::collector::FacetCounts;
@@ -29,6 +31,7 @@ use tantivy::{DateTime, DocAddress, TantivyDocument};
 use tracing::*;
 
 use crate::reader::ParagraphReaderService;
+use crate::schema::datetime_utc_to_timestamp;
 use crate::search_query::TermCollector;
 
 pub fn extract_labels<'a>(facets_iterator: impl Iterator<Item = CompactDocValue<'a>>) -> Vec<String> {
@@ -101,6 +104,7 @@ pub struct SearchIntResponse<'a> {
     pub results_per_page: i32,
     pub termc: TermCollector,
     pub searcher: tantivy::Searcher,
+    pub sort_field: OrderField,
 }
 
 pub struct SearchFacetsResponse<'a> {
@@ -133,13 +137,9 @@ impl From<SearchIntResponse<'_>> for ParagraphSearchResponse {
         let no_results = std::cmp::min(obtained, requested);
         let mut results: Vec<ParagraphResult> = Vec::with_capacity(no_results);
         let searcher = response.searcher;
-        for (_, doc_address) in response.top_docs.into_iter().take(no_results) {
+        for (score, doc_address) in response.top_docs.into_iter().take(no_results) {
             match searcher.doc::<TantivyDocument>(doc_address) {
                 Ok(doc) => {
-                    let score = ResultScore {
-                        bm25: 0.0,
-                        booster: 0.0,
-                    };
                     let schema = &response.text_service.schema;
                     let uuid = doc
                         .get_first(schema.uuid)
@@ -179,6 +179,8 @@ impl From<SearchIntResponse<'_>> for ParagraphSearchResponse {
                         .unwrap()
                         .to_string();
 
+                    let sort_value = Some(SortValue::Date(datetime_utc_to_timestamp(&score)));
+
                     let index = doc.get_first(schema.index).unwrap().as_u64().unwrap();
                     let mut terms: Vec<_> = response.termc.get_fterms(doc_address.doc_id).into_iter().collect();
                     terms.sort();
@@ -192,7 +194,7 @@ impl From<SearchIntResponse<'_>> for ParagraphSearchResponse {
                         split,
                         index,
                         matches: terms,
-                        score: Some(score),
+                        sort_value,
                         metadata: schema.metadata(&doc),
                     };
 
@@ -223,18 +225,22 @@ impl From<SearchIntResponse<'_>> for ParagraphSearchResponse {
 impl From<SearchBm25Response<'_>> for ParagraphSearchResponse {
     fn from(response: SearchBm25Response) -> Self {
         let total = response.total as i32;
+        let min_score = response.min_score;
         let obtained = response.top_docs.len();
         let requested = response.results_per_page as usize;
         let next_page = obtained > requested;
         let no_results = std::cmp::min(obtained, requested);
         let mut results: Vec<ParagraphResult> = Vec::with_capacity(no_results);
         let searcher = response.searcher;
-        for (score, doc_address) in response.top_docs.into_iter().take(no_results) {
+        for (i, (score, doc_address)) in response.top_docs.into_iter().take(no_results).enumerate() {
+            if score < min_score {
+                break;
+            }
             match searcher.doc::<TantivyDocument>(doc_address) {
                 Ok(doc) => {
                     let score = ResultScore {
                         bm25: score,
-                        booster: 0.0,
+                        booster: (response.total - i) as f32,
                     };
                     let schema = &response.text_service.schema;
                     let uuid = doc
@@ -281,7 +287,7 @@ impl From<SearchBm25Response<'_>> for ParagraphSearchResponse {
                         split,
                         index,
                         paragraph,
-                        score: Some(score),
+                        sort_value: Some(SortValue::Score(score)),
                         start: start_pos,
                         end: end_pos,
                         matches: terms,

{nidx_binding-6.9.4.post651 → nidx_binding-6.9.5.post653}/nidx_protos/nodereader.proto

@@ -54,7 +54,10 @@ message ResultScore {
 
 message DocumentResult {
     string uuid = 1;
-    ResultScore score = 3;
+    oneof sort_value {
+        ResultScore score = 3;
+        google.protobuf.Timestamp date = 6;
+    }
     string field = 4;
     repeated string labels = 5;
 }
@@ -86,7 +89,11 @@ message ParagraphResult {
     string paragraph = 6;
     string split = 7;
     uint64 index = 8;
-    ResultScore score = 9;
+    oneof sort_value {
+        ResultScore score = 9;
+        google.protobuf.Timestamp date = 13;
+    }
+
     repeated string matches = 10;
 
     // Metadata that can't be searched with but is returned on search results

{nidx_binding-6.9.4.post651 → nidx_binding-6.9.5.post653}/nidx_protos/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.9.4.post651"
+version = "6.9.5.post653"
 license = "AGPL-3.0-or-later"
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [{ name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.9.4.post651 → nidx_binding-6.9.5.post653}/nidx_text/src/reader.rs

@@ -21,13 +21,14 @@ use std::collections::HashMap;
 use std::fmt::Debug;
 use std::time::*;
 
-use crate::schema::decode_field_id;
+use crate::schema::{datetime_utc_to_timestamp, decode_field_id};
 use crate::search_query::filter_to_query;
 use crate::{DocumentSearchRequest, prefilter::*};
 
 use super::schema::TextSchema;
 use super::search_query;
 use itertools::Itertools;
+use nidx_protos::document_result::SortValue;
 use nidx_protos::order_by::{OrderField, OrderType};
 use nidx_protos::{
     DocumentItem, DocumentResult, DocumentSearchResponse, FacetResult, FacetResults, OrderBy, ResultScore,
@@ -39,8 +40,8 @@ use tantivy::collector::{Collector, Count, FacetCollector, FacetCounts, SegmentC
 use tantivy::columnar::Column;
 use tantivy::query::{AllQuery, BooleanQuery, Query, QueryParser, TermQuery};
 use tantivy::schema::Value;
-use tantivy::schema::*;
-use tantivy::{DocAddress, Index, IndexReader, Searcher};
+use tantivy::{DateTime, DocAddress, Index, IndexReader, Searcher};
+use tantivy::{Order, schema::*};
 use tracing::*;
 
 fn facet_count(facet: &str, facets_count: &FacetCounts) -> Vec<FacetResult> {
@@ -234,36 +235,33 @@ impl TextReaderService {
 }
 
 impl TextReaderService {
-    fn custom_order_collector(&self, order: OrderBy, limit: usize) -> impl Collector<Fruit = Vec<(i64, DocAddress)>> {
-        use tantivy::{DocId, SegmentReader};
-        let sorter = match order.r#type() {
-            OrderType::Desc => |t: i64| t,
-            OrderType::Asc => |t: i64| -t,
+    fn custom_order_collector(
+        &self,
+        order: OrderBy,
+        limit: usize,
+    ) -> impl Collector<Fruit = Vec<(DateTime, DocAddress)>> {
+        let order_field = match order.sort_by() {
+            OrderField::Created => "created",
+            OrderField::Modified => "modified",
         };
-        TopDocs::with_limit(limit).custom_score(move |segment_reader: &SegmentReader| {
-            let reader = match order.sort_by() {
-                OrderField::Created => segment_reader.fast_fields().date("created").unwrap(),
-                OrderField::Modified => segment_reader.fast_fields().date("modified").unwrap(),
-            };
-            move |doc: DocId| sorter(reader.values_for_doc(doc).next().unwrap().into_timestamp_secs())
-        })
+        let order_direction = match order.r#type() {
+            OrderType::Desc => Order::Desc,
+            OrderType::Asc => Order::Asc,
+        };
+        TopDocs::with_limit(limit).order_by_fast_field(order_field, order_direction)
     }
 
-    fn convert_int_order(&self, response: SearchResponse<i64>, searcher: &Searcher) -> DocumentSearchResponse {
+    fn convert_int_order(&self, response: SearchResponse<DateTime>, searcher: &Searcher) -> DocumentSearchResponse {
         let total = response.total as i32;
         let retrieved_results = response.results_per_page;
         let next_page = total > retrieved_results;
         let results_per_page = response.results_per_page as usize;
-        let result_stream = response.top_docs.into_iter().take(results_per_page).enumerate();
+        let result_stream = response.top_docs.into_iter().take(results_per_page);
         let mut results = Vec::with_capacity(results_per_page);
 
-        for (id, (_, doc_address)) in result_stream {
+        for (score, doc_address) in result_stream {
             match searcher.doc::<TantivyDocument>(doc_address) {
                 Ok(doc) => {
-                    let score = Some(ResultScore {
-                        bm25: 0.0,
-                        booster: id as f32,
-                    });
                     let uuid = String::from_utf8(
                         doc.get_first(self.schema.uuid)
                             .expect("document doesn't appear to have uuid.")
@@ -287,10 +285,12 @@ impl TextReaderService {
                         .filter(|x| x.starts_with("/l/"))
                         .collect_vec();
 
+                    let sort_value = Some(SortValue::Date(datetime_utc_to_timestamp(&score)));
+
                     let result = DocumentResult {
                         uuid,
                         field,
-                        score,
+                        sort_value,
                         labels,
                     };
                     results.push(result);
@@ -330,10 +330,10 @@ impl TextReaderService {
             }
             match searcher.doc::<TantivyDocument>(doc_address) {
                 Ok(doc) => {
-                    let score = Some(ResultScore {
+                    let score = ResultScore {
                         bm25: score,
                         booster: id as f32,
-                    });
+                    };
                     let uuid = String::from_utf8(
                         doc.get_first(self.schema.uuid)
                             .expect("document doesn't appear to have uuid.")
@@ -359,7 +359,7 @@ impl TextReaderService {
                     let result = DocumentResult {
                         uuid,
                         field,
-                        score,
+                        sort_value: Some(SortValue::Score(score)),
                         labels,
                     };
                     results.push(result);

{nidx_binding-6.9.4.post651 → nidx_binding-6.9.5.post653}/nidx_text/src/schema.rs

@@ -54,6 +54,13 @@ pub fn timestamp_to_datetime_utc(timestamp: &nidx_protos::prost_types::Timestamp
     DateTime::from_timestamp_secs(timestamp.seconds)
 }
 
+pub fn datetime_utc_to_timestamp(tantivy: &DateTime) -> nidx_protos::prost_types::Timestamp {
+    nidx_protos::prost_types::Timestamp {
+        seconds: tantivy.into_timestamp_secs(),
+        nanos: 0,
+    }
+}
+
 impl TextSchema {
     pub fn new(_version: u64) -> Self {
         let mut sb = Schema::builder();

{nidx_binding-6.9.4.post651 → nidx_binding-6.9.5.post653}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.9.4.post651"
+version = "6.9.5.post653"
 
 [project.urls]
 Homepage = "https://nuclia.com"