nidx-binding 6.7.0.post481__tar.gz → 6.7.0.post485__tar.gz

{nidx_binding-6.7.0.post481 → nidx_binding-6.7.0.post485}/Cargo.lock

@@ -2193,7 +2193,6 @@ dependencies = [
  "nidx_protos",
  "nidx_tantivy",
  "nidx_types",
- "nom",
  "once_cell",
  "regex",
  "serde_json",

{nidx_binding-6.7.0.post481 → nidx_binding-6.7.0.post485}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.7.0.post481
+Version: 6.7.0.post485
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.7.0.post481 → nidx_binding-6.7.0.post485}/nidx_paragraph/Cargo.toml

@@ -12,7 +12,6 @@ levenshtein_automata = "0.2.1"
 nidx_protos = { version = "0.1.0", path = "../nidx_protos" }
 nidx_tantivy = { version = "0.1.0", path = "../nidx_tantivy" }
 nidx_types = { version = "0.1.0", path = "../nidx_types" }
-nom = "7" # same version as used by tantivy
 once_cell = "1.20.2"
 regex = "1.11.1"
 serde_json = "1.0.132"

{nidx_binding-6.7.0.post481 → nidx_binding-6.7.0.post485}/nidx_paragraph/src/fuzzy_query.rs

@@ -184,12 +184,7 @@ pub struct FuzzyTermQuery {
 
 impl std::fmt::Debug for FuzzyTermQuery {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_fmt(format_args!(
-            "Fuzzy {{ term: {}, distance: {}, prefix: {} }}",
-            self.term.value().as_str().unwrap_or("<unknown>"),
-            self.distance,
-            self.prefix
-        ))
+        f.write_str("Fuzzy")
     }
 }
 impl FuzzyTermQuery {
@@ -215,11 +210,6 @@ impl FuzzyTermQuery {
         }
     }
 
-    #[cfg(test)]
-    pub fn is_prefix(&self) -> bool {
-        self.prefix
-    }
-
     fn specialized_weight(&self) -> tantivy::Result<AutomatonWeight<DfaWrapper>> {
         // LEV_BUILDER is a HashMap, whose `get` method returns an Option
         match LEV_BUILDER.get(&(self.distance, self.transposition_cost_one)) {

{nidx_binding-6.7.0.post481 → nidx_binding-6.7.0.post485}/nidx_paragraph/src/lib.rs

@@ -20,7 +20,6 @@
 
 mod fuzzy_query;
 mod query_io;
-mod query_parser;
 mod reader;
 mod request_types;
 mod resource_indexer;
@@ -28,6 +27,7 @@ mod schema;
 mod search_query;
 mod search_response;
 mod set_query;
+mod stop_words;
 
 use nidx_protos::{ParagraphItem, ParagraphSearchResponse, StreamRequest};
 use nidx_tantivy::{

{nidx_binding-6.7.0.post481 → nidx_binding-6.7.0.post485}/nidx_paragraph/src/reader.rs

@@ -19,23 +19,23 @@
 //
 
 use std::fmt::Debug;
-use std::time::{Duration, Instant};
+use std::time::Instant;
 
 use nidx_protos::order_by::{OrderField, OrderType};
 use nidx_protos::{OrderBy, ParagraphItem, ParagraphSearchResponse, StreamRequest};
 use nidx_types::prefilter::PrefilterResult;
 use tantivy::collector::{Collector, Count, FacetCollector, TopDocs};
-use tantivy::query::{AllQuery, Query};
+use tantivy::query::{AllQuery, Query, QueryParser};
 use tantivy::{DateTime, Order, schema::*};
 use tantivy::{DocAddress, Index, IndexReader};
 use tracing::*;
 
 use super::schema::ParagraphSchema;
-use crate::query_parser::FUZZY_DISTANCE;
 use crate::request_types::{ParagraphSearchRequest, ParagraphSuggestRequest};
 use crate::search_query::{SharedTermC, search_query, streaming_query, suggest_query};
 use crate::search_response::{SearchBm25Response, SearchFacetsResponse, SearchIntResponse, extract_labels};
 
+const FUZZY_DISTANCE: u8 = 1;
 const NUMBER_OF_RESULTS_SUGGEST: usize = 20;
 
 pub struct ParagraphReaderService {
@@ -65,17 +65,19 @@ impl ParagraphReaderService {
         request: &ParagraphSuggestRequest,
         prefilter: &PrefilterResult,
     ) -> anyhow::Result<ParagraphSearchResponse> {
-        let query = &request.body;
-        let (keyword, term_collector, fuzzy) = suggest_query(request, prefilter, &self.schema);
+        let parser = QueryParser::for_index(&self.index, vec![self.schema.text]);
+        let text = self.adapt_text(&parser, &request.body);
+        let (original, termc, fuzzied) =
+            suggest_query(&parser, &text, request, prefilter, &self.schema, FUZZY_DISTANCE);
 
         let searcher = self.reader.searcher();
         let topdocs = TopDocs::with_limit(NUMBER_OF_RESULTS_SUGGEST);
-        let mut results = searcher.search(&keyword, &topdocs)?;
+        let mut results = searcher.search(&original, &topdocs)?;
 
         if results.is_empty() {
             let topdocs = TopDocs::with_limit(NUMBER_OF_RESULTS_SUGGEST);
-            match searcher.search(&fuzzy, &topdocs) {
-                Ok(mut fuzzy) => results.append(&mut fuzzy),
+            match searcher.search(&fuzzied, &topdocs) {
+                Ok(mut fuzzied) => results.append(&mut fuzzied),
                 Err(err) => error!("{err:?} during suggest"),
             }
         }
@@ -84,10 +86,10 @@ impl ParagraphReaderService {
             total: results.len(),
             top_docs: results,
             facets_count: None,
-            facets: &[],
-            termc: term_collector.get_termc(),
+            facets: vec![],
+            termc: termc.get_termc(),
             text_service: self,
-            query,
+            query: &text,
             results_per_page: 10,
             searcher,
             min_score: 0.0,
@@ -111,46 +113,69 @@ impl ParagraphReaderService {
         request: &ParagraphSearchRequest,
         prefilter: &PrefilterResult,
     ) -> anyhow::Result<ParagraphSearchResponse> {
-        let mut time_tracker = TimeTracker::start();
+        let time = Instant::now();
+        let id = Some(&request.id);
+
+        let v = time.elapsed().as_millis();
+        debug!("{id:?} - Creating query: starts at {v} ms");
 
-        let shard_id = &request.id;
-        let text = &request.body;
+        let parser = QueryParser::for_index(&self.index, vec![self.schema.text]);
         let results = request.result_per_page as usize;
-        let (keyword_query, termc, fuzzy_query) = search_query(request, prefilter, &self.index, &self.schema);
-        let facets = request.facets();
-        trace!(
-            shard_id,
-            "Query parsing took {}µs",
-            time_tracker.checkpoint().as_micros()
-        );
 
+        let facets: Vec<_> = request
+            .faceted
+            .as_ref()
+            .iter()
+            .flat_map(|v| v.labels.iter())
+            .filter(|s| ParagraphReaderService::is_valid_facet(s))
+            .cloned()
+            .collect();
+        let text = self.adapt_text(&parser, &request.body);
+        let v = time.elapsed().as_millis();
+        debug!("{id:?} - Creating query: ends at {v} ms");
+
+        let v = time.elapsed().as_millis();
+        debug!("{id:?} - Searching: starts at {v} ms");
+
+        let advanced = request
+            .advanced_query
+            .as_ref()
+            .map(|query| parser.parse_query(query))
+            .transpose()?;
+        #[rustfmt::skip] let (original, termc, fuzzied) = search_query(
+            &parser,
+            &text,
+            request,
+            prefilter,
+            &self.schema,
+            FUZZY_DISTANCE,
+            advanced
+        );
         let searcher = Searcher {
             request,
             results,
-            facets,
-            text,
+            facets: &facets,
+            text: &text,
             only_faceted: request.only_faceted,
         };
-        // println!("Regular search with query: {:#?}", keyword_query);
-        let mut response = searcher.do_search(termc.clone(), keyword_query, self, request.min_score)?;
-        trace!(
-            shard_id,
-            "Keyword search took {}µs",
-            time_tracker.checkpoint().as_micros()
-        );
+        let mut response = searcher.do_search(termc.clone(), original, self, request.min_score)?;
+        let v = time.elapsed().as_millis();
+        debug!("{id:?} - Searching: ends at {v} ms");
 
         if response.results.is_empty() && request.result_per_page > 0 && request.min_score == 0 as f32 {
-            // println!("Fuzzy search with query: {:#?}", fuzzy_query);
-            let fuzzied = searcher.do_search(termc, fuzzy_query, self, request.min_score)?;
+            let v = time.elapsed().as_millis();
+            debug!("{id:?} - Applying fuzzy: starts at {v} ms");
+
+            let fuzzied = searcher.do_search(termc, fuzzied, self, request.min_score)?;
             response = fuzzied;
             response.fuzzy_distance = FUZZY_DISTANCE as i32;
-            trace!(
-                shard_id,
-                "Fallback fuzzy query took {}µs",
-                time_tracker.checkpoint().as_micros()
-            );
+            let v = time.elapsed().as_millis();
+            debug!("{id:?} - Applying fuzzy: ends at {v} ms");
         }
 
+        let v = time.elapsed().as_millis();
+        debug!("{id:?} - Producing results: starts at {v} ms");
+
         let total = response.results.len() as f32;
         let mut some_below_min_score: bool = false;
         let response_results = std::mem::take(&mut response.results);
@@ -175,47 +200,30 @@ impl ParagraphReaderService {
             response.next_page = false;
         }
 
-        trace!(
-            shard_id,
-            "Result processing took {}µs",
-            time_tracker.checkpoint().as_micros()
-        );
-        trace!(
-            shard_id,
-            "Paragraph search took {}µs",
-            time_tracker.elapsed().as_micros()
-        );
+        let v = time.elapsed().as_millis();
+        debug!("{id:?} - Producing results: starts at {v} ms");
+
+        let v = time.elapsed().as_millis();
+        debug!("{id:?} - Ending at: {v} ms");
 
         Ok(response)
     }
-}
-
-/// Small utility struct to track time in checkpoints.
-struct TimeTracker {
-    init: Instant,
-    checkpoint: Duration,
-}
-
-impl TimeTracker {
-    pub fn start() -> Self {
-        Self {
-            init: Instant::now(),
-            checkpoint: Duration::default(),
+    fn adapt_text(&self, parser: &QueryParser, text: &str) -> String {
+        // FIXME: after migrating from tantivy 0.22 -> 0.24, the query grammar
+        // now includes single quotes as special character. Queries having a
+        // single single quote now fail to parse. As a quick fix, we remove them
+        // all and replace them by a space.
+        let text = text.replace('\'', " ");
+        match text.trim() {
+            "" => text.to_string(),
+            text => parser
+                .parse_query(text)
+                .map(|_| text.to_string())
+                .unwrap_or_else(|_| format!("\"{}\"", text.replace('"', ""))),
         }
     }
-
-    /// Set a new checkpoint and return the duration since last one (or the
-    /// start if it's the first time).
-    pub fn checkpoint(&mut self) -> Duration {
-        let checkpoint = self.init.elapsed();
-        let elapsed = checkpoint - self.checkpoint;
-        self.checkpoint = checkpoint;
-        elapsed
-    }
-
-    /// Return the total elapsed duration since the start of the time tracking
-    pub fn elapsed(&self) -> Duration {
-        self.init.elapsed()
+    fn is_valid_facet(maybe_facet: &str) -> bool {
+        Facet::from_text(maybe_facet).is_ok()
     }
 }
 
@@ -271,7 +279,7 @@ impl Iterator for BatchProducer {
 struct Searcher<'a> {
     request: &'a ParagraphSearchRequest,
     results: usize,
-    facets: Vec<Facet>,
+    facets: &'a [String],
     text: &'a str,
     only_faceted: bool,
 }
@@ -303,7 +311,7 @@ impl Searcher<'_> {
             .facets
             .iter()
             .fold(FacetCollector::for_field("facets"), |mut collector, facet| {
-                collector.add_facet(facet.clone());
+                collector.add_facet(Facet::from(facet));
                 collector
             });
         if self.only_faceted {
@@ -312,7 +320,7 @@ impl Searcher<'_> {
             Ok(ParagraphSearchResponse::from(SearchFacetsResponse {
                 text_service: service,
                 facets_count: Some(facets_count),
-                facets: &self.facets,
+                facets: self.facets.to_vec(),
             }))
         } else if self.facets.is_empty() {
             // Only query no facets
@@ -325,7 +333,7 @@ impl Searcher<'_> {
                     Ok(ParagraphSearchResponse::from(SearchIntResponse {
                         total,
                         facets_count: None,
-                        facets: &self.facets,
+                        facets: self.facets.to_vec(),
                         top_docs,
                         termc: termc.get_termc(),
                         text_service: service,
@@ -341,7 +349,7 @@ impl Searcher<'_> {
                     Ok(ParagraphSearchResponse::from(SearchBm25Response {
                         total,
                         facets_count: None,
-                        facets: &self.facets,
+                        facets: self.facets.to_vec(),
                         top_docs,
                         termc: termc.get_termc(),
                         text_service: service,
@@ -364,7 +372,7 @@ impl Searcher<'_> {
                         total,
                         top_docs,
                         facets_count: Some(facets_count),
-                        facets: &self.facets,
+                        facets: self.facets.to_vec(),
                         termc: termc.get_termc(),
                         text_service: service,
                         query: self.text,
@@ -380,7 +388,7 @@ impl Searcher<'_> {
                         total,
                         top_docs,
                         facets_count: Some(facets_count),
-                        facets: &self.facets,
+                        facets: self.facets.to_vec(),
                         termc: termc.get_termc(),
                         text_service: service,
                         query: self.text,

{nidx_binding-6.7.0.post481 → nidx_binding-6.7.0.post485}/nidx_paragraph/src/request_types.rs

@@ -19,7 +19,6 @@
 //
 
 use nidx_types::query_language::BooleanExpression;
-use tantivy::schema::Facet;
 
 #[derive(Clone, Default)]
 pub struct ParagraphSearchRequest {
@@ -46,15 +45,3 @@ pub struct ParagraphSuggestRequest {
     /// Whether to do an OR/AND between prefilter results and filtering_formula
     pub filter_or: bool,
 }
-
-impl ParagraphSearchRequest {
-    // Helper method to extract valid facets from the request
-    pub fn facets(&self) -> Vec<Facet> {
-        self.faceted
-            .as_ref()
-            .iter()
-            .flat_map(|faceted| faceted.labels.iter())
-            .filter_map(|facet| Facet::from_text(facet).ok())
-            .collect()
-    }
-}

{nidx_binding-6.7.0.post481 → nidx_binding-6.7.0.post485}/nidx_paragraph/src/schema.rs

@@ -87,7 +87,6 @@ impl Default for ParagraphSchema {
         let uuid = sb.add_text_field("uuid", STRING | STORED);
         let field_uuid = sb.add_text_field("field_uuid", STRING);
         let paragraph = sb.add_text_field("paragraph", STRING | STORED);
-        // NOTE: review search query tokenization if we change how are we tokenizing this field
         let text = sb.add_text_field("text", TEXT);
         let start_pos = sb.add_u64_field("start_pos", num_options.clone());
         let end_pos = sb.add_u64_field("end_pos", num_options.clone());