nidx-binding 6.7.2.post490__tar.gz → 6.7.2.post495__tar.gz

{nidx_binding-6.7.2.post490 → nidx_binding-6.7.2.post495}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.7.2.post490
+Version: 6.7.2.post495
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.7.2.post490 → nidx_binding-6.7.2.post495}/nidx_paragraph/src/query_parser/fuzzy_parser.rs

@@ -30,6 +30,7 @@ use crate::fuzzy_query::FuzzyTermQuery;
 use crate::schema::ParagraphSchema;
 use crate::search_query::SharedTermC;
 
+use super::FallbackQuery;
 use super::keyword_parser;
 use super::tokenizer::Token;
 
@@ -53,7 +54,13 @@ pub const FUZZY_DISTANCE: u8 = 1;
 /// literals will be searched as fuzzy. The last literal can be searched as a
 /// fuzzy prefix if a suggest-like feature is desired.
 ///
-pub fn parse_fuzzy_query(query: &[Token], term_collector: SharedTermC, last_literal_as_prefix: bool) -> Box<dyn Query> {
+pub fn parse_fuzzy_query(
+    query: &[Token],
+    term_collector: SharedTermC,
+    last_literal_as_prefix: bool,
+) -> Result<Box<dyn Query>, FallbackQuery> {
+    let mut errors = vec![];
+
     let last_literal_index = if last_literal_as_prefix {
         query.iter().rposition(|token| matches!(token, Token::Literal(_)))
     } else {
@@ -92,8 +99,14 @@ pub fn parse_fuzzy_query(query: &[Token], term_collector: SharedTermC, last_lite
                 subqueries.push((Occur::Should, q));
             }
             Token::Quoted(quoted) => {
-                let q = keyword_parser::parse_quoted(&schema, quoted);
-                subqueries.push((Occur::Should, q));
+                match keyword_parser::parse_quoted(&schema, quoted) {
+                    Ok(q) => {
+                        subqueries.push((Occur::Should, q));
+                    }
+                    Err(error) => {
+                        errors.push(error);
+                    }
+                };
             }
             Token::Excluded(excluded) => {
                 let q = keyword_parser::parse_excluded(&schema, excluded);
@@ -102,13 +115,15 @@ pub fn parse_fuzzy_query(query: &[Token], term_collector: SharedTermC, last_lite
         }
     }
 
-    if subqueries.is_empty() {
+    let q = if subqueries.is_empty() {
         Box::new(AllQuery)
     } else if subqueries.len() == 1 {
         subqueries.pop().unwrap().1
     } else {
         Box::new(BooleanQuery::new(subqueries))
-    }
+    };
+
+    if errors.is_empty() { Ok(q) } else { Err((q, errors)) }
 }
 
 #[cfg(test)]
@@ -125,7 +140,7 @@ mod tests {
         let literal = "ab";
         assert!(literal.len() < MIN_FUZZY_LEN);
         let query = [Token::Literal(literal.into())];
-        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), false);
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), false).unwrap();
         assert!(fuzzy.is::<TermQuery>());
         let q = fuzzy.downcast::<TermQuery>().unwrap();
         assert_eq!(q.term().value().as_str(), Some(literal));
@@ -138,7 +153,7 @@ mod tests {
         let literal = "abcd";
         assert!(literal.len() >= MIN_FUZZY_LEN);
         let query = [Token::Literal(literal.into())];
-        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), false);
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), false).unwrap();
         assert!(fuzzy.is::<FuzzyTermQuery>());
         assert!(!fuzzy.downcast::<FuzzyTermQuery>().unwrap().is_prefix());
     }
@@ -154,18 +169,18 @@ mod tests {
         assert!(literal.len() >= MIN_FUZZY_PREFIX_LEN);
         let query = [Token::Literal(literal.into())];
 
-        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), false);
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), false).unwrap();
         assert!(fuzzy.is::<FuzzyTermQuery>());
         assert!(!fuzzy.downcast::<FuzzyTermQuery>().unwrap().is_prefix());
 
-        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), true);
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), true).unwrap();
         assert!(fuzzy.is::<FuzzyTermQuery>());
         assert!(fuzzy.downcast::<FuzzyTermQuery>().unwrap().is_prefix());
 
         // only the last term is fuzzy prefix
 
         let query = [Token::Literal(literal.into()), Token::Literal(literal.into())];
-        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), true);
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), true).unwrap();
         assert!(fuzzy.is::<BooleanQuery>());
         let q = fuzzy.downcast::<BooleanQuery>().unwrap();
         let clauses = q.clauses();
@@ -181,11 +196,11 @@ mod tests {
         assert!(literal.len() < MIN_FUZZY_PREFIX_LEN);
         let query = [Token::Literal(literal.into())];
 
-        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), false);
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), false).unwrap();
         assert!(fuzzy.is::<FuzzyTermQuery>());
         assert!(!fuzzy.downcast::<FuzzyTermQuery>().unwrap().is_prefix());
 
-        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), true);
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), true).unwrap();
         assert!(fuzzy.is::<FuzzyTermQuery>());
         assert!(!fuzzy.downcast::<FuzzyTermQuery>().unwrap().is_prefix());
     }

{nidx_binding-6.7.2.post490 → nidx_binding-6.7.2.post495}/nidx_paragraph/src/query_parser/keyword_parser.rs

@@ -18,18 +18,22 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 use tantivy::Term;
-use tantivy::query::{AllQuery, BooleanQuery, EmptyQuery, Occur, PhraseQuery, Query, TermQuery};
+use tantivy::query::{AllQuery, BooleanQuery, Occur, PhraseQuery, Query, TermQuery};
 use tantivy::schema::IndexRecordOption;
-use tracing::error;
 
 use crate::ParagraphSchema;
 
+use super::FallbackQuery;
 use super::tokenizer::Token;
 
 /// Convert a tokenized query into a tantivy keyword query
 ///
 /// Empty queries will match everything.
-pub fn parse_keyword_query<'a>(query: &'a [Token<'a>], schema: &ParagraphSchema) -> Box<dyn Query> {
+pub fn parse_keyword_query<'a>(
+    query: &'a [Token<'a>],
+    schema: &ParagraphSchema,
+) -> Result<Box<dyn Query>, FallbackQuery> {
+    let mut errors = vec![];
     let mut subqueries = vec![];
     for item in query {
         match item {
@@ -37,10 +41,10 @@ pub fn parse_keyword_query<'a>(query: &'a [Token<'a>], schema: &ParagraphSchema)
                 let q = parse_literal(schema, literal);
                 subqueries.push((Occur::Should, q));
             }
-            Token::Quoted(quoted) => {
-                let q = parse_quoted(schema, quoted);
-                subqueries.push((Occur::Should, q));
-            }
+            Token::Quoted(quoted) => match parse_quoted(schema, quoted) {
+                Ok(q) => subqueries.push((Occur::Should, q)),
+                Err(error) => errors.push(error),
+            },
             Token::Excluded(excluded) => {
                 let q = parse_excluded(schema, excluded);
                 subqueries.push((Occur::Should, q));
@@ -48,13 +52,15 @@ pub fn parse_keyword_query<'a>(query: &'a [Token<'a>], schema: &ParagraphSchema)
         }
     }
 
-    if subqueries.is_empty() {
+    let q = if subqueries.is_empty() {
         Box::new(AllQuery)
     } else if subqueries.len() == 1 {
         subqueries.pop().unwrap().1
     } else {
         Box::new(BooleanQuery::new(subqueries))
-    }
+    };
+
+    if errors.is_empty() { Ok(q) } else { Err((q, errors)) }
 }
 
 #[inline]
@@ -65,7 +71,7 @@ pub fn parse_literal(schema: &ParagraphSchema, literal: &str) -> Box<dyn Query>
     ))
 }
 
-pub fn parse_quoted(schema: &ParagraphSchema, quoted: &str) -> Box<dyn Query> {
+pub fn parse_quoted(schema: &ParagraphSchema, quoted: &str) -> Result<Box<dyn Query>, String> {
     let mut terms: Vec<Term> = quoted
         .split_whitespace()
         .map(|word| Term::from_field_text(schema.text, word))
@@ -75,18 +81,18 @@ pub fn parse_quoted(schema: &ParagraphSchema, quoted: &str) -> Box<dyn Query> {
     if terms.len() == 1 {
         // phrase queries must have more than one term, so we use a term query
         let term = terms.remove(0); // safe because terms.len() == 1
-        Box::new(TermQuery::new(term, IndexRecordOption::Basic))
+        Ok(Box::new(TermQuery::new(term, IndexRecordOption::Basic)))
     } else if terms.len() > 1 {
-        Box::new(PhraseQuery::new(terms))
+        Ok(Box::new(PhraseQuery::new(terms)))
     } else {
+        #[cfg(not(test))] // we don't want an assert on tests, we want the error
         debug_assert!(
             false,
             "Quoted content should have been validated to not only contain whitespaces"
         );
-        // we return a fallback to protect us from tokenizer errors, but this branch should never
+        // we return an error to protect us from tokenizer errors, but this branch should never
         // happen
-        error!("Keyword tokenizer build a query with a only whitespaces Quoted token!");
-        Box::new(EmptyQuery)
+        Err("Keyword tokenizer built a query with a only whitespaces Quoted token!".to_string())
     }
 }
 
@@ -111,7 +117,7 @@ mod tests {
     #[test]
     fn test_empty_query_is_all_query() {
         let schema = ParagraphSchema::new();
-        let query = parse_keyword_query(&[], &schema);
+        let query = parse_keyword_query(&[], &schema).unwrap();
         assert!(query.is::<AllQuery>());
     }
 
@@ -119,7 +125,7 @@ mod tests {
     fn test_one_clause_simplification() {
         let schema = ParagraphSchema::new();
 
-        let query = parse_keyword_query(&[Token::Literal("nucliadb".into())], &schema);
+        let query = parse_keyword_query(&[Token::Literal("nucliadb".into())], &schema).unwrap();
         let term = extract_term_from(&query);
         assert_eq!(*term, Term::from_field_text(schema.text, "nucliadb"));
     }
@@ -137,7 +143,7 @@ mod tests {
             Token::Literal("with".into()),
             Token::Quoted("superpowers".into()),
         ];
-        let r = downcast_boolean_query(parse_keyword_query(&query, &schema));
+        let r = downcast_boolean_query(parse_keyword_query(&query, &schema).unwrap());
         let clauses = r.clauses();
         assert_eq!(clauses.len(), 6);
 
@@ -188,6 +194,19 @@ mod tests {
         assert_eq!(*term, Term::from_field_text(schema.text, "superpowers"));
     }
 
+    #[test]
+    /// The tokenizer should not allow an empty (whitespaces) Quoted token
+    fn test_parse_quoted_error() {
+        let schema = ParagraphSchema::new();
+
+        let quoted = "   ";
+        let r = parse_quoted(&schema, quoted);
+        assert!(r.is_err());
+
+        let r = parse_keyword_query(&[Token::Quoted("   ".into())], &schema);
+        assert!(r.is_err());
+    }
+
     fn downcast_boolean_query(q: Box<dyn Query>) -> Box<BooleanQuery> {
         let q = q.downcast::<BooleanQuery>();
         assert!(q.is_ok(), "BooleanQuery expected");

{nidx_binding-6.7.2.post490 → nidx_binding-6.7.2.post495}/nidx_paragraph/src/query_parser/tokenizer.rs

@@ -165,7 +165,14 @@ fn tantivy_retokenize(tokens: Vec<Token>) -> Vec<Token> {
                     let token = stream.token();
                     tokens.push(token.text.clone())
                 }
-                vec![Token::Quoted(Cow::Owned(tokens.into_iter().join(" ")))]
+                let text = tokens.into_iter().join(" ");
+                // as with the nom tokenizer, empty quotes make no sense so we
+                // get rid of them
+                if !text.is_empty() {
+                    vec![Token::Quoted(Cow::Owned(text))]
+                } else {
+                    vec![]
+                }
             }
             Token::Excluded(Cow::Borrowed(excluded)) => {
                 let mut stream = tantivy_tokenizer.token_stream(excluded);
@@ -331,6 +338,12 @@ mod tests {
         ];
         let tokens = tokenize_query_infallible(query);
         assert_eq!(tokens, expected);
+
+        // quoted strings with special characters that tantivy tokenizer removes are removed
+        let query = r#"tantivy deletes "...""#;
+        let expected = vec![Literal("tantivy".into()), Literal("deletes".into())];
+        let tokens = tokenize_query_infallible(query);
+        assert_eq!(tokens, expected);
     }
 
     #[test]

{nidx_binding-6.7.2.post490 → nidx_binding-6.7.2.post495}/nidx_paragraph/src/query_parser.rs

@@ -24,6 +24,7 @@ mod tokenizer;
 
 use tantivy::query::Query;
 use tokenizer::{Token, tokenize_query_infallible};
+use tracing::error;
 
 use crate::{
     schema::ParagraphSchema,
@@ -33,6 +34,10 @@ use crate::{
 pub use fuzzy_parser::FUZZY_DISTANCE;
 use stop_words::remove_stop_words;
 
+/// Alias to make clippy happier. A FallbackQuery is no more than a query with a list of errors
+/// found while parsing
+pub type FallbackQuery = (Box<dyn Query>, Vec<String>);
+
 /// `Parser` is the nidx keyword grammar query parser
 ///
 /// It allows some behavior configuration and query parsing capabilities. Query
@@ -82,12 +87,29 @@ impl<'a> Parser<'a> {
         }
         let shared_term_collector = SharedTermC::from(term_collector);
 
-        let keyword = keyword_parser::parse_keyword_query(&tokenized, self.schema);
-        let fuzzy = fuzzy_parser::parse_fuzzy_query(
+        let keyword = match keyword_parser::parse_keyword_query(&tokenized, self.schema) {
+            Ok(q) => q,
+            Err((q, errors)) => {
+                for err in errors {
+                    error!(?query, parser = "keyword", err);
+                }
+                q
+            }
+        };
+
+        let fuzzy = match fuzzy_parser::parse_fuzzy_query(
             &tokenized,
             shared_term_collector.clone(),
             self.last_fuzzy_term_as_prefix,
-        );
+        ) {
+            Ok(q) => q,
+            Err((q, errors)) => {
+                for err in errors {
+                    error!(?query, parser = "fuzzy", err);
+                }
+                q
+            }
+        };
 
         ParsedQuery {
             keyword,
@@ -115,6 +137,16 @@ mod tests {
                 // keeps last term even if is a stop word
                 "nuclia database the",
             ),
+            (
+                "nuclia \"is\" a database for the",
+                // keeps last term even if is a stop word
+                "nuclia \"is\" database the",
+            ),
+            (
+                "nuclia \"...\" a database for the",
+                // keeps last term even if is a stop word
+                "nuclia database the",
+            ),
             ("is a for and", "and"),
             ("what does stop is?", "stop is"),
             ("", ""),

{nidx_binding-6.7.2.post490 → nidx_binding-6.7.2.post495}/nidx_protos/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.7.2.post490"
+version = "6.7.2.post495"
 license = "AGPL-3.0-or-later"
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [{ name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.7.2.post490 → nidx_binding-6.7.2.post495}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.7.2.post490"
+version = "6.7.2.post495"
 
 [project.urls]
 Homepage = "https://nuclia.com"