nidx-binding 6.7.1.post486__tar.gz → 6.7.1.post489__tar.gz

{nidx_binding-6.7.1.post486 → nidx_binding-6.7.1.post489}/Cargo.lock

@@ -2193,6 +2193,7 @@ dependencies = [
  "nidx_protos",
  "nidx_tantivy",
  "nidx_types",
+ "nom",
  "once_cell",
  "regex",
  "serde_json",

{nidx_binding-6.7.1.post486 → nidx_binding-6.7.1.post489}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.7.1.post486
+Version: 6.7.1.post489
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.7.1.post486 → nidx_binding-6.7.1.post489}/nidx_paragraph/Cargo.toml

@@ -12,6 +12,7 @@ levenshtein_automata = "0.2.1"
 nidx_protos = { version = "0.1.0", path = "../nidx_protos" }
 nidx_tantivy = { version = "0.1.0", path = "../nidx_tantivy" }
 nidx_types = { version = "0.1.0", path = "../nidx_types" }
+nom = "7" # same version as used by tantivy
 once_cell = "1.20.2"
 regex = "1.11.1"
 serde_json = "1.0.132"

{nidx_binding-6.7.1.post486 → nidx_binding-6.7.1.post489}/nidx_paragraph/src/fuzzy_query.rs

@@ -184,7 +184,12 @@ pub struct FuzzyTermQuery {
 
 impl std::fmt::Debug for FuzzyTermQuery {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_str("Fuzzy")
+        f.write_fmt(format_args!(
+            "Fuzzy {{ term: {}, distance: {}, prefix: {} }}",
+            self.term.value().as_str().unwrap_or("<unknown>"),
+            self.distance,
+            self.prefix
+        ))
     }
 }
 impl FuzzyTermQuery {
@@ -210,6 +215,11 @@ impl FuzzyTermQuery {
         }
     }
 
+    #[cfg(test)]
+    pub fn is_prefix(&self) -> bool {
+        self.prefix
+    }
+
     fn specialized_weight(&self) -> tantivy::Result<AutomatonWeight<DfaWrapper>> {
         // LEV_BUILDER is a HashMap, whose `get` method returns an Option
         match LEV_BUILDER.get(&(self.distance, self.transposition_cost_one)) {

{nidx_binding-6.7.1.post486 → nidx_binding-6.7.1.post489}/nidx_paragraph/src/lib.rs

@@ -20,6 +20,7 @@
 
 mod fuzzy_query;
 mod query_io;
+mod query_parser;
 mod reader;
 mod request_types;
 mod resource_indexer;
@@ -27,7 +28,6 @@ mod schema;
 mod search_query;
 mod search_response;
 mod set_query;
-mod stop_words;
 
 use nidx_protos::{ParagraphItem, ParagraphSearchResponse, StreamRequest};
 use nidx_tantivy::{

nidx_binding-6.7.1.post489/nidx_paragraph/src/query_parser/fuzzy_parser.rs

@@ -0,0 +1,192 @@
+// Copyright (C) 2021 Bosutech XXI S.L.
+//
+// nucliadb is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at info@nuclia.com.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+//
+
+use tantivy::Term;
+use tantivy::query::AllQuery;
+use tantivy::query::BooleanQuery;
+use tantivy::query::Occur;
+use tantivy::query::Query;
+use tantivy::query::TermQuery;
+use tantivy::schema::IndexRecordOption;
+
+use crate::fuzzy_query::FuzzyTermQuery;
+use crate::schema::ParagraphSchema;
+use crate::search_query::SharedTermC;
+
+use super::keyword_parser;
+use super::tokenizer::Token;
+
+/// Minimum length required to be considered a fuzzy word. Words with smaller
+/// length than this are considered too short to be fuzzy. This is done to avoid
+/// too much noise from short terms.
+const MIN_FUZZY_LEN: usize = 3;
+
+/// Minimum length required to be considered a fuzzy prefix. This is again don
+/// eto avoid too much noise from short term.
+const MIN_FUZZY_PREFIX_LEN: usize = 4;
+
+/// Levenshtein distance used by all fuzzy terms.
+pub const FUZZY_DISTANCE: u8 = 1;
+
+/// Convert a tokenized query into a tantivy fuzzy query
+///
+/// A fuzzy query will match similarly to a keyword query but some terms will be
+/// searched with certain Levenshtein distance. Quoted and excluded terms will
+/// remain the same, but literals will be elegible to be fuzzy. All long enough
+/// literals will be searched as fuzzy. The last literal can be searched as a
+/// fuzzy prefix if a suggest-like feature is desired.
+///
+pub fn parse_fuzzy_query(query: &[Token], term_collector: SharedTermC, last_literal_as_prefix: bool) -> Box<dyn Query> {
+    let last_literal_index = if last_literal_as_prefix {
+        query.iter().rposition(|token| matches!(token, Token::Literal(_)))
+    } else {
+        None
+    };
+
+    let mut subqueries = vec![];
+    let schema = ParagraphSchema::new();
+
+    for (i, item) in query.iter().enumerate() {
+        match item {
+            Token::Literal(literal) => {
+                let term = Term::from_field_text(schema.text, literal);
+                let distance = FUZZY_DISTANCE;
+                let transposition_cost_one = true;
+
+                let q: Box<dyn Query>;
+                if literal.len() < MIN_FUZZY_LEN {
+                    // to avoid noise, we don't want to match too short terms as fuzzy
+                    q = Box::new(TermQuery::new(term, IndexRecordOption::Basic));
+                } else if matches!(last_literal_index, Some(idx) if idx == i) && literal.len() >= MIN_FUZZY_PREFIX_LEN {
+                    q = Box::new(FuzzyTermQuery::new_prefix(
+                        term,
+                        distance,
+                        transposition_cost_one,
+                        term_collector.clone(),
+                    ));
+                } else {
+                    q = Box::new(FuzzyTermQuery::new(
+                        term,
+                        distance,
+                        transposition_cost_one,
+                        term_collector.clone(),
+                    ));
+                }
+                subqueries.push((Occur::Should, q));
+            }
+            Token::Quoted(quoted) => {
+                let q = keyword_parser::parse_quoted(&schema, quoted);
+                subqueries.push((Occur::Should, q));
+            }
+            Token::Excluded(excluded) => {
+                let q = keyword_parser::parse_excluded(&schema, excluded);
+                subqueries.push((Occur::Should, q));
+            }
+        }
+    }
+
+    if subqueries.is_empty() {
+        Box::new(AllQuery)
+    } else if subqueries.len() == 1 {
+        subqueries.pop().unwrap().1
+    } else {
+        Box::new(BooleanQuery::new(subqueries))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::search_query::TermCollector;
+
+    use super::*;
+
+    #[test]
+    fn test_short_literals_do_not_fuzzy() {
+        let term_collector = SharedTermC::from(TermCollector::new());
+
+        // literal shorter than MIN_FUZZY_LEN will become a TermQuery
+        let literal = "ab";
+        assert!(literal.len() < MIN_FUZZY_LEN);
+        let query = [Token::Literal(literal.into())];
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), false);
+        assert!(fuzzy.is::<TermQuery>());
+        let q = fuzzy.downcast::<TermQuery>().unwrap();
+        assert_eq!(q.term().value().as_str(), Some(literal));
+    }
+
+    #[test]
+    fn test_fuzzy_literals() {
+        let term_collector = SharedTermC::from(TermCollector::new());
+
+        let literal = "abcd";
+        assert!(literal.len() >= MIN_FUZZY_LEN);
+        let query = [Token::Literal(literal.into())];
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), false);
+        assert!(fuzzy.is::<FuzzyTermQuery>());
+        assert!(!fuzzy.downcast::<FuzzyTermQuery>().unwrap().is_prefix());
+    }
+
+    #[test]
+    fn test_fuzzy_prefix() {
+        let term_collector = SharedTermC::from(TermCollector::new());
+
+        // literals longer than the min fuzzy prefix become prefix if they are
+        // last and the flag is enabled
+
+        let literal = "abcd";
+        assert!(literal.len() >= MIN_FUZZY_PREFIX_LEN);
+        let query = [Token::Literal(literal.into())];
+
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), false);
+        assert!(fuzzy.is::<FuzzyTermQuery>());
+        assert!(!fuzzy.downcast::<FuzzyTermQuery>().unwrap().is_prefix());
+
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), true);
+        assert!(fuzzy.is::<FuzzyTermQuery>());
+        assert!(fuzzy.downcast::<FuzzyTermQuery>().unwrap().is_prefix());
+
+        // only the last term is fuzzy prefix
+
+        let query = [Token::Literal(literal.into()), Token::Literal(literal.into())];
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), true);
+        assert!(fuzzy.is::<BooleanQuery>());
+        let q = fuzzy.downcast::<BooleanQuery>().unwrap();
+        let clauses = q.clauses();
+        assert_eq!(clauses.len(), 2);
+        assert!(clauses[0].1.is::<FuzzyTermQuery>());
+        assert!(!clauses[0].1.downcast_ref::<FuzzyTermQuery>().unwrap().is_prefix());
+        assert!(clauses[1].1.is::<FuzzyTermQuery>());
+        assert!(clauses[1].1.downcast_ref::<FuzzyTermQuery>().unwrap().is_prefix());
+
+        // however, shorter terms won't become prefix
+
+        let literal = "abc";
+        assert!(literal.len() < MIN_FUZZY_PREFIX_LEN);
+        let query = [Token::Literal(literal.into())];
+
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), false);
+        assert!(fuzzy.is::<FuzzyTermQuery>());
+        assert!(!fuzzy.downcast::<FuzzyTermQuery>().unwrap().is_prefix());
+
+        let fuzzy = parse_fuzzy_query(&query, term_collector.clone(), true);
+        assert!(fuzzy.is::<FuzzyTermQuery>());
+        assert!(!fuzzy.downcast::<FuzzyTermQuery>().unwrap().is_prefix());
+    }
+}

nidx_binding-6.7.1.post489/nidx_paragraph/src/query_parser/keyword_parser.rs

@@ -0,0 +1,212 @@
+// Copyright (C) 2021 Bosutech XXI S.L.
+//
+// nucliadb is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at info@nuclia.com.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+//
+use tantivy::Term;
+use tantivy::query::{AllQuery, BooleanQuery, EmptyQuery, Occur, PhraseQuery, Query, TermQuery};
+use tantivy::schema::IndexRecordOption;
+use tracing::error;
+
+use crate::ParagraphSchema;
+
+use super::tokenizer::Token;
+
+/// Convert a tokenized query into a tantivy keyword query
+///
+/// Empty queries will match everything.
+pub fn parse_keyword_query<'a>(query: &'a [Token<'a>], schema: &ParagraphSchema) -> Box<dyn Query> {
+    let mut subqueries = vec![];
+    for item in query {
+        match item {
+            Token::Literal(literal) => {
+                let q = parse_literal(schema, literal);
+                subqueries.push((Occur::Should, q));
+            }
+            Token::Quoted(quoted) => {
+                let q = parse_quoted(schema, quoted);
+                subqueries.push((Occur::Should, q));
+            }
+            Token::Excluded(excluded) => {
+                let q = parse_excluded(schema, excluded);
+                subqueries.push((Occur::Should, q));
+            }
+        }
+    }
+
+    if subqueries.is_empty() {
+        Box::new(AllQuery)
+    } else if subqueries.len() == 1 {
+        subqueries.pop().unwrap().1
+    } else {
+        Box::new(BooleanQuery::new(subqueries))
+    }
+}
+
+#[inline]
+pub fn parse_literal(schema: &ParagraphSchema, literal: &str) -> Box<dyn Query> {
+    Box::new(TermQuery::new(
+        Term::from_field_text(schema.text, literal),
+        IndexRecordOption::Basic,
+    ))
+}
+
+pub fn parse_quoted(schema: &ParagraphSchema, quoted: &str) -> Box<dyn Query> {
+    let mut terms: Vec<Term> = quoted
+        .split_whitespace()
+        .map(|word| Term::from_field_text(schema.text, word))
+        .collect();
+
+    #[allow(clippy::comparison_chain)]
+    if terms.len() == 1 {
+        // phrase queries must have more than one term, so we use a term query
+        let term = terms.remove(0); // safe because terms.len() == 1
+        Box::new(TermQuery::new(term, IndexRecordOption::Basic))
+    } else if terms.len() > 1 {
+        Box::new(PhraseQuery::new(terms))
+    } else {
+        debug_assert!(
+            false,
+            "Quoted content should have been validated to not only contain whitespaces"
+        );
+        // we return a fallback to protect us from tokenizer errors, but this branch should never
+        // happen
+        error!("Keyword tokenizer build a query with a only whitespaces Quoted token!");
+        Box::new(EmptyQuery)
+    }
+}
+
+#[inline]
+pub fn parse_excluded(schema: &ParagraphSchema, excluded: &str) -> Box<dyn Query> {
+    Box::new(BooleanQuery::new(vec![
+        (Occur::Must, Box::new(AllQuery)),
+        (
+            Occur::MustNot,
+            Box::new(TermQuery::new(
+                Term::from_field_text(schema.text, excluded),
+                IndexRecordOption::Basic,
+            )),
+        ),
+    ]))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_empty_query_is_all_query() {
+        let schema = ParagraphSchema::new();
+        let query = parse_keyword_query(&[], &schema);
+        assert!(query.is::<AllQuery>());
+    }
+
+    #[test]
+    fn test_one_clause_simplification() {
+        let schema = ParagraphSchema::new();
+
+        let query = parse_keyword_query(&[Token::Literal("nucliadb".into())], &schema);
+        let term = extract_term_from(&query);
+        assert_eq!(*term, Term::from_field_text(schema.text, "nucliadb"));
+    }
+
+    #[test]
+    fn test_tantivy_query_conversion() {
+        let schema = ParagraphSchema::new();
+
+        // nucliadb -is a "RAG database" with "superpowers"
+        let query = vec![
+            Token::Literal("nucliadb".into()),
+            Token::Excluded("is".into()),
+            Token::Literal("a".into()),
+            Token::Quoted("RAG database".into()),
+            Token::Literal("with".into()),
+            Token::Quoted("superpowers".into()),
+        ];
+        let r = downcast_boolean_query(parse_keyword_query(&query, &schema));
+        let clauses = r.clauses();
+        assert_eq!(clauses.len(), 6);
+
+        // term: nucliadb
+        assert_eq!(clauses[0].0, Occur::Should);
+        let term = extract_term_from(&clauses[0].1);
+        assert_eq!(*term, Term::from_field_text(schema.text, "nucliadb"));
+
+        // excluded term: is
+        assert_eq!(clauses[1].0, Occur::Should);
+        let subquery = clauses[1]
+            .1
+            .downcast_ref::<BooleanQuery>()
+            .expect("BooleanQuery expected");
+        let subclauses = subquery.clauses();
+        assert_eq!(subclauses.len(), 2);
+        assert_eq!(subclauses[0].0, Occur::Must);
+        assert!(subclauses[0].1.downcast_ref::<AllQuery>().is_some());
+        assert_eq!(subclauses[1].0, Occur::MustNot);
+        let term = extract_term_from(&subclauses[1].1);
+        assert_eq!(*term, Term::from_field_text(schema.text, "is"));
+
+        // term: a
+        assert_eq!(clauses[2].0, Occur::Should);
+        let term = extract_term_from(&clauses[2].1);
+        assert_eq!(*term, Term::from_field_text(schema.text, "a"));
+
+        // exact: RAG database
+        assert_eq!(clauses[3].0, Occur::Should);
+        let terms = extract_phrase_terms_from(&clauses[3].1);
+        assert_eq!(
+            *terms,
+            vec![
+                Term::from_field_text(schema.text, "RAG"),
+                Term::from_field_text(schema.text, "database"),
+            ]
+        );
+
+        // term: with
+        assert_eq!(clauses[4].0, Occur::Should);
+        let term = extract_term_from(&clauses[4].1);
+        assert_eq!(*term, Term::from_field_text(schema.text, "with"));
+
+        // exact: superpowers.
+        // An exact match of 1 word is converted into a term query
+        assert_eq!(clauses[5].0, Occur::Should);
+        let term = extract_term_from(&clauses[5].1);
+        assert_eq!(*term, Term::from_field_text(schema.text, "superpowers"));
+    }
+
+    fn downcast_boolean_query(q: Box<dyn Query>) -> Box<BooleanQuery> {
+        let q = q.downcast::<BooleanQuery>();
+        assert!(q.is_ok(), "BooleanQuery expected");
+        q.unwrap()
+    }
+
+    #[allow(clippy::borrowed_box)]
+    fn extract_term_from(query: &Box<dyn Query>) -> &Term {
+        let q = query.downcast_ref::<TermQuery>();
+        assert!(q.is_some(), "TermQuery expected");
+        let q = q.unwrap();
+        q.term()
+    }
+
+    #[allow(clippy::borrowed_box)]
+    fn extract_phrase_terms_from(query: &Box<dyn Query>) -> Vec<Term> {
+        let q = query.downcast_ref::<PhraseQuery>();
+        assert!(q.is_some(), "PhraseQuery expected");
+        let q = q.unwrap();
+        q.phrase_terms()
+    }
+}

{nidx_binding-6.7.1.post486/nidx_paragraph/src → nidx_binding-6.7.1.post489/nidx_paragraph/src/query_parser}/stop_words.rs

@@ -22,6 +22,8 @@ use std::env;
 
 use lazy_static::lazy_static;
 
+use super::Token;
+
 lazy_static! {
     static ref STOP_WORDS: StopWords = build_stop_words();
 }
@@ -90,6 +92,37 @@ pub fn is_stop_word(word: &str) -> bool {
     STOP_WORDS.is_stop_word(word)
 }
 
+/// Consume a tokenized query and filter out any stop words from it.
+///
+/// A stop words is any literal matching the following criteria:
+/// - It is present in the given list of stop words
+/// - Is **not** the last term in the query
+///
+/// The last term of the query can be used as fuzzy prefix and must be preserved.
+pub fn remove_stop_words(mut query: Vec<Token>) -> Vec<Token> {
+    if query.is_empty() {
+        return query;
+    }
+    // as we don't want to remove the last literal, we pop the last element and
+    // always put it in the filtered query. We don't really care if the last
+    // it's a literal or not.
+    let last: Token<'_> = query.pop().unwrap(); // safe as query is not empty
+
+    query
+        .into_iter()
+        .filter(|token| !is_stop_word_token(token))
+        .chain([last])
+        .collect()
+}
+
+fn is_stop_word_token(token: &Token) -> bool {
+    if let Token::Literal(lit) = token {
+        is_stop_word(&lit.to_lowercase())
+    } else {
+        false
+    }
+}
+
 #[cfg(test)]
 mod tests {
 
@@ -122,4 +155,78 @@ mod tests {
             assert!(elapsed < 1000.0, "{}", elapsed);
         }
     }
+
+    #[test]
+    fn test_stop_word_tokens() {
+        assert!(is_stop_word_token(&Token::Literal("is".into())));
+        assert!(!is_stop_word_token(&Token::Excluded("is".into())));
+        assert!(!is_stop_word_token(&Token::Quoted("is".into())));
+
+        assert!(!is_stop_word_token(&Token::Literal("music".into())));
+        assert!(!is_stop_word_token(&Token::Excluded("music".into())));
+        assert!(!is_stop_word_token(&Token::Quoted("music".into())));
+    }
+
+    #[test]
+    fn test_stop_words_detection_is_case_insensitive() {
+        assert!(is_stop_word_token(&Token::Literal("is".into())));
+        assert!(is_stop_word_token(&Token::Literal("IS".into())));
+    }
+
+    #[test]
+    fn test_token_with_stop_word_detection() {
+        // Stop word in the middle is removed
+        let tokens = vec![
+            Token::Literal("music".into()),
+            Token::Literal("is".into()),
+            Token::Literal("classical".into()),
+        ];
+        let filtered = remove_stop_words(tokens);
+        assert_eq!(
+            filtered,
+            vec![Token::Literal("music".into()), Token::Literal("classical".into()),]
+        );
+
+        // Only literals are stop words
+        let tokens = vec![
+            Token::Literal("music".into()),
+            Token::Quoted("is".into()),
+            Token::Literal("classical".into()),
+        ];
+        let filtered = remove_stop_words(tokens);
+        assert_eq!(
+            filtered,
+            vec![
+                Token::Literal("music".into()),
+                Token::Quoted("is".into()),
+                Token::Literal("classical".into()),
+            ]
+        );
+
+        // Stop words at the end are not removed
+        let tokens = vec![
+            Token::Literal("classical".into()),
+            Token::Literal("music".into()),
+            Token::Literal("is".into()),
+        ];
+        let filtered = remove_stop_words(tokens);
+        assert_eq!(
+            filtered,
+            vec![
+                Token::Literal("classical".into()),
+                Token::Literal("music".into()),
+                Token::Literal("is".into()),
+            ]
+        );
+
+        // Everything is a stop word, but last is not removed
+        let tokens = vec![
+            Token::Literal("we".into()),
+            Token::Literal("shouldn't".into()),
+            Token::Literal("be".into()),
+            Token::Literal("here".into()),
+        ];
+        let filtered = remove_stop_words(tokens);
+        assert_eq!(filtered, vec![Token::Literal("here".into()),]);
+    }
 }