nidx-binding 6.7.2.post493__tar.gz → 6.7.2.post495__tar.gz

{nidx_binding-6.7.2.post493 → nidx_binding-6.7.2.post495}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.7.2.post493
+Version: 6.7.2.post495
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.7.2.post493 → nidx_binding-6.7.2.post495}/nidx_paragraph/src/query_parser/tokenizer.rs

@@ -165,7 +165,14 @@ fn tantivy_retokenize(tokens: Vec<Token>) -> Vec<Token> {
                     let token = stream.token();
                     tokens.push(token.text.clone())
                 }
-                vec![Token::Quoted(Cow::Owned(tokens.into_iter().join(" ")))]
+                let text = tokens.into_iter().join(" ");
+                // as with the nom tokenizer, empty quotes make no sense so we
+                // get rid of them
+                if !text.is_empty() {
+                    vec![Token::Quoted(Cow::Owned(text))]
+                } else {
+                    vec![]
+                }
             }
             Token::Excluded(Cow::Borrowed(excluded)) => {
                 let mut stream = tantivy_tokenizer.token_stream(excluded);
@@ -331,6 +338,12 @@ mod tests {
         ];
         let tokens = tokenize_query_infallible(query);
         assert_eq!(tokens, expected);
+
+        // quoted strings with special characters that tantivy tokenizer removes are removed
+        let query = r#"tantivy deletes "...""#;
+        let expected = vec![Literal("tantivy".into()), Literal("deletes".into())];
+        let tokens = tokenize_query_infallible(query);
+        assert_eq!(tokens, expected);
     }
 
     #[test]

{nidx_binding-6.7.2.post493 → nidx_binding-6.7.2.post495}/nidx_paragraph/src/query_parser.rs

@@ -137,6 +137,16 @@ mod tests {
                 // keeps last term even if is a stop word
                 "nuclia database the",
             ),
+            (
+                "nuclia \"is\" a database for the",
+                // keeps last term even if is a stop word
+                "nuclia \"is\" database the",
+            ),
+            (
+                "nuclia \"...\" a database for the",
+                // keeps last term even if is a stop word
+                "nuclia database the",
+            ),
             ("is a for and", "and"),
             ("what does stop is?", "stop is"),
             ("", ""),

{nidx_binding-6.7.2.post493 → nidx_binding-6.7.2.post495}/nidx_protos/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.7.2.post493"
+version = "6.7.2.post495"
 license = "AGPL-3.0-or-later"
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [{ name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.7.2.post493 → nidx_binding-6.7.2.post495}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.7.2.post493"
+version = "6.7.2.post495"
 
 [project.urls]
 Homepage = "https://nuclia.com"