nidx-binding 6.3.1.post23__tar.gz → 6.3.1.post33__tar.gz

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/Cargo.toml

@@ -2,6 +2,7 @@
 name = "nidx"
 version = "0.1.0"
 edition = "2021"
+license = "AGPL-3.0"
 
 [[bin]]
 name = "nidx"

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.1.post23
+Version: 6.3.1.post33
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/nidx_binding/Cargo.toml

@@ -2,6 +2,7 @@
 name = "nidx_binding"
 version = "0.1.0"
 edition = "2021"
+license = "AGPL-3.0"
 
 [lib]
 name = "nidx_binding"

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/nidx_paragraph/Cargo.toml

@@ -2,6 +2,7 @@
 name = "nidx_paragraph"
 version = "0.1.0"
 edition = "2021"
+license = "AGPL-3.0"
 
 [dependencies]
 anyhow = "1.0.92"

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/nidx_protos/Cargo.toml

@@ -3,6 +3,7 @@ name = "nidx_protos"
 version = "0.1.0"
 edition = "2021"
 build = "build.rs"
+license = "AGPL-3.0"
 
 [build-dependencies]
 tonic-build = "0.12.3"

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.1.post23"
+version = "6.3.1.post33"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/nidx_relation/Cargo.toml

@@ -2,6 +2,7 @@
 name = "nidx_relation"
 version = "0.1.0"
 edition = "2021"
+license = "AGPL-3.0"
 
 [dependencies]
 anyhow = "1.0.92"

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/nidx_tantivy/Cargo.toml

@@ -2,8 +2,7 @@
 name = "nidx_tantivy"
 version = "0.1.0"
 edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+license = "AGPL-3.0"
 
 [dependencies]
 anyhow = "1.0.91"

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/nidx_tests/Cargo.toml

@@ -2,8 +2,7 @@
 name = "nidx_tests"
 version = "0.1.0"
 edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+license = "AGPL-3.0"
 
 [dependencies]
 nidx_protos = { version = "0.1.0", path = "../nidx_protos" }

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/nidx_text/Cargo.toml

@@ -2,6 +2,7 @@
 name = "nidx_text"
 version = "0.1.0"
 edition = "2021"
+license = "AGPL-3.0"
 
 [dependencies]
 tempfile = "3"

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/nidx_types/Cargo.toml

@@ -2,6 +2,7 @@
 name = "nidx_types"
 version = "0.1.0"
 edition = "2021"
+license = "AGPL-3.0"
 
 [dependencies]
 uuid = "1.13.1"

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/nidx_vector/Cargo.toml

@@ -2,6 +2,7 @@
 name = "nidx_vector"
 version = "0.1.0"
 edition = "2021"
+license = "AGPL-3.0"
 
 [dependencies]
 nidx_types = { version = "0.1.0", path = "../nidx_types" }

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.1.post23"
+version = "6.3.1.post33"
 
 [project.urls]
 Homepage = "https://nuclia.com"

nidx_binding-6.3.1.post33/src/searcher/query_language.rs

@@ -0,0 +1,123 @@
+// Copyright (C) 2021 Bosutech XXI S.L.
+//
+// nucliadb is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at info@nuclia.com.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+use nidx_protos::{
+    filter_expression::{Expr, FacetFilter, FilterExpressionList},
+    FilterExpression,
+};
+use nidx_types::query_language::*;
+
+/// Extract an expression only involving some labels if it's an AND subset of the total expression
+pub fn extract_label_filters(expression: &FilterExpression, labels: &[&str]) -> Option<BooleanExpression> {
+    match expression.expr.as_ref().unwrap() {
+        Expr::Facet(FacetFilter {
+            facet,
+        }) if labels.contains(&facet.as_str()) => Some(BooleanExpression::Literal(facet.clone())),
+        Expr::BoolNot(not_expr) => {
+            extract_label_filters(not_expr.as_ref(), labels).map(|e| BooleanExpression::Not(Box::new(e)))
+        }
+        Expr::BoolAnd(FilterExpressionList {
+            operands,
+        }) => {
+            let relevant: Vec<_> = operands.iter().filter_map(|e| extract_label_filters(e, labels)).collect();
+            match &relevant[..] {
+                [] => None,
+                [expression] => Some(expression.clone()),
+                _ => Some(BooleanExpression::Operation(BooleanOperation {
+                    operator: Operator::And,
+                    operands: relevant,
+                })),
+            }
+        }
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_extract_label_filters() {
+        const LABELS: &[&str] = &["/v", "/w"];
+
+        let a = FilterExpression {
+            expr: Some(Expr::Facet(FacetFilter {
+                facet: "/a".into(),
+            })),
+        };
+        let v = FilterExpression {
+            expr: Some(Expr::Facet(FacetFilter {
+                facet: "/v".into(),
+            })),
+        };
+        let w = FilterExpression {
+            expr: Some(Expr::Facet(FacetFilter {
+                facet: "/w".into(),
+            })),
+        };
+
+        let not = |e| FilterExpression {
+            expr: Some(Expr::BoolNot(Box::new(e))),
+        };
+
+        // Literal
+        assert_eq!(extract_label_filters(&a, LABELS), None);
+        assert_eq!(extract_label_filters(&v, LABELS), Some(BooleanExpression::Literal("/v".into())));
+
+        // Not literal
+        assert_eq!(extract_label_filters(&not(a.clone()), LABELS), None);
+
+        assert_eq!(
+            extract_label_filters(&not(v.clone()), LABELS),
+            Some(BooleanExpression::Not(Box::new(BooleanExpression::Literal("/v".into()))))
+        );
+
+        // Or (not supported)
+        let or_expr = FilterExpression {
+            expr: Some(Expr::BoolOr(FilterExpressionList {
+                operands: vec![a.clone(), v.clone(), not(w.clone())],
+            })),
+        };
+        assert_eq!(extract_label_filters(&or_expr, LABELS), None);
+
+        // And
+        let expr = FilterExpression {
+            expr: Some(Expr::BoolAnd(FilterExpressionList {
+                operands: vec![a.clone(), v.clone(), not(w.clone())],
+            })),
+        };
+        let expected = BooleanExpression::Operation(BooleanOperation {
+            operator: Operator::And,
+            operands: vec![
+                BooleanExpression::Literal("/v".into()),
+                BooleanExpression::Not(Box::new(BooleanExpression::Literal("/w".into()))),
+            ],
+        });
+        assert_eq!(extract_label_filters(&expr, LABELS), Some(expected));
+
+        // Nested
+        let expr = FilterExpression {
+            expr: Some(Expr::BoolAnd(FilterExpressionList {
+                operands: vec![a, v, or_expr],
+            })),
+        };
+        assert_eq!(extract_label_filters(&expr, LABELS), Some(BooleanExpression::Literal("/v".into())));
+    }
+}

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/src/searcher/query_planner.rs

@@ -27,12 +27,9 @@ use nidx_types::prefilter::PrefilterResult;
 use nidx_types::query_language::*;
 use nidx_vector::VectorSearchRequest;
 use nidx_vector::SEGMENT_TAGS;
-use old_filter_compatibility::filter_from_search_request;
 
 use super::query_language::extract_label_filters;
 
-pub mod old_filter_compatibility;
-
 /// The queries a [`QueryPlan`] has decided to send to each index.
 #[derive(Default, Clone)]
 pub struct IndexQueries {
@@ -67,19 +64,7 @@ pub struct QueryPlan {
     pub index_queries: IndexQueries,
 }
 
-pub fn build_query_plan(mut search_request: SearchRequest) -> anyhow::Result<QueryPlan> {
-    // Backwards compatibility with old filters
-    let (field_filter, paragraph_filter) = filter_from_search_request(&search_request)?;
-    let has_old_filters = field_filter.is_some() || paragraph_filter.is_some();
-    let has_new_filters = search_request.field_filter.is_some() || search_request.paragraph_filter.is_some();
-    if has_old_filters && has_new_filters {
-        return Err(anyhow::anyhow!("Cannot specify old and new filters in the same request"));
-    }
-    if has_old_filters {
-        search_request.field_filter = field_filter;
-        search_request.paragraph_filter = paragraph_filter;
-    }
-
+pub fn build_query_plan(search_request: SearchRequest) -> anyhow::Result<QueryPlan> {
     let relations_request = compute_relations_request(&search_request);
     let texts_request = compute_texts_request(&search_request);
     let vectors_request = compute_vectors_request(&search_request)?;
@@ -186,7 +171,6 @@ fn compute_relations_request(search_request: &SearchRequest) -> Option<RelationS
         shard_id: search_request.shard.clone(),
         prefix: search_request.relation_prefix.clone(),
         subgraph: search_request.relation_subgraph.clone(),
-        reload: search_request.reload,
     })
 }
 
@@ -222,29 +206,37 @@ pub fn filter_to_boolean_expression(filter: FilterExpression) -> anyhow::Result<
 
 #[cfg(test)]
 mod tests {
-    use nidx_protos::{filter_expression::FacetFilter, Filter};
+    use nidx_protos::filter_expression::{FacetFilter, FilterExpressionList};
 
     use super::*;
     #[test]
     fn proper_propagation() {
-        let expression = serde_json::json!({
-            "and": [
-                { "literal": "this" },
-                { "literal": "and" },
-                { "literal": "that"},
-            ],
-        });
         #[allow(deprecated)]
         let request = SearchRequest {
-            filter: Some(Filter {
-                field_labels: vec!["this".to_string()],
-                paragraph_labels: vec!["and".to_string(), "that".to_string()],
-                labels_expression: expression.to_string(),
-                keywords_expression: "".to_string(),
-            }),
             result_per_page: 10,
             vector: vec![1.0],
             paragraph: true,
+            field_filter: Some(FilterExpression {
+                expr: Some(Expr::Facet(FacetFilter {
+                    facet: "this".into(),
+                })),
+            }),
+            paragraph_filter: Some(FilterExpression {
+                expr: Some(Expr::BoolAnd(FilterExpressionList {
+                    operands: vec![
+                        FilterExpression {
+                            expr: Some(Expr::Facet(FacetFilter {
+                                facet: "and".into(),
+                            })),
+                        },
+                        FilterExpression {
+                            expr: Some(Expr::Facet(FacetFilter {
+                                facet: "that".into(),
+                            })),
+                        },
+                    ],
+                })),
+            }),
             ..Default::default()
         };
         let query_plan = build_query_plan(request).unwrap();

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/src/searcher/shard_suggest.rs

@@ -29,10 +29,7 @@ use tracing::{instrument, Span};
 
 use crate::errors::{NidxError, NidxResult};
 
-use super::{
-    index_cache::IndexCache,
-    query_planner::{filter_to_boolean_expression, old_filter_compatibility::filter_from_suggest_request},
-};
+use super::{index_cache::IndexCache, query_planner::filter_to_boolean_expression};
 
 /// Max number of words accepted as a suggest query. This is useful for
 /// compounds with semantic meaning (like a name and a surname) but can add
@@ -99,18 +96,7 @@ fn blocking_suggest(
 
     let mut paragraph_request = None;
     if suggest_paragraphs {
-        let filter_expression = filter_from_suggest_request(&request)?;
-        if let Some(expr) = filter_expression {
-            if request.field_filter.is_some() {
-                return Err(anyhow::anyhow!("Cannot specify old and new filters in the same request"));
-            }
-            let prefilter = PreFilterRequest {
-                security: None,
-                filter_expression: Some(expr),
-            };
-
-            prefiltered = text_searcher.prefilter(&prefilter)?;
-        } else if let Some(expr) = &request.field_filter {
+        if let Some(expr) = &request.field_filter {
             let prefilter = PreFilterRequest {
                 security: None,
                 filter_expression: Some(expr.clone()),

{nidx_binding-6.3.1.post23 → nidx_binding-6.3.1.post33}/tests/test_date_range_search.rs

@@ -23,11 +23,13 @@ mod common;
 use std::collections::HashMap;
 
 use common::services::NidxFixture;
+use nidx_protos::filter_expression::date_range_filter::DateField;
+use nidx_protos::filter_expression::{DateRangeFilter, Expr, FilterExpressionList};
 use nidx_protos::prost_types::Timestamp;
 use nidx_protos::resource::ResourceStatus;
 use nidx_protos::{
-    IndexMetadata, IndexParagraph, IndexParagraphs, NewShardRequest, Resource, ResourceId, SearchRequest,
-    TextInformation, Timestamps, VectorIndexConfig, VectorSentence, VectorsetSentences,
+    FilterExpression, IndexMetadata, IndexParagraph, IndexParagraphs, NewShardRequest, Resource, ResourceId,
+    SearchRequest, TextInformation, VectorIndexConfig, VectorSentence, VectorsetSentences,
 };
 use sqlx::PgPool;
 use tonic::Request;
@@ -138,7 +140,6 @@ async fn test_date_range_search(pool: PgPool) -> Result<(), Box<dyn std::error::
     let request = SearchRequest {
         shard: shard_id.clone(),
         order: None,
-        timestamps: None,
         vectorset: "english".to_string(),
         vector: vec![4.0, 6.0, 7.0],
         page_number: 0,
@@ -158,11 +159,25 @@ async fn test_date_range_search(pool: PgPool) -> Result<(), Box<dyn std::error::
 
     // Time range allows everything
     let mut request_all_range = request.clone();
-    request_all_range.timestamps = Some(Timestamps {
-        from_modified: Some(base_time),
-        to_modified: None,
-        from_created: Some(base_time),
-        to_created: None,
+    request_all_range.field_filter = Some(FilterExpression {
+        expr: Some(Expr::BoolAnd(FilterExpressionList {
+            operands: vec![
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Modified.into(),
+                        since: Some(base_time),
+                        until: None,
+                    })),
+                },
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Created.into(),
+                        since: Some(base_time),
+                        until: None,
+                    })),
+                },
+            ],
+        })),
     });
     let result = fixture.searcher_client.search(request_all_range).await.unwrap();
     let result = result.into_inner();
@@ -171,11 +186,25 @@ async fn test_date_range_search(pool: PgPool) -> Result<(), Box<dyn std::error::
 
     // Time range allows only second batch
     let mut request_second_batch = request.clone();
-    request_second_batch.timestamps = Some(Timestamps {
-        from_modified: Some(base_time_plus_one),
-        to_modified: None,
-        from_created: Some(base_time_plus_one),
-        to_created: None,
+    request_second_batch.field_filter = Some(FilterExpression {
+        expr: Some(Expr::BoolAnd(FilterExpressionList {
+            operands: vec![
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Modified.into(),
+                        since: Some(base_time_plus_one),
+                        until: None,
+                    })),
+                },
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Created.into(),
+                        since: Some(base_time_plus_one),
+                        until: None,
+                    })),
+                },
+            ],
+        })),
     });
     let result = fixture.searcher_client.search(request_second_batch).await.unwrap();
     let result = result.into_inner();
@@ -184,11 +213,16 @@ async fn test_date_range_search(pool: PgPool) -> Result<(), Box<dyn std::error::
 
     // Time range allows only second batch, but with modified only
     let mut request_second_batch = request.clone();
-    request_second_batch.timestamps = Some(Timestamps {
-        from_modified: Some(base_time_plus_one),
-        to_modified: None,
-        from_created: None,
-        to_created: None,
+    request_second_batch.field_filter = Some(FilterExpression {
+        expr: Some(Expr::BoolAnd(FilterExpressionList {
+            operands: vec![FilterExpression {
+                expr: Some(Expr::Date(DateRangeFilter {
+                    field: DateField::Modified.into(),
+                    since: Some(base_time_plus_one),
+                    until: None,
+                })),
+            }],
+        })),
     });
     let result = fixture.searcher_client.search(request_second_batch).await.unwrap();
     let result = result.into_inner();
@@ -200,11 +234,25 @@ async fn test_date_range_search(pool: PgPool) -> Result<(), Box<dyn std::error::
 
     // Time range does not match any field, therefore the response has no results
     let mut request_second_batch = request.clone();
-    request_second_batch.timestamps = Some(Timestamps {
-        from_modified: Some(base_time_plus_two),
-        to_modified: None,
-        from_created: Some(base_time_plus_two),
-        to_created: None,
+    request_second_batch.field_filter = Some(FilterExpression {
+        expr: Some(Expr::BoolAnd(FilterExpressionList {
+            operands: vec![
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Modified.into(),
+                        since: Some(base_time_plus_two),
+                        until: None,
+                    })),
+                },
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Created.into(),
+                        since: Some(base_time_plus_two),
+                        until: None,
+                    })),
+                },
+            ],
+        })),
     });
     let result = fixture.searcher_client.search(request_second_batch).await.unwrap();
     let result = result.into_inner();
@@ -215,11 +263,25 @@ async fn test_date_range_search(pool: PgPool) -> Result<(), Box<dyn std::error::
 
     // Multiple timestamps are parsed as AND conditions
     let mut request_second_batch = request.clone();
-    request_second_batch.timestamps = Some(Timestamps {
-        from_modified: Some(base_time_plus_one),
-        to_modified: None,
-        from_created: Some(base_time_plus_two),
-        to_created: None,
+    request_second_batch.field_filter = Some(FilterExpression {
+        expr: Some(Expr::BoolAnd(FilterExpressionList {
+            operands: vec![
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Modified.into(),
+                        since: Some(base_time_plus_one),
+                        until: None,
+                    })),
+                },
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Created.into(),
+                        since: Some(base_time_plus_two),
+                        until: None,
+                    })),
+                },
+            ],
+        })),
     });
     let result = fixture.searcher_client.search(request_second_batch).await.unwrap();
     let result = result.into_inner();