nidx-binding 6.3.1.post20__tar.gz → 6.3.1.post26__tar.gz

{nidx_binding-6.3.1.post20 → nidx_binding-6.3.1.post26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.1.post20
+Version: 6.3.1.post26
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.1.post20 → nidx_binding-6.3.1.post26}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.1.post20"
+version = "6.3.1.post26"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.1.post20 → nidx_binding-6.3.1.post26}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.1.post20"
+version = "6.3.1.post26"
 
 [project.urls]
 Homepage = "https://nuclia.com"

nidx_binding-6.3.1.post26/src/searcher/query_language.rs

@@ -0,0 +1,123 @@
+// Copyright (C) 2021 Bosutech XXI S.L.
+//
+// nucliadb is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at info@nuclia.com.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+use nidx_protos::{
+    filter_expression::{Expr, FacetFilter, FilterExpressionList},
+    FilterExpression,
+};
+use nidx_types::query_language::*;
+
+/// Extract an expression only involving some labels if it's an AND subset of the total expression
+pub fn extract_label_filters(expression: &FilterExpression, labels: &[&str]) -> Option<BooleanExpression> {
+    match expression.expr.as_ref().unwrap() {
+        Expr::Facet(FacetFilter {
+            facet,
+        }) if labels.contains(&facet.as_str()) => Some(BooleanExpression::Literal(facet.clone())),
+        Expr::BoolNot(not_expr) => {
+            extract_label_filters(not_expr.as_ref(), labels).map(|e| BooleanExpression::Not(Box::new(e)))
+        }
+        Expr::BoolAnd(FilterExpressionList {
+            operands,
+        }) => {
+            let relevant: Vec<_> = operands.iter().filter_map(|e| extract_label_filters(e, labels)).collect();
+            match &relevant[..] {
+                [] => None,
+                [expression] => Some(expression.clone()),
+                _ => Some(BooleanExpression::Operation(BooleanOperation {
+                    operator: Operator::And,
+                    operands: relevant,
+                })),
+            }
+        }
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_extract_label_filters() {
+        const LABELS: &[&str] = &["/v", "/w"];
+
+        let a = FilterExpression {
+            expr: Some(Expr::Facet(FacetFilter {
+                facet: "/a".into(),
+            })),
+        };
+        let v = FilterExpression {
+            expr: Some(Expr::Facet(FacetFilter {
+                facet: "/v".into(),
+            })),
+        };
+        let w = FilterExpression {
+            expr: Some(Expr::Facet(FacetFilter {
+                facet: "/w".into(),
+            })),
+        };
+
+        let not = |e| FilterExpression {
+            expr: Some(Expr::BoolNot(Box::new(e))),
+        };
+
+        // Literal
+        assert_eq!(extract_label_filters(&a, LABELS), None);
+        assert_eq!(extract_label_filters(&v, LABELS), Some(BooleanExpression::Literal("/v".into())));
+
+        // Not literal
+        assert_eq!(extract_label_filters(&not(a.clone()), LABELS), None);
+
+        assert_eq!(
+            extract_label_filters(&not(v.clone()), LABELS),
+            Some(BooleanExpression::Not(Box::new(BooleanExpression::Literal("/v".into()))))
+        );
+
+        // Or (not supported)
+        let or_expr = FilterExpression {
+            expr: Some(Expr::BoolOr(FilterExpressionList {
+                operands: vec![a.clone(), v.clone(), not(w.clone())],
+            })),
+        };
+        assert_eq!(extract_label_filters(&or_expr, LABELS), None);
+
+        // And
+        let expr = FilterExpression {
+            expr: Some(Expr::BoolAnd(FilterExpressionList {
+                operands: vec![a.clone(), v.clone(), not(w.clone())],
+            })),
+        };
+        let expected = BooleanExpression::Operation(BooleanOperation {
+            operator: Operator::And,
+            operands: vec![
+                BooleanExpression::Literal("/v".into()),
+                BooleanExpression::Not(Box::new(BooleanExpression::Literal("/w".into()))),
+            ],
+        });
+        assert_eq!(extract_label_filters(&expr, LABELS), Some(expected));
+
+        // Nested
+        let expr = FilterExpression {
+            expr: Some(Expr::BoolAnd(FilterExpressionList {
+                operands: vec![a, v, or_expr],
+            })),
+        };
+        assert_eq!(extract_label_filters(&expr, LABELS), Some(BooleanExpression::Literal("/v".into())));
+    }
+}

{nidx_binding-6.3.1.post20 → nidx_binding-6.3.1.post26}/src/searcher/query_planner.rs

@@ -27,12 +27,9 @@ use nidx_types::prefilter::PrefilterResult;
 use nidx_types::query_language::*;
 use nidx_vector::VectorSearchRequest;
 use nidx_vector::SEGMENT_TAGS;
-use old_filter_compatibility::filter_from_search_request;
 
 use super::query_language::extract_label_filters;
 
-pub mod old_filter_compatibility;
-
 /// The queries a [`QueryPlan`] has decided to send to each index.
 #[derive(Default, Clone)]
 pub struct IndexQueries {
@@ -67,19 +64,7 @@ pub struct QueryPlan {
     pub index_queries: IndexQueries,
 }
 
-pub fn build_query_plan(mut search_request: SearchRequest) -> anyhow::Result<QueryPlan> {
-    // Backwards compatibility with old filters
-    let (field_filter, paragraph_filter) = filter_from_search_request(&search_request)?;
-    let has_old_filters = field_filter.is_some() || paragraph_filter.is_some();
-    let has_new_filters = search_request.field_filter.is_some() || search_request.paragraph_filter.is_some();
-    if has_old_filters && has_new_filters {
-        return Err(anyhow::anyhow!("Cannot specify old and new filters in the same request"));
-    }
-    if has_old_filters {
-        search_request.field_filter = field_filter;
-        search_request.paragraph_filter = paragraph_filter;
-    }
-
+pub fn build_query_plan(search_request: SearchRequest) -> anyhow::Result<QueryPlan> {
     let relations_request = compute_relations_request(&search_request);
     let texts_request = compute_texts_request(&search_request);
     let vectors_request = compute_vectors_request(&search_request)?;
@@ -186,7 +171,6 @@ fn compute_relations_request(search_request: &SearchRequest) -> Option<RelationS
         shard_id: search_request.shard.clone(),
         prefix: search_request.relation_prefix.clone(),
         subgraph: search_request.relation_subgraph.clone(),
-        reload: search_request.reload,
     })
 }
 
@@ -222,29 +206,37 @@ pub fn filter_to_boolean_expression(filter: FilterExpression) -> anyhow::Result<
 
 #[cfg(test)]
 mod tests {
-    use nidx_protos::{filter_expression::FacetFilter, Filter};
+    use nidx_protos::filter_expression::{FacetFilter, FilterExpressionList};
 
     use super::*;
     #[test]
     fn proper_propagation() {
-        let expression = serde_json::json!({
-            "and": [
-                { "literal": "this" },
-                { "literal": "and" },
-                { "literal": "that"},
-            ],
-        });
         #[allow(deprecated)]
         let request = SearchRequest {
-            filter: Some(Filter {
-                field_labels: vec!["this".to_string()],
-                paragraph_labels: vec!["and".to_string(), "that".to_string()],
-                labels_expression: expression.to_string(),
-                keywords_expression: "".to_string(),
-            }),
             result_per_page: 10,
             vector: vec![1.0],
             paragraph: true,
+            field_filter: Some(FilterExpression {
+                expr: Some(Expr::Facet(FacetFilter {
+                    facet: "this".into(),
+                })),
+            }),
+            paragraph_filter: Some(FilterExpression {
+                expr: Some(Expr::BoolAnd(FilterExpressionList {
+                    operands: vec![
+                        FilterExpression {
+                            expr: Some(Expr::Facet(FacetFilter {
+                                facet: "and".into(),
+                            })),
+                        },
+                        FilterExpression {
+                            expr: Some(Expr::Facet(FacetFilter {
+                                facet: "that".into(),
+                            })),
+                        },
+                    ],
+                })),
+            }),
             ..Default::default()
         };
         let query_plan = build_query_plan(request).unwrap();

{nidx_binding-6.3.1.post20 → nidx_binding-6.3.1.post26}/src/searcher/shard_suggest.rs

@@ -29,10 +29,7 @@ use tracing::{instrument, Span};
 
 use crate::errors::{NidxError, NidxResult};
 
-use super::{
-    index_cache::IndexCache,
-    query_planner::{filter_to_boolean_expression, old_filter_compatibility::filter_from_suggest_request},
-};
+use super::{index_cache::IndexCache, query_planner::filter_to_boolean_expression};
 
 /// Max number of words accepted as a suggest query. This is useful for
 /// compounds with semantic meaning (like a name and a surname) but can add
@@ -99,18 +96,7 @@ fn blocking_suggest(
 
     let mut paragraph_request = None;
     if suggest_paragraphs {
-        let filter_expression = filter_from_suggest_request(&request)?;
-        if let Some(expr) = filter_expression {
-            if request.field_filter.is_some() {
-                return Err(anyhow::anyhow!("Cannot specify old and new filters in the same request"));
-            }
-            let prefilter = PreFilterRequest {
-                security: None,
-                filter_expression: Some(expr),
-            };
-
-            prefiltered = text_searcher.prefilter(&prefilter)?;
-        } else if let Some(expr) = &request.field_filter {
+        if let Some(expr) = &request.field_filter {
             let prefilter = PreFilterRequest {
                 security: None,
                 filter_expression: Some(expr.clone()),

{nidx_binding-6.3.1.post20 → nidx_binding-6.3.1.post26}/tests/common/services.rs

@@ -19,14 +19,15 @@
 //
 
 use std::sync::Arc;
+use std::time::Duration;
 
 use nidx::api::grpc::ApiServer;
 use nidx::grpc_server::GrpcServer;
 use nidx::indexer::index_resource;
 use nidx::searcher::grpc::SearchServer;
 use nidx::searcher::shard_selector::ShardSelector;
-use nidx::searcher::{SyncStatus, SyncedSearcher};
-use nidx::settings::{EnvSettings, MetadataSettings, SearcherSettings, StorageSettings};
+use nidx::searcher::SyncedSearcher;
+use nidx::settings::{EnvSettings, MetadataSettings, StorageSettings};
 use nidx::{NidxMetadata, Settings};
 use nidx_protos::nidx::nidx_api_client::NidxApiClient;
 use nidx_protos::nidx::nidx_searcher_client::NidxSearcherClient;
@@ -34,7 +35,6 @@ use nidx_protos::Resource;
 use object_store::memory::InMemory;
 use sqlx::PgPool;
 use tempfile::tempdir;
-use tokio::sync::watch;
 use tokio_util::sync::CancellationToken;
 use tonic::transport::Channel;
 
@@ -43,7 +43,6 @@ pub struct NidxFixture {
     pub api_client: NidxApiClient<Channel>,
     pub settings: Settings,
     seq: i64,
-    sync_watcher: watch::Receiver<SyncStatus>,
 }
 
 impl NidxFixture {
@@ -66,11 +65,7 @@ impl NidxFixture {
                 telemetry: Default::default(),
                 work_path: None,
                 control_socket: None,
-                searcher: Some(SearcherSettings {
-                    // Reduced refresh interval for faster testing
-                    metadata_refresh_interval: 0.1,
-                    ..Default::default()
-                }),
+                searcher: Default::default(),
                 audit: None,
             },
         };
@@ -88,7 +83,6 @@ impl NidxFixture {
         let searcher_port = searcher_server.port()?;
         tokio::task::spawn(searcher_server.serve(searcher_api.into_router(), shutdown.clone()));
         let settings_copy = settings.clone();
-        let (sync_reporter, sync_watcher) = watch::channel(SyncStatus::Syncing);
         tokio::task::spawn(async move {
             searcher
                 .run(
@@ -96,7 +90,7 @@ impl NidxFixture {
                     settings_copy.searcher.clone().unwrap_or_default(),
                     shutdown.clone(),
                     ShardSelector::new_single(),
-                    Some(sync_reporter),
+                    None,
                     None,
                 )
                 .await
@@ -111,7 +105,6 @@ impl NidxFixture {
             api_client,
             settings,
             seq: 1,
-            sync_watcher,
         })
     }
 
@@ -129,10 +122,8 @@ impl NidxFixture {
         Ok(())
     }
 
-    pub async fn wait_sync(&mut self) {
-        // Wait for a new sync to start
-        self.sync_watcher.wait_for(|s| matches!(s, SyncStatus::Syncing)).await.unwrap();
-        // Wait for it to finish
-        self.sync_watcher.wait_for(|s| matches!(s, SyncStatus::Synced)).await.unwrap();
+    pub async fn wait_sync(&self) {
+        // TODO: Check the searcher has synced? For now, waiting twice the sync interval
+        tokio::time::sleep(Duration::from_secs(2)).await;
     }
 }

{nidx_binding-6.3.1.post20 → nidx_binding-6.3.1.post26}/tests/test_date_range_search.rs

@@ -23,11 +23,13 @@ mod common;
 use std::collections::HashMap;
 
 use common::services::NidxFixture;
+use nidx_protos::filter_expression::date_range_filter::DateField;
+use nidx_protos::filter_expression::{DateRangeFilter, Expr, FilterExpressionList};
 use nidx_protos::prost_types::Timestamp;
 use nidx_protos::resource::ResourceStatus;
 use nidx_protos::{
-    IndexMetadata, IndexParagraph, IndexParagraphs, NewShardRequest, Resource, ResourceId, SearchRequest,
-    TextInformation, Timestamps, VectorIndexConfig, VectorSentence, VectorsetSentences,
+    FilterExpression, IndexMetadata, IndexParagraph, IndexParagraphs, NewShardRequest, Resource, ResourceId,
+    SearchRequest, TextInformation, VectorIndexConfig, VectorSentence, VectorsetSentences,
 };
 use sqlx::PgPool;
 use tonic::Request;
@@ -138,7 +140,6 @@ async fn test_date_range_search(pool: PgPool) -> Result<(), Box<dyn std::error::
     let request = SearchRequest {
         shard: shard_id.clone(),
         order: None,
-        timestamps: None,
         vectorset: "english".to_string(),
         vector: vec![4.0, 6.0, 7.0],
         page_number: 0,
@@ -158,11 +159,25 @@ async fn test_date_range_search(pool: PgPool) -> Result<(), Box<dyn std::error::
 
     // Time range allows everything
     let mut request_all_range = request.clone();
-    request_all_range.timestamps = Some(Timestamps {
-        from_modified: Some(base_time),
-        to_modified: None,
-        from_created: Some(base_time),
-        to_created: None,
+    request_all_range.field_filter = Some(FilterExpression {
+        expr: Some(Expr::BoolAnd(FilterExpressionList {
+            operands: vec![
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Modified.into(),
+                        since: Some(base_time),
+                        until: None,
+                    })),
+                },
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Created.into(),
+                        since: Some(base_time),
+                        until: None,
+                    })),
+                },
+            ],
+        })),
     });
     let result = fixture.searcher_client.search(request_all_range).await.unwrap();
     let result = result.into_inner();
@@ -171,11 +186,25 @@ async fn test_date_range_search(pool: PgPool) -> Result<(), Box<dyn std::error::
 
     // Time range allows only second batch
     let mut request_second_batch = request.clone();
-    request_second_batch.timestamps = Some(Timestamps {
-        from_modified: Some(base_time_plus_one),
-        to_modified: None,
-        from_created: Some(base_time_plus_one),
-        to_created: None,
+    request_second_batch.field_filter = Some(FilterExpression {
+        expr: Some(Expr::BoolAnd(FilterExpressionList {
+            operands: vec![
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Modified.into(),
+                        since: Some(base_time_plus_one),
+                        until: None,
+                    })),
+                },
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Created.into(),
+                        since: Some(base_time_plus_one),
+                        until: None,
+                    })),
+                },
+            ],
+        })),
     });
     let result = fixture.searcher_client.search(request_second_batch).await.unwrap();
     let result = result.into_inner();
@@ -184,11 +213,16 @@ async fn test_date_range_search(pool: PgPool) -> Result<(), Box<dyn std::error::
 
     // Time range allows only second batch, but with modified only
     let mut request_second_batch = request.clone();
-    request_second_batch.timestamps = Some(Timestamps {
-        from_modified: Some(base_time_plus_one),
-        to_modified: None,
-        from_created: None,
-        to_created: None,
+    request_second_batch.field_filter = Some(FilterExpression {
+        expr: Some(Expr::BoolAnd(FilterExpressionList {
+            operands: vec![FilterExpression {
+                expr: Some(Expr::Date(DateRangeFilter {
+                    field: DateField::Modified.into(),
+                    since: Some(base_time_plus_one),
+                    until: None,
+                })),
+            }],
+        })),
     });
     let result = fixture.searcher_client.search(request_second_batch).await.unwrap();
     let result = result.into_inner();
@@ -200,11 +234,25 @@ async fn test_date_range_search(pool: PgPool) -> Result<(), Box<dyn std::error::
 
     // Time range does not match any field, therefore the response has no results
     let mut request_second_batch = request.clone();
-    request_second_batch.timestamps = Some(Timestamps {
-        from_modified: Some(base_time_plus_two),
-        to_modified: None,
-        from_created: Some(base_time_plus_two),
-        to_created: None,
+    request_second_batch.field_filter = Some(FilterExpression {
+        expr: Some(Expr::BoolAnd(FilterExpressionList {
+            operands: vec![
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Modified.into(),
+                        since: Some(base_time_plus_two),
+                        until: None,
+                    })),
+                },
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Created.into(),
+                        since: Some(base_time_plus_two),
+                        until: None,
+                    })),
+                },
+            ],
+        })),
     });
     let result = fixture.searcher_client.search(request_second_batch).await.unwrap();
     let result = result.into_inner();
@@ -215,11 +263,25 @@ async fn test_date_range_search(pool: PgPool) -> Result<(), Box<dyn std::error::
 
     // Multiple timestamps are parsed as AND conditions
     let mut request_second_batch = request.clone();
-    request_second_batch.timestamps = Some(Timestamps {
-        from_modified: Some(base_time_plus_one),
-        to_modified: None,
-        from_created: Some(base_time_plus_two),
-        to_created: None,
+    request_second_batch.field_filter = Some(FilterExpression {
+        expr: Some(Expr::BoolAnd(FilterExpressionList {
+            operands: vec![
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Modified.into(),
+                        since: Some(base_time_plus_one),
+                        until: None,
+                    })),
+                },
+                FilterExpression {
+                    expr: Some(Expr::Date(DateRangeFilter {
+                        field: DateField::Created.into(),
+                        since: Some(base_time_plus_two),
+                        until: None,
+                    })),
+                },
+            ],
+        })),
     });
     let result = fixture.searcher_client.search(request_second_batch).await.unwrap();
     let result = result.into_inner();