nidx-binding 6.3.1.post72__tar.gz → 6.3.1.post82__tar.gz

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.1.post72
+Version: 6.3.1.post82
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/nidx_paragraph/src/query_io.rs

@@ -29,7 +29,7 @@ fn translate_literal(literal: &str, schema: &ParagraphSchema) -> Box<dyn Query>
     Box::new(TermQuery::new(term, IndexRecordOption::Basic))
 }
 
-fn translate_not(inner: &BooleanExpression, schema: &ParagraphSchema) -> Box<dyn Query> {
+fn translate_not(inner: &BooleanExpression<String>, schema: &ParagraphSchema) -> Box<dyn Query> {
     let mut operands = Vec::with_capacity(2);
 
     // Check the following issue to see why the additional AllQuery is needed:
@@ -43,7 +43,7 @@ fn translate_not(inner: &BooleanExpression, schema: &ParagraphSchema) -> Box<dyn
     Box::new(BooleanQuery::new(operands))
 }
 
-fn translate_operation(operation: &BooleanOperation, schema: &ParagraphSchema) -> Box<dyn Query> {
+fn translate_operation(operation: &BooleanOperation<String>, schema: &ParagraphSchema) -> Box<dyn Query> {
     let operator = match operation.operator {
         Operator::And => Occur::Must,
         Operator::Or => Occur::Should,
@@ -59,7 +59,7 @@ fn translate_operation(operation: &BooleanOperation, schema: &ParagraphSchema) -
     Box::new(BooleanQuery::new(operands))
 }
 
-pub fn translate_expression(expression: &BooleanExpression, schema: &ParagraphSchema) -> Box<dyn Query> {
+pub fn translate_expression(expression: &BooleanExpression<String>, schema: &ParagraphSchema) -> Box<dyn Query> {
     match expression {
         BooleanExpression::Not(inner) => translate_not(inner, schema),
         BooleanExpression::Literal(literal) => translate_literal(literal, schema),

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/nidx_paragraph/src/request_types.rs

@@ -35,14 +35,14 @@ pub struct ParagraphSearchRequest {
     pub min_score: f32,
     pub security: Option<nidx_protos::utils::Security>,
 
-    pub filtering_formula: Option<BooleanExpression>,
+    pub filtering_formula: Option<BooleanExpression<String>>,
     /// Whether to do an OR/AND between prefilter results and filtering_formula
     pub filter_or: bool,
 }
 
 pub struct ParagraphSuggestRequest {
     pub body: String,
-    pub filtering_formula: Option<BooleanExpression>,
+    pub filtering_formula: Option<BooleanExpression<String>>,
     /// Whether to do an OR/AND between prefilter results and filtering_formula
     pub filter_or: bool,
 }

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/nidx_paragraph/src/search_query.rs

@@ -262,7 +262,7 @@ fn preprocess_raw_query(query: &str, tc: &mut TermCollector) -> ProcessedQuery {
 fn filter_query(
     schema: &ParagraphSchema,
     prefilter: &PrefilterResult,
-    paragraph_formula: &Option<BooleanExpression>,
+    paragraph_formula: &Option<BooleanExpression<String>>,
     filter_or: bool,
 ) -> Option<Box<dyn Query>> {
     let mut filter_terms = vec![];

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.1.post72"
+version = "6.3.1.post82"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/nidx_relation/src/graph_query_parser.rs

@@ -18,12 +18,15 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 use nidx_protos::relation_node::NodeType;
+use nidx_types::query_language::{BooleanExpression, BooleanOperation, Operator};
 use tantivy::query::{AllQuery, BooleanQuery, FuzzyTermQuery, Occur, Query, TermQuery};
 use tantivy::schema::{Field, IndexRecordOption};
 
 use crate::schema::Schema;
 use crate::{io_maps, schema};
 
+const DEFAULT_NODE_VALUE_FUZZY_DISTANCE: u8 = 2;
+
 #[derive(Clone)]
 pub struct FuzzyTerm {
     pub value: String,
@@ -46,7 +49,6 @@ pub struct Node {
 
 #[derive(Default, Clone)]
 pub struct Relation {
-    // TODO: fuzzy
     pub value: Option<String>,
 }
 
@@ -105,6 +107,8 @@ pub enum GraphQuery {
     PathQuery(PathQuery),
 }
 
+pub struct BoolGraphQuery(BooleanExpression<GraphQuery>);
+
 #[derive(Clone, Copy)]
 struct NodeSchemaFields {
     normalized_value: Field,
@@ -121,6 +125,33 @@ impl GraphQueryParser {
         Self { schema: Schema::new() }
     }
 
+    pub fn parse_bool(&self, query: BoolGraphQuery) -> Box<dyn Query> {
+        self.inner_parse_bool(query.0)
+    }
+
+    fn inner_parse_bool(&self, query: BooleanExpression<GraphQuery>) -> Box<dyn Query> {
+        match query {
+            BooleanExpression::Literal(query) => self.parse(query),
+            BooleanExpression::Not(subquery) => {
+                let mut subqueries = vec![];
+                subqueries.push((Occur::Must, Box::new(AllQuery) as Box<dyn Query>));
+                subqueries.push((Occur::MustNot, self.inner_parse_bool(*subquery)));
+                Box::new(BooleanQuery::new(subqueries))
+            }
+            BooleanExpression::Operation(operation) => {
+                let mut subqueries = vec![];
+                let occur = match operation.operator {
+                    Operator::And => Occur::Must,
+                    Operator::Or => Occur::Should,
+                };
+                for expression in operation.operands {
+                    subqueries.push((occur, self.inner_parse_bool(expression)));
+                }
+                Box::new(BooleanQuery::new(subqueries))
+            }
+        }
+    }
+
     pub fn parse(&self, query: GraphQuery) -> Box<dyn Query> {
         // REVIEW: if at some point we only want to return what the query really asks (nodes,
         // relations or paths), we may want to return the query and some kind of response builder
@@ -425,51 +456,79 @@ impl From<&str> for Term {
     }
 }
 
-impl TryFrom<&nidx_protos::graph_query::Query> for GraphQuery {
+impl TryFrom<&nidx_protos::graph_query::PathQuery> for BoolGraphQuery {
     type Error = anyhow::Error;
 
-    fn try_from(value: &nidx_protos::graph_query::Query) -> Result<Self, Self::Error> {
-        let query = match value {
-            nidx_protos::graph_query::Query::Node(query_node) => {
-                let node = Node::try_from(query_node)?;
-                Self::NodeQuery(NodeQuery::Node(Expression::Value(node)))
-            }
+    fn try_from(value: &nidx_protos::graph_query::PathQuery) -> Result<Self, Self::Error> {
+        let bool_expr = match &value.query {
+            Some(query) => match query {
+                nidx_protos::graph_query::path_query::Query::Path(path) => {
+                    let source = match &path.source {
+                        Some(source_pb) => Node::try_from(source_pb)?,
+                        None => Node::default(),
+                    };
+                    let relation = match &path.relation {
+                        Some(relation_pb) => Relation::try_from(relation_pb)?,
+                        None => Relation::default(),
+                    };
+                    let destination = match &path.destination {
+                        Some(destination_pb) => Node::try_from(destination_pb)?,
+                        None => Node::default(),
+                    };
+
+                    let path_query = if path.undirected {
+                        GraphQuery::PathQuery(PathQuery::UndirectedPath((
+                            Expression::Value(source),
+                            Expression::Value(relation),
+                            Expression::Value(destination),
+                        )))
+                    } else {
+                        GraphQuery::PathQuery(PathQuery::DirectedPath((
+                            Expression::Value(source),
+                            Expression::Value(relation),
+                            Expression::Value(destination),
+                        )))
+                    };
+
+                    BooleanExpression::Literal(path_query)
+                }
 
-            nidx_protos::graph_query::Query::Relation(relation) => {
-                Self::RelationQuery(RelationQuery(Expression::Value(Relation::try_from(relation)?)))
-            }
+                nidx_protos::graph_query::path_query::Query::BoolNot(bool_not) => {
+                    let subquery = BoolGraphQuery::try_from(bool_not.as_ref())?.0;
+                    BooleanExpression::Not(Box::new(subquery))
+                }
 
-            nidx_protos::graph_query::Query::Path(path) => {
-                let source = match &path.source {
-                    Some(source_pb) => Node::try_from(source_pb)?,
-                    None => Node::default(),
-                };
-                let relation = match &path.relation {
-                    Some(relation_pb) => Relation::try_from(relation_pb)?,
-                    None => Relation::default(),
-                };
-                let destination = match &path.destination {
-                    Some(destination_pb) => Node::try_from(destination_pb)?,
-                    None => Node::default(),
-                };
+                nidx_protos::graph_query::path_query::Query::BoolAnd(bool_and) => {
+                    BooleanExpression::Operation(BooleanOperation {
+                        operator: Operator::And,
+                        operands: bool_and
+                            .operands
+                            .iter()
+                            .map(|o| BoolGraphQuery::try_from(o).map(|x| x.0))
+                            .collect::<anyhow::Result<Vec<_>>>()?,
+                    })
+                }
 
-                if path.undirected {
-                    Self::PathQuery(PathQuery::UndirectedPath((
-                        Expression::Value(source),
-                        Expression::Value(relation),
-                        Expression::Value(destination),
-                    )))
-                } else {
-                    Self::PathQuery(PathQuery::DirectedPath((
-                        Expression::Value(source),
-                        Expression::Value(relation),
-                        Expression::Value(destination),
-                    )))
+                nidx_protos::graph_query::path_query::Query::BoolOr(bool_or) => {
+                    BooleanExpression::Operation(BooleanOperation {
+                        operator: Operator::Or,
+                        operands: bool_or
+                            .operands
+                            .iter()
+                            .map(|o| BoolGraphQuery::try_from(o).map(|x| x.0))
+                            .collect::<anyhow::Result<Vec<_>>>()?,
+                    })
                 }
-            }
+            },
+
+            None => BooleanExpression::Literal(GraphQuery::PathQuery(PathQuery::UndirectedPath((
+                Expression::Value(Node::default()),
+                Expression::Value(Relation::default()),
+                Expression::Value(Node::default()),
+            )))),
         };
 
-        Ok(query)
+        Ok(BoolGraphQuery(bool_expr))
     }
 }
 
@@ -477,16 +536,13 @@ impl TryFrom<&nidx_protos::graph_query::Node> for Node {
     type Error = anyhow::Error;
 
     fn try_from(node_pb: &nidx_protos::graph_query::Node) -> Result<Self, Self::Error> {
-        let value = node_pb.value.clone().map(|value| {
-            if node_pb.fuzzy_distance > 0 || node_pb.as_prefix {
-                Term::Fuzzy(FuzzyTerm {
-                    value: value,
-                    fuzzy_distance: node_pb.fuzzy_distance as u8,
-                    is_prefix: node_pb.as_prefix,
-                })
-            } else {
-                Term::Exact(value)
-            }
+        let value = node_pb.value.clone().map(|value| match node_pb.match_kind() {
+            nidx_protos::graph_query::node::MatchKind::Exact => Term::Exact(value),
+            nidx_protos::graph_query::node::MatchKind::Fuzzy => Term::Fuzzy(FuzzyTerm {
+                value: value,
+                fuzzy_distance: DEFAULT_NODE_VALUE_FUZZY_DISTANCE,
+                is_prefix: true,
+            }),
         });
         let node_type = node_pb.node_type.map(NodeType::try_from).transpose()?;
         let node_subtype = node_pb.node_subtype.clone();

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/nidx_relation/src/reader.rs

@@ -31,7 +31,7 @@ use tantivy::query::{BooleanQuery, Occur, Query};
 use tantivy::{DocAddress, Index, IndexReader, Searcher};
 
 use crate::graph_query_parser::{
-    Expression, FuzzyTerm, GraphQuery, GraphQueryParser, Node, NodeQuery, PathQuery, Relation, Term,
+    BoolGraphQuery, Expression, FuzzyTerm, GraphQuery, GraphQueryParser, Node, NodeQuery, Term,
 };
 use crate::schema::Schema;
 use crate::{io_maps, schema};
@@ -71,14 +71,14 @@ impl RelationsReaderService {
         let Some(query) = &request.query else {
             return Ok(GraphSearchResponse::default());
         };
-        let Some(query) = &query.query else {
+        let Some(query) = &query.path else {
             return Ok(GraphSearchResponse::default());
         };
 
         // Convert proto to tantivy query
-        let query = GraphQuery::try_from(query)?;
+        let query = BoolGraphQuery::try_from(query)?;
         let parser = GraphQueryParser::new();
-        let index_query = parser.parse(query);
+        let index_query = parser.parse_bool(query);
 
         // Tantivy searcher query
         let collector = TopDocs::with_limit(request.top_k as usize);
@@ -192,15 +192,11 @@ impl RelationsReaderService {
         for entry_point in bfs_request.entry_points.iter() {
             statements.push((
                 Occur::Should,
-                query_parser.parse(GraphQuery::PathQuery(PathQuery::UndirectedPath((
-                    Expression::Value(Node {
-                        value: Some(entry_point.value.clone().into()),
-                        node_type: Some(entry_point.ntype()),
-                        node_subtype: Some(entry_point.subtype.clone()),
-                    }),
-                    Expression::Value(Relation::default()),
-                    Expression::Value(Node::default()),
-                )))),
+                query_parser.parse(GraphQuery::NodeQuery(NodeQuery::Node(Expression::Value(Node {
+                    value: Some(entry_point.value.clone().into()),
+                    node_type: Some(entry_point.ntype()),
+                    node_subtype: Some(entry_point.subtype.clone()),
+                })))),
             ));
         }
 
@@ -215,20 +211,16 @@ impl RelationsReaderService {
             }
             statements.push((
                 Occur::MustNot,
-                query_parser.parse(GraphQuery::PathQuery(PathQuery::UndirectedPath((
-                    Expression::Or(
-                        deleted_nodes
-                            .node_values
-                            .iter()
-                            .map(|deleted_entity_value| Node {
-                                value: Some(deleted_entity_value.clone().into()),
-                                node_subtype: Some(deleted_nodes.node_subtype.clone()),
-                                ..Default::default()
-                            })
-                            .collect(),
-                    ),
-                    Expression::Value(Relation::default()),
-                    Expression::Value(Node::default()),
+                query_parser.parse(GraphQuery::NodeQuery(NodeQuery::Node(Expression::Or(
+                    deleted_nodes
+                        .node_values
+                        .iter()
+                        .map(|deleted_entity_value| Node {
+                            value: Some(deleted_entity_value.clone().into()),
+                            node_subtype: Some(deleted_nodes.node_subtype.clone()),
+                            ..Default::default()
+                        })
+                        .collect(),
                 )))),
             ));
         }
@@ -247,10 +239,8 @@ impl RelationsReaderService {
         if excluded_subtypes.len() > 0 {
             statements.push((
                 Occur::MustNot,
-                query_parser.parse(GraphQuery::PathQuery(PathQuery::UndirectedPath((
-                    Expression::Or(excluded_subtypes),
-                    Expression::Value(Relation::default()),
-                    Expression::Value(Node::default()),
+                query_parser.parse(GraphQuery::NodeQuery(NodeQuery::Node(Expression::Or(
+                    excluded_subtypes,
                 )))),
             ))
         }

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/nidx_types/src/query_language.rs

@@ -31,14 +31,14 @@ pub enum Operator {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
-pub struct BooleanOperation {
+pub struct BooleanOperation<T> {
     pub operator: Operator,
-    pub operands: Vec<BooleanExpression>,
+    pub operands: Vec<BooleanExpression<T>>,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
-pub enum BooleanExpression {
-    Literal(String),
-    Not(Box<BooleanExpression>),
-    Operation(BooleanOperation),
+pub enum BooleanExpression<T> {
+    Literal(T),
+    Not(Box<BooleanExpression<T>>),
+    Operation(BooleanOperation<T>),
 }

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/nidx_vector/src/data_point_provider/reader.rs

@@ -93,7 +93,7 @@ pub struct Reader {
     open_data_points: Vec<OpenDataPoint>,
 }
 
-fn segment_matches(expression: &BooleanExpression, labels: &HashSet<String>) -> bool {
+fn segment_matches(expression: &BooleanExpression<String>, labels: &HashSet<String>) -> bool {
     match expression {
         BooleanExpression::Literal(tag) => labels.contains(tag),
         BooleanExpression::Not(expr) => !segment_matches(expr, labels),
@@ -158,7 +158,7 @@ impl Reader {
     pub fn _search(
         &self,
         request: &dyn SearchRequest,
-        segment_filter: &Option<BooleanExpression>,
+        segment_filter: &Option<BooleanExpression<String>>,
     ) -> VectorR<Vec<Neighbour>> {
         let normalized_query;
         let query = if self.config.normalize_vectors {

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/nidx_vector/src/query_io.rs

@@ -25,7 +25,7 @@ fn map_literal(literal: &str) -> Clause {
     Clause::Atom(AtomClause::label(literal.to_string()))
 }
 
-fn map_not(inner: &BooleanExpression) -> Clause {
+fn map_not(inner: &BooleanExpression<String>) -> Clause {
     let operator = BooleanOperator::Not;
     let operand = map_expression(inner);
     let operands = vec![operand];
@@ -33,7 +33,7 @@ fn map_not(inner: &BooleanExpression) -> Clause {
     Clause::Compound(CompoundClause::new(operator, operands))
 }
 
-fn map_operation(operation: &BooleanOperation) -> Clause {
+fn map_operation(operation: &BooleanOperation<String>) -> Clause {
     let operator = match operation.operator {
         Operator::And => BooleanOperator::And,
         Operator::Or => BooleanOperator::Or,
@@ -43,7 +43,7 @@ fn map_operation(operation: &BooleanOperation) -> Clause {
     Clause::Compound(CompoundClause::new(operator, operands))
 }
 
-pub fn map_expression(expression: &BooleanExpression) -> Clause {
+pub fn map_expression(expression: &BooleanExpression<String>) -> Clause {
     match expression {
         BooleanExpression::Not(inner) => map_not(inner),
         BooleanExpression::Literal(literal) => map_literal(literal),

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/nidx_vector/src/request_types.rs

@@ -36,8 +36,8 @@ pub struct VectorSearchRequest {
     pub vector_set: String,
     pub min_score: f32,
 
-    pub filtering_formula: Option<BooleanExpression>,
-    pub segment_filtering_formula: Option<BooleanExpression>,
+    pub filtering_formula: Option<BooleanExpression<String>>,
+    pub segment_filtering_formula: Option<BooleanExpression<String>>,
     /// Whether to do an OR/AND between prefilter results and filtering_formula
     pub filter_or: bool,
 }

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.1.post72"
+version = "6.3.1.post82"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/src/searcher/query_language.rs

@@ -24,7 +24,7 @@ use nidx_protos::{
 use nidx_types::query_language::*;
 
 /// Extract an expression only involving some labels if it's an AND subset of the total expression
-pub fn extract_label_filters(expression: &FilterExpression, labels: &[&str]) -> Option<BooleanExpression> {
+pub fn extract_label_filters(expression: &FilterExpression, labels: &[&str]) -> Option<BooleanExpression<String>> {
     match expression.expr.as_ref().unwrap() {
         Expr::Facet(FacetFilter { facet }) if labels.contains(&facet.as_str()) => {
             Some(BooleanExpression::Literal(facet.clone()))

{nidx_binding-6.3.1.post72 → nidx_binding-6.3.1.post82}/src/searcher/query_planner.rs

@@ -184,14 +184,14 @@ fn compute_relations_request(search_request: &SearchRequest) -> Option<RelationS
     })
 }
 
-pub fn filter_to_boolean_expression(filter: FilterExpression) -> anyhow::Result<BooleanExpression> {
+pub fn filter_to_boolean_expression(filter: FilterExpression) -> anyhow::Result<BooleanExpression<String>> {
     match filter.expr.unwrap() {
         Expr::BoolAnd(and) => {
             let operands = and
                 .operands
                 .into_iter()
                 .map(filter_to_boolean_expression)
-                .collect::<anyhow::Result<Vec<BooleanExpression>>>()?;
+                .collect::<anyhow::Result<Vec<BooleanExpression<String>>>>()?;
             Ok(BooleanExpression::Operation(BooleanOperation {
                 operator: Operator::And,
                 operands,
@@ -202,7 +202,7 @@ pub fn filter_to_boolean_expression(filter: FilterExpression) -> anyhow::Result<
                 .operands
                 .into_iter()
                 .map(filter_to_boolean_expression)
-                .collect::<anyhow::Result<Vec<BooleanExpression>>>()?;
+                .collect::<anyhow::Result<Vec<BooleanExpression<String>>>>()?;
             Ok(BooleanExpression::Operation(BooleanOperation {
                 operator: Operator::Or,
                 operands,