nidx-binding 6.3.5.post303__tar.gz → 6.3.5.post305__tar.gz

{nidx_binding-6.3.5.post303 → nidx_binding-6.3.5.post305}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.5.post303
+Version: 6.3.5.post305
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.5.post303 → nidx_binding-6.3.5.post305}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.5.post303"
+version = "6.3.5.post305"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.5.post303 → nidx_binding-6.3.5.post305}/nidx_relation/src/graph_query_parser.rs

@@ -221,8 +221,6 @@ impl<'a> GraphQueryParser<'a> {
     }
 
     pub fn parse(&self, query: GraphQuery) -> Box<dyn Query> {
-        // REVIEW: if at some point we only want to return what the query really asks (nodes,
-        // relations or paths), we may want to return the query and some kind of response builder
         match query {
             GraphQuery::NodeQuery(query) => self.parse_node_query(query),
             GraphQuery::RelationQuery(query) => self.parse_relation_query(query),

{nidx_binding-6.3.5.post303 → nidx_binding-6.3.5.post305}/nidx_relation/src/io_maps.rs

@@ -108,19 +108,24 @@ pub fn doc_to_relation(schema: &Schema, doc: &TantivyDocument) -> ProtosRelation
     }
 }
 
-pub fn doc_to_index_relation(schema: &Schema, doc: &TantivyDocument) -> IndexRelation {
-    let resource_field_id = doc.get_first(schema.resource_field_id).map(|v| {
+pub fn doc_to_resource_field_id(schema: &Schema, doc: &TantivyDocument) -> Option<String> {
+    doc.get_first(schema.resource_field_id).map(|v| {
         let (rid, fid) = decode_field_id(v.as_bytes().unwrap());
         format!("{}/{}", rid.simple(), fid)
-    });
-    let facets = doc
-        .get_all(schema.facets)
+    })
+}
+
+pub fn doc_to_facets(schema: &Schema, doc: &TantivyDocument) -> Vec<String> {
+    doc.get_all(schema.facets)
         .map(|f| f.as_facet().unwrap().to_path_string())
-        .collect();
+        .collect()
+}
+
+pub fn doc_to_index_relation(schema: &Schema, doc: &TantivyDocument) -> IndexRelation {
     IndexRelation {
         relation: Some(doc_to_relation(schema, doc)),
-        resource_field_id,
-        facets,
+        resource_field_id: doc_to_resource_field_id(schema, doc),
+        facets: doc_to_facets(schema, doc),
     }
 }
 

{nidx_binding-6.3.5.post303 → nidx_binding-6.3.5.post305}/nidx_relation/src/lib.rs

@@ -212,13 +212,13 @@ impl RelationSearcher {
         self.reader.relation_search(request)
     }
 
-    #[instrument(name = "relation::graph_searcher", skip_all)]
+    #[instrument(name = "relation::graph_search", skip_all)]
     pub fn graph_search(
         &self,
         request: &GraphSearchRequest,
-        prefilter: PrefilterResult,
+        prefilter: &PrefilterResult,
     ) -> anyhow::Result<GraphSearchResponse> {
-        self.reader.graph_search(request, &prefilter)
+        self.reader.graph_search(request, prefilter)
     }
 
     #[instrument(name = "relation::suggest", skip_all)]
@@ -259,7 +259,7 @@ impl RelationSearcher {
             }),
             ..Default::default()
         };
-        let response = self.graph_search(&request, PrefilterResult::All)?;
+        let response = self.graph_search(&request, &PrefilterResult::All)?;
         Ok(response.nodes)
     }
 

{nidx_binding-6.3.5.post303 → nidx_binding-6.3.5.post305}/nidx_relation/src/reader.rs

@@ -169,6 +169,8 @@ impl RelationsReaderService {
                 relation: relation_idx as u32,
                 destination: destination_idx as u32,
                 metadata: io_maps::decode_metadata(&self.schema, &doc),
+                resource_field_id: io_maps::doc_to_resource_field_id(&self.schema, &doc),
+                facets: io_maps::doc_to_facets(&self.schema, &doc),
             })
         }
 
@@ -289,6 +291,7 @@ impl RelationsReaderService {
         })
     }
 
+    // TODO: remove unused method
     fn entities_subgraph_search(
         &self,
         request: &RelationSearchRequest,

{nidx_binding-6.3.5.post303 → nidx_binding-6.3.5.post305}/nidx_relation/tests/test_graph_search.rs

@@ -75,7 +75,7 @@ fn _search(
             kind: kind.into(),
             ..Default::default()
         },
-        prefilter,
+        &prefilter,
     )
 }
 

{nidx_binding-6.3.5.post303 → nidx_binding-6.3.5.post305}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.5.post303"
+version = "6.3.5.post305"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.3.5.post303 → nidx_binding-6.3.5.post305}/src/searcher/query_planner.rs

@@ -20,7 +20,12 @@
 
 use nidx_paragraph::ParagraphSearchRequest;
 use nidx_protos::filter_expression::Expr;
-use nidx_protos::{FilterExpression, FilterOperator, RelationSearchRequest, SearchRequest};
+use nidx_protos::graph_query::{self, PathQuery, path_query};
+use nidx_protos::graph_search_request::QueryKind;
+use nidx_protos::search_request::GraphSearch;
+use nidx_protos::{
+    EntitiesSubgraphRequest, FilterExpression, FilterOperator, GraphQuery, GraphSearchRequest, SearchRequest,
+};
 use nidx_text::DocumentSearchRequest;
 use nidx_text::prefilter::*;
 use nidx_types::prefilter::PrefilterResult;
@@ -37,7 +42,7 @@ pub struct IndexQueries {
     pub vectors_request: Option<VectorSearchRequest>,
     pub paragraphs_request: Option<ParagraphSearchRequest>,
     pub texts_request: Option<DocumentSearchRequest>,
-    pub relations_request: Option<RelationSearchRequest>,
+    pub relations_request: Option<GraphSearchRequest>,
 }
 
 impl IndexQueries {
@@ -65,7 +70,7 @@ pub struct QueryPlan {
 }
 
 pub fn build_query_plan(search_request: SearchRequest) -> anyhow::Result<QueryPlan> {
-    let relations_request = compute_relations_request(&search_request);
+    let graph_request = compute_relations_request(&search_request)?;
     let texts_request = compute_texts_request(&search_request);
     let vectors_request = compute_vectors_request(&search_request)?;
     let paragraphs_request = compute_paragraphs_request(&search_request)?;
@@ -79,7 +84,7 @@ pub fn build_query_plan(search_request: SearchRequest) -> anyhow::Result<QueryPl
             vectors_request,
             paragraphs_request,
             texts_request,
-            relations_request,
+            relations_request: graph_request,
         },
     })
 }
@@ -171,14 +176,153 @@ fn compute_vectors_request(search_request: &SearchRequest) -> anyhow::Result<Opt
     }))
 }
 
-fn compute_relations_request(search_request: &SearchRequest) -> Option<RelationSearchRequest> {
-    search_request.relation_subgraph.as_ref()?;
+#[allow(deprecated)]
+fn compute_relations_request(search_request: &SearchRequest) -> anyhow::Result<Option<GraphSearchRequest>> {
+    let graph_search = match (
+        search_request.relation_subgraph.as_ref(),
+        search_request.graph_search.as_ref(),
+    ) {
+        (None, None) => return Ok(None),
+        (Some(_), Some(_)) => {
+            return Err(anyhow::anyhow!(
+                "Entities subgraph and graph search are mutually exclusive features"
+            ));
+        }
+        (Some(bfs_request), None) => {
+            // Bw/c: we'll convert the entities subgraph request into a graph request
+            convert_entities_subgraph_into_graph_search(bfs_request)
+        }
+        (None, Some(graph_search)) => graph_search.clone(),
+    };
 
-    #[allow(deprecated)]
-    Some(RelationSearchRequest {
-        shard_id: search_request.shard.clone(),
-        subgraph: search_request.relation_subgraph.clone(),
-    })
+    Ok(Some(GraphSearchRequest {
+        query: graph_search.query,
+        top_k: std::cmp::max(search_request.result_per_page as u32, 20),
+        kind: QueryKind::Path.into(),
+        // we don't need to populate filters nor shard as they won't be used in search. Prefilter
+        // will be done with request filters and shard have been already obtained
+        ..Default::default()
+    }))
+}
+
+fn convert_entities_subgraph_into_graph_search(bfs_request: &EntitiesSubgraphRequest) -> GraphSearch {
+    if bfs_request.entry_points.is_empty() {
+        return GraphSearch { query: None };
+    }
+
+    // Entry points are source or target nodes we want to search for. We want any undirected
+    // path containing any entry point
+    let entry_points_queries: Vec<PathQuery> = bfs_request
+        .entry_points
+        .iter()
+        .map(|entry_point| PathQuery {
+            query: Some(path_query::Query::Path(graph_query::Path {
+                source: Some(graph_query::Node {
+                    value: Some(entry_point.value.clone()),
+                    node_type: Some(entry_point.ntype),
+                    node_subtype: Some(entry_point.subtype.clone()),
+                    ..Default::default()
+                }),
+                undirected: true,
+                ..Default::default()
+            })),
+        })
+        .collect();
+
+    // A query can specifiy nodes marked as deleted in the db (but not removed from the index).
+    // We want to exclude any path containing any of those nodes.
+    //
+    // The request groups values per subtype (to optimize request size) but, as we don't support
+    // OR at node value level, we'll split them.
+    let deleted_nodes_queries: Vec<PathQuery> = bfs_request
+        .deleted_entities
+        .iter()
+        .flat_map(|deleted_nodes| {
+            if deleted_nodes.node_values.is_empty() {
+                return None;
+            }
+            let subtype = &deleted_nodes.node_subtype;
+
+            let subqueries = deleted_nodes
+                .node_values
+                .iter()
+                .map(|deleted_entity_value| PathQuery {
+                    query: Some(path_query::Query::Path(graph_query::Path {
+                        source: Some(graph_query::Node {
+                            value: Some(deleted_entity_value.clone()),
+                            node_subtype: Some(subtype.clone()),
+                            ..Default::default()
+                        }),
+                        undirected: true,
+                        ..Default::default()
+                    })),
+                })
+                .collect::<Vec<_>>();
+            Some(subqueries)
+        })
+        .flatten()
+        .collect();
+
+    // Subtypes can also be marked as deleted in the db (but kept in the index). We also
+    // want to exclude any triplet containg a node with such subtypes
+    let excluded_subtypes_queries: Vec<PathQuery> = bfs_request
+        .deleted_groups
+        .iter()
+        .map(|deleted_subtype| PathQuery {
+            query: Some(path_query::Query::Path(graph_query::Path {
+                source: Some(graph_query::Node {
+                    node_subtype: Some(deleted_subtype.clone()),
+                    ..Default::default()
+                }),
+                undirected: true,
+                ..Default::default()
+            })),
+        })
+        .collect::<Vec<_>>();
+
+    let mut subqueries = vec![];
+    if !entry_points_queries.is_empty() {
+        // match any entry point
+        subqueries.push(PathQuery {
+            query: Some(path_query::Query::BoolOr(graph_query::BoolQuery {
+                operands: entry_points_queries,
+            })),
+        });
+    }
+
+    if !deleted_nodes_queries.is_empty() {
+        // exclude deleted nodes
+        subqueries.push(PathQuery {
+            query: Some(path_query::Query::BoolNot(Box::new(PathQuery {
+                query: Some(path_query::Query::BoolOr(graph_query::BoolQuery {
+                    operands: deleted_nodes_queries,
+                })),
+            }))),
+        });
+    }
+
+    if !excluded_subtypes_queries.is_empty() {
+        // exclude specific subtypes
+        subqueries.push(PathQuery {
+            query: Some(path_query::Query::BoolNot(Box::new(PathQuery {
+                query: Some(path_query::Query::BoolOr(graph_query::BoolQuery {
+                    operands: excluded_subtypes_queries,
+                })),
+            }))),
+        });
+    }
+
+    let graph_query = GraphQuery {
+        path: Some(PathQuery {
+            query: Some(path_query::Query::BoolAnd(graph_query::BoolQuery {
+                operands: subqueries,
+            })),
+        }),
+    };
+
+    GraphSearch {
+        query: Some(graph_query),
+    }
 }
 
 pub fn filter_to_boolean_expression(filter: FilterExpression) -> anyhow::Result<BooleanExpression<String>> {

{nidx_binding-6.3.5.post303 → nidx_binding-6.3.5.post305}/src/searcher/shard_search.rs

@@ -21,7 +21,10 @@
 use std::sync::Arc;
 
 use nidx_paragraph::ParagraphSearcher;
-use nidx_protos::{GraphSearchRequest, GraphSearchResponse, SearchRequest, SearchResponse};
+use nidx_protos::{
+    EntitiesSubgraphResponse, GraphSearchRequest, GraphSearchResponse, IndexRelation, Relation, RelationSearchResponse,
+    SearchRequest, SearchResponse,
+};
 use nidx_relation::RelationSearcher;
 use nidx_text::{TextSearcher, prefilter::PreFilterRequest};
 use nidx_types::prefilter::PrefilterResult;
@@ -131,7 +134,7 @@ fn blocking_search(
 
     let relation_task = index_queries
         .relations_request
-        .map(|request| move || relation_searcher.unwrap().search(&request));
+        .map(|request| move || relation_searcher.unwrap().graph_search(&request, prefilter));
 
     let vector_task = index_queries.vectors_request.map(|mut request| {
         request.id = search_id.clone();
@@ -169,11 +172,44 @@ fn blocking_search(
         }
     });
 
+    let graph_response = rrelation.transpose()?;
+
+    // Bw/c: convert graph search into relation response
+    let relation_response = graph_response.clone().map(|graph_response| {
+        let relations = graph_response
+            .graph
+            .into_iter()
+            .map(|path| {
+                let source = graph_response.nodes[path.source as usize].clone();
+                let relation = graph_response.relations[path.relation as usize].clone();
+                let destination = graph_response.nodes[path.destination as usize].clone();
+
+                IndexRelation {
+                    relation: Some(Relation {
+                        source: Some(source),
+                        to: Some(destination),
+                        relation: relation.relation_type,
+                        relation_label: relation.label,
+                        metadata: path.metadata,
+                    }),
+                    resource_field_id: path.resource_field_id,
+                    facets: path.facets,
+                }
+            })
+            .collect();
+
+        RelationSearchResponse {
+            subgraph: Some(EntitiesSubgraphResponse { relations }),
+            ..Default::default()
+        }
+    });
+
     Ok(SearchResponse {
         document: rtext.transpose()?,
         paragraph: rparagraph.transpose()?,
         vector: rvector.transpose()?,
-        relation: rrelation.transpose()?,
+        relation: relation_response,
+        graph: graph_response,
     })
 }
 
@@ -223,7 +259,7 @@ pub async fn graph_search(
     let results = tokio::task::spawn_blocking(move || {
         current.in_scope(|| {
             let searcher: &RelationSearcher = relation_searcher.as_ref().into();
-            searcher.graph_search(&graph_request, prefilter)
+            searcher.graph_search(&graph_request, &prefilter)
         })
     })
     .await??;

{nidx_binding-6.3.5.post303 → nidx_binding-6.3.5.post305}/tests/test_search_relations.rs

@@ -1115,6 +1115,7 @@ async fn relation_search(
     fixture: &mut NidxFixture,
     request: RelationSearchRequest,
 ) -> anyhow::Result<RelationSearchResponse> {
+    #[allow(deprecated)]
     let request = SearchRequest {
         shard: request.shard_id,
         vectorset: "english".to_string(),