nidx-binding 6.3.4.post161__tar.gz → 6.3.4.post178__tar.gz

{nidx_binding-6.3.4.post161 → nidx_binding-6.3.4.post178}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.4.post161
+Version: 6.3.4.post178
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.4.post161 → nidx_binding-6.3.4.post178}/nidx_paragraph/src/reader.rs

@@ -36,7 +36,7 @@ use crate::search_query::{SharedTermC, search_query, streaming_query, suggest_qu
 use crate::search_response::{SearchBm25Response, SearchFacetsResponse, SearchIntResponse, extract_labels};
 
 const FUZZY_DISTANCE: u8 = 1;
-const NUMBER_OF_RESULTS_SUGGEST: usize = 10;
+const NUMBER_OF_RESULTS_SUGGEST: usize = 20;
 
 pub struct ParagraphReaderService {
     pub index: Index,

{nidx_binding-6.3.4.post161 → nidx_binding-6.3.4.post178}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.4.post161"
+version = "6.3.4.post178"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.4.post161 → nidx_binding-6.3.4.post178}/nidx_relation/src/graph_query_parser.rs

@@ -19,10 +19,11 @@
 //
 use anyhow::anyhow;
 use nidx_protos::graph_query::FacetFilter;
+use nidx_protos::graph_query::node::MatchLocation;
 use nidx_protos::relation::RelationType;
 use nidx_protos::relation_node::NodeType;
 use nidx_types::query_language::{BooleanExpression, BooleanOperation, Operator};
-use tantivy::query::{AllQuery, BooleanQuery, FuzzyTermQuery, Occur, Query, TermQuery};
+use tantivy::query::{AllQuery, BooleanQuery, FuzzyTermQuery, Occur, Query, TermQuery, TermSetQuery};
 use tantivy::schema::{Facet, Field, IndexRecordOption};
 use tantivy::tokenizer::TokenizerManager;
 
@@ -41,7 +42,9 @@ pub struct FuzzyTerm {
 #[derive(Clone)]
 pub enum Term {
     Exact(String),
+    ExactWord(String),
     Fuzzy(FuzzyTerm),
+    FuzzyWord(FuzzyTerm),
 }
 
 #[derive(Default, Clone)]
@@ -467,8 +470,8 @@ impl<'a> GraphQueryParser<'a> {
 
     fn has_node_value(&self, value: &Term, exact_field: Field, tokenized_field: Field) -> Option<Box<dyn Query>> {
         let text_value = match value {
-            Term::Exact(value) => value,
-            Term::Fuzzy(fuzzy) => &fuzzy.value,
+            Term::Exact(value) | Term::ExactWord(value) => value,
+            Term::Fuzzy(fuzzy) | Term::FuzzyWord(fuzzy) => &fuzzy.value,
         };
         if text_value.is_empty() {
             return None;
@@ -482,42 +485,38 @@ impl<'a> GraphQueryParser<'a> {
             tokenized_terms.push(tantivy::Term::from_field_text(tokenized_field, &token.text));
         }
 
-        // TODO: Rethink this
-        // Current logic:
-        // - Exact match always match the search term against the full field
-        // - Fuzzy + prefix search works does a prefix fuzzy match of the whole entity name
-        // - Fuzzy search looks for entities containing all words in the term with a fuzzy match (tokenized)
-        //
-        // Questions:
-        // - Do we want exact match of a word in the entity? (kind of supported by setting fuzzy distance = 0)
         let query: Box<dyn Query> = match value {
             Term::Exact(_) => Box::new(TermQuery::new(exact_term, IndexRecordOption::Basic)),
-            Term::Fuzzy(fuzzy) => match fuzzy {
-                FuzzyTerm {
-                    fuzzy_distance,
-                    is_prefix: true,
-                    ..
-                } => Box::new(FuzzyTermQuery::new_prefix(exact_term, *fuzzy_distance, true)),
-
-                FuzzyTerm { fuzzy_distance, .. } => {
-                    if tokenized_terms.len() > 1 {
-                        Box::new(BooleanQuery::intersection(
-                            tokenized_terms
-                                .into_iter()
-                                .map(|term| -> Box<dyn Query> {
-                                    Box::new(FuzzyTermQuery::new(term, *fuzzy_distance, true))
-                                })
-                                .collect(),
-                        ))
-                    } else {
-                        Box::new(FuzzyTermQuery::new(
-                            tokenized_terms.into_iter().next().unwrap(),
-                            *fuzzy_distance,
-                            true,
-                        ))
-                    }
+
+            Term::ExactWord(_) => Box::new(TermSetQuery::new(tokenized_terms)),
+
+            Term::Fuzzy(fuzzy) => {
+                if fuzzy.is_prefix {
+                    Box::new(FuzzyTermQuery::new_prefix(exact_term, fuzzy.fuzzy_distance, true))
+                } else {
+                    Box::new(FuzzyTermQuery::new(exact_term, fuzzy.fuzzy_distance, true))
                 }
-            },
+            }
+
+            Term::FuzzyWord(fuzzy) => {
+                let query_builder = if fuzzy.is_prefix {
+                    FuzzyTermQuery::new_prefix
+                } else {
+                    FuzzyTermQuery::new
+                };
+
+                if tokenized_terms.len() == 1 {
+                    let tokenized_term = tokenized_terms.into_iter().next().unwrap();
+                    Box::new(query_builder(tokenized_term, fuzzy.fuzzy_distance, true))
+                } else {
+                    Box::new(BooleanQuery::intersection(
+                        tokenized_terms
+                            .into_iter()
+                            .map(|term| -> Box<dyn Query> { Box::new(query_builder(term, fuzzy.fuzzy_distance, true)) })
+                            .collect(),
+                    ))
+                }
+            }
         };
 
         Some(query)
@@ -704,13 +703,56 @@ impl TryFrom<&nidx_protos::graph_query::Node> for Node {
     type Error = anyhow::Error;
 
     fn try_from(node_pb: &nidx_protos::graph_query::Node) -> Result<Self, Self::Error> {
-        let value = node_pb.value.clone().map(|value| match node_pb.match_kind() {
-            nidx_protos::graph_query::node::MatchKind::Exact => Term::Exact(value),
-            nidx_protos::graph_query::node::MatchKind::Fuzzy => Term::Fuzzy(FuzzyTerm {
-                value,
-                fuzzy_distance: DEFAULT_NODE_VALUE_FUZZY_DISTANCE,
-                is_prefix: true,
-            }),
+        let value = node_pb.value.clone().map(|value| {
+            if let Some(match_kind) = node_pb.new_match_kind {
+                match match_kind {
+                    nidx_protos::graph_query::node::NewMatchKind::Exact(exact) => match exact.kind() {
+                        MatchLocation::Full => Term::Exact(value),
+                        MatchLocation::Prefix => Term::Fuzzy(FuzzyTerm {
+                            value,
+                            fuzzy_distance: 0,
+                            is_prefix: true,
+                        }),
+                        MatchLocation::Words => Term::ExactWord(value),
+                        MatchLocation::PrefixWords => Term::FuzzyWord(FuzzyTerm {
+                            value,
+                            fuzzy_distance: 0,
+                            is_prefix: true,
+                        }),
+                    },
+                    nidx_protos::graph_query::node::NewMatchKind::Fuzzy(fuzzy) => match fuzzy.kind() {
+                        MatchLocation::Full => Term::Fuzzy(FuzzyTerm {
+                            value,
+                            fuzzy_distance: fuzzy.distance as u8,
+                            is_prefix: false,
+                        }),
+                        MatchLocation::Prefix => Term::Fuzzy(FuzzyTerm {
+                            value,
+                            fuzzy_distance: fuzzy.distance as u8,
+                            is_prefix: true,
+                        }),
+                        MatchLocation::Words => Term::FuzzyWord(FuzzyTerm {
+                            value,
+                            fuzzy_distance: fuzzy.distance as u8,
+                            is_prefix: false,
+                        }),
+                        MatchLocation::PrefixWords => Term::FuzzyWord(FuzzyTerm {
+                            value,
+                            fuzzy_distance: fuzzy.distance as u8,
+                            is_prefix: true,
+                        }),
+                    },
+                }
+            } else {
+                match node_pb.match_kind() {
+                    nidx_protos::graph_query::node::MatchKind::DeprecatedExact => Term::Exact(value),
+                    nidx_protos::graph_query::node::MatchKind::DeprecatedFuzzy => Term::Fuzzy(FuzzyTerm {
+                        value,
+                        fuzzy_distance: DEFAULT_NODE_VALUE_FUZZY_DISTANCE,
+                        is_prefix: true,
+                    }),
+                }
+            }
         });
         let node_type = node_pb.node_type.map(NodeType::try_from).transpose()?;
         let node_subtype = node_pb.node_subtype.clone();

{nidx_binding-6.3.4.post161 → nidx_binding-6.3.4.post178}/nidx_relation/src/lib.rs

@@ -27,22 +27,27 @@ mod schema;
 mod top_unique_n;
 
 use anyhow::anyhow;
+use nidx_protos::graph_query;
+use nidx_protos::graph_query::node::FuzzyMatch;
+use nidx_protos::graph_query::node::MatchLocation;
+use nidx_protos::graph_query::node::NewMatchKind;
+use nidx_protos::graph_query::path_query;
+use nidx_protos::graph_search_request::QueryKind;
 use nidx_protos::{
-    GraphSearchRequest, GraphSearchResponse, RelationNode, RelationNodeFilter, RelationPrefixSearchRequest,
-    RelationSearchRequest, RelationSearchResponse, relation_node::NodeType, relation_prefix_search_request::Search,
-    resource::ResourceStatus,
+    GraphSearchRequest, GraphSearchResponse, RelationNode, RelationSearchRequest, RelationSearchResponse,
+    relation_node::NodeType, resource::ResourceStatus,
 };
 use nidx_tantivy::{
     TantivyIndexer, TantivyMeta, TantivySegmentMetadata,
     index_reader::{DeletionQueryBuilder, open_index_with_deletions},
 };
 use nidx_types::{OpenIndexMetadata, prefilter::PrefilterResult};
-use reader::{HashedRelationNode, RelationsReaderService};
+use reader::{FUZZY_DISTANCE, NUMBER_OF_RESULTS_SUGGEST, RelationsReaderService};
 use resource_indexer::index_relations;
 pub use schema::Schema as RelationSchema;
 use schema::encode_field_id;
 use serde::{Deserialize, Serialize};
-use std::{collections::HashSet, path::Path};
+use std::path::Path;
 use tantivy::{
     Term,
     directory::MmapDirectory,
@@ -217,36 +222,44 @@ impl RelationSearcher {
     }
 
     #[instrument(name = "relation::suggest", skip_all)]
-    pub fn suggest(&self, prefixes: Vec<String>) -> Vec<RelationNode> {
-        let requests = prefixes
-            .iter()
+    pub fn suggest(&self, prefixes: Vec<String>) -> anyhow::Result<Vec<RelationNode>> {
+        let subqueries: Vec<_> = prefixes
+            .into_iter()
             .filter(|prefix| prefix.len() >= MIN_SUGGEST_PREFIX_LENGTH)
-            .cloned()
-            .map(|prefix| RelationSearchRequest {
-                prefix: Some(RelationPrefixSearchRequest {
-                    search: Some(Search::Prefix(prefix)),
-                    node_filters: vec![RelationNodeFilter {
-                        node_type: NodeType::Entity.into(),
+            .map(|prefix| graph_query::PathQuery {
+                query: Some(path_query::Query::Path(graph_query::Path {
+                    source: Some(graph_query::Node {
+                        value: Some(prefix),
+                        node_type: Some(NodeType::Entity.into()),
+                        new_match_kind: Some(NewMatchKind::Fuzzy(FuzzyMatch {
+                            kind: MatchLocation::Prefix.into(),
+                            distance: FUZZY_DISTANCE as u32,
+                        })),
                         ..Default::default()
-                    }],
-                }),
-                ..Default::default()
-            });
-
-        let responses: Vec<_> = requests.map(|request| self.search(&request)).collect();
+                    }),
+                    undirected: true,
+                    ..Default::default()
+                })),
+            })
+            .collect();
 
-        // REVIEW: we are ignoring errors on search, we may want to, at least, log something
-        let entities = responses
-            .into_iter()
-            .flatten() // unwrap errors and continue with successful results
-            .flat_map(|response| response.prefix)
-            .flat_map(|prefix_response| prefix_response.nodes.into_iter());
-
-        // remove duplicate entities
-        let mut seen: HashSet<HashedRelationNode> = HashSet::new();
-        let mut ent_result = entities.collect::<Vec<_>>();
-        ent_result.retain(|e| seen.insert(e.clone().into()));
+        if subqueries.is_empty() {
+            return Ok(vec![]);
+        }
 
-        ent_result
+        let request = GraphSearchRequest {
+            kind: QueryKind::Nodes.into(),
+            top_k: NUMBER_OF_RESULTS_SUGGEST as u32,
+            query: Some(nidx_protos::GraphQuery {
+                path: Some(graph_query::PathQuery {
+                    query: Some(path_query::Query::BoolOr(graph_query::BoolQuery {
+                        operands: subqueries,
+                    })),
+                }),
+            }),
+            ..Default::default()
+        };
+        let response = self.graph_search(&request, PrefilterResult::All)?;
+        Ok(response.nodes)
     }
 }

{nidx_binding-6.3.4.post161 → nidx_binding-6.3.4.post178}/nidx_relation/src/reader.rs

@@ -42,8 +42,8 @@ use crate::schema::{Schema, decode_node, decode_relation, encode_field_id};
 use crate::top_unique_n::TopUniqueN;
 use crate::{RelationConfig, io_maps};
 
-const FUZZY_DISTANCE: u8 = 1;
-const NUMBER_OF_RESULTS_SUGGEST: usize = 10;
+pub const FUZZY_DISTANCE: u8 = 1;
+pub const NUMBER_OF_RESULTS_SUGGEST: usize = 20;
 // Hard limit until we have pagination in place
 const MAX_NUM_RELATIONS_RESULTS: usize = 500;
 
@@ -433,19 +433,20 @@ impl RelationsReaderService {
         let schema = &self.schema;
 
         match search {
+            // TODO: no longer used, we can remove it
             Search::Query(query) => {
                 let mut prefix_nodes_q = Vec::new();
 
                 let words: Vec<_> = query.split_whitespace().collect();
                 for word in words {
                     prefix_nodes_q.push(Node {
-                        value: Some(Term::Fuzzy(FuzzyTerm {
+                        value: Some(Term::FuzzyWord(FuzzyTerm {
                             value: word.to_string(),
                             fuzzy_distance: FUZZY_DISTANCE,
                             is_prefix: false,
                         })),
                         ..Default::default()
-                    })
+                    });
                 }
 
                 // add fuzzy query for all prefixes