nidx-binding 6.3.4.post167__tar.gz → 6.3.4.post181__tar.gz

{nidx_binding-6.3.4.post167 → nidx_binding-6.3.4.post181}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.4.post167
+Version: 6.3.4.post181
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.4.post167 → nidx_binding-6.3.4.post181}/nidx_paragraph/src/reader.rs

@@ -36,7 +36,7 @@ use crate::search_query::{SharedTermC, search_query, streaming_query, suggest_qu
 use crate::search_response::{SearchBm25Response, SearchFacetsResponse, SearchIntResponse, extract_labels};
 
 const FUZZY_DISTANCE: u8 = 1;
-const NUMBER_OF_RESULTS_SUGGEST: usize = 10;
+const NUMBER_OF_RESULTS_SUGGEST: usize = 20;
 
 pub struct ParagraphReaderService {
     pub index: Index,

{nidx_binding-6.3.4.post167 → nidx_binding-6.3.4.post181}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.4.post167"
+version = "6.3.4.post181"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.4.post167 → nidx_binding-6.3.4.post181}/nidx_relation/src/lib.rs

@@ -27,22 +27,27 @@ mod schema;
 mod top_unique_n;
 
 use anyhow::anyhow;
+use nidx_protos::graph_query;
+use nidx_protos::graph_query::node::FuzzyMatch;
+use nidx_protos::graph_query::node::MatchLocation;
+use nidx_protos::graph_query::node::NewMatchKind;
+use nidx_protos::graph_query::path_query;
+use nidx_protos::graph_search_request::QueryKind;
 use nidx_protos::{
-    GraphSearchRequest, GraphSearchResponse, RelationNode, RelationNodeFilter, RelationPrefixSearchRequest,
-    RelationSearchRequest, RelationSearchResponse, relation_node::NodeType, relation_prefix_search_request::Search,
-    resource::ResourceStatus,
+    GraphSearchRequest, GraphSearchResponse, RelationNode, RelationSearchRequest, RelationSearchResponse,
+    relation_node::NodeType, resource::ResourceStatus,
 };
 use nidx_tantivy::{
     TantivyIndexer, TantivyMeta, TantivySegmentMetadata,
     index_reader::{DeletionQueryBuilder, open_index_with_deletions},
 };
 use nidx_types::{OpenIndexMetadata, prefilter::PrefilterResult};
-use reader::{HashedRelationNode, RelationsReaderService};
+use reader::{FUZZY_DISTANCE, NUMBER_OF_RESULTS_SUGGEST, RelationsReaderService};
 use resource_indexer::index_relations;
 pub use schema::Schema as RelationSchema;
 use schema::encode_field_id;
 use serde::{Deserialize, Serialize};
-use std::{collections::HashSet, path::Path};
+use std::path::Path;
 use tantivy::{
     Term,
     directory::MmapDirectory,
@@ -217,36 +222,44 @@ impl RelationSearcher {
     }
 
     #[instrument(name = "relation::suggest", skip_all)]
-    pub fn suggest(&self, prefixes: Vec<String>) -> Vec<RelationNode> {
-        let requests = prefixes
-            .iter()
+    pub fn suggest(&self, prefixes: Vec<String>) -> anyhow::Result<Vec<RelationNode>> {
+        let subqueries: Vec<_> = prefixes
+            .into_iter()
             .filter(|prefix| prefix.len() >= MIN_SUGGEST_PREFIX_LENGTH)
-            .cloned()
-            .map(|prefix| RelationSearchRequest {
-                prefix: Some(RelationPrefixSearchRequest {
-                    search: Some(Search::Prefix(prefix)),
-                    node_filters: vec![RelationNodeFilter {
-                        node_type: NodeType::Entity.into(),
+            .map(|prefix| graph_query::PathQuery {
+                query: Some(path_query::Query::Path(graph_query::Path {
+                    source: Some(graph_query::Node {
+                        value: Some(prefix),
+                        node_type: Some(NodeType::Entity.into()),
+                        new_match_kind: Some(NewMatchKind::Fuzzy(FuzzyMatch {
+                            kind: MatchLocation::Prefix.into(),
+                            distance: FUZZY_DISTANCE as u32,
+                        })),
                         ..Default::default()
-                    }],
-                }),
-                ..Default::default()
-            });
-
-        let responses: Vec<_> = requests.map(|request| self.search(&request)).collect();
+                    }),
+                    undirected: true,
+                    ..Default::default()
+                })),
+            })
+            .collect();
 
-        // REVIEW: we are ignoring errors on search, we may want to, at least, log something
-        let entities = responses
-            .into_iter()
-            .flatten() // unwrap errors and continue with successful results
-            .flat_map(|response| response.prefix)
-            .flat_map(|prefix_response| prefix_response.nodes.into_iter());
-
-        // remove duplicate entities
-        let mut seen: HashSet<HashedRelationNode> = HashSet::new();
-        let mut ent_result = entities.collect::<Vec<_>>();
-        ent_result.retain(|e| seen.insert(e.clone().into()));
+        if subqueries.is_empty() {
+            return Ok(vec![]);
+        }
 
-        ent_result
+        let request = GraphSearchRequest {
+            kind: QueryKind::Nodes.into(),
+            top_k: NUMBER_OF_RESULTS_SUGGEST as u32,
+            query: Some(nidx_protos::GraphQuery {
+                path: Some(graph_query::PathQuery {
+                    query: Some(path_query::Query::BoolOr(graph_query::BoolQuery {
+                        operands: subqueries,
+                    })),
+                }),
+            }),
+            ..Default::default()
+        };
+        let response = self.graph_search(&request, PrefilterResult::All)?;
+        Ok(response.nodes)
     }
 }

{nidx_binding-6.3.4.post167 → nidx_binding-6.3.4.post181}/nidx_relation/src/reader.rs

@@ -42,8 +42,8 @@ use crate::schema::{Schema, decode_node, decode_relation, encode_field_id};
 use crate::top_unique_n::TopUniqueN;
 use crate::{RelationConfig, io_maps};
 
-const FUZZY_DISTANCE: u8 = 1;
-const NUMBER_OF_RESULTS_SUGGEST: usize = 10;
+pub const FUZZY_DISTANCE: u8 = 1;
+pub const NUMBER_OF_RESULTS_SUGGEST: usize = 20;
 // Hard limit until we have pagination in place
 const MAX_NUM_RELATIONS_RESULTS: usize = 500;
 
@@ -433,6 +433,7 @@ impl RelationsReaderService {
         let schema = &self.schema;
 
         match search {
+            // TODO: no longer used, we can remove it
             Search::Query(query) => {
                 let mut prefix_nodes_q = Vec::new();
 

{nidx_binding-6.3.4.post167 → nidx_binding-6.3.4.post181}/nidx_relation/tests/test_reader.rs

@@ -299,8 +299,8 @@ fn test_prefix_search() -> anyhow::Result<()> {
         ..Default::default()
     })?;
 
-    // max number of prefixes is fixed
-    assert_eq!(result.prefix.unwrap().nodes.len(), 10);
+    // max number of prefixes is fixed (to 20)
+    assert_eq!(result.prefix.unwrap().nodes.len(), 14);
 
     let result = reader.search(&RelationSearchRequest {
         prefix: Some(RelationPrefixSearchRequest {

{nidx_binding-6.3.4.post167 → nidx_binding-6.3.4.post181}/nidx_text/src/schema.rs

@@ -84,7 +84,7 @@ impl TextSchema {
         // Using a bytes field is slow due to tantivy's implementation being slow with many unique values.
         // A better implementation is tracked in https://github.com/quickwit-oss/tantivy/issues/2090
         let encoded_field_id = if version >= 2 {
-            Some(sb.add_u64_field("encoded_field_id", FAST))
+            Some(sb.add_u64_field("encoded_field_id", FAST | INDEXED))
         } else {
             None
         };

{nidx_binding-6.3.4.post167 → nidx_binding-6.3.4.post181}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.4.post167"
+version = "6.3.4.post181"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.3.4.post167 → nidx_binding-6.3.4.post181}/src/searcher/shard_suggest.rs

@@ -153,6 +153,7 @@ fn blocking_suggest(
     }
 
     if let Some(entities) = rrelation {
+        let entities = entities?;
         response.entity_results = Some(RelationPrefixSearchResponse { nodes: entities });
     }
 

{nidx_binding-6.3.4.post167 → nidx_binding-6.3.4.post181}/tests/test_search_relations.rs

@@ -73,10 +73,11 @@ async fn test_search_relations_by_prefix(pool: PgPool) -> Result<(), Box<dyn std
     assert!(response.prefix.is_some());
     let prefix_response = response.prefix.as_ref().unwrap();
     let results = &prefix_response.nodes;
-    // TODO this constants is spread between relations and paragraphs. It should
-    // be in a single place and common for everyone
-    const MAX_SUGGEST_RESULTS: usize = 10;
-    assert_eq!(results.len(), MAX_SUGGEST_RESULTS);
+    // This test will be removed soon, no need to update this
+    // // TODO this constants is spread between relations and paragraphs. It should
+    // // be in a single place and common for everyone
+    // const MAX_SUGGEST_RESULTS: usize = 20;
+    assert_eq!(results.len(), 15);
 
     // --------------------------------------------------------------
     // Test: prefixed search with "cat" term (some results)

{nidx_binding-6.3.4.post167 → nidx_binding-6.3.4.post181}/tests/test_suggest.rs

@@ -218,12 +218,13 @@ async fn test_suggest_entities(pool: PgPool) -> Result<(), Box<dyn std::error::E
         &["Barcelona", "Bárcenas"],
     );
 
-    // multiple words and result ordering
+    // multiple words
     let response = suggest_entities(&mut fixture, &shard.id, "Solomon Isa").await;
     assert!(response.entity_results.is_some());
     assert_eq!(response.entity_results.as_ref().unwrap().nodes.len(), 2);
-    assert!(response.entity_results.as_ref().unwrap().nodes[0].value == *"Solomon Islands");
-    assert!(response.entity_results.as_ref().unwrap().nodes[1].value == *"Israel");
+    // Due to tantivy limitations, scoring with fuzzy doesn't work as expected
+    // and the order of results is random
+    expect_entities(&response, &["Solomon Islands", "Israel"]);
 
     // Does not find resources by UUID prefix
     let pap_uuid = &shard.resources["pap"];