nidx-binding 6.9.3.post624__tar.gz → 6.9.3.post630__tar.gz

{nidx_binding-6.9.3.post624 → nidx_binding-6.9.3.post630}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.9.3.post624
+Version: 6.9.3.post630
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.9.3.post624 → nidx_binding-6.9.3.post630}/nidx_protos/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.9.3.post624"
+version = "6.9.3.post630"
 license = "AGPL-3.0-or-later"
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [{ name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.9.3.post624 → nidx_binding-6.9.3.post630}/nidx_vector/src/hnsw/ram_hnsw.rs

@@ -113,10 +113,31 @@ impl RAMHnsw {
     /// A bug in a previous version of this program could cause a node in layer N
     /// to link to a node in layer N-1. This breaks navigation accross layer N.
     /// This function will delete any such link from the graph.
-    pub fn fix_broken_links(&self) {
+    /// Also delete empty layers and entrypoints pointing to unexisting layers
+    pub fn fix_broken_graph(&mut self) {
+        // Fix links to a node not in this lauer
         for l in &self.layers[1..] {
             l.fix_broken_links();
         }
+
+        // Delete empty layers
+        while let Some(layer) = self.layers.last() {
+            if layer.out.is_empty() {
+                self.layers.pop();
+            } else {
+                break;
+            }
+        }
+
+        // If entrypoint point to non-existing layer, point it to the top-most layer
+        if self.entry_point.layer >= self.layers.len() {
+            self.entry_point.layer = self.layers.len() - 1;
+            let last_layer = self.layers.last().unwrap();
+            if !last_layer.contains(&self.entry_point.node) {
+                // If the current entrypoint node is not in the last layer, point to another node that is here
+                self.entry_point.node = *self.layers.last().unwrap().out.keys().next().unwrap();
+            }
+        }
     }
 }
 
@@ -172,7 +193,7 @@ mod tests {
         let mut graph = RAMHnsw::new();
         graph.layers.push(layer0);
         graph.layers.push(layer1);
-        graph.fix_broken_links();
+        graph.fix_broken_graph();
         assert!(graph.layers[1].out[&VectorAddr(0)].read().unwrap().is_empty());
     }
 }

{nidx_binding-6.9.3.post624 → nidx_binding-6.9.3.post630}/nidx_vector/src/hnsw/search.rs

@@ -21,7 +21,7 @@
 use bit_set::BitSet;
 use rustc_hash::FxHashSet;
 use std::cmp::{Ordering, Reverse};
-use std::collections::{BinaryHeap, HashMap, VecDeque};
+use std::collections::{BinaryHeap, HashMap};
 use std::time::Instant;
 use tracing::trace;
 
@@ -190,33 +190,29 @@ impl<'a, DR: DataRetriever> HnswSearcher<'a, DR> {
         }
     }
 
+    /// Breadth-first search to find the closest nodes to the entry-points that fulfill the filtering conditions
     fn closest_up_nodes<L: SearchableLayer>(
         &'a self,
-        entry_points: Vec<VectorAddr>,
+        entry_points: Vec<Cnx>,
         query: &SearchVector,
         layer: L,
         number_of_results: usize,
         mut filter: NodeFilter<'a>,
     ) -> Vec<(VectorAddr, f32)> {
-        // We just need to perform BFS, the replacement is the closest node to the actual
-        // best solution. This algorithm takes a lazy approach to computing the similarity of
-        // candidates.
-
         const MAX_VECTORS_TO_PRELOAD: u32 = 20_000;
         let mut results = Vec::new();
-        let inner_entry_points_iter = entry_points.iter().map(|VectorAddr(inner)| *inner as usize);
+        let inner_entry_points_iter = entry_points.iter().map(|Cnx(VectorAddr(inner), _)| *inner as usize);
         let mut visited_nodes: BitSet = BitSet::from_iter(inner_entry_points_iter);
-        let mut candidates = VecDeque::from(entry_points);
+        let mut candidates = entry_points;
+        candidates.sort_unstable();
 
         let mut preloaded = 0;
 
         loop {
-            let Some(candidate) = candidates.pop_front() else {
+            let Some(Cnx(candidate, candidate_similarity)) = candidates.pop() else {
                 break;
             };
 
-            let candidate_similarity = self.retriever.similarity(candidate, query);
-
             if candidate_similarity < self.retriever.min_score() {
                 break;
             }
@@ -226,22 +222,28 @@ impl<'a, DR: DataRetriever> HnswSearcher<'a, DR> {
             }
 
             if results.len() == number_of_results {
-                return results;
+                break;
             }
 
-            let mut sorted_out: Vec<_> = layer.get_out_edges(candidate).collect();
-            sorted_out.sort_by(|a, b| b.1.total_cmp(&a.1));
-            sorted_out.into_iter().for_each(|(new_candidate, _)| {
-                if !visited_nodes.contains(new_candidate.0 as usize) {
-                    visited_nodes.insert(new_candidate.0 as usize);
-                    candidates.push_back(new_candidate);
-
-                    if self.preload_nodes && preloaded < MAX_VECTORS_TO_PRELOAD {
+            if self.preload_nodes && preloaded < MAX_VECTORS_TO_PRELOAD {
+                for (new_candidate, _) in layer.get_out_edges(candidate) {
+                    if !visited_nodes.contains(new_candidate.0 as usize) {
                         self.retriever.will_need(new_candidate);
                         preloaded += 1;
                     }
                 }
-            });
+            }
+
+            for (new_candidate, _) in layer.get_out_edges(candidate) {
+                if visited_nodes.insert(new_candidate.0 as usize) {
+                    let new_similarity = self.retriever.similarity(new_candidate, query);
+
+                    if new_similarity > self.retriever.min_score() {
+                        candidates.push(Cnx(new_candidate, new_similarity));
+                    }
+                }
+            }
+            candidates.sort_unstable();
         }
 
         results
@@ -364,13 +366,13 @@ impl<'a, DR: DataRetriever> HnswSearcher<'a, DR> {
             let t = Instant::now();
             let reranked = rabitq::rerank_top(neighbours.collect(), k_neighbours, self.retriever, query)
                 .into_iter()
-                .map(|Reverse(Cnx(addr, _))| addr)
+                .map(|Reverse(c)| c)
                 .collect();
             let time = t.elapsed();
             trace!(?time, "HNSW search: reranking");
             reranked
         } else {
-            neighbours.map(|(addr, _)| addr).collect()
+            neighbours.map(|(addr, score)| Cnx(addr, score.score)).collect()
         };
 
         // Find k nodes that match the filter in the last layer

{nidx_binding-6.9.3.post624 → nidx_binding-6.9.3.post630}/nidx_vector/src/segment.rs

@@ -158,7 +158,7 @@ fn merge_indexes<DS: DataStore + 'static>(
         // If there are no deletions, we can reuse the first segment
         // HNSW since its indexes will match the the ones in data_store
         index = DiskHnsw::deserialize(&operants[0].index);
-        index.fix_broken_links();
+        index.fix_broken_graph();
         start_vector_index = operants[0].data_store.stored_vector_count();
     }
     let merged_vectors_count = data_store.stored_vector_count();

{nidx_binding-6.9.3.post624 → nidx_binding-6.9.3.post630}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.9.3.post624"
+version = "6.9.3.post630"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.9.3.post624 → nidx_binding-6.9.3.post630}/src/scheduler/merge_task.rs

@@ -30,12 +30,12 @@ use nidx_types::Seq;
 use super::{log_merge, vector_merge};
 
 pub struct MergeScheduler {
-    settings: MergeSettings,
+    merge_settings: MergeSettings,
 }
 
 impl MergeScheduler {
-    pub fn from_settings(settings: MergeSettings) -> Self {
-        Self { settings }
+    pub fn from_settings(merge_settings: MergeSettings) -> Self {
+        Self { merge_settings }
     }
 
     /// Enqueue merge jobs for segments older than `last_indexed_seq` that aren't
@@ -100,7 +100,7 @@ impl MergeScheduler {
                 HAVING COUNT(segments) > 1 OR index_deletion_window.id IS NOT NULL
             "#,
             i64::from(last_indexed_seq),
-            self.settings.max_deletions as i64
+            self.merge_settings.max_deletions as i64
         )
         .fetch_all(&meta.pool)
         .await?;
@@ -109,9 +109,9 @@ impl MergeScheduler {
             let segment_info: HashMap<_, _> = index.segments.iter().map(|(id, rec, f)| (*id, (*rec, *f))).collect();
             let merges = match index.kind {
                 IndexKind::Text | IndexKind::Paragraph | IndexKind::Relation => {
-                    log_merge::plan_merges(&self.settings.log_merge, index.segments)
+                    log_merge::plan_merges(&self.merge_settings.log, index.segments)
                 }
-                IndexKind::Vector => vector_merge::plan_merges(&self.settings.vector_merge, index.segments),
+                IndexKind::Vector => vector_merge::plan_merges(&self.merge_settings.vector, index.segments),
             };
 
             for m in merges {
@@ -166,7 +166,7 @@ mod tests {
 
         fn merge_scheduler() -> MergeScheduler {
             MergeScheduler::from_settings(MergeSettings {
-                log_merge: LogMergeSettings {
+                log: LogMergeSettings {
                     min_number_of_segments: 3,
                     ..Default::default()
                 },
@@ -326,7 +326,7 @@ mod tests {
         #[sqlx::test]
         async fn test_schedule_merges_with_segment_tags(pool: sqlx::PgPool) -> anyhow::Result<()> {
             let merge_scheduler = MergeScheduler::from_settings(MergeSettings {
-                log_merge: LogMergeSettings {
+                log: LogMergeSettings {
                     min_number_of_segments: 2,
                     ..Default::default()
                 },

{nidx_binding-6.9.3.post624 → nidx_binding-6.9.3.post630}/src/settings.rs

@@ -193,16 +193,16 @@ pub struct StorageSettings {
 #[serde(default)]
 pub struct MergeSettings {
     pub max_deletions: usize,
-    pub log_merge: LogMergeSettings,
-    pub vector_merge: VectorMergeSettings,
+    pub log: LogMergeSettings,
+    pub vector: VectorMergeSettings,
 }
 
 impl Default for MergeSettings {
     fn default() -> Self {
         Self {
             max_deletions: 500,
-            log_merge: Default::default(),
-            vector_merge: Default::default(),
+            log: Default::default(),
+            vector: Default::default(),
         }
     }
 }
@@ -434,7 +434,7 @@ mod tests {
         assert_eq!(settings.indexer.unwrap().nats_server, Some("localhost".to_string()));
         assert_eq!(settings.merge.max_deletions, 1234);
         assert_eq!(
-            settings.merge.log_merge.min_number_of_segments,
+            settings.merge.log.min_number_of_segments,
             LogMergeSettings::default().min_number_of_segments
         );
     }