nidx-binding 6.9.1.post596__tar.gz → 6.9.1.post600__tar.gz

{nidx_binding-6.9.1.post596 → nidx_binding-6.9.1.post600}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.9.1.post596
+Version: 6.9.1.post600
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.9.1.post596 → nidx_binding-6.9.1.post600}/nidx_protos/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.9.1.post596"
+version = "6.9.1.post600"
 license = "AGPL-3.0-or-later"
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [{ name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.9.1.post596 → nidx_binding-6.9.1.post600}/nidx_vector/src/data_store/v2/paragraph_store.rs

@@ -19,7 +19,11 @@
 //
 
 use memmap2::Mmap;
-use std::{fs::File, io::Write as _, path::Path};
+use std::{
+    fs::File,
+    io::{BufWriter, Write as _},
+    path::Path,
+};
 
 use crate::{
     VectorR,
@@ -112,8 +116,8 @@ impl ParagraphStore {
 }
 
 pub struct ParagraphStoreWriter {
-    data: File,
-    pos: File,
+    data: BufWriter<File>,
+    pos: BufWriter<File>,
     data_pos: u32,
     addr: u32,
 }
@@ -121,8 +125,8 @@ pub struct ParagraphStoreWriter {
 impl ParagraphStoreWriter {
     pub fn new(path: &Path) -> std::io::Result<Self> {
         Ok(Self {
-            pos: File::create(path.join(FILENAME_POS))?,
-            data: File::create(path.join(FILENAME_DATA))?,
+            pos: BufWriter::new(File::create(path.join(FILENAME_POS))?),
+            data: BufWriter::new(File::create(path.join(FILENAME_DATA))?),
             data_pos: 0,
             addr: 0,
         })
@@ -159,9 +163,9 @@ impl ParagraphStoreWriter {
         Ok(self.addr)
     }
 
-    pub fn close(self) -> std::io::Result<()> {
-        self.data.sync_all()?;
-        self.pos.sync_all()?;
+    pub fn close(&mut self) -> std::io::Result<()> {
+        self.data.flush()?;
+        self.pos.flush()?;
         Ok(())
     }
 }

{nidx_binding-6.9.1.post596 → nidx_binding-6.9.1.post600}/nidx_vector/src/data_store/v2/quant_vector_store.rs

@@ -20,7 +20,11 @@
 
 use lazy_static::lazy_static;
 use memmap2::Mmap;
-use std::{fs::File, io::Write as _, path::Path};
+use std::{
+    fs::File,
+    io::{BufWriter, Write as _},
+    path::Path,
+};
 
 use crate::{
     data_store::{OpenReason, VectorAddr},
@@ -86,13 +90,13 @@ impl QuantVectorStore {
 }
 
 pub struct QuantVectorStoreWriter {
-    output: File,
+    output: BufWriter<File>,
 }
 
 impl QuantVectorStoreWriter {
     pub fn new(path: &Path) -> std::io::Result<Self> {
         Ok(Self {
-            output: File::create(path.join(FILENAME))?,
+            output: BufWriter::new(File::create(path.join(FILENAME))?),
         })
     }
 
@@ -100,7 +104,7 @@ impl QuantVectorStoreWriter {
         self.output.write_all(vector)
     }
 
-    pub fn close(self) -> std::io::Result<()> {
-        self.output.sync_all()
+    pub fn close(&mut self) -> std::io::Result<()> {
+        self.output.flush()
     }
 }

{nidx_binding-6.9.1.post596 → nidx_binding-6.9.1.post600}/nidx_vector/src/data_store/v2/vector_store.rs

@@ -22,7 +22,7 @@ use lazy_static::lazy_static;
 use memmap2::Mmap;
 use std::{
     fs::File,
-    io::{Seek, SeekFrom, Write as _},
+    io::{BufWriter, Seek, SeekFrom, Write as _},
     path::Path,
 };
 
@@ -119,7 +119,7 @@ impl VectorStore {
 }
 
 pub struct VectorStoreWriter {
-    output: File,
+    output: BufWriter<File>,
     addr: u32,
     padding_bytes: usize,
 }
@@ -127,7 +127,7 @@ pub struct VectorStoreWriter {
 impl VectorStoreWriter {
     pub fn new(path: &Path, vector_type: &VectorType) -> std::io::Result<Self> {
         Ok(Self {
-            output: File::create(path.join(FILENAME))?,
+            output: BufWriter::new(File::create(path.join(FILENAME))?),
             addr: 0,
             padding_bytes: padding_bytes(vector_type),
         })
@@ -151,7 +151,7 @@ impl VectorStoreWriter {
         Ok((first_addr, last_addr))
     }
 
-    pub fn close(self) -> std::io::Result<()> {
-        self.output.sync_all()
+    pub fn close(mut self) -> std::io::Result<()> {
+        self.output.flush()
     }
 }

{nidx_binding-6.9.1.post596 → nidx_binding-6.9.1.post600}/nidx_vector/src/data_store/v2.rs

@@ -77,7 +77,7 @@ impl DataStoreV2 {
 
         paragraphs.close()?;
         vectors.close()?;
-        if let Some(quantized) = quantized {
+        if let Some(mut quantized) = quantized {
             quantized.close()?;
         }
 

{nidx_binding-6.9.1.post596 → nidx_binding-6.9.1.post600}/nidx_vector/src/searcher.rs

@@ -156,10 +156,11 @@ impl<'a> Fssc<'a> {
             buff: HashSet::with_capacity(size),
         }
     }
-    fn add(&mut self, candidate: ScoredParagraph<'a>, vector: Vec<u8>) {
-        if !self.with_duplicates && self.seen.contains(&vector) {
+    fn add(&mut self, candidate: ScoredParagraph<'a>, vector: &[u8]) {
+        if !self.with_duplicates && self.seen.contains(vector) {
             return;
         } else if !self.with_duplicates {
+            let vector = vector.to_vec();
             self.seen.insert(vector);
         }
 
@@ -260,37 +261,23 @@ impl Searcher {
         let min_score = request.min_score();
         let mut ffsv = Fssc::new(request.no_results(), with_duplicates);
 
-        let pool = rayon::ThreadPoolBuilder::new().build().unwrap();
-        let results: Vec<_> = pool.install(|| {
-            self.open_segments
-                .par_iter()
-                .flat_map(|open_segment| {
-                    // Skip this segment if it doesn't match the segment filter
-                    if !segment_filter
-                        .as_ref()
-                        .is_none_or(|f| segment_matches(f, open_segment.tags()))
-                    {
-                        return vec![];
-                    }
-                    let partial_solution =
-                        open_segment.search(query, filter, with_duplicates, no_results, &self.config, min_score);
-
-                    partial_solution
-                        .map(|candidate| {
-                            let addr = candidate.paragraph();
-                            let paragraph = open_segment.get_paragraph(addr);
-                            (
-                                ScoredParagraph::new(open_segment, addr, paragraph, candidate.score()),
-                                candidate.vector().to_vec(),
-                            )
-                        })
-                        .collect()
-                })
-                .collect()
-        });
-
-        for (paragraph, vector) in results {
-            ffsv.add(paragraph, vector);
+        for open_segment in &self.open_segments {
+            // Skip this segment if it doesn't match the segment filter
+            if !segment_filter
+                .as_ref()
+                .is_none_or(|f| segment_matches(f, open_segment.tags()))
+            {
+                continue;
+            }
+            let partial_solution =
+                open_segment.search(query, filter, with_duplicates, no_results, &self.config, min_score);
+
+            for candidate in partial_solution {
+                let addr = candidate.paragraph();
+                let paragraph = open_segment.get_paragraph(addr);
+                let scored_paragraph = ScoredParagraph::new(open_segment, addr, paragraph, candidate.score());
+                ffsv.add(scored_paragraph, candidate.vector());
+            }
         }
 
         Ok(ffsv.into())

{nidx_binding-6.9.1.post596 → nidx_binding-6.9.1.post600}/nidx_vector/src/segment.rs

@@ -180,8 +180,6 @@ fn merge_indexes<DS: DataStore + 'static>(
         index.advise(memmap2::Advice::Random)?;
     }
 
-    build_indexes(segment_path, &data_store)?;
-
     let metadata = VectorSegmentMetadata {
         path: segment_path.to_path_buf(),
         records: data_store.stored_paragraph_count() as usize,

{nidx_binding-6.9.1.post596 → nidx_binding-6.9.1.post600}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.9.1.post596"
+version = "6.9.1.post600"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.9.1.post596 → nidx_binding-6.9.1.post600}/src/segment_store.rs

@@ -107,6 +107,7 @@ pub async fn download_segment(
 
     let temp_dir2 = temp_dir.clone();
     let mut tar = tar::Archive::new(reader);
+    tar.set_preserve_mtime(false); // We don't care about metadata and we can save a few syscalls
     let result = tokio::task::spawn_blocking(move || tar.unpack(temp_dir2)).await?;
 
     if let Err(e) = result {