nidx-binding 6.9.1.post590__tar.gz → 6.9.1.post600__tar.gz

{nidx_binding-6.9.1.post590 → nidx_binding-6.9.1.post600}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.9.1.post590
+Version: 6.9.1.post600
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.9.1.post590 → nidx_binding-6.9.1.post600}/nidx_protos/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.9.1.post590"
+version = "6.9.1.post600"
 license = "AGPL-3.0-or-later"
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [{ name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.9.1.post590 → nidx_binding-6.9.1.post600}/nidx_vector/src/data_store/v2/paragraph_store.rs

@@ -19,7 +19,11 @@
 //
 
 use memmap2::Mmap;
-use std::{fs::File, io::Write as _, path::Path};
+use std::{
+    fs::File,
+    io::{BufWriter, Write as _},
+    path::Path,
+};
 
 use crate::{
     VectorR,
@@ -112,8 +116,8 @@ impl ParagraphStore {
 }
 
 pub struct ParagraphStoreWriter {
-    data: File,
-    pos: File,
+    data: BufWriter<File>,
+    pos: BufWriter<File>,
     data_pos: u32,
     addr: u32,
 }
@@ -121,8 +125,8 @@ pub struct ParagraphStoreWriter {
 impl ParagraphStoreWriter {
     pub fn new(path: &Path) -> std::io::Result<Self> {
         Ok(Self {
-            pos: File::create(path.join(FILENAME_POS))?,
-            data: File::create(path.join(FILENAME_DATA))?,
+            pos: BufWriter::new(File::create(path.join(FILENAME_POS))?),
+            data: BufWriter::new(File::create(path.join(FILENAME_DATA))?),
             data_pos: 0,
             addr: 0,
         })
@@ -159,9 +163,9 @@ impl ParagraphStoreWriter {
         Ok(self.addr)
     }
 
-    pub fn close(self) -> std::io::Result<()> {
-        self.data.sync_all()?;
-        self.pos.sync_all()?;
+    pub fn close(&mut self) -> std::io::Result<()> {
+        self.data.flush()?;
+        self.pos.flush()?;
         Ok(())
     }
 }

{nidx_binding-6.9.1.post590 → nidx_binding-6.9.1.post600}/nidx_vector/src/data_store/v2/quant_vector_store.rs

@@ -20,7 +20,11 @@
 
 use lazy_static::lazy_static;
 use memmap2::Mmap;
-use std::{fs::File, io::Write as _, path::Path};
+use std::{
+    fs::File,
+    io::{BufWriter, Write as _},
+    path::Path,
+};
 
 use crate::{
     data_store::{OpenReason, VectorAddr},
@@ -86,13 +90,13 @@ impl QuantVectorStore {
 }
 
 pub struct QuantVectorStoreWriter {
-    output: File,
+    output: BufWriter<File>,
 }
 
 impl QuantVectorStoreWriter {
     pub fn new(path: &Path) -> std::io::Result<Self> {
         Ok(Self {
-            output: File::create(path.join(FILENAME))?,
+            output: BufWriter::new(File::create(path.join(FILENAME))?),
         })
     }
 
@@ -100,7 +104,7 @@ impl QuantVectorStoreWriter {
         self.output.write_all(vector)
     }
 
-    pub fn close(self) -> std::io::Result<()> {
-        self.output.sync_all()
+    pub fn close(&mut self) -> std::io::Result<()> {
+        self.output.flush()
     }
 }

{nidx_binding-6.9.1.post590 → nidx_binding-6.9.1.post600}/nidx_vector/src/data_store/v2/vector_store.rs

@@ -22,7 +22,7 @@ use lazy_static::lazy_static;
 use memmap2::Mmap;
 use std::{
     fs::File,
-    io::{Seek, SeekFrom, Write as _},
+    io::{BufWriter, Seek, SeekFrom, Write as _},
     path::Path,
 };
 
@@ -119,7 +119,7 @@ impl VectorStore {
 }
 
 pub struct VectorStoreWriter {
-    output: File,
+    output: BufWriter<File>,
     addr: u32,
     padding_bytes: usize,
 }
@@ -127,7 +127,7 @@ pub struct VectorStoreWriter {
 impl VectorStoreWriter {
     pub fn new(path: &Path, vector_type: &VectorType) -> std::io::Result<Self> {
         Ok(Self {
-            output: File::create(path.join(FILENAME))?,
+            output: BufWriter::new(File::create(path.join(FILENAME))?),
             addr: 0,
             padding_bytes: padding_bytes(vector_type),
         })
@@ -151,7 +151,7 @@ impl VectorStoreWriter {
         Ok((first_addr, last_addr))
     }
 
-    pub fn close(self) -> std::io::Result<()> {
-        self.output.sync_all()
+    pub fn close(mut self) -> std::io::Result<()> {
+        self.output.flush()
     }
 }

{nidx_binding-6.9.1.post590 → nidx_binding-6.9.1.post600}/nidx_vector/src/data_store/v2.rs

@@ -77,7 +77,7 @@ impl DataStoreV2 {
 
         paragraphs.close()?;
         vectors.close()?;
-        if let Some(quantized) = quantized {
+        if let Some(mut quantized) = quantized {
             quantized.close()?;
         }
 

{nidx_binding-6.9.1.post590 → nidx_binding-6.9.1.post600}/nidx_vector/src/segment.rs

@@ -180,8 +180,6 @@ fn merge_indexes<DS: DataStore + 'static>(
         index.advise(memmap2::Advice::Random)?;
     }
 
-    build_indexes(segment_path, &data_store)?;
-
     let metadata = VectorSegmentMetadata {
         path: segment_path.to_path_buf(),
         records: data_store.stored_paragraph_count() as usize,

{nidx_binding-6.9.1.post590 → nidx_binding-6.9.1.post600}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.9.1.post590"
+version = "6.9.1.post600"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.9.1.post590 → nidx_binding-6.9.1.post600}/src/segment_store.rs

@@ -107,6 +107,7 @@ pub async fn download_segment(
 
     let temp_dir2 = temp_dir.clone();
     let mut tar = tar::Archive::new(reader);
+    tar.set_preserve_mtime(false); // We don't care about metadata and we can save a few syscalls
     let result = tokio::task::spawn_blocking(move || tar.unpack(temp_dir2)).await?;
 
     if let Err(e) = result {