nidx-binding 6.9.0.post534__tar.gz → 6.9.0.post567__tar.gz

{nidx_binding-6.9.0.post534 → nidx_binding-6.9.0.post567}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.9.0.post534
+Version: 6.9.0.post567
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.9.0.post534 → nidx_binding-6.9.0.post567}/nidx_protos/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.9.0.post534"
+version = "6.9.0.post567"
 license = "AGPL-3.0-or-later"
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [{ name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.9.0.post534 → nidx_binding-6.9.0.post567}/nidx_vector/Cargo.toml

@@ -24,8 +24,6 @@ fst = "0.4.7"
 bit-vec = "0.8.0"
 bincode = "2.0.1"
 rayon = "1.10.0"
-
-[target.'cfg(not(all(target_os="linux",target_arch="aarch64")))'.dependencies]
 simsimd = "4.3.1"
 
 [dev-dependencies]

{nidx_binding-6.9.0.post534 → nidx_binding-6.9.0.post567}/nidx_vector/src/config.rs

@@ -29,6 +29,8 @@ use crate::vector_types::*;
 pub mod flags {
     // pub const DATA_STORE_V2: &str = "data_store_v2";
     pub const FORCE_DATA_STORE_V1: &str = "force_data_store_v1"; // For testing of v1+v2 merges
+
+    pub const DISABLE_RABITQ_SEARCH: &str = "disable_rabitq_search"; // Do not use RaBitQ vectors in search
 }
 
 #[derive(Default, Debug, Clone, Copy, Serialize, Deserialize)]
@@ -60,6 +62,12 @@ impl VectorType {
         }
     }
 
+    pub fn decode<'a>(&self, data: &'a [u8]) -> &'a [f32] {
+        match self {
+            VectorType::DenseF32 { .. } => dense_f32::decode_vector(data),
+        }
+    }
+
     pub fn vector_alignment(&self) -> usize {
         match self {
             VectorType::DenseF32 { .. } => size_of::<f32>(),
@@ -107,6 +115,11 @@ impl VectorConfig {
             (Similarity::Cosine, VectorType::DenseF32 { .. }) => dense_f32::cosine_similarity,
         }
     }
+
+    pub fn quantizable_vectors(&self) -> bool {
+        matches!(self.similarity, Similarity::Dot)
+            && matches!(&self.vector_type, VectorType::DenseF32 { dimension } if dimension.is_multiple_of(64))
+    }
 }
 
 impl TryFrom<VectorIndexConfig> for VectorConfig {

{nidx_binding-6.9.0.post534 → nidx_binding-6.9.0.post567}/nidx_vector/src/data_store/v1.rs

@@ -21,6 +21,7 @@
 use crate::{
     config::{VectorConfig, VectorType},
     segment::Elem,
+    vector_types::rabitq,
 };
 use memmap2::Mmap;
 use node::Node;
@@ -42,6 +43,10 @@ pub struct DataStoreV1 {
 }
 
 impl DataStore for DataStoreV1 {
+    fn has_quantized(&self) -> bool {
+        false
+    }
+
     fn size_bytes(&self) -> usize {
         self.nodes.len()
     }
@@ -61,9 +66,16 @@ impl DataStore for DataStoreV1 {
         }
     }
 
+    fn get_quantized_vector(&self, _id: VectorAddr) -> rabitq::EncodedVector<'_> {
+        panic!("Store does not have quantized vectors")
+    }
+
     fn will_need(&self, id: VectorAddr) {
         store::will_need(&self.nodes, id.0 as usize, self.vector_len_bytes);
     }
+
+    fn will_need_quantized(&self, _id: VectorAddr) {}
+
     fn as_any(&self) -> &dyn Any {
         self
     }

nidx_binding-6.9.0.post567/nidx_vector/src/data_store/v2/quant_vector_store.rs

@@ -0,0 +1,106 @@
+// Copyright (C) 2021 Bosutech XXI S.L.
+//
+// nucliadb is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at info@nuclia.com.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+//
+
+use lazy_static::lazy_static;
+use memmap2::Mmap;
+use std::{fs::File, io::Write as _, path::Path};
+
+use crate::{
+    data_store::{OpenReason, VectorAddr},
+    vector_types::rabitq,
+};
+
+const FILENAME: &str = "vectors.quant";
+
+/// Storage for quantized vectors of fixed size, just the vectors, metadata is with the raw vectors
+pub struct QuantVectorStore {
+    data: Mmap,
+    vector_len_bytes: usize,
+}
+
+impl QuantVectorStore {
+    pub fn open(path: &Path, vector_len_bytes: usize, reason: &OpenReason) -> std::io::Result<Self> {
+        let data = unsafe { Mmap::map(&File::open(path.join(FILENAME))?)? };
+
+        #[cfg(not(target_os = "windows"))]
+        {
+            let advice = match reason {
+                OpenReason::Create => memmap2::Advice::Sequential,
+                OpenReason::Search => memmap2::Advice::Random,
+            };
+            data.advise(advice)?;
+        }
+
+        Ok(Self { data, vector_len_bytes })
+    }
+
+    pub fn get_vector(&self, addr: VectorAddr) -> rabitq::EncodedVector<'_> {
+        let start = self.record_start(addr);
+        rabitq::EncodedVector::from_bytes(&self.data[start..start + self.vector_len_bytes])
+    }
+
+    fn record_start(&self, VectorAddr(addr): VectorAddr) -> usize {
+        addr as usize * self.vector_len_bytes
+    }
+
+    #[cfg(not(target_os = "windows"))]
+    pub fn will_need(&self, addr: VectorAddr) {
+        lazy_static! {
+            static ref PAGE_SIZE: usize = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize;
+        };
+
+        // Align node pointer to the start page, as required by madvise
+        let start = self.data.as_ptr().wrapping_add(self.record_start(addr));
+        let offset = start.align_offset(*PAGE_SIZE);
+        let (start_page, advise_size) = if offset > 0 {
+            (
+                start.wrapping_add(offset).wrapping_sub(*PAGE_SIZE),
+                self.vector_len_bytes + *PAGE_SIZE - offset,
+            )
+        } else {
+            (start, self.vector_len_bytes)
+        };
+
+        unsafe { libc::madvise(start_page as *mut libc::c_void, advise_size, libc::MADV_WILLNEED) };
+    }
+
+    #[cfg(target_os = "windows")]
+    pub fn will_need(src: &[u8], id: usize, vector_len: usize) {}
+}
+
+pub struct QuantVectorStoreWriter {
+    output: File,
+}
+
+impl QuantVectorStoreWriter {
+    pub fn new(path: &Path) -> std::io::Result<Self> {
+        Ok(Self {
+            output: File::create(path.join(FILENAME))?,
+        })
+    }
+
+    pub fn write(&mut self, vector: &[u8]) -> std::io::Result<()> {
+        self.output.write_all(vector)
+    }
+
+    pub fn close(self) -> std::io::Result<()> {
+        self.output.sync_all()
+    }
+}

{nidx_binding-6.9.0.post534 → nidx_binding-6.9.0.post567}/nidx_vector/src/data_store/v2.rs

@@ -18,7 +18,13 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 //
 
-use crate::{VectorR, config::VectorType, segment::Elem};
+use crate::{
+    VectorR,
+    config::{VectorConfig, VectorType},
+    data_store::v2::quant_vector_store::{QuantVectorStore, QuantVectorStoreWriter},
+    segment::Elem,
+    vector_types::rabitq,
+};
 
 use super::{DataStore, OpenReason, ParagraphAddr, VectorAddr};
 pub use paragraph_store::StoredParagraph;
@@ -27,11 +33,13 @@ use std::path::Path;
 use vector_store::{VectorStore, VectorStoreWriter};
 
 mod paragraph_store;
+mod quant_vector_store;
 mod vector_store;
 
 pub struct DataStoreV2 {
     paragraphs: ParagraphStore,
     vectors: VectorStore,
+    quantized: Option<QuantVectorStore>,
 }
 
 impl DataStoreV2 {
@@ -39,20 +47,39 @@ impl DataStoreV2 {
         Ok(Self {
             vectors: VectorStore::open(path, vector_type, &reason)?,
             paragraphs: ParagraphStore::open(path, &reason)?,
+            quantized: QuantVectorStore::open(
+                path,
+                rabitq::EncodedVector::encoded_len(vector_type.dimension()),
+                &reason,
+            )
+            .ok(),
         })
     }
 
-    pub fn create(path: &Path, entries: Vec<Elem>, vector_type: &VectorType) -> VectorR<()> {
+    pub fn create(path: &Path, entries: Vec<Elem>, config: &VectorConfig) -> VectorR<()> {
         let mut paragraphs = ParagraphStoreWriter::new(path)?;
-        let mut vectors = VectorStoreWriter::new(path, vector_type)?;
+        let mut vectors = VectorStoreWriter::new(path, &config.vector_type)?;
+        let mut quantized = if config.quantizable_vectors() {
+            Some(QuantVectorStoreWriter::new(path)?)
+        } else {
+            None
+        };
 
         for (idx, elem) in (0..).zip(entries.into_iter()) {
-            let (first_vector, _) = vectors.write(idx, elem.vectors.iter().map(|v| vector_type.encode(v)))?;
+            let (first_vector, _) = vectors.write(idx, elem.vectors.iter().map(|v| config.vector_type.encode(v)))?;
+            if let Some(quantized) = &mut quantized {
+                for v in &elem.vectors {
+                    quantized.write(&rabitq::EncodedVector::encode(v))?;
+                }
+            }
             paragraphs.write(StoredParagraph::from_elem(&elem, first_vector))?;
         }
 
         paragraphs.close()?;
         vectors.close()?;
+        if let Some(quantized) = quantized {
+            quantized.close()?;
+        }
 
         Ok(())
     }
@@ -60,10 +87,15 @@ impl DataStoreV2 {
     pub fn merge(
         path: &Path,
         producers: Vec<(impl Iterator<Item = ParagraphAddr>, &dyn DataStore)>,
-        vector_type: &VectorType,
+        config: &VectorConfig,
     ) -> VectorR<()> {
         let mut paragraphs = ParagraphStoreWriter::new(path)?;
-        let mut vectors = VectorStoreWriter::new(path, vector_type)?;
+        let mut vectors = VectorStoreWriter::new(path, &config.vector_type)?;
+        let mut quantized = if config.quantizable_vectors() {
+            Some(QuantVectorStoreWriter::new(path)?)
+        } else {
+            None
+        };
 
         let mut p_idx = 0;
         for (alive, store) in producers {
@@ -74,6 +106,20 @@ impl DataStoreV2 {
 
                 // Write to new store
                 let (first_vector, last_vector) = vectors.write(p_idx, p_vectors)?;
+                if let Some(quantized) = &mut quantized {
+                    // Copy quantized vectors if they exist, calculate them if not
+                    if store.has_quantized() {
+                        for vec_addr in paragraph.vectors(&paragraph_addr) {
+                            quantized.write(store.get_quantized_vector(vec_addr).bytes())?;
+                        }
+                    } else {
+                        let p_vectors = paragraph.vectors(&paragraph_addr).map(|v| store.get_vector(v).vector());
+                        for v in p_vectors {
+                            quantized.write(&rabitq::EncodedVector::encode(config.vector_type.decode(v)))?;
+                        }
+                    }
+                }
+
                 paragraphs.write_paragraph_ref(paragraph, first_vector, last_vector - first_vector + 1)?;
                 p_idx += 1;
             }
@@ -87,6 +133,10 @@ impl DataStoreV2 {
 }
 
 impl DataStore for DataStoreV2 {
+    fn has_quantized(&self) -> bool {
+        self.quantized.is_some()
+    }
+
     fn size_bytes(&self) -> usize {
         self.vectors.size_bytes() + self.paragraphs.size_bytes()
     }
@@ -107,10 +157,21 @@ impl DataStore for DataStoreV2 {
         self.vectors.get_vector(id)
     }
 
+    fn get_quantized_vector(&self, id: VectorAddr) -> rabitq::EncodedVector<'_> {
+        let Some(quantized) = &self.quantized else {
+            panic!("Store does not have quantized vectors")
+        };
+        quantized.get_vector(id)
+    }
+
     fn will_need(&self, id: VectorAddr) {
         self.vectors.will_need(id);
     }
 
+    fn will_need_quantized(&self, id: VectorAddr) {
+        self.quantized.as_ref().unwrap().will_need(id);
+    }
+
     fn as_any(&self) -> &dyn std::any::Any {
         self
     }

{nidx_binding-6.9.0.post534 → nidx_binding-6.9.0.post567}/nidx_vector/src/data_store.rs

@@ -28,7 +28,7 @@ pub use v1::node::Node;
 pub use v2::DataStoreV2;
 use v2::StoredParagraph;
 
-use crate::{ParagraphAddr, VectorAddr};
+use crate::{ParagraphAddr, VectorAddr, vector_types::rabitq};
 
 pub enum OpenReason {
     Search,
@@ -94,8 +94,11 @@ pub trait DataStore: Sync + Send {
     fn stored_vector_count(&self) -> u32;
     fn get_paragraph(&self, id: ParagraphAddr) -> ParagraphRef<'_>;
     fn get_vector(&self, id: VectorAddr) -> VectorRef<'_>;
+    fn get_quantized_vector(&self, id: VectorAddr) -> rabitq::EncodedVector<'_>;
     fn will_need(&self, id: VectorAddr);
+    fn will_need_quantized(&self, id: VectorAddr);
     fn as_any(&self) -> &dyn Any;
+    fn has_quantized(&self) -> bool;
 }
 
 pub fn iter_paragraphs(data_store: &impl DataStore) -> impl Iterator<Item = ParagraphAddr> {

{nidx_binding-6.9.0.post534 → nidx_binding-6.9.0.post567}/nidx_vector/src/hnsw/ops_hnsw.rs

@@ -28,6 +28,7 @@ use std::cmp::{Ordering, Reverse};
 use std::collections::{BinaryHeap, HashMap, HashSet, VecDeque};
 
 use crate::inverted_index::FilterBitSet;
+use crate::vector_types::rabitq;
 use crate::{ParagraphAddr, VectorAddr};
 
 use super::params;
@@ -35,12 +36,22 @@ use super::*;
 
 /// Implementors of this trait can guide the hnsw search
 pub trait DataRetriever: std::marker::Sync {
-    fn similarity(&self, x: VectorAddr, y: VectorAddr) -> f32;
+    fn similarity(&self, x: VectorAddr, y: &SearchVector) -> f32;
+    fn similarity_upper_bound(&self, x: VectorAddr, y: &SearchVector) -> f32;
     fn paragraph(&self, x: VectorAddr) -> ParagraphAddr;
     fn get_vector(&self, x: VectorAddr) -> &[u8];
     /// Embeddings with smaller similarity should not be considered.
     fn min_score(&self) -> f32;
+    /// Preload all data for a vector + paragraph (needed for similarity + filtering)
     fn will_need(&self, x: VectorAddr);
+    /// Preload a vector (only the vector, for similarity comparison)
+    fn will_need_vector(&self, x: VectorAddr);
+}
+
+pub enum SearchVector {
+    Stored(VectorAddr),
+    Query(Vec<u8>),
+    RabitQ(rabitq::QueryVector),
 }
 
 /// Implementors of this trait are layers of an HNSW where a nearest neighbour search can be ran.
@@ -142,7 +153,7 @@ impl<'a, DR: DataRetriever> HnswOps<'a, DR> {
                     if let Some((_, edge)) = layer.get_out_edges(x).find(|&(z, _)| z == y) {
                         edge
                     } else {
-                        self.similarity(x, y)
+                        self.retriever.similarity(x, &SearchVector::Stored(y))
                     }
                 })
                 .all(|inter_sim| sim > inter_sim);
@@ -163,9 +174,6 @@ impl<'a, DR: DataRetriever> HnswOps<'a, DR> {
 
         results
     }
-    fn similarity(&self, x: VectorAddr, y: VectorAddr) -> f32 {
-        self.retriever.similarity(x, y)
-    }
     fn get_random_layer(&mut self) -> usize {
         let sample: f64 = self.layer_rng.sample(self.distribution);
         let picked_level = -sample.ln() * params::level_factor();
@@ -174,7 +182,7 @@ impl<'a, DR: DataRetriever> HnswOps<'a, DR> {
     fn closest_up_nodes<L: Layer>(
         &'a self,
         entry_points: Vec<VectorAddr>,
-        query: VectorAddr,
+        query: &SearchVector,
         layer: L,
         number_of_results: usize,
         mut filter: NodeFilter<'a, DR>,
@@ -196,7 +204,7 @@ impl<'a, DR: DataRetriever> HnswOps<'a, DR> {
                 break;
             };
 
-            let candidate_similarity = self.similarity(query, candidate);
+            let candidate_similarity = self.retriever.similarity(candidate, query);
 
             if candidate_similarity < self.retriever.min_score() {
                 break;
@@ -231,34 +239,43 @@ impl<'a, DR: DataRetriever> HnswOps<'a, DR> {
     }
     fn layer_search<L: Layer>(
         &self,
-        x: VectorAddr,
+        query: &SearchVector,
         layer: L,
         k_neighbours: usize,
         entry_points: &[VectorAddr],
     ) -> Neighbours {
+        // Nodes already visited
         let mut visited = FxHashSet::default();
+        // Nodes to visit
         let mut candidates = BinaryHeap::new();
+        // Best results so far
         let mut ms_neighbours = BinaryHeap::new();
+
+        // The initial candidates are the entry points
         for ep in entry_points.iter().copied() {
             visited.insert(ep);
-            let similarity = self.similarity(x, ep);
+            let similarity = self.retriever.similarity(ep, query);
             candidates.push(Cnx(ep, similarity));
             ms_neighbours.push(Reverse(Cnx(ep, similarity)));
         }
+
         loop {
             match (candidates.pop(), ms_neighbours.peek().cloned()) {
+                // No more candidates, done
                 (None, _) => break,
+                // Candidate is worse than worse result, done
                 (Some(Cnx(_, cs)), Some(Reverse(Cnx(_, ws)))) if cs < ws => break,
+                // Candidate is better than worse result
                 (Some(Cnx(cn, _)), Some(Reverse(Cnx(_, mut ws)))) => {
                     for (y, _) in layer.get_out_edges(cn) {
                         if self.preload_nodes && !visited.contains(&y) {
-                            self.retriever.will_need(y);
+                            self.retriever.will_need_vector(y);
                         }
                     }
                     for (y, _) in layer.get_out_edges(cn) {
                         if !visited.contains(&y) {
                             visited.insert(y);
-                            let similarity = self.similarity(x, y);
+                            let similarity = self.retriever.similarity(y, query);
                             if similarity > ws || ms_neighbours.len() < k_neighbours {
                                 candidates.push(Cnx(y, similarity));
                                 ms_neighbours.push(Reverse(Cnx(y, similarity)));
@@ -288,7 +305,8 @@ impl<'a, DR: DataRetriever> HnswOps<'a, DR> {
         mmax: usize,
     ) -> Vec<VectorAddr> {
         use params::*;
-        let neighbours = self.layer_search::<&RAMLayer>(x, layer, ef_construction(), entry_points);
+        let neighbours =
+            self.layer_search::<&RAMLayer>(&SearchVector::Stored(x), layer, ef_construction(), entry_points);
         let neighbours = self.select_neighbours_heuristic(m(), neighbours, layer);
         let mut needs_repair = HashSet::new();
         let mut result = Vec::with_capacity(neighbours.len());
@@ -325,7 +343,7 @@ impl<'a, DR: DataRetriever> HnswOps<'a, DR> {
                 for l in (0..=top_layer).rev() {
                     if l > level {
                         // Above insertion point, just search
-                        eps[0] = self.layer_search(x, &hnsw.layers[l], 1, &eps)[0].0;
+                        eps[0] = self.layer_search(&SearchVector::Stored(x), &hnsw.layers[l], 1, &eps)[0].0;
                     } else {
                         eps = self.layer_insert(x, &mut hnsw.layers[l], &eps, params::m_max_for_layer(l));
                     }
@@ -337,7 +355,7 @@ impl<'a, DR: DataRetriever> HnswOps<'a, DR> {
 
     pub fn search<H: Hnsw>(
         &self,
-        query: VectorAddr,
+        query: &SearchVector,
         hnsw: H,
         k_neighbours: usize,
         with_filter: &FilterBitSet,
@@ -362,9 +380,30 @@ impl<'a, DR: DataRetriever> HnswOps<'a, DR> {
             crnt_layer -= 1;
         }
 
+        // If using RabitQ, request more vectors to rerank later
+        let original_query = if let SearchVector::RabitQ(rq) = query {
+            Some(&SearchVector::Query(rq.original().to_vec()))
+        } else {
+            None
+        };
+        let last_layer_k = if original_query.is_some() {
+            std::cmp::min(k_neighbours * 100, 2000)
+        } else {
+            k_neighbours
+        };
+
         let entry_points: Vec<_> = neighbours.into_iter().map(|(node, _)| node).collect();
         let layer = hnsw.get_layer(crnt_layer);
-        let neighbors = self.layer_search(query, layer, k_neighbours, &entry_points);
+        let mut neighbors = self.layer_search(query, layer, last_layer_k, &entry_points);
+
+        // If using RabitQ, rerank using the original vectors
+        if let Some(query) = original_query {
+            neighbors = neighbors
+                .into_iter()
+                .map(|(addr, _)| (addr, self.retriever.similarity(addr, query)))
+                .collect();
+            neighbors.sort_unstable_by(|x, y| y.1.partial_cmp(&x.1).unwrap_or(Ordering::Equal));
+        }
 
         let filter = NodeFilter {
             filter: with_filter,
@@ -373,8 +412,15 @@ impl<'a, DR: DataRetriever> HnswOps<'a, DR> {
             vec_counter: RepCounter::new(!with_duplicates),
         };
         let layer_zero = hnsw.get_layer(0);
-        let entry_points: Vec<_> = neighbors.into_iter().map(|(node, _)| node).collect();
-        let mut filtered_result = self.closest_up_nodes(entry_points, query, layer_zero, k_neighbours, filter);
+        let entry_points: Vec<_> = neighbors.into_iter().take(k_neighbours).map(|(node, _)| node).collect();
+
+        let mut filtered_result = self.closest_up_nodes(
+            entry_points,
+            original_query.unwrap_or(query),
+            layer_zero,
+            k_neighbours,
+            filter,
+        );
 
         // order may be lost
         filtered_result.sort_by(|a, b| b.1.total_cmp(&a.1));

{nidx_binding-6.9.0.post534 → nidx_binding-6.9.0.post567}/nidx_vector/src/hnsw.rs

@@ -27,4 +27,5 @@ pub use disk_hnsw::DiskHnsw;
 pub use ops_hnsw::Cnx;
 pub use ops_hnsw::DataRetriever;
 pub use ops_hnsw::HnswOps;
+pub use ops_hnsw::SearchVector;
 pub use ram_hnsw::RAMHnsw;

{nidx_binding-6.9.0.post534 → nidx_binding-6.9.0.post567}/nidx_vector/src/lib.rs

@@ -21,7 +21,7 @@
 pub mod config;
 mod data_store;
 mod data_types;
-mod formula;
+pub mod formula;
 mod hnsw;
 mod indexer;
 mod inverted_index;
@@ -29,9 +29,9 @@ mod multivector;
 mod query_io;
 mod request_types;
 mod searcher;
-mod segment;
+pub mod segment;
 mod utils;
-mod vector_types;
+pub mod vector_types;
 
 use config::VectorConfig;
 use indexer::{ResourceWrapper, index_resource};
@@ -52,7 +52,7 @@ pub use request_types::VectorSearchRequest;
 #[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq, Hash)]
 pub struct ParagraphAddr(u32);
 #[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq, Hash)]
-pub struct VectorAddr(u32);
+pub struct VectorAddr(pub u32);
 
 #[derive(Clone, Serialize, Deserialize)]
 pub struct VectorSegmentMeta {

{nidx_binding-6.9.0.post534 → nidx_binding-6.9.0.post567}/nidx_vector/src/segment/tests.rs

@@ -32,13 +32,16 @@ use crate::segment::{self, Elem};
 const CONFIG: VectorConfig = VectorConfig {
     similarity: Similarity::Cosine,
     normalize_vectors: false,
-    vector_type: crate::config::VectorType::DenseF32 { dimension: 178 },
+    vector_type: crate::config::VectorType::DenseF32 { dimension: 128 },
     flags: vec![],
     vector_cardinality: VectorCardinality::Single,
 };
 
 fn create_query() -> Vec<f32> {
-    let v: Vec<_> = vec![rand::random::<f32>; 178].into_iter().map(|f| f()).collect();
+    let v: Vec<_> = vec![rand::random::<f32>; 128]
+        .into_iter()
+        .map(|f| f() * 2.0 - 1.0)
+        .collect();
     let mut modulus = 0.0;
     for w in &v {
         modulus += w * w;
@@ -61,12 +64,12 @@ fn simple_flow() {
     let mut expected_keys = vec![];
     for i in 0..50 {
         let key = format!("9cb39c75f8d9498d8f82d92b173011f5/f/field/0-{i}");
-        let vector = vec![rand::random::<f32>(); 178];
+        let vector = vec![rand::random::<f32>(); 128];
         elems.push(Elem::new(key.clone(), vector, labels.clone(), None));
         expected_keys.push(key);
     }
     let segment = segment::create(temp_dir.path(), elems, &CONFIG, HashSet::new()).unwrap();
-    let query = vec![rand::random::<f32>(); 178];
+    let query = vec![rand::random::<f32>(); 128];
     let no_results = 10;
     let formula = queries[..20].iter().fold(Formula::new(), |mut acc, i| {
         acc.extend(i.clone());