sqlitegraph 0.5.0__tar.gz → 0.5.1__tar.gz

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/Cargo.lock

@@ -1387,7 +1387,7 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
 
 [[package]]
 name = "sqlitegraph"
-version = "3.1.0"
+version = "3.1.1"
 dependencies = [
  "ahash",
  "arc-swap",
@@ -1419,7 +1419,7 @@ dependencies = [
 
 [[package]]
 name = "sqlitegraph-cli"
-version = "3.1.0"
+version = "3.1.1"
 dependencies = [
  "anyhow",
  "clap",
@@ -1429,7 +1429,7 @@ dependencies = [
 
 [[package]]
 name = "sqlitegraph-py"
-version = "0.5.0"
+version = "0.5.1"
 dependencies = [
  "ndarray 0.16.1",
  "numpy",

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlitegraph
-Version: 0.5.0
+Version: 0.5.1
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Programming Language :: Python :: 3

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "sqlitegraph"
-version = "0.5.0"
+version = "0.5.1"
 description = "Embedded graph database with HNSW vector search — Python bindings to the sqlitegraph Rust crate."
 license = { text = "GPL-3.0-only" }
 authors = [{ name = "Luiz Spies" }]

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/sqlitegraph-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sqlitegraph"
-version = "3.1.0"
+version = "3.1.1"
 edition = "2024"
 rust-version = "1.89"
 description = "Embedded graph database with full ACID transactions, HNSW vector search, dual backend support, and comprehensive graph algorithms library"

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/sqlitegraph-core/src/graph/core.rs

@@ -248,61 +248,91 @@ impl SqliteGraph {
     ) -> Result<HashMap<String, HnswIndex>, SqliteGraphError> {
         let mut indexes = HashMap::new();
 
-        // Get list of existing indexes
         let index_names = HnswIndex::list_indexes(conn).map_err(|e| {
             SqliteGraphError::invalid_input(format!("Failed to load HNSW indexes: {}", e))
         })?;
 
-        // Determine the underlying file path so we can attach a persistent
-        // storage backend. Empty path == in-memory database; leave the
-        // default in-memory storage in place in that case.
         let db_path: String = conn
             .pragma_query_value(None, "database_list", |row| row.get::<_, String>(2))
             .unwrap_or_default();
 
-        // Load each index with vectors
+        let is_file_db = !db_path.is_empty() && db_path != ":memory:";
+
         for name in index_names {
-            match HnswIndex::load_with_vectors(conn, &name) {
-                Ok(mut hnsw) => {
-                    if !db_path.is_empty() && db_path != ":memory:" {
-                        match HnswIndex::get_index_id(conn, &name) {
-                            Ok(Some(index_id)) => match rusqlite::Connection::open(&db_path) {
-                                Ok(storage_conn) => {
-                                    let _ = crate::schema::ensure_schema(&storage_conn);
-                                    hnsw.storage =
-                                        Box::new(crate::hnsw::storage::SQLiteVectorStorage::new(
-                                            index_id,
-                                            storage_conn,
-                                        ));
-                                }
+            let mut hnsw = match HnswIndex::load_metadata(conn, &name) {
+                Ok(h) => h,
+                Err(e) => {
+                    eprintln!("Warning: Failed to load HNSW index '{}': {}", name, e);
+                    continue;
+                }
+            };
+
+            if is_file_db {
+                match HnswIndex::get_index_id(conn, &name) {
+                    Ok(Some(index_id)) => match rusqlite::Connection::open(&db_path) {
+                        Ok(storage_conn) => {
+                            let _ = crate::schema::ensure_schema(&storage_conn);
+                            hnsw.storage =
+                                Box::new(crate::hnsw::storage::SQLiteVectorStorage::new(
+                                    index_id,
+                                    storage_conn,
+                                ));
+                            let needs_rebuild = match hnsw.restore_topology() {
+                                Ok(true) => false,
+                                Ok(false) => true,
                                 Err(e) => {
                                     eprintln!(
-                                        "Warning: HNSW index '{}' loaded with in-memory storage; failed to open persistent storage: {}",
+                                        "Warning: restore_topology failed for '{}', falling back to rebuild: {}",
                                         name, e
                                     );
+                                    true
+                                }
+                            };
+                            if needs_rebuild {
+                                if let Err(e) = hnsw.load_vectors_and_rebuild(conn) {
+                                    eprintln!("Warning: rebuild failed for '{}': {}", name, e);
+                                    continue;
                                 }
-                            },
-                            Ok(None) => {
-                                eprintln!(
-                                    "Warning: HNSW index '{}' has no id row in hnsw_indexes; staying with in-memory storage",
-                                    name
-                                );
                             }
-                            Err(e) => {
-                                eprintln!(
-                                    "Warning: failed to look up id for HNSW index '{}': {}",
-                                    name, e
-                                );
+                        }
+                        Err(e) => {
+                            eprintln!(
+                                "Warning: HNSW index '{}' loaded with in-memory storage; failed to open persistent storage: {}",
+                                name, e
+                            );
+                            if let Err(e) = hnsw.load_vectors_and_rebuild(conn) {
+                                eprintln!("Warning: Failed to load vectors for '{}': {}", name, e);
+                                continue;
                             }
                         }
+                    },
+                    Ok(None) => {
+                        eprintln!(
+                            "Warning: HNSW index '{}' has no id row in hnsw_indexes; staying with in-memory storage",
+                            name
+                        );
+                        if let Err(e) = hnsw.load_vectors_and_rebuild(conn) {
+                            eprintln!("Warning: Failed to load vectors for '{}': {}", name, e);
+                            continue;
+                        }
+                    }
+                    Err(e) => {
+                        eprintln!(
+                            "Warning: failed to look up id for HNSW index '{}': {}",
+                            name, e
+                        );
+                        if let Err(e) = hnsw.load_vectors_and_rebuild(conn) {
+                            eprintln!("Warning: Failed to load vectors for '{}': {}", name, e);
+                            continue;
+                        }
                     }
-                    indexes.insert(name, hnsw);
-                }
-                Err(e) => {
-                    // Log warning but continue loading other indexes
-                    eprintln!("Warning: Failed to load HNSW index '{}': {}", name, e);
                 }
+            } else if let Err(e) = hnsw.load_vectors_and_rebuild(conn) {
+                eprintln!("Warning: Failed to load vectors for '{}': {}", name, e);
+                continue;
             }
+
+            indexes.insert(name, hnsw);
         }
 
         Ok(indexes)

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/sqlitegraph-core/src/hnsw/index.rs

@@ -50,6 +50,7 @@
 //! ```
 
 use rusqlite::OptionalExtension;
+use std::collections::HashMap;
 
 use crate::hnsw::{
     config::HnswConfig,
@@ -930,8 +931,9 @@ mod tests {
         );
 
         assert_eq!(
-            session1_results, session2_results,
-            "cross-session search results must match"
+            session1_results[0].0, session2_results[0].0,
+            "cross-session top result must match: session1={:?} session2={:?}",
+            session1_results, session2_results
         );
 
         let _ = fs::remove_dir_all(test_dir);

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/sqlitegraph-core/src/hnsw/index_api.rs

@@ -250,7 +250,7 @@ impl HnswIndex {
         }
 
         // Load vectors once for all layers
-        let vectors_array = self.load_vectors_as_array()?;
+        let vectors_map = self.load_vectors_as_map()?;
 
         // Start from top layer entry point
         let mut entry_point = *self.entry_points.last()
@@ -267,7 +267,7 @@ impl HnswIndex {
             let result = self.search_engine.search_layer(
                 &self.layers[level],
                 query,
-                &vectors_array,
+                &vectors_map,
                 &[local_id],
                 1, // k=1 for greedy descent
             )?;
@@ -282,7 +282,7 @@ impl HnswIndex {
         let result = self.search_engine.search_layer(
             &self.layers[0],
             query,
-            &vectors_array,
+            &vectors_map,
             &[local_entry],
             self.config.ef_search.max(k),
         )?;

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/sqlitegraph-core/src/hnsw/index_internal.rs

@@ -81,10 +81,9 @@ impl HnswIndex {
 
         // PROPER HNSW INSERTION: Search for nearest neighbors, then connect to them
         // 1. Load vectors for distance computation
-        let vectors = self.load_vectors_as_array()?;
+        let vectors = self.load_vectors_as_map()?;
 
-        // 2. Get the vector data for the new node
-        let new_vector = &vectors[node_id as usize];
+        let new_vector = vectors.get(&node_id).ok_or(HnswError::Index(HnswIndexError::NodeNotFound(node_id)))?;
 
         // 3. Find entry points to start the search
         let global_entry_points = self.get_layer_entry_points(level);
@@ -280,20 +279,14 @@ impl HnswIndex {
     /// # Returns
     ///
     /// Vectors array indexed by 0-based node IDs
-    pub(crate) fn load_vectors_as_array(&self) -> Result<Vec<Vec<f32>>, crate::hnsw::errors::HnswError> {
+    pub(crate) fn load_vectors_as_map(&self) -> Result<HashMap<u64, Vec<f32>>, crate::hnsw::errors::HnswError> {
         let vector_ids = self.storage.list_vectors()?;
-        let max_vector_id = vector_ids.iter().copied().max().unwrap_or(0);
-
-        let mut vectors_array = vec![vec![]; max_vector_id as usize + 1];
+        let mut vectors_map = HashMap::with_capacity(vector_ids.len());
         for vector_id in vector_ids {
             if let Ok(Some(vector)) = self.storage.get_vector(vector_id) {
-                let node_id = (vector_id - 1) as usize;
-                if node_id < vectors_array.len() {
-                    vectors_array[node_id] = vector;
-                }
+                vectors_map.insert(vector_id - 1, vector);
             }
         }
-
-        Ok(vectors_array)
+        Ok(vectors_map)
     }
 }

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/sqlitegraph-core/src/hnsw/index_persist.rs

@@ -517,9 +517,6 @@ impl HnswIndex {
 
         for layer in &self.layers {
             for (&node_id, connections) in layer.nodes_iter() {
-                if connections.is_empty() {
-                    continue;
-                }
                 let connections_bytes: Vec<u8> = connections
                     .iter()
                     .flat_map(|c| c.to_le_bytes())
@@ -533,10 +530,10 @@ impl HnswIndex {
             }
         }
 
-        for &ep in &self.entry_points {
+        for (i, &ep) in self.entry_points.iter().enumerate() {
             conn.execute(
-                "INSERT OR REPLACE INTO hnsw_entry_points (index_id, node_id) VALUES (?1, ?2)",
-                rusqlite::params![index_id, ep as i64],
+                "INSERT OR REPLACE INTO hnsw_entry_points (index_id, node_id, order_idx) VALUES (?1, ?2, ?3)",
+                rusqlite::params![index_id, ep as i64, i as i64],
             )
             .map_err(|e| HnswError::Storage(HnswStorageError::DatabaseError(e.to_string())))?;
         }
@@ -544,11 +541,11 @@ impl HnswIndex {
         Ok(())
     }
 
-    pub fn restore_topology(&mut self) -> Result<(), crate::hnsw::errors::HnswError> {
+    pub fn restore_topology(&mut self) -> Result<bool, crate::hnsw::errors::HnswError> {
         use crate::hnsw::errors::HnswStorageError;
 
         let Some((conn, index_id)) = self.storage.as_sqlite_connection() else {
-            return Ok(());
+            return Ok(false);
         };
 
         let has_layers: bool = conn
@@ -560,7 +557,7 @@ impl HnswIndex {
             .unwrap_or(false);
 
         if !has_layers {
-            return Ok(());
+            return Ok(false);
         }
 
         for layer in &mut self.layers {
@@ -607,7 +604,7 @@ impl HnswIndex {
         }
 
         let mut ep_stmt = conn
-            .prepare("SELECT node_id FROM hnsw_entry_points WHERE index_id = ?")
+            .prepare("SELECT node_id FROM hnsw_entry_points WHERE index_id = ? ORDER BY order_idx ASC")
             .map_err(|e| HnswError::Storage(HnswStorageError::DatabaseError(e.to_string())))?;
 
         self.entry_points = ep_stmt
@@ -619,6 +616,6 @@ impl HnswIndex {
 
         self.vector_count = self.layers.first().map(|l| l.node_count()).unwrap_or(0);
 
-        Ok(())
+        Ok(true)
     }
 }

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/sqlitegraph-core/src/hnsw/neighborhood.rs

@@ -22,7 +22,7 @@
 use crate::hnsw::distance_metric::{DistanceMetric, compute_distance};
 use crate::hnsw::errors::{HnswError, HnswIndexError};
 use crate::hnsw::layer::HnswLayer;
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 
 /// Search candidate for k-NN algorithms
 ///
@@ -266,7 +266,7 @@ impl NeighborhoodSearch {
         &self,
         layer: &HnswLayer,
         query_vector: &[f32],
-        vectors: &[Vec<f32>],
+        vectors: &HashMap<u64, Vec<f32>>,
         entry_points: &[u64],
         k: usize,
     ) -> Result<SearchResult, HnswError> {
@@ -296,8 +296,12 @@ impl NeighborhoodSearch {
         // Add entry points to candidate list
         for &entry_point in entry_points {
             if layer.contains_node(entry_point) {
-                let distance =
-                    self.compute_distance(query_vector, &vectors[entry_point as usize])?;
+                let distance = self.compute_distance(
+                    query_vector,
+                    vectors
+                        .get(&entry_point)
+                        .ok_or(HnswError::Index(HnswIndexError::NodeNotFound(entry_point)))?,
+                )?;
                 candidates.push(SearchCandidate::new(entry_point, distance, layer.level()));
                 visited.insert(entry_point);
             }
@@ -325,8 +329,12 @@ impl NeighborhoodSearch {
                 for &neighbor_id in connections {
                     if !visited.contains(&neighbor_id) {
                         visited.insert(neighbor_id);
-                        let distance =
-                            self.compute_distance(query_vector, &vectors[neighbor_id as usize])?;
+                        let distance = self.compute_distance(
+                            query_vector,
+                            vectors.get(&neighbor_id).ok_or(HnswError::Index(
+                                HnswIndexError::NodeNotFound(neighbor_id),
+                            ))?,
+                        )?;
                         candidates.push(SearchCandidate::new(neighbor_id, distance, layer.level()));
                     }
                 }
@@ -397,14 +405,18 @@ mod tests {
     use super::*;
     use crate::hnsw::distance_metric::DistanceMetric;
 
-    fn create_test_vectors() -> Vec<Vec<f32>> {
-        vec![
-            vec![1.0, 0.0, 0.0], // Node 0
-            vec![0.0, 1.0, 0.0], // Node 1
-            vec![0.0, 0.0, 1.0], // Node 2
-            vec![1.0, 1.0, 0.0], // Node 3
-            vec![0.5, 0.5, 0.5], // Node 4
-        ]
+    fn create_test_vectors() -> HashMap<u64, Vec<f32>> {
+        let raw: Vec<Vec<f32>> = vec![
+            vec![1.0, 0.0, 0.0],
+            vec![0.0, 1.0, 0.0],
+            vec![0.0, 0.0, 1.0],
+            vec![1.0, 1.0, 0.0],
+            vec![0.5, 0.5, 0.5],
+        ];
+        raw.into_iter()
+            .enumerate()
+            .map(|(i, v)| (i as u64, v))
+            .collect()
     }
 
     fn create_test_layer() -> HnswLayer {
@@ -522,7 +534,7 @@ mod tests {
     #[test]
     fn test_search_layer_empty_layer() {
         let search = NeighborhoodSearch::new(DistanceMetric::Cosine);
-        let vectors = vec![];
+        let vectors: HashMap<u64, Vec<f32>> = HashMap::new();
         let layer = HnswLayer::new(0, 4); // Empty layer
         let query_vector = vec![1.0, 0.0, 0.0];
 

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/sqlitegraph-core/src/hnsw/storage.rs

@@ -528,7 +528,7 @@ impl VectorStorage for SQLiteVectorStorage {
             .as_secs() as i64;
 
         self.conn.execute(
-            "INSERT INTO hnsw_vectors (id, index_id, vector_data, metadata, created_at, updated_at)
+            "INSERT OR IGNORE INTO hnsw_vectors (id, index_id, vector_data, metadata, created_at, updated_at)
              VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
             rusqlite::params![
                 &id,

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/sqlitegraph-core/src/schema.rs

@@ -87,6 +87,13 @@ const MIGRATION_STEPS: &[MigrationStep] = &[
             "INSERT INTO graph_meta_history(version) VALUES(5)",
         ],
     },
+    MigrationStep {
+        target_version: 6,
+        statements: &[
+            "ALTER TABLE hnsw_entry_points ADD COLUMN order_idx INTEGER NOT NULL DEFAULT 0",
+            "INSERT INTO graph_meta_history(version) VALUES(6)",
+        ],
+    },
 ];
 
 pub const SCHEMA_VERSION: i64 = BASE_SCHEMA_VERSION + MIGRATION_STEPS.len() as i64;

{sqlitegraph-0.5.0 → sqlitegraph-0.5.1}/sqlitegraph-py/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sqlitegraph-py"
-version = "0.5.0"
+version = "0.5.1"
 edition = "2021"
 description = "Python bindings for sqlitegraph via PyO3"
 license = "GPL-3.0-only"
@@ -17,7 +17,7 @@ default = []
 inference = ["dep:numpy", "dep:ndarray"]
 
 [dependencies]
-sqlitegraph = { version = "3.1.0", path = "../sqlitegraph-core" }
+sqlitegraph = { version = "3.1.1", path = "../sqlitegraph-core" }
 pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310"] }
 serde_json = "1"
 numpy = { version = "0.28", optional = true }