arrowspace 0.24.5__tar.gz → 0.24.8__tar.gz

{arrowspace-0.24.5 → arrowspace-0.24.8}/Cargo.lock

@@ -134,6 +134,12 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "arc-swap"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
+
 [[package]]
 name = "arrow"
 version = "57.1.0"
@@ -351,9 +357,9 @@ dependencies = [
 
 [[package]]
 name = "arrowspace"
-version = "0.24.5"
+version = "0.24.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0771fca361d870c2109f3602ec5916fd6c5c5924738a3d4e6478309ee0c5658"
+checksum = "c5a852f7bbaee37066b280df74b048e33935ef26f9161f63ef0281cda26cb2ab"
 dependencies = [
  "approx 0.5.1",
  "arrow",
@@ -1181,11 +1187,12 @@ dependencies = [
 
 [[package]]
 name = "pyarrowspace"
-version = "0.24.5"
+version = "0.24.8"
 dependencies = [
  "arrowspace",
  "numpy",
  "pyo3",
+ "pyo3-log",
  "rayon",
  "smartcore",
 ]
@@ -1226,6 +1233,17 @@ dependencies = [
  "pyo3-build-config",
 ]
 
+[[package]]
+name = "pyo3-log"
+version = "0.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f8bae9ad5ba08b0b0ed2bb9c2bdbaeccc69cafca96d78cf0fbcea0d45d122bb"
+dependencies = [
+ "arc-swap",
+ "log",
+ "pyo3",
+]
+
 [[package]]
 name = "pyo3-macros"
 version = "0.27.1"

{arrowspace-0.24.5 → arrowspace-0.24.8}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pyarrowspace"
-version = "0.24.5"
+version = "0.24.8"
 edition = "2024"
 description = "Spectral vector search with taumode (λτ) indexing"
 authors = ["Lorenzo <tunedconsulting@gmail.com>"]
@@ -25,6 +25,7 @@ exclude = [
     "tests/output",
     "tests/small_datasets",
     "visualization_output",
+    "storage/"
 ]
 
 
@@ -38,8 +39,9 @@ name = "arrowspace"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrowspace = { version = "0.24.5", features = ["storage"] } #, path = "../arrowspace-rs"}
+arrowspace = { version = "0.24.6", features = ["storage"] } #, path = "../arrowspace-rs"}
 pyo3 = { version = "0.27.1", features = ["extension-module"] }
+pyo3-log = "0.13"
 numpy = "0.27.0"
 rayon = "1.8"
-smartcore = "0.4.6"
+smartcore = "0.4.8"

{arrowspace-0.24.5 → arrowspace-0.24.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arrowspace
-Version: 0.24.5
+Version: 0.24.8
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
@@ -34,9 +34,9 @@ Project-URL: Repository, https://github.com/tuned-org-uk/pyarrowspace.git
 
 # pyarrowspace
 
-Python bindings for [`arrowspace-rs`](https://github.com/Mec-iS/arrowspace-rs). This is experimental software meant for research at current state.
+Python bindings for [`arrowspace-rs`](https://github.com/Mec-iS/arrowspace-rs).
 
-This is the starting repository for `arrowspace`, it is made public as a showcase for the Python interface, to collect feedback and make public some results of the tests run. To run needs the `arrowspace-rs` Rust module in a sibling directory.
+`arrowspace` is a database for vectors supported by a graph representation and a key-value store. The main use-cases targeted are: AI search capabilities as advanced vector similarity, graph characterisation analysis and search, indexing of high-dimensional vectors. Design principles described in [this article](https://www.tuned.org.uk/posts/010_game_changer_unifying_vectors_and_features_graphs).
 
 For labs and tests please see [tests/](https://github.com/tuned-org-uk/pyarrowspace/tree/main/tests)
 

{arrowspace-0.24.5 → arrowspace-0.24.8}/README.md

@@ -1,8 +1,8 @@
 # pyarrowspace
 
-Python bindings for [`arrowspace-rs`](https://github.com/Mec-iS/arrowspace-rs). This is experimental software meant for research at current state.
+Python bindings for [`arrowspace-rs`](https://github.com/Mec-iS/arrowspace-rs).
 
-This is the starting repository for `arrowspace`, it is made public as a showcase for the Python interface, to collect feedback and make public some results of the tests run. To run needs the `arrowspace-rs` Rust module in a sibling directory.
+`arrowspace` is a database for vectors supported by a graph representation and a key-value store. The main use-cases targeted are: AI search capabilities as advanced vector similarity, graph characterisation analysis and search, indexing of high-dimensional vectors. Design principles described in [this article](https://www.tuned.org.uk/posts/010_game_changer_unifying_vectors_and_features_graphs).
 
 For labs and tests please see [tests/](https://github.com/tuned-org-uk/pyarrowspace/tree/main/tests)
 

{arrowspace-0.24.5 → arrowspace-0.24.8}/src/lib.rs

@@ -28,6 +28,16 @@ mod tests;
 #[cfg(test)]
 mod tests_python;
 
+use std::sync::Once;
+static INIT: Once = Once::new();
+
+/// Initialize logging for tests
+pub fn init() {
+    INIT.call_once(|| {
+        pyo3_log::init();
+    });
+}
+
 // ------------ Py wrappers ------------
 #[pyclass(name = "GraphLaplacian")]
 pub struct PyGraphLaplacian {
@@ -165,7 +175,7 @@ impl PyArrowSpace {
 
         dbg_println(format!("search: qlen={}, lambda_q={:.6}", v.len(), lambda_q));
 
-        let query = ArrowItem::new(v.to_vec(), lambda_q);
+        let query = ArrowItem::new(v, lambda_q);
         let k = graph_laplacian.graph_params.topk;
 
         Ok(self.inner.search_lambda_aware(&query, k, tau))
@@ -203,7 +213,7 @@ impl PyArrowSpace {
                 )));
             }
             
-            let query = ArrowItem::new(v.to_vec(), lambda_q);
+            let query = ArrowItem::new(v, lambda_q);
             results.push(self.inner.search_lambda_aware(&query, k, tau));
         }
         
@@ -232,7 +242,7 @@ impl PyArrowSpace {
 
         dbg_println(format!("search_hybrid: qlen={}, lambda_q={:.6}", v.len(), lambda_q));
 
-        let query = ArrowItem::new(v.to_vec(), lambda_q);
+        let query = ArrowItem::new(v, lambda_q);
         let k = graph_laplacian.graph_params.topk;
 
         Ok(self.inner.search_lambda_aware_hybrid(&query, k, tau))
@@ -303,8 +313,8 @@ impl PyArrowSpace {
         Ok(motifs)
     }
 
-        /// spot_subg_motives(gl: GraphLaplacian, cfg: dict) -> List[dict]
-    ///
+    /// spot_subg_motives(gl: GraphLaplacian, cfg: dict) -> List[dict]
+
     /// Runs energy-mode motif-based subgraph extraction and returns a list of
     /// subgraph dictionaries with:
     /// - "node_indices": List[int] centroid indices
@@ -439,16 +449,73 @@ impl PyArrowSpaceBuilder {
                 .with_lambda_graph(eps, k, topk, p, sigma)
                 .with_dims_reduction(true, Some(eps))
                 .with_seed(42)
-                .with_sparsity_check(false);
+                .with_sparsity_check(false)
         }
 
         dbg_println(format!("build: Processing {} rows × {} cols", nrows, ncols));
-        let (aspace, gl) = builder.build(rows);
+        let (aspace, gl) = py.detach(|| {
+            let (aspace, gl) = builder.build(rows);
+            
+            dbg_println(format!(
+                "build complete: nitems={}, nfeatures={}, lambdas={}",
+                aspace.nitems, aspace.nfeatures, aspace.lambdas().len()
+            ));
+
+            (aspace, gl)
+        });
+
+        Ok((
+            Py::new(py, PyArrowSpace { inner: aspace })?,
+            Py::new(py, PyGraphLaplacian { inner: gl })?,
+        ))
+    }
+
+    /// Same as `build(...)` but save computations on parquet files
+    #[staticmethod]
+    pub fn build_and_store(
+        py: Python<'_>,
+        graph_params: Option<&Bound<'_, PyDict>>,
+        items: PyReadonlyArray2<f64>,
+    ) -> PyResult<(Py<PyArrowSpace>, Py<PyGraphLaplacian>)> {
+        dbg_println("build: Converting numpy array to internal format");
         
-        dbg_println(format!(
-            "build complete: nitems={}, nfeatures={}, lambdas={}",
-            aspace.nitems, aspace.nfeatures, aspace.lambdas().len()
-        ));
+        let arr = items.as_array();
+        let (nrows, ncols) = (arr.shape()[0], arr.shape()[1]);
+        
+        let rows: Vec<Vec<f64>> = if nrows > 1000 {
+            use rayon::prelude::*;
+            (0..nrows)
+                .into_par_iter()
+                .map(|i| arr.row(i).to_owned().to_vec())
+                .collect()
+        } else {
+            (0..nrows)
+                .map(|i| arr.row(i).to_owned().to_vec())
+                .collect()
+        };
+
+        let mut builder = RustBuilder::new();
+        
+        if let Some((eps, k, topk, p, sigma)) = parse_graph_params(graph_params)? {
+            builder = builder
+                .with_lambda_graph(eps, k, topk, p, sigma)
+                .with_dims_reduction(true, Some(eps))
+                .with_seed(42)
+                .with_sparsity_check(false)
+                .with_persistence("./storage", "dataset".to_string());
+        }
+
+        dbg_println(format!("build: Processing {} rows × {} cols", nrows, ncols));
+        let (aspace, gl) = py.detach(|| {
+            let (aspace, gl) = builder.build(rows);
+            
+            dbg_println(format!(
+                "build complete: nitems={}, nfeatures={}, lambdas={}",
+                aspace.nitems, aspace.nfeatures, aspace.lambdas().len()
+            ));
+
+            (aspace, gl)
+        });
 
         Ok((
             Py::new(py, PyArrowSpace { inner: aspace })?,
@@ -494,20 +561,22 @@ impl PyArrowSpaceBuilder {
                 .with_dims_reduction(true, Some(eps))
                 .with_extra_dims_reduction(true)
                 .with_seed(999)
-                .with_inline_sampling(Some(SamplerType::Simple(0.6)))
+                .with_inline_sampling(Some(SamplerType::Simple(0.99)))
                 .with_spectral(false)
                 .with_sparsity_check(false);
         }
         
         dbg_println(format!("build_energy: Processing {} rows × {} cols", nrows, ncols));
         let (aspace, gl_energy) = py.detach(|| {
-            builder.build_energy(rows, e_params)
+            let (aspace, gl_energy) = builder.build_energy(rows, e_params);
+            
+            dbg_println(format!(
+                "build_energy complete: nitems={}, nfeatures={}, graph_nodes={}, lambdas={}",
+                aspace.nitems, aspace.nfeatures, gl_energy.nnodes, aspace.lambdas().len()
+            ));
+            
+            (aspace, gl_energy)
         });
-        
-        dbg_println(format!(
-            "build_energy complete: nitems={}, nfeatures={}, graph_nodes={}, lambdas={}",
-            aspace.nitems, aspace.nfeatures, gl_energy.nnodes, aspace.lambdas().len()
-        ));
 
         Ok((
             Py::new(py, PyArrowSpace { inner: aspace })?,

{arrowspace-0.24.5 → arrowspace-0.24.8}/tests/test_10_CVE_sorted_set_fixed_eta.py

@@ -36,7 +36,7 @@ STEPS_VALUES = [4, 6]
 
 # Standard graph params (used for both standard and energy builds)
 graph_params = {
-    "eps": 1.31,   # eps for 100K
+    "eps": 0.8,   # eps for 100K
     "k": 25,
     "topk": 15,
     "p": 2.0,