marsdb 0.1.0__tar.gz

marsdb-0.1.0/Cargo.toml

@@ -0,0 +1,32 @@
+[workspace]
+resolver = "2"
+members = [
+    "marsdb-storage",
+    "marsdb-graph",
+    "marsdb-query",
+    "marsdb",
+    "marsdb-cli",
+]
+# marsdb-python builds as a PyO3 extension-module (no libpython link, no
+# runnable test binary) via maturin, not plain `cargo build`/`cargo test` —
+# kept out of the workspace so it doesn't affect `cargo test --workspace`.
+exclude = ["marsdb-python"]
+
+[workspace.package]
+version = "0.1.0"
+edition = "2021"
+license = "MIT OR Apache-2.0"
+repository = "https://github.com/knoguchi/marsdb"
+homepage = "https://github.com/knoguchi/marsdb"
+
+[workspace.dependencies]
+redb = "2"
+serde = { version = "1", features = ["derive"] }
+postcard = { version = "1", features = ["alloc"] }
+thiserror = "1"
+pest = "2"
+pest_derive = "2"
+clap = { version = "4", features = ["derive"] }
+rustyline = "14"
+criterion = "0.5"
+tempfile = "3"

marsdb-0.1.0/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: marsdb
+Version: 0.1.0
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Summary: Embeddable property-graph database with an openCypher query subset.
+Home-Page: https://github.com/knoguchi/marsdb
+License: MIT OR Apache-2.0
+Requires-Python: >=3.8
+Project-URL: Repository, https://github.com/knoguchi/marsdb

marsdb-0.1.0/marsdb/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "marsdb"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+homepage.workspace = true
+description = "Embeddable property-graph database with an openCypher query subset: single binary, single file, optional in-memory mode."
+
+[lib]
+name = "marsdb"
+path = "src/lib.rs"
+
+[dependencies]
+marsdb-storage = { path = "../marsdb-storage", version = "0.1.0" }
+marsdb-graph = { path = "../marsdb-graph", version = "0.1.0" }
+marsdb-query = { path = "../marsdb-query", version = "0.1.0" }
+thiserror.workspace = true
+
+[dev-dependencies]
+tempfile = "3"
+criterion.workspace = true
+
+[[bench]]
+name = "cypher_ops"
+harness = false

marsdb-0.1.0/marsdb/benches/cypher_ops.rs

@@ -0,0 +1,62 @@
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use marsdb::Database;
+
+fn make_chain_create(hops: usize) -> String {
+    let mut chain = "(n0:Item {idx: 0})".to_string();
+    for i in 1..=hops {
+        chain.push_str(&format!("-[:R]->(n{i}:Item {{idx: {i}}})"));
+    }
+    format!("CREATE {chain}")
+}
+
+/// Isolates parsing cost from execution cost — answers the "where does the
+/// time go" question cleanly, in-process (no subprocess/CLI confounders).
+fn bench_parse_only(c: &mut Criterion) {
+    let mut group = c.benchmark_group("parse_only_by_hop_count");
+    for hops in [10usize, 100, 1_000] {
+        let cypher = make_chain_create(hops);
+        group.bench_with_input(BenchmarkId::from_parameter(hops), &hops, |b, _| {
+            b.iter(|| black_box(marsdb_query::parse(&cypher).unwrap()));
+        });
+    }
+    group.finish();
+}
+
+fn bench_execute_create(c: &mut Criterion) {
+    let mut group = c.benchmark_group("execute_create_by_hop_count");
+    for hops in [10usize, 100, 1_000] {
+        let cypher = make_chain_create(hops);
+        group.bench_with_input(BenchmarkId::from_parameter(hops), &hops, |b, _| {
+            b.iter(|| {
+                let db = Database::in_memory().unwrap();
+                black_box(db.execute(&cypher).unwrap())
+            });
+        });
+    }
+    group.finish();
+}
+
+fn bench_execute_match_1hop(c: &mut Criterion) {
+    let mut group = c.benchmark_group("execute_match_1hop_by_dataset_size");
+    for n in [100usize, 1_000, 10_000] {
+        let db = Database::in_memory().unwrap();
+        db.execute(&make_chain_create(n)).unwrap();
+        group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| {
+            b.iter(|| {
+                black_box(
+                    db.execute("MATCH (n:Item)-[:R]->(m:Item) RETURN m.idx LIMIT 10")
+                        .unwrap(),
+                )
+            });
+        });
+    }
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    bench_parse_only,
+    bench_execute_create,
+    bench_execute_match_1hop,
+);
+criterion_main!(benches);

marsdb-0.1.0/marsdb/src/lib.rs

@@ -0,0 +1,47 @@
+//! Embeddable property-graph database with an openCypher query subset:
+//! single binary, single file, optional in-memory mode.
+//!
+//! ```
+//! let db = marsdb::Database::in_memory().unwrap();
+//! db.execute("CREATE (a:Person {name: 'Alice'})-[:KNOWS]->(b:Person {name: 'Bob'})").unwrap();
+//! let result = db.execute("MATCH (n:Person) RETURN n.name").unwrap();
+//! assert_eq!(result.rows.len(), 2);
+//! ```
+
+use std::path::Path;
+
+pub use marsdb_query::{Literal, QueryResult, Value};
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("graph error: {0}")]
+    Graph(#[from] marsdb_graph::GraphError),
+    #[error("query error: {0}")]
+    Query(#[from] marsdb_query::QueryError),
+}
+
+pub struct Database {
+    store: marsdb_graph::GraphStore,
+}
+
+impl Database {
+    /// Open (creating if absent) a single-file, on-disk database.
+    pub fn open(path: impl AsRef<Path>) -> Result<Self, Error> {
+        Ok(Self {
+            store: marsdb_graph::GraphStore::open_file(path)?,
+        })
+    }
+
+    /// Open a purely in-memory database. Nothing is written to disk.
+    pub fn in_memory() -> Result<Self, Error> {
+        Ok(Self {
+            store: marsdb_graph::GraphStore::open_memory()?,
+        })
+    }
+
+    pub fn execute(&self, cypher: &str) -> Result<QueryResult, Error> {
+        let stmt = marsdb_query::parse(cypher)?;
+        let result = marsdb_query::Executor::new(&self.store).execute(&stmt)?;
+        Ok(result)
+    }
+}

marsdb-0.1.0/marsdb/tests/embed.rs

@@ -0,0 +1,36 @@
+//! Proves the crate is embeddable independent of the CLI subprocess.
+
+use marsdb::Database;
+
+#[test]
+fn in_memory_roundtrip() {
+    let db = Database::in_memory().unwrap();
+    db.execute("CREATE (a:Person {name: 'Alice'})-[:KNOWS]->(b:Person {name: 'Bob'})")
+        .unwrap();
+    let result = db.execute("MATCH (n:Person) RETURN n.name").unwrap();
+    assert_eq!(result.rows.len(), 2);
+}
+
+#[test]
+fn file_backed_persists_after_reopen() {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("test.db");
+
+    {
+        let db = Database::open(&path).unwrap();
+        db.execute("CREATE (a:Person {name: 'Alice'})").unwrap();
+    }
+
+    let db = Database::open(&path).unwrap();
+    let result = db.execute("MATCH (n:Person) RETURN n.name").unwrap();
+    assert_eq!(result.rows.len(), 1);
+}
+
+#[test]
+fn two_in_memory_databases_do_not_share_data() {
+    let a = Database::in_memory().unwrap();
+    let b = Database::in_memory().unwrap();
+    a.execute("CREATE (n:Person {name: 'Alice'})").unwrap();
+    let result = b.execute("MATCH (n:Person) RETURN n.name").unwrap();
+    assert_eq!(result.rows.len(), 0);
+}

marsdb-0.1.0/marsdb-graph/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "marsdb-graph"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+homepage.workspace = true
+description = "Property graph model and CRUD storage layer used internally by MarsDB."
+
+[dependencies]
+marsdb-storage = { path = "../marsdb-storage", version = "0.1.0" }
+redb.workspace = true
+serde.workspace = true
+postcard.workspace = true
+thiserror.workspace = true
+
+[dev-dependencies]
+tempfile = "3"
+criterion.workspace = true
+
+[[bench]]
+name = "graph_ops"
+harness = false

marsdb-0.1.0/marsdb-graph/benches/graph_ops.rs

@@ -0,0 +1,109 @@
+use std::collections::BTreeMap;
+
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use marsdb_graph::{Direction, GraphStore, PropertyValue};
+
+fn bench_create_node(c: &mut Criterion) {
+    let store = GraphStore::open_memory().unwrap();
+    c.bench_function("create_node", |b| {
+        b.iter(|| {
+            let mut props = BTreeMap::new();
+            props.insert("name".to_string(), PropertyValue::String("x".to_string()));
+            black_box(store.create_node("Item", props).unwrap())
+        });
+    });
+}
+
+fn bench_create_edge(c: &mut Criterion) {
+    let store = GraphStore::open_memory().unwrap();
+    let a = store.create_node("Item", BTreeMap::new()).unwrap();
+    let b_node = store.create_node("Item", BTreeMap::new()).unwrap();
+    c.bench_function("create_edge", |b| {
+        b.iter(|| black_box(store.create_edge("REL", a, b_node, BTreeMap::new()).unwrap()));
+    });
+}
+
+fn bench_get_node(c: &mut Criterion) {
+    let store = GraphStore::open_memory().unwrap();
+    let id = store.create_node("Item", BTreeMap::new()).unwrap();
+    c.bench_function("get_node", |b| {
+        b.iter(|| black_box(store.get_node(id).unwrap()));
+    });
+}
+
+fn bench_neighbors_1hop(c: &mut Criterion) {
+    let mut group = c.benchmark_group("neighbors_1hop_by_fanout");
+    for fanout in [1u64, 10, 100, 1_000] {
+        let store = GraphStore::open_memory().unwrap();
+        let center = store.create_node("Item", BTreeMap::new()).unwrap();
+        for _ in 0..fanout {
+            let n = store.create_node("Item", BTreeMap::new()).unwrap();
+            store.create_edge("REL", center, n, BTreeMap::new()).unwrap();
+        }
+        group.bench_with_input(BenchmarkId::from_parameter(fanout), &fanout, |b, _| {
+            b.iter(|| black_box(store.neighbors(center, Direction::Out, None).unwrap()));
+        });
+    }
+    group.finish();
+}
+
+fn bench_all_nodes_scan(c: &mut Criterion) {
+    let mut group = c.benchmark_group("all_nodes_scan_by_table_size");
+    for n in [100i64, 1_000, 10_000] {
+        let store = GraphStore::open_memory().unwrap();
+        for i in 0..n {
+            let mut props = BTreeMap::new();
+            props.insert("idx".to_string(), PropertyValue::Int(i));
+            store.create_node("Item", props).unwrap();
+        }
+        group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| {
+            b.iter(|| black_box(store.all_nodes(Some("Item")).unwrap()));
+        });
+    }
+    group.finish();
+}
+
+/// Follow-up on an earlier open question: does batching many writes into
+/// one transaction (via the `*_in_txn` API the query executor uses) matter
+/// versus one transaction per node? Answers it directly instead of guessing.
+fn bench_bulk_create_txn_strategy(c: &mut Criterion) {
+    const N: i64 = 1_000;
+    let mut group = c.benchmark_group("bulk_create_1000_nodes");
+
+    group.bench_function("one_txn_per_node", |b| {
+        b.iter(|| {
+            let store = GraphStore::open_memory().unwrap();
+            for i in 0..N {
+                let mut props = BTreeMap::new();
+                props.insert("idx".to_string(), PropertyValue::Int(i));
+                store.create_node("Item", props).unwrap();
+            }
+        });
+    });
+
+    group.bench_function("one_txn_total", |b| {
+        b.iter(|| {
+            let store = GraphStore::open_memory().unwrap();
+            let write_txn = store.begin_write().unwrap();
+            for i in 0..N {
+                let mut props = BTreeMap::new();
+                props.insert("idx".to_string(), PropertyValue::Int(i));
+                GraphStore::create_node_in_txn(&write_txn, "Item", props).unwrap();
+            }
+            GraphStore::commit(write_txn).unwrap();
+        });
+    });
+
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    bench_create_node,
+    bench_create_edge,
+    bench_get_node,
+    bench_neighbors_1hop,
+    bench_all_nodes_scan,
+    bench_bulk_create_txn_strategy,
+);
+criterion_main!(benches);

marsdb-0.1.0/marsdb-graph/src/encode.rs

@@ -0,0 +1,26 @@
+use std::collections::BTreeMap;
+
+use crate::error::GraphError;
+use crate::model::PropertyValue;
+
+#[derive(serde::Serialize, serde::Deserialize)]
+pub(crate) struct NodeRecord {
+    pub label_id: u32,
+    pub props: BTreeMap<String, PropertyValue>,
+}
+
+#[derive(serde::Serialize, serde::Deserialize)]
+pub(crate) struct EdgeRecord {
+    pub label_id: u32,
+    pub src: u64,
+    pub dst: u64,
+    pub props: BTreeMap<String, PropertyValue>,
+}
+
+pub(crate) fn encode<T: serde::Serialize>(value: &T) -> Result<Vec<u8>, GraphError> {
+    Ok(postcard::to_allocvec(value)?)
+}
+
+pub(crate) fn decode<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> Result<T, GraphError> {
+    Ok(postcard::from_bytes(bytes)?)
+}

marsdb-0.1.0/marsdb-graph/src/error.rs

@@ -0,0 +1,17 @@
+use crate::model::NodeId;
+
+#[derive(Debug, thiserror::Error)]
+pub enum GraphError {
+    #[error("storage error: {0}")]
+    Storage(#[from] marsdb_storage::StorageError),
+    #[error("table error: {0}")]
+    Table(#[from] redb::TableError),
+    #[error("storage error: {0}")]
+    RedbStorage(#[from] redb::StorageError),
+    #[error("commit error: {0}")]
+    Commit(#[from] redb::CommitError),
+    #[error("encode error: {0}")]
+    Encode(#[from] postcard::Error),
+    #[error("node {0:?} has incident edges; use detach delete")]
+    NodeHasEdges(NodeId),
+}

marsdb-0.1.0/marsdb-graph/src/id.rs

@@ -0,0 +1,15 @@
+use marsdb_storage::{ReadableTable, WriteTransaction};
+
+use crate::error::GraphError;
+
+/// Increment and return the counter stored at `key` in the `meta` table,
+/// within the given write transaction. Allocation is only durable once the
+/// caller commits `write_txn`, keeping id allocation inside the same
+/// crash-safety boundary as the row(s) it's used for.
+pub(crate) fn next_id(write_txn: &WriteTransaction, key: &str) -> Result<u64, GraphError> {
+    let mut meta = write_txn.open_table(marsdb_storage::tables::META)?;
+    let current = meta.get(key)?.map(|g| g.value()).unwrap_or(0);
+    let next = current + 1;
+    meta.insert(key, next)?;
+    Ok(next)
+}

marsdb-0.1.0/marsdb-graph/src/labels.rs

@@ -0,0 +1,47 @@
+use marsdb_storage::{ReadableTable, WriteTransaction};
+
+use crate::error::GraphError;
+use crate::id::next_id;
+
+/// Look up (or allocate) the u32 id interned for `label`, inside a write txn.
+pub(crate) fn intern_label(write_txn: &WriteTransaction, label: &str) -> Result<u32, GraphError> {
+    {
+        let l2i = write_txn.open_table(marsdb_storage::tables::LABEL_TO_ID)?;
+        let existing = l2i.get(label)?.map(|g| g.value());
+        if let Some(existing) = existing {
+            return Ok(existing);
+        }
+    }
+    let id = next_id(write_txn, "next_label_id")? as u32;
+    {
+        let mut l2i = write_txn.open_table(marsdb_storage::tables::LABEL_TO_ID)?;
+        l2i.insert(label, id)?;
+    }
+    {
+        let mut i2l = write_txn.open_table(marsdb_storage::tables::ID_TO_LABEL)?;
+        i2l.insert(id, label)?;
+    }
+    Ok(id)
+}
+
+/// Resolve a previously interned label id back to its string.
+///
+/// Takes `&WriteTransaction` (not `&ReadTransaction`) even for read-only
+/// callers: v1 drives an entire statement through one `WriteTransaction` —
+/// see `GraphStore::begin_write` — so every table access in this crate goes
+/// through that one transaction type, read or write.
+pub(crate) fn resolve_label(write_txn: &WriteTransaction, label_id: u32) -> Result<String, GraphError> {
+    let i2l = write_txn.open_table(marsdb_storage::tables::ID_TO_LABEL)?;
+    let value = i2l
+        .get(label_id)?
+        .expect("label id present in nodes/edges table must be interned");
+    Ok(value.value().to_string())
+}
+
+/// Look up the id for `label` without allocating one. Returns `None` if the
+/// label has never been used, meaning no rows can reference it.
+pub(crate) fn lookup_label_id(write_txn: &WriteTransaction, label: &str) -> Result<Option<u32>, GraphError> {
+    let l2i = write_txn.open_table(marsdb_storage::tables::LABEL_TO_ID)?;
+    let found = l2i.get(label)?.map(|g| g.value());
+    Ok(found)
+}

marsdb-0.1.0/marsdb-graph/src/lib.rs

@@ -0,0 +1,11 @@
+mod encode;
+mod error;
+mod id;
+mod labels;
+mod model;
+mod store;
+
+pub use error::GraphError;
+pub use marsdb_storage::WriteTransaction;
+pub use model::{AdjEntry, Direction, Edge, EdgeId, Node, NodeId, PropertyValue};
+pub use store::GraphStore;

marsdb-0.1.0/marsdb-graph/src/model.rs

@@ -0,0 +1,68 @@
+use std::collections::BTreeMap;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct NodeId(pub u64);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct EdgeId(pub u64);
+
+#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+pub enum PropertyValue {
+    Null,
+    Bool(bool),
+    Int(i64),
+    Float(f64),
+    String(String),
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Node {
+    pub id: NodeId,
+    pub label: String,
+    pub props: BTreeMap<String, PropertyValue>,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Edge {
+    pub id: EdgeId,
+    pub label: String,
+    pub src: NodeId,
+    pub dst: NodeId,
+    pub props: BTreeMap<String, PropertyValue>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Direction {
+    Out,
+    In,
+}
+
+/// A traversal-hop candidate read directly from an adjacency multimap entry,
+/// without touching the `edges`/`nodes` tables.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct AdjEntry {
+    pub edge_id: EdgeId,
+    pub other: NodeId,
+    pub label_id: u32,
+}
+
+impl AdjEntry {
+    pub(crate) fn encode(&self) -> [u8; 20] {
+        let mut buf = [0u8; 20];
+        buf[0..8].copy_from_slice(&self.edge_id.0.to_be_bytes());
+        buf[8..16].copy_from_slice(&self.other.0.to_be_bytes());
+        buf[16..20].copy_from_slice(&self.label_id.to_be_bytes());
+        buf
+    }
+
+    pub(crate) fn decode(bytes: &[u8]) -> Self {
+        let edge_id = u64::from_be_bytes(bytes[0..8].try_into().unwrap());
+        let other = u64::from_be_bytes(bytes[8..16].try_into().unwrap());
+        let label_id = u32::from_be_bytes(bytes[16..20].try_into().unwrap());
+        AdjEntry {
+            edge_id: EdgeId(edge_id),
+            other: NodeId(other),
+            label_id,
+        }
+    }
+}