lora-python 0.2.0__tar.gz → 0.4.0__tar.gz

{lora_python-0.2.0 → lora_python-0.4.0}/Cargo.lock

@@ -116,6 +116,15 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "bincode"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "bindgen"
 version = "0.72.1"
@@ -283,6 +292,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crc32fast"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "criterion"
 version = "0.8.2"
@@ -611,7 +629,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
 
 [[package]]
 name = "lora-analyzer"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
  "lora-ast",
  "lora-parser",
@@ -621,14 +639,14 @@ dependencies = [
 
 [[package]]
 name = "lora-ast"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
  "smallvec 2.0.0-alpha.12",
 ]
 
 [[package]]
 name = "lora-compiler"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
  "lora-analyzer",
  "lora-ast",
@@ -636,7 +654,7 @@ dependencies = [
 
 [[package]]
 name = "lora-database"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
  "anyhow",
  "criterion",
@@ -646,13 +664,14 @@ dependencies = [
  "lora-executor",
  "lora-parser",
  "lora-store",
+ "lora-wal",
  "serde_json",
  "tower",
 ]
 
 [[package]]
 name = "lora-executor"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
  "lora-analyzer",
  "lora-ast",
@@ -666,7 +685,7 @@ dependencies = [
 
 [[package]]
 name = "lora-ffi"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
  "lora-database",
  "lora-store",
@@ -676,7 +695,7 @@ dependencies = [
 
 [[package]]
 name = "lora-node"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
  "anyhow",
  "lora-database",
@@ -690,7 +709,7 @@ dependencies = [
 
 [[package]]
 name = "lora-parser"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
  "lora-ast",
  "pest",
@@ -701,7 +720,7 @@ dependencies = [
 
 [[package]]
 name = "lora-python"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
  "anyhow",
  "lora-database",
@@ -713,10 +732,11 @@ dependencies = [
 
 [[package]]
 name = "lora-server"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
  "anyhow",
  "axum",
+ "http-body-util",
  "lora-database",
  "serde",
  "serde_json",
@@ -726,15 +746,30 @@ dependencies = [
 
 [[package]]
 name = "lora-store"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
+ "bincode",
+ "crc32fast",
  "js-sys",
  "lora-ast",
+ "serde",
+ "thiserror",
+]
+
+[[package]]
+name = "lora-wal"
+version = "0.4.0"
+dependencies = [
+ "bincode",
+ "crc32fast",
+ "lora-store",
+ "serde",
+ "thiserror",
 ]
 
 [[package]]
 name = "lora-wasm"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
  "anyhow",
  "console_error_panic_hook",
@@ -749,7 +784,7 @@ dependencies = [
 
 [[package]]
 name = "lora_ruby"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
  "anyhow",
  "lora-database",

{lora_python-0.2.0 → lora_python-0.4.0}/Cargo.toml

@@ -1,10 +1,10 @@
 [workspace]
-members = ["crates/lora-ast", "crates/lora-parser", "crates/lora-analyzer", "crates/lora-compiler", "crates/lora-store", "crates/lora-executor", "crates/lora-database", "crates/lora-python"]
+members = ["crates/lora-ast", "crates/lora-parser", "crates/lora-analyzer", "crates/lora-compiler", "crates/lora-store", "crates/lora-wal", "crates/lora-executor", "crates/lora-database", "crates/lora-python"]
 resolver = "2"
 
 [workspace.package]
 edition = "2021"
-version = "0.2.0"
+version = "0.4.0"
 license = "BUSL-1.1"
 authors = ["LoraDB, Inc."]
 repository = "https://github.com/lora-db/lora"
@@ -15,13 +15,14 @@ rust-version = "1.87"
 # Internal crates — versions are kept in lockstep with [workspace.package].version
 # by `scripts/sync-versions.mjs`. Both `path` and `version` are set so that
 # `cargo publish` works (crates.io cannot resolve path-only deps).
-lora-ast = { path = "crates/lora-ast", version = "=0.2.0" }
-lora-parser = { path = "crates/lora-parser", version = "=0.2.0" }
-lora-analyzer = { path = "crates/lora-analyzer", version = "=0.2.0" }
-lora-compiler = { path = "crates/lora-compiler", version = "=0.2.0" }
-lora-store = { path = "crates/lora-store", version = "=0.2.0" }
-lora-executor = { path = "crates/lora-executor", version = "=0.2.0" }
-lora-database = { path = "crates/lora-database", version = "=0.2.0" }
+lora-ast = { path = "crates/lora-ast", version = "=0.4.0" }
+lora-parser = { path = "crates/lora-parser", version = "=0.4.0" }
+lora-analyzer = { path = "crates/lora-analyzer", version = "=0.4.0" }
+lora-compiler = { path = "crates/lora-compiler", version = "=0.4.0" }
+lora-store = { path = "crates/lora-store", version = "=0.4.0" }
+lora-wal = { path = "crates/lora-wal", version = "=0.4.0" }
+lora-executor = { path = "crates/lora-executor", version = "=0.4.0" }
+lora-database = { path = "crates/lora-database", version = "=0.4.0" }
 
 # External crates.
 anyhow = "1"
@@ -37,3 +38,5 @@ serde_json = "1.0.149"
 regex = "1"
 tower = "0.5.3"
 criterion = { version = "0.8.2", features = ["html_reports"] }
+bincode = "1.3"
+crc32fast = "1.4"

{lora_python-0.2.0 → lora_python-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lora-python
-Version: 0.2.0
+Version: 0.4.0
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
@@ -30,7 +30,7 @@ Project-URL: Repository, https://github.com/lora-db/lora
 
 # lora-python
 
-Python bindings for the [Lora](../../README.md) in-memory graph
+Python bindings for the [Lora](../../README.md) graph
 engine. Ships both a synchronous PyO3 `Database` class and an
 asyncio-compatible `AsyncDatabase` wrapper that never blocks the event loop.
 
@@ -64,6 +64,18 @@ for row in res["rows"]:
         print(n["properties"]["name"])
 ```
 
+Initialization rule:
+
+```python
+from lora_python import Database
+
+scratch = Database.create()            # in-memory
+persistent = Database.create("./app")  # persistent: directory string
+```
+
+If you want persistence, pass a directory string to `Database.create(...)`
+or `Database(...)`.
+
 ## Async usage (non-blocking)
 
 ```python
@@ -79,6 +91,13 @@ async def main():
 asyncio.run(main())
 ```
 
+Async initialization follows the same rule:
+
+```python
+db = await AsyncDatabase.create()            # in-memory
+db = await AsyncDatabase.create("./app")     # persistent: directory string
+```
+
 `AsyncDatabase.execute` dispatches the query onto the default asyncio
 thread pool via `asyncio.to_thread`. The PyO3 `Database.execute` releases
 the Python GIL for the duration of engine work, so other coroutines on the
@@ -125,6 +144,21 @@ Constructors and guards are exported from `lora_python.types`:
 
 All three are available as `lora_python.LoraError`, etc.
 
+## Persistence
+
+`Database.create("./app")`, `Database("./app")`, and
+`await AsyncDatabase.create("./app")` open or create a WAL-backed
+persistent database rooted at that directory. Reopening the same path
+replays committed writes before returning the handle.
+
+Call `db.close()` / `await db.close()` before reopening the same WAL
+directory inside one process.
+
+This first Python persistence slice intentionally stays small: the
+binding exposes WAL-backed initialization plus the existing
+`save_snapshot` / `load_snapshot` APIs, but not checkpoint, truncate,
+status, or sync-mode controls.
+
 ## Architecture
 
 ```

{lora_python-0.2.0 → lora_python-0.4.0}/README.md

@@ -1,6 +1,6 @@
 # lora-python
 
-Python bindings for the [Lora](../../README.md) in-memory graph
+Python bindings for the [Lora](../../README.md) graph
 engine. Ships both a synchronous PyO3 `Database` class and an
 asyncio-compatible `AsyncDatabase` wrapper that never blocks the event loop.
 
@@ -34,6 +34,18 @@ for row in res["rows"]:
         print(n["properties"]["name"])
 ```
 
+Initialization rule:
+
+```python
+from lora_python import Database
+
+scratch = Database.create()            # in-memory
+persistent = Database.create("./app")  # persistent: directory string
+```
+
+If you want persistence, pass a directory string to `Database.create(...)`
+or `Database(...)`.
+
 ## Async usage (non-blocking)
 
 ```python
@@ -49,6 +61,13 @@ async def main():
 asyncio.run(main())
 ```
 
+Async initialization follows the same rule:
+
+```python
+db = await AsyncDatabase.create()            # in-memory
+db = await AsyncDatabase.create("./app")     # persistent: directory string
+```
+
 `AsyncDatabase.execute` dispatches the query onto the default asyncio
 thread pool via `asyncio.to_thread`. The PyO3 `Database.execute` releases
 the Python GIL for the duration of engine work, so other coroutines on the
@@ -95,6 +114,21 @@ Constructors and guards are exported from `lora_python.types`:
 
 All three are available as `lora_python.LoraError`, etc.
 
+## Persistence
+
+`Database.create("./app")`, `Database("./app")`, and
+`await AsyncDatabase.create("./app")` open or create a WAL-backed
+persistent database rooted at that directory. Reopening the same path
+replays committed writes before returning the handle.
+
+Call `db.close()` / `await db.close()` before reopening the same WAL
+directory inside one process.
+
+This first Python persistence slice intentionally stays small: the
+binding exposes WAL-backed initialization plus the existing
+`save_snapshot` / `load_snapshot` APIs, but not checkpoint, truncate,
+status, or sync-mode controls.
+
 ## Architecture
 
 ```

{lora_python-0.2.0 → lora_python-0.4.0}/crates/lora-analyzer/src/analyzer.rs

@@ -5,10 +5,10 @@ use lora_ast::{
     ReadingClause, RelationshipPattern, Remove, RemoveItem, Return, Set, SetItem, SinglePartQuery,
     SingleQuery, Statement, Unwind, UpdatingClause, With,
 };
-use lora_store::GraphStorage;
+use lora_store::GraphCatalog;
 use std::collections::{BTreeMap, BTreeSet};
 
-pub struct Analyzer<'a, S: GraphStorage + ?Sized> {
+pub struct Analyzer<'a, S: GraphCatalog + ?Sized> {
     storage: &'a S,
     scopes: ScopeStack,
     symbols: SymbolTable,
@@ -22,7 +22,7 @@ enum PatternContext {
     Write,
 }
 
-impl<'a, S: GraphStorage + ?Sized> Analyzer<'a, S> {
+impl<'a, S: GraphCatalog + ?Sized> Analyzer<'a, S> {
     pub fn new(storage: &'a S) -> Self {
         Self {
             storage,

{lora_python-0.2.0 → lora_python-0.4.0}/crates/lora-database/Cargo.toml

@@ -23,6 +23,7 @@ lora-analyzer.workspace = true
 lora-compiler.workspace = true
 lora-executor.workspace = true
 lora-store.workspace = true
+lora-wal.workspace = true
 
 serde_json.workspace = true
 tower.workspace = true
@@ -49,3 +50,7 @@ harness = false
 [[bench]]
 name = "perf_smoke_benchmarks"
 harness = false
+
+[[bench]]
+name = "wal_benchmarks"
+harness = false

lora_python-0.4.0/crates/lora-database/benches/wal_benchmarks.rs

@@ -0,0 +1,205 @@
+//! WAL-aware microbenchmarks.
+//!
+//! These exercise the four durability profiles end-to-end through
+//! `Database::execute_with_params`:
+//!
+//! - **`no_wal`** — `Database::in_memory()` (the existing fast path;
+//!   serves as the baseline the others are compared against).
+//! - **`per_commit`** — `WalConfig::Enabled` with `SyncMode::PerCommit`
+//!   (fsync before every commit returns).
+//! - **`group`** — `WalConfig::Enabled` with `SyncMode::Group`
+//!   (write-only on commit, bg flusher fsyncs).
+//! - **`none`** — `WalConfig::Enabled` with `SyncMode::None`
+//!   (no fsync at all, OS-buffered).
+//!
+//! The shape that matters is *commit latency* — every iteration runs a
+//! single tiny `CREATE` statement so the engine work is negligible and
+//! the WAL path dominates. On NVMe the gap between `no_wal` and
+//! `per_commit` is roughly the cost of one `fsync` (50–200 µs); the
+//! gap between `per_commit` and `group` / `none` measures how much
+//! you save by deferring durability.
+//!
+//! There is also a small **`recovery`** bench that times opening a WAL
+//! with N committed transactions and re-applying them. This is the
+//! number that bounds startup time on a fresh process.
+//!
+//! Run with:
+//!   `cargo bench -p lora-database --bench wal_benchmarks`
+
+use std::path::PathBuf;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
+use lora_database::{Database, ExecuteOptions, ResultFormat, SyncMode, WalConfig};
+use std::hint::black_box;
+
+fn opts() -> Option<ExecuteOptions> {
+    Some(ExecuteOptions {
+        format: ResultFormat::Rows,
+    })
+}
+
+/// Per-iteration scratch directory. Criterion's `iter_batched_setup`
+/// runs setup once per batch, so this is called once per Criterion
+/// "sample" — fine even at sample_size=20.
+struct ScratchDir {
+    path: PathBuf,
+}
+
+impl ScratchDir {
+    fn new(tag: &str) -> Self {
+        let mut path = std::env::temp_dir();
+        path.push(format!(
+            "lora-wal-bench-{}-{}-{}",
+            tag,
+            std::process::id(),
+            SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap()
+                .as_nanos()
+        ));
+        std::fs::create_dir_all(&path).unwrap();
+        Self { path }
+    }
+}
+
+impl Drop for ScratchDir {
+    fn drop(&mut self) {
+        let _ = std::fs::remove_dir_all(&self.path);
+    }
+}
+
+fn enabled(dir: &std::path::Path, sync_mode: SyncMode) -> WalConfig {
+    WalConfig::Enabled {
+        dir: dir.to_path_buf(),
+        sync_mode,
+        segment_target_bytes: 8 * 1024 * 1024,
+    }
+}
+
+fn smoke_config() -> Criterion {
+    // Per-bench budget of ~3 s. fsync cost dominates per_commit so we
+    // cannot run this as cheaply as the in-memory smoke suite, but we
+    // also don't want a 30-second bench on every CI run.
+    Criterion::default()
+        .warm_up_time(Duration::from_millis(500))
+        .measurement_time(Duration::from_millis(2_500))
+        .sample_size(20)
+        .noise_threshold(0.10)
+}
+
+fn bench_commit_latency(c: &mut Criterion) {
+    let mut group = c.benchmark_group("wal/commit_latency");
+
+    // ---- baseline: no WAL ---------------------------------------------------
+    {
+        group.bench_function("no_wal", |b| {
+            b.iter_batched(
+                Database::in_memory,
+                |db| {
+                    black_box(db.execute("CREATE (:N {v: 1})", opts()).unwrap());
+                },
+                BatchSize::SmallInput,
+            );
+        });
+    }
+
+    // ---- PerCommit (fsync per commit) --------------------------------------
+    {
+        group.bench_function("per_commit", |b| {
+            b.iter_batched(
+                || {
+                    let dir = ScratchDir::new("per-commit");
+                    let db =
+                        Database::open_with_wal(enabled(&dir.path, SyncMode::PerCommit)).unwrap();
+                    (dir, db)
+                },
+                |(_dir, db)| {
+                    black_box(db.execute("CREATE (:N {v: 1})", opts()).unwrap());
+                },
+                BatchSize::SmallInput,
+            );
+        });
+    }
+
+    // ---- Group (write-only on commit, bg flusher fsyncs) -------------------
+    {
+        group.bench_function("group", |b| {
+            b.iter_batched(
+                || {
+                    let dir = ScratchDir::new("group");
+                    let db = Database::open_with_wal(enabled(
+                        &dir.path,
+                        SyncMode::Group { interval_ms: 50 },
+                    ))
+                    .unwrap();
+                    (dir, db)
+                },
+                |(_dir, db)| {
+                    black_box(db.execute("CREATE (:N {v: 1})", opts()).unwrap());
+                },
+                BatchSize::SmallInput,
+            );
+        });
+    }
+
+    // ---- None (no fsync at all) --------------------------------------------
+    {
+        group.bench_function("none", |b| {
+            b.iter_batched(
+                || {
+                    let dir = ScratchDir::new("none");
+                    let db = Database::open_with_wal(enabled(&dir.path, SyncMode::None)).unwrap();
+                    (dir, db)
+                },
+                |(_dir, db)| {
+                    black_box(db.execute("CREATE (:N {v: 1})", opts()).unwrap());
+                },
+                BatchSize::SmallInput,
+            );
+        });
+    }
+
+    group.finish();
+}
+
+fn bench_recovery(c: &mut Criterion) {
+    let mut group = c.benchmark_group("wal/recovery");
+
+    // Time how long Database::open_with_wal takes on a directory that
+    // has N committed transactions waiting to be replayed. This bounds
+    // startup time on a crash-recovery boot.
+    for n in [100usize, 1_000].iter().copied() {
+        // Build the WAL once outside the iter loop.
+        let dir = ScratchDir::new(&format!("recovery-{}", n));
+        {
+            let db = Database::open_with_wal(enabled(&dir.path, SyncMode::PerCommit)).unwrap();
+            for _ in 0..n {
+                db.execute("CREATE (:N {v: 1})", opts()).unwrap();
+            }
+            // Drop to release file handles; bg flusher (none here) joins.
+            drop(db);
+        }
+
+        group.bench_function(format!("replay_{}", n), |b| {
+            b.iter(|| {
+                let db = Database::open_with_wal(enabled(&dir.path, SyncMode::None)).unwrap();
+                black_box(db.node_count());
+            });
+        });
+
+        // Keep `dir` alive across the iter so the WAL files persist
+        // for every iteration. ScratchDir's Drop will clean up at
+        // bench end.
+        std::mem::forget(dir);
+    }
+
+    group.finish();
+}
+
+criterion_group! {
+    name = benches;
+    config = smoke_config();
+    targets = bench_commit_latency, bench_recovery,
+}
+criterion_main!(benches);