lora-python 0.8.1__tar.gz → 0.8.3__tar.gz

{lora_python-0.8.1 → lora_python-0.8.3}/Cargo.lock

@@ -797,7 +797,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
 
 [[package]]
 name = "lora-analyzer"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "lora-ast",
  "lora-parser",
@@ -807,14 +807,14 @@ dependencies = [
 
 [[package]]
 name = "lora-ast"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "smallvec 2.0.0-alpha.12",
 ]
 
 [[package]]
 name = "lora-binding-buffer"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "lora-database",
  "lora-store",
@@ -822,7 +822,7 @@ dependencies = [
 
 [[package]]
 name = "lora-compiler"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "lora-analyzer",
  "lora-ast",
@@ -830,7 +830,7 @@ dependencies = [
 
 [[package]]
 name = "lora-database"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -852,7 +852,7 @@ dependencies = [
 
 [[package]]
 name = "lora-executor"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "lora-analyzer",
  "lora-ast",
@@ -867,7 +867,7 @@ dependencies = [
 
 [[package]]
 name = "lora-ffi"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "anyhow",
  "lora-binding-buffer",
@@ -879,7 +879,7 @@ dependencies = [
 
 [[package]]
 name = "lora-node"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "anyhow",
  "lora-binding-buffer",
@@ -894,7 +894,7 @@ dependencies = [
 
 [[package]]
 name = "lora-parser"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "lora-ast",
  "pest",
@@ -905,7 +905,7 @@ dependencies = [
 
 [[package]]
 name = "lora-python"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "anyhow",
  "lora-database",
@@ -917,7 +917,7 @@ dependencies = [
 
 [[package]]
 name = "lora-server"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "anyhow",
  "axum",
@@ -931,7 +931,7 @@ dependencies = [
 
 [[package]]
 name = "lora-snapshot"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "argon2",
  "bincode",
@@ -946,7 +946,7 @@ dependencies = [
 
 [[package]]
 name = "lora-store"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "bincode",
  "crc32fast",
@@ -958,7 +958,7 @@ dependencies = [
 
 [[package]]
 name = "lora-wal"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "crc32fast",
  "lora-store",
@@ -968,7 +968,7 @@ dependencies = [
 
 [[package]]
 name = "lora-wasm"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "anyhow",
  "console_error_panic_hook",
@@ -984,7 +984,7 @@ dependencies = [
 
 [[package]]
 name = "lora_ruby"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "anyhow",
  "lora-database",

{lora_python-0.8.1 → lora_python-0.8.3}/Cargo.toml

@@ -4,7 +4,7 @@ resolver = "2"
 
 [workspace.package]
 edition = "2021"
-version = "0.8.1"
+version = "0.8.3"
 license = "BUSL-1.1"
 authors = ["LoraDB, Inc."]
 repository = "https://github.com/lora-db/lora"
@@ -15,16 +15,16 @@ rust-version = "1.87"
 # Internal crates — versions are kept in lockstep with [workspace.package].version
 # by `scripts/sync-versions.mjs`. Both `path` and `version` are set so that
 # `cargo publish` works (crates.io cannot resolve path-only deps).
-lora-ast = { path = "crates/lora-ast", version = "=0.8.1" }
-lora-parser = { path = "crates/lora-parser", version = "=0.8.1" }
-lora-analyzer = { path = "crates/lora-analyzer", version = "=0.8.1" }
-lora-compiler = { path = "crates/lora-compiler", version = "=0.8.1" }
-lora-store = { path = "crates/lora-store", version = "=0.8.1" }
-lora-snapshot = { path = "crates/lora-snapshot", version = "=0.8.1", default-features = false }
-lora-wal = { path = "crates/lora-wal", version = "=0.8.1" }
-lora-executor = { path = "crates/lora-executor", version = "=0.8.1" }
-lora-database = { path = "crates/lora-database", version = "=0.8.1" }
-lora-binding-buffer = { path = "crates/bindings/lora-binding-buffer", version = "=0.8.1" }
+lora-ast = { path = "crates/lora-ast", version = "=0.8.3" }
+lora-parser = { path = "crates/lora-parser", version = "=0.8.3" }
+lora-analyzer = { path = "crates/lora-analyzer", version = "=0.8.3" }
+lora-compiler = { path = "crates/lora-compiler", version = "=0.8.3" }
+lora-store = { path = "crates/lora-store", version = "=0.8.3" }
+lora-snapshot = { path = "crates/lora-snapshot", version = "=0.8.3", default-features = false }
+lora-wal = { path = "crates/lora-wal", version = "=0.8.3" }
+lora-executor = { path = "crates/lora-executor", version = "=0.8.3" }
+lora-database = { path = "crates/lora-database", version = "=0.8.3" }
+lora-binding-buffer = { path = "crates/bindings/lora-binding-buffer", version = "=0.8.3" }
 
 # External crates.
 anyhow = "1"

{lora_python-0.8.1 → lora_python-0.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lora-python
-Version: 0.8.1
+Version: 0.8.3
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8

{lora_python-0.8.1 → lora_python-0.8.3}/crates/lora-database/Cargo.toml

@@ -63,3 +63,7 @@ harness = false
 [[bench]]
 name = "concurrent_benchmarks"
 harness = false
+
+[[bench]]
+name = "concurrency_guard_benchmarks"
+harness = false

lora_python-0.8.3/crates/lora-database/benches/concurrency_guard_benchmarks.rs

@@ -0,0 +1,303 @@
+//! Tight local performance guard for concurrency work.
+//!
+//! `perf_smoke_benchmarks` is the broad CI canary and intentionally allows
+//! large runner noise. This suite is narrower and meant for phase-by-phase
+//! local checks while changing the concurrency/write/WAL plumbing:
+//!
+//! ```text
+//! cargo bench -p lora-database --bench concurrency_guard_benchmarks \
+//!     -- --output-format bencher > /tmp/lora-before.bencher
+//! # make one implementation step
+//! cargo bench -p lora-database --bench concurrency_guard_benchmarks \
+//!     -- --output-format bencher > /tmp/lora-after.bencher
+//! node scripts/check-bench-delta.mjs \
+//!     --baseline /tmp/lora-before.bencher \
+//!     --current /tmp/lora-after.bencher \
+//!     --threshold 1.15
+//! ```
+//!
+//! The cases here deliberately cover the hot surfaces most likely to regress
+//! while introducing concurrent writes: snapshot reads, live streams,
+//! auto-commit writes, explicit transactions, mixed read/write threads, and
+//! WAL-backed write paths.
+
+mod fixtures;
+
+use std::collections::BTreeMap;
+use std::hint::black_box;
+use std::path::PathBuf;
+use std::sync::{Arc, Barrier};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
+use fixtures::{build_node_graph, BenchDb, Scale};
+use lora_database::{
+    Database, ExecuteOptions, InMemoryGraph, LoraValue, ResultFormat, SyncMode, TransactionMode,
+    WalConfig,
+};
+
+fn opts() -> Option<ExecuteOptions> {
+    Some(ExecuteOptions {
+        format: ResultFormat::Rows,
+    })
+}
+
+fn guard_config() -> Criterion {
+    Criterion::default()
+        .warm_up_time(Duration::from_millis(400))
+        .measurement_time(Duration::from_millis(1_800))
+        .sample_size(30)
+        .noise_threshold(0.08)
+}
+
+struct ScratchDir {
+    path: PathBuf,
+}
+
+impl ScratchDir {
+    fn new(tag: &str) -> Self {
+        let mut path = std::env::temp_dir();
+        path.push(format!(
+            "lora-concurrency-guard-{}-{}-{}",
+            tag,
+            std::process::id(),
+            SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap()
+                .as_nanos()
+        ));
+        std::fs::create_dir_all(&path).unwrap();
+        Self { path }
+    }
+}
+
+impl Drop for ScratchDir {
+    fn drop(&mut self) {
+        let _ = std::fs::remove_dir_all(&self.path);
+    }
+}
+
+fn wal_config(dir: &std::path::Path, sync_mode: SyncMode) -> WalConfig {
+    WalConfig::Enabled {
+        dir: dir.to_path_buf(),
+        sync_mode,
+        segment_target_bytes: 8 * 1024 * 1024,
+    }
+}
+
+fn int_param(name: &str, value: i64) -> BTreeMap<String, LoraValue> {
+    let mut params = BTreeMap::new();
+    params.insert(name.to_string(), LoraValue::Int(value));
+    params
+}
+
+fn two_int_params(a: (&str, i64), b: (&str, i64)) -> BTreeMap<String, LoraValue> {
+    let mut params = BTreeMap::new();
+    params.insert(a.0.to_string(), LoraValue::Int(a.1));
+    params.insert(b.0.to_string(), LoraValue::Int(b.1));
+    params
+}
+
+fn bench_concurrency_guard(c: &mut Criterion) {
+    let mut group = c.benchmark_group("concurrency_guard");
+
+    // Snapshot read path: catches regressions in `LiveStore::load_full`,
+    // plan execution, and read-only WAL prechecks.
+    {
+        let db = build_node_graph(Scale::SMALL);
+        group.bench_function("read_scan_1k", |b| {
+            b.iter(|| {
+                black_box(
+                    db.service
+                        .execute("MATCH (n:Node) RETURN n.id", opts())
+                        .unwrap(),
+                );
+            });
+        });
+    }
+
+    // Live read stream: pins an Arc snapshot and drops after one row.
+    {
+        let db = build_node_graph(Scale::SMALL);
+        group.bench_function("stream_pull_one_1k", |b| {
+            b.iter(|| {
+                let mut stream = db.service.stream("MATCH (n:Node) RETURN n.id").unwrap();
+                black_box(stream.next_row().unwrap());
+            });
+        });
+    }
+
+    // Auto-commit create on one long-lived DB. This keeps the benchmark on the
+    // write hot path instead of measuring fixture setup.
+    {
+        let db = BenchDb::new();
+        let mut next = 0i64;
+        group.bench_function("write_create_one_steady", |b| {
+            b.iter(|| {
+                next += 1;
+                black_box(
+                    db.service
+                        .execute_with_params(
+                            "CREATE (:Guard {id: $id})",
+                            opts(),
+                            int_param("id", next),
+                        )
+                        .unwrap(),
+                );
+            });
+        });
+    }
+
+    // Existing-record write on a 1k graph. Future per-record commit
+    // validation should keep this close to the current single-writer cost.
+    {
+        let db = build_node_graph(Scale::SMALL);
+        let mut next = 0i64;
+        group.bench_function("write_set_existing_1k", |b| {
+            b.iter(|| {
+                next += 1;
+                black_box(
+                    db.service
+                        .execute_with_params(
+                            "MATCH (n:Node {id: $id}) SET n.value = $value",
+                            opts(),
+                            two_int_params(("id", 500), ("value", next)),
+                        )
+                        .unwrap(),
+                );
+            });
+        });
+    }
+
+    // Fixed explicit transaction cost. This should stay tiny while
+    // introducing concurrent write bookkeeping.
+    {
+        let db = build_node_graph(Scale::SMALL);
+        group.bench_function("tx_roundtrip_empty", |b| {
+            b.iter(|| {
+                let tx = db
+                    .service
+                    .begin_transaction(TransactionMode::ReadWrite)
+                    .unwrap();
+                tx.commit().unwrap();
+            });
+        });
+    }
+
+    // Explicit transaction write path, including buffering recorder and
+    // publish. Fresh DB per iteration so the committed write cannot
+    // accumulate into the next sample.
+    group.bench_function("tx_write_create_one", |b| {
+        b.iter_batched(
+            BenchDb::new,
+            |db| {
+                let mut tx = db
+                    .service
+                    .begin_transaction(TransactionMode::ReadWrite)
+                    .unwrap();
+                black_box(tx.execute("CREATE (:TxGuard {id: 1})", opts()).unwrap());
+                tx.commit().unwrap();
+            },
+            BatchSize::SmallInput,
+        );
+    });
+
+    // Mixed reader/writer pressure. Thread spawn overhead is intentional here:
+    // this is a coarse local guard for accidental global locks on read paths.
+    {
+        let db = Arc::new(build_node_graph(Scale::SMALL).service);
+        let mut next = 0i64;
+        group.bench_function("mixed_4_readers_1_writer", |b| {
+            b.iter(|| {
+                next += 1;
+                let barrier = Arc::new(Barrier::new(5));
+                let handles: Vec<_> = (0..4)
+                    .map(|_| {
+                        let db = db.clone();
+                        let barrier = barrier.clone();
+                        std::thread::spawn(move || {
+                            barrier.wait();
+                            black_box(db.execute("MATCH (n:Node) RETURN n.id", opts()).unwrap());
+                        })
+                    })
+                    .collect();
+
+                let writer = {
+                    let db = db.clone();
+                    let barrier = barrier.clone();
+                    std::thread::spawn(move || {
+                        barrier.wait();
+                        black_box(
+                            db.execute_with_params(
+                                "CREATE (:MixedGuard {id: $id})",
+                                opts(),
+                                int_param("id", next),
+                            )
+                            .unwrap(),
+                        );
+                    })
+                };
+
+                for handle in handles {
+                    handle.join().unwrap();
+                }
+                writer.join().unwrap();
+            });
+        });
+    }
+
+    // WAL write paths without the per-iteration directory setup measured.
+    // `None` isolates WAL encoding/flush-buffer overhead; cooperative Group
+    // isolates the path that future concurrent fsync coordination will touch.
+    {
+        let dir = ScratchDir::new("wal-none");
+        let db = Database::<InMemoryGraph>::open_with_wal(wal_config(&dir.path, SyncMode::None))
+            .unwrap();
+        let mut next = 0i64;
+        group.bench_function("wal_none_create_delete_one", |b| {
+            b.iter(|| {
+                next += 1;
+                black_box(
+                    db.execute_with_params(
+                        "CREATE (n:WalGuard {id: $id}) DELETE n",
+                        opts(),
+                        int_param("id", next),
+                    )
+                    .unwrap(),
+                );
+            });
+        });
+    }
+
+    {
+        let dir = ScratchDir::new("wal-group");
+        let db = Database::<InMemoryGraph>::open_with_wal(wal_config(
+            &dir.path,
+            SyncMode::Group { interval_ms: 50 },
+        ))
+        .unwrap();
+        let mut next = 0i64;
+        group.bench_function("wal_group_create_delete_one", |b| {
+            b.iter(|| {
+                next += 1;
+                black_box(
+                    db.execute_with_params(
+                        "CREATE (n:WalGuard {id: $id}) DELETE n",
+                        opts(),
+                        int_param("id", next),
+                    )
+                    .unwrap(),
+                );
+            });
+        });
+    }
+
+    group.finish();
+}
+
+criterion_group! {
+    name = benches;
+    config = guard_config();
+    targets = bench_concurrency_guard,
+}
+criterion_main!(benches);

{lora_python-0.8.1 → lora_python-0.8.3}/crates/lora-database/src/database/builder.rs

@@ -132,9 +132,8 @@ impl Database<InMemoryGraph> {
         let mirror: Arc<dyn WalMirror> = archive;
         let recorder = Arc::new(WalRecorder::new_with_mirror(wal, Some(mirror)));
         // Mark the archive dirty so a fresh named database is materialized as
-        // a portable ZIP. The archive writer coalesces this with any immediate
-        // follow-up writes and flushes it in the background, with a final flush
-        // on database drop.
+        // a portable ZIP. Follow-up writes refresh the archive on explicit
+        // `sync()` / checkpoint and on clean database drop.
         recorder.flush()?;
         Ok(Self::from_graph_with_wal(graph, recorder, None))
     }

{lora_python-0.8.1 → lora_python-0.8.3}/crates/lora-database/src/database/execute.rs

@@ -68,21 +68,17 @@ where
     /// 2. The executor runs; every primitive mutation fires
     ///    `MutationRecorder::record`, which buffers events in memory.
     /// 3. On Ok, `recorder.commit()` writes `TxBegin`, one batched
-    ///    mutation record, and `TxCommit` only when mutations occurred;
-    ///    the surrounding `recorder.flush()` runs only in that case so
-    ///    a read-only query never pays an `fsync`.
+    ///    mutation record, and `TxCommit` only when mutations occurred,
+    ///    then applies the WAL's configured single-thread flush policy.
     /// 4. On Err, `recorder.abort()` clears the pending batch. The
     ///    engine has no rollback, so the in-memory state may already
     ///    be partially mutated; the live handle is quarantined while
     ///    durable recovery stays atomic because no committed batch was
     ///    written.
-    /// 5. The recorder's poisoned flag is polled once (it also
-    ///    surfaces background-flusher fsync failures from
-    ///    `SyncMode::Group`). If set, the query fails loudly with the
-    ///    durability error so the caller can act on it; the WAL
-    ///    refuses further appends until the operator restarts the
-    ///    database, which recovers from the last consistent
-    ///    snapshot + WAL.
+    /// 5. The recorder's poisoned flag is polled once. If set, the query
+    ///    fails loudly with the durability error so the caller can act on it;
+    ///    the WAL refuses further appends until the operator restarts the
+    ///    database, which recovers from the last consistent snapshot + WAL.
     pub fn execute_with_params(
         &self,
         query: &str,
@@ -157,15 +153,11 @@ where
                 .map_err(anyhow::Error::from);
         }
 
-        // Mutating path. Drop the snapshot we used for compilation and
-        // route through the optimistic auto-commit path: build the
-        // working copy + mutate without holding any lock, then take
-        // the writer Mutex only briefly to do a CAS publish (replays
-        // buffered mutation events to the durable WAL inside the
-        // critical section). Multiple auto-commit writers can run
-        // their prep work in parallel; they only serialize at the
-        // commit point. On conflict (another writer published since
-        // we took our snapshot), retry from a fresh snapshot.
+        // Mutating path. Drop the snapshot used for compilation and route
+        // through the single-writer auto-commit path. The writer mutex is held
+        // for mutate + WAL commit so live state and WAL order stay identical.
+        // The method name still says "optimistic" because this is the seam a
+        // future write-set/CAS implementation can reuse.
         drop(store);
         debug_assert!(shape.is_mutating());
 

{lora_python-0.8.1 → lora_python-0.8.3}/crates/lora-database/src/database/mod.rs

@@ -26,7 +26,6 @@ use crate::explain::{QueryPlan, QueryProfile};
 use crate::live_store::LiveStore;
 use crate::plan_cache::PlanCache;
 use crate::snapshot::ManagedSnapshotStore;
-use crate::wal::write_scope::WalAbortPolicy;
 
 /// Minimal abstraction any transport can depend on to run Lora queries.
 ///
@@ -192,14 +191,12 @@ where
     /// Drop every node and relationship, returning WAL/archive errors to the
     /// caller.
     ///
-    /// When a WAL is attached, the clear is wrapped in `arm`/`commit` so the
-    /// `MutationEvent::Clear` fired by the store reaches the log inside a
-    /// transaction. If a failure happens after the in-memory graph has been
-    /// cleared, the recorder is poisoned by the failing WAL path and future
-    /// writes fail until the database is reopened from durable state.
+    /// When a WAL is attached, the buffered `MutationEvent::Clear` is appended
+    /// to the log on success. The clear runs in place against the live
+    /// graph via the same fast path as `with_logged_store_mut`, so a
+    /// large graph clear no longer pays an O(N+E) snapshot clone.
     pub fn try_clear(&self) -> Result<(), LoraError> {
-        let guard = self.write_store();
-        self.with_logged_write_guard(guard, WalAbortPolicy::AbortOnly, |store| {
+        self.with_logged_store_mut(|store| {
             store.clear();
             Ok(())
         })
@@ -236,13 +233,18 @@ where
 
     /// Run a closure with an exclusive borrow of the underlying store. Reserved
     /// for admin paths (restore, bulk load); regular mutation goes through
-    /// `execute_with_params`. The closure mutates a staged copy that is
-    /// published atomically when the closure returns.
+    /// `execute_with_params`. The closure mutates the live graph in place
+    /// via `Arc::make_mut`, so callers don't pay an O(N+E) snapshot clone
+    /// just to overwrite the graph. Concurrent readers, when present,
+    /// force a single CoW clone and keep observing their pre-mutation
+    /// snapshot via the `Arc<S>` they already hold.
     pub fn with_store_mut<R>(&self, f: impl FnOnce(&mut S) -> R) -> R {
-        let mut guard = self.write_store();
-        let result = f(&mut *guard);
-        guard.publish();
-        result
+        let _lock = self
+            .writer
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        let mut handle = self.store.write();
+        f(handle.as_mut())
     }
 }
 

lora_python-0.8.3/crates/lora-database/src/database/occ.rs

@@ -0,0 +1,52 @@
+//! Auto-commit write path for `InMemoryGraph`.
+//!
+//! This module is the dispatcher between [`Database::execute_with_params`]
+//! and the canonical mutating shape in [`super::write_guard`]. It
+//! builds a `MutableExecutor` against the live store and hands it to
+//! [`Database::run_with_durable_recorder`], which owns the writer
+//! mutex, recorder arm/commit/abort lifecycle, and the
+//! managed-snapshot trigger. The single-writer design (`Arc::make_mut`
+//! against the live `Arc<S>`) and the failure trade-off — a query
+//! that fails mid-execution can leave the live graph partially
+//! mutated, but never the durable log — are documented in
+//! [`super::write_guard`].
+
+use std::any::Any;
+use std::collections::BTreeMap;
+use std::sync::Arc;
+use std::time::Instant;
+
+use anyhow::Result;
+use lora_compiler::CompiledQuery;
+use lora_executor::{LoraValue, MutableExecutionContext, MutableExecutor, Row};
+use lora_store::{GraphStorage, GraphStorageMut};
+
+use crate::database::Database;
+
+impl<S> Database<S>
+where
+    S: GraphStorage + GraphStorageMut + Any + Clone + Send + Sync + 'static,
+{
+    /// Auto-commit a mutating query. Builds a `MutableExecutor`
+    /// against the staged graph and routes through the canonical
+    /// write shape in [`Database::run_with_durable_recorder`].
+    pub(crate) fn execute_mutating_optimistic(
+        &self,
+        params: BTreeMap<String, LoraValue>,
+        deadline: Option<Instant>,
+        compiled: &Arc<CompiledQuery>,
+    ) -> Result<Vec<Row>> {
+        self.run_with_durable_recorder(|staged| {
+            let mut executor = MutableExecutor::with_deadline(
+                MutableExecutionContext {
+                    storage: staged,
+                    params,
+                },
+                deadline,
+            );
+            executor
+                .execute_compiled_rows(compiled)
+                .map_err(anyhow::Error::from)
+        })
+    }
+}