lora-python 0.27.1__tar.gz → 0.28.0__tar.gz

{lora_python-0.27.1 → lora_python-0.28.0}/Cargo.lock

@@ -854,7 +854,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
 
 [[package]]
 name = "lora-analyzer"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "lora-ast",
  "lora-builtins-meta",
@@ -865,14 +865,14 @@ dependencies = [
 
 [[package]]
 name = "lora-ast"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "smallvec 2.0.0-alpha.12",
 ]
 
 [[package]]
 name = "lora-binding-buffer"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "lora-database",
  "lora-store",
@@ -880,11 +880,11 @@ dependencies = [
 
 [[package]]
 name = "lora-builtins-meta"
-version = "0.27.1"
+version = "0.28.0"
 
 [[package]]
 name = "lora-compiler"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "lora-analyzer",
  "lora-ast",
@@ -893,7 +893,7 @@ dependencies = [
 
 [[package]]
 name = "lora-database"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "anyhow",
  "chacha20poly1305",
@@ -918,7 +918,7 @@ dependencies = [
 
 [[package]]
 name = "lora-executor"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "blake3",
  "crc32fast",
@@ -941,7 +941,7 @@ dependencies = [
 
 [[package]]
 name = "lora-ffi"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "anyhow",
  "lora-binding-buffer",
@@ -953,7 +953,7 @@ dependencies = [
 
 [[package]]
 name = "lora-io"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "lora-executor",
  "lora-store",
@@ -964,7 +964,7 @@ dependencies = [
 
 [[package]]
 name = "lora-node"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "anyhow",
  "lora-binding-buffer",
@@ -979,7 +979,7 @@ dependencies = [
 
 [[package]]
 name = "lora-parser"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "lora-ast",
  "pest",
@@ -990,7 +990,7 @@ dependencies = [
 
 [[package]]
 name = "lora-python"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "anyhow",
  "lora-binding-buffer",
@@ -1003,7 +1003,7 @@ dependencies = [
 
 [[package]]
 name = "lora-query-wasm"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "console_error_panic_hook",
  "js-sys",
@@ -1017,7 +1017,7 @@ dependencies = [
 
 [[package]]
 name = "lora-server"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "anyhow",
  "axum",
@@ -1032,7 +1032,7 @@ dependencies = [
 
 [[package]]
 name = "lora-snapshot"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "argon2",
  "blake3",
@@ -1047,7 +1047,7 @@ dependencies = [
 
 [[package]]
 name = "lora-store"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "crc32fast",
  "jiff",
@@ -1062,7 +1062,7 @@ dependencies = [
 
 [[package]]
 name = "lora-wal"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "crc32fast",
  "lora-store",
@@ -1072,7 +1072,7 @@ dependencies = [
 
 [[package]]
 name = "lora-wasm"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "anyhow",
  "console_error_panic_hook",
@@ -1088,7 +1088,7 @@ dependencies = [
 
 [[package]]
 name = "lora_ruby"
-version = "0.27.1"
+version = "0.28.0"
 dependencies = [
  "anyhow",
  "lora-database",

{lora_python-0.27.1 → lora_python-0.28.0}/Cargo.toml

@@ -4,7 +4,7 @@ resolver = "2"
 
 [workspace.package]
 edition = "2021"
-version = "0.27.1"
+version = "0.28.0"
 license = "BUSL-1.1"
 authors = ["LoraDB, Inc."]
 repository = "https://github.com/lora-db/lora"
@@ -15,18 +15,18 @@ rust-version = "1.87"
 # Internal crates — versions are kept in lockstep with [workspace.package].version
 # by `scripts/sync-versions.mjs`. Both `path` and `version` are set so that
 # `cargo publish` works (crates.io cannot resolve path-only deps).
-lora-ast = { path = "crates/lora-ast", version = "=0.27.1" }
-lora-parser = { path = "crates/lora-parser", version = "=0.27.1" }
-lora-builtins-meta = { path = "crates/lora-builtins-meta", version = "=0.27.1" }
-lora-analyzer = { path = "crates/lora-analyzer", version = "=0.27.1" }
-lora-compiler = { path = "crates/lora-compiler", version = "=0.27.1" }
-lora-store = { path = "crates/lora-store", version = "=0.27.1" }
-lora-snapshot = { path = "crates/lora-snapshot", version = "=0.27.1", default-features = false }
-lora-io = { path = "crates/lora-io", version = "=0.27.1" }
-lora-wal = { path = "crates/lora-wal", version = "=0.27.1" }
-lora-executor = { path = "crates/lora-executor", version = "=0.27.1", default-features = false }
-lora-database = { path = "crates/lora-database", version = "=0.27.1" }
-lora-binding-buffer = { path = "crates/bindings/lora-binding-buffer", version = "=0.27.1" }
+lora-ast = { path = "crates/lora-ast", version = "=0.28.0" }
+lora-parser = { path = "crates/lora-parser", version = "=0.28.0" }
+lora-builtins-meta = { path = "crates/lora-builtins-meta", version = "=0.28.0" }
+lora-analyzer = { path = "crates/lora-analyzer", version = "=0.28.0" }
+lora-compiler = { path = "crates/lora-compiler", version = "=0.28.0" }
+lora-store = { path = "crates/lora-store", version = "=0.28.0" }
+lora-snapshot = { path = "crates/lora-snapshot", version = "=0.28.0", default-features = false }
+lora-io = { path = "crates/lora-io", version = "=0.28.0" }
+lora-wal = { path = "crates/lora-wal", version = "=0.28.0" }
+lora-executor = { path = "crates/lora-executor", version = "=0.28.0", default-features = false }
+lora-database = { path = "crates/lora-database", version = "=0.28.0" }
+lora-binding-buffer = { path = "crates/bindings/lora-binding-buffer", version = "=0.28.0" }
 
 # External crates.
 anyhow = "1"

{lora_python-0.27.1 → lora_python-0.28.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lora-python
-Version: 0.27.1
+Version: 0.28.0
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8

lora_python-0.28.0/crates/lora-database/examples/crash_probe.rs

@@ -0,0 +1,245 @@
+//! Kill-9 probe for a named database's write, `sync()` and open paths.
+//!
+//! Run with:
+//!
+//! ```text
+//! cargo run --release -p lora-database --example crash_probe -- <dir> <rounds> [preload]
+//! ```
+//!
+//! Each round starts a child that opens `<dir>/probe.loradb`, writes and
+//! calls `sync()` in a loop, and prints each write once it is
+//! acknowledged; the parent kills it with SIGKILL after a random delay,
+//! which lands in writes, in `sync()` and in the open itself. The parent
+//! then opens the database and checks that every acknowledged write is
+//! there and that the graph is whole: each id at most once, a chain of
+//! relationships between consecutive ids, and each node's counter equal
+//! to the number of updates acknowledged for it (or one more, for a write
+//! killed after it committed and before it was acknowledged).
+//!
+//! `preload` nodes are written before the first round, so a checkpoint
+//! takes long enough to be killed in the middle.
+
+use std::collections::BTreeMap;
+use std::io::{BufRead, BufReader};
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::time::{Duration, Instant};
+
+use lora_database::{Database, DatabaseOpenOptions, ExecuteOptions, ResultFormat};
+
+fn rows() -> Option<ExecuteOptions> {
+    Some(ExecuteOptions {
+        format: ResultFormat::Rows,
+    })
+}
+
+fn options(dir: &Path) -> DatabaseOpenOptions {
+    DatabaseOpenOptions {
+        segment_target_bytes: 256 * 1024,
+        ..DatabaseOpenOptions::default().with_database_dir(dir)
+    }
+}
+
+fn random_u64() -> u64 {
+    let mut bytes = [0u8; 8];
+    getrandom::getrandom(&mut bytes).unwrap();
+    u64::from_le_bytes(bytes)
+}
+
+fn query_rows(db: &Database<lora_database::InMemoryGraph>, query: &str) -> Vec<serde_json::Value> {
+    let result = db.execute(query, rows()).unwrap();
+    let json = serde_json::to_value(&result).unwrap();
+    json["rows"].as_array().cloned().unwrap_or_default()
+}
+
+/// Highest id in the database, or -1 when empty.
+fn max_id(db: &Database<lora_database::InMemoryGraph>) -> i64 {
+    query_rows(db, "MATCH (n:P) RETURN coalesce(max(n.id), -1) AS m")[0]["m"]
+        .as_i64()
+        .unwrap()
+}
+
+/// Child: write forever, printing `c <id>` after a create and `u <id>`
+/// after an update, and `s` after a sync.
+fn child(dir: &Path) {
+    let db = Database::open_named("probe", options(dir)).unwrap();
+    println!("open");
+    let mut next = max_id(&db) + 1;
+    loop {
+        let roll = random_u64() % 100;
+        if roll < 3 {
+            db.sync().unwrap();
+            println!("s");
+        } else if roll < 30 && next > 0 {
+            let id = (random_u64() % next as u64) as i64;
+            db.execute(
+                &format!("MATCH (n:P {{id: {id}}}) SET n.updates = n.updates + 1"),
+                rows(),
+            )
+            .unwrap();
+            println!("u {id}");
+        } else {
+            let id = next;
+            db.execute(
+                &format!(
+                    "CREATE (n:P {{id: {id}, updates: 0, name: 'p' + toString({id}), tags: ['a', {id}]}}) \
+                     WITH n OPTIONAL MATCH (prev:P {{id: {id} - 1}}) \
+                     FOREACH (_ IN CASE WHEN prev IS NULL THEN [] ELSE [1] END | CREATE (prev)-[:NEXT]->(n))"
+                ),
+                rows(),
+            )
+            .unwrap();
+            next += 1;
+            println!("c {id}");
+        }
+    }
+}
+
+struct Acked {
+    max_created: i64,
+    updates: BTreeMap<i64, i64>,
+    syncs: u64,
+}
+
+fn verify(dir: &Path, acked: &Acked) {
+    let db = Database::open_named("probe", options(dir)).unwrap();
+    let max = max_id(&db);
+    assert!(
+        max >= acked.max_created && max <= acked.max_created + 1,
+        "max id {max}, acknowledged {}",
+        acked.max_created
+    );
+    let counts = &query_rows(
+        &db,
+        "MATCH (n:P) RETURN count(n) AS n, count(DISTINCT n.id) AS d",
+    )[0];
+    assert_eq!(counts["n"].as_i64().unwrap(), max + 1, "node count");
+    assert_eq!(counts["d"].as_i64().unwrap(), max + 1, "distinct ids");
+    let chain = &query_rows(
+        &db,
+        "MATCH (a:P)-[:NEXT]->(b:P) RETURN count(*) AS n, sum(CASE WHEN b.id = a.id + 1 THEN 1 ELSE 0 END) AS ok",
+    )[0];
+    assert_eq!(chain["n"].as_i64().unwrap(), max.max(0), "chain length");
+    assert_eq!(chain["ok"].as_i64().unwrap(), max.max(0), "chain order");
+    let stored: BTreeMap<i64, i64> = query_rows(
+        &db,
+        "MATCH (n:P) WHERE n.updates > 0 RETURN n.id AS id, n.updates AS u",
+    )
+    .into_iter()
+    .map(|row| (row["id"].as_i64().unwrap(), row["u"].as_i64().unwrap()))
+    .collect();
+    let mut extra = 0;
+    for (id, want) in &acked.updates {
+        let got = stored.get(id).copied().unwrap_or(0);
+        assert!(
+            got >= *want,
+            "node {id}: {got} updates, {want} acknowledged"
+        );
+        extra += got - want;
+    }
+    for (id, got) in &stored {
+        if !acked.updates.contains_key(id) {
+            extra += got;
+        }
+    }
+    assert!(extra <= 1, "{extra} unacknowledged updates");
+}
+
+fn main() {
+    let args: Vec<String> = std::env::args().collect();
+    if args.get(1).map(String::as_str) == Some("child") {
+        child(Path::new(&args[2]));
+        return;
+    }
+    let dir = PathBuf::from(&args[1]);
+    let rounds: usize = args[2].parse().unwrap();
+    let preload: usize = args.get(3).map_or(0, |n| n.parse().unwrap());
+    std::fs::create_dir_all(&dir).unwrap();
+
+    if preload > 0 {
+        let db = Database::open_named("probe", options(&dir)).unwrap();
+        db.execute(
+            &format!(
+                "UNWIND list.range(0, {}) AS i CREATE (:Pad {{i: i, s: 'pad' + toString(i)}})",
+                preload - 1
+            ),
+            rows(),
+        )
+        .unwrap();
+        db.sync().unwrap();
+    }
+
+    let mut acked = {
+        let db = Database::open_named("probe", options(&dir)).unwrap();
+        let updates = query_rows(
+            &db,
+            "MATCH (n:P) WHERE n.updates > 0 RETURN n.id AS id, n.updates AS u",
+        )
+        .into_iter()
+        .map(|row| (row["id"].as_i64().unwrap(), row["u"].as_i64().unwrap()))
+        .collect();
+        Acked {
+            max_created: max_id(&db),
+            updates,
+            syncs: 0,
+        }
+    };
+
+    let exe = std::env::current_exe().unwrap();
+    let started = Instant::now();
+    let mut killed_in_open = 0;
+    for round in 0..rounds {
+        let mut child = Command::new(&exe)
+            .arg("child")
+            .arg(&dir)
+            .stdout(Stdio::piped())
+            .spawn()
+            .unwrap();
+        let stdout = child.stdout.take().unwrap();
+        let reader = std::thread::spawn(move || {
+            BufReader::new(stdout)
+                .lines()
+                .map_while(Result::ok)
+                .collect::<Vec<_>>()
+        });
+        std::thread::sleep(Duration::from_millis(50 + random_u64() % 1500));
+        child.kill().unwrap();
+        child.wait().unwrap();
+        let lines = reader.join().unwrap();
+        if !lines.iter().any(|l| l == "open") {
+            killed_in_open += 1;
+        }
+        for line in &lines {
+            match line.split_once(' ') {
+                Some(("c", id)) => acked.max_created = id.parse().unwrap(),
+                Some(("u", id)) => *acked.updates.entry(id.parse().unwrap()).or_insert(0) += 1,
+                _ if line == "s" => acked.syncs += 1,
+                _ => {}
+            }
+        }
+        verify(&dir, &acked);
+        // The verify's own open counts as one more clean open: an
+        // unacknowledged write it found is now part of the history.
+        let db = Database::open_named("probe", options(&dir)).unwrap();
+        acked.max_created = max_id(&db);
+        acked.updates = query_rows(
+            &db,
+            "MATCH (n:P) WHERE n.updates > 0 RETURN n.id AS id, n.updates AS u",
+        )
+        .into_iter()
+        .map(|row| (row["id"].as_i64().unwrap(), row["u"].as_i64().unwrap()))
+        .collect();
+        drop(db);
+        if (round + 1) % 10 == 0 {
+            println!(
+                "round {:>4}: {} nodes, {} syncs acknowledged, {} kills during open, {:.0} s",
+                round + 1,
+                acked.max_created + 1,
+                acked.syncs,
+                killed_in_open,
+                started.elapsed().as_secs_f64()
+            );
+        }
+    }
+    println!("ok: {rounds} rounds");
+}

lora_python-0.28.0/crates/lora-database/examples/sync_memory_probe.rs

@@ -0,0 +1,190 @@
+//! Heap probe for a named database's checkpoint (`sync()`) and open.
+//!
+//! Run with:
+//!
+//! ```text
+//! cargo run --release -p lora-database --example sync_memory_probe -- build <dir> <nodes>
+//! cargo run --release -p lora-database --example sync_memory_probe -- measure <dir>
+//! ```
+//!
+//! `build` writes a `.loradb` with `<nodes>` nodes (a unique key, a short
+//! hash, two integers, a float and a list each) and as many relationships.
+//! `writes <dir> <count>` times single-statement creates against it.
+//! `measure` opens it, calls `sync()` three times and drops it, reporting
+//! for each step the live heap after it and the peak during it. The
+//! allocator is wrapped with a live-byte counter, so the figures are the
+//! bytes the process asked for, whatever malloc then keeps resident; the
+//! resident set after each step is printed beside them.
+
+use std::alloc::{GlobalAlloc, Layout, System};
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicIsize, Ordering};
+use std::time::Instant;
+
+use lora_database::{Database, DatabaseOpenOptions, ExecuteOptions, ResultFormat};
+
+struct Counting;
+
+static LIVE: AtomicIsize = AtomicIsize::new(0);
+static PEAK: AtomicIsize = AtomicIsize::new(0);
+
+fn grew(by: isize) {
+    let live = LIVE.fetch_add(by, Ordering::Relaxed) + by;
+    PEAK.fetch_max(live, Ordering::Relaxed);
+}
+
+unsafe impl GlobalAlloc for Counting {
+    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+        let p = System.alloc(layout);
+        if !p.is_null() {
+            grew(layout.size() as isize);
+        }
+        p
+    }
+    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+        System.dealloc(ptr, layout);
+        LIVE.fetch_sub(layout.size() as isize, Ordering::Relaxed);
+    }
+    unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
+        let p = System.realloc(ptr, layout, new_size);
+        if !p.is_null() {
+            grew(new_size as isize - layout.size() as isize);
+        }
+        p
+    }
+}
+
+#[global_allocator]
+static GLOBAL: Counting = Counting;
+
+/// Resident set size from `ps`, in bytes: what malloc kept, beside the
+/// counter's view of what the process asked for.
+fn rss() -> isize {
+    std::process::Command::new("ps")
+        .args(["-o", "rss=", "-p", &std::process::id().to_string()])
+        .output()
+        .ok()
+        .and_then(|out| String::from_utf8(out.stdout).ok())
+        .and_then(|out| out.trim().parse::<isize>().ok())
+        .map_or(0, |kib| kib * 1024)
+}
+
+fn mib(bytes: isize) -> f64 {
+    bytes as f64 / (1024.0 * 1024.0)
+}
+
+/// Run `step`, printing the live heap after it and the peak during it.
+fn measure<T>(name: &str, step: impl FnOnce() -> T) -> T {
+    PEAK.store(LIVE.load(Ordering::Relaxed), Ordering::Relaxed);
+    let before = LIVE.load(Ordering::Relaxed);
+    let started = Instant::now();
+    let out = step();
+    let elapsed = started.elapsed();
+    let after = LIVE.load(Ordering::Relaxed);
+    let peak = PEAK.load(Ordering::Relaxed);
+    println!(
+        "{name:<10} live {:>8.1} MiB   peak {:>8.1} MiB   (+{:>7.1} over before)   rss {:>8.1} MiB   {:>6.2} s",
+        mib(after),
+        mib(peak),
+        mib(peak - before),
+        mib(rss()),
+        elapsed.as_secs_f64()
+    );
+    out
+}
+
+fn options(dir: &PathBuf) -> DatabaseOpenOptions {
+    DatabaseOpenOptions::default().with_database_dir(dir)
+}
+
+fn rows() -> Option<ExecuteOptions> {
+    Some(ExecuteOptions {
+        format: ResultFormat::Rows,
+    })
+}
+
+fn main() {
+    let args: Vec<String> = std::env::args().collect();
+    match args.get(1).map(String::as_str) {
+        Some("build") => {
+            let dir = PathBuf::from(&args[2]);
+            let nodes: usize = args[3].parse().expect("node count");
+            let db = Database::open_named("probe", options(&dir)).unwrap();
+            db.execute(
+                "CREATE CONSTRAINT probe_key FOR (n:Place) REQUIRE n.key IS UNIQUE",
+                rows(),
+            )
+            .unwrap();
+            let batch = 50_000;
+            let mut start = 0;
+            while start < nodes {
+                let end = (start + batch).min(nodes) - 1;
+                db.execute(
+                    &format!(
+                        "UNWIND list.range({start}, {end}) AS i \
+                         CREATE (:Place {{key: i, hash: 'h' + toString(i * 7919), \
+                         a: i % 1000, b: i * 3, lat: i / 1000.0, tags: ['x', i % 7]}})"
+                    ),
+                    rows(),
+                )
+                .unwrap();
+                start = end + 1;
+            }
+            let mut start = 1;
+            while start < nodes {
+                let end = (start + batch).min(nodes) - 1;
+                db.execute(
+                    &format!(
+                        "UNWIND list.range({start}, {end}) AS i \
+                         MATCH (a:Place {{key: i - 1}}), (b:Place {{key: i}}) \
+                         CREATE (a)-[:NEXT {{d: i % 13}}]->(b)"
+                    ),
+                    rows(),
+                )
+                .unwrap();
+                start = end + 1;
+            }
+            db.sync().unwrap();
+            println!(
+                "built {} nodes, {} relationships",
+                db.node_count(),
+                db.relationship_count()
+            );
+        }
+        Some("measure") => {
+            let dir = PathBuf::from(&args[2]);
+            let db = measure("open", || {
+                Database::open_named("probe", options(&dir)).unwrap()
+            });
+            for i in 1..=3 {
+                measure(&format!("sync {i}"), || db.sync().unwrap());
+            }
+            measure("dispose", || drop(db));
+        }
+        Some("writes") => {
+            // Single-statement creates against the built database: commit
+            // throughput while the archive worker rewrites the container
+            // behind them.
+            let dir = PathBuf::from(&args[2]);
+            let count: usize = args[3].parse().expect("write count");
+            let db = Database::open_named("probe", options(&dir)).unwrap();
+            let started = Instant::now();
+            for i in 0..count {
+                db.execute(&format!("CREATE (:Extra {{i: {i}}})"), rows())
+                    .unwrap();
+            }
+            let writes = started.elapsed();
+            let started = Instant::now();
+            drop(db);
+            println!(
+                "{count} writes in {:.2} s ({:.0}/s), dispose {:.2} s",
+                writes.as_secs_f64(),
+                count as f64 / writes.as_secs_f64(),
+                started.elapsed().as_secs_f64()
+            );
+        }
+        _ => eprintln!(
+            "usage: sync_memory_probe build <dir> <nodes> | measure <dir> | writes <dir> <count>"
+        ),
+    }
+}