tensorfs 0.3.69__tar.gz → 0.3.70__tar.gz

{tensorfs-0.3.69 → tensorfs-0.3.70}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.69"
+version = "0.3.70"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.69"
+version = "0.3.70"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.69 → tensorfs-0.3.70}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.69"
+version = "0.3.70"
 edition = "2021"

{tensorfs-0.3.69 → tensorfs-0.3.70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.69
+Version: 0.3.70
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.69 → tensorfs-0.3.70}/crates/tensorfs-core/src/bin/tfs/watch.rs

@@ -29,8 +29,6 @@ use std::process::{Child, ChildStdout, ExitStatus};
 use std::sync::mpsc::{self, RecvTimeoutError};
 use std::time::{Duration, Instant};
 
-use tensorfs_core::catalog::BUSY_TIMEOUT;
-
 /// How many times its own worst silence a worker is granted before the parent looks for
 /// inactivity, and how many times its own projected run it may spin without progress.
 /// Three doublings past the demonstrated maximum: the worst of n observed gaps
@@ -101,8 +99,11 @@ fn parse_progress(line: &str) -> Option<(u64, u64)> {
     Some((moved.parse().ok()?, admitted.parse().ok()?))
 }
 
+/// Patience before a worker has shown any gap to measure against.
+const FIRST_PATIENCE: Duration = Duration::from_secs(5);
+
 fn patience(worst_gap: Duration) -> Duration {
-    (worst_gap * PATIENCE_FACTOR).max(BUSY_TIMEOUT)
+    (worst_gap * PATIENCE_FACTOR).max(FIRST_PATIENCE)
 }
 
 /// What a silent-but-active worker is allowed: the whole declared input at the rate this

tensorfs-0.3.70/crates/tensorfs-core/src/catalog/lock.rs

@@ -0,0 +1,121 @@
+//! Waiting for another process's catalog lock by whether its holder can still release it.
+//!
+//! SQLite's locks are POSIX locks: they die with the process that holds them. A holder
+//! that is alive is committing or about to, so a writer waits for it for as long as it
+//! takes; a flat deadline refused a busy, progressing writer as if it were dead. The only
+//! holder that cannot release is one that is stopped (a debugger, SIGSTOP) or a zombie,
+//! and only then does a statement give up.
+//!
+//! Holders come from `/proc/locks`, matched by the catalog's inode (and its WAL `-shm`).
+//! The catalog file itself is never opened here: closing a second descriptor on it would
+//! drop every POSIX lock this process holds on it.
+
+use std::cell::RefCell;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use rusqlite::Connection;
+
+use crate::err::Result;
+
+thread_local! {
+    /// The catalog this thread's statements wait on. A connection is opened for each
+    /// operation just before it is used, on the thread that uses it, so the one opened last
+    /// here is the one a busy statement belongs to.
+    static WAITING_ON: RefCell<Option<PathBuf>> = const { RefCell::new(None) };
+}
+
+/// Install the waiting handler on one connection.
+pub(crate) fn wait_for_live_holders(connection: &Connection, path: &Path) -> Result<()> {
+    WAITING_ON.with(|waiting| *waiting.borrow_mut() = Some(path.to_path_buf()));
+    connection
+        .busy_handler(Some(busy))
+        .map_err(|error| super::sql("install catalog lock wait", error))
+}
+
+fn busy(attempt: i32) -> bool {
+    let path = WAITING_ON.with(|waiting| waiting.borrow().clone());
+    if path.is_some_and(|path| holders_cannot_release(&path)) {
+        return false;
+    }
+    std::thread::sleep(Duration::from_millis(attempt.clamp(1, 64) as u64));
+    true
+}
+
+/// Every process holding a write lock on the catalog is stopped or a zombie. Without a
+/// holder to name (the lock just went, or no `/proc`), keep asking.
+fn holders_cannot_release(path: &Path) -> bool {
+    let inodes: Vec<u64> = [path.to_path_buf(), shm(path)]
+        .iter()
+        .filter_map(|file| std::fs::metadata(file).ok())
+        .map(|metadata| std::os::unix::fs::MetadataExt::ino(&metadata))
+        .collect();
+    let Ok(locks) = std::fs::read_to_string("/proc/locks") else {
+        return false;
+    };
+    let holders = write_holders(&locks, &inodes, std::process::id());
+    !holders.is_empty() && holders.iter().all(|pid| stopped(*pid))
+}
+
+fn shm(path: &Path) -> PathBuf {
+    let mut name = path.as_os_str().to_owned();
+    name.push("-shm");
+    PathBuf::from(name)
+}
+
+/// Other processes' POSIX write locks on these inodes, from `/proc/locks` lines such as
+/// `3: POSIX  ADVISORY  WRITE 167671 103:02:204736108 1073741824 1073742335`.
+fn write_holders(locks: &str, inodes: &[u64], own: u32) -> Vec<u32> {
+    locks
+        .lines()
+        .filter_map(|line| {
+            let fields: Vec<&str> = line.split_whitespace().collect();
+            let at = fields.iter().position(|field| *field == "POSIX")?;
+            if fields.get(at + 2) != Some(&"WRITE") {
+                return None;
+            }
+            let pid: u32 = fields.get(at + 3)?.parse().ok()?;
+            let inode: u64 = fields.get(at + 4)?.rsplit(':').next()?.parse().ok()?;
+            (pid != own && inodes.contains(&inode)).then_some(pid)
+        })
+        .collect()
+}
+
+/// Stopped, traced-stopped, zombie or dead: a process that cannot release its lock.
+fn stopped(pid: u32) -> bool {
+    std::fs::read_to_string(format!("/proc/{pid}/stat"))
+        .ok()
+        .and_then(|stat| {
+            let state = stat
+                .rsplit_once(')')?
+                .1
+                .split_whitespace()
+                .next()?
+                .to_string();
+            Some(matches!(state.as_str(), "T" | "t" | "Z" | "X"))
+        })
+        .unwrap_or(false)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn only_other_processes_write_locks_on_the_catalog_are_holders() {
+        let locks = "\
+1: POSIX  ADVISORY  WRITE 4242 103:02:77 1073741824 1073742335
+2: POSIX  ADVISORY  READ  5151 103:02:77 1073741826 1073742335
+3: POSIX  ADVISORY  WRITE 6161 103:02:99 120 120
+4: FLOCK  ADVISORY  WRITE 7171 103:02:77 0 EOF
+5: POSIX  ADVISORY  WRITE 1000 103:02:77 0 EOF
+6: -> POSIX  ADVISORY  WRITE 8181 103:02:77 0 EOF
+";
+        assert_eq!(write_holders(locks, &[77], 1000), vec![4242, 8181]);
+        assert_eq!(
+            write_holders(locks, &[77, 99], 1000),
+            vec![4242, 6161, 8181]
+        );
+        assert!(write_holders(locks, &[5], 1000).is_empty());
+    }
+}

{tensorfs-0.3.69 → tensorfs-0.3.70}/crates/tensorfs-core/src/catalog.rs

@@ -6,7 +6,6 @@ use std::fs::{self, File};
 use std::os::unix::fs::MetadataExt;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
-use std::time::Duration;
 
 use fs2::FileExt;
 use rusqlite::{params, Connection};
@@ -14,10 +13,7 @@ use rusqlite::{params, Connection};
 use crate::err::{refuse, Code, Refusal, Result};
 use crate::storage::Projection;
 
-/// The longest one catalog call may SLEEP on another writer's lock before it refuses. It is
-/// the only wait a store operation can legitimately spend blocked and idle, so it is also
-/// the floor under any patience a supervisor grants a worker that has gone quiet.
-pub const BUSY_TIMEOUT: Duration = Duration::from_secs(5);
+mod lock;
 
 /// The catalog this build writes. A newer build may only ADD tables and columns (an
 /// incompatible change renames the table), so any catalog at this version or later that
@@ -62,11 +58,6 @@ fn sql(what: impl AsRef<str>, error: rusqlite::Error) -> Refusal {
     }
 }
 
-/// Whether a catalog refusal is SQLite's busy timeout expiring behind another writer.
-fn busy(refusal: &Refusal) -> bool {
-    refusal.code == Code::IO_FAILED && refusal.detail.ends_with("database is locked")
-}
-
 fn io(what: impl AsRef<str>, error: std::io::Error) -> Refusal {
     crate::store::classify_io(what, error)
 }
@@ -134,13 +125,32 @@ fn connect(path: &Path) -> Result<Connection> {
         }
         sql("open catalog", error)
     })?;
+    configure(&connection, path)?;
+    Ok(connection)
+}
+
+/// Journal and sync by what a crash can take from the disk the catalog lives on
+/// (`disk::DiskClass`): an ephemeral disk dies with the machine, so no sync buys anything;
+/// a persistent one keeps a WAL synced at checkpoints; a network filesystem keeps the
+/// rollback journal and a full sync, because WAL's shared memory is not coherent across
+/// hosts. The catalog is only ever opened by processes on one host otherwise.
+pub(crate) fn configure(connection: &Connection, path: &Path) -> Result<()> {
+    lock::wait_for_live_holders(connection, path)?;
+    let (journal, synchronous) = match crate::disk::DiskClass::of(path.parent().unwrap_or(path)) {
+        crate::disk::DiskClass::Ephemeral => ("wal", "OFF"),
+        crate::disk::DiskClass::Persistent => ("wal", "NORMAL"),
+        crate::disk::DiskClass::Network => ("delete", "FULL"),
+    };
     connection
-        .busy_timeout(BUSY_TIMEOUT)
-        .map_err(|error| sql("set catalog busy timeout", error))?;
+        .query_row(&format!("PRAGMA journal_mode={journal}"), [], |row| {
+            row.get::<_, String>(0)
+        })
+        .map_err(|error| sql("set catalog journal", error))?;
     connection
-        .execute_batch("PRAGMA foreign_keys=ON; PRAGMA synchronous=FULL;")
-        .map_err(|error| sql("configure catalog", error))?;
-    Ok(connection)
+        .execute_batch(&format!(
+            "PRAGMA foreign_keys=ON; PRAGMA synchronous={synchronous};"
+        ))
+        .map_err(|error| sql("configure catalog", error))
 }
 
 /// A connection to an existing catalog of exactly this schema.
@@ -390,22 +400,6 @@ impl Catalog {
         Ok(catalog)
     }
 
-    /// `open` for an object admission. A catalog busy past SQLite's timeout while this
-    /// process's own admission batches keep committing is a queue, not a failure: the
-    /// verified bytes wait for their turn instead of being thrown away and fetched again.
-    /// Busy with no batch committed in the meantime is another writer's lock, and refuses.
-    pub(crate) fn open_admitting(root: &Path) -> Result<Self> {
-        loop {
-            let committed = crate::stats::snapshot().admission_batches;
-            match Self::open(root) {
-                Err(refusal)
-                    if busy(&refusal) && crate::stats::snapshot().admission_batches > committed => {
-                }
-                opened => return opened,
-            }
-        }
-    }
-
     pub fn open(root: &Path) -> Result<Self> {
         let path = Self::path(root);
         validated(&path)?;
@@ -1626,6 +1620,89 @@ mod tests {
         ))
     }
 
+    /// Another process holding the catalog's write lock: this test binary, re-run as
+    /// `lock_holder_child`, until it has taken the lock.
+    fn holder(catalog: &Path, millis: u64) -> std::process::Child {
+        use std::io::BufRead;
+        let mut child = std::process::Command::new(std::env::current_exe().unwrap())
+            .args(["--exact", "catalog::tests::lock_holder_child", "--ignored"])
+            .args(["--nocapture", "--test-threads=1"])
+            .env("TENSORFS_TEST_HOLD", catalog)
+            .env("TENSORFS_TEST_HOLD_MS", millis.to_string())
+            .stdout(std::process::Stdio::piped())
+            .spawn()
+            .unwrap();
+        let mut lines = std::io::BufReader::new(child.stdout.take().unwrap()).lines();
+        let held = lines
+            .by_ref()
+            .map_while(std::result::Result::ok)
+            .any(|line| line.ends_with("held"));
+        assert!(held, "the holder never took the catalog lock");
+        std::thread::spawn(move || lines.for_each(drop));
+        child
+    }
+
+    #[test]
+    #[ignore = "run by holder() in a child process"]
+    fn lock_holder_child() {
+        let path = std::env::var("TENSORFS_TEST_HOLD").unwrap();
+        let millis: u64 = std::env::var("TENSORFS_TEST_HOLD_MS")
+            .unwrap()
+            .parse()
+            .unwrap();
+        let connection = Connection::open(path).unwrap();
+        connection.execute_batch("BEGIN IMMEDIATE").unwrap();
+        connection
+            .execute(
+                "INSERT INTO tensorfs_released_manifests VALUES ('held','by',?1)",
+                [std::process::id().to_string()],
+            )
+            .unwrap();
+        println!("held");
+        std::thread::sleep(std::time::Duration::from_millis(millis));
+        connection.execute_batch("COMMIT").unwrap();
+    }
+
+    fn write(catalog: &Catalog, name: &str) -> Result<usize> {
+        catalog
+            .connection()?
+            .execute(
+                "INSERT INTO tensorfs_released_manifests VALUES ('org',?1,'digest')",
+                [name],
+            )
+            .map_err(|error| sql("test write", error))
+    }
+
+    #[test]
+    fn a_live_lock_holder_is_waited_for_past_any_fixed_deadline() {
+        let root = temporary("live-holder");
+        fs::create_dir_all(&root).unwrap();
+        let catalog = Catalog::initialize(&root).unwrap();
+        let mut child = holder(&catalog.path, 7_000);
+        let started = std::time::Instant::now();
+        assert_eq!(write(&catalog, "after-holder").unwrap(), 1);
+        assert!(started.elapsed() >= std::time::Duration::from_secs(6));
+        assert!(child.wait().unwrap().success());
+        let _ = fs::remove_dir_all(root);
+    }
+
+    #[test]
+    fn a_stopped_lock_holder_is_refused() {
+        use rustix::process::{kill_process, Pid, Signal};
+        let root = temporary("stopped-holder");
+        fs::create_dir_all(&root).unwrap();
+        let catalog = Catalog::initialize(&root).unwrap();
+        let mut child = holder(&catalog.path, 60_000);
+        let pid = Pid::from_raw(child.id() as i32).unwrap();
+        kill_process(pid, Signal::STOP).unwrap();
+        let refused = write(&catalog, "behind-a-stopped-holder").unwrap_err();
+        assert!(refused.detail.contains("locked"), "{refused:?}");
+        kill_process(pid, Signal::KILL).unwrap();
+        child.wait().unwrap();
+        assert_eq!(write(&catalog, "after-the-holder-died").unwrap(), 1);
+        let _ = fs::remove_dir_all(root);
+    }
+
     #[test]
     fn catalog_has_seven_tables_and_opens_additive_future_versions() {
         let root = temporary("schema");

tensorfs-0.3.70/crates/tensorfs-core/src/derived/group_commit_tests.rs

@@ -0,0 +1,114 @@
+//! A live writer on a persistent disk syncs at its checkpoints, not per part, and still
+//! resumes from its last checkpoint after the machine loses its page cache.
+
+use super::*;
+use crate::disk::DiskClass;
+use crate::unsynced::testing::{from_an_earlier_boot, markers, persistent_root, tear};
+
+const TENSORS: usize = 6;
+
+fn declaration() -> Declaration {
+    let mut declaration = super::tests::created_declaration();
+    let prototype = declaration.components[0].add[0].clone();
+    declaration.components[0].add = (0..TENSORS)
+        .map(|index| {
+            let mut tensor = prototype.clone();
+            tensor.key = format!("weight_{index}");
+            tensor.shape = vec![128];
+            tensor.parts[0].shape = vec![128];
+            tensor
+        })
+        .collect();
+    declaration.order = (0..TENSORS)
+        .map(|index| ("model".into(), format!("weight_{index}")))
+        .collect();
+    declaration.max_new_bytes = (TENSORS * 512) as u64;
+    declaration
+}
+
+fn store(name: &str) -> (Store, Meta) {
+    let store = Store::init(&persistent_root(name)).unwrap();
+    assert_eq!(store.disk_class(), DiskClass::Persistent);
+    let meta = Meta::open(&store).unwrap();
+    (store, meta)
+}
+
+fn add(store: &Store, meta: &Meta, id: &str, session: u64, writer: &Begin, index: usize) -> Part {
+    add_indexed_part(
+        store,
+        meta,
+        id,
+        session,
+        &writer.additions,
+        None,
+        ("model", &format!("weight_{index}"), "value"),
+        &mut [index as u8 + 1; 512].as_slice(),
+    )
+    .unwrap()
+    .part
+}
+
+#[test]
+fn a_power_loss_after_a_checkpoint_resumes_from_it_to_the_same_manifest() {
+    let id = format!("sha256:{}", "6c".repeat(32));
+
+    // The reference: one uninterrupted writer.
+    let (clean, clean_meta) = store("group-commit-clean");
+    let writer = begin(&clean, &clean_meta, &id, 1, declaration(), None).unwrap();
+    let live = writer_store(&clean).unwrap();
+    for index in 0..TENSORS {
+        add(&live, &clean_meta, &id, 1, &writer, index);
+    }
+    let expected = commit(&live, &clean_meta, &id, 1).unwrap();
+    finish_writer(&live).unwrap();
+    release_guards(&clean_meta, Some(writer.writer_hold), writer.source_leases);
+    assert!(
+        markers(clean.root()).is_empty(),
+        "a committed writer retires its epoch"
+    );
+
+    // A checkpoint after three parts; three more admitted unsynced; then the machine goes
+    // down and the page cache holding those three is lost.
+    let (crashed, meta) = store("group-commit-crash");
+    let root = crashed.root().to_path_buf();
+    let writer = begin(&crashed, &meta, &id, 1, declaration(), None).unwrap();
+    let live = writer_store(&crashed).unwrap();
+    assert_eq!(
+        markers(&root).len(),
+        1,
+        "a persistent writer admits under one epoch"
+    );
+    for index in 0..3 {
+        add(&live, &meta, &id, 1, &writer, index);
+    }
+    let (_, head) = checkpoint_progress(&live, &meta, &id, 1, "request", "model", None).unwrap();
+    let lost: Vec<Part> = (3..TENSORS)
+        .map(|index| add(&live, &meta, &id, 1, &writer, index))
+        .collect();
+    fence(&meta, &id, 1).unwrap();
+    release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+    drop(live);
+    from_an_earlier_boot(&markers(&root).pop().unwrap());
+    for segment in lost.iter().flat_map(Part::segments) {
+        tear(&crashed.blob_path(&segment.sha256));
+    }
+
+    // Opening repairs the torn objects; the resumed writer keeps what its checkpoint made
+    // durable and recomputes the rest.
+    let reopened = Store::open(&root).unwrap();
+    let meta = Meta::open(&reopened).unwrap();
+    let head = head.head.expect("the checkpoint names a head");
+    let writer = begin(&reopened, &meta, &id, 2, declaration(), Some(&head)).unwrap();
+    assert_eq!(completed(&meta, &id, 2).unwrap().0.len(), 3);
+    let live = writer_store(&reopened).unwrap();
+    for index in 3..TENSORS {
+        add(&live, &meta, &id, 2, &writer, index);
+    }
+    let resumed = commit(&live, &meta, &id, 2).unwrap();
+    finish_writer(&live).unwrap();
+    release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+    assert_eq!(resumed.manifest, expected.manifest);
+    assert!(markers(&root).is_empty());
+    std::fs::remove_dir_all(clean.root()).unwrap();
+    std::fs::remove_dir_all(&root).unwrap();
+}

{tensorfs-0.3.69 → tensorfs-0.3.70}/crates/tensorfs-core/src/derived/roots.rs

@@ -123,6 +123,20 @@ fn io(error: std::io::Error) -> Refusal {
 fn directory(store: &Store) -> PathBuf {
     store.root().join("roots/derived")
 }
+
+/// A root must survive the machine on every disk but an ephemeral one, whose Store dies with
+/// its container. A parts journal follows the Store's admissions instead ([`Store::flushes`]):
+/// under a writer's epoch its lines become durable with that writer's next checkpoint.
+fn durable(store: &Store) -> bool {
+    store.disk_class() != crate::disk::DiskClass::Ephemeral
+}
+
+/// Create the root directory; true when this call created it (so its parents need a sync).
+fn create_directory(store: &Store) -> Result<bool> {
+    let created = !directory(store).is_dir();
+    fs::create_dir_all(directory(store)).map_err(io)?;
+    Ok(created)
+}
 fn path(store: &Store, transaction: &str) -> Result<PathBuf> {
     let transaction = super::transaction_id(transaction)?;
     Ok(directory(store).join(format!(
@@ -150,7 +164,7 @@ pub(super) fn append_parts(store: &Store, transaction: &str, objects: &[ObjectRe
         return Ok(());
     }
     let path = parts_path(store, transaction)?;
-    fs::create_dir_all(directory(store)).map_err(io)?;
+    let parents = create_directory(store)?;
     let created = !path.exists();
     let mut file = OpenOptions::new()
         .read(true)
@@ -170,11 +184,12 @@ pub(super) fn append_parts(store: &Store, transaction: &str, objects: &[ObjectRe
         vec![b'\n']
     };
     bytes.extend(parts_line(objects));
-    file.write_all(&bytes)
-        .and_then(|()| file.sync_data())
-        .map_err(io)?;
-    if created {
-        sync_directories(store)?;
+    file.write_all(&bytes).map_err(io)?;
+    if store.flushes() {
+        file.sync_data().map_err(io)?;
+        if created {
+            sync_directories(store, parents)?;
+        }
     }
     Ok(())
 }
@@ -182,7 +197,7 @@ pub(super) fn append_parts(store: &Store, transaction: &str, objects: &[ObjectRe
 /// Replace the journal with exactly these objects (a writer's merged progress).
 pub(super) fn rewrite_parts(store: &Store, transaction: &str, objects: &[ObjectRef]) -> Result<()> {
     let path = parts_path(store, transaction)?;
-    fs::create_dir_all(directory(store)).map_err(io)?;
+    let parents = create_directory(store)?;
     let temporary = path.with_extension(format!("parts-tmp-{}", crate::meta::now_nanos_unique()));
     let mut file = OpenOptions::new()
         .write(true)
@@ -194,11 +209,15 @@ pub(super) fn rewrite_parts(store: &Store, transaction: &str, objects: &[ObjectR
     } else {
         parts_line(objects)
     };
-    file.write_all(&bytes)
-        .and_then(|()| file.sync_all())
-        .map_err(io)?;
+    file.write_all(&bytes).map_err(io)?;
+    if store.flushes() {
+        file.sync_all().map_err(io)?;
+    }
     fs::rename(&temporary, &path).map_err(io)?;
-    sync_directories(store)
+    if store.flushes() {
+        sync_directories(store, parents)?;
+    }
+    Ok(())
 }
 
 /// Every object named by any transaction's parts journal.
@@ -235,12 +254,15 @@ pub(super) fn journaled(store: &Store) -> Result<Vec<ObjectRef>> {
     Ok(objects)
 }
 
-fn sync_directories(store: &Store) -> Result<()> {
-    for directory in [
+/// A rename needs its own directory synced; the parents only when this write created it.
+fn sync_directories(store: &Store, parents: bool) -> Result<()> {
+    let own = [directory(store)];
+    let all = [
         directory(store),
         store.root().join("roots"),
         store.root().to_path_buf(),
-    ] {
+    ];
+    for directory in if parents { &all[..] } else { &own[..] } {
         File::open(directory)
             .and_then(|file| file.sync_all())
             .map_err(io)?;
@@ -301,25 +323,20 @@ pub(super) fn write(store: &Store, root: &Root) -> Result<()> {
         return refuse(Code::COUNT_CAP, "private derived root exceeds its byte cap");
     }
     let path = path(store, &root.transaction)?;
-    fs::create_dir_all(directory(store)).map_err(io)?;
+    let parents = create_directory(store)?;
     let temporary = path.with_extension(format!("tmp-{}", crate::meta::now_nanos_unique()));
     let mut file = OpenOptions::new()
         .write(true)
         .create_new(true)
         .open(&temporary)
         .map_err(io)?;
-    file.write_all(&bytes)
-        .and_then(|()| file.sync_all())
-        .map_err(io)?;
+    file.write_all(&bytes).map_err(io)?;
+    if durable(store) {
+        file.sync_all().map_err(io)?;
+    }
     fs::rename(&temporary, &path).map_err(io)?;
-    for directory in [
-        directory(store),
-        store.root().join("roots"),
-        store.root().to_path_buf(),
-    ] {
-        File::open(directory)
-            .and_then(|file| file.sync_all())
-            .map_err(io)?;
+    if durable(store) {
+        sync_directories(store, parents)?;
     }
     Ok(())
 }
@@ -396,6 +413,7 @@ pub(super) fn remove(store: &Store, transaction: &str) -> Result<()> {
         Err(error) => return Err(io(error)),
     }
     match fs::remove_file(path(store, transaction)?) {
+        Ok(()) if !durable(store) => Ok(()),
         Ok(()) => File::open(directory(store))
             .and_then(|file| file.sync_all())
             .map_err(io),

{tensorfs-0.3.69 → tensorfs-0.3.70}/crates/tensorfs-core/src/derived.rs

@@ -2346,6 +2346,19 @@ fn planned_row(
     })
 }
 
+/// The Store a live writer admits through. On a persistent disk its parts and progress
+/// documents are not synced one by one: each checkpoint (and the commit) makes everything
+/// admitted since the previous one durable at once, before the root names it, so a crash
+/// loses at most the work after the last checkpoint. An ephemeral disk needs no sync at all.
+pub fn writer_store(store: &Store) -> Result<Store> {
+    Ok(store.syncing(crate::unsynced::Epoch::begin_every(store, u64::MAX)?))
+}
+
+/// The writer is done: sync anything admitted since its last checkpoint and retire its epoch.
+pub fn finish_writer(store: &Store) -> Result<()> {
+    store.finish_admitted()
+}
+
 pub fn begin(
     store: &Store,
     meta: &Meta,
@@ -2810,6 +2823,8 @@ impl CheckpointCursor {
         let Some(mut root) = root else {
             return Ok(None);
         };
+        // Everything the new head names is durable before the root publishes it.
+        store.sync_admitted()?;
         root.checkpoint = extended.head.head.clone();
         roots::write(store, &root)?;
         self.tip
@@ -2935,6 +2950,7 @@ pub fn checkpoint_progress(
         objects,
         None,
     )?;
+    store.sync_admitted()?;
     let mut txn = meta.txn_for(transaction)?;
     let row = open_row(&mut txn.rows, transaction, writer_session)?;
     retain_progress(store, row, exported.head.head.as_ref())?;
@@ -3609,6 +3625,7 @@ pub fn commit(
     walk.require_held(store, &crate::ingest::custody::live(store.root())?)?;
     let (inherited_payload_objects, inherited_payload_bytes) =
         inherit_observation(&header, &row_before)?;
+    store.sync_admitted()?;
 
     let mut txn = meta.txn_for(&transaction)?;
     let row = open_row(&mut txn.rows, &transaction, writer_session)?;
@@ -4598,3 +4615,6 @@ mod companion_tests;
 
 #[cfg(test)]
 mod part_rows_tests;
+
+#[cfg(test)]
+mod group_commit_tests;

{tensorfs-0.3.69 → tensorfs-0.3.70}/crates/tensorfs-core/src/meta.rs

@@ -222,12 +222,7 @@ impl Meta {
 
     fn connection(&self) -> Result<Connection> {
         let connection = Connection::open(&self.db).map_err(|error| sql("open catalog", error))?;
-        connection
-            .busy_timeout(std::time::Duration::from_secs(5))
-            .map_err(|error| sql("set catalog busy timeout", error))?;
-        connection
-            .execute_batch("PRAGMA foreign_keys=ON; PRAGMA synchronous=FULL;")
-            .map_err(|error| sql("configure catalog", error))?;
+        crate::catalog::configure(&connection, &self.db)?;
         Ok(connection)
     }