tensorfs 0.3.69__tar.gz → 0.3.71__tar.gz

{tensorfs-0.3.69 → tensorfs-0.3.71}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.69"
+version = "0.3.71"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.69"
+version = "0.3.71"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.69 → tensorfs-0.3.71}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.69"
+version = "0.3.71"
 edition = "2021"

{tensorfs-0.3.69 → tensorfs-0.3.71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.69
+Version: 0.3.71
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.69 → tensorfs-0.3.71}/crates/tensorfs-core/src/bin/tfs/ingest.rs

@@ -445,7 +445,7 @@ pub fn cmd_bank(dialect: &str, converter: &str, f: &Flags) -> ExitCode {
     let mut reviewed = Vec::new();
     for (component, path) in &srcs {
         let (full, raw) = ok!(source_core::read_carrier_at(path));
-        let h = if conv.name == "single_file.identity/1" {
+        let h = if conv.dialect == "safetensors.single_file" {
             ok!(convert::single_file_component(component, &full))
         } else {
             full
@@ -584,11 +584,11 @@ pub fn cmd_bank(dialect: &str, converter: &str, f: &Flags) -> ExitCode {
             .collect::<Vec<_>>()
             .join(",");
         let variant = flag(f, "spec-variant").and_then(|value| value.parse().ok());
-        let mut prepared = match source_core::prepare_with_registry(
+        let mut prepared = match source_core::prepare(
             &target,
             &unnamed(&srcs),
             &CarrierSet::default(),
-            &reg,
+            Some(&reg),
             variant,
         ) {
             Ok(prepared) => prepared,
@@ -849,17 +849,22 @@ fn prepare_closed_source_plan(
     if closed.registry != BUILTIN_REGISTRY {
         flags.push(("registry".to_string(), closed.registry.clone()));
     }
+    let as_is = closed.converter == convert::IDENTITY;
+    if as_is {
+        flags.push(("source-profile".to_string(), source_core::AS_IS.to_string()));
+    }
     for (component, source) in &closed.sources {
         flags.push((
             "source".to_string(),
             format!("{component}={}", source.path.to_string_lossy()),
         ));
     }
-    let mut prepared = source_core::prepare_with_registry(
+    // The plan was re-derived above, so its converter is the planner's: as-is has no registry.
+    let mut prepared = source_core::prepare(
         &closed.target,
         &closed.sources,
         &closed.set,
-        &registry,
+        (!as_is).then_some(&registry),
         None,
     )
     .map_err(bail)?;
@@ -989,15 +994,17 @@ pub(crate) fn construction_order(f: &Flags) -> Result<Vec<(String, String)>, Exi
 
 /// Headers in, plan out. The one function `plan`, `run` and the worker all go through, so
 /// a preview and a run cannot disagree about what was going to happen.
+/// `--source-profile as-is/1` stores the sources as they are, unclassified.
 fn prepare(f: &Flags, target: &str) -> Result<Prepared, ExitCode> {
     let sources = sources(f)?;
     let registry = load_registry(f)?;
     let variant = flag(f, "spec-variant").and_then(|value| value.parse().ok());
-    source_core::prepare_with_registry(
+    let as_is = flag(f, "source-profile") == Some(source_core::AS_IS);
+    source_core::prepare(
         target,
         &unnamed(&sources),
         &CarrierSet::default(),
-        &registry,
+        (!as_is).then_some(&registry),
         variant,
     )
     .map_err(bail)
@@ -1206,6 +1213,7 @@ pub fn cmd_run(root: &Path, target: &str, f: &Flags) -> ExitCode {
                 | "tenant"
                 | "isolation-tier"
                 | "checkpoint-bytes"
+                | "source-profile"
         ) {
             argv.push(format!("--{k}"));
             argv.push(v.clone());

{tensorfs-0.3.69 → tensorfs-0.3.71}/crates/tensorfs-core/src/bin/tfs/watch.rs

@@ -29,8 +29,6 @@ use std::process::{Child, ChildStdout, ExitStatus};
 use std::sync::mpsc::{self, RecvTimeoutError};
 use std::time::{Duration, Instant};
 
-use tensorfs_core::catalog::BUSY_TIMEOUT;
-
 /// How many times its own worst silence a worker is granted before the parent looks for
 /// inactivity, and how many times its own projected run it may spin without progress.
 /// Three doublings past the demonstrated maximum: the worst of n observed gaps
@@ -101,8 +99,11 @@ fn parse_progress(line: &str) -> Option<(u64, u64)> {
     Some((moved.parse().ok()?, admitted.parse().ok()?))
 }
 
+/// Patience before a worker has shown any gap to measure against.
+const FIRST_PATIENCE: Duration = Duration::from_secs(5);
+
 fn patience(worst_gap: Duration) -> Duration {
-    (worst_gap * PATIENCE_FACTOR).max(BUSY_TIMEOUT)
+    (worst_gap * PATIENCE_FACTOR).max(FIRST_PATIENCE)
 }
 
 /// What a silent-but-active worker is allowed: the whole declared input at the rate this

tensorfs-0.3.71/crates/tensorfs-core/src/catalog/lock.rs

@@ -0,0 +1,121 @@
+//! Waiting for another process's catalog lock by whether its holder can still release it.
+//!
+//! SQLite's locks are POSIX locks: they die with the process that holds them. A holder
+//! that is alive is committing or about to, so a writer waits for it for as long as it
+//! takes; a flat deadline refused a busy, progressing writer as if it were dead. The only
+//! holder that cannot release is one that is stopped (a debugger, SIGSTOP) or a zombie,
+//! and only then does a statement give up.
+//!
+//! Holders come from `/proc/locks`, matched by the catalog's inode (and its WAL `-shm`).
+//! The catalog file itself is never opened here: closing a second descriptor on it would
+//! drop every POSIX lock this process holds on it.
+
+use std::cell::RefCell;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use rusqlite::Connection;
+
+use crate::err::Result;
+
+thread_local! {
+    /// The catalog this thread's statements wait on. A connection is opened for each
+    /// operation just before it is used, on the thread that uses it, so the one opened last
+    /// here is the one a busy statement belongs to.
+    static WAITING_ON: RefCell<Option<PathBuf>> = const { RefCell::new(None) };
+}
+
+/// Install the waiting handler on one connection.
+pub(crate) fn wait_for_live_holders(connection: &Connection, path: &Path) -> Result<()> {
+    WAITING_ON.with(|waiting| *waiting.borrow_mut() = Some(path.to_path_buf()));
+    connection
+        .busy_handler(Some(busy))
+        .map_err(|error| super::sql("install catalog lock wait", error))
+}
+
+fn busy(attempt: i32) -> bool {
+    let path = WAITING_ON.with(|waiting| waiting.borrow().clone());
+    if path.is_some_and(|path| holders_cannot_release(&path)) {
+        return false;
+    }
+    std::thread::sleep(Duration::from_millis(attempt.clamp(1, 64) as u64));
+    true
+}
+
+/// Every process holding a write lock on the catalog is stopped or a zombie. Without a
+/// holder to name (the lock just went, or no `/proc`), keep asking.
+fn holders_cannot_release(path: &Path) -> bool {
+    let inodes: Vec<u64> = [path.to_path_buf(), shm(path)]
+        .iter()
+        .filter_map(|file| std::fs::metadata(file).ok())
+        .map(|metadata| std::os::unix::fs::MetadataExt::ino(&metadata))
+        .collect();
+    let Ok(locks) = std::fs::read_to_string("/proc/locks") else {
+        return false;
+    };
+    let holders = write_holders(&locks, &inodes, std::process::id());
+    !holders.is_empty() && holders.iter().all(|pid| stopped(*pid))
+}
+
+fn shm(path: &Path) -> PathBuf {
+    let mut name = path.as_os_str().to_owned();
+    name.push("-shm");
+    PathBuf::from(name)
+}
+
+/// Other processes' POSIX write locks on these inodes, from `/proc/locks` lines such as
+/// `3: POSIX  ADVISORY  WRITE 167671 103:02:204736108 1073741824 1073742335`.
+fn write_holders(locks: &str, inodes: &[u64], own: u32) -> Vec<u32> {
+    locks
+        .lines()
+        .filter_map(|line| {
+            let fields: Vec<&str> = line.split_whitespace().collect();
+            let at = fields.iter().position(|field| *field == "POSIX")?;
+            if fields.get(at + 2) != Some(&"WRITE") {
+                return None;
+            }
+            let pid: u32 = fields.get(at + 3)?.parse().ok()?;
+            let inode: u64 = fields.get(at + 4)?.rsplit(':').next()?.parse().ok()?;
+            (pid != own && inodes.contains(&inode)).then_some(pid)
+        })
+        .collect()
+}
+
+/// Stopped, traced-stopped, zombie or dead: a process that cannot release its lock.
+fn stopped(pid: u32) -> bool {
+    std::fs::read_to_string(format!("/proc/{pid}/stat"))
+        .ok()
+        .and_then(|stat| {
+            let state = stat
+                .rsplit_once(')')?
+                .1
+                .split_whitespace()
+                .next()?
+                .to_string();
+            Some(matches!(state.as_str(), "T" | "t" | "Z" | "X"))
+        })
+        .unwrap_or(false)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn only_other_processes_write_locks_on_the_catalog_are_holders() {
+        let locks = "\
+1: POSIX  ADVISORY  WRITE 4242 103:02:77 1073741824 1073742335
+2: POSIX  ADVISORY  READ  5151 103:02:77 1073741826 1073742335
+3: POSIX  ADVISORY  WRITE 6161 103:02:99 120 120
+4: FLOCK  ADVISORY  WRITE 7171 103:02:77 0 EOF
+5: POSIX  ADVISORY  WRITE 1000 103:02:77 0 EOF
+6: -> POSIX  ADVISORY  WRITE 8181 103:02:77 0 EOF
+";
+        assert_eq!(write_holders(locks, &[77], 1000), vec![4242, 8181]);
+        assert_eq!(
+            write_holders(locks, &[77, 99], 1000),
+            vec![4242, 6161, 8181]
+        );
+        assert!(write_holders(locks, &[5], 1000).is_empty());
+    }
+}

{tensorfs-0.3.69 → tensorfs-0.3.71}/crates/tensorfs-core/src/catalog.rs

@@ -6,7 +6,6 @@ use std::fs::{self, File};
 use std::os::unix::fs::MetadataExt;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
-use std::time::Duration;
 
 use fs2::FileExt;
 use rusqlite::{params, Connection};
@@ -14,10 +13,7 @@ use rusqlite::{params, Connection};
 use crate::err::{refuse, Code, Refusal, Result};
 use crate::storage::Projection;
 
-/// The longest one catalog call may SLEEP on another writer's lock before it refuses. It is
-/// the only wait a store operation can legitimately spend blocked and idle, so it is also
-/// the floor under any patience a supervisor grants a worker that has gone quiet.
-pub const BUSY_TIMEOUT: Duration = Duration::from_secs(5);
+mod lock;
 
 /// The catalog this build writes. A newer build may only ADD tables and columns (an
 /// incompatible change renames the table), so any catalog at this version or later that
@@ -62,11 +58,6 @@ fn sql(what: impl AsRef<str>, error: rusqlite::Error) -> Refusal {
     }
 }
 
-/// Whether a catalog refusal is SQLite's busy timeout expiring behind another writer.
-fn busy(refusal: &Refusal) -> bool {
-    refusal.code == Code::IO_FAILED && refusal.detail.ends_with("database is locked")
-}
-
 fn io(what: impl AsRef<str>, error: std::io::Error) -> Refusal {
     crate::store::classify_io(what, error)
 }
@@ -134,13 +125,32 @@ fn connect(path: &Path) -> Result<Connection> {
         }
         sql("open catalog", error)
     })?;
+    configure(&connection, path)?;
+    Ok(connection)
+}
+
+/// Journal and sync by what a crash can take from the disk the catalog lives on
+/// (`disk::DiskClass`): an ephemeral disk dies with the machine, so no sync buys anything;
+/// a persistent one keeps a WAL synced at checkpoints; a network filesystem keeps the
+/// rollback journal and a full sync, because WAL's shared memory is not coherent across
+/// hosts. The catalog is only ever opened by processes on one host otherwise.
+pub(crate) fn configure(connection: &Connection, path: &Path) -> Result<()> {
+    lock::wait_for_live_holders(connection, path)?;
+    let (journal, synchronous) = match crate::disk::DiskClass::of(path.parent().unwrap_or(path)) {
+        crate::disk::DiskClass::Ephemeral => ("wal", "OFF"),
+        crate::disk::DiskClass::Persistent => ("wal", "NORMAL"),
+        crate::disk::DiskClass::Network => ("delete", "FULL"),
+    };
     connection
-        .busy_timeout(BUSY_TIMEOUT)
-        .map_err(|error| sql("set catalog busy timeout", error))?;
+        .query_row(&format!("PRAGMA journal_mode={journal}"), [], |row| {
+            row.get::<_, String>(0)
+        })
+        .map_err(|error| sql("set catalog journal", error))?;
     connection
-        .execute_batch("PRAGMA foreign_keys=ON; PRAGMA synchronous=FULL;")
-        .map_err(|error| sql("configure catalog", error))?;
-    Ok(connection)
+        .execute_batch(&format!(
+            "PRAGMA foreign_keys=ON; PRAGMA synchronous={synchronous};"
+        ))
+        .map_err(|error| sql("configure catalog", error))
 }
 
 /// A connection to an existing catalog of exactly this schema.
@@ -390,22 +400,6 @@ impl Catalog {
         Ok(catalog)
     }
 
-    /// `open` for an object admission. A catalog busy past SQLite's timeout while this
-    /// process's own admission batches keep committing is a queue, not a failure: the
-    /// verified bytes wait for their turn instead of being thrown away and fetched again.
-    /// Busy with no batch committed in the meantime is another writer's lock, and refuses.
-    pub(crate) fn open_admitting(root: &Path) -> Result<Self> {
-        loop {
-            let committed = crate::stats::snapshot().admission_batches;
-            match Self::open(root) {
-                Err(refusal)
-                    if busy(&refusal) && crate::stats::snapshot().admission_batches > committed => {
-                }
-                opened => return opened,
-            }
-        }
-    }
-
     pub fn open(root: &Path) -> Result<Self> {
         let path = Self::path(root);
         validated(&path)?;
@@ -1626,6 +1620,89 @@ mod tests {
         ))
     }
 
+    /// Another process holding the catalog's write lock: this test binary, re-run as
+    /// `lock_holder_child`, until it has taken the lock.
+    fn holder(catalog: &Path, millis: u64) -> std::process::Child {
+        use std::io::BufRead;
+        let mut child = std::process::Command::new(std::env::current_exe().unwrap())
+            .args(["--exact", "catalog::tests::lock_holder_child", "--ignored"])
+            .args(["--nocapture", "--test-threads=1"])
+            .env("TENSORFS_TEST_HOLD", catalog)
+            .env("TENSORFS_TEST_HOLD_MS", millis.to_string())
+            .stdout(std::process::Stdio::piped())
+            .spawn()
+            .unwrap();
+        let mut lines = std::io::BufReader::new(child.stdout.take().unwrap()).lines();
+        let held = lines
+            .by_ref()
+            .map_while(std::result::Result::ok)
+            .any(|line| line.ends_with("held"));
+        assert!(held, "the holder never took the catalog lock");
+        std::thread::spawn(move || lines.for_each(drop));
+        child
+    }
+
+    #[test]
+    #[ignore = "run by holder() in a child process"]
+    fn lock_holder_child() {
+        let path = std::env::var("TENSORFS_TEST_HOLD").unwrap();
+        let millis: u64 = std::env::var("TENSORFS_TEST_HOLD_MS")
+            .unwrap()
+            .parse()
+            .unwrap();
+        let connection = Connection::open(path).unwrap();
+        connection.execute_batch("BEGIN IMMEDIATE").unwrap();
+        connection
+            .execute(
+                "INSERT INTO tensorfs_released_manifests VALUES ('held','by',?1)",
+                [std::process::id().to_string()],
+            )
+            .unwrap();
+        println!("held");
+        std::thread::sleep(std::time::Duration::from_millis(millis));
+        connection.execute_batch("COMMIT").unwrap();
+    }
+
+    fn write(catalog: &Catalog, name: &str) -> Result<usize> {
+        catalog
+            .connection()?
+            .execute(
+                "INSERT INTO tensorfs_released_manifests VALUES ('org',?1,'digest')",
+                [name],
+            )
+            .map_err(|error| sql("test write", error))
+    }
+
+    #[test]
+    fn a_live_lock_holder_is_waited_for_past_any_fixed_deadline() {
+        let root = temporary("live-holder");
+        fs::create_dir_all(&root).unwrap();
+        let catalog = Catalog::initialize(&root).unwrap();
+        let mut child = holder(&catalog.path, 7_000);
+        let started = std::time::Instant::now();
+        assert_eq!(write(&catalog, "after-holder").unwrap(), 1);
+        assert!(started.elapsed() >= std::time::Duration::from_secs(6));
+        assert!(child.wait().unwrap().success());
+        let _ = fs::remove_dir_all(root);
+    }
+
+    #[test]
+    fn a_stopped_lock_holder_is_refused() {
+        use rustix::process::{kill_process, Pid, Signal};
+        let root = temporary("stopped-holder");
+        fs::create_dir_all(&root).unwrap();
+        let catalog = Catalog::initialize(&root).unwrap();
+        let mut child = holder(&catalog.path, 60_000);
+        let pid = Pid::from_raw(child.id() as i32).unwrap();
+        kill_process(pid, Signal::STOP).unwrap();
+        let refused = write(&catalog, "behind-a-stopped-holder").unwrap_err();
+        assert!(refused.detail.contains("locked"), "{refused:?}");
+        kill_process(pid, Signal::KILL).unwrap();
+        child.wait().unwrap();
+        assert_eq!(write(&catalog, "after-the-holder-died").unwrap(), 1);
+        let _ = fs::remove_dir_all(root);
+    }
+
     #[test]
     fn catalog_has_seven_tables_and_opens_additive_future_versions() {
         let root = temporary("schema");

{tensorfs-0.3.69 → tensorfs-0.3.71}/crates/tensorfs-core/src/derived/companion_tests.rs

@@ -196,6 +196,50 @@ fn ordinary_companions_survive_derivation_without_copying_tensor_backing_files()
     std::fs::remove_dir_all(store.root()).unwrap();
 }
 
+#[test]
+fn a_verified_store_object_is_a_companion_by_reference_and_never_a_new_byte() {
+    let (store, meta, id) = fixture();
+    let vocab = vec![0x7b; 256 << 10]; // over the inline bound: references have none
+    let object = store
+        .put_stream(
+            &mut vocab.as_slice(),
+            Some(&ObjectRef::of(&vocab)),
+            &Fault::default(),
+        )
+        .unwrap()
+        .obj;
+    let mut d = super::tests::created_declaration();
+    d.objects = vec![("tokenizer/vocab.json".into(), object.clone())];
+    let encoded = d.canonical_work_bytes().unwrap();
+    assert_eq!(
+        Declaration::parse_text(std::str::from_utf8(&encoded).unwrap()).unwrap(),
+        d
+    );
+    let writer = begin(&store, &meta, &id, 1, d.clone(), None).unwrap();
+    write_weight(&store, &meta, &id);
+    let receipt = commit(&store, &meta, &id, 1).unwrap();
+    release(&meta, writer);
+    assert!(!receipt.added_objects.contains(&object));
+    let header = checkpoint::load_header(&store, &receipt.header).unwrap();
+    let (name, asset) = &header.assets[0];
+    assert_eq!(
+        (name.as_str(), &asset.segments),
+        ("tokenizer/vocab.json", &vec![object])
+    );
+    let mut read = Vec::new();
+    checkpoint::read_asset(&store, name, asset, &mut read).unwrap();
+    assert_eq!(read, vocab);
+
+    // An object this Store does not hold is refused at commit, never published.
+    let next = format!("sha256:{}", "c3".repeat(32));
+    d.objects = vec![("tokenizer/vocab.json".into(), ObjectRef::of(b"absent"))];
+    let writer = begin(&store, &meta, &next, 1, d, None).unwrap();
+    write_weight(&store, &meta, &next);
+    assert!(commit(&store, &meta, &next, 1).is_err());
+    release(&meta, writer);
+    std::fs::remove_dir_all(store.root()).unwrap();
+}
+
 #[test]
 fn companion_paths_size_count_and_quotas_are_native_refusals() {
     for path in [

tensorfs-0.3.71/crates/tensorfs-core/src/derived/group_commit_tests.rs

@@ -0,0 +1,114 @@
+//! A live writer on a persistent disk syncs at its checkpoints, not per part, and still
+//! resumes from its last checkpoint after the machine loses its page cache.
+
+use super::*;
+use crate::disk::DiskClass;
+use crate::unsynced::testing::{from_an_earlier_boot, markers, persistent_root, tear};
+
+const TENSORS: usize = 6;
+
+fn declaration() -> Declaration {
+    let mut declaration = super::tests::created_declaration();
+    let prototype = declaration.components[0].add[0].clone();
+    declaration.components[0].add = (0..TENSORS)
+        .map(|index| {
+            let mut tensor = prototype.clone();
+            tensor.key = format!("weight_{index}");
+            tensor.shape = vec![128];
+            tensor.parts[0].shape = vec![128];
+            tensor
+        })
+        .collect();
+    declaration.order = (0..TENSORS)
+        .map(|index| ("model".into(), format!("weight_{index}")))
+        .collect();
+    declaration.max_new_bytes = (TENSORS * 512) as u64;
+    declaration
+}
+
+fn store(name: &str) -> (Store, Meta) {
+    let store = Store::init(&persistent_root(name)).unwrap();
+    assert_eq!(store.disk_class(), DiskClass::Persistent);
+    let meta = Meta::open(&store).unwrap();
+    (store, meta)
+}
+
+fn add(store: &Store, meta: &Meta, id: &str, session: u64, writer: &Begin, index: usize) -> Part {
+    add_indexed_part(
+        store,
+        meta,
+        id,
+        session,
+        &writer.additions,
+        None,
+        ("model", &format!("weight_{index}"), "value"),
+        &mut [index as u8 + 1; 512].as_slice(),
+    )
+    .unwrap()
+    .part
+}
+
+#[test]
+fn a_power_loss_after_a_checkpoint_resumes_from_it_to_the_same_manifest() {
+    let id = format!("sha256:{}", "6c".repeat(32));
+
+    // The reference: one uninterrupted writer.
+    let (clean, clean_meta) = store("group-commit-clean");
+    let writer = begin(&clean, &clean_meta, &id, 1, declaration(), None).unwrap();
+    let live = writer_store(&clean).unwrap();
+    for index in 0..TENSORS {
+        add(&live, &clean_meta, &id, 1, &writer, index);
+    }
+    let expected = commit(&live, &clean_meta, &id, 1).unwrap();
+    finish_writer(&live).unwrap();
+    release_guards(&clean_meta, Some(writer.writer_hold), writer.source_leases);
+    assert!(
+        markers(clean.root()).is_empty(),
+        "a committed writer retires its epoch"
+    );
+
+    // A checkpoint after three parts; three more admitted unsynced; then the machine goes
+    // down and the page cache holding those three is lost.
+    let (crashed, meta) = store("group-commit-crash");
+    let root = crashed.root().to_path_buf();
+    let writer = begin(&crashed, &meta, &id, 1, declaration(), None).unwrap();
+    let live = writer_store(&crashed).unwrap();
+    assert_eq!(
+        markers(&root).len(),
+        1,
+        "a persistent writer admits under one epoch"
+    );
+    for index in 0..3 {
+        add(&live, &meta, &id, 1, &writer, index);
+    }
+    let (_, head) = checkpoint_progress(&live, &meta, &id, 1, "request", "model", None).unwrap();
+    let lost: Vec<Part> = (3..TENSORS)
+        .map(|index| add(&live, &meta, &id, 1, &writer, index))
+        .collect();
+    fence(&meta, &id, 1).unwrap();
+    release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+    drop(live);
+    from_an_earlier_boot(&markers(&root).pop().unwrap());
+    for segment in lost.iter().flat_map(Part::segments) {
+        tear(&crashed.blob_path(&segment.sha256));
+    }
+
+    // Opening repairs the torn objects; the resumed writer keeps what its checkpoint made
+    // durable and recomputes the rest.
+    let reopened = Store::open(&root).unwrap();
+    let meta = Meta::open(&reopened).unwrap();
+    let head = head.head.expect("the checkpoint names a head");
+    let writer = begin(&reopened, &meta, &id, 2, declaration(), Some(&head)).unwrap();
+    assert_eq!(completed(&meta, &id, 2).unwrap().0.len(), 3);
+    let live = writer_store(&reopened).unwrap();
+    for index in 3..TENSORS {
+        add(&live, &meta, &id, 2, &writer, index);
+    }
+    let resumed = commit(&live, &meta, &id, 2).unwrap();
+    finish_writer(&live).unwrap();
+    release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+    assert_eq!(resumed.manifest, expected.manifest);
+    assert!(markers(&root).is_empty());
+    std::fs::remove_dir_all(clean.root()).unwrap();
+    std::fs::remove_dir_all(&root).unwrap();
+}