tensorfs 0.3.64__tar.gz → 0.3.66__tar.gz

{tensorfs-0.3.64 → tensorfs-0.3.66}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.64"
+version = "0.3.66"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.64"
+version = "0.3.66"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.64 → tensorfs-0.3.66}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.64"
+version = "0.3.66"
 edition = "2021"

{tensorfs-0.3.64 → tensorfs-0.3.66}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.64
+Version: 0.3.66
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.64 → tensorfs-0.3.66}/crates/tensorfs-core/src/bin/tfs/fetch.rs

@@ -1358,6 +1358,7 @@ struct PullProgress {
 struct PullProgressState {
     total_bytes: u64,
     held_bytes: u64,
+    wanted_objects: u64,
     origin_bytes: u64,
     cache_bytes: u64,
     origin_objects: u64,
@@ -1371,6 +1372,7 @@ impl PullProgress {
             state: Mutex::new(PullProgressState {
                 total_bytes: 0,
                 held_bytes: 0,
+                wanted_objects: 0,
                 origin_bytes: 0,
                 cache_bytes: 0,
                 origin_objects: 0,
@@ -1385,6 +1387,7 @@ impl PullProgress {
         let mut state = self.state.lock().unwrap();
         state.total_bytes = plan.declared_bytes();
         state.held_bytes = plan.held_bytes();
+        state.wanted_objects = plan.wanted.len() as u64;
         state.last = Some(Instant::now());
         print_progress(&state);
     }
@@ -1447,11 +1450,12 @@ impl PullProgress {
 ///
 /// Every counter is always present, including a zero. A consumer that reads an absent key
 /// as zero cannot tell a cache that answered nothing from a build that never reported the
-/// door at all, and those are the two states worth telling apart.
+/// door at all, and those are the two states worth telling apart. `wanted_objects` is the
+/// plan's count of objects to land; a pod host sizes this pull's storage reservation from it.
 fn print_progress(state: &PullProgressState) {
     eprintln!(
-        "{{\"event\":\"pull.progress\",\"total_bytes\":{},\"held_bytes\":{},\"origin_bytes\":{},\"cache_bytes\":{},\"origin_objects\":{},\"cache_objects\":{}}}",
-        state.total_bytes, state.held_bytes, state.origin_bytes, state.cache_bytes,
+        "{{\"event\":\"pull.progress\",\"total_bytes\":{},\"held_bytes\":{},\"wanted_objects\":{},\"origin_bytes\":{},\"cache_bytes\":{},\"origin_objects\":{},\"cache_objects\":{}}}",
+        state.total_bytes, state.held_bytes, state.wanted_objects, state.origin_bytes, state.cache_bytes,
         state.origin_objects, state.cache_objects
     );
 }

{tensorfs-0.3.64 → tensorfs-0.3.66}/crates/tensorfs-core/src/catalog.rs

@@ -62,6 +62,11 @@ fn sql(what: impl AsRef<str>, error: rusqlite::Error) -> Refusal {
     }
 }
 
+/// Whether a catalog refusal is SQLite's busy timeout expiring behind another writer.
+fn busy(refusal: &Refusal) -> bool {
+    refusal.code == Code::IO_FAILED && refusal.detail.ends_with("database is locked")
+}
+
 fn io(what: impl AsRef<str>, error: std::io::Error) -> Refusal {
     crate::store::classify_io(what, error)
 }
@@ -115,7 +120,20 @@ fn connect(path: &Path) -> Result<Connection> {
         .entry(path.to_path_buf())
         .or_default() += 1;
     crate::stats::catalog_opened();
-    let connection = Connection::open(path).map_err(|error| sql("open catalog", error))?;
+    let connection = Connection::open(path).map_err(|error| {
+        // SQLite reports a full descriptor table only as "unable to open"; the table
+        // itself answers whether that is what happened.
+        if crate::descriptors::available().is_some_and(|free| free == 0) {
+            crate::descriptors::exhausted(&std::io::Error::from_raw_os_error(
+                rustix::io::Errno::MFILE.raw_os_error(),
+            ));
+            return Refusal {
+                code: Code::FD_HEADROOM,
+                detail: format!("open catalog: no file descriptor ({error})"),
+            };
+        }
+        sql("open catalog", error)
+    })?;
     connection
         .busy_timeout(BUSY_TIMEOUT)
         .map_err(|error| sql("set catalog busy timeout", error))?;
@@ -372,6 +390,22 @@ impl Catalog {
         Ok(catalog)
     }
 
+    /// `open` for an object admission. A catalog busy past SQLite's timeout while this
+    /// process's own admission batches keep committing is a queue, not a failure: the
+    /// verified bytes wait for their turn instead of being thrown away and fetched again.
+    /// Busy with no batch committed in the meantime is another writer's lock, and refuses.
+    pub(crate) fn open_admitting(root: &Path) -> Result<Self> {
+        loop {
+            let committed = crate::stats::snapshot().admission_batches;
+            match Self::open(root) {
+                Err(refusal)
+                    if busy(&refusal) && crate::stats::snapshot().admission_batches > committed => {
+                }
+                opened => return opened,
+            }
+        }
+    }
+
     pub fn open(root: &Path) -> Result<Self> {
         let path = Self::path(root);
         validated(&path)?;
@@ -1473,16 +1507,7 @@ impl WriterGuard {
             .read(true)
             .write(true)
             .open(&path)
-            .map_err(|error| {
-                if crate::descriptors::exhausted(&error) {
-                    Refusal {
-                        code: Code::FD_HEADROOM,
-                        detail: format!("create writer marker: {error}"),
-                    }
-                } else {
-                    io("create writer marker", error)
-                }
-            })?;
+            .map_err(|error| io("create writer marker", error))?;
         file.try_lock_exclusive()
             .map_err(|error| io("lock writer marker", error))?;
         Ok(Self {

{tensorfs-0.3.64 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived/part_rows_tests.rs

@@ -44,6 +44,7 @@ fn add(store: &Store, meta: &Meta, id: &str, writer: &Begin, key: &str, byte: u8
         id,
         1,
         &writer.additions,
+        None,
         ("model", key, "value"),
         &mut [byte; 512].as_slice(),
     )
@@ -112,6 +113,7 @@ fn parts_are_rows_that_resume_refuse_conflicts_and_stay_retained() {
         &id,
         2,
         &resumed.additions,
+        None,
         ("model", "weight_2", "value"),
         &mut [3u8; 512].as_slice(),
     )
@@ -167,3 +169,158 @@ fn an_older_builds_inline_row_and_a_torn_journal_line_are_read() {
     assert!(!journal.exists());
     std::fs::remove_dir_all(store.root()).unwrap();
 }
+
+#[test]
+fn indexed_checkpoints_write_the_chain_full_checkpoints_write() {
+    const COUNT: usize = 40;
+    let mut stores = Vec::new();
+    for label in ["full", "indexed"] {
+        let (store, meta, id, mut declaration) = fixture(label);
+        let prototype = declaration.components[0].add[0].clone();
+        declaration.components[0].add = (0..COUNT)
+            .map(|index| {
+                let mut tensor = prototype.clone();
+                tensor.key = format!("weight_{index:03}");
+                tensor
+            })
+            .collect();
+        declaration.order = (0..COUNT)
+            .map(|index| ("model".into(), format!("weight_{index:03}")))
+            .collect();
+        declaration.configs = vec![ConfigDeclaration::Add {
+            target: "model.json".into(),
+        }];
+        declaration.max_new_bytes = (COUNT * 512 + 64) as u64;
+        let writer = begin(&store, &meta, &id, 1, declaration.clone(), None).unwrap();
+        stores.push((store, meta, id, declaration, writer));
+    }
+    let mut cursor = None;
+    let (mut full_prior, mut indexed_prior) = (None, None);
+    let mut heads: Vec<ObjectRef> = Vec::new();
+    for index in 0..COUNT {
+        let key = format!("weight_{index:03}");
+        let payload = [index as u8; 512];
+        let (store, meta, id, _, writer) = &stores[0];
+        add_indexed_part(
+            store,
+            meta,
+            id,
+            1,
+            &writer.additions,
+            None,
+            ("model", &key, "value"),
+            &mut payload.as_slice(),
+        )
+        .unwrap();
+        let full = checkpoint_progress(store, meta, id, 1, "request", "model", full_prior.as_ref())
+            .unwrap();
+        let (store, meta, id, _, writer) = &stores[1];
+        add_indexed_part(
+            store,
+            meta,
+            id,
+            1,
+            &writer.additions,
+            cursor.as_mut(),
+            ("model", &key, "value"),
+            &mut payload.as_slice(),
+        )
+        .unwrap();
+        // An older acknowledged head sends the writer down the full path, which re-seeds it.
+        let previous = if index == 25 {
+            heads.get(heads.len() - 3).cloned()
+        } else {
+            indexed_prior.clone()
+        };
+        let indexed = checkpoint_indexed(
+            store,
+            meta,
+            id,
+            1,
+            "request",
+            "model",
+            previous.as_ref(),
+            &mut cursor,
+        )
+        .unwrap();
+        assert_eq!(full, indexed, "checkpoint {index} diverged");
+        assert_eq!(cursor.as_ref().unwrap().tip.head, indexed.1);
+        full_prior = full.1.head;
+        indexed_prior = indexed.1.head;
+        heads.push(indexed_prior.clone().unwrap());
+        if index == 20 {
+            for (store, meta, id, _, _) in &stores {
+                add_config(meta, id, 1, "model.json", &mut b"{}".as_slice()).unwrap();
+                let _ = store;
+            }
+            cursor = None;
+        }
+    }
+    let (store, meta, id, declaration, writer) = stores.pop().unwrap();
+    let head = indexed_prior.unwrap();
+    let canonical = String::from_utf8(declaration.clone().canonical_work_bytes().unwrap()).unwrap();
+    let restored = read_progress(&store, &head, &id, &canonical, &[]).unwrap();
+    assert_eq!(restored.added_parts.len(), COUNT);
+    assert_eq!(restored.added_configs.len(), 1);
+    fence(&meta, &id, 1).unwrap();
+    release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+    let resumed = begin(&store, &meta, &id, 2, declaration, Some(&head)).unwrap();
+    assert_eq!(completed(&meta, &id, 2).unwrap().0.len(), COUNT);
+    fence(&meta, &id, 2).unwrap();
+    release_guards(&meta, Some(resumed.writer_hold), resumed.source_leases);
+    std::fs::remove_dir_all(store.root()).unwrap();
+    for (store, meta, id, _, writer) in stores {
+        fence(&meta, &id, 1).unwrap();
+        release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+        std::fs::remove_dir_all(store.root()).unwrap();
+    }
+}
+
+#[test]
+fn an_indexed_checkpoint_does_not_reread_its_chain() {
+    let (store, meta, id, declaration) = fixture("no-reread");
+    let writer = begin(&store, &meta, &id, 1, declaration, None).unwrap();
+    let mut cursor = None;
+    let mut prior: Option<ObjectRef> = None;
+    let mut first = None;
+    for (index, key) in ["weight_0", "weight_1", "weight_2"].into_iter().enumerate() {
+        add_indexed_part(
+            &store,
+            &meta,
+            &id,
+            1,
+            &writer.additions,
+            cursor.as_mut(),
+            ("model", key, "value"),
+            &mut [index as u8 + 1; 512].as_slice(),
+        )
+        .unwrap();
+        let (_, head) = checkpoint_indexed(
+            &store,
+            &meta,
+            &id,
+            1,
+            "request",
+            "model",
+            prior.as_ref(),
+            &mut cursor,
+        )
+        .unwrap();
+        prior = head.head;
+        first.get_or_insert(prior.clone().unwrap());
+        if index == 0 {
+            // The chain's root link is gone: a checkpoint that walked the chain now fails.
+            let root = first.clone().unwrap();
+            std::fs::remove_file(store.object_path(root.sha256.trim_start_matches("sha256:")))
+                .unwrap();
+            assert!(
+                checkpoint_progress(&store, &meta, &id, 1, "request", "model", prior.as_ref())
+                    .is_err()
+            );
+        }
+    }
+    assert_eq!(cursor.as_ref().unwrap().parts, 3);
+    fence(&meta, &id, 1).unwrap();
+    release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+    std::fs::remove_dir_all(store.root()).unwrap();
+}

{tensorfs-0.3.64 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived/progress.rs

@@ -21,6 +21,16 @@ fn part_key(part: &AddedPart) -> (&str, &str, &str) {
 }
 
 impl Delta {
+    /// The parts a writer accepted since its last checkpoint, sorted as a delta stores them.
+    pub(super) fn accepted(work: ObjectRef, mut parts: Vec<AddedPart>) -> Self {
+        parts.sort_by(|left, right| part_key(left).cmp(&part_key(right)));
+        Self {
+            work,
+            parts,
+            configs: Vec::new(),
+        }
+    }
+
     pub(super) fn between(
         row: &TransactionRow,
         prior: Option<&TransactionRow>,

{tensorfs-0.3.64 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived/resume_tests.rs

@@ -981,6 +981,7 @@ fn profile_derived_transaction_scaling() {
         let mut part_ms = 0.0;
         let mut checkpoint_ms = 0.0;
         let mut prior = None;
+        let mut cursor = None;
         for index in 0..writes {
             let key = format!("weight_{index:04}");
             let started = Instant::now();
@@ -990,15 +991,24 @@ fn profile_derived_transaction_scaling() {
                 &id,
                 1,
                 &writer.additions,
+                cursor.as_mut(),
                 ("model", &key, "value"),
                 &mut [index as u8; 16].as_slice(),
             )
             .unwrap();
             part_ms += started.elapsed().as_secs_f64() * 1000.0;
             let started = Instant::now();
-            let (_, head) =
-                checkpoint_progress(&store, &meta, &id, 1, "profile", "model", prior.as_ref())
-                    .unwrap();
+            let (_, head) = checkpoint_indexed(
+                &store,
+                &meta,
+                &id,
+                1,
+                "profile",
+                "model",
+                prior.as_ref(),
+                &mut cursor,
+            )
+            .unwrap();
             prior = head.head;
             checkpoint_ms += started.elapsed().as_secs_f64() * 1000.0;
         }

{tensorfs-0.3.64 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived.rs

@@ -2711,6 +2711,158 @@ pub fn completed(
     ))
 }
 
+/// A writer's position in its checkpoint chain: the chain's tip and held blobs, its fixed
+/// plan and work context, and the parts it accepted since. With it a checkpoint writes and
+/// verifies only those parts instead of re-reading the chain it extends.
+#[derive(Debug, Clone)]
+pub struct CheckpointCursor {
+    chain: String,
+    plan: String,
+    work: ObjectRef,
+    tip: crate::durability::Tip,
+    parts: usize,
+    pending: Vec<AddedPart>,
+}
+
+impl CheckpointCursor {
+    /// Seed from a checkpoint the full path just wrote or confirmed. `None` when there is
+    /// no head yet to extend.
+    fn at(
+        store: &Store,
+        meta: &Meta,
+        transaction: &str,
+        writer_session: u64,
+        chain: String,
+        plan: &str,
+        head: &crate::durability::Head,
+    ) -> Result<Option<Self>> {
+        let Some(tip) = head.head.as_ref() else {
+            return Ok(None);
+        };
+        let mut rows: Vec<_> = meta.derived_row(transaction)?.into_iter().collect();
+        let row = open_row(&mut rows, transaction, writer_session)?;
+        let context = progress::context(row);
+        let held = crate::durability::local_chain(store, tip, plan)?
+            .blobs
+            .into_iter()
+            .map(|object| object.sha256)
+            .collect();
+        Ok(Some(Self {
+            chain,
+            plan: plan.to_string(),
+            work: ObjectRef::of(&crate::canon::write(&context.to_value())),
+            tip: crate::durability::Tip {
+                head: head.clone(),
+                held,
+            },
+            parts: row.added_parts.len(),
+            pending: Vec::new(),
+        }))
+    }
+
+    /// Extend the chain by the pending parts, or `None` when anything but this writer's own
+    /// accepted parts moved since the cursor was taken.
+    fn advance(
+        &mut self,
+        store: &Store,
+        meta: &Meta,
+        transaction: &str,
+        writer_session: u64,
+        chain: &str,
+        previous: Option<&ObjectRef>,
+    ) -> Result<Option<(String, crate::durability::Head)>> {
+        let tip = self.tip.head.head.clone();
+        let root = roots::read(store, transaction)?;
+        if self.chain != chain
+            || previous.is_some_and(|previous| Some(previous) != tip.as_ref())
+            || root.as_ref().and_then(|root| root.checkpoint.as_ref()) != tip.as_ref()
+        {
+            return Ok(None);
+        }
+        meta.check_session(transaction, writer_session)?;
+        if meta.part_count(transaction)? != self.parts + self.pending.len() {
+            return Ok(None);
+        }
+        if self.pending.is_empty() {
+            return Ok(Some((self.plan.clone(), self.tip.head.clone())));
+        }
+        let parts = std::mem::take(&mut self.pending);
+        let objects = parts
+            .iter()
+            .flat_map(|added| added.part.segments().iter().cloned())
+            .collect();
+        let count = parts.len();
+        let delta = progress::Delta::accepted(self.work.clone(), parts);
+        let policy = crate::durability::Policy {
+            chain,
+            operation: transaction,
+            plan: &self.plan,
+            interval: crate::durability::INTERVAL_BYTES,
+        };
+        let extended = crate::durability::extend(
+            store,
+            &policy,
+            &self.tip,
+            &crate::canon::write(&delta.to_value()),
+            objects,
+            None,
+        )?;
+        let Some(mut root) = root else {
+            return Ok(None);
+        };
+        root.checkpoint = extended.head.head.clone();
+        roots::write(store, &root)?;
+        self.tip
+            .held
+            .extend(extended.named.into_iter().map(|object| object.sha256));
+        self.tip.head = extended.head;
+        self.parts += count;
+        Ok(Some((self.plan.clone(), self.tip.head.clone())))
+    }
+}
+
+/// Checkpoint through the writer's cursor, which each full checkpoint re-seeds.
+#[allow(clippy::too_many_arguments)]
+pub fn checkpoint_indexed(
+    store: &Store,
+    meta: &Meta,
+    transaction: &str,
+    writer_session: u64,
+    operation: &str,
+    slot: &str,
+    previous: Option<&ObjectRef>,
+    cursor: &mut Option<CheckpointCursor>,
+) -> Result<(String, crate::durability::Head)> {
+    let chain = format!("{operation}/{slot}");
+    if let Some(current) = cursor.as_mut() {
+        if let Some(advanced) =
+            current.advance(store, meta, transaction, writer_session, &chain, previous)?
+        {
+            return Ok(advanced);
+        }
+    }
+    *cursor = None;
+    let (plan, head) = checkpoint_progress(
+        store,
+        meta,
+        transaction,
+        writer_session,
+        operation,
+        slot,
+        previous,
+    )?;
+    *cursor = CheckpointCursor::at(
+        store,
+        meta,
+        transaction,
+        writer_session,
+        chain,
+        &plan,
+        &head,
+    )?;
+    Ok((plan, head))
+}
+
 pub fn checkpoint_progress(
     store: &Store,
     meta: &Meta,
@@ -3115,6 +3267,7 @@ pub fn add_part<R: Read>(
         transaction,
         writer_session,
         &additions,
+        None,
         value,
         reader,
     )
@@ -3122,12 +3275,14 @@ pub fn add_part<R: Read>(
 
 /// Accept one part. The cost is this part's bytes plus one indexed SQL row: the
 /// declaration is not re-read and earlier accepted parts are neither read nor rewritten.
+#[allow(clippy::too_many_arguments)]
 pub fn add_indexed_part<R: Read>(
     store: &Store,
     meta: &Meta,
     transaction: &str,
     writer_session: u64,
     additions: &Additions,
+    cursor: Option<&mut CheckpointCursor>,
     value: (&str, &str, &str),
     reader: &mut R,
 ) -> Result<crate::checkpoint::Written> {
@@ -3168,16 +3323,17 @@ pub fn add_indexed_part<R: Read>(
     // The private root names the objects before SQL accepts the part, so GC never
     // observes an accepted part whose bytes it may reclaim.
     roots::append_parts(store, &transaction, written.part.segments())?;
-    meta.accept_part(
-        &transaction,
-        writer_session,
-        &AddedPart {
-            component: component.to_string(),
-            key: tensor_key.to_string(),
-            role: role.to_string(),
-            part: written.part.clone(),
-        },
-    )?;
+    let added = AddedPart {
+        component: component.to_string(),
+        key: tensor_key.to_string(),
+        role: role.to_string(),
+        part: written.part.clone(),
+    };
+    if meta.accept_part(&transaction, writer_session, &added)? {
+        if let Some(cursor) = cursor {
+            cursor.pending.push(added);
+        }
+    }
     Ok(written)
 }