tensorfs 0.3.63__tar.gz → 0.3.65__tar.gz

{tensorfs-0.3.63 → tensorfs-0.3.65}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.63"
+version = "0.3.65"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.63"
+version = "0.3.65"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.63 → tensorfs-0.3.65}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.63"
+version = "0.3.65"
 edition = "2021"

{tensorfs-0.3.63 → tensorfs-0.3.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.63
+Version: 0.3.65
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.63 → tensorfs-0.3.65}/crates/tensorfs-core/src/bin/tfs/fetch.rs

@@ -182,6 +182,7 @@ fn policy_of(flags: &Flags) -> SourcePolicy {
             .unwrap_or_default(),
         allow_local: crate::flag_on(flags, "allow-local"),
         max_redirects: crate::flag_num(flags, "max-redirects", 0u32),
+        ..Default::default()
     }
 }
 
@@ -334,12 +335,13 @@ fn pull_with_budget(
     // And what admitting them cost: a pull whose streams stop growing while its batches
     // take most of the wall time is held by the disk, not the link.
     eprintln!(
-        "{{\"event\":\"pull.streams\",\"model\":{},\"seconds\":{:.3},\"bytes_moved\":{},\"streams\":{},\"levels\":[{}],\"admission\":{{\"objects\":{},\"batches\":{},\"seconds\":{:.3}}}}}",
+        "{{\"event\":\"pull.streams\",\"model\":{},\"seconds\":{:.3},\"bytes_moved\":{},\"streams\":{},\"levels\":[{}],\"backoffs\":{},\"admission\":{{\"objects\":{},\"batches\":{},\"seconds\":{:.3}}}}}",
         quoted(&report.model),
         seconds,
         report.bytes_moved,
         report.streams,
         levels.join(","),
+        report.backoffs,
         after.admitted_objects - before.admitted_objects,
         after.admission_batches - before.admission_batches,
         (after.admission_nanos - before.admission_nanos) as f64 / 1e9,

{tensorfs-0.3.63 → tensorfs-0.3.65}/crates/tensorfs-core/src/catalog.rs

@@ -1473,7 +1473,16 @@ impl WriterGuard {
             .read(true)
             .write(true)
             .open(&path)
-            .map_err(|error| io("create writer marker", error))?;
+            .map_err(|error| {
+                if crate::descriptors::exhausted(&error) {
+                    Refusal {
+                        code: Code::FD_HEADROOM,
+                        detail: format!("create writer marker: {error}"),
+                    }
+                } else {
+                    io("create writer marker", error)
+                }
+            })?;
         file.try_lock_exclusive()
             .map_err(|error| io("lock writer marker", error))?;
         Ok(Self {

{tensorfs-0.3.63 → tensorfs-0.3.65}/crates/tensorfs-core/src/derived/part_rows_tests.rs

@@ -44,6 +44,7 @@ fn add(store: &Store, meta: &Meta, id: &str, writer: &Begin, key: &str, byte: u8
         id,
         1,
         &writer.additions,
+        None,
         ("model", key, "value"),
         &mut [byte; 512].as_slice(),
     )
@@ -112,6 +113,7 @@ fn parts_are_rows_that_resume_refuse_conflicts_and_stay_retained() {
         &id,
         2,
         &resumed.additions,
+        None,
         ("model", "weight_2", "value"),
         &mut [3u8; 512].as_slice(),
     )
@@ -167,3 +169,158 @@ fn an_older_builds_inline_row_and_a_torn_journal_line_are_read() {
     assert!(!journal.exists());
     std::fs::remove_dir_all(store.root()).unwrap();
 }
+
+#[test]
+fn indexed_checkpoints_write_the_chain_full_checkpoints_write() {
+    const COUNT: usize = 40;
+    let mut stores = Vec::new();
+    for label in ["full", "indexed"] {
+        let (store, meta, id, mut declaration) = fixture(label);
+        let prototype = declaration.components[0].add[0].clone();
+        declaration.components[0].add = (0..COUNT)
+            .map(|index| {
+                let mut tensor = prototype.clone();
+                tensor.key = format!("weight_{index:03}");
+                tensor
+            })
+            .collect();
+        declaration.order = (0..COUNT)
+            .map(|index| ("model".into(), format!("weight_{index:03}")))
+            .collect();
+        declaration.configs = vec![ConfigDeclaration::Add {
+            target: "model.json".into(),
+        }];
+        declaration.max_new_bytes = (COUNT * 512 + 64) as u64;
+        let writer = begin(&store, &meta, &id, 1, declaration.clone(), None).unwrap();
+        stores.push((store, meta, id, declaration, writer));
+    }
+    let mut cursor = None;
+    let (mut full_prior, mut indexed_prior) = (None, None);
+    let mut heads: Vec<ObjectRef> = Vec::new();
+    for index in 0..COUNT {
+        let key = format!("weight_{index:03}");
+        let payload = [index as u8; 512];
+        let (store, meta, id, _, writer) = &stores[0];
+        add_indexed_part(
+            store,
+            meta,
+            id,
+            1,
+            &writer.additions,
+            None,
+            ("model", &key, "value"),
+            &mut payload.as_slice(),
+        )
+        .unwrap();
+        let full = checkpoint_progress(store, meta, id, 1, "request", "model", full_prior.as_ref())
+            .unwrap();
+        let (store, meta, id, _, writer) = &stores[1];
+        add_indexed_part(
+            store,
+            meta,
+            id,
+            1,
+            &writer.additions,
+            cursor.as_mut(),
+            ("model", &key, "value"),
+            &mut payload.as_slice(),
+        )
+        .unwrap();
+        // An older acknowledged head sends the writer down the full path, which re-seeds it.
+        let previous = if index == 25 {
+            heads.get(heads.len() - 3).cloned()
+        } else {
+            indexed_prior.clone()
+        };
+        let indexed = checkpoint_indexed(
+            store,
+            meta,
+            id,
+            1,
+            "request",
+            "model",
+            previous.as_ref(),
+            &mut cursor,
+        )
+        .unwrap();
+        assert_eq!(full, indexed, "checkpoint {index} diverged");
+        assert_eq!(cursor.as_ref().unwrap().tip.head, indexed.1);
+        full_prior = full.1.head;
+        indexed_prior = indexed.1.head;
+        heads.push(indexed_prior.clone().unwrap());
+        if index == 20 {
+            for (store, meta, id, _, _) in &stores {
+                add_config(meta, id, 1, "model.json", &mut b"{}".as_slice()).unwrap();
+                let _ = store;
+            }
+            cursor = None;
+        }
+    }
+    let (store, meta, id, declaration, writer) = stores.pop().unwrap();
+    let head = indexed_prior.unwrap();
+    let canonical = String::from_utf8(declaration.clone().canonical_work_bytes().unwrap()).unwrap();
+    let restored = read_progress(&store, &head, &id, &canonical, &[]).unwrap();
+    assert_eq!(restored.added_parts.len(), COUNT);
+    assert_eq!(restored.added_configs.len(), 1);
+    fence(&meta, &id, 1).unwrap();
+    release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+    let resumed = begin(&store, &meta, &id, 2, declaration, Some(&head)).unwrap();
+    assert_eq!(completed(&meta, &id, 2).unwrap().0.len(), COUNT);
+    fence(&meta, &id, 2).unwrap();
+    release_guards(&meta, Some(resumed.writer_hold), resumed.source_leases);
+    std::fs::remove_dir_all(store.root()).unwrap();
+    for (store, meta, id, _, writer) in stores {
+        fence(&meta, &id, 1).unwrap();
+        release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+        std::fs::remove_dir_all(store.root()).unwrap();
+    }
+}
+
+#[test]
+fn an_indexed_checkpoint_does_not_reread_its_chain() {
+    let (store, meta, id, declaration) = fixture("no-reread");
+    let writer = begin(&store, &meta, &id, 1, declaration, None).unwrap();
+    let mut cursor = None;
+    let mut prior: Option<ObjectRef> = None;
+    let mut first = None;
+    for (index, key) in ["weight_0", "weight_1", "weight_2"].into_iter().enumerate() {
+        add_indexed_part(
+            &store,
+            &meta,
+            &id,
+            1,
+            &writer.additions,
+            cursor.as_mut(),
+            ("model", key, "value"),
+            &mut [index as u8 + 1; 512].as_slice(),
+        )
+        .unwrap();
+        let (_, head) = checkpoint_indexed(
+            &store,
+            &meta,
+            &id,
+            1,
+            "request",
+            "model",
+            prior.as_ref(),
+            &mut cursor,
+        )
+        .unwrap();
+        prior = head.head;
+        first.get_or_insert(prior.clone().unwrap());
+        if index == 0 {
+            // The chain's root link is gone: a checkpoint that walked the chain now fails.
+            let root = first.clone().unwrap();
+            std::fs::remove_file(store.object_path(root.sha256.trim_start_matches("sha256:")))
+                .unwrap();
+            assert!(
+                checkpoint_progress(&store, &meta, &id, 1, "request", "model", prior.as_ref())
+                    .is_err()
+            );
+        }
+    }
+    assert_eq!(cursor.as_ref().unwrap().parts, 3);
+    fence(&meta, &id, 1).unwrap();
+    release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+    std::fs::remove_dir_all(store.root()).unwrap();
+}

{tensorfs-0.3.63 → tensorfs-0.3.65}/crates/tensorfs-core/src/derived/progress.rs

@@ -21,6 +21,16 @@ fn part_key(part: &AddedPart) -> (&str, &str, &str) {
 }
 
 impl Delta {
+    /// The parts a writer accepted since its last checkpoint, sorted as a delta stores them.
+    pub(super) fn accepted(work: ObjectRef, mut parts: Vec<AddedPart>) -> Self {
+        parts.sort_by(|left, right| part_key(left).cmp(&part_key(right)));
+        Self {
+            work,
+            parts,
+            configs: Vec::new(),
+        }
+    }
+
     pub(super) fn between(
         row: &TransactionRow,
         prior: Option<&TransactionRow>,

{tensorfs-0.3.63 → tensorfs-0.3.65}/crates/tensorfs-core/src/derived/resume_tests.rs

@@ -981,6 +981,7 @@ fn profile_derived_transaction_scaling() {
         let mut part_ms = 0.0;
         let mut checkpoint_ms = 0.0;
         let mut prior = None;
+        let mut cursor = None;
         for index in 0..writes {
             let key = format!("weight_{index:04}");
             let started = Instant::now();
@@ -990,15 +991,24 @@ fn profile_derived_transaction_scaling() {
                 &id,
                 1,
                 &writer.additions,
+                cursor.as_mut(),
                 ("model", &key, "value"),
                 &mut [index as u8; 16].as_slice(),
             )
             .unwrap();
             part_ms += started.elapsed().as_secs_f64() * 1000.0;
             let started = Instant::now();
-            let (_, head) =
-                checkpoint_progress(&store, &meta, &id, 1, "profile", "model", prior.as_ref())
-                    .unwrap();
+            let (_, head) = checkpoint_indexed(
+                &store,
+                &meta,
+                &id,
+                1,
+                "profile",
+                "model",
+                prior.as_ref(),
+                &mut cursor,
+            )
+            .unwrap();
             prior = head.head;
             checkpoint_ms += started.elapsed().as_secs_f64() * 1000.0;
         }

{tensorfs-0.3.63 → tensorfs-0.3.65}/crates/tensorfs-core/src/derived.rs

@@ -2711,6 +2711,158 @@ pub fn completed(
     ))
 }
 
+/// A writer's position in its checkpoint chain: the chain's tip and held blobs, its fixed
+/// plan and work context, and the parts it accepted since. With it a checkpoint writes and
+/// verifies only those parts instead of re-reading the chain it extends.
+#[derive(Debug, Clone)]
+pub struct CheckpointCursor {
+    chain: String,
+    plan: String,
+    work: ObjectRef,
+    tip: crate::durability::Tip,
+    parts: usize,
+    pending: Vec<AddedPart>,
+}
+
+impl CheckpointCursor {
+    /// Seed from a checkpoint the full path just wrote or confirmed. `None` when there is
+    /// no head yet to extend.
+    fn at(
+        store: &Store,
+        meta: &Meta,
+        transaction: &str,
+        writer_session: u64,
+        chain: String,
+        plan: &str,
+        head: &crate::durability::Head,
+    ) -> Result<Option<Self>> {
+        let Some(tip) = head.head.as_ref() else {
+            return Ok(None);
+        };
+        let mut rows: Vec<_> = meta.derived_row(transaction)?.into_iter().collect();
+        let row = open_row(&mut rows, transaction, writer_session)?;
+        let context = progress::context(row);
+        let held = crate::durability::local_chain(store, tip, plan)?
+            .blobs
+            .into_iter()
+            .map(|object| object.sha256)
+            .collect();
+        Ok(Some(Self {
+            chain,
+            plan: plan.to_string(),
+            work: ObjectRef::of(&crate::canon::write(&context.to_value())),
+            tip: crate::durability::Tip {
+                head: head.clone(),
+                held,
+            },
+            parts: row.added_parts.len(),
+            pending: Vec::new(),
+        }))
+    }
+
+    /// Extend the chain by the pending parts, or `None` when anything but this writer's own
+    /// accepted parts moved since the cursor was taken.
+    fn advance(
+        &mut self,
+        store: &Store,
+        meta: &Meta,
+        transaction: &str,
+        writer_session: u64,
+        chain: &str,
+        previous: Option<&ObjectRef>,
+    ) -> Result<Option<(String, crate::durability::Head)>> {
+        let tip = self.tip.head.head.clone();
+        let root = roots::read(store, transaction)?;
+        if self.chain != chain
+            || previous.is_some_and(|previous| Some(previous) != tip.as_ref())
+            || root.as_ref().and_then(|root| root.checkpoint.as_ref()) != tip.as_ref()
+        {
+            return Ok(None);
+        }
+        meta.check_session(transaction, writer_session)?;
+        if meta.part_count(transaction)? != self.parts + self.pending.len() {
+            return Ok(None);
+        }
+        if self.pending.is_empty() {
+            return Ok(Some((self.plan.clone(), self.tip.head.clone())));
+        }
+        let parts = std::mem::take(&mut self.pending);
+        let objects = parts
+            .iter()
+            .flat_map(|added| added.part.segments().iter().cloned())
+            .collect();
+        let count = parts.len();
+        let delta = progress::Delta::accepted(self.work.clone(), parts);
+        let policy = crate::durability::Policy {
+            chain,
+            operation: transaction,
+            plan: &self.plan,
+            interval: crate::durability::INTERVAL_BYTES,
+        };
+        let extended = crate::durability::extend(
+            store,
+            &policy,
+            &self.tip,
+            &crate::canon::write(&delta.to_value()),
+            objects,
+            None,
+        )?;
+        let Some(mut root) = root else {
+            return Ok(None);
+        };
+        root.checkpoint = extended.head.head.clone();
+        roots::write(store, &root)?;
+        self.tip
+            .held
+            .extend(extended.named.into_iter().map(|object| object.sha256));
+        self.tip.head = extended.head;
+        self.parts += count;
+        Ok(Some((self.plan.clone(), self.tip.head.clone())))
+    }
+}
+
+/// Checkpoint through the writer's cursor, which each full checkpoint re-seeds.
+#[allow(clippy::too_many_arguments)]
+pub fn checkpoint_indexed(
+    store: &Store,
+    meta: &Meta,
+    transaction: &str,
+    writer_session: u64,
+    operation: &str,
+    slot: &str,
+    previous: Option<&ObjectRef>,
+    cursor: &mut Option<CheckpointCursor>,
+) -> Result<(String, crate::durability::Head)> {
+    let chain = format!("{operation}/{slot}");
+    if let Some(current) = cursor.as_mut() {
+        if let Some(advanced) =
+            current.advance(store, meta, transaction, writer_session, &chain, previous)?
+        {
+            return Ok(advanced);
+        }
+    }
+    *cursor = None;
+    let (plan, head) = checkpoint_progress(
+        store,
+        meta,
+        transaction,
+        writer_session,
+        operation,
+        slot,
+        previous,
+    )?;
+    *cursor = CheckpointCursor::at(
+        store,
+        meta,
+        transaction,
+        writer_session,
+        chain,
+        &plan,
+        &head,
+    )?;
+    Ok((plan, head))
+}
+
 pub fn checkpoint_progress(
     store: &Store,
     meta: &Meta,
@@ -3115,6 +3267,7 @@ pub fn add_part<R: Read>(
         transaction,
         writer_session,
         &additions,
+        None,
         value,
         reader,
     )
@@ -3122,12 +3275,14 @@ pub fn add_part<R: Read>(
 
 /// Accept one part. The cost is this part's bytes plus one indexed SQL row: the
 /// declaration is not re-read and earlier accepted parts are neither read nor rewritten.
+#[allow(clippy::too_many_arguments)]
 pub fn add_indexed_part<R: Read>(
     store: &Store,
     meta: &Meta,
     transaction: &str,
     writer_session: u64,
     additions: &Additions,
+    cursor: Option<&mut CheckpointCursor>,
     value: (&str, &str, &str),
     reader: &mut R,
 ) -> Result<crate::checkpoint::Written> {
@@ -3168,16 +3323,17 @@ pub fn add_indexed_part<R: Read>(
     // The private root names the objects before SQL accepts the part, so GC never
     // observes an accepted part whose bytes it may reclaim.
     roots::append_parts(store, &transaction, written.part.segments())?;
-    meta.accept_part(
-        &transaction,
-        writer_session,
-        &AddedPart {
-            component: component.to_string(),
-            key: tensor_key.to_string(),
-            role: role.to_string(),
-            part: written.part.clone(),
-        },
-    )?;
+    let added = AddedPart {
+        component: component.to_string(),
+        key: tensor_key.to_string(),
+        role: role.to_string(),
+        part: written.part.clone(),
+    };
+    if meta.accept_part(&transaction, writer_session, &added)? {
+        if let Some(cursor) = cursor {
+            cursor.pending.push(added);
+        }
+    }
     Ok(written)
 }
 

tensorfs-0.3.65/crates/tensorfs-core/src/descriptors.rs

@@ -0,0 +1,112 @@
+//! File descriptors as a budget: raised to what the process may have, measured before a
+//! transfer sizes itself, and an exhausted table reported as backpressure.
+//!
+//! Run 1297 (2026-09-27) grew a pull toward 128 streams under a 1,024 soft RLIMIT_NOFILE;
+//! each object in flight holds a socket, a temp file, writer markers and a catalog
+//! connection, and the table ran out mid-pull (`name resolution failed (Too many open
+//! files)`). A Go supervisor raises its own soft limit but hands its children the original.
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Once;
+
+/// Descriptors one object stream may hold at once, and what it is spent on: the object
+/// socket and one pooled idle socket (2); the temp file and the two writer-marker files of
+/// its admission, nested once more by the catalog record (5); the catalog connection and its
+/// journal (2); and the transient DNS socket or contender of a retry (3).
+pub const PER_STREAM: u64 = 12;
+
+/// Descriptors left for everything that is not a stream: stdio, the Store's own locks, the
+/// hub calls, the progress channel and whatever the caller opens next.
+pub const MARGIN: u64 = 64;
+
+static RAISE: Once = Once::new();
+static EXHAUSTED: AtomicU64 = AtomicU64::new(0);
+
+/// Raise the soft RLIMIT_NOFILE to the hard limit, once per process. Idempotent; a refused
+/// raise leaves the limit where it was and the budget below measures that.
+pub fn raise() {
+    RAISE.call_once(|| {
+        use rustix::process::{getrlimit, setrlimit, Resource, Rlimit};
+        let limit = getrlimit(Resource::Nofile);
+        if let (Some(soft), Some(hard)) = (limit.current, limit.maximum) {
+            if soft < hard {
+                let _ = setrlimit(
+                    Resource::Nofile,
+                    Rlimit {
+                        current: Some(hard),
+                        maximum: Some(hard),
+                    },
+                );
+            }
+        }
+    });
+}
+
+/// Descriptors this process can still open now: soft limit less those open and `MARGIN`.
+/// `None` when the platform cannot say (no `/proc`, or no soft limit).
+pub fn available() -> Option<u64> {
+    let soft = rustix::process::getrlimit(rustix::process::Resource::Nofile).current?;
+    let open = std::fs::read_dir("/proc/self/fd").ok()?.count() as u64;
+    Some(soft.saturating_sub(open).saturating_sub(MARGIN))
+}
+
+/// Object streams the descriptor table allows now: at most `wanted`, at least one.
+pub fn streams_within(wanted: usize) -> usize {
+    streams_for(wanted, available())
+}
+
+/// `streams_within` against a stated number of free descriptors.
+pub fn streams_for(wanted: usize, available: Option<u64>) -> usize {
+    let fits = available.map_or(usize::MAX, |free| {
+        usize::try_from(free / PER_STREAM).unwrap_or(usize::MAX)
+    });
+    wanted.min(fits).max(1)
+}
+
+/// Whether `error` is the descriptor table being full, counting it when it is. Call it
+/// where an open, socket or resolution fails, before the error loses its errno.
+pub fn exhausted(error: &std::io::Error) -> bool {
+    let full = matches!(
+        error.raw_os_error(),
+        Some(code) if code == rustix::io::Errno::MFILE.raw_os_error()
+            || code == rustix::io::Errno::NFILE.raw_os_error()
+    );
+    if full {
+        EXHAUSTED.fetch_add(1, Ordering::Relaxed);
+    }
+    full
+}
+
+/// How many times this process has found its descriptor table full. A caller diffs two
+/// reads around its own work.
+pub fn exhaustions() -> u64 {
+    EXHAUSTED.load(Ordering::Relaxed)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_budget_is_descriptors_over_per_stream_cost() {
+        assert_eq!(streams_for(128, Some(PER_STREAM * 10)), 10);
+        assert_eq!(streams_for(8, Some(PER_STREAM * 10)), 8);
+        assert_eq!(streams_for(128, Some(0)), 1, "one stream is always allowed");
+        assert_eq!(
+            streams_for(128, None),
+            128,
+            "an unmeasurable table is not a shortage"
+        );
+    }
+
+    #[test]
+    fn only_a_full_table_counts_as_exhaustion() {
+        let before = exhaustions();
+        assert!(exhausted(&std::io::Error::from_raw_os_error(
+            rustix::io::Errno::MFILE.raw_os_error()
+        )));
+        assert!(!exhausted(&std::io::Error::from_raw_os_error(
+            rustix::io::Errno::CONNREFUSED.raw_os_error()
+        )));
+        assert!(exhaustions() > before);
+    }
+}