tensorfs 0.3.66__tar.gz → 0.3.68__tar.gz

{tensorfs-0.3.66 → tensorfs-0.3.68}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.66"
+version = "0.3.68"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.66"
+version = "0.3.68"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.66 → tensorfs-0.3.68}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.66"
+version = "0.3.68"
 edition = "2021"

{tensorfs-0.3.66 → tensorfs-0.3.68}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.66
+Version: 0.3.68
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.66 → tensorfs-0.3.68}/crates/tensorfs-core/src/bin/tfs/fetch.rs

@@ -257,6 +257,8 @@ fn pull_with_budget(
     flags: &Flags,
     budget: Option<(u64, u64)>,
 ) -> ExitCode {
+    // Before the open, which is where an unclean shutdown's recovery runs.
+    let before = tensorfs_core::stats::snapshot();
     let store = ok!(Store::open(root));
     let Some(base) = flag(flags, "hub") else {
         eprintln!("--hub <base-url> is required");
@@ -302,7 +304,6 @@ fn pull_with_budget(
     let observe = |_object: &ObjectRef, moved: u64, door: ObjectSource| progress.saw(moved, door);
     request.on_plan = Some(&planned);
     request.on_object = Some(&observe);
-    let before = tensorfs_core::stats::snapshot();
     let started = std::time::Instant::now();
     let report = ok!(transport::pull(&request));
     let seconds = started.elapsed().as_secs_f64();
@@ -333,9 +334,12 @@ fn pull_with_budget(
         })
         .collect();
     // And what admitting them cost: a pull whose streams stop growing while its batches
-    // take most of the wall time is held by the disk, not the link.
+    // take most of the wall time is held by the disk, not the link. `durability` says
+    // which syncs the disk the Store lives on made it pay, and what an unclean shutdown
+    // before this pull cost to repair.
+    let durability = &report.durability;
     eprintln!(
-        "{{\"event\":\"pull.streams\",\"model\":{},\"seconds\":{:.3},\"bytes_moved\":{},\"streams\":{},\"levels\":[{}],\"backoffs\":{},\"admission\":{{\"objects\":{},\"batches\":{},\"seconds\":{:.3}}}}}",
+        "{{\"event\":\"pull.streams\",\"model\":{},\"seconds\":{:.3},\"bytes_moved\":{},\"streams\":{},\"levels\":[{}],\"backoffs\":{},\"admission\":{{\"objects\":{},\"batches\":{},\"seconds\":{:.3}}},\"durability\":{{\"class\":{},\"filesystem\":{},\"syncs\":{},\"sync_seconds\":{:.3},\"recovery\":{{\"rehashed\":{},\"removed\":{},\"seconds\":{:.3}}}}}}}",
         quoted(&report.model),
         seconds,
         report.bytes_moved,
@@ -345,6 +349,13 @@ fn pull_with_budget(
         after.admitted_objects - before.admitted_objects,
         after.admission_batches - before.admission_batches,
         (after.admission_nanos - before.admission_nanos) as f64 / 1e9,
+        quoted(durability.class.as_str()),
+        quoted(durability.filesystem),
+        durability.syncs,
+        durability.seconds,
+        after.recovery_rehashed - before.recovery_rehashed,
+        after.recovery_removed - before.recovery_removed,
+        (after.recovery_nanos - before.recovery_nanos) as f64 / 1e9,
     );
     println!(
         "model        {}@{} lane {}",

{tensorfs-0.3.66 → tensorfs-0.3.68}/crates/tensorfs-core/src/bin/tfs/main.rs

@@ -904,6 +904,11 @@ fn cmd_store(root: &Path, verb: &str, flags: &Flags) -> ExitCode {
             None => "(none)".to_string(),
         }
     );
+    println!(
+        "durability: {} ({})",
+        s.disk_class().as_str(),
+        s.disk().kind
+    );
     ExitCode::SUCCESS
 }
 

{tensorfs-0.3.66 → tensorfs-0.3.68}/crates/tensorfs-core/src/bin/tfs/rd.rs

@@ -133,9 +133,13 @@ pub fn cmd_plan(root: &Path, hex: &str, f: &Flags) -> ExitCode {
     let mut rev = construction_order(&h);
     rev.reverse();
     println!(
-        "contract:   reversed caller traversal refuses: {}",
+        "contract:   reversed caller traversal plans in its own order: {}",
         match read::plan_for_traversal(&h, &rev, &all_components(&h), window) {
-            Err(e) if e.code == Code::TRAVERSAL_ORDER_MISMATCH => "YES",
+            Ok(p)
+                if p.items.first().is_some_and(|it| it
+                    .what
+                    .starts_with(&format!("{}/{}#", rev[0].0, rev[0].1))) =>
+                "YES",
             _ => "NO — DEFECT",
         }
     );
@@ -991,7 +995,7 @@ pub fn cmd_arms(root: &Path, hex: &str) -> ExitCode {
     short.pop();
     a.arm(
         "traversal missing a tensor",
-        Code::TRAVERSAL_ORDER_MISMATCH,
+        Code::TRAVERSAL_INCOMPLETE,
         read::plan_for_traversal(&h, &short, &all_components(&h), 1 << 20),
     );
 
@@ -1026,7 +1030,7 @@ pub fn cmd_arms(root: &Path, hex: &str) -> ExitCode {
         holed.pop();
         a.arm(
             "a tensor missing WITHIN a requested component",
-            Code::TRAVERSAL_ORDER_MISMATCH,
+            Code::TRAVERSAL_INCOMPLETE,
             read::plan_for_traversal(&h, &holed, only_one, 1 << 20),
         );
     }
@@ -1042,7 +1046,7 @@ pub fn cmd_arms(root: &Path, hex: &str) -> ExitCode {
     }
     a.arm(
         "a tensor from a component the plan did not request",
-        Code::TRAVERSAL_ORDER_MISMATCH,
+        Code::TRAVERSAL_INCOMPLETE,
         read::plan_for_traversal(&h, &smuggled, only_one, 1 << 20),
     );
 

tensorfs-0.3.68/crates/tensorfs-core/src/disk.rs

@@ -0,0 +1,153 @@
+//! What a crash can take from a Store, read off the filesystem the Store actually lives on.
+//!
+//! An fsync protects bytes against the machine going down before its page cache is written.
+//! Whether that protection is worth paying for depends on where the bytes live, and only the
+//! mount can say that — never a flag, never an environment variable.
+//!
+//! The rule, from `statfs(2)` of the Store root (Linux):
+//!
+//! - **Ephemeral** — tmpfs, ramfs, overlayfs. RAM, or a container's writable layer, which the
+//!   platform discards with the machine or container (a RunPod container disk is erased when
+//!   the pod stops). Nothing a crash could leave behind is ever read again, and a process
+//!   crash leaves the page cache intact, so no fsync buys anything.
+//! - **Network** — NFS, SMB/CIFS, FUSE, Ceph, 9p, AFS, Coda, GFS2, OCFS2, Lustre, GPFS.
+//!   Another machine may read it, and close-to-open is the only ordering that machine sees:
+//!   bytes are flushed before their name is published, as before.
+//! - **Persistent** — everything else (ext4, xfs, btrfs, zfs, ...), a `statfs` that fails,
+//!   and every non-Linux platform. The safe side: it survives a crash of this machine.
+
+use std::path::Path;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum DiskClass {
+    Ephemeral,
+    Persistent,
+    Network,
+}
+
+impl DiskClass {
+    /// The class of the filesystem holding `path`.
+    pub fn of(path: &Path) -> DiskClass {
+        filesystem(path).class
+    }
+
+    pub fn as_str(self) -> &'static str {
+        match self {
+            DiskClass::Ephemeral => "ephemeral",
+            DiskClass::Persistent => "persistent",
+            DiskClass::Network => "network",
+        }
+    }
+}
+
+/// The filesystem holding a path: its class and the name it was recognised by.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Filesystem {
+    pub class: DiskClass,
+    /// `ext4`, `overlayfs`, `nfs`, ...; `other` for an unrecognised magic, `unknown` when
+    /// `statfs` failed or the platform has no magic.
+    pub kind: &'static str,
+}
+
+#[cfg(target_os = "linux")]
+pub fn filesystem(path: &Path) -> Filesystem {
+    match rustix::fs::statfs(path) {
+        #[allow(clippy::unnecessary_cast)]
+        Ok(stat) => classify((stat.f_type as u64) & 0xffff_ffff),
+        Err(_) => Filesystem {
+            class: DiskClass::Persistent,
+            kind: "unknown",
+        },
+    }
+}
+
+#[cfg(not(target_os = "linux"))]
+pub fn filesystem(_path: &Path) -> Filesystem {
+    Filesystem {
+        class: DiskClass::Persistent,
+        kind: "unknown",
+    }
+}
+
+/// Superblock magics from `linux/magic.h` and the filesystems' own headers.
+#[cfg(target_os = "linux")]
+fn classify(magic: u64) -> Filesystem {
+    use DiskClass::*;
+    let (class, kind) = match magic {
+        0x0102_1994 => (Ephemeral, "tmpfs"),
+        0x8584_58f6 => (Ephemeral, "ramfs"),
+        0x794c_7630 => (Ephemeral, "overlayfs"),
+        0x6969 => (Network, "nfs"),
+        0x517b => (Network, "smb"),
+        0xff53_4d42 => (Network, "cifs"),
+        0xfe53_4d42 => (Network, "smb2"),
+        0x6573_5546 => (Network, "fuse"),
+        0x00c3_6400 => (Network, "ceph"),
+        0x0102_1997 => (Network, "9p"),
+        0x5346_414f | 0x6b41_4653 => (Network, "afs"),
+        0x7375_7245 => (Network, "coda"),
+        0x0116_1970 => (Network, "gfs2"),
+        0x7461_636f => (Network, "ocfs2"),
+        0x0bd0_0bd0 => (Network, "lustre"),
+        0x4750_4653 => (Network, "gpfs"),
+        0xef53 => (Persistent, "ext4"),
+        0x5846_5342 => (Persistent, "xfs"),
+        0x9123_683e => (Persistent, "btrfs"),
+        0x2fc1_2fc1 => (Persistent, "zfs"),
+        0xf2f5_2010 => (Persistent, "f2fs"),
+        _ => (Persistent, "other"),
+    };
+    Filesystem { class, kind }
+}
+
+/// This kernel's boot, or `None` where there is no such identity. Page cache belongs to a
+/// boot: bytes written under this boot and not yet on disk are still readable until it ends.
+/// A container sees its host's boot, which is the one that owns the page cache.
+pub(crate) fn boot_id() -> Option<String> {
+    #[cfg(target_os = "linux")]
+    {
+        let id = std::fs::read_to_string("/proc/sys/kernel/random/boot_id").ok()?;
+        let id = id.trim();
+        (id.len() == 36 && id.bytes().all(|b| b.is_ascii_hexdigit() || b == b'-'))
+            .then(|| id.to_string())
+    }
+    #[cfg(not(target_os = "linux"))]
+    {
+        None
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn magics_name_their_class() {
+        assert_eq!(classify(0x794c_7630).class, DiskClass::Ephemeral);
+        assert_eq!(classify(0x0102_1994).class, DiskClass::Ephemeral);
+        assert_eq!(classify(0x6969).class, DiskClass::Network);
+        assert_eq!(classify(0x6573_5546).class, DiskClass::Network);
+        assert_eq!(classify(0xef53).class, DiskClass::Persistent);
+        assert_eq!(classify(0x1234_5678).class, DiskClass::Persistent);
+    }
+
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn real_mounts_classify() {
+        if Path::new("/dev/shm").is_dir() {
+            assert_eq!(filesystem(Path::new("/dev/shm")).kind, "tmpfs");
+            assert_eq!(DiskClass::of(Path::new("/dev/shm")), DiskClass::Ephemeral);
+        }
+        assert_eq!(
+            DiskClass::of(Path::new("/proc/self")),
+            DiskClass::Persistent,
+            "an unrecognised filesystem is the safe side"
+        );
+        assert_eq!(
+            DiskClass::of(Path::new("/no/such/path/anywhere")),
+            DiskClass::Persistent
+        );
+        assert!(boot_id().is_some());
+    }
+}

{tensorfs-0.3.66 → tensorfs-0.3.68}/crates/tensorfs-core/src/lib.rs

@@ -20,6 +20,7 @@ pub mod checkpoint_root;
 pub mod corpus;
 pub mod derived;
 pub mod descriptors;
+pub mod disk;
 pub mod dtype;
 pub mod durability;
 pub mod err;
@@ -53,6 +54,7 @@ pub mod stats;
 pub mod storage;
 pub mod store;
 pub mod transport;
+pub mod unsynced;
 pub mod vectors;
 
 /// This TensorFS's release version.

{tensorfs-0.3.66 → tensorfs-0.3.68}/crates/tensorfs-core/src/read.rs

@@ -24,7 +24,7 @@ use std::time::Instant;
 
 use crate::checkpoint;
 use crate::err::{refuse, Code, Refusal, Result};
-use crate::header::{Asset, Body, Header, Located, Part};
+use crate::header::{Asset, Body, Header, Located, Part, Tensor};
 use crate::ids::ObjectRef;
 use crate::manifest::Manifest;
 use crate::meta::{Hold, Meta};
@@ -594,48 +594,43 @@ impl ReadPlan {
     }
 }
 
-fn build(header: &Header, scope: &[&str], window: u64) -> Result<ReadPlan> {
+fn build(ordered: &[(&str, &str, &Tensor)], window: u64) -> Result<ReadPlan> {
     let mut items = Vec::new();
     let mut dest: u64 = 0;
-    for (comp, tensors) in &header.components {
-        if !scope.contains(&comp.as_str()) {
-            continue;
-        }
-        for (key, tensor) in tensors {
-            for (role, part) in &tensor.parts {
-                let what = format!("{comp}/{key}#{role}");
-                if let Body::Inline(b) = &part.body {
-                    let n = part.nbytes(&what)?;
+    for (comp, key, tensor) in ordered {
+        for (role, part) in &tensor.parts {
+            let what = format!("{comp}/{key}#{role}");
+            if let Body::Inline(b) = &part.body {
+                let n = part.nbytes(&what)?;
+                items.push(PlanItem {
+                    what: what.clone(),
+                    dest_off: dest,
+                    len: n,
+                    source: Source::Inline(b.clone()),
+                });
+                dest += n;
+                continue;
+            }
+            for segment in part.segments() {
+                let mut off = 0u64;
+                while off < segment.length {
+                    let n = if window == 0 {
+                        segment.length - off
+                    } else {
+                        window.min(segment.length - off)
+                    };
                     items.push(PlanItem {
                         what: what.clone(),
                         dest_off: dest,
                         len: n,
-                        source: Source::Inline(b.clone()),
+                        source: Source::Object(ObjectRange {
+                            obj: segment.clone(),
+                            off,
+                            len: n,
+                        }),
                     });
                     dest += n;
-                    continue;
-                }
-                for segment in part.segments() {
-                    let mut off = 0u64;
-                    while off < segment.length {
-                        let n = if window == 0 {
-                            segment.length - off
-                        } else {
-                            window.min(segment.length - off)
-                        };
-                        items.push(PlanItem {
-                            what: what.clone(),
-                            dest_off: dest,
-                            len: n,
-                            source: Source::Object(ObjectRange {
-                                obj: segment.clone(),
-                                off,
-                                len: n,
-                            }),
-                        });
-                        dest += n;
-                        off += n;
-                    }
+                    off += n;
                 }
             }
         }
@@ -658,8 +653,10 @@ fn tally(items: &[PlanItem]) -> (u64, u64, usize) {
     (bytes, io_bytes, inline)
 }
 
-/// The read planner. Runtime's traversal must exactly equal the stored construction order;
-/// after that equality gate, planning walks the header once and never performs key lookup.
+/// The read planner. The destination follows the CALLER's traversal, which is the order it
+/// constructs in; the header's tensor order is a storage choice of whoever wrote it. The two
+/// are written by independently versioned peers, so a permutation is planned, never refused:
+/// only a traversal that names a different set of tensors is.
 ///
 /// COMPLETENESS SCOPES TO THE REQUESTED COMPONENT SET (#570b). `requested` is the caller's
 /// explicit declaration of which components this plan is for; the traversal must name every
@@ -716,34 +713,38 @@ pub fn plan_for_traversal(
         }
     }
 
-    let named: Vec<(&str, &str)> = traversal
-        .iter()
-        .map(|(c, k)| (c.as_str(), k.as_str()))
-        .collect();
-    let have: Vec<(&str, &str)> = header
+    let mut carried: HashMap<(&str, &str), &Tensor> = header
         .tensors()
         .filter(|(c, _, _)| scope.contains(&c.as_str()))
-        .map(|(c, k, _)| (c.as_str(), k.as_str()))
+        .map(|(c, k, t)| ((c.as_str(), k.as_str()), t))
         .collect();
-    if named != have {
-        let at = named
-            .iter()
-            .zip(have.iter())
-            .position(|(got, want)| got != want)
-            .unwrap_or(named.len().min(have.len()));
-        let got = named.get(at).map(|(c, k)| format!("{c}/{k}"));
-        let want = have.get(at).map(|(c, k)| format!("{c}/{k}"));
+    let mut ordered = Vec::with_capacity(traversal.len());
+    for (c, k) in traversal {
+        let Some(tensor) = carried.remove(&(c.as_str(), k.as_str())) else {
+            return refuse(
+                Code::TRAVERSAL_INCOMPLETE,
+                format!(
+                    "the traversal names {c}/{k}, which requested components {scope:?} do \
+                     not carry, or names it twice"
+                ),
+            );
+        };
+        ordered.push((c.as_str(), k.as_str(), tensor));
+    }
+    if !carried.is_empty() {
+        let mut missing: Vec<String> = carried.keys().map(|(c, k)| format!("{c}/{k}")).collect();
+        missing.sort();
+        missing.truncate(4);
         return refuse(
-            Code::TRAVERSAL_ORDER_MISMATCH,
+            Code::TRAVERSAL_INCOMPLETE,
             format!(
-                "construction order differs at row {at}: caller {got:?}, header {want:?}; \
-                 caller has {} rows and requested components {scope:?} carry {}",
-                named.len(),
-                have.len()
+                "the traversal omits {} tensor(s) requested components {scope:?} carry: \
+                 {missing:?}",
+                carried.len()
             ),
         );
     }
-    build(header, &scope, window)
+    build(&ordered, window)
 }
 
 // ---------------------------------------------------------------- the reader pool

{tensorfs-0.3.66 → tensorfs-0.3.68}/crates/tensorfs-core/src/staging.rs

@@ -204,7 +204,9 @@ pub fn stage<R: Read>(store: &Store, r: &mut R, want: &ObjectRef) -> Result<Carr
             return Err(io(format!("link {}", destination.display()), error));
         }
     }
-    crate::store::fsync_dir(&dir)?;
+    if store.flushes() {
+        crate::store::fsync_dir(&dir)?;
+    }
     let _ = fs::remove_file(&verified.tmp);
     drop(verified.file);
     Ok(Carrier {

{tensorfs-0.3.66 → tensorfs-0.3.68}/crates/tensorfs-core/src/stats.rs

@@ -9,6 +9,11 @@ static REPO_CACHE_VERIFIED_READS: AtomicU64 = AtomicU64::new(0);
 static ADMISSION_BATCHES: AtomicU64 = AtomicU64::new(0);
 static ADMITTED_OBJECTS: AtomicU64 = AtomicU64::new(0);
 static ADMISSION_NANOS: AtomicU64 = AtomicU64::new(0);
+static SYNCS: AtomicU64 = AtomicU64::new(0);
+static SYNC_NANOS: AtomicU64 = AtomicU64::new(0);
+static RECOVERY_REHASHED: AtomicU64 = AtomicU64::new(0);
+static RECOVERY_REMOVED: AtomicU64 = AtomicU64::new(0);
+static RECOVERY_NANOS: AtomicU64 = AtomicU64::new(0);
 
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
 pub struct Stats {
@@ -24,6 +29,14 @@ pub struct Stats {
     pub admitted_objects: u64,
     /// Wall time those batches spent linking, fsyncing and committing.
     pub admission_nanos: u64,
+    /// Filesystem syncs pulls ran on persistent disks, and their wall time.
+    pub syncs: u64,
+    pub sync_nanos: u64,
+    /// Objects rehashed after an unclean shutdown, those found torn and removed, and the
+    /// wall time the recovery took.
+    pub recovery_rehashed: u64,
+    pub recovery_removed: u64,
+    pub recovery_nanos: u64,
 }
 
 pub fn snapshot() -> Stats {
@@ -34,6 +47,11 @@ pub fn snapshot() -> Stats {
         admission_batches: ADMISSION_BATCHES.load(Ordering::Relaxed),
         admitted_objects: ADMITTED_OBJECTS.load(Ordering::Relaxed),
         admission_nanos: ADMISSION_NANOS.load(Ordering::Relaxed),
+        syncs: SYNCS.load(Ordering::Relaxed),
+        sync_nanos: SYNC_NANOS.load(Ordering::Relaxed),
+        recovery_rehashed: RECOVERY_REHASHED.load(Ordering::Relaxed),
+        recovery_removed: RECOVERY_REMOVED.load(Ordering::Relaxed),
+        recovery_nanos: RECOVERY_NANOS.load(Ordering::Relaxed),
     }
 }
 
@@ -69,3 +87,18 @@ pub(crate) fn admission_batch(objects: usize, spent: std::time::Duration) {
         Ordering::Relaxed,
     );
 }
+
+fn nanos(spent: std::time::Duration) -> u64 {
+    u64::try_from(spent.as_nanos()).unwrap_or(u64::MAX)
+}
+
+pub(crate) fn synced(spent: std::time::Duration) {
+    SYNCS.fetch_add(1, Ordering::Relaxed);
+    SYNC_NANOS.fetch_add(nanos(spent), Ordering::Relaxed);
+}
+
+pub(crate) fn recovered(recovery: &crate::unsynced::Recovery, spent: std::time::Duration) {
+    RECOVERY_REHASHED.fetch_add(recovery.rehashed, Ordering::Relaxed);
+    RECOVERY_REMOVED.fetch_add(recovery.removed, Ordering::Relaxed);
+    RECOVERY_NANOS.fetch_add(nanos(spent), Ordering::Relaxed);
+}