tensorfs 0.3.92__tar.gz → 0.3.93__tar.gz

{tensorfs-0.3.92 → tensorfs-0.3.93}/Cargo.lock

@@ -449,7 +449,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.92"
+version = "0.3.93"
 dependencies = [
  "fs2",
  "libc",
@@ -464,7 +464,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-plane"
-version = "0.3.92"
+version = "0.3.93"
 dependencies = [
  "libc",
  "libloading",
@@ -474,7 +474,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.92"
+version = "0.3.93"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.92 → tensorfs-0.3.93}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-plane", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.92"
+version = "0.3.93"
 edition = "2021"

{tensorfs-0.3.92 → tensorfs-0.3.93}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.92
+Version: 0.3.93
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.92 → tensorfs-0.3.93}/crates/tensorfs-plane/API.md

@@ -99,18 +99,7 @@ ts = p.replicate(ws, devices, regions=None, priority=0, pin=False)  # [Ticket]
   wait meanwhile; unregister on eviction ~0.03 ms/MiB): want the pinned tier at admission or
   between stages, never while this process computes. A host-only plane registers nothing.
 - device: map the regions (slab-sized handles), then copy from the pinned tier if it holds the
-  region; else, for a `want` of bytes the page cache holds, straight from read-only mappings
-  of their objects (no host copy, no wait for a pinned fill, which then runs beside it;
-  counted in `host_copy_bytes` and, as its part, `mapped_copy_bytes`); else after a pinned
-  fill that is running (one disk read
-  serves both); else disk → staging buffers → device. A demand miss (`acquire`) and a
-  streaming ring never map. An object that cannot be mapped (not ext4, xfs, btrfs, tmpfs or
-  overlayfs; not at its declared length; pages gone from the cache) sends its region through
-  staging (`map_refused`). A mapping's fault is a signal, not an error: a disk read error on a
-  page evicted between the check and the copy kills the process with SIGBUS. The cache query
-  runs on the caller before the fill is queued and opens each object of the wanted regions
-  (a record lookup, a rehash when the record is missing): a device `want` can block on a
-  first open.
+  region, after it if it is filling, else disk → staging buffers → device.
 - `replicate`: one pinned fill (if the pinned budget allows), then one copy per device.
 - A `want` of regions already resident only updates their priority/pin: no bytes move and
   nothing is evicted. To re-prioritise without any chance of a fill, use:
@@ -212,13 +201,8 @@ p.stats() -> {
   "devices": [{"ordinal", "uuid", "granularity", "exportable", "budget", "committed", "mapped",
                "idle", "leased_bytes", "held_bytes", "over_budget_unreleasable", "reserved_va",
                "copies", "host_copy_bytes", "host_copy_ns", "disk_copy_bytes", "disk_copy_ns",
-               "mapped_copy_bytes", "mapped_copy_ns", "misses", "evictions", "evicted_bytes",
-               "trims"}],
-  # host_copy_* counts every copy from host memory with no staging: the pinned tier's and the
-  # mappings' (mapped_copy_* is that part of it; subtract it for the tier alone, whose rate a
-  # reader of the host pair otherwise sees blended). host + disk copies are every byte that
-  # landed on the device. A pinned fill served by the page cache counts in fill_bytes and
-  # cached_bytes, never in disk_read_bytes.
+               "mapped_copy_bytes", "mapped_copy_ns",
+               "misses", "evictions", "evicted_bytes", "trims"}],
   "sets":    [{"id", "name", "nbytes", "regions", "host_ready_bytes", "host_memfd_bytes",
                "evictions", "evicted_bytes", "devices": [{"ordinal", "ready_bytes"}]}],
   "stages":  [{"ws", "device", "tag", "acquires", "skipped", "misses", "late", "late_bytes",
@@ -297,17 +281,19 @@ an unclosed plane ends.
 
 ## Changelog
 
+- v3.4 2026-10-02: the mapped device copy of v3.3 is removed; a device fill is again pinned
+  tier, else after a filling tier, else staging. In real first loads it never acted at full
+  CPU (the tier was ready first) and was slower where it did (L4 held to 4 CPUs, warm cache:
+  SDXL load to denoise 6.44 s -> 7.54 s, Anima 5.54 s -> 8.31 s): the pinned fill copied the
+  same bytes beside it. `mapped_copy_bytes`, `mapped_copy_ns` and `map_refused` stay in
+  `stats()` and read 0. Kept from v3.3: the page-cache query that answers on overlayfs and
+  where seccomp refuses `cachestat` (a pinned fill there copies cached bytes instead of
+  re-reading them), and `stats()["host"]["disk_read_bytes"]`.
+
 - v3.3 2026-10-02: `stats()["host"]["disk_read_bytes"]` exists. The plane has counted it
   since v3.1 and this file listed it, but the Python facade never carried it: a caller read 0
-  whatever the disk did, so a "nothing read from disk" check on it could not fail.
-
-- v3.3 2026-10-02 (warm loads bound by the plane's own host copy): a device `want` of bytes
-  the page cache holds copies straight from mappings of their objects, beside any pinned
-  fill. `host_copy_*` counts them with the pinned tier's copies (host + disk copies stay every
-  byte that landed on the device); `mapped_copy_*` is their part of it. The cache query now also answers where `cachestat` cannot:
-  overlayfs and a seccomp filter that refuses it. Pod-visible: there the pinned fill copies
-  freshly downloaded weights from the page cache where it re-read them by O_DIRECT, so the
-  tier and the cache copy coexist until the kernel reclaims the cache.
+  whatever the disk did, so a "nothing read from disk" check on it could not fail. (v3.3 also
+  added a device copy from page-cache mappings; v3.4 removed it.)
 
 - v3.2 2026-10-02 (W3: fd and thread growth per weight set): a plane collected unclosed or
   poisoned is torn down at its last handle (threads, device memory, pinned memfds) once its

{tensorfs-0.3.92 → tensorfs-0.3.93}/crates/tensorfs-plane/src/cursor.rs

@@ -166,7 +166,7 @@ fn submit(inner: &Arc<Inner>, c: &mut Core, id: u64, k: u64, off: u64, waits: Ve
                 c.timings.push_back(Timing {
                     dev: d,
                     ws: ws.0,
-                    kind: TimingKind::Copy { from: cp.from, host_ns: cp.host_ns },
+                    kind: TimingKind::Copy { from_host },
                     a: cp.start.clone(),
                     b: cp.ready.clone(),
                     bytes: cp.bytes,

{tensorfs-0.3.92 → tensorfs-0.3.93}/crates/tensorfs-plane/src/device.rs

@@ -11,7 +11,7 @@ use std::sync::{Arc, Condvar, Mutex};
 use std::thread::JoinHandle;
 
 use crate::cuda::{self, AccessDesc, AllocationProp, CUcontext, CUevent, CUstream, CtxGuard, Handle};
-use crate::io::{Batch, Dst, IoTally, Mapping, Mode, Readers, Source};
+use crate::io::{Batch, Dst, IoTally, Mode, Readers, Source};
 use crate::layout::{Item, ItemSource, REGION_ALIGN};
 use crate::{Error, Result};
 
@@ -368,31 +368,14 @@ pub enum CopySrc {
         items: Vec<Item>,
         base: u64,
     },
-    /// Bytes the page cache holds, read through mappings of their objects: no host copy. An
-    /// object that cannot be mapped sends the whole copy through staging instead.
-    Mapped {
-        source: Arc<Source>,
-        items: Vec<Item>,
-        base: u64,
-    },
 }
 unsafe impl Send for CopySrc {}
 
-/// Where a copy's bytes came from, for its counters.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum CopyFrom {
-    Pinned,
-    Disk,
-    Mapped,
-}
-
 pub struct Copied {
     pub ready: EvRef,
     pub start: EvRef,
     pub bytes: u64,
-    pub from: CopyFrom,
-    /// Host time before `start` that belongs to the copy: mapping and faulting its source in.
-    pub host_ns: u64,
+    pub from_host: bool,
 }
 
 pub struct CopyJob {
@@ -491,16 +474,8 @@ fn mover_main(m: MoverCtx, rx: Receiver<CopyJob>, ok: Sender<Result<()>>) {
     };
     while let Ok(job) = rx.recv() {
         let done = job.done;
-        let (r, maps) = run_copy(&m, stream, &mut bounces, job.waits, job.dst, job.len, job.src);
-        // The driver stages a pageable source after the call returns, so a mapped copy's
-        // mappings outlive its event. The region is reported first: its consumer queues behind
-        // the event with no host wait.
-        let mapped = r.as_ref().ok().filter(|_| !maps.is_empty()).map(|cp| cp.ready.clone());
+        let r = run_copy(&m, stream, &mut bounces, job.waits, job.dst, job.len, job.src);
         done(r);
-        match mapped.map(|ready| ready.sync()) {
-            Some(Err(_)) => std::mem::forget(maps), // the copy's end is unknown: it may still read them
-            _ => drop(maps),
-        }
     }
     if let Ok(d) = cuda::driver() {
         // SAFETY: our stream and buffers; the stream is drained first.
@@ -523,9 +498,6 @@ struct Waiter {
 /// no reader still writes a bounce: a failed copy is drained before it is reported, so no
 /// caller unmaps, unregisters or reuses a range a copy still touches. A drain that itself fails
 /// is ambiguous device state: `Poisoned`.
-///
-/// The mappings a queued mapped copy reads come back with it, for the caller to hold past its
-/// event. A failed one's are unmapped here, after the drain, and never if the drain failed.
 fn run_copy(
     m: &MoverCtx,
     stream: CUstream,
@@ -534,36 +506,20 @@ fn run_copy(
     dst: u64,
     len: u64,
     src: CopySrc,
-) -> (Result<Copied>, Vec<Mapping>) {
-    let began = std::time::Instant::now();
-    let (src, maps) = match src {
-        CopySrc::Mapped { source, items, base } => match source.map(&items, &m.tally) {
-            Ok(maps) => (CopySrc::Mapped { source, items, base }, maps),
-            Err(_) => {
-                // Nothing is queued yet: the same bytes go through the staging buffers.
-                m.tally.map_refused.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-                (CopySrc::Disk { source, items, base }, Vec::new())
-            }
-        },
-        src => (src, Vec::new()),
-    };
-    let host_ns = if maps.is_empty() { 0 } else { began.elapsed().as_nanos() as u64 };
-    let r = enqueue_copy(m, stream, bounces, waits, dst, len, src, &maps).map(|cp| Copied { host_ns, ..cp });
-    let Err(e) = r else { return (r, maps) };
-    let drained = crate::faults::hit("drain").and_then(|_| cuda::driver()).and_then(|d| {
-        // SAFETY: our own stream.
+) -> Result<Copied> {
+    let r = enqueue_copy(m, stream, bounces, waits, dst, len, src);
+    let Err(e) = r else { return r };
+    let d = cuda::driver()?;
+    // SAFETY: our own stream.
+    let drained = crate::faults::hit("drain").and_then(|_| {
         cuda::check("cuStreamSynchronize", unsafe { (d.stream_synchronize)(stream) })
     });
     match drained {
-        Ok(()) => (Err(e), Vec::new()),
-        Err(drain) => {
-            std::mem::forget(maps); // a copy may still read them
-            (Err(Error::Poisoned(format!("a failed copy ({e}) could not be drained: {drain}"))), Vec::new())
-        }
+        Ok(()) => Err(e),
+        Err(drain) => Err(Error::Poisoned(format!("a failed copy ({e}) could not be drained: {drain}"))),
     }
 }
 
-#[allow(clippy::too_many_arguments)]
 fn enqueue_copy(
     m: &MoverCtx,
     stream: CUstream,
@@ -572,7 +528,6 @@ fn enqueue_copy(
     dst: u64,
     len: u64,
     src: CopySrc,
-    maps: &[Mapping],
 ) -> Result<Copied> {
     let d = cuda::driver()?;
     let start = m.events.get(true)?;
@@ -595,22 +550,7 @@ fn enqueue_copy(
             })?;
             crate::faults::hit("host-copy-recorded")?;
             ready.record(stream)?;
-            Ok(Copied { ready, start, bytes: len, from: CopyFrom::Pinned, host_ns: 0 })
-        }
-        CopySrc::Mapped { items, base, .. } => {
-            wait_all(&waits)?;
-            crate::faults::gate("copy", stream)?;
-            start.record(stream)?;
-            for (it, map) in items.iter().zip(maps).filter(|(it, _)| it.len > 0) {
-                crate::faults::hit("mapped-copy")?;
-                // SAFETY: the mapping covers `it.len` bytes and the mover keeps it past this
-                // copy's event.
-                cuda::check("cuMemcpyHtoDAsync", unsafe {
-                    (d.memcpy_htod_async)(dst + (it.offset - base), map.ptr() as *const libc::c_void, it.len as usize, stream)
-                })?;
-            }
-            ready.record(stream)?;
-            Ok(Copied { ready, start, bytes: len, from: CopyFrom::Mapped, host_ns: 0 })
+            Ok(Copied { ready, start, bytes: len, from_host: true })
         }
         CopySrc::Disk { source, items, base } => {
             let cap = m.staging.1;
@@ -697,7 +637,7 @@ fn enqueue_copy(
                 start.record(stream)?;
             }
             ready.record(stream)?;
-            Ok(Copied { ready, start, bytes: len, from: CopyFrom::Disk, host_ns: 0 })
+            Ok(Copied { ready, start, bytes: len, from_host: false })
         }
     }
 }

{tensorfs-0.3.92 → tensorfs-0.3.93}/crates/tensorfs-plane/src/faults.rs

@@ -1,5 +1,5 @@
-//! Fault points on the copy and fill paths, armed only by tests (cargo feature `faults`).
-//! Without the feature every point is a no-op.
+//! Fault points on the copy path, armed only by tests (cargo feature `faults`). Without the
+//! feature every point is a no-op.
 
 #[cfg(feature = "faults")]
 mod imp {
@@ -74,24 +74,12 @@ mod imp {
         // SAFETY: our stream; the host function only waits on the flag.
         cuda::check("cuLaunchHostFunc", unsafe { (d.launch_host_func)(stream, hold, p) })
     }
-
-    /// Holds the calling thread at `point` until the gate armed there opens.
-    pub(crate) fn wait(point: &'static str) {
-        let f = {
-            let mut g = GATES.lock().unwrap();
-            let Some(i) = g.iter().position(|(p, _)| *p == point) else { return };
-            g.remove(i).1
-        };
-        while !f.load(Ordering::Acquire) {
-            std::thread::sleep(std::time::Duration::from_millis(1));
-        }
-    }
 }
 
 #[cfg(feature = "faults")]
 pub use imp::{arm, arm_gate, Gate};
 #[cfg(feature = "faults")]
-pub(crate) use imp::{gate, hit, wait};
+pub(crate) use imp::{gate, hit};
 
 #[cfg(not(feature = "faults"))]
 #[inline(always)]
@@ -104,7 +92,3 @@ pub(crate) fn hit(_point: &'static str) -> crate::Result<()> {
 pub(crate) fn gate(_point: &'static str, _stream: crate::cuda::CUstream) -> crate::Result<()> {
     Ok(())
 }
-
-#[cfg(not(feature = "faults"))]
-#[inline(always)]
-pub(crate) fn wait(_point: &'static str) {}

{tensorfs-0.3.92 → tensorfs-0.3.93}/crates/tensorfs-plane/src/io.rs

@@ -2,8 +2,7 @@
 //! descriptors the plane opens through the store's own verification (`open_verified`). A
 //! pinned-tier fill copies what the page cache already holds and reads the rest by O_DIRECT
 //! into the destination (no second host copy), else through `tensorfs_core::read::read_into`.
-//! A device copy of bytes the page cache holds reads them through a read-only mapping, with no
-//! host copy at all. Each mode's bytes are counted.
+//! Each mode's bytes are counted.
 
 use std::collections::HashMap;
 use std::ffi::CString;
@@ -30,8 +29,6 @@ pub struct IoTally {
     pub inline_bytes: AtomicU64,
     /// Objects whose filesystem refused O_DIRECT: their bytes went buffered.
     pub direct_refused: AtomicU64,
-    /// Device copies whose objects could not be mapped: they went through staging.
-    pub map_refused: AtomicU64,
 }
 
 /// How a fill wants its bytes: `Cache` for the pinned tier (O_DIRECT for what the page cache
@@ -45,94 +42,8 @@ pub enum Mode {
 struct Fds {
     buffered: File,
     direct: Option<File>,
-    fs: Fs,
-}
-
-/// What the plane may do with an object by its filesystem. A fault on a mapping is a signal,
-/// not an error, so only local filesystems are mapped.
-#[derive(Clone, Copy, PartialEq, Eq)]
-enum Fs {
-    Local,
-    /// Local too, but its files report no page cache of their own (a container's root).
-    Overlay,
-    Other,
-}
-
-impl Fs {
-    fn of(f: &File) -> Fs {
-        // SAFETY: out-param of the right type on our own descriptor.
-        let mut fs: libc::statfs = unsafe { std::mem::zeroed() };
-        if unsafe { libc::fstatfs(f.as_raw_fd(), &mut fs) } != 0 {
-            return Fs::Other;
-        }
-        #[allow(clippy::unnecessary_cast)] // f_type is not an i64 on every target
-        match fs.f_type as i64 {
-            0x794c7630 => Fs::Overlay,
-            // ext4, xfs, btrfs, tmpfs
-            0xEF53 | 0x58465342 | 0x9123683E | 0x01021994 => Fs::Local,
-            _ => Fs::Other,
-        }
-    }
-}
-
-/// One item's bytes for a device copy, with no host copy: a populated read-only mapping of its
-/// object range (unmapped on drop), or the inline bytes themselves.
-pub enum Mapping {
-    Inline(Arc<[u8]>),
-    File { base: *mut u8, span: usize, at: usize },
-}
-// SAFETY: a read-only mapping (or shared immutable bytes) owned by whoever holds the value.
-unsafe impl Send for Mapping {}
-
-impl Mapping {
-    pub fn ptr(&self) -> *const u8 {
-        match self {
-            Mapping::Inline(b) => b.as_ptr(),
-            // SAFETY: `at` lies inside the mapping.
-            Mapping::File { base, at, .. } => unsafe { base.add(*at) },
-        }
-    }
-
-    /// `[off, off+len)` of `f`, which the page cache must hold: a cold range is refused, never
-    /// read from disk through the mapping. Populated and checked again, so the copy that
-    /// follows is not the range's first reader.
-    fn file(f: &File, off: u64, len: u64) -> Result<Mapping> {
-        if !resident(f, off, len) {
-            return Err(Error::Io("the page cache let go of the range".into()));
-        }
-        let at = (off % 4096) as usize;
-        let span = at + len as usize;
-        // SAFETY: a fresh read-only shared mapping of our descriptor.
-        let p = unsafe {
-            libc::mmap(
-                std::ptr::null_mut(),
-                span,
-                libc::PROT_READ,
-                libc::MAP_SHARED | libc::MAP_POPULATE,
-                f.as_raw_fd(),
-                (off - at as u64) as libc::off_t,
-            )
-        };
-        if p == libc::MAP_FAILED {
-            return Err(Error::Io(format!("mmap: {}", std::io::Error::last_os_error())));
-        }
-        let m = Mapping::File { base: p as *mut u8, span, at };
-        let mut pages = vec![0u8; span.div_ceil(4096)];
-        // SAFETY: the vector holds one byte per page of the mapping.
-        if unsafe { libc::mincore(p, span, pages.as_mut_ptr()) } != 0 || pages.iter().any(|b| b & 1 == 0) {
-            return Err(Error::Io("the page cache let go of a mapped range".into()));
-        }
-        Ok(m)
-    }
-}
-
-impl Drop for Mapping {
-    fn drop(&mut self) {
-        if let Mapping::File { base, span, .. } = self {
-            // SAFETY: our own mapping, no longer read by anyone (the holder's contract).
-            unsafe { libc::munmap(*base as *mut libc::c_void, *span) };
-        }
-    }
+    /// On overlayfs (a container's root): the file reports no page cache of its own.
+    overlay: bool,
 }
 
 pub struct Source {
@@ -173,7 +84,16 @@ fn cached(f: &Fds, off: u64, len: u64) -> bool {
     if rc == 0 && st.nr_cache >= (off + len).div_ceil(page) - off / page {
         return true;
     }
-    (rc != 0 || f.fs == Fs::Overlay) && resident(&f.buffered, off, len)
+    (rc != 0 || f.overlay) && resident(&f.buffered, off, len)
+}
+
+fn overlay(f: &File) -> bool {
+    const OVERLAYFS_SUPER_MAGIC: i64 = 0x794c7630;
+    // SAFETY: out-param of the right type on our own descriptor.
+    let mut fs: libc::statfs = unsafe { std::mem::zeroed() };
+    #[allow(clippy::unnecessary_cast)] // f_type is not an i64 on every target
+    let magic = unsafe { libc::fstatfs(f.as_raw_fd(), &mut fs) == 0 }.then_some(fs.f_type as i64);
+    magic == Some(OVERLAYFS_SUPER_MAGIC)
 }
 
 /// Every page of `[off, off+len)` is in core, by `mincore(2)` over a mapping (never faults).
@@ -250,50 +170,11 @@ impl Source {
         } else {
             None
         };
-        let fs = Fs::of(&buffered);
-        let f = Arc::new(Fds { buffered, direct, fs });
+        let f = Arc::new(Fds { overlay: overlay(&buffered), buffered, direct });
         self.fds.lock().unwrap().insert(obj.sha256.clone(), f.clone());
         Ok(f)
     }
 
-    /// Whether the page cache holds every object byte of `items`. An object that will not open
-    /// counts as not held: its reader reports the error.
-    pub(crate) fn cached(&self, items: &[Item], tally: &IoTally) -> bool {
-        items.iter().all(|it| match &it.source {
-            ItemSource::Inline(_) => true,
-            ItemSource::Object(o) => {
-                self.fds(&o.obj, tally).is_ok_and(|f| cached(&f, o.off, it.len))
-            }
-        })
-    }
-
-    /// `items` mapped for a device copy. Refuses (the caller stages instead) an object on a
-    /// filesystem where a fault could be a network error, one not at its declared length, and
-    /// pages the cache no longer holds.
-    pub fn map(&self, items: &[Item], tally: &IoTally) -> Result<Vec<Mapping>> {
-        crate::faults::hit("map")?;
-        let guard = self.lease.read().unwrap();
-        let lease = guard.as_ref().ok_or(Error::Closed)?;
-        let one = |it: &Item| -> Result<Mapping> {
-            let o = match &it.source {
-                ItemSource::Inline(b) => return Ok(Mapping::Inline(b.clone())),
-                ItemSource::Object(_) if it.len == 0 => return Ok(Mapping::Inline(Arc::from([]))),
-                ItemSource::Object(o) => o,
-            };
-            lease.covers(o)?;
-            let fds = self.fds(&o.obj, tally)?;
-            if fds.fs == Fs::Other {
-                return Err(Error::Invalid(format!("{}: not on a local filesystem", o.obj.sha256)));
-            }
-            let size = fds.buffered.metadata().map_err(|e| Error::Io(format!("fstat {}: {e}", o.obj.sha256)))?.len();
-            if size != o.obj.length {
-                return Err(Error::Io(format!("{} is {size} bytes, declared {}", o.obj.sha256, o.obj.length)));
-            }
-            Mapping::file(&fds.buffered, o.off, it.len)
-        };
-        items.iter().map(one).collect()
-    }
-
     /// Fill `item.len` bytes at `dst`. `Mode::Cache` reads a range the page cache fully
     /// holds by copy and everything else by O_DIRECT when alignment allows, which may DMA up
     /// to the next `PART_ALIGN` boundary past the item: the layout reserves that as padding.