tensorfs 0.3.91__tar.gz → 0.3.92__tar.gz

{tensorfs-0.3.91 → tensorfs-0.3.92}/Cargo.lock

@@ -449,7 +449,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.91"
+version = "0.3.92"
 dependencies = [
  "fs2",
  "libc",
@@ -464,7 +464,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-plane"
-version = "0.3.91"
+version = "0.3.92"
 dependencies = [
  "libc",
  "libloading",
@@ -474,7 +474,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.91"
+version = "0.3.92"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.91 → tensorfs-0.3.92}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-plane", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.91"
+version = "0.3.92"
 edition = "2021"

{tensorfs-0.3.91 → tensorfs-0.3.92}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.91
+Version: 0.3.92
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.91 → tensorfs-0.3.92}/crates/tensorfs-plane/API.md

@@ -99,7 +99,18 @@ ts = p.replicate(ws, devices, regions=None, priority=0, pin=False)  # [Ticket]
   wait meanwhile; unregister on eviction ~0.03 ms/MiB): want the pinned tier at admission or
   between stages, never while this process computes. A host-only plane registers nothing.
 - device: map the regions (slab-sized handles), then copy from the pinned tier if it holds the
-  region, after it if it is filling, else disk → staging buffers → device.
+  region; else, for a `want` of bytes the page cache holds, straight from read-only mappings
+  of their objects (no host copy, no wait for a pinned fill, which then runs beside it;
+  counted in `host_copy_bytes` and, as its part, `mapped_copy_bytes`); else after a pinned
+  fill that is running (one disk read
+  serves both); else disk → staging buffers → device. A demand miss (`acquire`) and a
+  streaming ring never map. An object that cannot be mapped (not ext4, xfs, btrfs, tmpfs or
+  overlayfs; not at its declared length; pages gone from the cache) sends its region through
+  staging (`map_refused`). A mapping's fault is a signal, not an error: a disk read error on a
+  page evicted between the check and the copy kills the process with SIGBUS. The cache query
+  runs on the caller before the fill is queued and opens each object of the wanted regions
+  (a record lookup, a rehash when the record is missing): a device `want` can block on a
+  first open.
 - `replicate`: one pinned fill (if the pinned budget allows), then one copy per device.
 - A `want` of regions already resident only updates their priority/pin: no bytes move and
   nothing is evicted. To re-prioritise without any chance of a fill, use:
@@ -195,13 +206,19 @@ are equal-sized).
 p.stats() -> {
   "host":    {"budget", "used", "ready_bytes", "registered_bytes", "memfd_bytes",
               "cached_bytes", "direct_bytes", "buffered_bytes", "inline_bytes", "direct_refused",
-              "disk_read_bytes",
+              "map_refused", "disk_read_bytes",
               "fills", "fill_bytes", "fill_ns", "evictions", "evicted_bytes", "register_failed",
               "trims"},
   "devices": [{"ordinal", "uuid", "granularity", "exportable", "budget", "committed", "mapped",
                "idle", "leased_bytes", "held_bytes", "over_budget_unreleasable", "reserved_va",
                "copies", "host_copy_bytes", "host_copy_ns", "disk_copy_bytes", "disk_copy_ns",
-               "misses", "evictions", "evicted_bytes", "trims"}],
+               "mapped_copy_bytes", "mapped_copy_ns", "misses", "evictions", "evicted_bytes",
+               "trims"}],
+  # host_copy_* counts every copy from host memory with no staging: the pinned tier's and the
+  # mappings' (mapped_copy_* is that part of it; subtract it for the tier alone, whose rate a
+  # reader of the host pair otherwise sees blended). host + disk copies are every byte that
+  # landed on the device. A pinned fill served by the page cache counts in fill_bytes and
+  # cached_bytes, never in disk_read_bytes.
   "sets":    [{"id", "name", "nbytes", "regions", "host_ready_bytes", "host_memfd_bytes",
                "evictions", "evicted_bytes", "devices": [{"ordinal", "ready_bytes"}]}],
   "stages":  [{"ws", "device", "tag", "acquires", "skipped", "misses", "late", "late_bytes",
@@ -280,6 +297,18 @@ an unclosed plane ends.
 
 ## Changelog
 
+- v3.3 2026-10-02: `stats()["host"]["disk_read_bytes"]` exists. The plane has counted it
+  since v3.1 and this file listed it, but the Python facade never carried it: a caller read 0
+  whatever the disk did, so a "nothing read from disk" check on it could not fail.
+
+- v3.3 2026-10-02 (warm loads bound by the plane's own host copy): a device `want` of bytes
+  the page cache holds copies straight from mappings of their objects, beside any pinned
+  fill. `host_copy_*` counts them with the pinned tier's copies (host + disk copies stay every
+  byte that landed on the device); `mapped_copy_*` is their part of it. The cache query now also answers where `cachestat` cannot:
+  overlayfs and a seccomp filter that refuses it. Pod-visible: there the pinned fill copies
+  freshly downloaded weights from the page cache where it re-read them by O_DIRECT, so the
+  tier and the cache copy coexist until the kernel reclaims the cache.
+
 - v3.2 2026-10-02 (W3: fd and thread growth per weight set): a plane collected unclosed or
   poisoned is torn down at its last handle (threads, device memory, pinned memfds) once its
   devices are synchronized; before, its reader threads stayed and a poisoned plane kept its

{tensorfs-0.3.91 → tensorfs-0.3.92}/crates/tensorfs-plane/src/cursor.rs

@@ -166,7 +166,7 @@ fn submit(inner: &Arc<Inner>, c: &mut Core, id: u64, k: u64, off: u64, waits: Ve
                 c.timings.push_back(Timing {
                     dev: d,
                     ws: ws.0,
-                    kind: TimingKind::Copy { from_host },
+                    kind: TimingKind::Copy { from: cp.from, host_ns: cp.host_ns },
                     a: cp.start.clone(),
                     b: cp.ready.clone(),
                     bytes: cp.bytes,

{tensorfs-0.3.91 → tensorfs-0.3.92}/crates/tensorfs-plane/src/device.rs

@@ -11,7 +11,7 @@ use std::sync::{Arc, Condvar, Mutex};
 use std::thread::JoinHandle;
 
 use crate::cuda::{self, AccessDesc, AllocationProp, CUcontext, CUevent, CUstream, CtxGuard, Handle};
-use crate::io::{Batch, Dst, IoTally, Mode, Readers, Source};
+use crate::io::{Batch, Dst, IoTally, Mapping, Mode, Readers, Source};
 use crate::layout::{Item, ItemSource, REGION_ALIGN};
 use crate::{Error, Result};
 
@@ -368,14 +368,31 @@ pub enum CopySrc {
         items: Vec<Item>,
         base: u64,
     },
+    /// Bytes the page cache holds, read through mappings of their objects: no host copy. An
+    /// object that cannot be mapped sends the whole copy through staging instead.
+    Mapped {
+        source: Arc<Source>,
+        items: Vec<Item>,
+        base: u64,
+    },
 }
 unsafe impl Send for CopySrc {}
 
+/// Where a copy's bytes came from, for its counters.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum CopyFrom {
+    Pinned,
+    Disk,
+    Mapped,
+}
+
 pub struct Copied {
     pub ready: EvRef,
     pub start: EvRef,
     pub bytes: u64,
-    pub from_host: bool,
+    pub from: CopyFrom,
+    /// Host time before `start` that belongs to the copy: mapping and faulting its source in.
+    pub host_ns: u64,
 }
 
 pub struct CopyJob {
@@ -474,8 +491,16 @@ fn mover_main(m: MoverCtx, rx: Receiver<CopyJob>, ok: Sender<Result<()>>) {
     };
     while let Ok(job) = rx.recv() {
         let done = job.done;
-        let r = run_copy(&m, stream, &mut bounces, job.waits, job.dst, job.len, job.src);
+        let (r, maps) = run_copy(&m, stream, &mut bounces, job.waits, job.dst, job.len, job.src);
+        // The driver stages a pageable source after the call returns, so a mapped copy's
+        // mappings outlive its event. The region is reported first: its consumer queues behind
+        // the event with no host wait.
+        let mapped = r.as_ref().ok().filter(|_| !maps.is_empty()).map(|cp| cp.ready.clone());
         done(r);
+        match mapped.map(|ready| ready.sync()) {
+            Some(Err(_)) => std::mem::forget(maps), // the copy's end is unknown: it may still read them
+            _ => drop(maps),
+        }
     }
     if let Ok(d) = cuda::driver() {
         // SAFETY: our stream and buffers; the stream is drained first.
@@ -498,6 +523,9 @@ struct Waiter {
 /// no reader still writes a bounce: a failed copy is drained before it is reported, so no
 /// caller unmaps, unregisters or reuses a range a copy still touches. A drain that itself fails
 /// is ambiguous device state: `Poisoned`.
+///
+/// The mappings a queued mapped copy reads come back with it, for the caller to hold past its
+/// event. A failed one's are unmapped here, after the drain, and never if the drain failed.
 fn run_copy(
     m: &MoverCtx,
     stream: CUstream,
@@ -506,20 +534,36 @@ fn run_copy(
     dst: u64,
     len: u64,
     src: CopySrc,
-) -> Result<Copied> {
-    let r = enqueue_copy(m, stream, bounces, waits, dst, len, src);
-    let Err(e) = r else { return r };
-    let d = cuda::driver()?;
-    // SAFETY: our own stream.
-    let drained = crate::faults::hit("drain").and_then(|_| {
+) -> (Result<Copied>, Vec<Mapping>) {
+    let began = std::time::Instant::now();
+    let (src, maps) = match src {
+        CopySrc::Mapped { source, items, base } => match source.map(&items, &m.tally) {
+            Ok(maps) => (CopySrc::Mapped { source, items, base }, maps),
+            Err(_) => {
+                // Nothing is queued yet: the same bytes go through the staging buffers.
+                m.tally.map_refused.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+                (CopySrc::Disk { source, items, base }, Vec::new())
+            }
+        },
+        src => (src, Vec::new()),
+    };
+    let host_ns = if maps.is_empty() { 0 } else { began.elapsed().as_nanos() as u64 };
+    let r = enqueue_copy(m, stream, bounces, waits, dst, len, src, &maps).map(|cp| Copied { host_ns, ..cp });
+    let Err(e) = r else { return (r, maps) };
+    let drained = crate::faults::hit("drain").and_then(|_| cuda::driver()).and_then(|d| {
+        // SAFETY: our own stream.
         cuda::check("cuStreamSynchronize", unsafe { (d.stream_synchronize)(stream) })
     });
     match drained {
-        Ok(()) => Err(e),
-        Err(drain) => Err(Error::Poisoned(format!("a failed copy ({e}) could not be drained: {drain}"))),
+        Ok(()) => (Err(e), Vec::new()),
+        Err(drain) => {
+            std::mem::forget(maps); // a copy may still read them
+            (Err(Error::Poisoned(format!("a failed copy ({e}) could not be drained: {drain}"))), Vec::new())
+        }
     }
 }
 
+#[allow(clippy::too_many_arguments)]
 fn enqueue_copy(
     m: &MoverCtx,
     stream: CUstream,
@@ -528,6 +572,7 @@ fn enqueue_copy(
     dst: u64,
     len: u64,
     src: CopySrc,
+    maps: &[Mapping],
 ) -> Result<Copied> {
     let d = cuda::driver()?;
     let start = m.events.get(true)?;
@@ -550,7 +595,22 @@ fn enqueue_copy(
             })?;
             crate::faults::hit("host-copy-recorded")?;
             ready.record(stream)?;
-            Ok(Copied { ready, start, bytes: len, from_host: true })
+            Ok(Copied { ready, start, bytes: len, from: CopyFrom::Pinned, host_ns: 0 })
+        }
+        CopySrc::Mapped { items, base, .. } => {
+            wait_all(&waits)?;
+            crate::faults::gate("copy", stream)?;
+            start.record(stream)?;
+            for (it, map) in items.iter().zip(maps).filter(|(it, _)| it.len > 0) {
+                crate::faults::hit("mapped-copy")?;
+                // SAFETY: the mapping covers `it.len` bytes and the mover keeps it past this
+                // copy's event.
+                cuda::check("cuMemcpyHtoDAsync", unsafe {
+                    (d.memcpy_htod_async)(dst + (it.offset - base), map.ptr() as *const libc::c_void, it.len as usize, stream)
+                })?;
+            }
+            ready.record(stream)?;
+            Ok(Copied { ready, start, bytes: len, from: CopyFrom::Mapped, host_ns: 0 })
         }
         CopySrc::Disk { source, items, base } => {
             let cap = m.staging.1;
@@ -637,7 +697,7 @@ fn enqueue_copy(
                 start.record(stream)?;
             }
             ready.record(stream)?;
-            Ok(Copied { ready, start, bytes: len, from_host: false })
+            Ok(Copied { ready, start, bytes: len, from: CopyFrom::Disk, host_ns: 0 })
         }
     }
 }

{tensorfs-0.3.91 → tensorfs-0.3.92}/crates/tensorfs-plane/src/faults.rs

@@ -1,5 +1,5 @@
-//! Fault points on the copy path, armed only by tests (cargo feature `faults`). Without the
-//! feature every point is a no-op.
+//! Fault points on the copy and fill paths, armed only by tests (cargo feature `faults`).
+//! Without the feature every point is a no-op.
 
 #[cfg(feature = "faults")]
 mod imp {
@@ -74,12 +74,24 @@ mod imp {
         // SAFETY: our stream; the host function only waits on the flag.
         cuda::check("cuLaunchHostFunc", unsafe { (d.launch_host_func)(stream, hold, p) })
     }
+
+    /// Holds the calling thread at `point` until the gate armed there opens.
+    pub(crate) fn wait(point: &'static str) {
+        let f = {
+            let mut g = GATES.lock().unwrap();
+            let Some(i) = g.iter().position(|(p, _)| *p == point) else { return };
+            g.remove(i).1
+        };
+        while !f.load(Ordering::Acquire) {
+            std::thread::sleep(std::time::Duration::from_millis(1));
+        }
+    }
 }
 
 #[cfg(feature = "faults")]
 pub use imp::{arm, arm_gate, Gate};
 #[cfg(feature = "faults")]
-pub(crate) use imp::{gate, hit};
+pub(crate) use imp::{gate, hit, wait};
 
 #[cfg(not(feature = "faults"))]
 #[inline(always)]
@@ -92,3 +104,7 @@ pub(crate) fn hit(_point: &'static str) -> crate::Result<()> {
 pub(crate) fn gate(_point: &'static str, _stream: crate::cuda::CUstream) -> crate::Result<()> {
     Ok(())
 }
+
+#[cfg(not(feature = "faults"))]
+#[inline(always)]
+pub(crate) fn wait(_point: &'static str) {}

{tensorfs-0.3.91 → tensorfs-0.3.92}/crates/tensorfs-plane/src/io.rs

@@ -2,7 +2,8 @@
 //! descriptors the plane opens through the store's own verification (`open_verified`). A
 //! pinned-tier fill copies what the page cache already holds and reads the rest by O_DIRECT
 //! into the destination (no second host copy), else through `tensorfs_core::read::read_into`.
-//! Each mode's bytes are counted.
+//! A device copy of bytes the page cache holds reads them through a read-only mapping, with no
+//! host copy at all. Each mode's bytes are counted.
 
 use std::collections::HashMap;
 use std::ffi::CString;
@@ -29,6 +30,8 @@ pub struct IoTally {
     pub inline_bytes: AtomicU64,
     /// Objects whose filesystem refused O_DIRECT: their bytes went buffered.
     pub direct_refused: AtomicU64,
+    /// Device copies whose objects could not be mapped: they went through staging.
+    pub map_refused: AtomicU64,
 }
 
 /// How a fill wants its bytes: `Cache` for the pinned tier (O_DIRECT for what the page cache
@@ -42,6 +45,94 @@ pub enum Mode {
 struct Fds {
     buffered: File,
     direct: Option<File>,
+    fs: Fs,
+}
+
+/// What the plane may do with an object by its filesystem. A fault on a mapping is a signal,
+/// not an error, so only local filesystems are mapped.
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum Fs {
+    Local,
+    /// Local too, but its files report no page cache of their own (a container's root).
+    Overlay,
+    Other,
+}
+
+impl Fs {
+    fn of(f: &File) -> Fs {
+        // SAFETY: out-param of the right type on our own descriptor.
+        let mut fs: libc::statfs = unsafe { std::mem::zeroed() };
+        if unsafe { libc::fstatfs(f.as_raw_fd(), &mut fs) } != 0 {
+            return Fs::Other;
+        }
+        #[allow(clippy::unnecessary_cast)] // f_type is not an i64 on every target
+        match fs.f_type as i64 {
+            0x794c7630 => Fs::Overlay,
+            // ext4, xfs, btrfs, tmpfs
+            0xEF53 | 0x58465342 | 0x9123683E | 0x01021994 => Fs::Local,
+            _ => Fs::Other,
+        }
+    }
+}
+
+/// One item's bytes for a device copy, with no host copy: a populated read-only mapping of its
+/// object range (unmapped on drop), or the inline bytes themselves.
+pub enum Mapping {
+    Inline(Arc<[u8]>),
+    File { base: *mut u8, span: usize, at: usize },
+}
+// SAFETY: a read-only mapping (or shared immutable bytes) owned by whoever holds the value.
+unsafe impl Send for Mapping {}
+
+impl Mapping {
+    pub fn ptr(&self) -> *const u8 {
+        match self {
+            Mapping::Inline(b) => b.as_ptr(),
+            // SAFETY: `at` lies inside the mapping.
+            Mapping::File { base, at, .. } => unsafe { base.add(*at) },
+        }
+    }
+
+    /// `[off, off+len)` of `f`, which the page cache must hold: a cold range is refused, never
+    /// read from disk through the mapping. Populated and checked again, so the copy that
+    /// follows is not the range's first reader.
+    fn file(f: &File, off: u64, len: u64) -> Result<Mapping> {
+        if !resident(f, off, len) {
+            return Err(Error::Io("the page cache let go of the range".into()));
+        }
+        let at = (off % 4096) as usize;
+        let span = at + len as usize;
+        // SAFETY: a fresh read-only shared mapping of our descriptor.
+        let p = unsafe {
+            libc::mmap(
+                std::ptr::null_mut(),
+                span,
+                libc::PROT_READ,
+                libc::MAP_SHARED | libc::MAP_POPULATE,
+                f.as_raw_fd(),
+                (off - at as u64) as libc::off_t,
+            )
+        };
+        if p == libc::MAP_FAILED {
+            return Err(Error::Io(format!("mmap: {}", std::io::Error::last_os_error())));
+        }
+        let m = Mapping::File { base: p as *mut u8, span, at };
+        let mut pages = vec![0u8; span.div_ceil(4096)];
+        // SAFETY: the vector holds one byte per page of the mapping.
+        if unsafe { libc::mincore(p, span, pages.as_mut_ptr()) } != 0 || pages.iter().any(|b| b & 1 == 0) {
+            return Err(Error::Io("the page cache let go of a mapped range".into()));
+        }
+        Ok(m)
+    }
+}
+
+impl Drop for Mapping {
+    fn drop(&mut self) {
+        if let Mapping::File { base, span, .. } = self {
+            // SAFETY: our own mapping, no longer read by anyone (the holder's contract).
+            unsafe { libc::munmap(*base as *mut libc::c_void, *span) };
+        }
+    }
 }
 
 pub struct Source {
@@ -54,9 +145,11 @@ pub struct Source {
 }
 
 /// Whether the page cache holds every page of `[off, off+len)`, from `cachestat(2)`: a pure
-/// query (unlike an `RWF_NOWAIT` read, which starts readahead on a miss). `None` where the
-/// kernel lacks it (< 6.5): the caller treats the range as cold.
-fn cached(f: &File, off: u64, len: u64) -> Option<bool> {
+/// query (unlike an `RWF_NOWAIT` read, which starts readahead on a miss). Where that cannot
+/// answer, the pages are counted through a mapping: a kernel without it (< 6.5), a container
+/// whose seccomp filter refuses it, and overlayfs (a container's root), whose files report no
+/// pages of their own.
+fn cached(f: &Fds, off: u64, len: u64) -> bool {
     #[repr(C)]
     struct Range {
         off: u64,
@@ -75,12 +168,31 @@ fn cached(f: &File, off: u64, len: u64) -> Option<bool> {
     let r = Range { off, len };
     let mut st = Stat::default();
     // SAFETY: the documented cachestat ABI; both structs outlive the call.
-    let rc = unsafe { libc::syscall(SYS_CACHESTAT, f.as_raw_fd(), &r, &mut st, 0) };
-    if rc != 0 {
-        return None;
-    }
+    let rc = unsafe { libc::syscall(SYS_CACHESTAT, f.buffered.as_raw_fd(), &r, &mut st, 0) };
     let page = 4096u64;
-    Some(st.nr_cache >= (off + len).div_ceil(page) - off / page)
+    if rc == 0 && st.nr_cache >= (off + len).div_ceil(page) - off / page {
+        return true;
+    }
+    (rc != 0 || f.fs == Fs::Overlay) && resident(&f.buffered, off, len)
+}
+
+/// Every page of `[off, off+len)` is in core, by `mincore(2)` over a mapping (never faults).
+fn resident(f: &File, off: u64, len: u64) -> bool {
+    let start = off - off % 4096;
+    let n = (off + len - start) as usize;
+    if n == 0 {
+        return true;
+    }
+    // SAFETY: a read-only shared mapping of our descriptor, unmapped below.
+    let p = unsafe { libc::mmap(std::ptr::null_mut(), n, libc::PROT_READ, libc::MAP_SHARED, f.as_raw_fd(), start as libc::off_t) };
+    if p == libc::MAP_FAILED {
+        return false;
+    }
+    let mut pages = vec![0u8; n.div_ceil(4096)];
+    // SAFETY: the vector holds one byte per page of the mapping.
+    let ok = unsafe { libc::mincore(p, n, pages.as_mut_ptr()) } == 0 && pages.iter().all(|b| b & 1 == 1);
+    unsafe { libc::munmap(p, n) };
+    ok
 }
 
 impl Source {
@@ -138,11 +250,50 @@ impl Source {
         } else {
             None
         };
-        let f = Arc::new(Fds { buffered, direct });
+        let fs = Fs::of(&buffered);
+        let f = Arc::new(Fds { buffered, direct, fs });
         self.fds.lock().unwrap().insert(obj.sha256.clone(), f.clone());
         Ok(f)
     }
 
+    /// Whether the page cache holds every object byte of `items`. An object that will not open
+    /// counts as not held: its reader reports the error.
+    pub(crate) fn cached(&self, items: &[Item], tally: &IoTally) -> bool {
+        items.iter().all(|it| match &it.source {
+            ItemSource::Inline(_) => true,
+            ItemSource::Object(o) => {
+                self.fds(&o.obj, tally).is_ok_and(|f| cached(&f, o.off, it.len))
+            }
+        })
+    }
+
+    /// `items` mapped for a device copy. Refuses (the caller stages instead) an object on a
+    /// filesystem where a fault could be a network error, one not at its declared length, and
+    /// pages the cache no longer holds.
+    pub fn map(&self, items: &[Item], tally: &IoTally) -> Result<Vec<Mapping>> {
+        crate::faults::hit("map")?;
+        let guard = self.lease.read().unwrap();
+        let lease = guard.as_ref().ok_or(Error::Closed)?;
+        let one = |it: &Item| -> Result<Mapping> {
+            let o = match &it.source {
+                ItemSource::Inline(b) => return Ok(Mapping::Inline(b.clone())),
+                ItemSource::Object(_) if it.len == 0 => return Ok(Mapping::Inline(Arc::from([]))),
+                ItemSource::Object(o) => o,
+            };
+            lease.covers(o)?;
+            let fds = self.fds(&o.obj, tally)?;
+            if fds.fs == Fs::Other {
+                return Err(Error::Invalid(format!("{}: not on a local filesystem", o.obj.sha256)));
+            }
+            let size = fds.buffered.metadata().map_err(|e| Error::Io(format!("fstat {}: {e}", o.obj.sha256)))?.len();
+            if size != o.obj.length {
+                return Err(Error::Io(format!("{} is {size} bytes, declared {}", o.obj.sha256, o.obj.length)));
+            }
+            Mapping::file(&fds.buffered, o.off, it.len)
+        };
+        items.iter().map(one).collect()
+    }
+
     /// Fill `item.len` bytes at `dst`. `Mode::Cache` reads a range the page cache fully
     /// holds by copy and everything else by O_DIRECT when alignment allows, which may DMA up
     /// to the next `PART_ALIGN` boundary past the item: the layout reserves that as padding.
@@ -169,7 +320,7 @@ impl Source {
         let end = o.off + item.len;
         let aligned = (dst as u64).wrapping_sub(o.off).is_multiple_of(a) && (end.is_multiple_of(a) || item.part_end);
         let direct = match (mode, &fds.direct) {
-            (Mode::Cache, Some(f)) if aligned && !cached(&fds.buffered, o.off, item.len).unwrap_or(false) => f,
+            (Mode::Cache, Some(f)) if aligned && !cached(&fds, o.off, item.len) => f,
             _ => {
                 read::read_into(lease, o, buf)?;
                 let t = if mode == Mode::Buffered { &tally.buffered_bytes } else { &tally.cached_bytes };