tensorfs 0.9.0__tar.gz → 0.9.2__tar.gz

{tensorfs-0.9.0 → tensorfs-0.9.2}/Cargo.lock

@@ -559,7 +559,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.9.0"
+version = "0.9.2"
 dependencies = [
  "base64",
  "fs2",
@@ -580,7 +580,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-plane"
-version = "0.9.0"
+version = "0.9.2"
 dependencies = [
  "libc",
  "libloading",
@@ -590,7 +590,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.9.0"
+version = "0.9.2"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.9.0 → tensorfs-0.9.2}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-plane", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.9.0"
+version = "0.9.2"
 edition = "2021"

{tensorfs-0.9.0 → tensorfs-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.9.0
+Version: 0.9.2
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.9.0 → tensorfs-0.9.2}/crates/tensorfs-core/src/ensure.rs

@@ -280,13 +280,9 @@ impl Parts {
                     landing = self.landed.wait(landing).unwrap();
                 }
             };
+            let landed = Landed(self, &mine);
             let pulled = self.pull(request, &mine);
-            let mut landing = self.landing.lock().unwrap();
-            for object in &mine {
-                landing.remove(&object.sha256);
-            }
-            self.landed.notify_all();
-            drop(landing);
+            drop(landed);
             pulled?;
         }
     }
@@ -448,7 +444,8 @@ fn land(
 
     let stop = (Mutex::new(false), Condvar::new());
     let pulled = std::thread::scope(|scope| {
-        scope.spawn(|| {
+        // A sampler the container refuses to start leaves the pull running unsampled.
+        let _ = std::thread::Builder::new().spawn_scoped(scope, || {
             let (mut at, mut last) = (Instant::now(), meter.moved.load(Ordering::Relaxed));
             let mut stopped = stop.0.lock().unwrap();
             while !*stopped {
@@ -466,10 +463,8 @@ fn land(
                 });
             }
         });
-        let pulled = attempts(request, &store, &closure, &counters, &on_plan, &on_object);
-        *stop.0.lock().unwrap() = true;
-        stop.1.notify_all();
-        pulled
+        let _stop = Stop(&stop);
+        attempts(request, &store, closure, &counters, &on_plan, &on_object)
     });
     let report = pulled?;
     record.bytes_done = total;
@@ -518,7 +513,8 @@ fn land(
 fn ticking<T>(every: Duration, tick: &(dyn Fn() + Sync), work: impl FnOnce() -> T) -> T {
     let stop = (Mutex::new(false), Condvar::new());
     std::thread::scope(|scope| {
-        scope.spawn(|| {
+        // A ticker the container refuses to start leaves the work running untold.
+        let _ = std::thread::Builder::new().spawn_scoped(scope, || {
             let mut stopped = stop.0.lock().unwrap();
             loop {
                 stopped = stop.1.wait_timeout(stopped, every).unwrap().0;
@@ -528,13 +524,37 @@ fn ticking<T>(every: Duration, tick: &(dyn Fn() + Sync), work: impl FnOnce() ->
                 tick();
             }
         });
-        let out = work();
-        *stop.0.lock().unwrap() = true;
-        stop.1.notify_all();
-        out
+        let _stop = Stop(&stop);
+        work()
     })
 }
 
+/// Takes a fetch's objects off the landing set as its pull ends, by return or by unwind alike:
+/// the fetches waiting on them never wait forever.
+struct Landed<'a>(&'a Parts, &'a [ObjectRef]);
+
+impl Drop for Landed<'_> {
+    fn drop(&mut self) {
+        let mut landing = self.0.landing.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+        for object in self.1 {
+            landing.remove(&object.sha256);
+        }
+        self.0.landed.notify_all();
+    }
+}
+
+/// Stops a sampling thread as its work ends, by return or by unwind alike: a panic in the
+/// work (a thread the container refused) would otherwise leave its scope waiting forever on
+/// a sampler that never stops, and the pull's flight held (2026-10-15).
+struct Stop<'a>(&'a (Mutex<bool>, Condvar));
+
+impl Drop for Stop<'_> {
+    fn drop(&mut self) {
+        *self.0 .0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = true;
+        self.0 .1.notify_all();
+    }
+}
+
 /// Pull until done. A pull that failed on the link is re-attempted; one that landed nothing
 /// waits a sample first. The ensure is `STALLED` only once no object has landed for the
 /// ledger's own floor: a blip that spent one attempt's asks in milliseconds measured nothing.

{tensorfs-0.9.0 → tensorfs-0.9.2}/crates/tensorfs-core/src/registry.rs

@@ -120,6 +120,7 @@ fn mined(name: &str) -> EncodingVectors {
         }
         "nvfp4-w4a4" => include_bytes!("../../../vectors/mined/nvfp4-w4a4.json"),
         "mxfp8" => include_bytes!("../../../vectors/mined/mxfp8.json"),
+        "int8-convrot256" => include_bytes!("../../../vectors/mined/int8-convrot256.json"),
         other => panic!("no mined vector set {other:?}"),
     };
     EncodingVectors::parse_fixture(bytes).expect("mined vectors are frozen canonical data")
@@ -266,6 +267,23 @@ pub fn seeds() -> Vec<Seed> {
             ],
             Some(mined("mxfp8")),
         ),
+        // int8 W8A8 with ComfyUI's ConvRot: each row is rotated by the orthonormal regular
+        // Hadamard H4(x)H4(x)H4(x)H4 per 256 input channels, then quantized per output row.
+        // value[i,j] = scale[i] * (data[i] . H)[j] / 16, the integer Hadamard sum times the
+        // f32 scale in f32, then the logical cast. Mined from cozy-runtime's own producer.
+        seed(
+            "int8-convrot256/1",
+            float.clone(),
+            Some(2),
+            vec![
+                ("data", role(set(&[Dtype::I8]), Relation::Same)),
+                (
+                    "scale",
+                    role(set(&[Dtype::F32]), dims(vec![Dim::Axis { axis: 0 }])),
+                ),
+            ],
+            Some(mined("int8-convrot256")),
+        ),
         Seed {
             alias: "svdq-microscale/1",
             vectors: None,

{tensorfs-0.9.0 → tensorfs-0.9.2}/crates/tensorfs-core/src/transport/download.rs

@@ -40,7 +40,8 @@ use std::io::Read;
 use std::os::unix::fs::{FileExt as _, OpenOptionsExt};
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
-use std::sync::{Arc, Condvar, Mutex};
+use std::panic::AssertUnwindSafe;
+use std::sync::{Arc, Condvar, Mutex, PoisonError};
 use std::time::{Duration, Instant};
 
 /// One request's bytes. Measured on a pod at 128 workers: 4, 8 and 16 MiB chunks all move
@@ -302,12 +303,31 @@ impl Download<'_> {
             landed: AtomicU64::new(0),
             writers: Writers::new(),
         };
-        std::thread::scope(|scope| {
-            for slot in 0..streams {
-                let run = &run;
-                scope.spawn(move || run.work(slot));
+        let streams = std::thread::scope(|scope| {
+            // A thread the container refuses to start is backpressure, never a panic: the
+            // queue runs on the streams that started, and with none this thread is the walk's
+            // one stream. A stream that panics ends the walk: its chunk would keep its peers
+            // waiting.
+            let mut started = 0;
+            while started < streams {
+                let (run, slot) = (&run, started);
+                let spawned = std::thread::Builder::new().spawn_scoped(scope, move || {
+                    if std::panic::catch_unwind(AssertUnwindSafe(|| run.work(slot))).is_err() {
+                        let refusal = Refusal { code: Code::IO_FAILED, detail: "a stream panicked".into() };
+                        run.fail(&mut run.state.lock().unwrap_or_else(PoisonError::into_inner), refusal);
+                        run.wake.notify_all();
+                    }
+                });
+                if spawned.is_err() {
+                    break;
+                }
+                started += 1;
+            }
+            match started {
+                0 => run.work(0),
+                _ => run.watch(&mut observer),
             }
-            run.watch(&mut observer);
+            started.max(1)
         });
         if let Some(observer) = observer.as_mut() {
             observer.moved(run.landed.load(Ordering::Relaxed));

{tensorfs-0.9.0 → tensorfs-0.9.2}/crates/tensorfs-core/src/unsynced.rs

@@ -167,24 +167,29 @@ fn dead(markers: Vec<Found>) -> Vec<(Found, File)> {
 fn sync_files(paths: &[PathBuf]) -> Result<()> {
     let next = AtomicUsize::new(0);
     let failed: Mutex<Option<Refusal>> = Mutex::new(None);
-    std::thread::scope(|scope| {
-        for _ in 0..SYNC_THREADS.min(paths.len()) {
-            scope.spawn(|| {
-                while let Some(path) = paths.get(next.fetch_add(1, Ordering::Relaxed)) {
-                    match File::open(path).and_then(|file| file.sync_data()) {
-                        Ok(()) => {}
-                        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
-                        Err(error) => {
-                            failed
-                                .lock()
-                                .unwrap_or_else(|error| error.into_inner())
-                                .get_or_insert(io(format!("sync {}", path.display()), error));
-                            return;
-                        }
-                    }
+    let sync = || {
+        while let Some(path) = paths.get(next.fetch_add(1, Ordering::Relaxed)) {
+            match File::open(path).and_then(|file| file.sync_data()) {
+                Ok(()) => {}
+                Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+                Err(error) => {
+                    failed
+                        .lock()
+                        .unwrap_or_else(|error| error.into_inner())
+                        .get_or_insert(io(format!("sync {}", path.display()), error));
+                    return;
                 }
-            });
+            }
+        }
+    };
+    // This thread syncs too, so a thread the container refuses only slows the batch.
+    std::thread::scope(|scope| {
+        for _ in 1..SYNC_THREADS.min(paths.len()) {
+            if std::thread::Builder::new().spawn_scoped(scope, sync).is_err() {
+                break;
+            }
         }
+        sync();
     });
     match failed
         .into_inner()

{tensorfs-0.9.0 → tensorfs-0.9.2}/crates/tensorfs-core/tests/conform.rs

@@ -311,6 +311,51 @@ fn nvfp4_inner(c: &Case, swap: bool, skip_global: bool) -> Option<Vec<u8>> {
     Some(out)
 }
 
+/// int8-convrot256: value[i,j] = f32(sum_k data[i,k] * H[k,j]) * scale[i] / 16 over each
+/// 256-group, H = H4(x)H4(x)H4(x)H4 with H4's -1 where the two base-4 digits sum to 3.
+fn int8_convrot256(c: &Case) -> Option<Vec<u8>> {
+    int8_convrot_inner(c, false, false)
+}
+fn int8_convrot_unrotated(c: &Case) -> Option<Vec<u8>> {
+    int8_convrot_inner(c, true, false)
+}
+fn int8_convrot_scale_by_column(c: &Case) -> Option<Vec<u8>> {
+    int8_convrot_inner(c, false, true)
+}
+
+fn hadamard_sign(k: usize, j: usize) -> i32 {
+    let mut sign = 1;
+    for digit in 0..4 {
+        if (k >> (2 * digit)) % 4 + (j >> (2 * digit)) % 4 == 3 {
+            sign = -sign;
+        }
+    }
+    sign
+}
+
+fn int8_convrot_inner(c: &Case, unrotated: bool, wrong_axis: bool) -> Option<Vec<u8>> {
+    let d = role(c, "data")?;
+    let s = role(c, "scale")?;
+    let rows = *c.logical_shape.first()? as usize;
+    let cols = *c.logical_shape.get(1)? as usize;
+    let mut out = Vec::new();
+    for i in 0..rows {
+        for j in 0..cols {
+            let group = j - j % 256;
+            let q = |k: usize| d.bits[i * cols + group + k] as i8 as i32;
+            let sum = if unrotated {
+                q(j % 256) * 16
+            } else {
+                (0..256).map(|k| q(k) * hadamard_sign(k, j % 256)).sum()
+            };
+            let o = if wrong_axis { (j % rows) * 4 } else { i * 4 };
+            let sc = f32::from_le_bytes(s.bits[o..o + 4].try_into().ok()?);
+            put_logical(c.logical_dtype, sum as f32 * sc * 0.0625, &mut out);
+        }
+    }
+    Some(out)
+}
+
 fn candidates(alias: &str) -> Vec<Candidate> {
     let mk = |name, wrong, decode| Candidate {
         name,
@@ -342,6 +387,11 @@ fn candidates(alias: &str) -> Vec<Candidate> {
             mk("fp8-scaled-scalar:scale-ignored", true, fp8_scaled_no_scale),
             mk("fp8-scaled-scalar:f32-math", true, fp8_scaled_f32_math),
         ],
+        "int8-convrot256/1" => vec![
+            mk("int8-convrot256", false, int8_convrot256),
+            mk("int8-convrot256:unrotated", true, int8_convrot_unrotated),
+            mk("int8-convrot256:scale-by-column", true, int8_convrot_scale_by_column),
+        ],
         "mxfp8/1" => vec![
             mk("mxfp8", false, mxfp8),
             mk("mxfp8:block-16", true, mxfp8_wrong_block),

{tensorfs-0.9.0 → tensorfs-0.9.2}/crates/tensorfs-core/tests/ensure.rs

@@ -977,3 +977,80 @@ fn a_checkpoint_held_in_part_resolves_once_and_pulls_each_object_once() {
     let foreign = vec![ObjectRef::of(b"not of this checkpoint")];
     assert_eq!(fetch(&foreign).unwrap_err().code, tensorfs_core::err::Code::NOT_CONTAINED);
 }
+
+/// A pod's container caps its tasks (a vast RTX 5060 Ti: `pids.max` 1280) below the threads
+/// concurrent pulls start. A stream thread the cap refuses is backpressure: each pull lands on
+/// the streams that started, or on its own thread when none did. On 0.9.0 the refusal panicked
+/// past the pull's sampler and `tfs ensure` never returned, holding the manifest's flight
+/// (TensorD 0.5.30 prepare wedge, 2026-10-15).
+#[test]
+fn concurrent_ensures_whose_stream_threads_the_container_refuses_land() {
+    const TASKS: u64 = 64;
+    let cap = format!("TasksMax={TASKS}");
+    let scope = ["systemd-run", "--user", "--scope", "-q", "-p", cap.as_str()];
+    let usable = Command::new(scope[0])
+        .args(&scope[1..])
+        .arg("true")
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status()
+        .is_ok_and(|status| status.success());
+    if !usable {
+        eprintln!("SKIPPED: no systemd user manager to create a task-limited cgroup");
+        return;
+    }
+    let dir = temporary("tasks");
+    // Two cold pulls of 251 objects each, in one task-limited cgroup: each asks for a stream
+    // per object.
+    let pulls: Vec<(PathBuf, Arc<Checkpoint>, Hub)> = (0..2u8)
+        .map(|at| {
+            let root = dir.join(format!("store-{at}"));
+            Store::init(&root).unwrap();
+            let checkpoint = Arc::new(checkpoint(251, 4096, 9 + at));
+            let hub = hub(Arc::clone(&checkpoint), None, Duration::ZERO, serve_all());
+            (root, checkpoint, hub)
+        })
+        .collect();
+    let script: Vec<String> = pulls
+        .iter()
+        .enumerate()
+        .map(|(at, (root, _, hub))| {
+            let inner = ensure(root, &hub.base, &[]);
+            let args: Vec<String> = inner.get_args().map(|a| format!("'{}'", a.to_str().unwrap())).collect();
+            format!("{TFS} {} > {}/out-{at} &", args.join(" "), dir.display())
+        })
+        .collect();
+    let mut child = Command::new(scope[0])
+        .args(&scope[1..])
+        .args(["sh", "-c", &format!("{} wait", script.join(" "))])
+        .env("TFS_CREDENTIAL", "")
+        .spawn()
+        .unwrap();
+    // Test harness bound on a wedged pull; the product has no such limit.
+    let until = std::time::Instant::now() + Duration::from_secs(90);
+    while child.try_wait().unwrap().is_none() {
+        if std::time::Instant::now() > until {
+            let _ = child.kill();
+            panic!("the ensures never ended under {cap}");
+        }
+        std::thread::sleep(Duration::from_millis(100));
+    }
+    let mut streams = 0;
+    for (at, (root, checkpoint, _)) in pulls.iter().enumerate() {
+        let stdout = std::fs::read_to_string(dir.join(format!("out-{at}"))).unwrap();
+        let last = stdout.lines().last().map(parse);
+        let ok = last.as_ref().is_some_and(|line| text(line, "event") == "ensure.result");
+        let run = Run { ok, lines: stdout.lines().map(parse).collect(), stdout };
+        assert_result(&run, checkpoint);
+        let log = std::fs::read_to_string(root.join("logs/transport.log")).unwrap();
+        let walk = log.lines().rfind(|line| line.contains(" walk ")).unwrap().to_string();
+        streams += walk
+            .split_whitespace()
+            .find_map(|field| field.strip_prefix("streams="))
+            .unwrap()
+            .parse::<u64>()
+            .unwrap();
+    }
+    assert!(streams < 2 * 251, "the cap refused none of their streams");
+    let _ = std::fs::remove_dir_all(&dir);
+}

{tensorfs-0.9.0 → tensorfs-0.9.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "tensorfs"
-version = "0.9.0"
+version = "0.9.2"
 description = "TensorFS — canonical tensor artifact substrate"
 requires-python = ">=3.12"
 classifiers = [