tensorfs 0.3.46__tar.gz → 0.3.47__tar.gz

{tensorfs-0.3.46 → tensorfs-0.3.47}/Cargo.lock

@@ -421,7 +421,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.46"
+version = "0.3.47"
 dependencies = [
  "fs2",
  "libc",
@@ -435,7 +435,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.46"
+version = "0.3.47"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.46 → tensorfs-0.3.47}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.46"
+version = "0.3.47"
 edition = "2021"

{tensorfs-0.3.46 → tensorfs-0.3.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.46
+Version: 0.3.47
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.46 → tensorfs-0.3.47}/crates/tensorfs-core/Cargo.toml

@@ -11,7 +11,7 @@ path = "src/bin/tfs/main.rs"
 fs2 = "0.4"
 minicbor = { version = "2.3.0", default-features = false, features = ["alloc"] }
 rusqlite = { version = "0.32", features = ["bundled"] }
-rustix = { version = "1.1.4", default-features = false, features = ["fs", "std"] }
+rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "net", "event"] }
 # tfs-049: the ONE transport (src/transport/). rustls is the one TLS stack, used only
 # there; scripts/fence.py refuses it anywhere else so a second mover cannot grow.
 rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }

{tensorfs-0.3.46 → tensorfs-0.3.47}/crates/tensorfs-core/src/transport/http.rs

@@ -82,6 +82,7 @@ fn transfer_verdict(
     since: Instant,
     waiting_for: &str,
 ) -> Result<()> {
+    ledger.check_running()?;
     if judge != Judge::Transfer {
         return Ok(());
     }
@@ -131,6 +132,7 @@ fn transfer_verdict(
 /// its own window and advances only when transport bytes actually move; slow
 /// progress may span arbitrarily many windows without exhausting a wall budget.
 fn stream_verdict(judge: Judge, ledger: &Ledger, advanced: Instant, phase: &str) -> Result<()> {
+    ledger.check_running()?;
     if judge != Judge::Stream || advanced.elapsed() <= ledger.patience() {
         return Ok(());
     }
@@ -271,27 +273,86 @@ impl Wire {
     }
 }
 
-fn connect(checked: &CheckedUrl, deadline: Deadline, fail: Code) -> Result<TcpStream> {
+fn connect(
+    checked: &CheckedUrl,
+    deadline: Deadline,
+    ledger: &Ledger,
+    judge: Judge,
+    fail: Code,
+) -> Result<TcpStream> {
+    use rustix::event::{poll, PollFd, PollFlags, Timespec};
+    use rustix::net::{self, AddressFamily, SocketFlags, SocketType};
     let mut last: Option<std::io::Error> = None;
-    for addr in &checked.addrs {
-        let attempt = match deadline.remaining()? {
-            Some(left) => TcpStream::connect_timeout(addr, left),
-            None => TcpStream::connect(addr),
+    'addresses: for addr in &checked.addrs {
+        ledger.check_running()?;
+        let family = if addr.is_ipv4() {
+            AddressFamily::INET
+        } else {
+            AddressFamily::INET6
         };
-        match attempt {
-            Ok(sock) => {
-                let _ = sock.set_nodelay(true);
-                return Ok(sock);
+        let socket = match net::socket_with(
+            family,
+            SocketType::STREAM,
+            SocketFlags::NONBLOCK | SocketFlags::CLOEXEC,
+            Some(net::ipproto::TCP),
+        ) {
+            Ok(socket) => socket,
+            Err(error) => {
+                last = Some(error.into());
+                continue;
+            }
+        };
+        let since = Instant::now();
+        match net::connect(&socket, addr) {
+            Ok(()) => {}
+            Err(rustix::io::Errno::INPROGRESS) => loop {
+                // Poll one pending connection instead of restarting it at each
+                // sample. Cancellation and existing silence rules remain active
+                // without imposing a new total connection deadline.
+                transfer_verdict(judge, ledger, since, "a TCP connection")?;
+                stream_verdict(judge, ledger, since, "TCP connect")?;
+                let tick = deadline
+                    .remaining()?
+                    .map_or(ledger.sample(), |left| left.min(ledger.sample()));
+                let timeout = Timespec {
+                    tv_sec: tick.as_secs() as _,
+                    tv_nsec: tick.subsec_nanos() as _,
+                };
+                let mut descriptors = [PollFd::new(&socket, PollFlags::OUT)];
+                match poll(&mut descriptors, Some(&timeout)) {
+                    Ok(0) | Err(rustix::io::Errno::INTR) => continue,
+                    Ok(_) => match net::sockopt::socket_error(&socket) {
+                        Ok(Ok(())) => break,
+                        Ok(Err(error)) | Err(error) => {
+                            last = Some(error.into());
+                            continue 'addresses;
+                        }
+                    },
+                    Err(error) => {
+                        last = Some(error.into());
+                        continue 'addresses;
+                    }
+                }
+            },
+            Err(error) => {
+                last = Some(error.into());
+                continue;
             }
-            Err(error) => last = Some(error),
         }
+        let socket: TcpStream = socket.into();
+        socket.set_nonblocking(false).map_err(|error| Refusal {
+            code: fail,
+            detail: format!("configure connected socket: {error}"),
+        })?;
+        let _ = socket.set_nodelay(true);
+        return Ok(socket);
     }
     refuse(
         fail,
         format!(
             "{}: no checked address accepted a connection ({})",
             checked.host,
-            last.map(|e| e.to_string()).unwrap_or_default()
+            last.map(|error| error.to_string()).unwrap_or_default()
         ),
     )
 }
@@ -425,7 +486,11 @@ impl BodyReader<'_> {
     /// One raw read off the wire under the sampling rule: the socket timeout is the
     /// RESOLUTION; the deadline and (when judged) the measured patience carry the verdict.
     fn step(&mut self, buf: &mut [u8]) -> Result<usize> {
+        // Time spent by the consumer hashing or writing the prior chunk is not
+        // network silence. This wait starts when it asks for another chunk.
+        self.advanced = Instant::now();
         loop {
+            self.ledger.check_running()?;
             let tick = match self.deadline.remaining()? {
                 Some(left) => left.min(self.ledger.sample()),
                 None => self.ledger.sample(),
@@ -435,7 +500,9 @@ impl BodyReader<'_> {
                 Ok(n) => {
                     if n > 0 {
                         let now = Instant::now();
-                        self.ledger.taught(now - self.advanced);
+                        if self.judge == Judge::Stream {
+                            self.ledger.taught(now - self.advanced);
+                        }
                         self.advanced = now;
                     }
                     return Ok(n);
@@ -645,8 +712,9 @@ pub fn request<'l>(
     write_silence: Option<Duration>,
     fail: Code,
 ) -> Result<Response<'l>> {
+    ledger.check_running()?;
     let opened = Instant::now();
-    let sock = connect(checked, deadline, fail)?;
+    let sock = connect(checked, deadline, ledger, judge, fail)?;
     let mut wire = if checked.https {
         let name = ServerName::try_from(checked.host.clone()).map_err(|_| Refusal {
             code: Code::SOURCE_NOT_ALLOWED,
@@ -697,7 +765,9 @@ pub fn request<'l>(
 
     let (status, response_headers, spill) =
         read_head(&mut wire, deadline, ledger, judge, opened, fail)?;
-    ledger.taught(opened.elapsed());
+    if judge == Judge::Stream {
+        ledger.taught(opened.elapsed());
+    }
     // A HEAD THAT ARRIVED IS PROGRESS. Without this, the second page of a paginated closure
     // would be judged from the start of the first, and a pull whose hub is merely slow
     // would condemn its own second call for the first call's honest latency.

{tensorfs-0.3.46 → tensorfs-0.3.47}/crates/tensorfs-core/src/transport/ledger.rs

@@ -31,6 +31,7 @@
 //! paid H100s sat at exactly zero for fifteen minutes with neither plane complaining.
 
 use crate::err::{refuse, Code, Result};
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Mutex;
 use std::time::{Duration, Instant};
 
@@ -134,6 +135,7 @@ pub struct Still {
 
 /// The one liveness ledger a pull shares across its streams.
 pub struct Ledger {
+    cancelled: AtomicBool,
     sample: Duration,
     worst_gap: Mutex<Duration>,
     progress: Mutex<Progress>,
@@ -150,6 +152,7 @@ impl Ledger {
     /// in milliseconds and judge by exactly the production rule.
     pub fn with_resolution(sample: Duration) -> Ledger {
         Ledger {
+            cancelled: AtomicBool::new(false),
             sample: sample.max(Duration::from_millis(1)),
             worst_gap: Mutex::new(Duration::ZERO),
             progress: Mutex::new(Progress {
@@ -181,6 +184,23 @@ impl Ledger {
         Instant::now()
     }
 
+    /// A terminal object failure stops the whole pull, including already-open
+    /// HTTP streams. Completed objects remain admitted for the next pull.
+    pub(crate) fn cancel(&self) {
+        self.cancelled.store(true, Ordering::Release);
+    }
+
+    pub(crate) fn check_running(&self) -> Result<()> {
+        if self.cancelled.load(Ordering::Acquire) {
+            refuse(
+                Code::TRANSFER_FAILED,
+                "the pull stopped after another object failed",
+            )
+        } else {
+            Ok(())
+        }
+    }
+
     pub fn sample(&self) -> Duration {
         self.sample
     }

{tensorfs-0.3.46 → tensorfs-0.3.47}/crates/tensorfs-core/src/transport/pull.rs

@@ -207,6 +207,7 @@ pub fn fetch_authorized(
     // declared life, which is the only place that question can be answered.
     let mut stale: Option<u16> = None;
     for attempt in 1..=FETCH_ATTEMPTS {
+        ledger.check_running()?;
         let last = attempt == FETCH_ATTEMPTS;
         // A refused re-authorization must still name the origin's answer. It is the whole
         // diagnosis: "403, and re-minting was not honest" and "403, and the hub would not
@@ -604,6 +605,7 @@ pub(super) struct Leases<'a> {
 
 impl ObjectUrls for Leases<'_> {
     fn url_for(&self, at: usize, object: &ObjectRef, stale: bool) -> Result<String> {
+        self.ledger.check_running()?;
         if stale && !self.retire(&object.sha256)? {
             // The refused lease was NOT spent: the hub's own declared life still had more
             // than half of it left, so the origin's refusal is not about time and asking to
@@ -623,6 +625,7 @@ impl ObjectUrls for Leases<'_> {
             return Ok(url);
         }
         let _minting = self.minting.lock().unwrap();
+        self.ledger.check_running()?;
         // Another stream may have minted this window while this one waited for the right
         // to call. Asking again before calling is what makes one window cost one call.
         if let Some(url) = self.fresh(&object.sha256) {
@@ -773,7 +776,7 @@ pub struct WalkOutcome {
 
 /// Walk `plan.wanted` with `streams` objects in flight. One is the serial walk and it
 /// stays reachable, because a link where concurrency does not help is a real link. The
-/// first refusal wins; streams already moving finish their object and stop.
+/// first refusal wins and cancels in-flight network waits. Verified objects remain local.
 ///
 /// **`cache` is the source order, and it is the whole reason a second pod is cheap.** The
 /// order is local Store, then the optional mounted cache, then the origin — the Store half
@@ -885,11 +888,14 @@ pub fn fetch_wanted(
                         }
                     }
                     Err(refusal) => {
-                        stop.store(true, Ordering::Relaxed);
                         let mut slot = first.lock().unwrap();
                         if slot.is_none() {
+                            // Keep the actual failure before waking peers, so their
+                            // cancellation errors cannot replace its diagnosis.
                             *slot = Some(refusal);
+                            ledger.cancel();
                         }
+                        stop.store(true, Ordering::Relaxed);
                         return;
                     }
                 }

{tensorfs-0.3.46 → tensorfs-0.3.47}/crates/tensorfs-core/src/transport/tests.rs

@@ -4094,3 +4094,334 @@ fn downloaded_cache_root_is_protected_then_evicted_and_refetched() {
     assert!(store.repository_path(&repository).exists());
     let _ = std::fs::remove_dir_all(root);
 }
+
+#[test]
+fn failed_object_cancels_stalled_siblings_and_preserves_completed_bytes() {
+    use std::collections::BTreeMap;
+    let root = temporary("failed-sibling");
+    let store = Store::init(&root).unwrap();
+    let good_bytes = vec![b'g'; 4096];
+    let good = ObjectRef::of(&good_bytes);
+    let failed = ObjectRef::of(b"missing object");
+    let stalled = ObjectRef::of(b"unfinished object");
+    store
+        .put_stream(
+            &mut good_bytes.as_slice(),
+            Some(&good),
+            &crate::store::Fault::default(),
+        )
+        .unwrap();
+    let ledger = Arc::new(Ledger::with_resolution(Duration::from_millis(20)));
+    ledger.taught(Duration::from_secs(5));
+    let progress = Arc::clone(&ledger);
+    let failed_path = format!("/{}", failed.sha256);
+    let peer = origin(Arc::new(move |request| {
+        if request.path == failed_path {
+            let until = std::time::Instant::now() + Duration::from_secs(2);
+            while progress.still().moved == 0 {
+                assert!(
+                    std::time::Instant::now() < until,
+                    "peer fixture never started"
+                );
+                std::thread::sleep(Duration::from_millis(1));
+            }
+            return status(404, b"missing object");
+        }
+        Serve::Stall {
+            total: 17,
+            prefix: b"unfinished".to_vec(),
+        }
+    }));
+    let mut objects = vec![good.clone(), failed.clone(), stalled.clone()];
+    objects.sort_by(|a, b| a.sha256.cmp(&b.sha256));
+    let (plan, _) = FetchPlan::of_objects(&store, "failed-sibling", &objects).unwrap();
+    let urls: BTreeMap<String, String> = objects
+        .iter()
+        .map(|object| {
+            (
+                object.sha256.clone(),
+                format!("{}/{}", peer.base, object.sha256),
+            )
+        })
+        .collect();
+    let started = std::time::Instant::now();
+    let refusal = super::pull::fetch_wanted(
+        &store,
+        &plan,
+        &urls,
+        &local_policy(),
+        &Anonymous,
+        Deadline::after_seconds(Some(3.0)),
+        &ledger,
+        2,
+        None,
+    )
+    .unwrap_err();
+    assert!(
+        refusal.detail.contains("404"),
+        "original refusal lost: {refusal}"
+    );
+    assert!(
+        started.elapsed() < Duration::from_secs(1),
+        "failed pull waited on its stalled sibling: {:?}",
+        started.elapsed()
+    );
+    assert!(store.record_valid(&good.sha256).is_ok());
+    assert!(store.record_valid(&stalled.sha256).is_err());
+    let (retry, _) = FetchPlan::of_objects(&store, "retry-failed-sibling", &objects).unwrap();
+    assert_eq!(retry.held, vec![good]);
+    assert_eq!(retry.wanted.len(), 2);
+    let _ = std::fs::remove_dir_all(root);
+}
+
+#[test]
+fn complete_content_length_body_does_not_wait_for_connection_close() {
+    let root = temporary("complete-keepalive");
+    let store = Store::init(&root).unwrap();
+    let bytes = b"complete bounded body";
+    let object = ObjectRef::of(bytes);
+    let (plan, _) =
+        FetchPlan::of_objects(&store, "keepalive", std::slice::from_ref(&object)).unwrap();
+    let grant = DeliveryGrant::mint(&plan, &object).unwrap();
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let address = listener.local_addr().unwrap();
+    let (close_peer, hold_open) = std::sync::mpsc::channel();
+    let peer = std::thread::spawn(move || {
+        let (mut socket, _) = listener.accept().unwrap();
+        read_request(&mut socket).unwrap();
+        write!(
+            socket,
+            "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: keep-alive\r\n\r\n",
+            bytes.len()
+        )
+        .unwrap();
+        socket.write_all(bytes).unwrap();
+        // Keep TCP open until the complete Store admission returns. Disk fsync
+        // time is unrelated to HTTP framing and may exceed a test socket timer.
+        let _ = hold_open.recv();
+    });
+    let ledger = Ledger::with_resolution(Duration::from_millis(20));
+    let result = fetch_url(
+        &store,
+        &grant,
+        &format!("http://{address}/body"),
+        &local_policy(),
+        &Anonymous,
+        Deadline::after_seconds(Some(1.0)),
+        &ledger,
+        None,
+    );
+    close_peer.send(()).unwrap();
+    peer.join().unwrap();
+    let result = result.unwrap();
+    assert_eq!(result.admitted.object, object);
+    assert!(store.record_valid(&object.sha256).is_ok());
+    let _ = std::fs::remove_dir_all(root);
+}
+
+#[test]
+fn slow_control_answer_does_not_teach_object_stream_patience() {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let address = listener.local_addr().unwrap();
+    let peer = std::thread::spawn(move || {
+        let (mut socket, _) = listener.accept().unwrap();
+        read_request(&mut socket).unwrap();
+        std::thread::sleep(Duration::from_millis(200));
+        socket
+            .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\n{}")
+            .unwrap();
+    });
+    let ledger = Ledger::with_resolution(Duration::from_millis(10));
+    let checked = local_policy()
+        .check(&format!("http://{address}/presign"))
+        .unwrap();
+    let response = super::http::request(
+        &super::http::Client::new(),
+        &checked,
+        "GET",
+        &[],
+        None,
+        Deadline::after_seconds(Some(1.0)),
+        &ledger,
+        super::http::Judge::Transfer,
+        None,
+        Code::HUB_UNREACHABLE,
+    )
+    .unwrap();
+    assert_eq!(response.read_capped(2).unwrap(), b"{}");
+    peer.join().unwrap();
+    assert_eq!(
+        ledger.patience(),
+        ledger.floor(),
+        "Hub latency inflated the unrelated object-stream timeout"
+    );
+}
+
+#[test]
+fn consumer_pause_does_not_teach_object_stream_patience() {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let address = listener.local_addr().unwrap();
+    let (resume, paused) = std::sync::mpsc::channel();
+    let peer = std::thread::spawn(move || {
+        let (mut socket, _) = listener.accept().unwrap();
+        read_request(&mut socket).unwrap();
+        socket
+            .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\na")
+            .unwrap();
+        paused.recv().unwrap();
+        socket.write_all(b"b").unwrap();
+    });
+    let ledger = Ledger::with_resolution(Duration::from_millis(10));
+    let checked = local_policy()
+        .check(&format!("http://{address}/object"))
+        .unwrap();
+    let mut body = super::http::request(
+        &super::http::Client::new(),
+        &checked,
+        "GET",
+        &[],
+        None,
+        Deadline::after_seconds(Some(1.0)),
+        &ledger,
+        super::http::Judge::Stream,
+        None,
+        Code::TRANSFER_FAILED,
+    )
+    .unwrap()
+    .into_body();
+    let mut byte = [0];
+    body.read_exact(&mut byte).unwrap();
+    assert_eq!(&byte, b"a");
+    // Simulate time spent hashing or writing between successive body reads.
+    std::thread::sleep(Duration::from_millis(200));
+    resume.send(()).unwrap();
+    body.read_exact(&mut byte).unwrap();
+    assert_eq!(&byte, b"b");
+    peer.join().unwrap();
+    assert!(
+        ledger.patience() < Duration::from_millis(200),
+        "consumer work inflated the unrelated object-stream timeout"
+    );
+}
+
+#[test]
+fn pull_cancellation_interrupts_tls_headers_and_partial_body_waits() {
+    for phase in ["tls", "headers", "body"] {
+        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+        let address = listener.local_addr().unwrap();
+        let (entered, waiting) = std::sync::mpsc::channel();
+        let peer = std::thread::spawn(move || {
+            let (mut socket, _) = listener.accept().unwrap();
+            socket
+                .set_read_timeout(Some(Duration::from_secs(2)))
+                .unwrap();
+            let mut input = [0u8; 8192];
+            if phase == "tls" {
+                assert!(socket.read(&mut input).unwrap() > 0);
+            } else {
+                read_request(&mut socket).unwrap();
+                if phase == "body" {
+                    socket
+                        .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 99\r\n\r\nx")
+                        .unwrap();
+                }
+            }
+            entered.send(()).unwrap();
+            while matches!(socket.read(&mut input),Ok(n) if n>0) {}
+        });
+        let ledger = Arc::new(Ledger::with_resolution(Duration::from_millis(20)));
+        ledger.taught(Duration::from_secs(5));
+        let abort = Arc::clone(&ledger);
+        let cancelling = std::thread::spawn(move || {
+            waiting.recv_timeout(Duration::from_secs(1)).unwrap();
+            abort.cancel();
+        });
+        let scheme = if phase == "tls" { "https" } else { "http" };
+        let checked = local_policy()
+            .check(&format!("{scheme}://{address}/object"))
+            .unwrap();
+        let started = std::time::Instant::now();
+        let result = super::http::request(
+            &super::http::Client::new(),
+            &checked,
+            "GET",
+            &[],
+            None,
+            Deadline::after_seconds(Some(1.0)),
+            &ledger,
+            super::http::Judge::Stream,
+            None,
+            Code::TRANSFER_FAILED,
+        );
+        let refusal = match result {
+            Ok(response) => response.read_capped(99).unwrap_err(),
+            Err(refusal) => refusal,
+        };
+        assert!(
+            refusal.detail.contains("another object failed"),
+            "{phase}: {refusal}"
+        );
+        assert!(
+            started.elapsed() < Duration::from_millis(500),
+            "{phase} ignored pull cancellation"
+        );
+        cancelling.join().unwrap();
+        peer.join().unwrap();
+    }
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn pull_cancellation_interrupts_a_pending_tcp_connection() {
+    use rustix::net::{self, AddressFamily, SocketFlags, SocketType};
+    let socket = net::socket_with(
+        AddressFamily::INET,
+        SocketType::STREAM,
+        SocketFlags::CLOEXEC,
+        Some(net::ipproto::TCP),
+    )
+    .unwrap();
+    net::bind(
+        &socket,
+        &std::net::SocketAddrV4::new(std::net::Ipv4Addr::LOCALHOST, 0),
+    )
+    .unwrap();
+    net::listen(&socket, 0).unwrap();
+    let listener: TcpListener = socket.into();
+    let address = listener.local_addr().unwrap();
+    // Fill Linux's one-entry backlog and deliberately never accept it. A second
+    // connection must keep its same nonblocking socket while polling cancellation.
+    let _occupied = TcpStream::connect(address).unwrap();
+    let ledger = Arc::new(Ledger::with_resolution(Duration::from_millis(20)));
+    ledger.taught(Duration::from_secs(5));
+    let abort = Arc::clone(&ledger);
+    let cancelling = std::thread::spawn(move || {
+        std::thread::sleep(Duration::from_millis(50));
+        abort.cancel();
+    });
+    let checked = local_policy()
+        .check(&format!("http://{address}/object"))
+        .unwrap();
+    let started = std::time::Instant::now();
+    let refusal = match super::http::request(
+        &super::http::Client::new(),
+        &checked,
+        "GET",
+        &[],
+        None,
+        Deadline::after_seconds(Some(1.0)),
+        &ledger,
+        super::http::Judge::Stream,
+        None,
+        Code::TRANSFER_FAILED,
+    ) {
+        Err(refusal) => refusal,
+        Ok(_) => panic!("full listener unexpectedly answered"),
+    };
+    assert!(
+        refusal.detail.contains("another object failed"),
+        "{refusal}"
+    );
+    assert!(started.elapsed() < Duration::from_millis(500));
+    cancelling.join().unwrap();
+}

{tensorfs-0.3.46 → tensorfs-0.3.47}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "tensorfs"
-version = "0.3.46"
+version = "0.3.47"
 description = "TensorFS — canonical tensor artifact substrate"
 requires-python = ">=3.12"
 classifiers = [