tensorfs 0.3.43__tar.gz → 0.3.44__tar.gz

{tensorfs-0.3.43 → tensorfs-0.3.44}/Cargo.lock

@@ -421,7 +421,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.43"
+version = "0.3.44"
 dependencies = [
  "fs2",
  "libc",
@@ -435,7 +435,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.43"
+version = "0.3.44"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.43 → tensorfs-0.3.44}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.43"
+version = "0.3.44"
 edition = "2021"

{tensorfs-0.3.43 → tensorfs-0.3.44}/PKG-INFO

@@ -1,8 +1,12 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.43
+Version: 0.3.44
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust
+Requires-Dist: torch>=2.11 ; extra == 'torch'
+Requires-Dist: torchao==0.18.0 ; extra == 'torch'
+Requires-Dist: numpy>=1.26 ; extra == 'torch'
+Provides-Extra: torch
 Summary: TensorFS — canonical tensor artifact substrate
 Requires-Python: >=3.12

{tensorfs-0.3.43 → tensorfs-0.3.44}/crates/tensorfs-core/src/transport/http.rs

@@ -21,6 +21,11 @@
 //! exactly zero bytes. A CONTROL-plane exchange now answers to `Judge::Transfer`: the
 //! pull's own progress, which is a measurement, and never to this call's own duration,
 //! which would be a constant.
+//!
+//! Object GETs also apply their existing measured stream-silence rule while
+//! handshaking, writing and reading headers (h3a-070). Previously Judge::Stream
+//! was ignored in those phases, so body retries were unreachable for a silent
+//! TLS peer. Phase progress resets that phase's clock; no object wall limit is added.
 
 use crate::err::{refuse, Code, Refusal, Result};
 use crate::transport::ledger::{Deadline, Ledger};
@@ -122,6 +127,24 @@ fn transfer_verdict(
     )
 }
 
+/// Object-stream silence is judged before the body too. Each phase starts with
+/// its own window and advances only when transport bytes actually move; slow
+/// progress may span arbitrarily many windows without exhausting a wall budget.
+fn stream_verdict(judge: Judge, ledger: &Ledger, advanced: Instant, phase: &str) -> Result<()> {
+    if judge != Judge::Stream || advanced.elapsed() <= ledger.patience() {
+        return Ok(());
+    }
+    refuse(
+        Code::TRANSFER_FAILED,
+        format!(
+            "the object stream stopped during {phase}: silent {:.1} s against the \
+             {:.1} s this pull's measured pace allows",
+            advanced.elapsed().as_secs_f64(),
+            ledger.patience().as_secs_f64(),
+        ),
+    )
+}
+
 pub struct Client {
     tls: Arc<rustls::ClientConfig>,
 }
@@ -283,6 +306,7 @@ fn tls_handshake(
     fail: Code,
 ) -> Result<()> {
     let sample = ledger.sample();
+    let mut advanced = Instant::now();
     while conn.is_handshaking() {
         transfer_verdict(judge, ledger, since, "a TLS handshake")?;
         let tick = match deadline.remaining()? {
@@ -293,8 +317,11 @@ fn tls_handshake(
         let _ = sock.set_write_timeout(Some(tick.max(Duration::from_millis(1))));
         if conn.wants_write() {
             match conn.write_tls(sock) {
+                Ok(n) if n > 0 => advanced = Instant::now(),
                 Ok(_) => {}
-                Err(e) if would_block(&e) => {}
+                Err(e) if would_block(&e) => {
+                    stream_verdict(judge, ledger, advanced, "TLS handshake")?;
+                }
                 Err(e) => return refuse(fail, format!("TLS handshake write: {e}")),
             }
             continue;
@@ -302,12 +329,15 @@ fn tls_handshake(
         match conn.read_tls(sock) {
             Ok(0) => return refuse(fail, "origin closed the connection mid-handshake"),
             Ok(_) => {
+                advanced = Instant::now();
                 conn.process_new_packets().map_err(|e| Refusal {
                     code: fail,
                     detail: format!("TLS refused: {e}"),
                 })?;
             }
-            Err(e) if would_block(&e) => {}
+            Err(e) if would_block(&e) => {
+                stream_verdict(judge, ledger, advanced, "TLS handshake")?;
+            }
             Err(e) => return refuse(fail, format!("TLS handshake read: {e}")),
         }
     }
@@ -733,6 +763,7 @@ fn write_all(
                 Err(e) => return refuse(fail, format!("the connection failed mid-write: {e}")),
             }
         }
+        stream_verdict(judge, ledger, advanced, "request write")?;
         if let Some(bound) = silence {
             if advanced.elapsed() > bound {
                 return refuse(
@@ -811,6 +842,7 @@ fn read_step_ticking(
     since: Instant,
     fail: Code,
 ) -> Result<usize> {
+    let advanced = Instant::now();
     loop {
         transfer_verdict(judge, ledger, since, "a response head")?;
         let tick = match deadline.remaining()? {
@@ -820,7 +852,9 @@ fn read_step_ticking(
         wire.set_read_timeout(tick);
         match wire.read_step(buf) {
             Ok(n) => return Ok(n),
-            Err(e) if would_block(&e) => {}
+            Err(e) if would_block(&e) => {
+                stream_verdict(judge, ledger, advanced, "response head")?;
+            }
             Err(e) => return refuse(fail, format!("the connection failed: {e}")),
         }
     }

{tensorfs-0.3.43 → tensorfs-0.3.44}/crates/tensorfs-core/src/transport/tests.rs

@@ -57,6 +57,8 @@ impl Request {
 }
 
 enum Serve {
+    /// Read the request, then hold the socket without sending response headers.
+    StallHead,
     /// Content-Length framed whole answer.
     Whole {
         status: u16,
@@ -146,6 +148,11 @@ fn handle(mut socket: TcpStream, dispatch: Dispatch, hits: Arc<Mutex<HashMap<Str
         .entry(request.path.clone())
         .or_insert(0) += 1;
     match dispatch(&request) {
+        Serve::StallHead => {
+            let mut sink = [0u8; 16];
+            let _ = socket.set_read_timeout(Some(Duration::from_secs(30)));
+            let _ = socket.read(&mut sink);
+        }
         Serve::Whole {
             status,
             body,
@@ -915,6 +922,120 @@ fn black_holed_stream_is_stalled_by_the_ledger_and_reasked() {
     let _ = std::fs::remove_dir_all(root);
 }
 
+fn prebody_silence_is_judged(https: bool) {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let port = listener.local_addr().unwrap().port();
+    std::thread::spawn(move || {
+        let (mut socket, _) = listener.accept().unwrap();
+        socket
+            .set_read_timeout(Some(Duration::from_secs(3)))
+            .unwrap();
+        // Accept HTTP request bytes or a TLS ClientHello, but never answer.
+        let mut bytes = [0u8; 8192];
+        while matches!(socket.read(&mut bytes), Ok(n) if n > 0) {}
+    });
+    let scheme = if https { "https" } else { "http" };
+    let checked = local_policy()
+        .check(&format!("{scheme}://127.0.0.1:{port}/object"))
+        .unwrap();
+    let ledger = Ledger::with_resolution(Duration::from_millis(25));
+    ledger.moved(1024); // A pull that has already moved bytes, as on the real pod.
+    let response = super::http::request(
+        &super::http::Client::new(),
+        &checked,
+        "GET",
+        &[],
+        None,
+        Deadline::after_seconds(Some(1.0)),
+        &ledger,
+        super::http::Judge::Stream,
+        None,
+        Code::TRANSFER_FAILED,
+    );
+    let refusal = match response {
+        Err(refusal) => refusal,
+        Ok(_) => panic!("silent origin unexpectedly answered"),
+    };
+    assert_eq!(refusal.code, Code::TRANSFER_FAILED, "{refusal}");
+    assert!(
+        refusal.detail.contains(if https {
+            "TLS handshake"
+        } else {
+            "response head"
+        }),
+        "{refusal}"
+    );
+}
+
+#[test]
+fn an_object_stalled_before_headers_is_judged_by_stream_progress() {
+    prebody_silence_is_judged(false);
+}
+
+#[test]
+fn an_object_stalled_during_tls_is_judged_by_stream_progress() {
+    prebody_silence_is_judged(true);
+}
+
+#[test]
+fn a_prebody_stall_is_reasked_without_discarding_completed_objects() {
+    let root = temporary("head-stall-retry");
+    let store = Store::init(&root).unwrap();
+    let checkpoint = Arc::new(checkpoint(2, 64 * 1024));
+    let victim = checkpoint.objects[0].sha256.clone();
+    let selected = victim.clone();
+    let origin = hub_origin(Arc::clone(&checkpoint), move |hex, attempt| {
+        (hex == selected && attempt == 1).then_some(Serve::StallHead)
+    });
+    let policy = SourcePolicy::default();
+    let report = pull(&request_for(&store, &origin.base, &policy)).unwrap();
+    assert_eq!(report.fetched, 4);
+    for object in &checkpoint.objects {
+        assert!(store.record_valid(&object.sha256).is_ok());
+        assert_eq!(
+            origin.hits(&format!("/obj/{}", object.sha256)),
+            if object.sha256 == victim { 2 } else { 1 }
+        );
+    }
+    let _ = std::fs::remove_dir_all(root);
+}
+
+#[test]
+fn fragmented_headers_can_progress_across_multiple_stream_windows() {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let port = listener.local_addr().unwrap().port();
+    std::thread::spawn(move || {
+        let (mut socket, _) = listener.accept().unwrap();
+        read_request(&mut socket).unwrap();
+        socket.write_all(b"HTTP/1.1 200 OK\r\n").unwrap();
+        for _ in 0..10 {
+            std::thread::sleep(Duration::from_millis(50));
+            socket.write_all(b"X-Progress: yes\r\n").unwrap();
+        }
+        socket.write_all(b"Content-Length: 3\r\n\r\nabc").unwrap();
+    });
+    let checked = local_policy()
+        .check(&format!("http://127.0.0.1:{port}/object"))
+        .unwrap();
+    let ledger = Ledger::with_resolution(Duration::from_millis(25));
+    let started = std::time::Instant::now();
+    let response = super::http::request(
+        &super::http::Client::new(),
+        &checked,
+        "GET",
+        &[],
+        None,
+        Deadline::none(),
+        &ledger,
+        super::http::Judge::Stream,
+        None,
+        Code::TRANSFER_FAILED,
+    )
+    .unwrap();
+    assert_eq!(response.read_capped(3).unwrap(), b"abc");
+    assert!(started.elapsed() > ledger.floor() * 2);
+}
+
 #[test]
 fn slow_origin_teaches_the_ledger_and_is_left_alone() {
     let root = temporary("slow");
@@ -923,21 +1044,26 @@ fn slow_origin_teaches_the_ledger_and_is_left_alone() {
     let slow = checkpoint.objects[0].sha256.clone();
     let slow_for = slow.clone();
     let bodies = checkpoint.bodies.clone();
-    // 400 ms to first byte teaches the ledger this origin pauses; the 200 ms drip gaps
-    // that follow are far over the 150 ms noise floor and must NOT be called stalls.
+    // A successful 400 ms first answer teaches this origin's latency; subsequent
+    // 800 ms gaps exceed the initial floor but remain below that measured term.
+    // Unlike the old body-only rule,
+    // initial pre-header silence is now bounded, so this fixture gives the first
+    // answer a sufficient 600 ms floor. Production timing constants are unchanged.
     let origin = hub_origin(Arc::clone(&checkpoint), move |hex, _| {
         if hex == slow_for {
             return Some(Serve::Drip {
                 body: bodies[hex].clone(),
                 chunk: 1024,
-                gap: Duration::from_millis(200),
+                gap: Duration::from_millis(800),
                 ttfb: Duration::from_millis(400),
             });
         }
         None
     });
     let policy = SourcePolicy::default();
-    let report = pull(&request_for(&store, &origin.base, &policy)).unwrap();
+    let mut request = request_for(&store, &origin.base, &policy);
+    request.sample_seconds = 0.1;
+    let report = pull(&request).unwrap();
     assert_eq!(report.fetched, 3);
     assert_eq!(origin.hits(&format!("/obj/{slow}")), 1); // one ask; never condemned
     let _ = std::fs::remove_dir_all(root);

{tensorfs-0.3.43 → tensorfs-0.3.44}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "tensorfs"
-version = "0.3.43"
+version = "0.3.44"
 description = "TensorFS — canonical tensor artifact substrate"
 requires-python = ">=3.12"
 classifiers = [
@@ -13,6 +13,9 @@ classifiers = [
     "Programming Language :: Rust",
 ]
 
+[project.optional-dependencies]
+torch = ["torch>=2.11", "torchao==0.18.0", "numpy>=1.26"]
+
 [tool.maturin]
 manifest-path = "crates/tensorfs-py/Cargo.toml"
 module-name = "tensorfs._ext"

{tensorfs-0.3.43 → tensorfs-0.3.44}/python/tensorfs/__init__.py

@@ -4,8 +4,9 @@ Marshal only. Every byte-level fact — hashing, canonical bytes, geometry, veri
 leases, the reader pool — comes from the extension, and `scripts/fence.py` enforces that
 this package computes none of them. What lives here is types and names.
 
-Torch is OUTSIDE the boundary. A read fills a buffer YOU own and hands back plain data;
-element sizes come from `DTYPES`, which is the compiled dtype enum, exported once::
+The default inference reader returns owned Torch tensors (install `tensorfs[torch]`).
+Torch loads only when tensor access is requested. Storage and metadata remain lightweight.
+The lower-level read interface still fills caller-owned buffers::
 
     import torch, tensorfs
 
@@ -35,7 +36,21 @@ from dataclasses import dataclass
 from typing import Literal, TypedDict, cast
 
 from . import errors
-from .derived import Derivation, OutputCapability, SourceCapability, SourceInspection, derive
+from .derived import (
+    Derivation,
+    OutputCapability,
+    SourceCapability,
+    SourceInspection,
+    derive,
+)
+from ._tensors import (
+    TensorAccessError,
+    TensorAllocation,
+    TensorReader,
+    from_parts,
+    physical_tensors,
+    tensors,
+)
 from ._ext import (
     _fit,
     Batch,
@@ -182,12 +197,19 @@ def fit(
         ),
     )
 
+
 #: The closed carrier/logical dtype enum and its element sizes. The ONE export of dtype
 #: knowledge: a consumer sizing a `torch.frombuffer` reads it from here, never from a
 #: table of its own.
 DTYPES: dict[str, int] = dtypes()
 
 __all__ = [
+    "TensorAccessError",
+    "TensorAllocation",
+    "TensorReader",
+    "from_parts",
+    "physical_tensors",
+    "tensors",
     "Derivation",
     "OutputCapability",
     "SourceCapability",