tensorfs 0.18.2__tar.gz → 0.19.4__tar.gz

{tensorfs-0.18.2 → tensorfs-0.19.4}/Cargo.lock

@@ -244,6 +244,16 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "backon"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef"
+dependencies = [
+ "fastrand",
+ "tokio",
+]
+
 [[package]]
 name = "base16ct"
 version = "0.2.0"
@@ -344,7 +354,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "calcifer"
-version = "0.18.3"
+version = "0.19.8"
 dependencies = [
  "async-trait",
  "axum",
@@ -496,7 +506,7 @@ checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486"
 
 [[package]]
 name = "cli"
-version = "0.18.3"
+version = "0.19.8"
 dependencies = [
  "anstream",
  "anstyle",
@@ -636,7 +646,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
 name = "cozy"
-version = "0.18.3"
+version = "0.19.8"
 dependencies = [
  "calcifer",
  "cli",
@@ -1167,7 +1177,7 @@ dependencies = [
 
 [[package]]
 name = "executor-protocol"
-version = "0.18.3"
+version = "0.19.8"
 dependencies = [
  "base64 0.23.1",
  "pyo3",
@@ -1691,7 +1701,7 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "hub-client"
-version = "0.18.3"
+version = "0.19.8"
 dependencies = [
  "aws-lc-rs",
  "base64 0.23.1",
@@ -2557,7 +2567,7 @@ checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
 
 [[package]]
 name = "platform"
-version = "0.18.3"
+version = "0.19.8"
 dependencies = [
  "flate2",
  "getrandom 0.3.4",
@@ -2630,7 +2640,7 @@ dependencies = [
 
 [[package]]
 name = "protocol"
-version = "0.18.3"
+version = "0.19.8"
 dependencies = [
  "base64 0.23.1",
  "bytes",
@@ -2644,6 +2654,7 @@ dependencies = [
  "reqwest",
  "rustls",
  "saphyr",
+ "saphyr-parser",
  "serde",
  "serde_ignored",
  "serde_json",
@@ -3914,7 +3925,7 @@ dependencies = [
 
 [[package]]
 name = "tensord"
-version = "0.18.3"
+version = "0.19.8"
 dependencies = [
  "async-trait",
  "aws-lc-rs",
@@ -3931,6 +3942,7 @@ dependencies = [
  "hyper",
  "hyper-util",
  "libc",
+ "pep440_rs",
  "platform",
  "protocol",
  "rcgen",
@@ -3963,17 +3975,15 @@ dependencies = [
 
 [[package]]
 name = "tensorfs"
-version = "0.18.2"
+version = "0.19.4"
 dependencies = [
  "async-trait",
+ "backon",
  "base64 0.23.1",
  "bytes",
  "faer",
  "futures",
  "hmac",
- "http-body-util",
- "hyper",
- "hyper-util",
  "libc",
  "libloading 0.9.0",
  "pep440_rs",
@@ -3983,6 +3993,7 @@ dependencies = [
  "rand_chacha",
  "rand_distr",
  "rayon",
+ "reqwest",
  "rustix",
  "rustls",
  "serde",
@@ -3990,8 +4001,6 @@ dependencies = [
  "sha2",
  "thiserror 2.0.21",
  "tokio",
- "tokio-rustls",
- "tower-service",
  "tracing",
  "tracing-subscriber",
  "webpki-roots",
@@ -3999,7 +4008,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.18.2"
+version = "0.19.4"
 dependencies = [
  "async-trait",
  "executor-protocol",
@@ -4434,7 +4443,7 @@ dependencies = [
 
 [[package]]
 name = "varena"
-version = "0.18.2"
+version = "0.19.4"
 dependencies = [
  "libloading 0.9.0",
  "thiserror 2.0.21",

{tensorfs-0.18.2 → tensorfs-0.19.4}/Cargo.toml

@@ -7,7 +7,7 @@ exclude = ["vendor/faer"]
 members = ["crates/executor-protocol", "crates/platform", "crates/tensorfs", "crates/tensorfs-py", "crates/varena"]
 
 [workspace.package]
-version = "0.18.3"
+version = "0.19.8"
 edition = "2024"
 rust-version = "1.99"
 license = "MIT"
@@ -16,6 +16,7 @@ homepage = "https://github.com/cozy-creator/cozy-creator"
 publish = false
 
 [workspace.dependencies]
+backon = { version = "1.6", default-features = false, features = ["std", "tokio-sleep"] }
 calcifer = { path = "crates/calcifer" }
 cli = { path = "crates/cli" }
 executor-protocol = { path = "crates/executor-protocol" }
@@ -66,6 +67,7 @@ rusqlite = { version = "0.40", features = ["bundled"] }
 rustix = { version = "1.1.4", default-features = false, features = ["std"] }
 rustls = { version = "0.23.46", default-features = false, features = ["aws_lc_rs", "std", "tls12"] }
 saphyr = { version = "0.2.2", default-features = false }
+saphyr-parser = { version = "0.2.2", default-features = false }
 self-replace = "1.5"
 semver = "1"
 serde = { version = "1", features = ["derive"] }

{tensorfs-0.18.2 → tensorfs-0.19.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.18.2
+Version: 0.19.4
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{tensorfs-0.18.2 → tensorfs-0.19.4}/crates/tensorfs/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tensorfs"
-version = "0.18.2"
+version = "0.19.4"
 edition = "2021"
 license.workspace = true
 publish.workspace = true
@@ -17,7 +17,7 @@ required-features = ["store"]
 default = ["store"]
 # `Writer::quantize` and its SVDQ fit: faer and its tree. The wheel's; `cozy` never quantizes.
 quantize = ["store", "dep:faer", "dep:rayon", "dep:pulp", "dep:rand", "dep:rand_chacha", "dep:rand_distr"]
-store = ["dep:hmac", "dep:rustix", "dep:rustls", "dep:webpki-roots", "dep:tokio", "dep:tokio-rustls", "dep:hyper", "dep:hyper-util", "dep:tower-service", "dep:async-trait", "dep:http-body-util", "dep:bytes", "dep:futures", "dep:tracing-subscriber"]
+store = ["dep:hmac", "dep:rustix", "dep:rustls", "dep:webpki-roots", "dep:tokio", "dep:reqwest", "dep:backon", "dep:async-trait", "dep:bytes", "dep:futures", "dep:tracing-subscriber"]
 
 [dependencies]
 # SVDQ producer revision pins its factorization and random-sketch implementation.
@@ -39,18 +39,15 @@ serde_json = { workspace = true, features = ["float_roundtrip"] }
 sha2.workspace = true
 thiserror.workspace = true
 rustix = { workspace = true, features = ["net", "event"], optional = true }
-# The one HTTP client (src/fetch/net.rs). rustls is the one TLS stack, used only there.
+# The one HTTP client (src/fetch/net.rs): reqwest, over rustls with the webpki roots.
+reqwest = { workspace = true, optional = true }
 rustls = { workspace = true, optional = true }
 webpki-roots = { workspace = true, optional = true }
-# Requests in flight are tasks, never a thread each: hyper's pooled HTTP/1.1 client over our
-# connector (egress policy, TLS).
+# Requests in flight are tasks, never a thread each.
 tokio = { workspace = true, optional = true }
-tokio-rustls = { workspace = true, optional = true }
-hyper = { workspace = true, features = ["client", "http1"], optional = true }
-hyper-util = { workspace = true, features = ["client-legacy", "http1"], optional = true }
-tower-service = { workspace = true, optional = true }
+# Exponential backoff with jitter between attempts (src/fetch/ask.rs).
+backon = { workspace = true, optional = true }
 async-trait = { workspace = true, optional = true }
-http-body-util = { workspace = true, optional = true }
 bytes = { workspace = true, optional = true }
 futures = { version = "0.3.31", default-features = false, features = ["std", "executor"], optional = true }
 tracing.workspace = true

{tensorfs-0.18.2 → tensorfs-0.19.4}/crates/tensorfs/README.md

@@ -384,8 +384,8 @@ Public data needs no account. [Distribution](spec/distribution.md) defines endpo
 ## Storage and ownership
 
 The shared local store is internal. Its default root is `~/.cache/tensorfs`; `cache_dir=` and `TENSORFS_HOME` override it. Processes
-share immutable objects and coordinated records. Admission hashes bytes before installing them by rename; ordinary
-opens check lengths. Saved local roots are retained separately from evictable download-cache entries; deleting a header file only removes that handle. Explicit verification rehashes on demand. TensorFS is a library and has no daemon.
+share immutable objects and coordinated records. Admission hashes bytes before installing them by rename; a download
+cut or killed resumes from the bytes it left, which the object's digest still judges; ordinary opens check lengths. Saved local roots are retained separately from evictable download-cache entries; deleting a header file only removes that handle. Explicit verification rehashes on demand. TensorFS is a library and has no daemon.
 
 Runtime and TensorD use this same public API. Callers read and write the shared store and upload with a supplied
 credential. TensorFS owns I/O and decoding; Runtime manages residency of framework-owned buffers. Installing the wheel does

tensorfs-0.19.4/crates/tensorfs/src/fetch/ask.rs

@@ -0,0 +1,125 @@
+//! The one retry rule, for every request (object_store's and the AWS SDKs'): a transport failure (no
+//! connection, 30 s with no bytes, a body cut off) or a "not now" answer is asked again after
+//! backon's exponential backoff, 100 ms doubling to 15 s with jitter, or the wait the answer states.
+//! An attempt that moved bytes starts the count over; [`ATTEMPTS`] in a row that moved nothing end it.
+
+use std::time::Duration;
+
+use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder};
+
+use super::net::{shown, Client, GetError, HeaderMap};
+use super::token::TokenProvider;
+use crate::err::{Code, Refusal};
+
+/// Attempts in a row that moved nothing before a request fails (object_store's `max_retries`).
+pub const ATTEMPTS: usize = 10;
+
+/// An answer that means "not now": 499 is a proxy's own dropped exchange (the Hub's blobs, run
+/// 15391). 501 and 505 say the ask can never be answered.
+pub(crate) fn weather(status: u16) -> bool {
+    matches!(status, 408 | 425 | 429 | 499 | 500..=599) && !matches!(status, 501 | 505)
+}
+
+/// A failure the next attempt may not have.
+pub(crate) fn transient(refusal: &Refusal) -> bool {
+    matches!(refusal.code, Code::TRANSFER_FAILED | Code::HUB_UNREACHABLE | Code::FD_HEADROOM)
+}
+
+/// The pauses between one request's attempts.
+pub(crate) struct Backoff(ExponentialBackoff);
+
+impl Backoff {
+    pub(crate) fn new() -> Backoff {
+        let builder = ExponentialBuilder::default()
+            .with_min_delay(Duration::from_millis(100))
+            .with_factor(2.0)
+            .with_max_delay(Duration::from_secs(15))
+            .with_max_times(ATTEMPTS - 1)
+            .with_jitter();
+        Backoff(builder.build())
+    }
+
+    /// The pause after a failed attempt (`moved`: it gained bytes; `stated`: the answer's
+    /// Retry-After), or None once [`ATTEMPTS`] in a row moved nothing.
+    pub(crate) fn after(&mut self, moved: bool, stated: Option<Duration>) -> Option<Duration> {
+        if moved {
+            *self = Backoff::new();
+        }
+        let pause = self.0.next()?;
+        Some(stated.unwrap_or(pause))
+    }
+}
+
+/// Where [`get`] puts a body: each frame with the byte it starts at.
+pub(crate) type Sink<'a> = dyn FnMut(u64, &[u8]) -> Result<(), Refusal> + Send + 'a;
+
+/// GET `url` (`range`: from, and through) into `sink`, asked again by the rule from the byte reached;
+/// a 200 to an open-ended ask starts the body over. The answer's headers, or the status of an answer
+/// neither data nor weather.
+pub(crate) async fn get(
+    client: &Client,
+    url: &str,
+    headers: &[(String, String)],
+    provider: Option<&dyn TokenProvider>,
+    range: Option<(u64, Option<u64>)>,
+    sink: &mut Sink<'_>,
+) -> Result<Result<HeaderMap, u16>, GetError> {
+    let (mut at, through) = range.unwrap_or((0, None));
+    let mut backoff = Backoff::new();
+    loop {
+        let from = at;
+        let (why, wait) = match attempt(client, url, headers, provider, &mut at, through, sink).await {
+            Ok(Ok(done)) => return Ok(done),
+            Ok(Err((why, wait))) => (why, wait),
+            Err(GetError::Exchange(refusal)) if transient(&refusal) => (refusal.detail, None),
+            Err(error) => return Err(error),
+        };
+        tracing::info!(url = %shown(url), at, %why, "asked again");
+        match backoff.after(at > from, wait) {
+            Some(pause) => tokio::time::sleep(pause).await,
+            None => {
+                let detail = format!("{why}; {ATTEMPTS} attempts in a row moved nothing");
+                return Err(GetError::Exchange(Refusal { code: Code::HUB_UNREACHABLE, detail }));
+            }
+        }
+    }
+}
+
+type Tried = Result<Result<HeaderMap, u16>, (String, Option<Duration>)>;
+
+async fn attempt(
+    client: &Client,
+    url: &str,
+    headers: &[(String, String)],
+    provider: Option<&dyn TokenProvider>,
+    at: &mut u64,
+    through: Option<u64>,
+    sink: &mut Sink<'_>,
+) -> Result<Tried, GetError> {
+    let mut headers = headers.to_vec();
+    if *at > 0 || through.is_some() {
+        let through = through.map_or(String::new(), |t| t.to_string());
+        headers.push(("range".into(), format!("bytes={at}-{through}")));
+    }
+    let mut answer = client.get_authorized(url, &headers, provider).await?;
+    let status = answer.status();
+    match status {
+        _ if weather(status) => {
+            let wait = answer.stated_wait();
+            return Ok(Err((format!("GET {}: {status} {}", shown(url), answer.said().await), wait)));
+        }
+        200 if through.is_none() => *at = 0,
+        206 if answer.content_range().is_some_and(|(first, _)| first == *at) => {}
+        _ => return Ok(Ok(Err(status))),
+    }
+    loop {
+        match answer.frame().await {
+            Ok(Some(frame)) => {
+                sink(*at, &frame).map_err(GetError::Caller)?;
+                *at += frame.len() as u64;
+            }
+            Ok(None) => return Ok(Ok(Ok(answer.response.headers().clone()))),
+            Err(refusal) => return Ok(Err((format!("GET {}: {}", shown(url), refusal.detail), None))),
+        }
+    }
+}