tensorfs 0.3.73__tar.gz → 0.3.75__tar.gz

{tensorfs-0.3.73 → tensorfs-0.3.75}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.73"
+version = "0.3.75"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.73"
+version = "0.3.75"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.73 → tensorfs-0.3.75}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.73"
+version = "0.3.75"
 edition = "2021"

{tensorfs-0.3.73 → tensorfs-0.3.75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.73
+Version: 0.3.75
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.73 → tensorfs-0.3.75}/crates/tensorfs-core/src/bin/tfs/fetch.rs

@@ -192,9 +192,8 @@ fn deadline_of(flags: &Flags) -> Deadline {
 
 /// The ONE credential spelling, reaching the child explicitly (tfs-050): `--credential-file
 /// <path>` or the deliberately-set `TFS_CREDENTIAL` — never argv (world-readable in /proc),
-/// never the inherited environment at large. The value is `bearer <token>`, scoped to
-/// exactly the named hosts, or empty for anonymous — which is what a rented pod passes,
-/// the hub's closure and presign routes taking no credential at all.
+/// never the inherited environment at large. The value is `bearer <token>` or
+/// `worker <id> <token>`, scoped to exactly the named hosts, or empty for anonymous.
 fn credential_of(
     flags: &Flags,
     hosts: Vec<String>,

{tensorfs-0.3.73 → tensorfs-0.3.75}/crates/tensorfs-core/src/bin/tfs/main.rs

@@ -39,9 +39,10 @@
 //!   tfs fill <root> <plan>       resumable fill: skip only verified objects
 //!
 //! The ONE transport (tfs-048/tfs-049/tfs-050). The credential is `--credential-file` or
-//! TFS_CREDENTIAL (`bearer <token>`, or empty for anonymous), never bare argv; a
-//! URL of `-` arrives on stdin. Flags: --hub <base> --lane --session --streams <n>
-//! --streams-start <n> --timeout <s> --allow-hosts a,b,.c --allow-local --max-redirects <n>.
+//! TFS_CREDENTIAL (`bearer <token>`, `worker <id> <token>`, or empty for anonymous), never
+//! bare argv; a URL of `-` arrives on stdin. Flags: --hub <base> --lane --session --streams <n>
+//! --streams-start <n> --timeout <s> --allow-hosts a,b,.c --allow-local --max-redirects <n>
+//! --ca-file <pem> (a private hub's CA, trusted beside the webpki set).
 //! `--streams` is the most objects in flight (default 128, memory-bounded); the walk starts
 //! at `--streams-start` (default 16) and doubles while measured goodput follows.
 //! `tfs fetch` reads and backfills the Store's bound repo cache between the local Store
@@ -202,6 +203,17 @@ fn accept_printed_ids(args: Vec<String>) -> Vec<String> {
 
 fn main() -> ExitCode {
     let (args, flags) = split_flags(std::env::args().skip(1).collect());
+    if let Some(path) = flag(&flags, "ca-file") {
+        let trusted = fs::read(path)
+            .map_err(|e| format!("IO_FAILED: read {path}: {e}"))
+            .and_then(|pem| {
+                tensorfs_core::transport::trust_roots(&pem).map_err(|r| format!("{r} ({path})"))
+            });
+        if let Err(refusal) = trusted {
+            eprintln!("REFUSED {refusal}");
+            return ExitCode::FAILURE;
+        }
+    }
     let args = accept_printed_ids(args);
     let a: Vec<&str> = args.iter().map(|s| s.as_str()).collect();
     match a.as_slice() {

{tensorfs-0.3.73 → tensorfs-0.3.75}/crates/tensorfs-core/src/err.rs

@@ -189,6 +189,7 @@ codes![
     REDIRECT_REFUSED,
     HUB_UNREACHABLE,
     HUB_REFUSED,
+    TLS_UNTRUSTED,
     REF_NOT_FOUND,
     CREDENTIAL_REQUIRED,
     TRANSFER_FAILED,

{tensorfs-0.3.73 → tensorfs-0.3.75}/crates/tensorfs-core/src/transport/http.rs

@@ -30,10 +30,11 @@
 use crate::err::{refuse, Code, Refusal, Result};
 use crate::transport::ledger::{Deadline, Ledger};
 use crate::transport::policy::CheckedUrl;
+use rustls::pki_types::pem::PemObject;
 use rustls::pki_types::{CertificateDer, ServerName};
 use std::io::{ErrorKind, Read, Write};
 use std::net::TcpStream;
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, Mutex, OnceLock};
 use std::time::{Duration, Instant};
 
 const HEAD_MAX_BYTES: usize = 64 * 1024;
@@ -211,13 +212,35 @@ struct Idle {
     wire: Wire,
 }
 
+/// Authorities this process trusts beside the webpki set: a private hub's CA, named by its
+/// caller as a location (`tfs --ca-file`). Set once, before the first exchange.
+static EXTRA_ROOTS: OnceLock<Vec<CertificateDer<'static>>> = OnceLock::new();
+
+/// Trust the PEM certificates in `pem` for every client this process builds.
+pub fn trust_roots(pem: &[u8]) -> Result<usize> {
+    let roots: Vec<_> = CertificateDer::pem_slice_iter(pem)
+        .collect::<std::result::Result<_, _>>()
+        .map_err(|e| Refusal {
+            code: Code::UNKNOWN_FORMAT,
+            detail: format!("not a PEM certificate bundle: {e}"),
+        })?;
+    let (usable, _) = rustls::RootCertStore::empty().add_parsable_certificates(roots.clone());
+    if roots.is_empty() || usable != roots.len() {
+        return refuse(Code::UNKNOWN_FORMAT, "holds no usable CA certificate");
+    }
+    EXTRA_ROOTS.set(roots).map_err(|_| Refusal {
+        code: Code::UNKNOWN_FORMAT,
+        detail: "this process already chose its trust roots".into(),
+    })?;
+    Ok(usable)
+}
+
 impl Client {
     pub fn new() -> Client {
-        Client::with_extra_roots(&[])
+        Client::with_extra_roots(EXTRA_ROOTS.get().map_or(&[], Vec::as_slice))
     }
 
-    /// `extra_roots` exists for tests that stand up a loopback TLS origin under a
-    /// self-signed authority; production callers pass none and trust the webpki set.
+    /// Trusts the webpki set plus `extra_roots`.
     pub fn with_extra_roots(extra_roots: &[CertificateDer<'static>]) -> Client {
         let mut roots = rustls::RootCertStore::empty();
         roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
@@ -482,7 +505,9 @@ fn connect(
     )
 }
 
+#[allow(clippy::too_many_arguments)]
 fn tls_handshake(
+    host: &str,
     sock: &mut TcpStream,
     conn: &mut rustls::ClientConnection,
     deadline: Deadline,
@@ -516,10 +541,18 @@ fn tls_handshake(
             Ok(0) => return refuse(fail, "origin closed the connection mid-handshake"),
             Ok(_) => {
                 advanced = Instant::now();
-                conn.process_new_packets().map_err(|e| Refusal {
-                    code: fail,
-                    detail: format!("TLS refused: {e}"),
-                })?;
+                // A certificate or protocol verdict is this client's, and re-asking reaches
+                // it again: refused typed, never retried as weather. The alert goes out so
+                // the origin logs why.
+                if let Err(e) = conn.process_new_packets() {
+                    let _ = conn.write_tls(sock);
+                    return refuse(
+                        Code::TLS_UNTRUSTED,
+                        format!(
+                            "{host}: TLS refused: {e}; a private hub names its CA with --ca-file"
+                        ),
+                    );
+                }
             }
             Err(e) if would_block(&e) => {
                 stream_verdict(judge, ledger, advanced, "TLS handshake")?;
@@ -1041,7 +1074,16 @@ pub fn send<'l>(
                 })?;
             let mut sock = sock;
             let mut conn = Box::new(conn);
-            tls_handshake(&mut sock, &mut conn, deadline, ledger, judge, opened, fail)?;
+            tls_handshake(
+                &checked.host,
+                &mut sock,
+                &mut conn,
+                deadline,
+                ledger,
+                judge,
+                opened,
+                fail,
+            )?;
             Wire::Tls { sock, conn }
         } else {
             Wire::Plain(sock)

{tensorfs-0.3.73 → tensorfs-0.3.75}/crates/tensorfs-core/src/transport/hub.rs

@@ -36,9 +36,10 @@
 //! **The credential arrives as a provider trait, never a credential type**, so one
 //! implementation serves whatever a caller holds. It is asked per host, so a credential
 //! stays scoped to the hosts its provider names and cannot leak onto a presigned object
-//! host or a redirect target. A rented pod holds nothing at all: the hub's closure and
-//! presign routes are PUBLIC (tensorhub owner ruling 2026-09-03, which deleted the signed
-//! DownloadDelegation these calls used to present), so the pod fetches as `Anonymous`.
+//! host or a redirect target. Published checkpoints need none; a machine reads its
+//! owner's unpublished checkpoints by presenting its worker capability
+//! (`worker <id> <token>` → `x-cozy-worker-id` + `x-cozy-worker-token`), which the hub
+//! scopes to the machine owner's account.
 
 use crate::canon::{self, as_arr, as_obj, as_str, as_uint, Value};
 use crate::err::{refuse, Code, Refusal, Result};
@@ -103,9 +104,10 @@ impl CredentialProvider for ScopedHeaders {
 /// The ONE credential spelling every surface passes (tfs-050): exactly one scoped value,
 /// handed explicitly — a file, one deliberately-set variable — never inherited.
 ///
-/// - `bearer <token>` → `authorization: Bearer <token>`
-/// - empty            → anonymous, which is what a rented pod uses: the hub's closure and
-///   presign routes take no credential.
+/// - `bearer <token>`      → `authorization: Bearer <token>`
+/// - `worker <id> <token>` → `x-cozy-worker-id` + `x-cozy-worker-token`: a machine's
+///   worker capability, which the hub's closure and presign read as its owner's grant
+/// - empty                 → anonymous: published checkpoints only.
 pub fn credential_from_spec(spec: &str, hosts: Vec<String>) -> Result<ScopedHeaders> {
     let spec = spec.trim();
     if spec.is_empty() {
@@ -114,16 +116,20 @@ pub fn credential_from_spec(spec: &str, hosts: Vec<String>) -> Result<ScopedHead
             headers: Vec::new(),
         });
     }
-    let mut parts = spec.split_whitespace();
-    let headers = match (parts.next(), parts.next()) {
-        (Some("bearer"), Some(token)) if parts.next().is_none() => {
+    let parts: Vec<&str> = spec.split_whitespace().collect();
+    let headers = match parts.as_slice() {
+        ["bearer", token] => {
             vec![("authorization".to_string(), format!("Bearer {token}"))]
         }
+        ["worker", id, token] => vec![
+            ("x-cozy-worker-id".to_string(), id.to_string()),
+            ("x-cozy-worker-token".to_string(), token.to_string()),
+        ],
         _ => {
             return refuse(
                 Code::CREDENTIAL_REQUIRED,
-                "a credential is `bearer <token>` or empty — one scoped value, passed \
-                 explicitly",
+                "a credential is `bearer <token>`, `worker <id> <token>` or empty — one \
+                 scoped value, passed explicitly",
             )
         }
     };

{tensorfs-0.3.73 → tensorfs-0.3.75}/crates/tensorfs-core/src/transport/mod.rs

@@ -49,7 +49,7 @@ mod speculate;
 mod tests;
 
 pub use api::{api_get, fetch_prefix, probe_length, Prefix};
-pub use http::Client;
+pub use http::{trust_roots, Client};
 pub use hub::{
     closure, closure_all, credential_from_spec, presign, Anonymous, Closure, CredentialProvider,
     HostToken, Presigned, ScopedHeaders,

{tensorfs-0.3.73 → tensorfs-0.3.75}/crates/tensorfs-core/src/transport/tests.rs

@@ -3,7 +3,7 @@
 //! no hub. The ledger tests run at millisecond resolution: the cadence is configuration,
 //! the RULE under test is production's.
 
-use super::hub::{Anonymous, HostToken};
+use super::hub::{credential_from_spec, Anonymous, HostToken};
 use super::ledger::{Deadline, Ledger};
 use super::policy::{self, blocked, SourcePolicy};
 use super::pull::{absolute_location, fetch_url, pull, ObjectSource, PullRequest};
@@ -2134,6 +2134,56 @@ fn credential_headers_reach_the_hub_and_not_the_object_host() {
     let _ = std::fs::remove_dir_all(root);
 }
 
+#[test]
+fn a_worker_capability_reaches_the_hub_calls_as_its_two_headers() {
+    let root = temporary("worker-cred");
+    let store = Store::init(&root).unwrap();
+    let checkpoint = Arc::new(checkpoint(1, 4 * 1024));
+    let checkpoint_for = Arc::clone(&checkpoint);
+    let base_slot: Arc<Mutex<String>> = Arc::default();
+    let dispatch_base = Arc::clone(&base_slot);
+    let dispatch: Dispatch = Arc::new(move |request: &Request| {
+        let base = dispatch_base.lock().unwrap().clone();
+        let machine = request.header("x-cozy-worker-id") == Some("om-machine")
+            && request.header("x-cozy-worker-token") == Some("worker-token");
+        match (request.method.as_str(), request.path.as_str()) {
+            ("POST", _) if !machine => status(
+                401,
+                br#"{"error":{"code":"worker.unauthorized","message":"m"}}"#,
+            ),
+            ("POST", "/v1/tensorfs/closure") => ok_json(&closure_value(&checkpoint_for, &base)),
+            ("POST", "/v1/tensorfs/presign") => ok_json(&presign_value(&request.body, &base)),
+            ("GET", path) => match checkpoint_for.bodies.get(&path["/obj/".len()..]) {
+                Some(body) => status(200, body),
+                None => status(404, b"absent"),
+            },
+            _ => status(404, b"no route"),
+        }
+    });
+    let origin = origin(dispatch);
+    *base_slot.lock().unwrap() = origin.base.clone();
+    let policy = SourcePolicy::default();
+    let credential =
+        credential_from_spec("worker om-machine worker-token", vec!["127.0.0.1".into()]).unwrap();
+    let mut request = PullRequest::new(&store, &origin.base, "acme/model", &credential, &policy);
+    request.sample_seconds = 0.025;
+    pull(&request).unwrap();
+    let anonymous_root = temporary("worker-cred-anon");
+    let anonymous = Store::init(&anonymous_root).unwrap();
+    let mut refused = PullRequest::new(&anonymous, &origin.base, "acme/model", &Anonymous, &policy);
+    refused.sample_seconds = 0.025;
+    assert!(pull(&refused)
+        .unwrap_err()
+        .detail
+        .contains("worker.unauthorized"));
+    for spec in ["worker om-machine", "worker a b c", "machine a b"] {
+        let refusal = credential_from_spec(spec, vec![]).err().unwrap();
+        assert_eq!(refusal.code, Code::CREDENTIAL_REQUIRED, "{spec}");
+    }
+    let _ = std::fs::remove_dir_all(root);
+    let _ = std::fs::remove_dir_all(anonymous_root);
+}
+
 // ---------------------------------------------------------------- the predicate
 
 #[test]

tensorfs-0.3.75/crates/tensorfs-core/tests/hub_tls.rs

@@ -0,0 +1,124 @@
+//! A hub whose certificate this machine does not trust is a verdict, not weather: `tfs fetch`
+//! refuses it typed on the first handshake, and `--ca-file` names the private CA that makes
+//! the same hub reachable.
+
+use std::io::{Read, Write};
+use std::net::{SocketAddr, TcpListener, TcpStream};
+use std::process::Command;
+use std::sync::{mpsc, Arc};
+
+const ENVELOPE: &str =
+    r#"{"error":{"code":"checkpoint_not_found","message":"no such checkpoint"}}"#;
+
+/// A loopback hub under a self-signed authority that answers every call it can read with a
+/// typed 404. Every accepted peer is reported before its handshake.
+fn private_hub() -> (u16, String, mpsc::Receiver<SocketAddr>) {
+    let key = rcgen::generate_simple_self_signed(vec!["127.0.0.1".into()]).unwrap();
+    let cert = rustls::pki_types::CertificateDer::from(key.cert.der().to_vec());
+    let private = rustls::pki_types::PrivateKeyDer::try_from(key.key_pair.serialize_der())
+        .unwrap()
+        .clone_key();
+    let config = rustls::ServerConfig::builder_with_provider(Arc::new(
+        rustls::crypto::ring::default_provider(),
+    ))
+    .with_safe_default_protocol_versions()
+    .unwrap()
+    .with_no_client_auth()
+    .with_single_cert(vec![cert], private)
+    .unwrap();
+    let config = Arc::new(config);
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let port = listener.local_addr().unwrap().port();
+    let (peers, accepted) = mpsc::channel();
+    std::thread::spawn(move || {
+        for socket in listener.incoming() {
+            let Ok(socket) = socket else { return };
+            let _ = peers.send(socket.peer_addr().unwrap());
+            let conn = rustls::ServerConnection::new(Arc::clone(&config)).unwrap();
+            let mut stream = rustls::StreamOwned::new(conn, socket);
+            let mut head = Vec::new();
+            let mut chunk = [0u8; 4096];
+            while !head.windows(4).any(|w| w == b"\r\n\r\n") {
+                match stream.read(&mut chunk) {
+                    Ok(n) if n > 0 => head.extend_from_slice(&chunk[..n]),
+                    _ => break,
+                }
+            }
+            if head.windows(4).any(|w| w == b"\r\n\r\n") {
+                let answer = format!(
+                    "HTTP/1.1 404 Not Found\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{ENVELOPE}",
+                    ENVELOPE.len()
+                );
+                let _ = stream.write_all(answer.as_bytes());
+                stream.conn.send_close_notify();
+                let _ = stream.flush();
+            }
+        }
+    });
+    (port, key.cert.pem(), accepted)
+}
+
+/// How many connections reached the hub before this call: a sentinel connection is accepted
+/// after every one queued ahead of it.
+fn connections(port: u16, accepted: &mpsc::Receiver<SocketAddr>) -> usize {
+    let sentinel = TcpStream::connect(("127.0.0.1", port)).unwrap();
+    let mine = sentinel.local_addr().unwrap();
+    accepted.iter().take_while(|peer| *peer != mine).count()
+}
+
+fn fetch(root: &std::path::Path, port: u16, extra: &[&str]) -> (bool, String) {
+    let out = Command::new(env!("CARGO_BIN_EXE_tfs"))
+        .args(["fetch", root.to_str().unwrap(), "proof/model"])
+        .args(["--hub", &format!("https://127.0.0.1:{port}")])
+        .args(extra)
+        .env("TFS_CREDENTIAL", "")
+        .output()
+        .unwrap();
+    (
+        out.status.success(),
+        String::from_utf8_lossy(&out.stderr).into_owned(),
+    )
+}
+
+#[test]
+fn an_untrusted_hub_is_refused_once_and_a_named_ca_reaches_it() {
+    let dir = std::env::temp_dir().join(format!("tfs-hub-tls-{}", std::process::id()));
+    let _ = std::fs::remove_dir_all(&dir);
+    std::fs::create_dir_all(&dir).unwrap();
+    let root = dir.join("store");
+    let init = Command::new(env!("CARGO_BIN_EXE_tfs"))
+        .args(["store", "init", root.to_str().unwrap()])
+        .output()
+        .unwrap();
+    assert!(
+        init.status.success(),
+        "{}",
+        String::from_utf8_lossy(&init.stderr)
+    );
+    let (port, pem, accepted) = private_hub();
+
+    let (ok, stderr) = fetch(&root, port, &[]);
+    assert!(
+        !ok && stderr.contains("REFUSED TLS_UNTRUSTED") && stderr.contains("127.0.0.1"),
+        "{stderr}"
+    );
+    assert_eq!(
+        connections(port, &accepted),
+        1,
+        "an untrusted hub was asked again: {stderr}"
+    );
+
+    let ca = dir.join("hub-ca.pem");
+    std::fs::write(&ca, pem).unwrap();
+    let (ok, stderr) = fetch(&root, port, &["--ca-file", ca.to_str().unwrap()]);
+    assert!(
+        !ok && stderr.contains("REFUSED REF_NOT_FOUND") && stderr.contains("checkpoint_not_found"),
+        "{stderr}"
+    );
+
+    let garbage = dir.join("garbage.pem");
+    std::fs::write(&garbage, "not a certificate").unwrap();
+    let (ok, stderr) = fetch(&root, port, &["--ca-file", garbage.to_str().unwrap()]);
+    assert!(!ok && stderr.contains("REFUSED UNKNOWN_FORMAT"), "{stderr}");
+    let _ = std::fs::remove_dir_all(&dir);
+}

{tensorfs-0.3.73 → tensorfs-0.3.75}/crates/tensorfs-py/src/store.rs

@@ -3590,9 +3590,9 @@ pub fn transfer_streams(wanted: usize) -> usize {
 /// plan against the real store, the 16-way walk presigning `wanted` as it reaches it, and
 /// the completion proof, all in the compiled core.
 ///
-/// `credential` is the one spelling (`bearer <token>`, or empty for anonymous — which is
-/// what a rented pod uses, the hub's closure and presign routes taking no credential),
-/// scoped to the hub's host. `timeout` is the caller's deadline or None for none —
+/// `credential` is the one spelling (`bearer <token>`, `worker <id> <token>` — a machine's
+/// worker capability, which reads its owner's unpublished checkpoints — or empty for
+/// anonymous, published only), scoped to the hub's host. `timeout` is the caller's deadline or None for none —
 /// nothing below invents a number of seconds. `sample_seconds` is the liveness ledger's
 /// RESOLUTION (configuration, never a verdict); leave it defaulted outside tests.
 #[pyfunction]

{tensorfs-0.3.73 → tensorfs-0.3.75}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "tensorfs"
-version = "0.3.73"
+version = "0.3.75"
 description = "TensorFS — canonical tensor artifact substrate"
 requires-python = ">=3.12"
 classifiers = [

{tensorfs-0.3.73 → tensorfs-0.3.75}/python/tensorfs/_ext.pyi

@@ -278,6 +278,7 @@ class StoreEra(Refusal): ...
 class StoreRootAbsent(Refusal): ...
 class SweepSetIncomplete(Refusal): ...
 class TensorSchemaMismatch(Refusal): ...
+class TlsUntrusted(Refusal): ...
 class TotalBytesCap(Refusal): ...
 class TotalRefsCap(Refusal): ...
 class TrailingBytes(Refusal): ...

{tensorfs-0.3.73 → tensorfs-0.3.75}/python/tensorfs/errors.py

@@ -149,6 +149,7 @@ from ._ext import (
     StoreRootAbsent,
     SweepSetIncomplete,
     TensorSchemaMismatch,
+    TlsUntrusted,
     TotalBytesCap,
     TotalRefsCap,
     TrailingBytes,
@@ -306,6 +307,7 @@ __all__ = [
     "StoreRootAbsent",
     "SweepSetIncomplete",
     "TensorSchemaMismatch",
+    "TlsUntrusted",
     "TotalBytesCap",
     "TotalRefsCap",
     "TrailingBytes",