tensorfs 0.3.74__tar.gz → 0.3.75__tar.gz

{tensorfs-0.3.74 → tensorfs-0.3.75}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.74"
+version = "0.3.75"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.74"
+version = "0.3.75"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.74 → tensorfs-0.3.75}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.74"
+version = "0.3.75"
 edition = "2021"

{tensorfs-0.3.74 → tensorfs-0.3.75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.74
+Version: 0.3.75
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.74 → tensorfs-0.3.75}/crates/tensorfs-core/src/bin/tfs/main.rs

@@ -41,7 +41,8 @@
 //! The ONE transport (tfs-048/tfs-049/tfs-050). The credential is `--credential-file` or
 //! TFS_CREDENTIAL (`bearer <token>`, `worker <id> <token>`, or empty for anonymous), never
 //! bare argv; a URL of `-` arrives on stdin. Flags: --hub <base> --lane --session --streams <n>
-//! --streams-start <n> --timeout <s> --allow-hosts a,b,.c --allow-local --max-redirects <n>.
+//! --streams-start <n> --timeout <s> --allow-hosts a,b,.c --allow-local --max-redirects <n>
+//! --ca-file <pem> (a private hub's CA, trusted beside the webpki set).
 //! `--streams` is the most objects in flight (default 128, memory-bounded); the walk starts
 //! at `--streams-start` (default 16) and doubles while measured goodput follows.
 //! `tfs fetch` reads and backfills the Store's bound repo cache between the local Store
@@ -202,6 +203,17 @@ fn accept_printed_ids(args: Vec<String>) -> Vec<String> {
 
 fn main() -> ExitCode {
     let (args, flags) = split_flags(std::env::args().skip(1).collect());
+    if let Some(path) = flag(&flags, "ca-file") {
+        let trusted = fs::read(path)
+            .map_err(|e| format!("IO_FAILED: read {path}: {e}"))
+            .and_then(|pem| {
+                tensorfs_core::transport::trust_roots(&pem).map_err(|r| format!("{r} ({path})"))
+            });
+        if let Err(refusal) = trusted {
+            eprintln!("REFUSED {refusal}");
+            return ExitCode::FAILURE;
+        }
+    }
     let args = accept_printed_ids(args);
     let a: Vec<&str> = args.iter().map(|s| s.as_str()).collect();
     match a.as_slice() {

{tensorfs-0.3.74 → tensorfs-0.3.75}/crates/tensorfs-core/src/err.rs

@@ -189,6 +189,7 @@ codes![
     REDIRECT_REFUSED,
     HUB_UNREACHABLE,
     HUB_REFUSED,
+    TLS_UNTRUSTED,
     REF_NOT_FOUND,
     CREDENTIAL_REQUIRED,
     TRANSFER_FAILED,

{tensorfs-0.3.74 → tensorfs-0.3.75}/crates/tensorfs-core/src/transport/http.rs

@@ -30,10 +30,11 @@
 use crate::err::{refuse, Code, Refusal, Result};
 use crate::transport::ledger::{Deadline, Ledger};
 use crate::transport::policy::CheckedUrl;
+use rustls::pki_types::pem::PemObject;
 use rustls::pki_types::{CertificateDer, ServerName};
 use std::io::{ErrorKind, Read, Write};
 use std::net::TcpStream;
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, Mutex, OnceLock};
 use std::time::{Duration, Instant};
 
 const HEAD_MAX_BYTES: usize = 64 * 1024;
@@ -211,13 +212,35 @@ struct Idle {
     wire: Wire,
 }
 
+/// Authorities this process trusts beside the webpki set: a private hub's CA, named by its
+/// caller as a location (`tfs --ca-file`). Set once, before the first exchange.
+static EXTRA_ROOTS: OnceLock<Vec<CertificateDer<'static>>> = OnceLock::new();
+
+/// Trust the PEM certificates in `pem` for every client this process builds.
+pub fn trust_roots(pem: &[u8]) -> Result<usize> {
+    let roots: Vec<_> = CertificateDer::pem_slice_iter(pem)
+        .collect::<std::result::Result<_, _>>()
+        .map_err(|e| Refusal {
+            code: Code::UNKNOWN_FORMAT,
+            detail: format!("not a PEM certificate bundle: {e}"),
+        })?;
+    let (usable, _) = rustls::RootCertStore::empty().add_parsable_certificates(roots.clone());
+    if roots.is_empty() || usable != roots.len() {
+        return refuse(Code::UNKNOWN_FORMAT, "holds no usable CA certificate");
+    }
+    EXTRA_ROOTS.set(roots).map_err(|_| Refusal {
+        code: Code::UNKNOWN_FORMAT,
+        detail: "this process already chose its trust roots".into(),
+    })?;
+    Ok(usable)
+}
+
 impl Client {
     pub fn new() -> Client {
-        Client::with_extra_roots(&[])
+        Client::with_extra_roots(EXTRA_ROOTS.get().map_or(&[], Vec::as_slice))
     }
 
-    /// `extra_roots` exists for tests that stand up a loopback TLS origin under a
-    /// self-signed authority; production callers pass none and trust the webpki set.
+    /// Trusts the webpki set plus `extra_roots`.
     pub fn with_extra_roots(extra_roots: &[CertificateDer<'static>]) -> Client {
         let mut roots = rustls::RootCertStore::empty();
         roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
@@ -482,7 +505,9 @@ fn connect(
     )
 }
 
+#[allow(clippy::too_many_arguments)]
 fn tls_handshake(
+    host: &str,
     sock: &mut TcpStream,
     conn: &mut rustls::ClientConnection,
     deadline: Deadline,
@@ -516,10 +541,18 @@ fn tls_handshake(
             Ok(0) => return refuse(fail, "origin closed the connection mid-handshake"),
             Ok(_) => {
                 advanced = Instant::now();
-                conn.process_new_packets().map_err(|e| Refusal {
-                    code: fail,
-                    detail: format!("TLS refused: {e}"),
-                })?;
+                // A certificate or protocol verdict is this client's, and re-asking reaches
+                // it again: refused typed, never retried as weather. The alert goes out so
+                // the origin logs why.
+                if let Err(e) = conn.process_new_packets() {
+                    let _ = conn.write_tls(sock);
+                    return refuse(
+                        Code::TLS_UNTRUSTED,
+                        format!(
+                            "{host}: TLS refused: {e}; a private hub names its CA with --ca-file"
+                        ),
+                    );
+                }
             }
             Err(e) if would_block(&e) => {
                 stream_verdict(judge, ledger, advanced, "TLS handshake")?;
@@ -1041,7 +1074,16 @@ pub fn send<'l>(
                 })?;
             let mut sock = sock;
             let mut conn = Box::new(conn);
-            tls_handshake(&mut sock, &mut conn, deadline, ledger, judge, opened, fail)?;
+            tls_handshake(
+                &checked.host,
+                &mut sock,
+                &mut conn,
+                deadline,
+                ledger,
+                judge,
+                opened,
+                fail,
+            )?;
             Wire::Tls { sock, conn }
         } else {
             Wire::Plain(sock)

{tensorfs-0.3.74 → tensorfs-0.3.75}/crates/tensorfs-core/src/transport/mod.rs

@@ -49,7 +49,7 @@ mod speculate;
 mod tests;
 
 pub use api::{api_get, fetch_prefix, probe_length, Prefix};
-pub use http::Client;
+pub use http::{trust_roots, Client};
 pub use hub::{
     closure, closure_all, credential_from_spec, presign, Anonymous, Closure, CredentialProvider,
     HostToken, Presigned, ScopedHeaders,

tensorfs-0.3.75/crates/tensorfs-core/tests/hub_tls.rs

@@ -0,0 +1,124 @@
+//! A hub whose certificate this machine does not trust is a verdict, not weather: `tfs fetch`
+//! refuses it typed on the first handshake, and `--ca-file` names the private CA that makes
+//! the same hub reachable.
+
+use std::io::{Read, Write};
+use std::net::{SocketAddr, TcpListener, TcpStream};
+use std::process::Command;
+use std::sync::{mpsc, Arc};
+
+const ENVELOPE: &str =
+    r#"{"error":{"code":"checkpoint_not_found","message":"no such checkpoint"}}"#;
+
+/// A loopback hub under a self-signed authority that answers every call it can read with a
+/// typed 404. Every accepted peer is reported before its handshake.
+fn private_hub() -> (u16, String, mpsc::Receiver<SocketAddr>) {
+    let key = rcgen::generate_simple_self_signed(vec!["127.0.0.1".into()]).unwrap();
+    let cert = rustls::pki_types::CertificateDer::from(key.cert.der().to_vec());
+    let private = rustls::pki_types::PrivateKeyDer::try_from(key.key_pair.serialize_der())
+        .unwrap()
+        .clone_key();
+    let config = rustls::ServerConfig::builder_with_provider(Arc::new(
+        rustls::crypto::ring::default_provider(),
+    ))
+    .with_safe_default_protocol_versions()
+    .unwrap()
+    .with_no_client_auth()
+    .with_single_cert(vec![cert], private)
+    .unwrap();
+    let config = Arc::new(config);
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let port = listener.local_addr().unwrap().port();
+    let (peers, accepted) = mpsc::channel();
+    std::thread::spawn(move || {
+        for socket in listener.incoming() {
+            let Ok(socket) = socket else { return };
+            let _ = peers.send(socket.peer_addr().unwrap());
+            let conn = rustls::ServerConnection::new(Arc::clone(&config)).unwrap();
+            let mut stream = rustls::StreamOwned::new(conn, socket);
+            let mut head = Vec::new();
+            let mut chunk = [0u8; 4096];
+            while !head.windows(4).any(|w| w == b"\r\n\r\n") {
+                match stream.read(&mut chunk) {
+                    Ok(n) if n > 0 => head.extend_from_slice(&chunk[..n]),
+                    _ => break,
+                }
+            }
+            if head.windows(4).any(|w| w == b"\r\n\r\n") {
+                let answer = format!(
+                    "HTTP/1.1 404 Not Found\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{ENVELOPE}",
+                    ENVELOPE.len()
+                );
+                let _ = stream.write_all(answer.as_bytes());
+                stream.conn.send_close_notify();
+                let _ = stream.flush();
+            }
+        }
+    });
+    (port, key.cert.pem(), accepted)
+}
+
+/// How many connections reached the hub before this call: a sentinel connection is accepted
+/// after every one queued ahead of it.
+fn connections(port: u16, accepted: &mpsc::Receiver<SocketAddr>) -> usize {
+    let sentinel = TcpStream::connect(("127.0.0.1", port)).unwrap();
+    let mine = sentinel.local_addr().unwrap();
+    accepted.iter().take_while(|peer| *peer != mine).count()
+}
+
+fn fetch(root: &std::path::Path, port: u16, extra: &[&str]) -> (bool, String) {
+    let out = Command::new(env!("CARGO_BIN_EXE_tfs"))
+        .args(["fetch", root.to_str().unwrap(), "proof/model"])
+        .args(["--hub", &format!("https://127.0.0.1:{port}")])
+        .args(extra)
+        .env("TFS_CREDENTIAL", "")
+        .output()
+        .unwrap();
+    (
+        out.status.success(),
+        String::from_utf8_lossy(&out.stderr).into_owned(),
+    )
+}
+
+#[test]
+fn an_untrusted_hub_is_refused_once_and_a_named_ca_reaches_it() {
+    let dir = std::env::temp_dir().join(format!("tfs-hub-tls-{}", std::process::id()));
+    let _ = std::fs::remove_dir_all(&dir);
+    std::fs::create_dir_all(&dir).unwrap();
+    let root = dir.join("store");
+    let init = Command::new(env!("CARGO_BIN_EXE_tfs"))
+        .args(["store", "init", root.to_str().unwrap()])
+        .output()
+        .unwrap();
+    assert!(
+        init.status.success(),
+        "{}",
+        String::from_utf8_lossy(&init.stderr)
+    );
+    let (port, pem, accepted) = private_hub();
+
+    let (ok, stderr) = fetch(&root, port, &[]);
+    assert!(
+        !ok && stderr.contains("REFUSED TLS_UNTRUSTED") && stderr.contains("127.0.0.1"),
+        "{stderr}"
+    );
+    assert_eq!(
+        connections(port, &accepted),
+        1,
+        "an untrusted hub was asked again: {stderr}"
+    );
+
+    let ca = dir.join("hub-ca.pem");
+    std::fs::write(&ca, pem).unwrap();
+    let (ok, stderr) = fetch(&root, port, &["--ca-file", ca.to_str().unwrap()]);
+    assert!(
+        !ok && stderr.contains("REFUSED REF_NOT_FOUND") && stderr.contains("checkpoint_not_found"),
+        "{stderr}"
+    );
+
+    let garbage = dir.join("garbage.pem");
+    std::fs::write(&garbage, "not a certificate").unwrap();
+    let (ok, stderr) = fetch(&root, port, &["--ca-file", garbage.to_str().unwrap()]);
+    assert!(!ok && stderr.contains("REFUSED UNKNOWN_FORMAT"), "{stderr}");
+    let _ = std::fs::remove_dir_all(&dir);
+}

{tensorfs-0.3.74 → tensorfs-0.3.75}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "tensorfs"
-version = "0.3.74"
+version = "0.3.75"
 description = "TensorFS — canonical tensor artifact substrate"
 requires-python = ">=3.12"
 classifiers = [

{tensorfs-0.3.74 → tensorfs-0.3.75}/python/tensorfs/_ext.pyi

@@ -278,6 +278,7 @@ class StoreEra(Refusal): ...
 class StoreRootAbsent(Refusal): ...
 class SweepSetIncomplete(Refusal): ...
 class TensorSchemaMismatch(Refusal): ...
+class TlsUntrusted(Refusal): ...
 class TotalBytesCap(Refusal): ...
 class TotalRefsCap(Refusal): ...
 class TrailingBytes(Refusal): ...

{tensorfs-0.3.74 → tensorfs-0.3.75}/python/tensorfs/errors.py

@@ -149,6 +149,7 @@ from ._ext import (
     StoreRootAbsent,
     SweepSetIncomplete,
     TensorSchemaMismatch,
+    TlsUntrusted,
     TotalBytesCap,
     TotalRefsCap,
     TrailingBytes,
@@ -306,6 +307,7 @@ __all__ = [
     "StoreRootAbsent",
     "SweepSetIncomplete",
     "TensorSchemaMismatch",
+    "TlsUntrusted",
     "TotalBytesCap",
     "TotalRefsCap",
     "TrailingBytes",