tensorfs 0.3.92__tar.gz → 0.6.0__tar.gz

{tensorfs-0.3.92 → tensorfs-0.6.0}/Cargo.lock

@@ -26,6 +26,15 @@ version = "2.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
 
+[[package]]
+name = "block-buffer"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "cc"
 version = "1.4.4"
@@ -48,12 +57,41 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
 
+[[package]]
+name = "cpufeatures"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crypto-common"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
 [[package]]
 name = "deranged"
 version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
 
+[[package]]
+name = "digest"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+]
+
 [[package]]
 name = "errno"
 version = "0.3.14"
@@ -92,6 +130,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "generic-array"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.17"
@@ -127,6 +175,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
 [[package]]
 name = "libc"
 version = "0.2.189"
@@ -160,6 +214,12 @@ version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
 
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
 [[package]]
 name = "minicbor"
 version = "2.3.0"
@@ -381,6 +441,21 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "ryu-js"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04d056b875a9d2e6cb9a61d127afee9ac5999b9f87bcb32079d1318e505be714"
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+]
+
 [[package]]
 name = "serde_core"
 version = "1.0.229"
@@ -401,6 +476,41 @@ dependencies = [
  "syn 3.0.4",
 ]
 
+[[package]]
+name = "serde_json"
+version = "1.0.151"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "serde_json_canonicalizer"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe52319a927259afbfa5180c5157cd8167edfd3e8c254f9558c7fef44c5649f2"
+dependencies = [
+ "ryu-js",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "sha2"
+version = "0.10.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
 [[package]]
 name = "shlex"
 version = "2.0.1"
@@ -449,8 +559,9 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.92"
+version = "0.6.0"
 dependencies = [
+ "base64",
  "fs2",
  "libc",
  "minicbor",
@@ -459,12 +570,16 @@ dependencies = [
  "rusqlite",
  "rustix",
  "rustls",
+ "serde",
+ "serde_json",
+ "serde_json_canonicalizer",
+ "sha2",
  "webpki-roots 0.26.11",
 ]
 
 [[package]]
 name = "tensorfs-plane"
-version = "0.3.92"
+version = "0.6.0"
 dependencies = [
  "libc",
  "libloading",
@@ -474,7 +589,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.92"
+version = "0.6.0"
 dependencies = [
  "pyo3",
  "tensorfs-core",
@@ -520,6 +635,12 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
 
+[[package]]
+name = "typenum"
+version = "1.20.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"
@@ -703,3 +824,9 @@ name = "zeroize"
 version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

{tensorfs-0.3.92 → tensorfs-0.6.0}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-plane", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.92"
+version = "0.6.0"
 edition = "2021"

{tensorfs-0.3.92 → tensorfs-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.92
+Version: 0.6.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

tensorfs-0.6.0/crates/tensorfs-core/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "tensorfs-core"
+version.workspace = true
+edition.workspace = true
+
+[[bin]]
+name = "tfs"
+path = "src/bin/tfs/main.rs"
+required-features = ["store"]
+
+[features]
+# The store, downloader, ingest, GC and everything that opens one. Without it: documents,
+# headers, read plans and the fit function only (what a plane-only executor links).
+default = ["store"]
+store = ["dep:fs2", "dep:rusqlite", "dep:rustix", "dep:rustls", "dep:webpki-roots", "dep:nix"]
+
+[dependencies]
+base64 = "0.22"
+fs2 = { version = "0.4", optional = true }
+minicbor = { version = "2.3.0", default-features = false, features = ["alloc"] }
+rusqlite = { version = "0.32", features = ["bundled"], optional = true }
+serde = "1"
+serde_json = { version = "1", features = ["float_roundtrip"] }
+serde_json_canonicalizer = "0.3"
+sha2 = "0.10"
+rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "net", "event", "process"], optional = true }
+# tfs-049: the ONE transport (src/transport/). rustls is the one TLS stack, used only
+# there; scripts/fence.py refuses it anywhere else so a second mover cannot grow.
+rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"], optional = true }
+webpki-roots = { version = "0.26", optional = true }
+
+[target.'cfg(target_os = "linux")'.dependencies]
+nix = { version = "0.31.3", default-features = false, features = ["fs", "user"], optional = true }
+
+[build-dependencies]
+sha2 = "0.10"
+
+[dev-dependencies]
+libc = "0.2"
+rcgen = "0.13"

{tensorfs-0.3.92 → tensorfs-0.6.0}/crates/tensorfs-core/build.rs

@@ -1,7 +1,5 @@
 //! Capture actual native source/dependency/build inputs, including same-version edits.
-#[allow(dead_code)]
-#[path = "src/sha256.rs"]
-mod sha256;
+use sha2::{Digest, Sha256};
 use std::{env, fs, path::Path};
 fn collect(path: &Path, files: &mut Vec<std::path::PathBuf>) {
     if path.is_dir() {
@@ -26,13 +24,13 @@ fn main() {
         collect(Path::new(path), &mut files);
     }
     files.sort();
-    let mut hash = sha256::Sha256::new();
+    let mut hash = Sha256::new();
     for file in files {
         let name = file.to_string_lossy();
         let bytes = fs::read(&file).unwrap();
-        hash.update(&(name.len() as u64).to_le_bytes());
+        hash.update((name.len() as u64).to_le_bytes());
         hash.update(name.as_bytes());
-        hash.update(&(bytes.len() as u64).to_le_bytes());
+        hash.update((bytes.len() as u64).to_le_bytes());
         hash.update(&bytes);
     }
     for key in [
@@ -45,15 +43,13 @@ fn main() {
         println!("cargo:rerun-if-env-changed={key}");
         hash.update(key.as_bytes());
         hash.update(env::var(key).unwrap_or_default().as_bytes());
-        hash.update(&[0]);
+        hash.update([0]);
     }
     let rustc = std::process::Command::new(env::var("RUSTC").unwrap())
         .arg("--version")
         .output()
         .unwrap();
     hash.update(&rustc.stdout);
-    println!(
-        "cargo:rustc-env=TENSORFS_IMPLEMENTATION_SHA256={}",
-        sha256::hex(&hash.finish())
-    );
+    let digest: String = hash.finalize().iter().map(|b| format!("{b:02x}")).collect();
+    println!("cargo:rustc-env=TENSORFS_IMPLEMENTATION_SHA256={digest}");
 }

tensorfs-0.6.0/crates/tensorfs-core/examples/filesystem_parent_death_gate.rs

@@ -0,0 +1,89 @@
+//! CPU-only process fixture compiling the actual private filesystem guard, not a copy.
+
+#[cfg(target_os = "linux")]
+#[path = "../src/filesystem_identity.rs"]
+mod filesystem_identity;
+#[cfg(target_os = "linux")]
+use tensorfs_core::store;
+
+#[cfg(target_os = "linux")]
+fn main() -> std::io::Result<()> {
+    use std::fs;
+    use std::io::{Read, Write};
+    use std::os::unix::fs::MetadataExt;
+    use std::panic::{catch_unwind, AssertUnwindSafe};
+    use std::path::PathBuf;
+
+    use rustix::process::{
+        getpid, parent_process_death_signal, set_parent_process_death_signal, Signal,
+    };
+
+    let args: Vec<String> = std::env::args().collect();
+    let root = PathBuf::from(&args[2]);
+    if args[1] == "setup" {
+        store::Store::init(&root).expect("real native Store init");
+        return Ok(());
+    }
+    let signal = match args[3].as_str() {
+        "0" => None,
+        "9" => Some(Signal::KILL),
+        "15" => Some(Signal::TERM),
+        _ => panic!("fixture supports disabled, SIGKILL and SIGTERM"),
+    };
+    set_parent_process_death_signal(signal)?;
+    let before = parent_process_death_signal()?.map_or(0, Signal::as_raw);
+    let mut in_scope = before;
+    let mut file_uid = 0;
+    if args[1] == "store" {
+        store::Store::open(&root).expect("real native foreign-owner Store open");
+    } else {
+        let mut operation = || -> std::io::Result<()> {
+            let _guard = filesystem_identity::FilesystemIdentity::for_store_root(&root)?;
+            in_scope = parent_process_death_signal()?.map_or(0, Signal::as_raw);
+            match args[1].as_str() {
+                "success" => {
+                    let path = root.join("guard-fixture-data");
+                    fs::write(&path, b"real scoped native I/O")?;
+                    assert_eq!(fs::read(&path)?, b"real scoped native I/O");
+                    file_uid = fs::metadata(&path)?.uid();
+                }
+                "error" => {
+                    assert_eq!(
+                        fs::read(root.join("absent-fixture-file"))
+                            .unwrap_err()
+                            .kind(),
+                        std::io::ErrorKind::NotFound
+                    );
+                    return Err(std::io::Error::other("owned I/O failure"));
+                }
+                "unwind" => panic!("owned scope unwind"),
+                "wait" => {
+                    println!(
+                        "{{\"ready\":true,\"guard\":{in_scope},\"pid\":{}}}",
+                        getpid().as_raw_nonzero()
+                    );
+                    std::io::stdout().flush()?;
+                    std::io::stdin().read_exact(&mut [0])?;
+                }
+                _ => panic!("unknown fixture operation"),
+            }
+            Ok(())
+        };
+        match args[1].as_str() {
+            "unwind" => assert!(catch_unwind(AssertUnwindSafe(operation)).is_err()),
+            "error" => assert!(operation().is_err()),
+            _ => operation()?,
+        }
+    }
+    let after = parent_process_death_signal()?.map_or(0, Signal::as_raw);
+    let maps = fs::read_to_string("/proc/self/maps")?;
+    let cuda_mapped = maps.contains("libcuda") || maps.contains("libnvidia");
+    println!("{{\"before\":{before},\"in_scope\":{in_scope},\"after\":{after},\"file_uid\":{file_uid},\"cuda_mapped\":{cuda_mapped}}}");
+    Ok(())
+}
+
+#[cfg(not(target_os = "linux"))]
+fn main() {
+    eprintln!("Linux-only parent-death credential fixture");
+    std::process::exit(2);
+}

tensorfs-0.6.0/crates/tensorfs-core/examples/mine-vectors.rs

@@ -0,0 +1,152 @@
+//! PRODUCER-MINED vector rows (scripts/mine-vectors.py) into the canonical vector document
+//! `vectors/mined/<name>.json`:
+//!
+//!     cargo run -p tensorfs-core --example mine-vectors -- <rows> <name> [<match>]
+
+use std::process::ExitCode;
+
+use tensorfs_core::dtype::Dtype;
+use tensorfs_core::vectors::{Case, EncodingVectors};
+
+macro_rules! ok {
+    ($e:expr) => {
+        match $e {
+            Ok(v) => v,
+            Err(e) => {
+                eprintln!("REFUSED {e}");
+                return ExitCode::FAILURE;
+            }
+        }
+    };
+}
+
+fn main() -> ExitCode {
+    let args: Vec<String> = std::env::args().skip(1).collect();
+    match args.as_slice() {
+        [rows, name] => mine(std::path::Path::new(rows), name, ""),
+        [rows, name, want] => mine(std::path::Path::new(rows), name, want),
+        _ => {
+            eprintln!("usage: mine-vectors <rows> <name> [<match>]");
+            ExitCode::from(2)
+        }
+    }
+}
+
+/// Turn PRODUCER-MINED rows into the canonical vector document. The rows carry raw bytes
+/// and nothing else; canonicalisation happens here, once, in the one writer.
+fn mine(rows: &std::path::Path, name: &str, want: &str) -> ExitCode {
+    let text = match std::fs::read_to_string(rows) {
+        Ok(t) => t,
+        Err(e) => {
+            eprintln!("{}: {e}", rows.display());
+            return ExitCode::FAILURE;
+        }
+    };
+    let shape = |s: &str| -> Vec<u64> {
+        if s.is_empty() {
+            vec![]
+        } else {
+            s.split(',').filter_map(|d| d.parse().ok()).collect()
+        }
+    };
+
+    let mut cases: Vec<Case> = Vec::new();
+    let mut cur: Option<Case> = None;
+    for line in text.lines() {
+        let t: Vec<&str> = line.split_whitespace().collect();
+        match t.first().copied() {
+            Some("case") => {
+                if let Some(c) = cur.take() {
+                    cases.push(c);
+                }
+                cur = Some(Case {
+                    name: t[1].to_string(),
+                    logical_dtype: Dtype::F32,
+                    logical_shape: vec![],
+                    roles: vec![],
+                    expect_logical_bits: vec![],
+                });
+            }
+            Some("logical") => {
+                if let Some(c) = cur.as_mut() {
+                    c.logical_dtype = ok!(Dtype::parse(t[1]));
+                    c.logical_shape = shape(t.get(2).copied().unwrap_or(""));
+                }
+            }
+            Some("expect") => {
+                if let Some(c) = cur.as_mut() {
+                    c.expect_logical_bits = ok!(tensorfs_core::b64::decode(t[1]));
+                }
+            }
+            Some("role") => {
+                if let Some(c) = cur.as_mut() {
+                    let (rname, dt) = (t[1].to_string(), ok!(Dtype::parse(t[2])));
+                    // a rank-0 role prints an empty shape field, so the payload shifts left
+                    let (sh, b64) = if t.len() >= 5 {
+                        (shape(t[3]), t[4])
+                    } else {
+                        (vec![], t[3])
+                    };
+                    c.roles.push((
+                        rname,
+                        tensorfs_core::vectors::RoleBits {
+                            dtype: dt,
+                            shape: sh,
+                            bits: ok!(tensorfs_core::b64::decode(b64)),
+                        },
+                    ));
+                }
+            }
+            _ => {}
+        }
+    }
+    if let Some(c) = cur.take() {
+        cases.push(c);
+    }
+    cases.retain(|c| c.name.contains(want));
+    for c in cases.iter_mut() {
+        c.roles.sort_by(|a, b| a.0.cmp(&b.0));
+    }
+    if cases.is_empty() {
+        eprintln!("no mined case matches {want:?}");
+        return ExitCode::from(2);
+    }
+    let v = EncodingVectors { cases };
+    let bytes = v.fixture_bytes();
+    if let Err(e) = EncodingVectors::parse_fixture(&bytes) {
+        {
+        eprintln!("REFUSED {e}");
+        return ExitCode::FAILURE;
+    }
+    }
+    let d: std::path::PathBuf = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("../../vectors/mined")
+        .components()
+        .collect();
+    if let Err(e) = std::fs::create_dir_all(&d) {
+        eprintln!("{}: {e}", d.display());
+        return ExitCode::FAILURE;
+    }
+    let path = d.join(format!("{name}.json"));
+    if let Err(e) = std::fs::write(&path, &bytes) {
+        eprintln!("{}: {e}", path.display());
+        return ExitCode::FAILURE;
+    }
+    println!("mined {} ({} B)", path.display(), bytes.len());
+    println!("  cases     {}", v.cases.len());
+    println!(
+        "  fixture_id {}",
+        tensorfs_core::ids::object_id(&v.fixture_bytes())
+    );
+    for c in &v.cases {
+        println!(
+            "    {:<34} logical {} {:?}, roles {:?}",
+            c.name,
+            c.logical_dtype.name(),
+            c.logical_shape,
+            c.roles.iter().map(|(n, _)| n.as_str()).collect::<Vec<_>>()
+        );
+    }
+    ExitCode::SUCCESS
+}
+

{tensorfs-0.3.92 → tensorfs-0.6.0}/crates/tensorfs-core/examples/resident_conversion_profile.rs

@@ -105,7 +105,7 @@ fn main() {
         .filter(|s| s.alias == "plain/1")
         .map(|s| (s.alias.to_string(), s.spec))
         .collect::<Vec<_>>();
-    let mut plan = convert::plan(conv, &sources, &target, &specs, &Default::default()).unwrap();
+    let mut plan = convert::plan(conv, &sources, &target, &specs).unwrap();
     let order = plan
         .ops
         .iter()

tensorfs-0.6.0/crates/tensorfs-core/src/b64.rs

@@ -0,0 +1,33 @@
+//! Canonical padded RFC 4648 base64: the `base64` crate's STANDARD engine, which requires
+//! canonical padding and refuses trailing bits, so a decode re-encodes to its input.
+
+use base64::{engine::general_purpose::STANDARD, Engine};
+
+use crate::err::{Code, Refusal, Result};
+
+pub fn encode(data: &[u8]) -> String {
+    STANDARD.encode(data)
+}
+
+pub fn decode(s: &str) -> Result<Vec<u8>> {
+    STANDARD.decode(s).map_err(|e| Refusal {
+        code: Code::BAD_BASE64,
+        detail: e.to_string(),
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn round_trips_and_refuses_what_is_not_canonical() {
+        for data in [&b""[..], b"f", b"fo", b"foo", b"foob", b"fooba", b"foobar"] {
+            assert_eq!(decode(&encode(data)).unwrap(), data);
+        }
+        assert_eq!(encode(b"foobar"), "Zm9vYmFy");
+        for bad in ["Zg", "Zg=", "Zh==", "Zm9v\n", " Zm9v", "Zm9v====", "Zm=9", "Z===", "Zm9vYg", "Zm9vYh==", "@@@@"] {
+            assert_eq!(decode(bad).unwrap_err().code, Code::BAD_BASE64, "{bad:?}");
+        }
+    }
+}