tensorfs 0.3.41__tar.gz → 0.3.43__tar.gz

{tensorfs-0.3.41 → tensorfs-0.3.43}/Cargo.lock

@@ -421,9 +421,10 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.41"
+version = "0.3.43"
 dependencies = [
  "fs2",
+ "libc",
  "minicbor",
  "rcgen",
  "rusqlite",
@@ -434,7 +435,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.41"
+version = "0.3.43"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.41 → tensorfs-0.3.43}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.41"
+version = "0.3.43"
 edition = "2021"

{tensorfs-0.3.41 → tensorfs-0.3.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.41
+Version: 0.3.43
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.41 → tensorfs-0.3.43}/crates/tensorfs-core/Cargo.toml

@@ -18,4 +18,5 @@ rustls = { version = "0.23", default-features = false, features = ["ring", "std"
 webpki-roots = "0.26"
 
 [dev-dependencies]
+libc = "0.2"
 rcgen = "0.13"

{tensorfs-0.3.41 → tensorfs-0.3.43}/crates/tensorfs-core/src/bin/tfs/ingest.rs

@@ -1506,6 +1506,27 @@ pub fn cmd_worker(root: &Path, target: &str, f: &Flags) -> ExitCode {
     let build = tensorfs_core::ids::object_id(
         format!("{}|{}", p.conv.name, env!("CARGO_PKG_VERSION")).as_bytes(),
     );
+    let source_bytes = match p.files.iter().try_fold(0u64, |total, source| {
+        total.checked_add(source.header.file_len)
+    }) {
+        Some(bytes) => bytes,
+        None => {
+            return bail(tensorfs_core::err::Refusal {
+                code: Code::ARITH_OVERFLOW,
+                detail: "conversion input byte count overflow".into(),
+            })
+        }
+    };
+    let logical_output_bytes = ok!(p.plan.declared_bytes());
+    let logical_output_observation = match i64::try_from(logical_output_bytes) {
+        Ok(bytes) => bytes,
+        Err(_) => {
+            return bail(tensorfs_core::err::Refusal {
+                code: Code::ARITH_OVERFLOW,
+                detail: "conversion output exceeds signed accounting extent".into(),
+            })
+        }
+    };
     let profile = IngestProfile {
         class: "normal".to_string(),
         source_dialect: p.conv.dialect.to_string(),
@@ -1518,7 +1539,10 @@ pub fn cmd_worker(root: &Path, target: &str, f: &Flags) -> ExitCode {
             v.dedup_by(|a, b| a.sha256 == b.sha256);
             v
         },
-        max_source_bytes: p.files.iter().map(|s| s.header.file_len).sum(),
+        // The receipt quota counts materialized bytes. A reviewed permutation can
+        // duplicate a low-rank factor into several canonical projections; its exact
+        // planned output, not only the carrier length, must fit the same bound.
+        max_source_bytes: source_bytes.max(logical_output_bytes),
         max_tensors: limits::MAX_TENSORS as u64,
     };
     ok!(profile.validate(&registry::platform_digests()));
@@ -1547,6 +1571,10 @@ pub fn cmd_worker(root: &Path, target: &str, f: &Flags) -> ExitCode {
     let isolation_tier: u64 = flag_num(f, "isolation-tier", 1);
     out.obs
         .push(("source_bytes".to_string(), profile.max_source_bytes as i64));
+    out.obs.push((
+        "logical_output_bytes".to_string(),
+        logical_output_observation,
+    ));
     out.obs.push(("golden_cases".to_string(), golden as i64));
     out.obs
         .push(("isolation_tier".to_string(), isolation_tier as i64));

{tensorfs-0.3.41 → tensorfs-0.3.43}/crates/tensorfs-core/src/cbor.rs

@@ -436,6 +436,26 @@ fn encode_part(e: &mut Encoder<Bounded>, role: &str, part: &Part) -> Result<()>
     Ok(())
 }
 
+pub(crate) fn encoded_tensor_identity(tensor: &Tensor) -> Result<String> {
+    let mut encoder = Encoder::new(Bounded { bytes: Vec::new() });
+    emit("tensor identity", encoder.array(5))?;
+    emit(
+        "tensor identity domain",
+        encoder.str("tensorfs.encoded-tensor/1"),
+    )?;
+    emit("tensor identity dtype", encoder.u8(tensor.dtype.code()))?;
+    encode_shape(&mut encoder, &tensor.shape)?;
+    emit("tensor identity encoding", encoder.str(&tensor.encoding))?;
+    emit(
+        "tensor identity parts",
+        encoder.array(tensor.parts.len() as u64),
+    )?;
+    for (role, part) in &tensor.parts {
+        encode_part(&mut encoder, role, part)?;
+    }
+    Ok(crate::ids::object_id(&encoder.into_writer().bytes))
+}
+
 fn decode_part(d: &mut Decoder<'_>, what: &str) -> Result<(String, Part)> {
     exact_array(d, what, 4)?;
     let role = text(d, &format!("{what}.role"), limits::MAX_NAME_BYTES)?;

tensorfs-0.3.43/crates/tensorfs-core/src/checkpoint_root.rs

@@ -0,0 +1,314 @@
+//! Independent input custody for an exact, already-materialized repository checkpoint.
+//! These filesystem roots retain bytes; they do not claim a computation or issue a receipt.
+
+use std::fs::{self, File, OpenOptions};
+use std::io::{Read, Write};
+use std::os::unix::fs::OpenOptionsExt;
+use std::path::PathBuf;
+
+use fs2::FileExt;
+
+use crate::canon::{Fields, Value};
+use crate::catalog::WriterGuard;
+use crate::err::{refuse, Code, Refusal, Result};
+use crate::ids::{hex64, prefixed, Doc, ObjectRef, Plain};
+use crate::repository::{Repository, RepositoryName};
+use crate::storage::HeldKey;
+use crate::store::{Store, O_NOFOLLOW};
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct CheckpointRoot {
+    pub owner: String,
+    pub repository: RepositoryName,
+    pub manifest: ObjectRef,
+    pub released: bool,
+}
+
+impl Plain for CheckpointRoot {
+    const MAX_BYTES: usize = 4096;
+
+    fn from_value(value: &Value) -> Result<Self> {
+        let mut fields = Fields::new("CheckpointRoot", value)?;
+        let owner = prefixed("checkpoint root owner", fields.req_str("owner")?)?;
+        let repository = repository(fields.req_str("repository")?)?;
+        let manifest = ObjectRef::from_value("checkpoint root manifest", fields.req("manifest")?)?;
+        let released = match fields.req("released")? {
+            Value::Bool(value) => *value,
+            _ => return refuse(Code::WRONG_TYPE, "checkpoint root release must be boolean"),
+        };
+        fields.done()?;
+        if manifest.length == 0 || manifest.length > crate::limits::INT_MAX as u64 {
+            return refuse(Code::LENGTH_MISMATCH, "checkpoint root manifest is empty");
+        }
+        Ok(Self {
+            owner,
+            repository,
+            manifest,
+            released,
+        })
+    }
+
+    fn to_value(&self) -> Value {
+        Value::obj(vec![
+            ("owner", Value::str(self.owner.clone())),
+            (
+                "repository",
+                Value::str(format!("{}/{}", self.repository.org, self.repository.name)),
+            ),
+            ("manifest", self.manifest.to_value()),
+            ("released", Value::Bool(self.released)),
+        ])
+    }
+}
+
+pub fn repository(value: &str) -> Result<RepositoryName> {
+    let (org, name) = value.split_once('/').ok_or_else(|| Refusal {
+        code: Code::KEY_GRAMMAR,
+        detail: "checkpoint source repository must be org/name".into(),
+    })?;
+    RepositoryName::new(org, name)
+}
+
+fn io(error: std::io::Error) -> Refusal {
+    Refusal {
+        code: Code::IO_FAILED,
+        detail: format!("checkpoint root: {error}"),
+    }
+}
+
+fn directory(store: &Store) -> PathBuf {
+    store.root().join("roots/checkpoints")
+}
+
+fn path(store: &Store, owner: &str) -> Result<PathBuf> {
+    prefixed("checkpoint root owner", owner)?;
+    Ok(directory(store).join(format!("{}.json", &owner[7..])))
+}
+
+/// Inspect the native root, including its permanent release tombstone.
+pub fn read(store: &Store, owner: &str) -> Result<Option<CheckpointRoot>> {
+    let file = match OpenOptions::new()
+        .read(true)
+        .custom_flags(O_NOFOLLOW)
+        .open(path(store, owner)?)
+    {
+        Ok(file) => file,
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
+        Err(error) => return Err(io(error)),
+    };
+    let mut bytes = Vec::new();
+    file.take(CheckpointRoot::MAX_BYTES as u64 + 1)
+        .read_to_end(&mut bytes)
+        .map_err(io)?;
+    let root = CheckpointRoot::parse(&bytes)?;
+    if root.owner != owner {
+        return refuse(
+            Code::CROSS_SUBJECT_REPLAY,
+            "checkpoint root path names another owner",
+        );
+    }
+    Ok(Some(root))
+}
+
+fn lock(store: &Store, owner: &str) -> Result<File> {
+    let path = path(store, owner)?.with_extension("lock");
+    fs::create_dir_all(directory(store)).map_err(io)?;
+    let file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create(true)
+        .truncate(false)
+        .custom_flags(O_NOFOLLOW)
+        .open(path)
+        .map_err(io)?;
+    file.lock_exclusive().map_err(io)?;
+    Ok(file)
+}
+
+fn write(store: &Store, root: &CheckpointRoot) -> Result<()> {
+    let path = path(store, &root.owner)?;
+    let temporary = path.with_extension(format!("tmp-{}", crate::meta::now_nanos_unique()));
+    let result = (|| {
+        let mut file = OpenOptions::new()
+            .write(true)
+            .create_new(true)
+            .open(&temporary)
+            .map_err(io)?;
+        file.write_all(&root.canonical_bytes())
+            .and_then(|()| file.sync_all())
+            .map_err(io)?;
+        fs::rename(&temporary, path).map_err(io)?;
+        for directory in [
+            directory(store),
+            store.root().join("roots"),
+            store.root().to_path_buf(),
+        ] {
+            File::open(directory)
+                .and_then(|file| file.sync_all())
+                .map_err(io)?;
+        }
+        Ok(())
+    })();
+    let _ = fs::remove_file(temporary);
+    result
+}
+
+fn identity(owner: &str, source: &str, manifest: ObjectRef) -> Result<CheckpointRoot> {
+    hex64("checkpoint root manifest", &manifest.sha256)?;
+    if manifest.length == 0 || manifest.length > crate::limits::INT_MAX as u64 {
+        return refuse(Code::LENGTH_MISMATCH, "checkpoint root manifest is empty");
+    }
+    Ok(CheckpointRoot {
+        owner: prefixed("checkpoint root owner", owner)?,
+        repository: repository(source)?,
+        manifest,
+        released: false,
+    })
+}
+
+fn same_subject(root: &CheckpointRoot, wanted: &CheckpointRoot) -> Result<()> {
+    if root.repository != wanted.repository || root.manifest != wanted.manifest {
+        return refuse(
+            Code::CROSS_SUBJECT_REPLAY,
+            "checkpoint root already names another subject",
+        );
+    }
+    Ok(())
+}
+
+fn verify(store: &Store, manifest: &ObjectRef) -> Result<()> {
+    let model = store.read_manifest(manifest)?;
+    let closure = crate::checkpoint::walk_cozytensors(store, &model)?;
+    closure.require_resident(store)?;
+    for object in closure.distinct() {
+        let record = store.record_valid(&object.sha256).map_err(|why| Refusal {
+            code: Code::OBJECT_CORRUPT,
+            detail: format!("checkpoint root closure has no current verification record: {why}"),
+        })?;
+        if record.length != object.length {
+            return refuse(
+                Code::LENGTH_MISMATCH,
+                "checkpoint root closure record changed length",
+            );
+        }
+    }
+    Ok(())
+}
+
+/// Retain the runtime closure of an exact native repository checkpoint.
+pub fn retain(
+    store: &Store,
+    owner: &str,
+    source: &str,
+    manifest: ObjectRef,
+) -> Result<CheckpointRoot> {
+    let wanted = identity(owner, source, manifest)?;
+    let _gc = WriterGuard::acquire(store.root())?;
+    let _owner = lock(store, owner)?;
+    if let Some(root) = read(store, owner)? {
+        same_subject(&root, &wanted)?;
+        if root.released {
+            return refuse(
+                Code::TRANSACTION_CLOSED,
+                "checkpoint root was permanently released",
+            );
+        }
+        verify(store, &root.manifest)?;
+        return Ok(root);
+    }
+    verify_source(store, &wanted.repository, &wanted.manifest)?;
+    write(store, &wanted)?;
+    Ok(wanted)
+}
+
+/// Validate all source inputs before the execution owner creates any recipient roots.
+pub fn check_source(store: &Store, source: &str, manifest: &ObjectRef) -> Result<()> {
+    let repository = repository(source)?;
+    let _gc = WriterGuard::acquire(store.root())?;
+    verify_source(store, &repository, manifest)
+}
+
+fn verify_source(store: &Store, repository: &RepositoryName, manifest: &ObjectRef) -> Result<()> {
+    let file = OpenOptions::new()
+        .read(true)
+        .custom_flags(O_NOFOLLOW)
+        .open(store.repository_path(repository))
+        .map_err(|error| {
+            if error.kind() == std::io::ErrorKind::NotFound {
+                Refusal {
+                    code: Code::REPOSITORY_ABSENT,
+                    detail: "checkpoint source repository is absent".into(),
+                }
+            } else {
+                io(error)
+            }
+        })?;
+    let mut body = Vec::new();
+    file.take(Repository::MAX_BYTES as u64 + 1)
+        .read_to_end(&mut body)
+        .map_err(io)?;
+    let actual = Repository::parse(&body)?;
+    if &actual.repo != repository
+        || !actual
+            .checkpoints
+            .iter()
+            .any(|row| &row.manifest == manifest)
+    {
+        return refuse(
+            Code::ROOT_ABSENT,
+            "checkpoint has no exact source repository membership",
+        );
+    }
+    verify(store, manifest)
+}
+
+/// Release this exact owner permanently, even before its first acquisition.
+pub fn release(
+    store: &Store,
+    owner: &str,
+    source: &str,
+    manifest: ObjectRef,
+) -> Result<CheckpointRoot> {
+    let mut wanted = identity(owner, source, manifest)?;
+    let _gc = WriterGuard::acquire(store.root())?;
+    let _owner = lock(store, owner)?;
+    if let Some(root) = read(store, owner)? {
+        same_subject(&root, &wanted)?;
+        if root.released {
+            return Ok(root);
+        }
+    }
+    wanted.released = true;
+    write(store, &wanted)?;
+    Ok(wanted)
+}
+
+pub(crate) fn held_objects(store: &Store) -> Result<Vec<HeldKey>> {
+    let entries = match fs::read_dir(directory(store)) {
+        Ok(entries) => entries,
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
+        Err(error) => return Err(io(error)),
+    };
+    let mut objects = Vec::new();
+    for entry in entries {
+        let entry = entry.map_err(io)?;
+        let name = entry.file_name();
+        let name = name.to_string_lossy();
+        let Some(hex) = name.strip_suffix(".json") else {
+            continue;
+        };
+        let root = read(store, &format!("sha256:{hex}"))?.ok_or_else(|| Refusal {
+            code: Code::ROOT_ABSENT,
+            detail: "checkpoint root disappeared during census".into(),
+        })?;
+        if !root.released {
+            objects.push(HeldKey {
+                key: crate::storage::manifest_key(&root.manifest.sha256)?,
+                kind: "cozytensors".into(),
+                length: root.manifest.length,
+                sha256: root.manifest.sha256,
+            });
+        }
+    }
+    Ok(objects)
+}

{tensorfs-0.3.41 → tensorfs-0.3.43}/crates/tensorfs-core/src/gc.rs

@@ -134,7 +134,13 @@ pub fn session_holds(store: &Store) -> Result<(Vec<HeldKey>, Vec<String>)> {
         .collect::<Result<Vec<_>>>()?;
     holds.extend(crate::derived::retained_objects(store)?);
     holds.extend(crate::source_artifact::held_objects(store)?);
-    holds.sort_by(|left, right| left.key.cmp(&right.key));
+    holds.extend(crate::checkpoint_root::held_objects(store)?);
+    // A full snapshot obligation wins over a runtime-only pin of the same manifest.
+    holds.sort_by(|left, right| {
+        left.key
+            .cmp(&right.key)
+            .then_with(|| (left.kind == "cozytensors").cmp(&(right.kind == "cozytensors")))
+    });
     if holds
         .windows(2)
         .any(|pair| pair[0].key == pair[1].key && pair[0].length != pair[1].length)
@@ -165,7 +171,7 @@ pub(crate) fn retained_manifests(store: &Store) -> Result<Vec<crate::ids::Object
     roots.extend(
         holds
             .into_iter()
-            .filter(|hold| hold.kind == "manifest")
+            .filter(|hold| hold.is_manifest())
             .map(|hold| crate::ids::ObjectRef {
                 sha256: hold.sha256,
                 length: hold.length,

{tensorfs-0.3.41 → tensorfs-0.3.43}/crates/tensorfs-core/src/header.rs

@@ -93,6 +93,14 @@ pub struct Tensor {
     pub parts: Vec<(String, Part)>,
 }
 
+impl Tensor {
+    /// Opaque equality of encoded geometry and bytes, independent of tensor names
+    /// and unrelated header configs or encodings. This is not a Store capability.
+    pub fn encoded_identity(&self) -> Result<String> {
+        crate::cbor::encoded_tensor_identity(self)
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Asset {
     pub logical_sha256: String,

{tensorfs-0.3.41 → tensorfs-0.3.43}/crates/tensorfs-core/src/ingest/convert.rs

@@ -53,7 +53,7 @@ pub struct Converter {
     pub validated_on_real_bytes: bool,
 }
 
-pub const CONVERTERS: [Converter; 4] = [
+pub const CONVERTERS: [Converter; 5] = [
     Converter {
         name: "diffusers.identity/1",
         dialect: "safetensors.diffusers",
@@ -79,6 +79,13 @@ pub const CONVERTERS: [Converter; 4] = [
               REFERENCE; nothing is re-hashed and nothing is re-uploaded.",
         validated_on_real_bytes: true,
     },
+    Converter {
+        name: "h3.lora/1",
+        dialect: "safetensors.h3_lora",
+        class: Class::Permutation,
+        doc: "Native H3 LoRA A/B and Kohya down/up factors: split head-interleaved QKV B rows, swap SwiGLU B halves, preserve A and alpha without arithmetic.",
+        validated_on_real_bytes: false,
+    },
     Converter {
         name: "h3.native/2",
         dialect: "safetensors.h3_native",
@@ -376,7 +383,12 @@ impl Plan {
         let mut n = 0u64;
         for o in &self.ops {
             for r in &o.roles {
-                n += checked_bytes(&o.out_key, &r.shape, r.dtype)?;
+                n = n
+                    .checked_add(checked_bytes(&o.out_key, &r.shape, r.dtype)?)
+                    .ok_or_else(|| crate::err::Refusal {
+                        code: Code::ARITH_OVERFLOW,
+                        detail: "conversion output byte count overflow".into(),
+                    })?;
             }
         }
         Ok(n)
@@ -947,6 +959,15 @@ fn plan_carrier(
     cands: &Candidates<'_>,
     p: &mut Plan,
 ) -> Result<()> {
+    if conv.name == "h3.lora/1" {
+        if allowed != ["plain/1"] {
+            return refuse(
+                Code::UNKNOWN_ENCODING,
+                "H3 LoRA ingestion preserves floating point factors in plain/1",
+            );
+        }
+        return super::h3_lora::plan(component, file, h, cands.plain, p);
+    }
     let alias = Allowed(allowed);
     p.header_bytes += h.header_bytes;
     let plain_map = role_map("plain/1")?;
@@ -1738,6 +1759,23 @@ pub fn h3_geometry(
 /// error at ~90% numerically while nothing raised.
 pub fn golden_cases(c: &Converter) -> Vec<(&'static str, Xform)> {
     match c.name {
+        "h3.lora/1" => vec![
+            (
+                "H3 LoRA SwiGLU B rows",
+                Xform::SwapHalves {
+                    half_bytes: 14336 * 16 * 2,
+                },
+            ),
+            (
+                "H3 LoRA QKV B rows",
+                Xform::QkvSplit {
+                    groups: H3_HEADS,
+                    shares: [1, 1, 1],
+                    take: 0,
+                    unit_bytes: H3_HEAD_DIM * 16 * 2,
+                },
+            ),
+        ],
         "h3.native/2" => vec![
             (
                 "h3-swiglu gate/value rows",

{tensorfs-0.3.41 → tensorfs-0.3.43}/crates/tensorfs-core/src/ingest/fingerprint.rs

@@ -855,7 +855,9 @@ pub fn validate_source_member(member: &str) -> Result<()> {
 }
 
 pub fn validate_source_member_prefix(prefix: &str) -> Result<()> {
-    if !prefix.ends_with('/') || validate_source_member(&format!("{prefix}member")).is_err() {
+    if !prefix.is_empty()
+        && (!prefix.ends_with('/') || validate_source_member(&format!("{prefix}member")).is_err())
+    {
         return refuse(
             Code::KEY_GRAMMAR,
             format!("source member prefix {prefix:?} is not a safe canonical prefix"),
@@ -867,8 +869,9 @@ pub fn validate_source_member_prefix(prefix: &str) -> Result<()> {
 /// Decoded `__metadata__` facts the border is willing to carry as EVIDENCE STAMPS. Foreign
 /// JSON is decoded through this strict list or discarded — it never survives as itself.
 pub fn evidence_stamps(h: &SourceHeader) -> Result<Vec<(String, String)>> {
-    const KEPT: [&str; 4] = [
+    const KEPT: [&str; 5] = [
         "modelspec.architecture",
+        "ss_h3_training_mode",
         "modelspec.prediction_type",
         "format",
         "scaled_fp8",