tensorfs 0.3.62__tar.gz → 0.3.63__tar.gz

{tensorfs-0.3.62 → tensorfs-0.3.63}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.62"
+version = "0.3.63"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.62"
+version = "0.3.63"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.62 → tensorfs-0.3.63}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.62"
+version = "0.3.63"
 edition = "2021"

{tensorfs-0.3.62 → tensorfs-0.3.63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.62
+Version: 0.3.63
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.62 → tensorfs-0.3.63}/crates/tensorfs-core/src/derived/part_rows_tests.rs

@@ -25,7 +25,12 @@ fn fixture(label: &str) -> (Store, Meta, String, Declaration) {
         .map(|index| ("model".into(), format!("weight_{index}")))
         .collect();
     declaration.max_new_bytes = 3 * 512;
-    (store, meta, format!("sha256:{}", "7a".repeat(32)), declaration)
+    (
+        store,
+        meta,
+        format!("sha256:{}", "7a".repeat(32)),
+        declaration,
+    )
 }
 
 fn catalog(store: &Store) -> rusqlite::Connection {
@@ -55,9 +60,16 @@ fn stored_rows(store: &Store, id: &str) -> (usize, Option<i64>, i64) {
         )
         .unwrap();
     let value = crate::canon::parse_canonical(&bytes, limits::DOC_MAX_BYTES).unwrap();
-    let inline = TransactionRow::from_value(&value).unwrap().added_parts.len();
+    let inline = TransactionRow::from_value(&value)
+        .unwrap()
+        .added_parts
+        .len();
     let parts = connection
-        .query_row("SELECT count(*) FROM tensorfs_derived_parts WHERE id=?1", [id], |row| row.get(0))
+        .query_row(
+            "SELECT count(*) FROM tensorfs_derived_parts WHERE id=?1",
+            [id],
+            |row| row.get(0),
+        )
         .unwrap();
     (inline, open, parts)
 }
@@ -74,14 +86,18 @@ fn parts_are_rows_that_resume_refuse_conflicts_and_stay_retained() {
     add(&store, &meta, &id, &writer, "weight_0", 1).unwrap();
     assert_eq!(stored_rows(&store, &id).2, 2);
     assert_eq!(
-        add(&store, &meta, &id, &writer, "weight_0", 9).unwrap_err().code,
+        add(&store, &meta, &id, &writer, "weight_0", 9)
+            .unwrap_err()
+            .code,
         Code::TRANSACTION_CONFLICT
     );
     // A fenced writer cannot add. Once it is gone, GC keeps its accepted bytes through
     // the append-only journal alone, and its successor resumes with both parts.
     fence(&meta, &id, 1).unwrap();
     assert_eq!(
-        add(&store, &meta, &id, &writer, "weight_2", 3).unwrap_err().code,
+        add(&store, &meta, &id, &writer, "weight_2", 3)
+            .unwrap_err()
+            .code,
         Code::WRITER_FENCED
     );
     release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
@@ -135,7 +151,9 @@ fn an_older_builds_inline_row_and_a_torn_journal_line_are_read() {
     std::fs::write(&journal, torn).unwrap();
 
     assert_eq!(
-        add(&store, &meta, &id, &writer, "weight_0", 9).unwrap_err().code,
+        add(&store, &meta, &id, &writer, "weight_0", 9)
+            .unwrap_err()
+            .code,
         Code::TRANSACTION_CONFLICT
     );
     add(&store, &meta, &id, &writer, "weight_1", 2).unwrap();

{tensorfs-0.3.62 → tensorfs-0.3.63}/crates/tensorfs-core/src/derived/roots.rs

@@ -2,8 +2,8 @@
 //! metadata journals lifecycle; GC never needs that database to discover retained bytes.
 
 use std::fs::{self, File, OpenOptions};
-use std::os::unix::fs::FileExt;
 use std::io::{Read, Write};
+use std::os::unix::fs::FileExt;
 use std::path::PathBuf;
 
 use crate::canon::{Fields, Value};
@@ -164,7 +164,11 @@ pub(super) fn append_parts(store: &Store, transaction: &str, objects: &[ObjectRe
         file.read_exact_at(&mut last, length - 1).map_err(io)?;
     }
     // Terminate a line torn by a dead writer so this one parses on its own.
-    let mut bytes = if last[0] == b'\n' { Vec::new() } else { vec![b'\n'] };
+    let mut bytes = if last[0] == b'\n' {
+        Vec::new()
+    } else {
+        vec![b'\n']
+    };
     bytes.extend(parts_line(objects));
     file.write_all(&bytes)
         .and_then(|()| file.sync_data())
@@ -185,7 +189,11 @@ pub(super) fn rewrite_parts(store: &Store, transaction: &str, objects: &[ObjectR
         .create_new(true)
         .open(&temporary)
         .map_err(io)?;
-    let bytes = if objects.is_empty() { Vec::new() } else { parts_line(objects) };
+    let bytes = if objects.is_empty() {
+        Vec::new()
+    } else {
+        parts_line(objects)
+    };
     file.write_all(&bytes)
         .and_then(|()| file.sync_all())
         .map_err(io)?;
@@ -211,8 +219,12 @@ pub(super) fn journaled(store: &Store) -> Result<Vec<ObjectRef>> {
             Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
             Err(error) => return Err(io(error)),
         };
-        for line in bytes.split(|byte| *byte == b'\n').filter(|line| !line.is_empty()) {
-            let Ok(value) = crate::canon::parse_canonical(line, crate::limits::DOC_MAX_BYTES) else {
+        for line in bytes
+            .split(|byte| *byte == b'\n')
+            .filter(|line| !line.is_empty())
+        {
+            let Ok(value) = crate::canon::parse_canonical(line, crate::limits::DOC_MAX_BYTES)
+            else {
                 continue;
             };
             for value in crate::canon::as_arr("PrivateDerivedParts", "objects", &value)? {

{tensorfs-0.3.62 → tensorfs-0.3.63}/crates/tensorfs-core/src/derived.rs

@@ -1168,11 +1168,19 @@ pub(crate) fn part_bytes(part: &Part) -> Vec<u8> {
 }
 
 impl AddedPart {
-    pub(crate) fn stored(component: String, key: String, role: String, bytes: &[u8]) -> Result<Self> {
+    pub(crate) fn stored(
+        component: String,
+        key: String,
+        role: String,
+        bytes: &[u8],
+    ) -> Result<Self> {
         name("added part component", &component)?;
         self::key("added part tensor", &key)?;
         name("added part role", &role)?;
-        let part = part_from_value(&crate::canon::parse_canonical(bytes, limits::DOC_MAX_BYTES)?)?;
+        let part = part_from_value(&crate::canon::parse_canonical(
+            bytes,
+            limits::DOC_MAX_BYTES,
+        )?)?;
         Ok(Self {
             component,
             key,
@@ -3005,7 +3013,11 @@ impl Additions {
             for tensor in &component.add {
                 for part in &tensor.parts {
                     parts.insert(
-                        (component.target.clone(), tensor.key.clone(), part.role.clone()),
+                        (
+                            component.target.clone(),
+                            tensor.key.clone(),
+                            part.role.clone(),
+                        ),
                         part.clone(),
                     );
                 }
@@ -3025,7 +3037,11 @@ impl Additions {
     fn get(&self, (component, tensor_key, role): (&str, &str, &str)) -> Result<&PartDeclaration> {
         let part = self
             .parts
-            .get(&(component.to_string(), tensor_key.to_string(), role.to_string()))
+            .get(&(
+                component.to_string(),
+                tensor_key.to_string(),
+                role.to_string(),
+            ))
             .ok_or_else(|| Refusal {
                 code: Code::MISSING_FIELD,
                 detail: format!("{component}/{tensor_key}#{role} is not a declared addition"),
@@ -3093,7 +3109,15 @@ pub fn add_part<R: Read>(
     reader: &mut R,
 ) -> Result<crate::checkpoint::Written> {
     let additions = Additions::load(meta, transaction, writer_session)?;
-    add_indexed_part(store, meta, transaction, writer_session, &additions, value, reader)
+    add_indexed_part(
+        store,
+        meta,
+        transaction,
+        writer_session,
+        &additions,
+        value,
+        reader,
+    )
 }
 
 /// Accept one part. The cost is this part's bytes plus one indexed SQL row: the

{tensorfs-0.3.62 → tensorfs-0.3.63}/crates/tensorfs-core/src/meta.rs

@@ -172,9 +172,11 @@ impl Meta {
             .optional()
             .map_err(|error| sql("read accepted part", error))?
             .or_else(|| {
-                legacy.iter().flatten().find(|old| old.same_value(added)).map(|old| {
-                    crate::derived::part_bytes(&old.part)
-                })
+                legacy
+                    .iter()
+                    .flatten()
+                    .find(|old| old.same_value(added))
+                    .map(|old| crate::derived::part_bytes(&old.part))
             });
         match existing {
             Some(existing) if existing == bytes => {

tensorfs-0.3.63/crates/tensorfs-core/tests/anima_hf_profile.rs

@@ -0,0 +1,166 @@
+//! Real header-only evidence for the Hugging Face Diffusers release of Anima Base v1.0: its four
+//! components classify against the entries already banked from Civitai 2945208, so the source
+//! profile is the only new data. No tensor payload is downloaded or read.
+use std::{fs, path::PathBuf};
+use tensorfs_core::{
+    canon::{self, Value},
+    dtype::Dtype,
+    err::Code,
+    ids::ObjectRef,
+    ingest::{
+        fingerprint, preflight,
+        source::{self, BUILTIN_REGISTRY, BUILTIN_REGISTRY_BYTES},
+    },
+    limits,
+    providers::MemberHead,
+};
+
+const PROFILE: &str = "hf/circlestone-labs/anima-base-v1.0-diffusers/bf16/1";
+
+fn root() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../vectors/anima-hf-headers")
+}
+
+fn field<'a>(value: &'a Value, name: &str) -> &'a Value {
+    let Value::Obj(rows) = value else {
+        panic!("object")
+    };
+    &rows.iter().find(|(key, _)| key == name).unwrap().1
+}
+
+fn text(value: &Value) -> &str {
+    let Value::Str(value) = value else {
+        panic!("string")
+    };
+    value
+}
+
+fn number(value: &Value) -> u64 {
+    let Value::Int(value) = value else {
+        panic!("integer")
+    };
+    (*value).try_into().unwrap()
+}
+
+fn heads() -> Vec<MemberHead> {
+    let manifest = canon::parse(
+        &fs::read(root().join("MANIFEST.json")).unwrap(),
+        limits::DOC_MAX_BYTES,
+    )
+    .unwrap();
+    let Value::Arr(rows) = field(&manifest, "members") else {
+        panic!("members")
+    };
+    rows.iter()
+        .map(|row| {
+            let head = fs::read(root().join(text(field(row, "header_file")))).unwrap();
+            assert_eq!(
+                ObjectRef::of(&head).sha256,
+                text(field(row, "header_sha256"))
+            );
+            MemberHead {
+                member: text(field(row, "member")).into(),
+                length: number(field(row, "full_length")),
+                head,
+            }
+        })
+        .collect()
+}
+
+fn scratch(name: &str) -> PathBuf {
+    std::env::temp_dir().join(format!(
+        "anima-hf-{name}-{}-{}",
+        std::process::id(),
+        tensorfs_core::meta::now_nanos_unique()
+    ))
+}
+
+#[test]
+fn anima_hf_diffusers_plans_against_the_banked_keysets_without_payloads() {
+    let heads = heads();
+    let path = scratch("plan");
+    let decision = preflight::plan(
+        BUILTIN_REGISTRY,
+        BUILTIN_REGISTRY_BYTES,
+        &heads,
+        &[PROFILE.into()],
+        &path,
+    )
+    .unwrap();
+    assert!(!path.exists());
+    assert_eq!(decision.plans.len(), 1);
+    assert_eq!(decision.plans[0].constructs, 1189);
+    assert_eq!(decision.plans[0].components.len(), 4);
+    assert_eq!(decision.plans[0].converter, "diffusers.identity/1");
+
+    let staged = preflight::stage(&path, &heads).unwrap();
+    let native = source::plan_source(
+        BUILTIN_REGISTRY,
+        BUILTIN_REGISTRY_BYTES,
+        staged.carriers(),
+        Some(PROFILE),
+        None,
+    )
+    .unwrap();
+    assert_eq!(native.session, decision.plans[0].session);
+    let recorded = canon::parse(
+        &fs::read(root().join("construction_order.json")).unwrap(),
+        limits::DOC_MAX_BYTES,
+    )
+    .unwrap();
+    let actual = Value::arr(
+        native
+            .construction_order
+            .iter()
+            .map(|(component, key)| Value::arr(vec![Value::str(component), Value::str(key)]))
+            .collect(),
+    );
+    assert_eq!(actual, recorded);
+
+    let registry = fingerprint::FingerprintRegistry::parse(BUILTIN_REGISTRY_BYTES).unwrap();
+    let carriers = source::CarrierSet::of(staged.carriers());
+    for (component, member, count) in [
+        (
+            "transformer",
+            "transformer/diffusion_pytorch_model.safetensors",
+            567,
+        ),
+        ("text_encoder", "text_encoder/model.safetensors", 310),
+        (
+            "text_conditioner",
+            "text_conditioner/diffusion_pytorch_model.safetensors",
+            118,
+        ),
+        ("vae", "vae/diffusion_pytorch_model.safetensors", 194),
+    ] {
+        let carrier = staged
+            .carriers()
+            .iter()
+            .find(|row| row.member.as_deref() == Some(member))
+            .unwrap();
+        let (header, _) = source::read_carrier(carrier, &carriers).unwrap();
+        let observed = fingerprint::fingerprint(component, &header).unwrap();
+        assert_eq!(observed.dtypes, vec![(Dtype::Bf16, count)]);
+        let banked = registry.authorize(component, &observed).unwrap();
+        assert!(banked.provenance.source.contains("2945208"), "{component}");
+    }
+    drop(staged);
+    assert!(!path.exists());
+}
+
+#[test]
+fn anima_hf_refuses_a_missing_component_before_payload_transfer() {
+    let mut heads = heads();
+    heads.retain(|row| row.member != "vae/diffusion_pytorch_model.safetensors");
+    let path = scratch("missing");
+    let error = preflight::plan(
+        BUILTIN_REGISTRY,
+        BUILTIN_REGISTRY_BYTES,
+        &heads,
+        &[PROFILE.into()],
+        &path,
+    )
+    .unwrap_err();
+    assert_ne!(error.code, Code::IO_FAILED, "{error:?}");
+    assert!(!path.exists());
+}

{tensorfs-0.3.62 → tensorfs-0.3.63}/crates/tensorfs-core/tests/qwen_image21_profile.rs

@@ -160,7 +160,7 @@ fn qwen_image21_refuses_missing_shard_before_payload_transfer() {
 }
 
 #[test]
-fn qwen_image21_refuses_same_keys_with_changed_precision() {
+fn qwen_image21_same_keys_at_another_precision_classify_by_key_set() {
     let mut heads = heads();
     for row in &mut heads {
         if row.member.starts_with("transformer/") && row.member.ends_with(".safetensors") {
@@ -181,15 +181,16 @@ fn qwen_image21_refuses_same_keys_with_changed_precision() {
             row.head[8..].copy_from_slice(&encoded);
         }
     }
+    // Owner ruling: the reviewed key set and component classify; conversion validates dtypes.
     let path = scratch("precision");
-    let error = preflight::plan(
+    let decision = preflight::plan(
         BUILTIN_REGISTRY,
         BUILTIN_REGISTRY_BYTES,
         &heads,
         &[PROFILE.into()],
         &path,
     )
-    .unwrap_err();
-    assert_eq!(error.code, Code::UNREGISTERED_FINGERPRINT);
+    .unwrap();
+    assert_eq!(decision.plans[0].constructs, 1285);
     assert!(!path.exists());
 }

{tensorfs-0.3.62 → tensorfs-0.3.63}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "tensorfs"
-version = "0.3.62"
+version = "0.3.63"
 description = "TensorFS — canonical tensor artifact substrate"
 requires-python = ">=3.12"
 classifiers = [