tensorfs 0.3.66__tar.gz → 0.3.67__tar.gz

{tensorfs-0.3.66 → tensorfs-0.3.67}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.66"
+version = "0.3.67"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.66"
+version = "0.3.67"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.66 → tensorfs-0.3.67}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.66"
+version = "0.3.67"
 edition = "2021"

{tensorfs-0.3.66 → tensorfs-0.3.67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.66
+Version: 0.3.67
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.66 → tensorfs-0.3.67}/crates/tensorfs-core/src/bin/tfs/rd.rs

@@ -133,9 +133,13 @@ pub fn cmd_plan(root: &Path, hex: &str, f: &Flags) -> ExitCode {
     let mut rev = construction_order(&h);
     rev.reverse();
     println!(
-        "contract:   reversed caller traversal refuses: {}",
+        "contract:   reversed caller traversal plans in its own order: {}",
         match read::plan_for_traversal(&h, &rev, &all_components(&h), window) {
-            Err(e) if e.code == Code::TRAVERSAL_ORDER_MISMATCH => "YES",
+            Ok(p)
+                if p.items.first().is_some_and(|it| it
+                    .what
+                    .starts_with(&format!("{}/{}#", rev[0].0, rev[0].1))) =>
+                "YES",
             _ => "NO — DEFECT",
         }
     );
@@ -991,7 +995,7 @@ pub fn cmd_arms(root: &Path, hex: &str) -> ExitCode {
     short.pop();
     a.arm(
         "traversal missing a tensor",
-        Code::TRAVERSAL_ORDER_MISMATCH,
+        Code::TRAVERSAL_INCOMPLETE,
         read::plan_for_traversal(&h, &short, &all_components(&h), 1 << 20),
     );
 
@@ -1026,7 +1030,7 @@ pub fn cmd_arms(root: &Path, hex: &str) -> ExitCode {
         holed.pop();
         a.arm(
             "a tensor missing WITHIN a requested component",
-            Code::TRAVERSAL_ORDER_MISMATCH,
+            Code::TRAVERSAL_INCOMPLETE,
             read::plan_for_traversal(&h, &holed, only_one, 1 << 20),
         );
     }
@@ -1042,7 +1046,7 @@ pub fn cmd_arms(root: &Path, hex: &str) -> ExitCode {
     }
     a.arm(
         "a tensor from a component the plan did not request",
-        Code::TRAVERSAL_ORDER_MISMATCH,
+        Code::TRAVERSAL_INCOMPLETE,
         read::plan_for_traversal(&h, &smuggled, only_one, 1 << 20),
     );
 

{tensorfs-0.3.66 → tensorfs-0.3.67}/crates/tensorfs-core/src/read.rs

@@ -24,7 +24,7 @@ use std::time::Instant;
 
 use crate::checkpoint;
 use crate::err::{refuse, Code, Refusal, Result};
-use crate::header::{Asset, Body, Header, Located, Part};
+use crate::header::{Asset, Body, Header, Located, Part, Tensor};
 use crate::ids::ObjectRef;
 use crate::manifest::Manifest;
 use crate::meta::{Hold, Meta};
@@ -594,48 +594,43 @@ impl ReadPlan {
     }
 }
 
-fn build(header: &Header, scope: &[&str], window: u64) -> Result<ReadPlan> {
+fn build(ordered: &[(&str, &str, &Tensor)], window: u64) -> Result<ReadPlan> {
     let mut items = Vec::new();
     let mut dest: u64 = 0;
-    for (comp, tensors) in &header.components {
-        if !scope.contains(&comp.as_str()) {
-            continue;
-        }
-        for (key, tensor) in tensors {
-            for (role, part) in &tensor.parts {
-                let what = format!("{comp}/{key}#{role}");
-                if let Body::Inline(b) = &part.body {
-                    let n = part.nbytes(&what)?;
+    for (comp, key, tensor) in ordered {
+        for (role, part) in &tensor.parts {
+            let what = format!("{comp}/{key}#{role}");
+            if let Body::Inline(b) = &part.body {
+                let n = part.nbytes(&what)?;
+                items.push(PlanItem {
+                    what: what.clone(),
+                    dest_off: dest,
+                    len: n,
+                    source: Source::Inline(b.clone()),
+                });
+                dest += n;
+                continue;
+            }
+            for segment in part.segments() {
+                let mut off = 0u64;
+                while off < segment.length {
+                    let n = if window == 0 {
+                        segment.length - off
+                    } else {
+                        window.min(segment.length - off)
+                    };
                     items.push(PlanItem {
                         what: what.clone(),
                         dest_off: dest,
                         len: n,
-                        source: Source::Inline(b.clone()),
+                        source: Source::Object(ObjectRange {
+                            obj: segment.clone(),
+                            off,
+                            len: n,
+                        }),
                     });
                     dest += n;
-                    continue;
-                }
-                for segment in part.segments() {
-                    let mut off = 0u64;
-                    while off < segment.length {
-                        let n = if window == 0 {
-                            segment.length - off
-                        } else {
-                            window.min(segment.length - off)
-                        };
-                        items.push(PlanItem {
-                            what: what.clone(),
-                            dest_off: dest,
-                            len: n,
-                            source: Source::Object(ObjectRange {
-                                obj: segment.clone(),
-                                off,
-                                len: n,
-                            }),
-                        });
-                        dest += n;
-                        off += n;
-                    }
+                    off += n;
                 }
             }
         }
@@ -658,8 +653,10 @@ fn tally(items: &[PlanItem]) -> (u64, u64, usize) {
     (bytes, io_bytes, inline)
 }
 
-/// The read planner. Runtime's traversal must exactly equal the stored construction order;
-/// after that equality gate, planning walks the header once and never performs key lookup.
+/// The read planner. The destination follows the CALLER's traversal, which is the order it
+/// constructs in; the header's tensor order is a storage choice of whoever wrote it. The two
+/// are written by independently versioned peers, so a permutation is planned, never refused:
+/// only a traversal that names a different set of tensors is.
 ///
 /// COMPLETENESS SCOPES TO THE REQUESTED COMPONENT SET (#570b). `requested` is the caller's
 /// explicit declaration of which components this plan is for; the traversal must name every
@@ -716,34 +713,38 @@ pub fn plan_for_traversal(
         }
     }
 
-    let named: Vec<(&str, &str)> = traversal
-        .iter()
-        .map(|(c, k)| (c.as_str(), k.as_str()))
-        .collect();
-    let have: Vec<(&str, &str)> = header
+    let mut carried: HashMap<(&str, &str), &Tensor> = header
         .tensors()
         .filter(|(c, _, _)| scope.contains(&c.as_str()))
-        .map(|(c, k, _)| (c.as_str(), k.as_str()))
+        .map(|(c, k, t)| ((c.as_str(), k.as_str()), t))
         .collect();
-    if named != have {
-        let at = named
-            .iter()
-            .zip(have.iter())
-            .position(|(got, want)| got != want)
-            .unwrap_or(named.len().min(have.len()));
-        let got = named.get(at).map(|(c, k)| format!("{c}/{k}"));
-        let want = have.get(at).map(|(c, k)| format!("{c}/{k}"));
+    let mut ordered = Vec::with_capacity(traversal.len());
+    for (c, k) in traversal {
+        let Some(tensor) = carried.remove(&(c.as_str(), k.as_str())) else {
+            return refuse(
+                Code::TRAVERSAL_INCOMPLETE,
+                format!(
+                    "the traversal names {c}/{k}, which requested components {scope:?} do \
+                     not carry, or names it twice"
+                ),
+            );
+        };
+        ordered.push((c.as_str(), k.as_str(), tensor));
+    }
+    if !carried.is_empty() {
+        let mut missing: Vec<String> = carried.keys().map(|(c, k)| format!("{c}/{k}")).collect();
+        missing.sort();
+        missing.truncate(4);
         return refuse(
-            Code::TRAVERSAL_ORDER_MISMATCH,
+            Code::TRAVERSAL_INCOMPLETE,
             format!(
-                "construction order differs at row {at}: caller {got:?}, header {want:?}; \
-                 caller has {} rows and requested components {scope:?} carry {}",
-                named.len(),
-                have.len()
+                "the traversal omits {} tensor(s) requested components {scope:?} carry: \
+                 {missing:?}",
+                carried.len()
             ),
         );
     }
-    build(header, &scope, window)
+    build(&ordered, window)
 }
 
 // ---------------------------------------------------------------- the reader pool

{tensorfs-0.3.66 → tensorfs-0.3.67}/crates/tensorfs-core/tests/anima_hf_profile.rs

@@ -6,6 +6,7 @@ use tensorfs_core::{
     canon::{self, Value},
     dtype::Dtype,
     err::Code,
+    header::{Header, Part, Tensor},
     ids::ObjectRef,
     ingest::{
         fingerprint, preflight,
@@ -13,6 +14,7 @@ use tensorfs_core::{
     },
     limits,
     providers::MemberHead,
+    read, registry,
 };
 
 const PROFILE: &str = "hf/circlestone-labs/anima-base-v1.0-diffusers/bf16/1";
@@ -175,3 +177,93 @@ fn anima_hf_selects_its_profile_without_being_named() {
     );
     assert!(!path.exists());
 }
+
+/// The checkpoint stores the transformer in its files' order; Runtime constructs it in module
+/// order and run 1404 asked for `patch_embed.proj.weight` first. Over the real 567 keys the
+/// planner fills the caller's order and refuses only a traversal naming different tensors.
+#[test]
+fn anima_transformer_reads_in_the_callers_construction_order() {
+    let recorded = canon::parse(
+        &fs::read(root().join("construction_order.json")).unwrap(),
+        limits::DOC_MAX_BYTES,
+    )
+    .unwrap();
+    let Value::Arr(rows) = recorded else {
+        panic!("rows")
+    };
+    let stored: Vec<String> = rows
+        .iter()
+        .filter_map(|row| match row {
+            Value::Arr(pair) if text(&pair[0]) == "transformer" => Some(text(&pair[1]).into()),
+            _ => None,
+        })
+        .collect();
+    assert_eq!(stored.len(), 567);
+    let plain = registry::seeds()
+        .into_iter()
+        .find(|seed| seed.alias == "plain/1")
+        .unwrap()
+        .spec;
+    let tensors = stored
+        .iter()
+        .map(|key| {
+            let tensor = Tensor {
+                dtype: Dtype::Bf16,
+                shape: vec![1],
+                encoding: plain.object_id(),
+                parts: vec![("value".into(), Part::plan(Dtype::Bf16, vec![1], &[0, 0]))],
+            };
+            (key.clone(), tensor)
+        })
+        .collect();
+    let header = Header {
+        configs: Vec::new(),
+        assets: Vec::new(),
+        encodings: vec![plain],
+        components: vec![("transformer".into(), tensors)],
+    };
+    let header = Header::parse(&header.canonical_bytes().unwrap()).unwrap();
+    let modules = [
+        "patch_embed",
+        "time_embed",
+        "transformer_blocks",
+        "norm_out",
+        "proj_out",
+    ];
+    let rank = |key: &String| {
+        let mut parts = key.split('.');
+        let first = parts.next().unwrap();
+        let module = modules.iter().position(|m| *m == first).unwrap();
+        let block = parts
+            .next()
+            .and_then(|n| n.parse::<u32>().ok())
+            .unwrap_or(0);
+        (module, block, key.clone())
+    };
+    let mut constructed = stored.clone();
+    constructed.sort_by_key(rank);
+    assert_ne!(constructed, stored, "a real permutation");
+    let traversal: Vec<(String, String)> = constructed
+        .iter()
+        .map(|key| ("transformer".into(), key.clone()))
+        .collect();
+    let scope = ["transformer".to_string()];
+    let plan = read::plan_for_traversal(&header, &traversal, &scope, 1 << 20).unwrap();
+    let order: Vec<&str> = plan.items.iter().map(|item| item.what.as_str()).collect();
+    let expected: Vec<String> = constructed
+        .iter()
+        .map(|key| format!("transformer/{key}#value"))
+        .collect();
+    assert_eq!(order, expected);
+    assert_eq!(order[0], "transformer/patch_embed.proj.weight#value");
+    assert_eq!(plan.bytes, 2 * 567);
+
+    let mut holed = traversal.clone();
+    holed.remove(100);
+    let error = read::plan_for_traversal(&header, &holed, &scope, 1 << 20).unwrap_err();
+    assert_eq!(error.code, Code::TRAVERSAL_INCOMPLETE);
+    let mut repeated = traversal.clone();
+    repeated[100] = repeated[0].clone();
+    let error = read::plan_for_traversal(&header, &repeated, &scope, 1 << 20).unwrap_err();
+    assert_eq!(error.code, Code::TRAVERSAL_INCOMPLETE);
+}

{tensorfs-0.3.66 → tensorfs-0.3.67}/crates/tensorfs-core/tests/read_plan.rs

@@ -80,7 +80,7 @@ fn selection_does_not_weaken_the_complete_plan_gate() {
         plan_for_traversal(&header, &traversal[1..], &["model".into()], 128)
             .unwrap_err()
             .code,
-        Code::TRAVERSAL_ORDER_MISMATCH
+        Code::TRAVERSAL_INCOMPLETE
     );
     let full = plan_for_traversal(&header, &traversal, &["model".into()], 128).unwrap();
     assert_eq!(

{tensorfs-0.3.66 → tensorfs-0.3.67}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "tensorfs"
-version = "0.3.66"
+version = "0.3.67"
 description = "TensorFS — canonical tensor artifact substrate"
 requires-python = ">=3.12"
 classifiers = [