tensorfs 0.3.89__tar.gz → 0.3.90__tar.gz

{tensorfs-0.3.89 → tensorfs-0.3.90}/Cargo.lock

@@ -449,7 +449,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.89"
+version = "0.3.90"
 dependencies = [
  "fs2",
  "libc",
@@ -464,7 +464,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-plane"
-version = "0.3.89"
+version = "0.3.90"
 dependencies = [
  "libc",
  "libloading",
@@ -474,7 +474,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.89"
+version = "0.3.90"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.89 → tensorfs-0.3.90}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-plane", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.89"
+version = "0.3.90"
 edition = "2021"

{tensorfs-0.3.89 → tensorfs-0.3.90}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.89
+Version: 0.3.90
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.89 → tensorfs-0.3.90}/crates/tensorfs-core/src/bin/tfs/ingest.rs

@@ -575,6 +575,7 @@ pub fn cmd_bank(dialect: &str, converter: &str, f: &Flags) -> ExitCode {
         }
         let profile = SourceProfile {
             auto_select: true,
+            grammar: None,
             name: name.to_string(),
             components,
         };
@@ -589,7 +590,7 @@ pub fn cmd_bank(dialect: &str, converter: &str, f: &Flags) -> ExitCode {
             &target,
             &unnamed(&srcs),
             &CarrierSet::default(),
-            Some(&reg),
+            source_core::Classify::Banked(&reg),
             variant,
         ) {
             Ok(prepared) => prepared,
@@ -659,6 +660,7 @@ pub fn cmd_source_plan(f: &Flags) -> ExitCode {
 
 struct ClosedSourcePlan {
     converter: String,
+    profile: String,
     order: Vec<(String, String)>,
     registry: String,
     session: String,
@@ -828,6 +830,7 @@ fn read_closed_source_plan(path: &Path) -> Result<ClosedSourcePlan, ExitCode> {
     }
     Ok(ClosedSourcePlan {
         converter,
+        profile,
         order,
         registry,
         session,
@@ -850,10 +853,7 @@ fn prepare_closed_source_plan(
     if closed.registry != BUILTIN_REGISTRY {
         flags.push(("registry".to_string(), closed.registry.clone()));
     }
-    let as_is = closed.converter == convert::IDENTITY;
-    if as_is {
-        flags.push(("source-profile".to_string(), source_core::AS_IS.to_string()));
-    }
+    flags.push(("source-profile".to_string(), closed.profile.clone()));
     for (component, source) in &closed.sources {
         flags.push((
             "source".to_string(),
@@ -865,7 +865,7 @@ fn prepare_closed_source_plan(
         &closed.target,
         &closed.sources,
         &closed.set,
-        (!as_is).then_some(&registry),
+        source_core::classify(&registry, &closed.profile).map_err(bail)?,
         None,
     )
     .map_err(bail)?;
@@ -1000,12 +1000,12 @@ fn prepare(f: &Flags, target: &str) -> Result<Prepared, ExitCode> {
     let sources = sources(f)?;
     let registry = load_registry(f)?;
     let variant = flag(f, "spec-variant").and_then(|value| value.parse().ok());
-    let as_is = flag(f, "source-profile") == Some(source_core::AS_IS);
     source_core::prepare(
         target,
         &unnamed(&sources),
         &CarrierSet::default(),
-        (!as_is).then_some(&registry),
+        source_core::classify(&registry, flag(f, "source-profile").unwrap_or_default())
+            .map_err(bail)?,
         variant,
     )
     .map_err(bail)

{tensorfs-0.3.89 → tensorfs-0.3.90}/crates/tensorfs-core/src/err.rs

@@ -134,6 +134,9 @@ codes![
     PLACEMENT_UNFIT,
     CONVERT_SIZE_CAP,
     SEAM_UNBANKED,
+    // An adapter factor targets a module the reviewed converter cannot map; dropping it
+    // would change the adapter.
+    ADAPTER_TARGET_UNSUPPORTED,
     // ingest border: the temporary candidate transaction
     QUOTA_EXHAUSTED,
     ROOT_EXPIRED,

{tensorfs-0.3.89 → tensorfs-0.3.90}/crates/tensorfs-core/src/fit.rs

@@ -8,10 +8,13 @@
 use crate::capability::CapabilityRecords;
 use crate::dtype::Dtype;
 use crate::err::{refuse, Code, Result};
-use crate::header::{Closure, Header};
+use crate::header::{Body, Closure, Header};
 use crate::ids::ascii_name;
 use crate::limits;
 
+/// How many skipped keys the warning names.
+const WARNED_KEYS: usize = 5;
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct TensorRequirement {
     pub shape: Vec<u64>,
@@ -102,6 +105,7 @@ impl TensorRequirements {
     }
 }
 
+/// Where the checkpoint came from. Accepted and echoed; it no longer changes the verdict.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Custody {
     Canonical,
@@ -139,7 +143,11 @@ pub struct Routes {
 pub enum Fit {
     Ok {
         routes: Routes,
+        /// Stored tensors of a constructed component the code does not build, as
+        /// `component/key`: skipped, never loaded.
         ignored: Vec<String>,
+        /// What those tensors store, in bytes.
+        ignored_bytes: u64,
         device: Option<String>,
     },
     ComponentMissing {
@@ -162,6 +170,7 @@ pub enum Fit {
         required: Dtype,
         stored: Dtype,
     },
+    /// No longer answered: a stored key the code does not build is skipped (`Ok.ignored`).
     ExtraTensor {
         component: String,
         key: String,
@@ -207,13 +216,35 @@ impl Fit {
         matches!(self, Self::Ok { .. })
     }
 
+    /// The one line a caller shows when stored tensors were skipped: how many, how many
+    /// bytes, and the first few. A truncated model reads "420 stored tensor(s), 9.8 GB".
+    pub fn warning(&self) -> Option<String> {
+        let Self::Ok {
+            ignored,
+            ignored_bytes,
+            ..
+        } = self
+        else {
+            return None;
+        };
+        if ignored.is_empty() {
+            return None;
+        }
+        let more = match ignored.len().saturating_sub(WARNED_KEYS) {
+            0 => String::new(),
+            more => format!(", and {more} more"),
+        };
+        Some(format!(
+            "{} stored tensor(s) the code does not build were skipped, not loaded: {} B ({}{more})",
+            ignored.len(),
+            ignored_bytes,
+            ignored[..ignored.len().min(WARNED_KEYS)].join(", "),
+        ))
+    }
+
     pub fn text(&self) -> String {
         match self {
-            Self::Ok {
-                routes,
-                ignored,
-                device,
-            } => {
+            Self::Ok { routes, device, .. } => {
                 let mut text = format!(
                     "fits (routes: verbatim x{}, decoded_float x{}, encoded_gemm x{}",
                     routes.verbatim, routes.decoded_float, routes.encoded_gemm
@@ -221,13 +252,10 @@ impl Fit {
                 if let Some(device) = device {
                     text.push_str(&format!("; encoded leaves qualified on {device}"));
                 }
-                if !ignored.is_empty() {
-                    text.push_str(&format!(
-                        "; {} stored key(s) ignored on local custody",
-                        ignored.len()
-                    ));
-                }
                 text.push(')');
+                if let Some(warning) = self.warning() {
+                    text.push_str(&format!("; {warning}"));
+                }
                 text
             }
             Self::ComponentMissing { component, present } => format!(
@@ -258,7 +286,7 @@ impl Fit {
                 stored.name()
             ),
             Self::ExtraTensor { component, key } => {
-                format!("{component}.{key}: stored but not constructed on a canonical checkpoint")
+                format!("{component}.{key}: stored but not constructed")
             }
             Self::EncodingUnsupported {
                 component,
@@ -325,8 +353,9 @@ pub fn fit(
 ) -> Result<Fit> {
     header.validate(&Closure::default())?;
     let plain = plain_digest();
+    let _ = custody;
     let mut routes = Routes::default();
-    let mut ignored = Vec::new();
+    let (mut ignored, mut ignored_bytes) = (Vec::new(), 0u64);
     let mut encoded = Vec::new();
 
     for (component, required_tensors) in requirements.components() {
@@ -389,20 +418,22 @@ pub fn fit(
             }
         }
 
-        for (key, _) in stored_tensors {
+        // A stored key the code does not build is skipped, on every custody: the code may
+        // build less than an older checkpoint stores (Qwen's text encoder dropped `lm_head`).
+        for (key, stored) in stored_tensors {
             if required_tensors
                 .iter()
                 .any(|(required_key, _)| required_key == key)
             {
                 continue;
             }
-            if custody == Custody::Canonical {
-                return Ok(Fit::ExtraTensor {
-                    component: component.clone(),
-                    key: key.clone(),
-                });
-            }
             ignored.push(format!("{component}/{key}"));
+            for (_, part) in &stored.parts {
+                ignored_bytes += match &part.body {
+                    Body::Segments(objects) => objects.iter().map(|object| object.length).sum(),
+                    Body::Inline(bytes) => bytes.len() as u64,
+                };
+            }
         }
     }
 
@@ -436,6 +467,7 @@ pub fn fit(
     Ok(Fit::Ok {
         routes,
         ignored,
+        ignored_bytes,
         device: device.map(str::to_string),
     })
 }

{tensorfs-0.3.89 → tensorfs-0.3.90}/crates/tensorfs-core/src/ingest/fingerprint.rs

@@ -204,9 +204,20 @@ pub struct SourceProfile {
     /// Whether headers alone may choose this format. False requires the caller
     /// to declare the reviewed format; shapes cannot prove semantic row layout.
     pub auto_select: bool,
+    /// Set when a converter's planner, not banked key sets, recognizes the source.
+    pub grammar: Option<Grammar>,
     pub components: Vec<SourceProfileComponent>,
 }
 
+/// A carrier matches when every key starts with `key_prefix` and the reviewed `converter`
+/// plans it: the planner is the key grammar, so any valid subset of its reviewed modules
+/// matches without a banked key list. Its keys construct in sorted order.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Grammar {
+    pub converter: String,
+    pub key_prefix: String,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct SourceProfileComponent {
     pub component: String,
@@ -684,6 +695,13 @@ impl FingerprintRegistry {
                             if !profile.auto_select {
                                 fields.push(("auto_select", Value::Bool(false)));
                             }
+                            if let Some(grammar) = &profile.grammar {
+                                let converter = Value::str(grammar.converter.clone());
+                                let key_prefix = Value::str(grammar.key_prefix.clone());
+                                let grammar =
+                                    vec![("converter", converter), ("key_prefix", key_prefix)];
+                                fields.push(("grammar", Value::obj(grammar)));
+                            }
                             Value::obj(fields)
                         })
                         .collect(),
@@ -705,6 +723,19 @@ fn parse_source_profiles(value: &Value) -> Result<Vec<SourceProfile>> {
             Some(_) => return refuse(Code::WRONG_TYPE, "SourceProfile.auto_select must be bool"),
         };
         ascii_name("source profile", &name, limits::MAX_NAME_BYTES)?;
+        let grammar = match fields.opt("grammar") {
+            None => None,
+            Some(value) => {
+                let mut grammar = Fields::new("Grammar", value)?;
+                let converter = grammar.req_str("converter")?.to_string();
+                let key_prefix = grammar.req_str("key_prefix")?.to_string();
+                grammar.done()?;
+                Some(Grammar {
+                    converter,
+                    key_prefix,
+                })
+            }
+        };
         let mut components = Vec::new();
         for value in as_arr("SourceProfile", "components", fields.req("components")?)? {
             let mut component_fields = Fields::new("SourceProfileComponent", value)?;
@@ -795,7 +826,7 @@ fn parse_source_profiles(value: &Value) -> Result<Vec<SourceProfile>> {
                     ),
                 );
             }
-            if variants.is_empty() {
+            if variants.is_empty() && grammar.is_none() {
                 return refuse(
                     Code::MISSING_FIELD,
                     format!("source profile {name:?} component {component:?} has no variants"),
@@ -853,6 +884,7 @@ fn parse_source_profiles(value: &Value) -> Result<Vec<SourceProfile>> {
         profiles.push(SourceProfile {
             name,
             auto_select,
+            grammar,
             components,
         });
     }
@@ -1061,11 +1093,13 @@ mod tests {
         registry.source_profiles = vec![
             SourceProfile {
                 auto_select: true,
+                grammar: None,
                 name: "profile/a".into(),
                 components: vec![source_component(Some("a/model.index.json"), None)],
             },
             SourceProfile {
                 auto_select: true,
+                grammar: None,
                 name: "profile/b".into(),
                 components: vec![
                     source_component(Some("b/model.safetensors"), None),
@@ -1089,6 +1123,7 @@ mod tests {
         let mut registry = registry();
         registry.source_profiles = vec![SourceProfile {
             auto_select: true,
+            grammar: None,
             name: "profile/prefix".into(),
             components: vec![source_component(None, Some("weights/"))],
         }];

{tensorfs-0.3.89 → tensorfs-0.3.90}/crates/tensorfs-core/src/ingest/h3_lora.rs

@@ -6,7 +6,8 @@
 //! checkpoint's head interleave is reordered before training, not exported again
 //! in adapter state dictionaries. This is structural conversion, not permission
 //! to apply a FL2VA-trained adapter to REF2VA. Runtime owns that binding and never
-//! parses foreign keys.
+//! parses foreign keys. This planner is also the key grammar of the
+//! `h3/native-lora-*/2` profiles: any complete pairs on the reviewed projections.
 
 use std::collections::BTreeMap;
 
@@ -160,6 +161,29 @@ pub(super) fn plan(
     plain: &EncodingSpec,
     p: &mut Plan,
 ) -> Result<()> {
+    let adaln: Vec<&str> = header
+        .tensors
+        .iter()
+        .map(|tensor| tensor.key.as_str())
+        .filter(|key| key.contains("adaln_proj"))
+        .collect();
+    if !adaln.is_empty() {
+        // Named in full: these are the factors a conversion would have to drop.
+        let more = match adaln.len().saturating_sub(4) {
+            0 => String::new(),
+            more => format!(", and {more} more"),
+        };
+        return refuse(
+            Code::ADAPTER_TARGET_UNSUPPORTED,
+            format!(
+                "{} AdaLN modulation factor(s) ({}{more}). h3.lora/2 maps attention and MLP \
+                 projections only, and dropping AdaLN pairs changes the adapter. Use a repack \
+                 with the AdaLN pairs removed, or store the file as-is.",
+                adaln.len(),
+                adaln[..adaln.len().min(4)].join(", "),
+            ),
+        );
+    }
     let mut pairs: BTreeMap<String, Pair<'_>> = BTreeMap::new();
     for tensor in &header.tensors {
         if !matches!(tensor.dtype, Dtype::F16 | Dtype::Bf16 | Dtype::F32) {