tensorfs 0.3.56__tar.gz → 0.3.57__tar.gz

{tensorfs-0.3.56 → tensorfs-0.3.57}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.56"
+version = "0.3.57"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.56"
+version = "0.3.57"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.56 → tensorfs-0.3.57}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.56"
+version = "0.3.57"
 edition = "2021"

{tensorfs-0.3.56 → tensorfs-0.3.57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.56
+Version: 0.3.57
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.56 → tensorfs-0.3.57}/crates/tensorfs-core/Cargo.toml

@@ -11,7 +11,7 @@ path = "src/bin/tfs/main.rs"
 fs2 = "0.4"
 minicbor = { version = "2.3.0", default-features = false, features = ["alloc"] }
 rusqlite = { version = "0.32", features = ["bundled"] }
-rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "net", "event"] }
+rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "net", "event", "process"] }
 # tfs-049: the ONE transport (src/transport/). rustls is the one TLS stack, used only
 # there; scripts/fence.py refuses it anywhere else so a second mover cannot grow.
 rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }

{tensorfs-0.3.56 → tensorfs-0.3.57}/crates/tensorfs-core/src/bin/tfs/repo.rs

@@ -80,9 +80,6 @@ pub fn cmd_manifest_build(entries_path: &Path, flags: &Flags) -> ExitCode {
             Ok(path) => path.to_string(),
             Err(error) => return bail(error),
         };
-        if let Err(error) = fields.done() {
-            return bail(error);
-        }
         files.push((path, blob));
     }
     let manifest = match Manifest::from_files(files) {

{tensorfs-0.3.56 → tensorfs-0.3.57}/crates/tensorfs-core/src/cache_roots.rs

@@ -316,6 +316,7 @@ mod tests {
                     expected_revision: 0,
                     remove: vec![],
                     set: vec![crate::repository::ReleaseLane {
+                        extra: Default::default(),
                         lane: "bf16".into(),
                         manifest,
                     }],

{tensorfs-0.3.56 → tensorfs-0.3.57}/crates/tensorfs-core/src/canon.rs

@@ -36,6 +36,14 @@ impl Value {
         v.sort_by(|a, b| a.0.cmp(&b.0));
         Value::Obj(v)
     }
+    /// `obj` plus the fields a parse carried through unread.
+    pub fn obj_with(pairs: Vec<(&str, Value)>, extra: &Extra) -> Value {
+        let mut v: Vec<(String, Value)> =
+            pairs.into_iter().map(|(k, x)| (k.to_string(), x)).collect();
+        v.extend(extra.0.iter().cloned());
+        v.sort_by(|a, b| a.0.cmp(&b.0));
+        Value::Obj(v)
+    }
     pub fn str(s: impl Into<String>) -> Value {
         Value::Str(s.into())
     }
@@ -357,7 +365,47 @@ impl<'a> P<'a> {
     }
 }
 
-// ---------------------------------------------------------------- strict field reader
+// ---------------------------------------------------------------- field reader
+
+/// Fields this build does not read. A stored document carries them through unchanged, so a
+/// newer writer's additions keep their bytes, identity and meaning across an older reader.
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct Extra(Vec<(String, Value)>);
+
+impl Extra {
+    /// Every `{sha256, length}` object anywhere in the unread fields. Reachability treats
+    /// them as referenced: GC never deletes what a newer writer may have named.
+    pub fn refs(&self) -> Vec<crate::ids::ObjectRef> {
+        fn walk(value: &Value, out: &mut Vec<crate::ids::ObjectRef>) {
+            match value {
+                Value::Arr(items) => items.iter().for_each(|item| walk(item, out)),
+                Value::Obj(pairs) => {
+                    let sha256 = pairs.iter().find_map(|(k, v)| match (k.as_str(), v) {
+                        ("sha256", Value::Str(s)) => Some(s),
+                        _ => None,
+                    });
+                    let length = pairs.iter().find_map(|(k, v)| match (k.as_str(), v) {
+                        ("length", Value::Int(n)) if *n >= 0 => Some(*n as u64),
+                        _ => None,
+                    });
+                    if let (Some(sha256), Some(length)) = (sha256, length) {
+                        if crate::ids::hex64("unread reference", sha256).is_ok() {
+                            out.push(crate::ids::ObjectRef {
+                                sha256: sha256.clone(),
+                                length,
+                            });
+                        }
+                    }
+                    pairs.iter().for_each(|(_, v)| walk(v, out));
+                }
+                _ => {}
+            }
+        }
+        let mut out = Vec::new();
+        self.0.iter().for_each(|(_, v)| walk(v, &mut out));
+        out
+    }
+}
 
 pub struct Fields<'a> {
     what: &'static str,
@@ -396,6 +444,8 @@ impl<'a> Fields<'a> {
     pub fn req_uint(&mut self, key: &str) -> Result<u64> {
         as_uint(self.what, key, self.req(key)?)
     }
+    /// End a document this build must understand completely: a command, or a record whose
+    /// unread field could change what it means.
     pub fn done(self) -> Result<()> {
         match self.left.first() {
             None => Ok(()),
@@ -405,6 +455,16 @@ impl<'a> Fields<'a> {
             ),
         }
     }
+
+    /// End a document that tolerates additive fields, keeping them for re-emission.
+    pub fn rest(self) -> Extra {
+        Extra(
+            self.left
+                .into_iter()
+                .map(|(k, v)| (k.to_string(), v.clone()))
+                .collect(),
+        )
+    }
 }
 
 pub fn as_str<'a>(what: &str, key: &str, v: &'a Value) -> Result<&'a str> {

{tensorfs-0.3.56 → tensorfs-0.3.57}/crates/tensorfs-core/src/catalog.rs

@@ -19,6 +19,9 @@ use crate::storage::Projection;
 /// the floor under any patience a supervisor grants a worker that has gone quiet.
 pub const BUSY_TIMEOUT: Duration = Duration::from_secs(5);
 
+/// The catalog this build writes. A newer build may only ADD tables and columns (an
+/// incompatible change renames the table), so any catalog at this version or later that
+/// holds every table and column below opens.
 const SCHEMA_VERSION: u64 = 2;
 
 const SCHEMA: &[(&str, &[&str])] = &[
@@ -63,50 +66,25 @@ fn io(what: impl AsRef<str>, error: std::io::Error) -> Refusal {
     crate::store::classify_io(what, error)
 }
 
+/// Every table and column this build uses is present. Additions by a newer build are its own.
 fn verify_schema(connection: &Connection) -> Result<()> {
-    let mut statement = connection
-        .prepare(
-            "SELECT name FROM sqlite_master
-             WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
-        )
-        .map_err(|error| sql("prepare catalog schema census", error))?;
-    let tables: Vec<String> = statement
-        .query_map([], |row| row.get(0))
-        .map_err(|error| sql("read catalog schema census", error))?
-        .collect::<std::result::Result<_, _>>()
-        .map_err(|error| sql("collect catalog schema census", error))?;
-    let expected: Vec<String> = SCHEMA.iter().map(|(name, _)| (*name).to_string()).collect();
-    if tables != expected {
-        return refuse(
-            Code::UNKNOWN_FORMAT,
-            "tensorfs.sqlite is not the exact v1 catalog; remove it and rebuild from repository files",
-        );
-    }
     for (table, expected_columns) in SCHEMA {
         let mut columns = connection
-            .prepare("SELECT name FROM pragma_table_info(?1) ORDER BY cid")
+            .prepare("SELECT name FROM pragma_table_info(?1)")
             .map_err(|error| sql("prepare catalog column census", error))?;
         let actual: Vec<String> = columns
             .query_map([table], |row| row.get(0))
             .map_err(|error| sql("read catalog column census", error))?
             .collect::<std::result::Result<_, _>>()
             .map_err(|error| sql("collect catalog column census", error))?;
-        let expected: Vec<String> = expected_columns
+        if let Some(missing) = expected_columns
             .iter()
-            .map(|name| (*name).to_string())
-            .collect();
-        let strict: u64 = connection
-            .query_row(
-                "SELECT strict FROM pragma_table_list WHERE schema='main' AND name=?1",
-                [table],
-                |row| row.get(0),
-            )
-            .map_err(|error| sql("read catalog strict-table census", error))?;
-        if actual != expected || strict != 1 {
+            .find(|column| !actual.iter().any(|name| name == *column))
+        {
             return refuse(
                 Code::UNKNOWN_FORMAT,
                 format!(
-                    "tensorfs.sqlite table {table} is not the exact v1 shape; remove the catalog and rebuild from repository files"
+                    "tensorfs.sqlite table {table} has no column {missing}; remove the catalog and rebuild from repository files"
                 ),
             );
         }
@@ -162,10 +140,12 @@ fn validated(path: &Path) -> Result<Connection> {
     let version: u64 = connection
         .query_row("PRAGMA user_version", [], |row| row.get(0))
         .map_err(|error| sql("read catalog version", error))?;
-    if version != SCHEMA_VERSION {
+    if version < SCHEMA_VERSION {
         return refuse(
             Code::UNKNOWN_FORMAT,
-            format!("tensorfs.sqlite user_version {version} != {SCHEMA_VERSION}"),
+            format!(
+                "tensorfs.sqlite user_version {version} predates {SCHEMA_VERSION}; remove it and rebuild from repository files"
+            ),
         );
     }
     verify_schema(&connection)?;
@@ -410,7 +390,7 @@ impl Catalog {
         let version: u64 = connection
             .query_row("PRAGMA user_version", [], |row| row.get(0))
             .map_err(|error| sql("read catalog version", error))?;
-        if version == SCHEMA_VERSION {
+        if version >= SCHEMA_VERSION {
             connection
                 .execute_batch("COMMIT")
                 .map_err(|error| sql("commit catalog open", error))?;
@@ -421,7 +401,7 @@ impl Catalog {
             return refuse(
                 Code::UNKNOWN_FORMAT,
                 format!(
-                    "tensorfs.sqlite user_version {version} is not supported; remove it and rebuild from repository files"
+                    "tensorfs.sqlite user_version {version} predates {SCHEMA_VERSION}; remove it and rebuild from repository files"
                 ),
             );
         }
@@ -1418,7 +1398,7 @@ mod tests {
     }
 
     #[test]
-    fn catalog_has_seven_tables_and_refuses_future_versions_without_mutation() {
+    fn catalog_has_seven_tables_and_opens_additive_future_versions() {
         let root = temporary("schema");
         fs::create_dir_all(&root).unwrap();
         let catalog = Catalog::initialize(&root).unwrap();
@@ -1448,17 +1428,41 @@ mod tests {
                 "tensorfs_verified_blobs",
             ]
         );
-        connection.execute_batch("PRAGMA user_version=3").unwrap();
+        // A newer build added a table and a column: this build still opens and uses it.
+        connection
+            .execute_batch(
+                "CREATE TABLE tensorfs_future(value TEXT);
+                 ALTER TABLE tensorfs_holds ADD COLUMN note TEXT;
+                 PRAGMA user_version=3;",
+            )
+            .unwrap();
         drop(connection);
-        assert_eq!(
-            Catalog::initialize(&root).unwrap_err().code,
-            Code::UNKNOWN_FORMAT
-        );
+        let reopened = Catalog::initialize(&root).unwrap();
+        let guard = reopened.begin_operation("op", "org", "repo").unwrap();
+        reopened
+            .hold(
+                "op",
+                &crate::storage::blob_key(&"ab".repeat(32)).unwrap(),
+                7,
+            )
+            .unwrap();
+        drop(guard);
+        drop(reopened);
+        Catalog::open(&root).unwrap();
         let connection = rusqlite::Connection::open(Catalog::path(&root)).unwrap();
         let version: u64 = connection
             .query_row("PRAGMA user_version", [], |row| row.get(0))
             .unwrap();
         assert_eq!(version, 3);
+        // A catalog without a column this build uses is genuinely unreadable.
+        connection
+            .execute_batch("ALTER TABLE tensorfs_verified_blobs DROP COLUMN mtime_ns")
+            .unwrap();
+        drop(connection);
+        assert_eq!(
+            Catalog::initialize(&root).unwrap_err().code,
+            Code::UNKNOWN_FORMAT
+        );
         let _ = fs::remove_dir_all(root);
     }
 

{tensorfs-0.3.56 → tensorfs-0.3.57}/crates/tensorfs-core/src/derived/resume_tests.rs

@@ -437,6 +437,7 @@ fn source_identity_is_locked_and_source_closure_stays_rooted_after_writer_death(
                     repo: repo.clone(),
                     remove: Vec::new(),
                     set: vec![ReleaseLane {
+                        extra: Default::default(),
                         lane: "main".into(),
                         manifest: receipt.manifest.clone(),
                     }],

{tensorfs-0.3.56 → tensorfs-0.3.57}/crates/tensorfs-core/src/derived.rs

@@ -4191,6 +4191,7 @@ mod tests {
                     repo,
                     remove: Vec::new(),
                     set: vec![ReleaseLane {
+                        extra: Default::default(),
                         lane: "main".into(),
                         manifest: manifest_ref.clone(),
                     }],

{tensorfs-0.3.56 → tensorfs-0.3.57}/crates/tensorfs-core/src/fetch.rs

@@ -35,26 +35,35 @@ pub fn parse_refs_jsonl(bytes: &[u8]) -> Result<Vec<ObjectRef>> {
         .split(|byte| *byte == b'\n')
         .filter(|line| !line.is_empty())
     {
-        let value = canon::parse_canonical(line, 16 * 1024)?;
+        let value = canon::parse(line, 16 * 1024)?;
         let mut fields = Fields::new("FetchRef", &value)?;
         let length = fields.req_uint("length")?;
         let sha256 = hex64("FetchRef.sha256", fields.req_str("sha256")?)?;
-        fields.done()?;
         result.push(ObjectRef { sha256, length });
     }
     if result.is_empty() {
         return refuse(Code::MISSING_FIELD, "fetch refs are empty");
     }
-    if result
+    sorted_refs("fetch refs", result)
+}
+
+/// Digest order with identical repeats merged. One digest at two lengths still refuses.
+fn sorted_refs(what: &str, mut refs: Vec<ObjectRef>) -> Result<Vec<ObjectRef>> {
+    refs.sort_by(|a, b| a.sha256.cmp(&b.sha256).then(a.length.cmp(&b.length)));
+    refs.dedup();
+    if let Some(pair) = refs
         .windows(2)
-        .any(|pair| pair[0].sha256 >= pair[1].sha256)
+        .find(|pair| pair[0].sha256 == pair[1].sha256)
     {
         return refuse(
-            Code::SORT_ORDER,
-            "fetch refs must be strictly sorted and unique by digest",
+            Code::LENGTH_MISMATCH,
+            format!(
+                "{what} name sha256:{} at {} B and at {} B",
+                pair[0].sha256, pair[0].length, pair[1].length
+            ),
         );
     }
-    Ok(result)
+    Ok(refs)
 }
 
 pub fn final_key(sha256: &str) -> Result<String> {
@@ -148,16 +157,7 @@ impl FetchPlan {
                  something a worker can pull",
             );
         }
-        let mut sorted: Vec<ObjectRef> = declared.to_vec();
-        sorted.sort_by(|a, b| a.sha256.cmp(&b.sha256));
-        sorted.dedup_by(|a, b| a.sha256 == b.sha256);
-        if sorted.len() != declared.len() {
-            return refuse(
-                Code::SORT_ORDER,
-                "the declared object set lists one digest twice — two opinions about how many \
-                 bytes this pull moves",
-            );
-        }
+        let mut sorted = sorted_refs("the declared object set", declared.to_vec())?;
         if let Some(manifest) = manifest {
             if !sorted.iter().any(|object| object.sha256 == manifest.sha256) {
                 sorted.push(manifest.clone());
@@ -318,7 +318,7 @@ fn presence(store: &Store, o: &ObjectRef) -> Result<Presence> {
 
 impl FetchPlan {
     pub fn parse(bytes: &[u8]) -> Result<Self> {
-        Self::from_value(&canon::parse_canonical(bytes, limits::DOC_MAX_BYTES)?)
+        Self::from_value(&canon::parse(bytes, limits::DOC_MAX_BYTES)?)
     }
     pub fn canonical_bytes(&self) -> Vec<u8> {
         canon::write(&self.to_value())
@@ -343,20 +343,9 @@ impl FetchPlan {
         for o in as_arr("FetchPlan", "wanted", f.req("wanted")?)? {
             wanted.push(ObjectRef::from_value("FetchPlan.wanted", o)?);
         }
-        f.done()?;
         ascii_name("FetchPlan.session", &session, limits::MAX_NAME_BYTES)?;
-        for (what, set) in [("held", &held), ("wanted", &wanted)] {
-            let mut ids: Vec<&str> = set.iter().map(|o| o.sha256.as_str()).collect();
-            let given = ids.clone();
-            ids.sort_unstable();
-            ids.dedup();
-            if ids != given {
-                return refuse(
-                    Code::SORT_ORDER,
-                    format!("FetchPlan.{what} must be sorted and unique by digest"),
-                );
-            }
-        }
+        let held = sorted_refs("FetchPlan.held", held)?;
+        let wanted = sorted_refs("FetchPlan.wanted", wanted)?;
         if held.is_empty() && wanted.is_empty() {
             return refuse(Code::MISSING_FIELD, "a fetch plan covers no object");
         }
@@ -921,7 +910,6 @@ impl DeliveryGrant {
         };
         let object = ObjectRef::from_value("DeliveryGrant.object", f.req("object")?)?;
         let session = f.req_str("session")?.to_string();
-        f.done()?;
         ascii_name("DeliveryGrant.session", &session, limits::MAX_NAME_BYTES)?;
         Ok(DeliveryGrant {
             session,

{tensorfs-0.3.56 → tensorfs-0.3.57}/crates/tensorfs-core/src/ingest/fingerprint.rs

@@ -508,34 +508,28 @@ impl FingerprintRegistry {
                 provenance,
             });
         }
+        // A newer registry's additional top-level sections belong to a newer reader.
         let source_profiles = match f.opt("source_profiles") {
             None => Vec::new(),
             Some(value) => parse_source_profiles(value)?,
         };
-        f.done()?;
-        let mut seen: Vec<(&str, &str, &str)> = entries
-            .iter()
-            .map(|e| {
-                (
-                    e.keyset_digest.as_str(),
-                    e.component.as_str(),
-                    e.tensor_schema_digest.as_str(),
-                )
-            })
-            .collect();
-        let given = seen.clone();
-        seen.sort();
-        if seen != given {
-            return refuse(
-                Code::SORT_ORDER,
-                "registry entries must be sorted by (keyset_digest, component, tensor_schema_digest)",
-            );
-        }
-        seen.dedup();
-        if seen.len() != entries.len() {
+        let key = |e: &Banked| {
+            (
+                e.keyset_digest.clone(),
+                e.component.clone(),
+                e.tensor_schema_digest.clone(),
+            )
+        };
+        entries.sort_by_key(key);
+        entries.dedup();
+        if entries
+            .windows(2)
+            .any(|pair| key(&pair[0]) == key(&pair[1]))
+        {
             return refuse(
                 Code::DUPLICATE_KEY,
-                "two entries bank the same (keyset_digest, component, tensor_schema_digest)",
+                "two entries bank the same (keyset_digest, component, tensor_schema_digest) \
+                 with different verdicts",
             );
         }
         for profile in &source_profiles {
@@ -997,12 +991,12 @@ mod tests {
         assert!(!String::from_utf8_lossy(&bytes).contains("format"));
         assert_eq!(FingerprintRegistry::parse(&bytes).unwrap(), registry);
 
-        let old = String::from_utf8(bytes)
+        let newer = String::from_utf8(bytes)
             .unwrap()
             .replacen('{', r#"{"format":"retired","#, 1);
         assert_eq!(
-            FingerprintRegistry::parse(old.as_bytes()).unwrap_err().code,
-            Code::UNKNOWN_FIELD
+            FingerprintRegistry::parse(newer.as_bytes()).unwrap(),
+            registry
         );
     }
 

{tensorfs-0.3.56 → tensorfs-0.3.57}/crates/tensorfs-core/src/ingest/preflight.rs

@@ -262,9 +262,21 @@ pub fn plan(
     }
     let mut sorted: Vec<&str> = profiles.iter().map(String::as_str).collect();
     sorted.sort();
-    if sorted.windows(2).any(|window| window[0] == window[1]) {
-        return refuse(Code::DUPLICATE_KEY, "source profiles must be unique");
+    sorted.dedup();
+    let mut unique: Vec<MemberHead> = Vec::with_capacity(heads.len());
+    for head in heads {
+        match unique.iter().find(|kept| kept.member == head.member) {
+            Some(kept) if kept == head => {}
+            Some(_) => {
+                return refuse(
+                    Code::DUPLICATE_KEY,
+                    format!("{} is listed twice with different heads", head.member),
+                )
+            }
+            None => unique.push(head.clone()),
+        }
     }
+    let heads = unique.as_slice();
     let staged = stage(scratch, heads)?;
     let mut plans = Vec::with_capacity(profiles.len());
     for profile in &sorted {