tensorfs 0.3.65__tar.gz → 0.3.67__tar.gz

{tensorfs-0.3.65 → tensorfs-0.3.67}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.65"
+version = "0.3.67"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.65"
+version = "0.3.67"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.65 → tensorfs-0.3.67}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.65"
+version = "0.3.67"
 edition = "2021"

{tensorfs-0.3.65 → tensorfs-0.3.67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.65
+Version: 0.3.67
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.65 → tensorfs-0.3.67}/crates/tensorfs-core/src/bin/tfs/fetch.rs

@@ -1358,6 +1358,7 @@ struct PullProgress {
 struct PullProgressState {
     total_bytes: u64,
     held_bytes: u64,
+    wanted_objects: u64,
     origin_bytes: u64,
     cache_bytes: u64,
     origin_objects: u64,
@@ -1371,6 +1372,7 @@ impl PullProgress {
             state: Mutex::new(PullProgressState {
                 total_bytes: 0,
                 held_bytes: 0,
+                wanted_objects: 0,
                 origin_bytes: 0,
                 cache_bytes: 0,
                 origin_objects: 0,
@@ -1385,6 +1387,7 @@ impl PullProgress {
         let mut state = self.state.lock().unwrap();
         state.total_bytes = plan.declared_bytes();
         state.held_bytes = plan.held_bytes();
+        state.wanted_objects = plan.wanted.len() as u64;
         state.last = Some(Instant::now());
         print_progress(&state);
     }
@@ -1447,11 +1450,12 @@ impl PullProgress {
 ///
 /// Every counter is always present, including a zero. A consumer that reads an absent key
 /// as zero cannot tell a cache that answered nothing from a build that never reported the
-/// door at all, and those are the two states worth telling apart.
+/// door at all, and those are the two states worth telling apart. `wanted_objects` is the
+/// plan's count of objects to land; a pod host sizes this pull's storage reservation from it.
 fn print_progress(state: &PullProgressState) {
     eprintln!(
-        "{{\"event\":\"pull.progress\",\"total_bytes\":{},\"held_bytes\":{},\"origin_bytes\":{},\"cache_bytes\":{},\"origin_objects\":{},\"cache_objects\":{}}}",
-        state.total_bytes, state.held_bytes, state.origin_bytes, state.cache_bytes,
+        "{{\"event\":\"pull.progress\",\"total_bytes\":{},\"held_bytes\":{},\"wanted_objects\":{},\"origin_bytes\":{},\"cache_bytes\":{},\"origin_objects\":{},\"cache_objects\":{}}}",
+        state.total_bytes, state.held_bytes, state.wanted_objects, state.origin_bytes, state.cache_bytes,
         state.origin_objects, state.cache_objects
     );
 }

{tensorfs-0.3.65 → tensorfs-0.3.67}/crates/tensorfs-core/src/bin/tfs/rd.rs

@@ -133,9 +133,13 @@ pub fn cmd_plan(root: &Path, hex: &str, f: &Flags) -> ExitCode {
     let mut rev = construction_order(&h);
     rev.reverse();
     println!(
-        "contract:   reversed caller traversal refuses: {}",
+        "contract:   reversed caller traversal plans in its own order: {}",
         match read::plan_for_traversal(&h, &rev, &all_components(&h), window) {
-            Err(e) if e.code == Code::TRAVERSAL_ORDER_MISMATCH => "YES",
+            Ok(p)
+                if p.items.first().is_some_and(|it| it
+                    .what
+                    .starts_with(&format!("{}/{}#", rev[0].0, rev[0].1))) =>
+                "YES",
             _ => "NO — DEFECT",
         }
     );
@@ -991,7 +995,7 @@ pub fn cmd_arms(root: &Path, hex: &str) -> ExitCode {
     short.pop();
     a.arm(
         "traversal missing a tensor",
-        Code::TRAVERSAL_ORDER_MISMATCH,
+        Code::TRAVERSAL_INCOMPLETE,
         read::plan_for_traversal(&h, &short, &all_components(&h), 1 << 20),
     );
 
@@ -1026,7 +1030,7 @@ pub fn cmd_arms(root: &Path, hex: &str) -> ExitCode {
         holed.pop();
         a.arm(
             "a tensor missing WITHIN a requested component",
-            Code::TRAVERSAL_ORDER_MISMATCH,
+            Code::TRAVERSAL_INCOMPLETE,
             read::plan_for_traversal(&h, &holed, only_one, 1 << 20),
         );
     }
@@ -1042,7 +1046,7 @@ pub fn cmd_arms(root: &Path, hex: &str) -> ExitCode {
     }
     a.arm(
         "a tensor from a component the plan did not request",
-        Code::TRAVERSAL_ORDER_MISMATCH,
+        Code::TRAVERSAL_INCOMPLETE,
         read::plan_for_traversal(&h, &smuggled, only_one, 1 << 20),
     );
 

{tensorfs-0.3.65 → tensorfs-0.3.67}/crates/tensorfs-core/src/catalog.rs

@@ -62,6 +62,11 @@ fn sql(what: impl AsRef<str>, error: rusqlite::Error) -> Refusal {
     }
 }
 
+/// Whether a catalog refusal is SQLite's busy timeout expiring behind another writer.
+fn busy(refusal: &Refusal) -> bool {
+    refusal.code == Code::IO_FAILED && refusal.detail.ends_with("database is locked")
+}
+
 fn io(what: impl AsRef<str>, error: std::io::Error) -> Refusal {
     crate::store::classify_io(what, error)
 }
@@ -115,7 +120,20 @@ fn connect(path: &Path) -> Result<Connection> {
         .entry(path.to_path_buf())
         .or_default() += 1;
     crate::stats::catalog_opened();
-    let connection = Connection::open(path).map_err(|error| sql("open catalog", error))?;
+    let connection = Connection::open(path).map_err(|error| {
+        // SQLite reports a full descriptor table only as "unable to open"; the table
+        // itself answers whether that is what happened.
+        if crate::descriptors::available().is_some_and(|free| free == 0) {
+            crate::descriptors::exhausted(&std::io::Error::from_raw_os_error(
+                rustix::io::Errno::MFILE.raw_os_error(),
+            ));
+            return Refusal {
+                code: Code::FD_HEADROOM,
+                detail: format!("open catalog: no file descriptor ({error})"),
+            };
+        }
+        sql("open catalog", error)
+    })?;
     connection
         .busy_timeout(BUSY_TIMEOUT)
         .map_err(|error| sql("set catalog busy timeout", error))?;
@@ -372,6 +390,22 @@ impl Catalog {
         Ok(catalog)
     }
 
+    /// `open` for an object admission. A catalog busy past SQLite's timeout while this
+    /// process's own admission batches keep committing is a queue, not a failure: the
+    /// verified bytes wait for their turn instead of being thrown away and fetched again.
+    /// Busy with no batch committed in the meantime is another writer's lock, and refuses.
+    pub(crate) fn open_admitting(root: &Path) -> Result<Self> {
+        loop {
+            let committed = crate::stats::snapshot().admission_batches;
+            match Self::open(root) {
+                Err(refusal)
+                    if busy(&refusal) && crate::stats::snapshot().admission_batches > committed => {
+                }
+                opened => return opened,
+            }
+        }
+    }
+
     pub fn open(root: &Path) -> Result<Self> {
         let path = Self::path(root);
         validated(&path)?;
@@ -1473,16 +1507,7 @@ impl WriterGuard {
             .read(true)
             .write(true)
             .open(&path)
-            .map_err(|error| {
-                if crate::descriptors::exhausted(&error) {
-                    Refusal {
-                        code: Code::FD_HEADROOM,
-                        detail: format!("create writer marker: {error}"),
-                    }
-                } else {
-                    io("create writer marker", error)
-                }
-            })?;
+            .map_err(|error| io("create writer marker", error))?;
         file.try_lock_exclusive()
             .map_err(|error| io("lock writer marker", error))?;
         Ok(Self {

{tensorfs-0.3.65 → tensorfs-0.3.67}/crates/tensorfs-core/src/ingest/preflight.rs

@@ -239,6 +239,26 @@ fn profile_plan(plan: &SourcePlan) -> ProfilePlan {
     }
 }
 
+/// The one reviewed profile these member heads plan under, chosen from their headers alone:
+/// several matching profiles that plan identically are one answer; different plans refuse
+/// `AMBIGUOUS_CLASSIFICATION` naming them, and none refuses `UNREGISTERED_FINGERPRINT`.
+pub fn select_profile(
+    registry_locator: &str,
+    registry_bytes: &[u8],
+    heads: &[MemberHead],
+    scratch: &Path,
+) -> Result<String> {
+    let staged = stage(scratch, heads)?;
+    source::plan_source(
+        registry_locator,
+        registry_bytes,
+        staged.carriers(),
+        None,
+        None,
+    )
+    .map(|plan| plan.profile)
+}
+
 /// Decide the conversion plan for every requested profile, from headers alone.
 ///
 /// One [`source::plan_source`] call per profile, over one carrier set — which is precisely

{tensorfs-0.3.65 → tensorfs-0.3.67}/crates/tensorfs-core/src/ingest/source.rs

@@ -690,12 +690,6 @@ fn plan_source_prepared(
         }
         for assignments in profile_assignments(profile, &registry, carriers, &headers) {
             matches.push((profile, assignments));
-            if matches.len() >= 2 {
-                break;
-            }
-        }
-        if matches.len() >= 2 {
-            break;
         }
     }
     if matches.is_empty() {
@@ -704,20 +698,61 @@ fn plan_source_prepared(
             "carrier set matches no reviewed source profile",
         );
     }
-    if matches.len() != 1 {
-        let profiles: Vec<&str> = matches
+    // Every match is planned. Matches whose plans are identical (the same sources, order,
+    // target and session under different profile names) are one answer, named by the most
+    // specific profile; only genuinely different plans are ambiguous.
+    let mut planned = Vec::with_capacity(matches.len());
+    let mut first_error = None;
+    for (profile, assignments) in matches {
+        match plan_match(
+            profile,
+            assignments,
+            carriers,
+            &set,
+            &registry,
+            registry_locator,
+            registry_bytes,
+            spec_variant,
+        ) {
+            Ok(result) => planned.push((profile, result)),
+            Err(error) => {
+                first_error.get_or_insert(error);
+            }
+        }
+    }
+    if planned.is_empty() {
+        return Err(first_error.expect("every match was planned or refused"));
+    }
+    let same = |left: &SourcePlan, right: &SourcePlan| {
+        (
+            &left.session,
+            &left.construction_order,
+            &left.sources,
+            &left.target,
+        ) == (
+            &right.session,
+            &right.construction_order,
+            &right.sources,
+            &right.target,
+        )
+    };
+    if planned
+        .iter()
+        .any(|(_, (plan, _))| !same(plan, &planned[0].1 .0))
+    {
+        let mut profiles: Vec<&str> = planned
             .iter()
             .map(|(profile, _)| profile.name.as_str())
             .collect();
-        let mut members: Vec<String> = matches
+        profiles.dedup();
+        let mut members: Vec<String> = planned
             .iter()
-            .flat_map(|(_, assignments)| {
-                assignments.iter().map(|assignment| {
-                    let carrier = &carriers[assignment.carrier];
-                    carrier
-                        .member
+            .flat_map(|(_, (plan, _))| {
+                plan.sources.iter().map(|source| {
+                    source
+                        .source_member
                         .clone()
-                        .unwrap_or_else(|| carrier.path.display().to_string())
+                        .unwrap_or_else(|| source.path.display().to_string())
                 })
             })
             .collect();
@@ -731,7 +766,42 @@ fn plan_source_prepared(
             ),
         );
     }
-    let (profile, mut assignments) = matches.remove(0);
+    planned.sort_by_key(|(profile, _)| {
+        (
+            std::cmp::Reverse(specificity(profile)),
+            profile.name.clone(),
+        )
+    });
+    let (_, (plan, prepared)) = planned.swap_remove(0);
+    Ok((plan, prepared))
+}
+
+/// How exactly a profile binds its components: provider members, then member prefixes.
+fn specificity(profile: &SourceProfile) -> (usize, usize) {
+    let exact = profile
+        .components
+        .iter()
+        .filter(|component| component.source_member.is_some())
+        .count();
+    let prefixed = profile
+        .components
+        .iter()
+        .filter(|component| component.source_member_prefix.is_some())
+        .count();
+    (exact, prefixed)
+}
+
+#[allow(clippy::too_many_arguments)]
+fn plan_match(
+    profile: &SourceProfile,
+    mut assignments: Vec<SourceAssignment>,
+    carriers: &[CarrierInput],
+    set: &CarrierSet,
+    registry: &FingerprintRegistry,
+    registry_locator: &str,
+    registry_bytes: &[u8],
+    spec_variant: Option<usize>,
+) -> Result<(SourcePlan, Prepared)> {
     assignments.sort_by(|left, right| left.component.cmp(&right.component));
     let sources: Vec<(String, CarrierInput)> = assignments
         .iter()
@@ -762,7 +832,7 @@ fn plan_source_prepared(
                 .map(|key| (component.component.clone(), key.clone()))
         })
         .collect();
-    let mut prepared = prepare_with_registry(&target, &sources, &set, &registry, spec_variant)?;
+    let mut prepared = prepare_with_registry(&target, &sources, set, registry, spec_variant)?;
     prepared.plan.apply_order(&order)?;
     prepared.plan.check_converts_something()?;
     let plan = SourcePlan {

{tensorfs-0.3.65 → tensorfs-0.3.67}/crates/tensorfs-core/src/read.rs

@@ -24,7 +24,7 @@ use std::time::Instant;
 
 use crate::checkpoint;
 use crate::err::{refuse, Code, Refusal, Result};
-use crate::header::{Asset, Body, Header, Located, Part};
+use crate::header::{Asset, Body, Header, Located, Part, Tensor};
 use crate::ids::ObjectRef;
 use crate::manifest::Manifest;
 use crate::meta::{Hold, Meta};
@@ -594,48 +594,43 @@ impl ReadPlan {
     }
 }
 
-fn build(header: &Header, scope: &[&str], window: u64) -> Result<ReadPlan> {
+fn build(ordered: &[(&str, &str, &Tensor)], window: u64) -> Result<ReadPlan> {
     let mut items = Vec::new();
     let mut dest: u64 = 0;
-    for (comp, tensors) in &header.components {
-        if !scope.contains(&comp.as_str()) {
-            continue;
-        }
-        for (key, tensor) in tensors {
-            for (role, part) in &tensor.parts {
-                let what = format!("{comp}/{key}#{role}");
-                if let Body::Inline(b) = &part.body {
-                    let n = part.nbytes(&what)?;
+    for (comp, key, tensor) in ordered {
+        for (role, part) in &tensor.parts {
+            let what = format!("{comp}/{key}#{role}");
+            if let Body::Inline(b) = &part.body {
+                let n = part.nbytes(&what)?;
+                items.push(PlanItem {
+                    what: what.clone(),
+                    dest_off: dest,
+                    len: n,
+                    source: Source::Inline(b.clone()),
+                });
+                dest += n;
+                continue;
+            }
+            for segment in part.segments() {
+                let mut off = 0u64;
+                while off < segment.length {
+                    let n = if window == 0 {
+                        segment.length - off
+                    } else {
+                        window.min(segment.length - off)
+                    };
                     items.push(PlanItem {
                         what: what.clone(),
                         dest_off: dest,
                         len: n,
-                        source: Source::Inline(b.clone()),
+                        source: Source::Object(ObjectRange {
+                            obj: segment.clone(),
+                            off,
+                            len: n,
+                        }),
                     });
                     dest += n;
-                    continue;
-                }
-                for segment in part.segments() {
-                    let mut off = 0u64;
-                    while off < segment.length {
-                        let n = if window == 0 {
-                            segment.length - off
-                        } else {
-                            window.min(segment.length - off)
-                        };
-                        items.push(PlanItem {
-                            what: what.clone(),
-                            dest_off: dest,
-                            len: n,
-                            source: Source::Object(ObjectRange {
-                                obj: segment.clone(),
-                                off,
-                                len: n,
-                            }),
-                        });
-                        dest += n;
-                        off += n;
-                    }
+                    off += n;
                 }
             }
         }
@@ -658,8 +653,10 @@ fn tally(items: &[PlanItem]) -> (u64, u64, usize) {
     (bytes, io_bytes, inline)
 }
 
-/// The read planner. Runtime's traversal must exactly equal the stored construction order;
-/// after that equality gate, planning walks the header once and never performs key lookup.
+/// The read planner. The destination follows the CALLER's traversal, which is the order it
+/// constructs in; the header's tensor order is a storage choice of whoever wrote it. The two
+/// are written by independently versioned peers, so a permutation is planned, never refused:
+/// only a traversal that names a different set of tensors is.
 ///
 /// COMPLETENESS SCOPES TO THE REQUESTED COMPONENT SET (#570b). `requested` is the caller's
 /// explicit declaration of which components this plan is for; the traversal must name every
@@ -716,34 +713,38 @@ pub fn plan_for_traversal(
         }
     }
 
-    let named: Vec<(&str, &str)> = traversal
-        .iter()
-        .map(|(c, k)| (c.as_str(), k.as_str()))
-        .collect();
-    let have: Vec<(&str, &str)> = header
+    let mut carried: HashMap<(&str, &str), &Tensor> = header
         .tensors()
         .filter(|(c, _, _)| scope.contains(&c.as_str()))
-        .map(|(c, k, _)| (c.as_str(), k.as_str()))
+        .map(|(c, k, t)| ((c.as_str(), k.as_str()), t))
         .collect();
-    if named != have {
-        let at = named
-            .iter()
-            .zip(have.iter())
-            .position(|(got, want)| got != want)
-            .unwrap_or(named.len().min(have.len()));
-        let got = named.get(at).map(|(c, k)| format!("{c}/{k}"));
-        let want = have.get(at).map(|(c, k)| format!("{c}/{k}"));
+    let mut ordered = Vec::with_capacity(traversal.len());
+    for (c, k) in traversal {
+        let Some(tensor) = carried.remove(&(c.as_str(), k.as_str())) else {
+            return refuse(
+                Code::TRAVERSAL_INCOMPLETE,
+                format!(
+                    "the traversal names {c}/{k}, which requested components {scope:?} do \
+                     not carry, or names it twice"
+                ),
+            );
+        };
+        ordered.push((c.as_str(), k.as_str(), tensor));
+    }
+    if !carried.is_empty() {
+        let mut missing: Vec<String> = carried.keys().map(|(c, k)| format!("{c}/{k}")).collect();
+        missing.sort();
+        missing.truncate(4);
         return refuse(
-            Code::TRAVERSAL_ORDER_MISMATCH,
+            Code::TRAVERSAL_INCOMPLETE,
             format!(
-                "construction order differs at row {at}: caller {got:?}, header {want:?}; \
-                 caller has {} rows and requested components {scope:?} carry {}",
-                named.len(),
-                have.len()
+                "the traversal omits {} tensor(s) requested components {scope:?} carry: \
+                 {missing:?}",
+                carried.len()
             ),
         );
     }
-    build(header, &scope, window)
+    build(&ordered, window)
 }
 
 // ---------------------------------------------------------------- the reader pool

{tensorfs-0.3.65 → tensorfs-0.3.67}/crates/tensorfs-core/src/store.rs

@@ -158,6 +158,14 @@ const EDQUOT: i32 = 69;
 /// the owner re-issued the fetch, which re-planned, re-asked, and failed at the same byte.
 /// On rented hardware, forever. Naming the condition is what lets a caller stop.
 pub(crate) fn classify_io(what: impl AsRef<str>, e: std::io::Error) -> Refusal {
+    // A full descriptor table is the process's shortage, not the Store's: every open in
+    // the Store says so the same way, and a transfer treats it as backpressure.
+    if crate::descriptors::exhausted(&e) {
+        return Refusal {
+            code: Code::FD_HEADROOM,
+            detail: format!("{}: no file descriptor ({e})", what.as_ref()),
+        };
+    }
     if matches!(e.raw_os_error(), Some(ENOSPC) | Some(EDQUOT)) {
         return Refusal {
             code: Code::CAPACITY_EXHAUSTED,
@@ -1362,16 +1370,7 @@ impl Store {
                 .create_new(true)
                 .mode(0o600)
                 .open(&tmp)
-                .map_err(|e| {
-                    if crate::descriptors::exhausted(&e) {
-                        Refusal {
-                            code: Code::FD_HEADROOM,
-                            detail: format!("create temp {}: {e}", tmp.display()),
-                        }
-                    } else {
-                        io(format!("create temp {}", tmp.display()), e)
-                    }
-                })?;
+                .map_err(|e| io(format!("create temp {}", tmp.display()), e))?;
             match f.try_lock_exclusive() {
                 Ok(()) => break (tmp, f),
                 // Only a reaper can hold the lock on a name this process just minted: it
@@ -1473,7 +1472,10 @@ impl Store {
         fault: &Fault,
         operation: Option<&str>,
     ) -> Result<Put> {
-        let catalog = crate::catalog::Catalog::open(&self.root)?;
+        let catalog = match crate::catalog::Catalog::open_admitting(&self.root) {
+            Ok(catalog) => catalog,
+            Err(refusal) => return cleanup(&verified.tmp, refusal),
+        };
         let _writer = crate::catalog::WriterGuard::acquire(&self.root)?;
         let object = ObjectRef {
             sha256: verified.sha256.clone(),