tensorfs 0.9.2__tar.gz → 0.9.3__tar.gz

{tensorfs-0.9.2 → tensorfs-0.9.3}/Cargo.lock

@@ -559,7 +559,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.9.2"
+version = "0.9.3"
 dependencies = [
  "base64",
  "fs2",
@@ -580,7 +580,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-plane"
-version = "0.9.2"
+version = "0.9.3"
 dependencies = [
  "libc",
  "libloading",
@@ -590,7 +590,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.9.2"
+version = "0.9.3"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.9.2 → tensorfs-0.9.3}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-plane", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.9.2"
+version = "0.9.3"
 edition = "2021"

{tensorfs-0.9.2 → tensorfs-0.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.9.2
+Version: 0.9.3
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.9.2 → tensorfs-0.9.3}/crates/tensorfs-core/src/gc.rs

@@ -107,7 +107,12 @@ pub fn session_holds(store: &Store) -> Result<(Vec<HeldKey>, Vec<String>)> {
     let mut lines: BTreeMap<String, Vec<u8>> = BTreeMap::new();
     let mut sessions = Vec::new();
     for session in tx::sessions(store) {
-        let root = tx::read_session(store.root(), &session)?;
+        // A root another writer removes (its operation released) or has not yet written
+        // holds nothing; it is not this walk's failure.
+        let root = match tx::read_session(store.root(), &session) {
+            Err(absent) if absent.code == Code::ROOT_ABSENT => continue,
+            root => root?,
+        };
         // An object the operation's publication custodian already holds is not in flight:
         // the journal stops naming it, so the local copy is ordinary unreferenced bytes.
         let custody = crate::ingest::custody::read_session(store.root(), &session)?;
@@ -540,4 +545,51 @@ mod tests {
         assert!(!present(&leased));
         fs::remove_dir_all(root).unwrap();
     }
+
+    /// A concurrent ingest that creates and releases its own session root (run 6801 failed
+    /// composing on ROOT_ABSENT naming its neighbour 6802's released conversion) never fails
+    /// a walk of the others, which keep their holds.
+    #[test]
+    fn a_session_released_during_the_walk_fails_no_one() {
+        let root = std::env::temp_dir().join(format!(
+            "tensorfs-gc-released-{}-{}",
+            std::process::id(),
+            meta::now_nanos_unique()
+        ));
+        let store = Store::init(&root).unwrap();
+        let held = store
+            .put_stream(
+                &mut &b"a live session's candidate"[..],
+                None,
+                &Fault::default(),
+            )
+            .unwrap()
+            .obj;
+        tx::ensure_session_root(&root, "live", "_tensorfs").unwrap();
+        tx::name_candidates(&root, "live", std::slice::from_ref(&held)).unwrap();
+        let holds = |store: &Store| {
+            let (holds, sessions) = session_holds(store).unwrap();
+            assert!(sessions.contains(&"live".to_string()), "{sessions:?}");
+            assert!(holds.iter().any(|hold| hold.sha256 == held.sha256));
+        };
+        // Its directory is still there, its root already gone.
+        fs::create_dir_all(tx::candidate_dir(&root, "leaving")).unwrap();
+        holds(&store);
+        let churn = std::thread::spawn({
+            let root = root.clone();
+            let held = held.clone();
+            move || {
+                for _ in 0..300 {
+                    tx::ensure_session_root(&root, "churn", "_tensorfs").unwrap();
+                    tx::name_candidates(&root, "churn", std::slice::from_ref(&held)).unwrap();
+                    fs::remove_dir_all(tx::candidate_dir(&root, "churn")).unwrap();
+                }
+            }
+        });
+        while !churn.is_finished() {
+            holds(&store);
+        }
+        churn.join().unwrap();
+        fs::remove_dir_all(root).unwrap();
+    }
 }

{tensorfs-0.9.2 → tensorfs-0.9.3}/crates/tensorfs-core/src/ingest/transaction.rs

@@ -219,18 +219,21 @@ pub fn write_session(store_root: &Path, root: &IngestSession) -> Result<()> {
     };
     let dir = candidate_dir(store_root, &root.session);
     fs::create_dir_all(&dir).map_err(|e| io("mkdir ingest candidate", e))?;
-    fs::write(dir.join("session.json"), root.canonical_bytes())
-        .map_err(|e| io("write ingest session", e))
+    // Renamed into place: a concurrent reader sees the whole root or none, never a torn one.
+    let staged = dir.join(format!("session.json.{}", crate::meta::now_nanos_unique()));
+    fs::write(&staged, root.canonical_bytes()).map_err(|e| io("write ingest session", e))?;
+    fs::rename(&staged, dir.join("session.json")).map_err(|e| io("install ingest session", e))
 }
 
 pub fn read_session(store_root: &Path, session: &str) -> Result<IngestSession> {
     let p = candidate_dir(store_root, session).join("session.json");
     match fs::read(&p) {
         Ok(b) => IngestSession::parse(&b),
-        Err(_) => refuse(
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => refuse(
             Code::ROOT_ABSENT,
             format!("no ingest session {session:?}: reaped, never created, or already installed"),
         ),
+        Err(e) => Err(io("read ingest session", e)),
     }
 }
 

{tensorfs-0.9.2 → tensorfs-0.9.3}/crates/tensorfs-core/src/providers.rs

@@ -247,7 +247,7 @@ fn list_selected_observed(
     credentials: &dyn CredentialProvider,
     deadline: Deadline,
     ledger: &Ledger,
-    files: Option<&[String]>,
+    wanted: Option<&dyn Fn(&str) -> bool>,
     mut unnameable: Option<&mut Vec<String>>,
 ) -> Result<Vec<SourceMember>> {
     let policy = endpoints.policy(uri);
@@ -286,7 +286,9 @@ fn list_selected_observed(
                     let Some(path) = json_str(row, "path") else {
                         return refuse(Code::MISSING_FIELD, "a tree row names no path");
                     };
-                    if files.is_some_and(|files| !files.iter().any(|file| file == path)) {
+                    // Before any check of its own: a gated repository masks the digests of
+                    // carriers a metadata-only selection never reads.
+                    if wanted.is_some_and(|wanted| !wanted(path)) {
                         continue;
                     }
                     if !member_path(path) {
@@ -442,7 +444,8 @@ fn list_selected_observed(
             members
         }
     };
-    if members.is_empty() {
+    // A selection that matches nothing is an answer; a source with no member is not.
+    if members.is_empty() && wanted.is_none() {
         return refuse(Code::MISSING_FIELD, "the source reaches no member");
     }
     if members.len() > MEMBER_MAX {
@@ -1283,20 +1286,17 @@ fn resolve_with(
     })
 }
 
-/// Resolve only explicitly named ordinary files at one immutable HF revision.
-/// No suffix has special meaning here: selecting Python bytes never executes them,
-/// and selecting an index does not expand it into tensor shards. Tensor carriers
-/// continue to use `resolve` and reviewed profile/carrier selection.
-///
-/// A name ending in `/` selects every ordinary file under that folder (a tokenizer or
-/// processor, whose file set varies by class); tensor carriers there are never selected.
+/// Resolve the ordinary files `wanted` selects at one immutable HF revision, from one
+/// listing; `None` when it selects none. Selecting Python bytes never executes them, and
+/// selecting an index does not expand it into tensor shards: tensor carriers are never
+/// selected here, they continue to use `resolve` and reviewed profile/carrier selection.
 pub fn resolve_files(
     uri: &SourceUri,
-    files: &[String],
+    wanted: &dyn Fn(&str) -> bool,
     endpoints: &Endpoints,
     credentials: &dyn CredentialProvider,
     deadline: Deadline,
-) -> Result<Resolution> {
+) -> Result<Option<Resolution>> {
     let SourceUri::HuggingFace {
         org,
         repo,
@@ -1308,70 +1308,32 @@ pub fn resolve_files(
             "ordinary file selection requires HuggingFace",
         );
     };
-    if files.is_empty() || files.len() > MEMBER_MAX {
-        return refuse(
-            Code::COUNT_CAP,
-            "ordinary file selection count is outside the bound",
-        );
-    }
-    let (folders, mut names): (Vec<String>, Vec<String>) =
-        files.iter().cloned().partition(|name| name.ends_with('/'));
-    names.sort();
-    names.dedup();
-    if names
-        .iter()
-        .chain(&folders)
-        .map(|name| name.trim_end_matches('/'))
-        .any(|name| !member_path(name) || crate::manifest::check_path(name).is_err())
-    {
-        return refuse(
-            Code::PATH_ILLEGAL,
-            "ordinary file selection requires exact member or folder paths",
-        );
-    }
     let ledger = Ledger::new();
     let policy = endpoints.policy(uri);
-    let filter = folders.is_empty().then_some(names.as_slice());
-    let listed =
-        list_selected_observed(uri, endpoints, credentials, deadline, &ledger, filter, None)?;
-    for folder in &folders {
-        let before = names.len();
-        names.extend(
-            listed
-                .iter()
-                .filter(|m| m.member.starts_with(folder.as_str()))
-                .filter(|m| !is_safetensors(&m.member) && !is_index(&m.member))
-                .map(|m| m.member.clone()),
-        );
-        if names.len() == before {
-            return refuse(
-                Code::MISSING_FIELD,
-                format!("the source has no ordinary file under {folder:?}"),
-            );
-        }
+    let ordinary = |m: &str| {
+        !is_safetensors(m) && !is_index(m) && crate::manifest::check_path(m).is_ok() && wanted(m)
+    };
+    let listed = list_selected_observed(
+        uri,
+        endpoints,
+        credentials,
+        deadline,
+        &ledger,
+        Some(&ordinary),
+        None,
+    )?;
+    let mut selected: Vec<&SourceMember> = listed.iter().collect();
+    selected.sort_by(|a, b| a.member.cmp(&b.member));
+    selected.dedup_by(|a, b| a.member == b.member);
+    if selected.is_empty() {
+        return Ok(None);
     }
-    names.sort();
-    names.dedup();
-    if names.len() > MEMBER_MAX {
+    if selected.len() > MEMBER_MAX {
         return refuse(
             Code::COUNT_CAP,
             "ordinary file selection count is outside the bound",
         );
     }
-    // Establish the whole roster before fetching any body. A missing later member
-    // must not turn a partial requested tree into a successful smaller selection.
-    let selected = names
-        .iter()
-        .map(|name| {
-            listed
-                .iter()
-                .find(|member| &member.member == name)
-                .ok_or_else(|| Refusal {
-                    code: Code::MISSING_FIELD,
-                    detail: format!("the source does not contain requested file {name:?}"),
-                })
-        })
-        .collect::<Result<Vec<_>>>()?;
     let listed_bytes = selected.iter().try_fold(0u64, |total, member| {
         total.checked_add(member.length.unwrap_or(0))
     });
@@ -1442,11 +1404,11 @@ pub fn resolve_files(
         });
     }
     let canonical = format!("hf://{org}/{repo}@{revision}");
-    Ok(Resolution {
+    Ok(Some(Resolution {
         selection_sha256: selection_digest(&canonical, &resolved),
         canonical,
         members: resolved,
-    })
+    }))
 }
 
 #[cfg(test)]
@@ -2278,17 +2240,14 @@ mod tests {
             allow_local: true,
             ..Endpoints::default()
         };
-        let mut names = vec![
-            "config.json".into(),
-            "tokenizer files/tokenizer.json".into(),
-        ];
-        let selected =
-            resolve_files(&uri, &names, &endpoints, &Anonymous, Deadline::none()).unwrap();
-        names.reverse();
-        assert_eq!(
-            selected,
-            resolve_files(&uri, &names, &endpoints, &Anonymous, Deadline::none()).unwrap()
-        );
+        let wanted = |m: &str| m == "config.json" || m == "tokenizer files/tokenizer.json";
+        let resolved = || {
+            resolve_files(&uri, &wanted, &endpoints, &Anonymous, Deadline::none())
+                .unwrap()
+                .unwrap()
+        };
+        let selected = resolved();
+        assert_eq!(selected, resolved());
         assert_eq!(
             selected
                 .members
@@ -2359,21 +2318,18 @@ mod tests {
     }
 
     #[test]
-    fn explicit_files_refuse_missing_and_over_budget_before_fetching_bodies() {
-        for (names, listing, code) in [
+    fn files_select_nothing_absent_and_refuse_over_budget_before_fetching_bodies() {
+        let oversized = format!(
+            r#"[{{"type":"file","path":"config.json","size":{}}}]"#,
+            FILE_SELECTION_MAX_BYTES + 1
+        );
+        for (listing, name, outcome) in [
             (
-                vec!["absent.json".into(), "config.json".into()],
                 r#"[{"type":"file","path":"config.json","size":2}]"#.to_string(),
-                Code::MISSING_FIELD,
-            ),
-            (
-                vec!["config.json".into()],
-                format!(
-                    r#"[{{"type":"file","path":"config.json","size":{}}}]"#,
-                    FILE_SELECTION_MAX_BYTES + 1
-                ),
-                Code::SIZE_CAP,
+                "absent.json",
+                None,
             ),
+            (oversized, "config.json", Some(Code::SIZE_CAP)),
         ] {
             let (base, _, handle) =
                 origin(1, move |_| ok_body(listing.as_bytes(), "application/json"));
@@ -2383,33 +2339,14 @@ mod tests {
                 allow_local: true,
                 ..Endpoints::default()
             };
-            assert_eq!(
-                resolve_files(&uri, &names, &endpoints, &Anonymous, Deadline::none())
-                    .unwrap_err()
-                    .code,
-                code
-            );
+            let wanted = |m: &str| m == name;
+            let resolved = resolve_files(&uri, &wanted, &endpoints, &Anonymous, Deadline::none());
+            match outcome {
+                None => assert_eq!(resolved.unwrap(), None),
+                Some(code) => assert_eq!(resolved.unwrap_err().code, code),
+            }
             handle.join().unwrap();
         }
-        let uri = SourceUri::parse(&format!("hf://org/repo@{}", "ab".repeat(20))).unwrap();
-        for names in [
-            vec!["../config.json".into()],
-            vec!["*.json".into()],
-            vec!["/config.json".into()],
-        ] {
-            assert_eq!(
-                resolve_files(
-                    &uri,
-                    &names,
-                    &Endpoints::default(),
-                    &Anonymous,
-                    Deadline::none()
-                )
-                .unwrap_err()
-                .code,
-                Code::PATH_ILLEGAL
-            );
-        }
     }
 
     #[test]
@@ -2434,14 +2371,10 @@ mod tests {
             allow_local: true,
             ..Endpoints::default()
         };
-        let selected = resolve_files(
-            &uri,
-            &["tokenizer/".into(), "unet/config.json".into()],
-            &endpoints,
-            &Anonymous,
-            Deadline::none(),
-        )
-        .unwrap();
+        let wanted = |m: &str| m.starts_with("tokenizer/") || m == "unet/config.json";
+        let selected = resolve_files(&uri, &wanted, &endpoints, &Anonymous, Deadline::none())
+            .unwrap()
+            .unwrap();
         let members: Vec<&str> = selected.members.iter().map(|m| m.member.as_str()).collect();
         assert_eq!(
             members,
@@ -2451,17 +2384,10 @@ mod tests {
                 "unet/config.json"
             ]
         );
+        let vae = |m: &str| m.starts_with("vae/");
         assert_eq!(
-            resolve_files(
-                &uri,
-                &["vae/".into()],
-                &endpoints,
-                &Anonymous,
-                Deadline::none()
-            )
-            .unwrap_err()
-            .code,
-            Code::MISSING_FIELD
+            resolve_files(&uri, &vae, &endpoints, &Anonymous, Deadline::none()).unwrap(),
+            None
         );
         handle.join().unwrap();
         assert!(!seen
@@ -2489,16 +2415,11 @@ mod tests {
             allow_local: true,
             ..Endpoints::default()
         };
+        let config = |m: &str| m == "config.json";
         assert_eq!(
-            resolve_files(
-                &uri,
-                &["config.json".into()],
-                &endpoints,
-                &Anonymous,
-                Deadline::none()
-            )
-            .unwrap_err()
-            .code,
+            resolve_files(&uri, &config, &endpoints, &Anonymous, Deadline::none())
+                .unwrap_err()
+                .code,
             Code::LENGTH_MISMATCH
         );
         handle.join().unwrap();
@@ -2524,14 +2445,10 @@ mod tests {
             allow_local: true,
             ..Endpoints::default()
         };
-        let selected = resolve_files(
-            &uri,
-            &["config.json".into()],
-            &endpoints,
-            &Anonymous,
-            Deadline::none(),
-        )
-        .unwrap();
+        let config = |m: &str| m == "config.json";
+        let selected = resolve_files(&uri, &config, &endpoints, &Anonymous, Deadline::none())
+            .unwrap()
+            .unwrap();
         let root = temporary("changed-files");
         let store = Store::init(&root).unwrap();
         let producer = ObjectRef::of(b"changed-files-producer").id();
@@ -2575,14 +2492,10 @@ mod tests {
             allow_local: true,
             ..Endpoints::default()
         };
-        let selected = resolve_files(
-            &uri,
-            &["empty.py".into()],
-            &endpoints,
-            &Anonymous,
-            Deadline::none(),
-        )
-        .unwrap();
+        let empty = |m: &str| m == "empty.py";
+        let selected = resolve_files(&uri, &empty, &endpoints, &Anonymous, Deadline::none())
+            .unwrap()
+            .unwrap();
         assert_eq!(selected.members[0].object, ObjectRef::of(b""));
         let root = temporary("empty-files");
         let store = Store::init(&root).unwrap();