tensorfs 0.3.60__tar.gz → 0.3.62__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tensorfs-0.3.60 → tensorfs-0.3.62}/Cargo.lock +2 -2
- {tensorfs-0.3.60 → tensorfs-0.3.62}/Cargo.toml +1 -1
- {tensorfs-0.3.60 → tensorfs-0.3.62}/PKG-INFO +1 -1
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/asset_update.rs +3 -8
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/bin/tfs/fetch.rs +32 -1
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/bin/tfs/main.rs +11 -7
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/bin/tfs/repo.rs +10 -19
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/bin/tfs/snap.rs +4 -1
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/canon.rs +25 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/catalog.rs +183 -4
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/checkpoint.rs +7 -10
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/checkpoint_root.rs +21 -3
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/derived/companion_tests.rs +1 -1
- tensorfs-0.3.62/crates/tensorfs-core/src/derived/part_rows_tests.rs +151 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/derived/resume_tests.rs +2 -1
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/derived/retention.rs +18 -1
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/derived/roots.rs +128 -1
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/derived.rs +180 -88
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ingest/fingerprint.rs +24 -1
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ingest/source.rs +96 -4
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ingest/transaction.rs +27 -5
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/lib.rs +3 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/manifest.rs +50 -13
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/meta.rs +197 -7
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/project.rs +2 -2
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/source_artifact.rs +20 -3
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/stats.rs +21 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/storage.rs +3 -3
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/store.rs +72 -41
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/http.rs +26 -19
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/http_reuse_tests.rs +84 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/mod.rs +4 -1
- tensorfs-0.3.62/crates/tensorfs-core/src/transport/pace.rs +236 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/pull.rs +177 -100
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/push.rs +5 -1
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/speculate.rs +11 -3
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/tests.rs +254 -1
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/additive_fields.rs +43 -13
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/checkpoint_roots.rs +26 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-py/src/lib.rs +3 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-py/src/store.rs +27 -13
- {tensorfs-0.3.60 → tensorfs-0.3.62}/pyproject.toml +1 -1
- {tensorfs-0.3.60 → tensorfs-0.3.62}/python/tensorfs/_ext.pyi +5 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/Cargo.toml +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/build.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/examples/h3-swiglu-layout.md +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/examples/optional-cache-backfill.md +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/examples/resident-conversion.md +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/examples/resident_conversion_profile.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/b64.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/bin/tfs/ckpt.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/bin/tfs/conform.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/bin/tfs/ingest.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/bin/tfs/rd.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/bin/tfs/watch.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/cache_roots.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/capability.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/cbor.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/corpus.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/derived/adoption.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/derived/estimate.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/derived/graft_tests.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/derived/progress.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/dtype.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/durability.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/err.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/fetch.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/fill.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/fit.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/gc.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/header.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/home.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ids.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ingest/carrier.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ingest/convert.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ingest/custody.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ingest/gguf.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ingest/h3_lora.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ingest/journal.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ingest/mod.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ingest/preflight.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/ingest/stamp.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/jcs.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/limits.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/machine.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/providers.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/read.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/receipt.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/reclaim.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/registry.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/relation.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/repo_cache/identity.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/repo_cache.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/repository.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/sha256.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/spec.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/staging.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/api.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/hub.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/ledger.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/memory.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/policy.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/prefix.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/transport/ranged.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/src/vectors.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/fd_headroom.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/fetch_url_progress.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/fit.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/fixtures/repo_apply/blobs.jsonl +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_list.jsonl +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_repository.json +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_rows.jsonl +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/fixtures/repo_apply/header.cbor +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/fixtures/repo_apply/manifest.json +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/fixtures/repo_apply/mutation.json +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/foreign_json.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/h3_lora.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/h3_swiglu.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/home.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/import_tree.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/instrument_profiles.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/liveness.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/qwen_image21_profile.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/read_plan.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/reclaim.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/reclaim_run.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/repo_cache.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/repo_store.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/resident_admission.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/sha256_backends.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/sharded_cas.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/source_artifacts.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/source_download.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/source_plan.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/source_prefixes.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/source_preflight.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/staging.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/store_boundaries.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/tensor_schema.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/transfer_memory.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/trust_index.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/version.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-core/tests/write_through.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-py/Cargo.toml +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-py/src/docs.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-py/src/errors.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/crates/tensorfs-py/src/ingest.rs +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/python/tensorfs/__init__.py +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/python/tensorfs/_derived_channel.py +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/python/tensorfs/_source_channel.py +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/python/tensorfs/_tensors.py +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/python/tensorfs/derived.py +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/python/tensorfs/errors.py +0 -0
- {tensorfs-0.3.60 → tensorfs-0.3.62}/python/tensorfs/py.typed +0 -0
|
@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
|
439
439
|
|
|
440
440
|
[[package]]
|
|
441
441
|
name = "tensorfs-core"
|
|
442
|
-
version = "0.3.
|
|
442
|
+
version = "0.3.62"
|
|
443
443
|
dependencies = [
|
|
444
444
|
"fs2",
|
|
445
445
|
"libc",
|
|
@@ -454,7 +454,7 @@ dependencies = [
|
|
|
454
454
|
|
|
455
455
|
[[package]]
|
|
456
456
|
name = "tensorfs-py"
|
|
457
|
-
version = "0.3.
|
|
457
|
+
version = "0.3.62"
|
|
458
458
|
dependencies = [
|
|
459
459
|
"pyo3",
|
|
460
460
|
"tensorfs-core",
|
|
@@ -11,7 +11,7 @@ use crate::checkpoint;
|
|
|
11
11
|
use crate::err::{refuse, Code, Refusal, Result};
|
|
12
12
|
use crate::header::Asset;
|
|
13
13
|
use crate::ids::ObjectRef;
|
|
14
|
-
use crate::manifest::
|
|
14
|
+
use crate::manifest::Manifest;
|
|
15
15
|
use crate::store::Store;
|
|
16
16
|
|
|
17
17
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
@@ -42,13 +42,7 @@ fn source_object(manifest: &Manifest, path: &str) -> Result<ObjectRef> {
|
|
|
42
42
|
.entries()
|
|
43
43
|
.iter()
|
|
44
44
|
.find_map(|(candidate, entry)| {
|
|
45
|
-
(candidate == path).then(||
|
|
46
|
-
Entry::File(object) => Ok(object.clone()),
|
|
47
|
-
Entry::CozyTensors(_) => Err(Refusal {
|
|
48
|
-
code: Code::WRONG_TYPE,
|
|
49
|
-
detail: format!("asset source {path:?} is a CozyTensors header, not a file"),
|
|
50
|
-
}),
|
|
51
|
-
})
|
|
45
|
+
(candidate == path).then(|| entry.materializable(path).cloned())
|
|
52
46
|
})
|
|
53
47
|
.unwrap_or_else(|| {
|
|
54
48
|
Err(Refusal {
|
|
@@ -187,6 +181,7 @@ mod tests {
|
|
|
187
181
|
use crate::checkpoint;
|
|
188
182
|
use crate::header::{Body, Header, Part, Tensor};
|
|
189
183
|
use crate::manifest::Draft;
|
|
184
|
+
use crate::manifest::Entry;
|
|
190
185
|
use crate::store::Fault;
|
|
191
186
|
use crate::{dtype::Dtype, registry};
|
|
192
187
|
|
|
@@ -273,7 +273,8 @@ fn pull_with_budget(
|
|
|
273
273
|
let mut request = PullRequest::new(&store, base, refspec, &credential, &policy);
|
|
274
274
|
request.lane = flag(flags, "lane").unwrap_or("");
|
|
275
275
|
request.session = flag(flags, "session").unwrap_or("");
|
|
276
|
-
request.streams = crate::flag_num(flags, "streams", transport::
|
|
276
|
+
request.streams = crate::flag_num(flags, "streams", transport::PULL_STREAMS);
|
|
277
|
+
request.streams_start = crate::flag_num(flags, "streams-start", transport::STREAMS);
|
|
277
278
|
request.deadline = deadline_of(flags);
|
|
278
279
|
request.download_budget = budget;
|
|
279
280
|
// THE WALK REPORTS WHILE IT WALKS.
|
|
@@ -300,7 +301,11 @@ fn pull_with_budget(
|
|
|
300
301
|
let observe = |_object: &ObjectRef, moved: u64, door: ObjectSource| progress.saw(moved, door);
|
|
301
302
|
request.on_plan = Some(&planned);
|
|
302
303
|
request.on_object = Some(&observe);
|
|
304
|
+
let before = tensorfs_core::stats::snapshot();
|
|
305
|
+
let started = std::time::Instant::now();
|
|
303
306
|
let report = ok!(transport::pull(&request));
|
|
307
|
+
let seconds = started.elapsed().as_secs_f64();
|
|
308
|
+
let after = tensorfs_core::stats::snapshot();
|
|
304
309
|
progress.settle();
|
|
305
310
|
if !tensorfs_core::repo_cache::finish_optional_backfills() {
|
|
306
311
|
eprintln!("repo cache replication incomplete; local fetch remains complete");
|
|
@@ -313,6 +318,32 @@ fn pull_with_budget(
|
|
|
313
318
|
quoted(&report.model), quoted(&report.manifest), quoted(&report.scope),
|
|
314
319
|
report.cache_writes.bytes(), report.bytes_cached, report.bytes_total,
|
|
315
320
|
);
|
|
321
|
+
// What each stream level bought, so a slow pull says whether more streams helped
|
|
322
|
+
// (one GET was the limit) or not (the link, disk or origin was).
|
|
323
|
+
let levels: Vec<String> = report
|
|
324
|
+
.levels
|
|
325
|
+
.iter()
|
|
326
|
+
.map(|level| {
|
|
327
|
+
format!(
|
|
328
|
+
"[{},{}]",
|
|
329
|
+
level.streams,
|
|
330
|
+
level.bytes_per_second.round() as u64
|
|
331
|
+
)
|
|
332
|
+
})
|
|
333
|
+
.collect();
|
|
334
|
+
// And what admitting them cost: a pull whose streams stop growing while its batches
|
|
335
|
+
// take most of the wall time is held by the disk, not the link.
|
|
336
|
+
eprintln!(
|
|
337
|
+
"{{\"event\":\"pull.streams\",\"model\":{},\"seconds\":{:.3},\"bytes_moved\":{},\"streams\":{},\"levels\":[{}],\"admission\":{{\"objects\":{},\"batches\":{},\"seconds\":{:.3}}}}}",
|
|
338
|
+
quoted(&report.model),
|
|
339
|
+
seconds,
|
|
340
|
+
report.bytes_moved,
|
|
341
|
+
report.streams,
|
|
342
|
+
levels.join(","),
|
|
343
|
+
after.admitted_objects - before.admitted_objects,
|
|
344
|
+
after.admission_batches - before.admission_batches,
|
|
345
|
+
(after.admission_nanos - before.admission_nanos) as f64 / 1e9,
|
|
346
|
+
);
|
|
316
347
|
println!(
|
|
317
348
|
"model {}@{} lane {}",
|
|
318
349
|
report.model, report.release, report.lane
|
|
@@ -42,7 +42,9 @@
|
|
|
42
42
|
//! The ONE transport (tfs-048/tfs-049/tfs-050). The credential is `--credential-file` or
|
|
43
43
|
//! TFS_CREDENTIAL (`bearer <token>`, or empty for anonymous), never bare argv; a
|
|
44
44
|
//! URL of `-` arrives on stdin. Flags: --hub <base> --lane --session --streams <n>
|
|
45
|
-
//! --timeout <s> --allow-hosts a,b,.c --allow-local --max-redirects <n>.
|
|
45
|
+
//! --streams-start <n> --timeout <s> --allow-hosts a,b,.c --allow-local --max-redirects <n>.
|
|
46
|
+
//! `--streams` is the most objects in flight (default 128, memory-bounded); the walk starts
|
|
47
|
+
//! at `--streams-start` (default 16) and doubles while measured goodput follows.
|
|
46
48
|
//! `tfs fetch` reads and backfills the Store's bound repo cache between the local Store
|
|
47
49
|
//! and the origin. It is bound by `tfs store ensure --repo-cache` and never by a flag here.
|
|
48
50
|
//!
|
|
@@ -2205,16 +2207,17 @@ fn build_corpus() -> TfsResult<(Vec<Case>, Vec<EncodingSpec>)> {
|
|
|
2205
2207
|
"carrier no longer equals the logical dtype",
|
|
2206
2208
|
));
|
|
2207
2209
|
|
|
2208
|
-
//
|
|
2210
|
+
// A kind a newer TensorFS wrote is opaque here: it parses, and only materializing refuses.
|
|
2209
2211
|
let manifest_bytes = snap2.canonical_bytes();
|
|
2210
|
-
cases.push(
|
|
2211
|
-
"manifest
|
|
2212
|
-
"entry-kind-
|
|
2212
|
+
cases.push(ok_case(
|
|
2213
|
+
"manifest",
|
|
2214
|
+
"entry-kind-future",
|
|
2213
2215
|
"manifest",
|
|
2214
2216
|
sub(&manifest_bytes, "\"kind\":\"file\"", "\"kind\":\"symlink\""),
|
|
2215
|
-
"
|
|
2216
|
-
"only file and cozytensors entries exist",
|
|
2217
|
+
"an entry kind from a newer TensorFS is carried opaquely",
|
|
2217
2218
|
));
|
|
2219
|
+
|
|
2220
|
+
// --- location-typed Manifest reds
|
|
2218
2221
|
let mut draft = Draft::of(&snap);
|
|
2219
2222
|
draft.entries.clear();
|
|
2220
2223
|
cases.push(red_case(
|
|
@@ -2257,6 +2260,7 @@ fn build_corpus() -> TfsResult<(Vec<Case>, Vec<EncodingSpec>)> {
|
|
|
2257
2260
|
// --- the border's own documents (tfs-003)
|
|
2258
2261
|
let g = "ingest";
|
|
2259
2262
|
let session = tensorfs_core::ingest::transaction::IngestSession {
|
|
2263
|
+
tensorfs: None,
|
|
2260
2264
|
session: "e3b0c44298fc1c14".to_string(),
|
|
2261
2265
|
tenant: "acme".to_string(),
|
|
2262
2266
|
candidates: vec![ObjectRef::of(b"a candidate header")],
|
|
@@ -9,7 +9,7 @@ use tensorfs_core::err::{Code, Refusal};
|
|
|
9
9
|
use tensorfs_core::gc;
|
|
10
10
|
use tensorfs_core::header::{Body, Header};
|
|
11
11
|
use tensorfs_core::ids::{hex64, prefixed, Doc, ObjectRef};
|
|
12
|
-
use tensorfs_core::manifest::
|
|
12
|
+
use tensorfs_core::manifest::Manifest;
|
|
13
13
|
use tensorfs_core::reclaim;
|
|
14
14
|
use tensorfs_core::repository::{self, Mutation, Repository};
|
|
15
15
|
use tensorfs_core::storage::{
|
|
@@ -738,13 +738,7 @@ pub fn cmd_manifest_inspect(path: &Path, flags: &Flags) -> ExitCode {
|
|
|
738
738
|
.iter()
|
|
739
739
|
.map(|(path, entry)| {
|
|
740
740
|
tensorfs_core::canon::write(&Value::obj(vec![
|
|
741
|
-
(
|
|
742
|
-
"kind",
|
|
743
|
-
Value::str(match entry {
|
|
744
|
-
Entry::File(_) => "file",
|
|
745
|
-
Entry::CozyTensors(_) => "cozytensors",
|
|
746
|
-
}),
|
|
747
|
-
),
|
|
741
|
+
("kind", Value::str(entry.kind())),
|
|
748
742
|
("length", Value::uint(entry.blob().length)),
|
|
749
743
|
("path", Value::str(path.clone())),
|
|
750
744
|
("sha256", Value::str(entry.blob().sha256.clone())),
|
|
@@ -855,7 +849,7 @@ fn closure_refs(
|
|
|
855
849
|
// Runtime roles precede ordinary snapshot paths so an aliased README/sample path cannot
|
|
856
850
|
// relabel a required config, model asset, or tensor part as an ignorable file.
|
|
857
851
|
for (_, entry) in manifest.entries() {
|
|
858
|
-
if let
|
|
852
|
+
if let Some(object) = entry.content() {
|
|
859
853
|
insert(&mut refs, "file", object)?;
|
|
860
854
|
}
|
|
861
855
|
}
|
|
@@ -1206,7 +1200,7 @@ pub fn cmd_reclaim_run(root: &Path, flags: &Flags) -> ExitCode {
|
|
|
1206
1200
|
mod tests {
|
|
1207
1201
|
use super::*;
|
|
1208
1202
|
use std::collections::HashSet;
|
|
1209
|
-
use tensorfs_core::manifest::Draft;
|
|
1203
|
+
use tensorfs_core::manifest::{Draft, Entry};
|
|
1210
1204
|
|
|
1211
1205
|
#[test]
|
|
1212
1206
|
fn inspect_closure_preserves_runtime_roles_over_aliased_files() {
|
|
@@ -1246,10 +1240,7 @@ mod tests {
|
|
|
1246
1240
|
let file_ids: HashSet<&str> = manifest
|
|
1247
1241
|
.entries()
|
|
1248
1242
|
.iter()
|
|
1249
|
-
.filter_map(|(_, entry)|
|
|
1250
|
-
Entry::File(object) => Some(object.sha256.as_str()),
|
|
1251
|
-
Entry::CozyTensors(_) => None,
|
|
1252
|
-
})
|
|
1243
|
+
.filter_map(|(_, entry)| entry.content().map(|object| object.sha256.as_str()))
|
|
1253
1244
|
.collect();
|
|
1254
1245
|
|
|
1255
1246
|
assert!(header.assets.iter().any(|(_, asset)| asset
|
|
@@ -1268,11 +1259,11 @@ mod tests {
|
|
|
1268
1259
|
.filter(|(kind, _)| *kind != "file")
|
|
1269
1260
|
.map(|(_, object)| object.sha256.as_str())
|
|
1270
1261
|
.collect();
|
|
1271
|
-
assert!(manifest
|
|
1272
|
-
|
|
1262
|
+
assert!(manifest
|
|
1263
|
+
.entries()
|
|
1264
|
+
.iter()
|
|
1265
|
+
.any(|(_, entry)| entry.content().is_some_and(|object| {
|
|
1273
1266
|
!runtime_ids.contains(object.sha256.as_str()) && refs[&object.sha256].0 == "file"
|
|
1274
|
-
}
|
|
1275
|
-
Entry::CozyTensors(_) => false,
|
|
1276
|
-
}));
|
|
1267
|
+
})));
|
|
1277
1268
|
}
|
|
1278
1269
|
}
|
|
@@ -104,7 +104,7 @@ fn referenced_blobs(header: &Header, draft: &Draft) -> Result<BTreeMap<String, u
|
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
for (_, entry) in &draft.entries {
|
|
107
|
-
if let
|
|
107
|
+
if let Some(object) = entry.content() {
|
|
108
108
|
add_ref(&mut refs, object, "manifest file")?;
|
|
109
109
|
}
|
|
110
110
|
}
|
|
@@ -257,6 +257,9 @@ pub fn cmd_show(root: &Path, hex: &str) -> ExitCode {
|
|
|
257
257
|
blob.id(),
|
|
258
258
|
blob.length
|
|
259
259
|
),
|
|
260
|
+
Entry::Other { kind, blob } => {
|
|
261
|
+
println!(" {kind:<12} {path} {} ({} B)", blob.id(), blob.length)
|
|
262
|
+
}
|
|
260
263
|
}
|
|
261
264
|
}
|
|
262
265
|
ExitCode::SUCCESS
|
|
@@ -456,6 +456,31 @@ impl<'a> Fields<'a> {
|
|
|
456
456
|
}
|
|
457
457
|
}
|
|
458
458
|
|
|
459
|
+
/// End a record whose unread field could keep bytes alive (a GC root): refuse, naming the
|
|
460
|
+
/// newer TensorFS that wrote it, rather than drop what that version may rely on.
|
|
461
|
+
pub fn done_written_by(self, writer: Option<&str>) -> Result<()> {
|
|
462
|
+
let Some((k, _)) = self.left.first() else {
|
|
463
|
+
return Ok(());
|
|
464
|
+
};
|
|
465
|
+
let by = writer.map_or_else(
|
|
466
|
+
|| "a newer TensorFS".to_string(),
|
|
467
|
+
|writer| format!("TensorFS {writer}"),
|
|
468
|
+
);
|
|
469
|
+
let upgrade = writer.map_or_else(
|
|
470
|
+
|| "upgrade TensorFS".to_string(),
|
|
471
|
+
|writer| format!("upgrade to tensorfs>={writer}"),
|
|
472
|
+
);
|
|
473
|
+
refuse(
|
|
474
|
+
Code::UNKNOWN_FIELD,
|
|
475
|
+
format!(
|
|
476
|
+
"{}: field {k:?} was written by {by}; this is TensorFS {} — {upgrade}. It is \
|
|
477
|
+
refused rather than dropped because it may keep bytes alive",
|
|
478
|
+
self.what,
|
|
479
|
+
crate::VERSION
|
|
480
|
+
),
|
|
481
|
+
)
|
|
482
|
+
}
|
|
483
|
+
|
|
459
484
|
/// End a document that tolerates additive fields, keeping them for re-emission.
|
|
460
485
|
pub fn rest(self) -> Extra {
|
|
461
486
|
Extra(
|
|
@@ -903,8 +903,9 @@ impl Catalog {
|
|
|
903
903
|
hold_on(&self.connection()?, operation, key, length)
|
|
904
904
|
}
|
|
905
905
|
|
|
906
|
-
///
|
|
907
|
-
///
|
|
906
|
+
/// A HELD admission: the hold is written before the object is published, so an
|
|
907
|
+
/// operation that cannot hold it publishes nothing. Unheld admissions, which is every
|
|
908
|
+
/// pull, take `record_grouped` instead.
|
|
908
909
|
pub(crate) fn admit_blob(
|
|
909
910
|
&self,
|
|
910
911
|
operation: Option<&str>,
|
|
@@ -914,7 +915,7 @@ impl Catalog {
|
|
|
914
915
|
// One admission at a time per catalog in this process, holding the write lock from
|
|
915
916
|
// before publication: streams landing objects together queue here instead of in
|
|
916
917
|
// SQLite's busy timeout, where a lost race refused bytes already verified.
|
|
917
|
-
let turn =
|
|
918
|
+
let turn = held_admissions(&self.path);
|
|
918
919
|
let _turn = turn.lock().unwrap_or_else(|error| error.into_inner());
|
|
919
920
|
let mut connection = self.connection()?;
|
|
920
921
|
let transaction = connection
|
|
@@ -941,6 +942,117 @@ impl Catalog {
|
|
|
941
942
|
Ok(verified.is_some())
|
|
942
943
|
}
|
|
943
944
|
|
|
945
|
+
/// The object's existing hold and verification become durable together. Filesystem
|
|
946
|
+
/// publication has already happened, on the caller's thread, and precedes this record,
|
|
947
|
+
/// which is written with FULL synchronization.
|
|
948
|
+
///
|
|
949
|
+
/// Records queue per catalog in this process, and whichever arrives while no batch is
|
|
950
|
+
/// committing leads the next one: one immediate transaction and one commit make every
|
|
951
|
+
/// waiting record durable. One object alone costs what it always did. Before this,
|
|
952
|
+
/// directory creation, link, directory fsync and commit ran for one object at a time
|
|
953
|
+
/// under one lock, several fsyncs each, and on a disk busy flushing those same objects'
|
|
954
|
+
/// bodies that queue, not the network, set a pull's rate.
|
|
955
|
+
pub(crate) fn record_grouped(&self, recording: Recording) -> Result<()> {
|
|
956
|
+
let group = admission_group(&self.path);
|
|
957
|
+
let mut state = group
|
|
958
|
+
.state
|
|
959
|
+
.lock()
|
|
960
|
+
.unwrap_or_else(|error| error.into_inner());
|
|
961
|
+
let ticket = state.next;
|
|
962
|
+
state.next += 1;
|
|
963
|
+
state.queue.push((ticket, recording));
|
|
964
|
+
loop {
|
|
965
|
+
if let Some(result) = state.done.remove(&ticket) {
|
|
966
|
+
return result;
|
|
967
|
+
}
|
|
968
|
+
if state.leading {
|
|
969
|
+
state = group
|
|
970
|
+
.turned
|
|
971
|
+
.wait(state)
|
|
972
|
+
.unwrap_or_else(|error| error.into_inner());
|
|
973
|
+
continue;
|
|
974
|
+
}
|
|
975
|
+
state.leading = true;
|
|
976
|
+
let batch = std::mem::take(&mut state.queue);
|
|
977
|
+
let mut lead = Lead {
|
|
978
|
+
group: &group,
|
|
979
|
+
tickets: batch.iter().map(|(ticket, _)| *ticket).collect(),
|
|
980
|
+
results: Vec::new(),
|
|
981
|
+
};
|
|
982
|
+
drop(state);
|
|
983
|
+
let started = std::time::Instant::now();
|
|
984
|
+
let objects = batch.len();
|
|
985
|
+
lead.results = self.record_batch(batch);
|
|
986
|
+
crate::stats::admission_batch(objects, started.elapsed());
|
|
987
|
+
drop(lead);
|
|
988
|
+
state = group
|
|
989
|
+
.state
|
|
990
|
+
.lock()
|
|
991
|
+
.unwrap_or_else(|error| error.into_inner());
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
fn record_batch(&self, batch: Vec<(u64, Recording)>) -> Vec<(u64, Result<()>)> {
|
|
996
|
+
let fail_all = |batch: Vec<(u64, Recording)>, error: Refusal| {
|
|
997
|
+
batch
|
|
998
|
+
.into_iter()
|
|
999
|
+
.map(|(ticket, _)| (ticket, Err(error.clone())))
|
|
1000
|
+
.collect::<Vec<_>>()
|
|
1001
|
+
};
|
|
1002
|
+
let mut connection = match self.connection() {
|
|
1003
|
+
Ok(connection) => connection,
|
|
1004
|
+
Err(error) => return fail_all(batch, error),
|
|
1005
|
+
};
|
|
1006
|
+
let mut transaction =
|
|
1007
|
+
match connection.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) {
|
|
1008
|
+
Ok(transaction) => transaction,
|
|
1009
|
+
Err(error) => return fail_all(batch, sql("begin blob admission", error)),
|
|
1010
|
+
};
|
|
1011
|
+
let mut results = Vec::with_capacity(batch.len());
|
|
1012
|
+
let mut written = Vec::with_capacity(batch.len());
|
|
1013
|
+
// Each object's row sits behind its own savepoint: one refused row refuses that
|
|
1014
|
+
// object, as it did when every object had a transaction of its own.
|
|
1015
|
+
for (ticket, recording) in batch {
|
|
1016
|
+
let outcome = transaction
|
|
1017
|
+
.savepoint()
|
|
1018
|
+
.map_err(|error| sql("begin object admission", error))
|
|
1019
|
+
.and_then(|savepoint| {
|
|
1020
|
+
if let Some(row) = &recording.record {
|
|
1021
|
+
write_verification_on(&savepoint, row)?;
|
|
1022
|
+
}
|
|
1023
|
+
savepoint
|
|
1024
|
+
.commit()
|
|
1025
|
+
.map_err(|error| sql("record object admission", error))
|
|
1026
|
+
});
|
|
1027
|
+
match outcome {
|
|
1028
|
+
Ok(()) => written.push((ticket, recording.record)),
|
|
1029
|
+
Err(error) => results.push((ticket, Err(error))),
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
match transaction
|
|
1033
|
+
.commit()
|
|
1034
|
+
.map_err(|error| sql("commit blob admission", error))
|
|
1035
|
+
{
|
|
1036
|
+
Ok(()) => {
|
|
1037
|
+
let trust = Trust::live(&self.path);
|
|
1038
|
+
for (ticket, record) in written {
|
|
1039
|
+
if let (Some(row), Some(trust)) = (&record, &trust) {
|
|
1040
|
+
trust.remember(row);
|
|
1041
|
+
}
|
|
1042
|
+
results.push((ticket, Ok(())));
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
Err(error) => {
|
|
1046
|
+
results.extend(
|
|
1047
|
+
written
|
|
1048
|
+
.into_iter()
|
|
1049
|
+
.map(|(ticket, _)| (ticket, Err(error.clone()))),
|
|
1050
|
+
);
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
results
|
|
1054
|
+
}
|
|
1055
|
+
|
|
944
1056
|
pub fn held_keys(&self) -> Result<Vec<crate::storage::HeldKey>> {
|
|
945
1057
|
let connection = self.connection()?;
|
|
946
1058
|
let mut statement = connection
|
|
@@ -1084,7 +1196,7 @@ impl Catalog {
|
|
|
1084
1196
|
}
|
|
1085
1197
|
}
|
|
1086
1198
|
|
|
1087
|
-
fn
|
|
1199
|
+
fn held_admissions(path: &Path) -> Arc<Mutex<()>> {
|
|
1088
1200
|
static TURNS: OnceLock<Mutex<HashMap<PathBuf, Arc<Mutex<()>>>>> = OnceLock::new();
|
|
1089
1201
|
TURNS
|
|
1090
1202
|
.get_or_init(Default::default)
|
|
@@ -1095,6 +1207,73 @@ fn admissions(path: &Path) -> Arc<Mutex<()>> {
|
|
|
1095
1207
|
.clone()
|
|
1096
1208
|
}
|
|
1097
1209
|
|
|
1210
|
+
/// One published object's catalog rows, handed to the admission group.
|
|
1211
|
+
pub(crate) struct Recording {
|
|
1212
|
+
/// The verification row, when this call installed the object.
|
|
1213
|
+
pub record: Option<crate::store::VRecord>,
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
#[derive(Default)]
|
|
1217
|
+
struct AdmissionGroup {
|
|
1218
|
+
state: Mutex<GroupState>,
|
|
1219
|
+
turned: std::sync::Condvar,
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
#[derive(Default)]
|
|
1223
|
+
struct GroupState {
|
|
1224
|
+
next: u64,
|
|
1225
|
+
leading: bool,
|
|
1226
|
+
queue: Vec<(u64, Recording)>,
|
|
1227
|
+
done: HashMap<u64, Result<()>>,
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
fn admission_group(path: &Path) -> Arc<AdmissionGroup> {
|
|
1231
|
+
static GROUPS: OnceLock<Mutex<HashMap<PathBuf, Arc<AdmissionGroup>>>> = OnceLock::new();
|
|
1232
|
+
GROUPS
|
|
1233
|
+
.get_or_init(Default::default)
|
|
1234
|
+
.lock()
|
|
1235
|
+
.unwrap_or_else(|error| error.into_inner())
|
|
1236
|
+
.entry(path.to_path_buf())
|
|
1237
|
+
.or_default()
|
|
1238
|
+
.clone()
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
/// A leader hands its batch's results back however it leaves, so a panic inside one batch
|
|
1242
|
+
/// cannot strand the admissions queued behind it.
|
|
1243
|
+
struct Lead<'a> {
|
|
1244
|
+
group: &'a AdmissionGroup,
|
|
1245
|
+
tickets: Vec<u64>,
|
|
1246
|
+
results: Vec<(u64, Result<()>)>,
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
impl Drop for Lead<'_> {
|
|
1250
|
+
fn drop(&mut self) {
|
|
1251
|
+
let mut state = self
|
|
1252
|
+
.group
|
|
1253
|
+
.state
|
|
1254
|
+
.lock()
|
|
1255
|
+
.unwrap_or_else(|error| error.into_inner());
|
|
1256
|
+
state.leading = false;
|
|
1257
|
+
let mut answered = std::collections::HashSet::new();
|
|
1258
|
+
for (ticket, result) in self.results.drain(..) {
|
|
1259
|
+
answered.insert(ticket);
|
|
1260
|
+
state.done.insert(ticket, result);
|
|
1261
|
+
}
|
|
1262
|
+
for ticket in &self.tickets {
|
|
1263
|
+
if !answered.contains(ticket) {
|
|
1264
|
+
state.done.insert(
|
|
1265
|
+
*ticket,
|
|
1266
|
+
Err(Refusal {
|
|
1267
|
+
code: Code::IO_FAILED,
|
|
1268
|
+
detail: "the admission batch holding this object did not finish".into(),
|
|
1269
|
+
}),
|
|
1270
|
+
);
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
self.group.turned.notify_all();
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1098
1277
|
fn hold_on(connection: &Connection, operation: &str, key: &str, length: u64) -> Result<()> {
|
|
1099
1278
|
let kind = if key.starts_with("blobs/") {
|
|
1100
1279
|
"blob"
|
|
@@ -523,17 +523,14 @@ pub fn walk(store: &Store, snap: &Manifest) -> Result<Walk> {
|
|
|
523
523
|
.map(|r| (r.obj.sha256.clone(), r.obj.length))
|
|
524
524
|
.collect();
|
|
525
525
|
for (path, e) in snap.entries() {
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
});
|
|
534
|
-
}
|
|
526
|
+
if let Some(o) = e.content() {
|
|
527
|
+
if !accounted.contains(&(o.sha256.clone(), o.length)) {
|
|
528
|
+
w.siblings.push(path.clone());
|
|
529
|
+
w.objects.push(Reached {
|
|
530
|
+
kind: "asset",
|
|
531
|
+
obj: o.clone(),
|
|
532
|
+
});
|
|
535
533
|
}
|
|
536
|
-
crate::manifest::Entry::CozyTensors(_) => {}
|
|
537
534
|
}
|
|
538
535
|
}
|
|
539
536
|
Ok(w)
|
|
@@ -22,6 +22,8 @@ pub struct CheckpointRoot {
|
|
|
22
22
|
pub repository: RepositoryName,
|
|
23
23
|
pub manifest: ObjectRef,
|
|
24
24
|
pub released: bool,
|
|
25
|
+
/// The newest TensorFS that wrote this root.
|
|
26
|
+
pub tensorfs: Option<String>,
|
|
25
27
|
}
|
|
26
28
|
|
|
27
29
|
impl Plain for CheckpointRoot {
|
|
@@ -36,11 +38,18 @@ impl Plain for CheckpointRoot {
|
|
|
36
38
|
Value::Bool(value) => *value,
|
|
37
39
|
_ => return refuse(Code::WRONG_TYPE, "checkpoint root release must be boolean"),
|
|
38
40
|
};
|
|
39
|
-
fields
|
|
41
|
+
let tensorfs = fields
|
|
42
|
+
.opt("tensorfs")
|
|
43
|
+
.map(|value| {
|
|
44
|
+
crate::canon::as_str("CheckpointRoot", "tensorfs", value).map(str::to_string)
|
|
45
|
+
})
|
|
46
|
+
.transpose()?;
|
|
47
|
+
fields.done_written_by(tensorfs.as_deref())?;
|
|
40
48
|
if manifest.length == 0 || manifest.length > crate::limits::INT_MAX as u64 {
|
|
41
49
|
return refuse(Code::LENGTH_MISMATCH, "checkpoint root manifest is empty");
|
|
42
50
|
}
|
|
43
51
|
Ok(Self {
|
|
52
|
+
tensorfs,
|
|
44
53
|
owner,
|
|
45
54
|
repository,
|
|
46
55
|
manifest,
|
|
@@ -49,7 +58,7 @@ impl Plain for CheckpointRoot {
|
|
|
49
58
|
}
|
|
50
59
|
|
|
51
60
|
fn to_value(&self) -> Value {
|
|
52
|
-
|
|
61
|
+
let mut fields = vec![
|
|
53
62
|
("owner", Value::str(self.owner.clone())),
|
|
54
63
|
(
|
|
55
64
|
"repository",
|
|
@@ -57,7 +66,11 @@ impl Plain for CheckpointRoot {
|
|
|
57
66
|
),
|
|
58
67
|
("manifest", self.manifest.to_value()),
|
|
59
68
|
("released", Value::Bool(self.released)),
|
|
60
|
-
]
|
|
69
|
+
];
|
|
70
|
+
if let Some(tensorfs) = &self.tensorfs {
|
|
71
|
+
fields.push(("tensorfs", Value::str(tensorfs.clone())));
|
|
72
|
+
}
|
|
73
|
+
Value::obj(fields)
|
|
61
74
|
}
|
|
62
75
|
}
|
|
63
76
|
|
|
@@ -126,6 +139,10 @@ fn lock(store: &Store, owner: &str) -> Result<File> {
|
|
|
126
139
|
}
|
|
127
140
|
|
|
128
141
|
fn write(store: &Store, root: &CheckpointRoot) -> Result<()> {
|
|
142
|
+
let root = &CheckpointRoot {
|
|
143
|
+
tensorfs: Some(crate::VERSION.into()),
|
|
144
|
+
..root.clone()
|
|
145
|
+
};
|
|
129
146
|
let path = path(store, &root.owner)?;
|
|
130
147
|
let temporary = path.with_extension(format!("tmp-{}", crate::meta::now_nanos_unique()));
|
|
131
148
|
let result = (|| {
|
|
@@ -159,6 +176,7 @@ fn identity(owner: &str, source: &str, manifest: ObjectRef) -> Result<Checkpoint
|
|
|
159
176
|
return refuse(Code::LENGTH_MISMATCH, "checkpoint root manifest is empty");
|
|
160
177
|
}
|
|
161
178
|
Ok(CheckpointRoot {
|
|
179
|
+
tensorfs: Some(crate::VERSION.into()),
|
|
162
180
|
owner: prefixed("checkpoint root owner", owner)?,
|
|
163
181
|
repository: repository(source)?,
|
|
164
182
|
manifest,
|
|
@@ -191,7 +191,7 @@ fn ordinary_companions_survive_derivation_without_copying_tensor_backing_files()
|
|
|
191
191
|
.push(("LICENSE".into(), b"explicit licence".to_vec()));
|
|
192
192
|
assert_eq!(
|
|
193
193
|
companion_files(&d, &sources).unwrap()["LICENSE"],
|
|
194
|
-
ObjectRef::of(b"explicit licence")
|
|
194
|
+
Entry::File(ObjectRef::of(b"explicit licence"))
|
|
195
195
|
);
|
|
196
196
|
std::fs::remove_dir_all(store.root()).unwrap();
|
|
197
197
|
}
|