tensorfs 0.3.67__tar.gz → 0.3.68__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tensorfs-0.3.67 → tensorfs-0.3.68}/Cargo.lock +2 -2
- {tensorfs-0.3.67 → tensorfs-0.3.68}/Cargo.toml +1 -1
- {tensorfs-0.3.67 → tensorfs-0.3.68}/PKG-INFO +1 -1
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/bin/tfs/fetch.rs +14 -3
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/bin/tfs/main.rs +5 -0
- tensorfs-0.3.68/crates/tensorfs-core/src/disk.rs +153 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/lib.rs +2 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/staging.rs +3 -1
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/stats.rs +33 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/store.rs +84 -16
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/pull.rs +15 -1
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/tests.rs +242 -47
- tensorfs-0.3.68/crates/tensorfs-core/src/unsynced.rs +674 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-py/src/lib.rs +5 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-py/src/store.rs +77 -15
- {tensorfs-0.3.67 → tensorfs-0.3.68}/pyproject.toml +1 -1
- {tensorfs-0.3.67 → tensorfs-0.3.68}/python/tensorfs/_ext.pyi +6 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/Cargo.toml +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/build.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/examples/h3-swiglu-layout.md +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/examples/optional-cache-backfill.md +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/examples/resident-conversion.md +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/examples/resident_conversion_profile.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/asset_update.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/b64.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/bin/tfs/ckpt.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/bin/tfs/conform.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/bin/tfs/ingest.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/bin/tfs/rd.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/bin/tfs/repo.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/bin/tfs/snap.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/bin/tfs/watch.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/cache_roots.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/canon.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/capability.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/catalog.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/cbor.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/checkpoint.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/checkpoint_root.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/corpus.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/derived/adoption.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/derived/companion_tests.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/derived/estimate.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/derived/graft_tests.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/derived/part_rows_tests.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/derived/progress.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/derived/resume_tests.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/derived/retention.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/derived/roots.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/derived.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/descriptors.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/dtype.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/durability.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/err.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/fetch.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/fill.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/fit.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/gc.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/header.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/home.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ids.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ingest/carrier.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ingest/convert.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ingest/custody.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ingest/fingerprint.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ingest/gguf.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ingest/h3_lora.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ingest/journal.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ingest/mod.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ingest/preflight.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ingest/source.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ingest/stamp.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/ingest/transaction.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/jcs.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/limits.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/machine.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/manifest.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/meta.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/project.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/providers.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/read.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/receipt.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/reclaim.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/registry.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/relation.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/repo_cache/identity.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/repo_cache.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/repository.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/sha256.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/source_artifact.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/spec.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/storage.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/api.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/http.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/http_reuse_tests.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/hub.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/ledger.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/memory.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/mod.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/pace.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/policy.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/prefix.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/push.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/ranged.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/transport/speculate.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/src/vectors.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/additive_fields.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/anima_hf_profile.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/checkpoint_roots.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/civitai_sdxl_profile.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/fd_budget.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/fd_headroom.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/fetch_url_progress.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/fit.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/fixtures/repo_apply/blobs.jsonl +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_list.jsonl +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_repository.json +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_rows.jsonl +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/fixtures/repo_apply/header.cbor +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/fixtures/repo_apply/manifest.json +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/fixtures/repo_apply/mutation.json +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/foreign_json.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/h3_lora.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/h3_swiglu.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/home.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/import_tree.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/instrument_profiles.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/liveness.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/qwen_image21_profile.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/read_plan.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/reclaim.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/reclaim_run.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/repo_cache.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/repo_store.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/resident_admission.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/sha256_backends.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/sharded_cas.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/source_artifacts.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/source_download.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/source_plan.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/source_prefixes.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/source_preflight.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/staging.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/store_boundaries.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/tensor_schema.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/transfer_memory.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/trust_index.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/version.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-core/tests/write_through.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-py/Cargo.toml +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-py/src/docs.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-py/src/errors.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/crates/tensorfs-py/src/ingest.rs +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/python/tensorfs/__init__.py +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/python/tensorfs/_derived_channel.py +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/python/tensorfs/_source_channel.py +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/python/tensorfs/_tensors.py +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/python/tensorfs/derived.py +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/python/tensorfs/errors.py +0 -0
- {tensorfs-0.3.67 → tensorfs-0.3.68}/python/tensorfs/py.typed +0 -0
|
@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
|
439
439
|
|
|
440
440
|
[[package]]
|
|
441
441
|
name = "tensorfs-core"
|
|
442
|
-
version = "0.3.
|
|
442
|
+
version = "0.3.68"
|
|
443
443
|
dependencies = [
|
|
444
444
|
"fs2",
|
|
445
445
|
"libc",
|
|
@@ -454,7 +454,7 @@ dependencies = [
|
|
|
454
454
|
|
|
455
455
|
[[package]]
|
|
456
456
|
name = "tensorfs-py"
|
|
457
|
-
version = "0.3.
|
|
457
|
+
version = "0.3.68"
|
|
458
458
|
dependencies = [
|
|
459
459
|
"pyo3",
|
|
460
460
|
"tensorfs-core",
|
|
@@ -257,6 +257,8 @@ fn pull_with_budget(
|
|
|
257
257
|
flags: &Flags,
|
|
258
258
|
budget: Option<(u64, u64)>,
|
|
259
259
|
) -> ExitCode {
|
|
260
|
+
// Before the open, which is where an unclean shutdown's recovery runs.
|
|
261
|
+
let before = tensorfs_core::stats::snapshot();
|
|
260
262
|
let store = ok!(Store::open(root));
|
|
261
263
|
let Some(base) = flag(flags, "hub") else {
|
|
262
264
|
eprintln!("--hub <base-url> is required");
|
|
@@ -302,7 +304,6 @@ fn pull_with_budget(
|
|
|
302
304
|
let observe = |_object: &ObjectRef, moved: u64, door: ObjectSource| progress.saw(moved, door);
|
|
303
305
|
request.on_plan = Some(&planned);
|
|
304
306
|
request.on_object = Some(&observe);
|
|
305
|
-
let before = tensorfs_core::stats::snapshot();
|
|
306
307
|
let started = std::time::Instant::now();
|
|
307
308
|
let report = ok!(transport::pull(&request));
|
|
308
309
|
let seconds = started.elapsed().as_secs_f64();
|
|
@@ -333,9 +334,12 @@ fn pull_with_budget(
|
|
|
333
334
|
})
|
|
334
335
|
.collect();
|
|
335
336
|
// And what admitting them cost: a pull whose streams stop growing while its batches
|
|
336
|
-
// take most of the wall time is held by the disk, not the link.
|
|
337
|
+
// take most of the wall time is held by the disk, not the link. `durability` says
|
|
338
|
+
// which syncs the disk the Store lives on made it pay, and what an unclean shutdown
|
|
339
|
+
// before this pull cost to repair.
|
|
340
|
+
let durability = &report.durability;
|
|
337
341
|
eprintln!(
|
|
338
|
-
"{{\"event\":\"pull.streams\",\"model\":{},\"seconds\":{:.3},\"bytes_moved\":{},\"streams\":{},\"levels\":[{}],\"backoffs\":{},\"admission\":{{\"objects\":{},\"batches\":{},\"seconds\":{:.3}}}}}",
|
|
342
|
+
"{{\"event\":\"pull.streams\",\"model\":{},\"seconds\":{:.3},\"bytes_moved\":{},\"streams\":{},\"levels\":[{}],\"backoffs\":{},\"admission\":{{\"objects\":{},\"batches\":{},\"seconds\":{:.3}}},\"durability\":{{\"class\":{},\"filesystem\":{},\"syncs\":{},\"sync_seconds\":{:.3},\"recovery\":{{\"rehashed\":{},\"removed\":{},\"seconds\":{:.3}}}}}}}",
|
|
339
343
|
quoted(&report.model),
|
|
340
344
|
seconds,
|
|
341
345
|
report.bytes_moved,
|
|
@@ -345,6 +349,13 @@ fn pull_with_budget(
|
|
|
345
349
|
after.admitted_objects - before.admitted_objects,
|
|
346
350
|
after.admission_batches - before.admission_batches,
|
|
347
351
|
(after.admission_nanos - before.admission_nanos) as f64 / 1e9,
|
|
352
|
+
quoted(durability.class.as_str()),
|
|
353
|
+
quoted(durability.filesystem),
|
|
354
|
+
durability.syncs,
|
|
355
|
+
durability.seconds,
|
|
356
|
+
after.recovery_rehashed - before.recovery_rehashed,
|
|
357
|
+
after.recovery_removed - before.recovery_removed,
|
|
358
|
+
(after.recovery_nanos - before.recovery_nanos) as f64 / 1e9,
|
|
348
359
|
);
|
|
349
360
|
println!(
|
|
350
361
|
"model {}@{} lane {}",
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
//! What a crash can take from a Store, read off the filesystem the Store actually lives on.
|
|
2
|
+
//!
|
|
3
|
+
//! An fsync protects bytes against the machine going down before its page cache is written.
|
|
4
|
+
//! Whether that protection is worth paying for depends on where the bytes live, and only the
|
|
5
|
+
//! mount can say that — never a flag, never an environment variable.
|
|
6
|
+
//!
|
|
7
|
+
//! The rule, from `statfs(2)` of the Store root (Linux):
|
|
8
|
+
//!
|
|
9
|
+
//! - **Ephemeral** — tmpfs, ramfs, overlayfs. RAM, or a container's writable layer, which the
|
|
10
|
+
//! platform discards with the machine or container (a RunPod container disk is erased when
|
|
11
|
+
//! the pod stops). Nothing a crash could leave behind is ever read again, and a process
|
|
12
|
+
//! crash leaves the page cache intact, so no fsync buys anything.
|
|
13
|
+
//! - **Network** — NFS, SMB/CIFS, FUSE, Ceph, 9p, AFS, Coda, GFS2, OCFS2, Lustre, GPFS.
|
|
14
|
+
//! Another machine may read it, and close-to-open is the only ordering that machine sees:
|
|
15
|
+
//! bytes are flushed before their name is published, as before.
|
|
16
|
+
//! - **Persistent** — everything else (ext4, xfs, btrfs, zfs, ...), a `statfs` that fails,
|
|
17
|
+
//! and every non-Linux platform. The safe side: it survives a crash of this machine.
|
|
18
|
+
|
|
19
|
+
use std::path::Path;
|
|
20
|
+
|
|
21
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
|
22
|
+
pub enum DiskClass {
|
|
23
|
+
Ephemeral,
|
|
24
|
+
Persistent,
|
|
25
|
+
Network,
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
impl DiskClass {
|
|
29
|
+
/// The class of the filesystem holding `path`.
|
|
30
|
+
pub fn of(path: &Path) -> DiskClass {
|
|
31
|
+
filesystem(path).class
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
pub fn as_str(self) -> &'static str {
|
|
35
|
+
match self {
|
|
36
|
+
DiskClass::Ephemeral => "ephemeral",
|
|
37
|
+
DiskClass::Persistent => "persistent",
|
|
38
|
+
DiskClass::Network => "network",
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/// The filesystem holding a path: its class and the name it was recognised by.
|
|
44
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
45
|
+
pub struct Filesystem {
|
|
46
|
+
pub class: DiskClass,
|
|
47
|
+
/// `ext4`, `overlayfs`, `nfs`, ...; `other` for an unrecognised magic, `unknown` when
|
|
48
|
+
/// `statfs` failed or the platform has no magic.
|
|
49
|
+
pub kind: &'static str,
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
#[cfg(target_os = "linux")]
|
|
53
|
+
pub fn filesystem(path: &Path) -> Filesystem {
|
|
54
|
+
match rustix::fs::statfs(path) {
|
|
55
|
+
#[allow(clippy::unnecessary_cast)]
|
|
56
|
+
Ok(stat) => classify((stat.f_type as u64) & 0xffff_ffff),
|
|
57
|
+
Err(_) => Filesystem {
|
|
58
|
+
class: DiskClass::Persistent,
|
|
59
|
+
kind: "unknown",
|
|
60
|
+
},
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
#[cfg(not(target_os = "linux"))]
|
|
65
|
+
pub fn filesystem(_path: &Path) -> Filesystem {
|
|
66
|
+
Filesystem {
|
|
67
|
+
class: DiskClass::Persistent,
|
|
68
|
+
kind: "unknown",
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/// Superblock magics from `linux/magic.h` and the filesystems' own headers.
|
|
73
|
+
#[cfg(target_os = "linux")]
|
|
74
|
+
fn classify(magic: u64) -> Filesystem {
|
|
75
|
+
use DiskClass::*;
|
|
76
|
+
let (class, kind) = match magic {
|
|
77
|
+
0x0102_1994 => (Ephemeral, "tmpfs"),
|
|
78
|
+
0x8584_58f6 => (Ephemeral, "ramfs"),
|
|
79
|
+
0x794c_7630 => (Ephemeral, "overlayfs"),
|
|
80
|
+
0x6969 => (Network, "nfs"),
|
|
81
|
+
0x517b => (Network, "smb"),
|
|
82
|
+
0xff53_4d42 => (Network, "cifs"),
|
|
83
|
+
0xfe53_4d42 => (Network, "smb2"),
|
|
84
|
+
0x6573_5546 => (Network, "fuse"),
|
|
85
|
+
0x00c3_6400 => (Network, "ceph"),
|
|
86
|
+
0x0102_1997 => (Network, "9p"),
|
|
87
|
+
0x5346_414f | 0x6b41_4653 => (Network, "afs"),
|
|
88
|
+
0x7375_7245 => (Network, "coda"),
|
|
89
|
+
0x0116_1970 => (Network, "gfs2"),
|
|
90
|
+
0x7461_636f => (Network, "ocfs2"),
|
|
91
|
+
0x0bd0_0bd0 => (Network, "lustre"),
|
|
92
|
+
0x4750_4653 => (Network, "gpfs"),
|
|
93
|
+
0xef53 => (Persistent, "ext4"),
|
|
94
|
+
0x5846_5342 => (Persistent, "xfs"),
|
|
95
|
+
0x9123_683e => (Persistent, "btrfs"),
|
|
96
|
+
0x2fc1_2fc1 => (Persistent, "zfs"),
|
|
97
|
+
0xf2f5_2010 => (Persistent, "f2fs"),
|
|
98
|
+
_ => (Persistent, "other"),
|
|
99
|
+
};
|
|
100
|
+
Filesystem { class, kind }
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/// This kernel's boot, or `None` where there is no such identity. Page cache belongs to a
|
|
104
|
+
/// boot: bytes written under this boot and not yet on disk are still readable until it ends.
|
|
105
|
+
/// A container sees its host's boot, which is the one that owns the page cache.
|
|
106
|
+
pub(crate) fn boot_id() -> Option<String> {
|
|
107
|
+
#[cfg(target_os = "linux")]
|
|
108
|
+
{
|
|
109
|
+
let id = std::fs::read_to_string("/proc/sys/kernel/random/boot_id").ok()?;
|
|
110
|
+
let id = id.trim();
|
|
111
|
+
(id.len() == 36 && id.bytes().all(|b| b.is_ascii_hexdigit() || b == b'-'))
|
|
112
|
+
.then(|| id.to_string())
|
|
113
|
+
}
|
|
114
|
+
#[cfg(not(target_os = "linux"))]
|
|
115
|
+
{
|
|
116
|
+
None
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
#[cfg(test)]
|
|
121
|
+
mod tests {
|
|
122
|
+
use super::*;
|
|
123
|
+
|
|
124
|
+
#[cfg(target_os = "linux")]
|
|
125
|
+
#[test]
|
|
126
|
+
fn magics_name_their_class() {
|
|
127
|
+
assert_eq!(classify(0x794c_7630).class, DiskClass::Ephemeral);
|
|
128
|
+
assert_eq!(classify(0x0102_1994).class, DiskClass::Ephemeral);
|
|
129
|
+
assert_eq!(classify(0x6969).class, DiskClass::Network);
|
|
130
|
+
assert_eq!(classify(0x6573_5546).class, DiskClass::Network);
|
|
131
|
+
assert_eq!(classify(0xef53).class, DiskClass::Persistent);
|
|
132
|
+
assert_eq!(classify(0x1234_5678).class, DiskClass::Persistent);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
#[cfg(target_os = "linux")]
|
|
136
|
+
#[test]
|
|
137
|
+
fn real_mounts_classify() {
|
|
138
|
+
if Path::new("/dev/shm").is_dir() {
|
|
139
|
+
assert_eq!(filesystem(Path::new("/dev/shm")).kind, "tmpfs");
|
|
140
|
+
assert_eq!(DiskClass::of(Path::new("/dev/shm")), DiskClass::Ephemeral);
|
|
141
|
+
}
|
|
142
|
+
assert_eq!(
|
|
143
|
+
DiskClass::of(Path::new("/proc/self")),
|
|
144
|
+
DiskClass::Persistent,
|
|
145
|
+
"an unrecognised filesystem is the safe side"
|
|
146
|
+
);
|
|
147
|
+
assert_eq!(
|
|
148
|
+
DiskClass::of(Path::new("/no/such/path/anywhere")),
|
|
149
|
+
DiskClass::Persistent
|
|
150
|
+
);
|
|
151
|
+
assert!(boot_id().is_some());
|
|
152
|
+
}
|
|
153
|
+
}
|
|
@@ -20,6 +20,7 @@ pub mod checkpoint_root;
|
|
|
20
20
|
pub mod corpus;
|
|
21
21
|
pub mod derived;
|
|
22
22
|
pub mod descriptors;
|
|
23
|
+
pub mod disk;
|
|
23
24
|
pub mod dtype;
|
|
24
25
|
pub mod durability;
|
|
25
26
|
pub mod err;
|
|
@@ -53,6 +54,7 @@ pub mod stats;
|
|
|
53
54
|
pub mod storage;
|
|
54
55
|
pub mod store;
|
|
55
56
|
pub mod transport;
|
|
57
|
+
pub mod unsynced;
|
|
56
58
|
pub mod vectors;
|
|
57
59
|
|
|
58
60
|
/// This TensorFS's release version.
|
|
@@ -204,7 +204,9 @@ pub fn stage<R: Read>(store: &Store, r: &mut R, want: &ObjectRef) -> Result<Carr
|
|
|
204
204
|
return Err(io(format!("link {}", destination.display()), error));
|
|
205
205
|
}
|
|
206
206
|
}
|
|
207
|
-
|
|
207
|
+
if store.flushes() {
|
|
208
|
+
crate::store::fsync_dir(&dir)?;
|
|
209
|
+
}
|
|
208
210
|
let _ = fs::remove_file(&verified.tmp);
|
|
209
211
|
drop(verified.file);
|
|
210
212
|
Ok(Carrier {
|
|
@@ -9,6 +9,11 @@ static REPO_CACHE_VERIFIED_READS: AtomicU64 = AtomicU64::new(0);
|
|
|
9
9
|
static ADMISSION_BATCHES: AtomicU64 = AtomicU64::new(0);
|
|
10
10
|
static ADMITTED_OBJECTS: AtomicU64 = AtomicU64::new(0);
|
|
11
11
|
static ADMISSION_NANOS: AtomicU64 = AtomicU64::new(0);
|
|
12
|
+
static SYNCS: AtomicU64 = AtomicU64::new(0);
|
|
13
|
+
static SYNC_NANOS: AtomicU64 = AtomicU64::new(0);
|
|
14
|
+
static RECOVERY_REHASHED: AtomicU64 = AtomicU64::new(0);
|
|
15
|
+
static RECOVERY_REMOVED: AtomicU64 = AtomicU64::new(0);
|
|
16
|
+
static RECOVERY_NANOS: AtomicU64 = AtomicU64::new(0);
|
|
12
17
|
|
|
13
18
|
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
|
14
19
|
pub struct Stats {
|
|
@@ -24,6 +29,14 @@ pub struct Stats {
|
|
|
24
29
|
pub admitted_objects: u64,
|
|
25
30
|
/// Wall time those batches spent linking, fsyncing and committing.
|
|
26
31
|
pub admission_nanos: u64,
|
|
32
|
+
/// Filesystem syncs pulls ran on persistent disks, and their wall time.
|
|
33
|
+
pub syncs: u64,
|
|
34
|
+
pub sync_nanos: u64,
|
|
35
|
+
/// Objects rehashed after an unclean shutdown, those found torn and removed, and the
|
|
36
|
+
/// wall time the recovery took.
|
|
37
|
+
pub recovery_rehashed: u64,
|
|
38
|
+
pub recovery_removed: u64,
|
|
39
|
+
pub recovery_nanos: u64,
|
|
27
40
|
}
|
|
28
41
|
|
|
29
42
|
pub fn snapshot() -> Stats {
|
|
@@ -34,6 +47,11 @@ pub fn snapshot() -> Stats {
|
|
|
34
47
|
admission_batches: ADMISSION_BATCHES.load(Ordering::Relaxed),
|
|
35
48
|
admitted_objects: ADMITTED_OBJECTS.load(Ordering::Relaxed),
|
|
36
49
|
admission_nanos: ADMISSION_NANOS.load(Ordering::Relaxed),
|
|
50
|
+
syncs: SYNCS.load(Ordering::Relaxed),
|
|
51
|
+
sync_nanos: SYNC_NANOS.load(Ordering::Relaxed),
|
|
52
|
+
recovery_rehashed: RECOVERY_REHASHED.load(Ordering::Relaxed),
|
|
53
|
+
recovery_removed: RECOVERY_REMOVED.load(Ordering::Relaxed),
|
|
54
|
+
recovery_nanos: RECOVERY_NANOS.load(Ordering::Relaxed),
|
|
37
55
|
}
|
|
38
56
|
}
|
|
39
57
|
|
|
@@ -69,3 +87,18 @@ pub(crate) fn admission_batch(objects: usize, spent: std::time::Duration) {
|
|
|
69
87
|
Ordering::Relaxed,
|
|
70
88
|
);
|
|
71
89
|
}
|
|
90
|
+
|
|
91
|
+
fn nanos(spent: std::time::Duration) -> u64 {
|
|
92
|
+
u64::try_from(spent.as_nanos()).unwrap_or(u64::MAX)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
pub(crate) fn synced(spent: std::time::Duration) {
|
|
96
|
+
SYNCS.fetch_add(1, Ordering::Relaxed);
|
|
97
|
+
SYNC_NANOS.fetch_add(nanos(spent), Ordering::Relaxed);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
pub(crate) fn recovered(recovery: &crate::unsynced::Recovery, spent: std::time::Duration) {
|
|
101
|
+
RECOVERY_REHASHED.fetch_add(recovery.rehashed, Ordering::Relaxed);
|
|
102
|
+
RECOVERY_REMOVED.fetch_add(recovery.removed, Ordering::Relaxed);
|
|
103
|
+
RECOVERY_NANOS.fetch_add(nanos(spent), Ordering::Relaxed);
|
|
104
|
+
}
|
|
@@ -8,7 +8,10 @@
|
|
|
8
8
|
//!
|
|
9
9
|
//! Admission is `writing → verified-temp → installed`: temp in `tmp/`, hash while writing,
|
|
10
10
|
//! fsync, 0444, then a no-clobber `link()` into place plus a parent-directory fsync. A
|
|
11
|
-
//! crash at any stage leaves a temp, never a partial object under `objects/`.
|
|
11
|
+
//! crash at any stage leaves a temp, never a partial object under `objects/`. Which of those
|
|
12
|
+
//! syncs an admission pays depends on the disk the Store lives on ([`crate::disk`]): none on
|
|
13
|
+
//! an ephemeral one, and on a persistent one none inside a pull, whose batch sync
|
|
14
|
+
//! covers them ([`crate::unsynced`]).
|
|
12
15
|
//!
|
|
13
16
|
//! `contains` is a PRESENCE HINT and nothing more. Trusting bytes requires a durable
|
|
14
17
|
//! verification record bound to object id, length, file identity (dev, ino), size and
|
|
@@ -305,6 +308,10 @@ pub struct Store {
|
|
|
305
308
|
budget: Option<u64>,
|
|
306
309
|
/// This process's verified-blob index for the catalog, shared with every other handle.
|
|
307
310
|
trust: std::sync::Arc<crate::catalog::Trust>,
|
|
311
|
+
/// The filesystem this Store lives on, and so what a crash can take from it.
|
|
312
|
+
disk: crate::disk::Filesystem,
|
|
313
|
+
/// The pull whose batch sync covers this handle's admissions (persistent disk).
|
|
314
|
+
epoch: Option<std::sync::Arc<crate::unsynced::Epoch>>,
|
|
308
315
|
}
|
|
309
316
|
|
|
310
317
|
impl std::fmt::Debug for Store {
|
|
@@ -313,6 +320,7 @@ impl std::fmt::Debug for Store {
|
|
|
313
320
|
.field("root", &self.root)
|
|
314
321
|
.field("observed", &self.progress.is_some())
|
|
315
322
|
.field("repo_cache", &self.cache.as_ref().map(|c| c.root()))
|
|
323
|
+
.field("disk", &self.disk)
|
|
316
324
|
.finish()
|
|
317
325
|
}
|
|
318
326
|
}
|
|
@@ -518,6 +526,8 @@ impl Store {
|
|
|
518
526
|
cache: read_repo_cache_binding(root),
|
|
519
527
|
budget: read_disk_budget(root),
|
|
520
528
|
trust: crate::catalog::Trust::of(root),
|
|
529
|
+
disk: crate::disk::filesystem(root),
|
|
530
|
+
epoch: None,
|
|
521
531
|
})
|
|
522
532
|
}
|
|
523
533
|
|
|
@@ -549,6 +559,45 @@ impl Store {
|
|
|
549
559
|
&self.root
|
|
550
560
|
}
|
|
551
561
|
|
|
562
|
+
/// The filesystem this Store lives on. See [`crate::disk`].
|
|
563
|
+
pub fn disk(&self) -> crate::disk::Filesystem {
|
|
564
|
+
self.disk
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
pub fn disk_class(&self) -> crate::disk::DiskClass {
|
|
568
|
+
self.disk.class
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
/// Whether an admission makes its own bytes and name durable before it reports: not on
|
|
572
|
+
/// an ephemeral disk, and not under a pull's epoch, whose batch sync covers them.
|
|
573
|
+
pub(crate) fn flushes(&self) -> bool {
|
|
574
|
+
match self.disk.class {
|
|
575
|
+
crate::disk::DiskClass::Ephemeral => false,
|
|
576
|
+
crate::disk::DiskClass::Network => true,
|
|
577
|
+
crate::disk::DiskClass::Persistent => self.epoch.is_none(),
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
/// The same Store, its admissions covered by `epoch`.
|
|
582
|
+
pub(crate) fn syncing(&self, epoch: Option<std::sync::Arc<crate::unsynced::Epoch>>) -> Store {
|
|
583
|
+
Store {
|
|
584
|
+
epoch,
|
|
585
|
+
..self.clone()
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/// The same Store as if it lived on a disk of `class`: benchmarks of one real disk.
|
|
590
|
+
#[cfg(test)]
|
|
591
|
+
pub(crate) fn with_disk_class(self, class: crate::disk::DiskClass) -> Store {
|
|
592
|
+
Store {
|
|
593
|
+
disk: crate::disk::Filesystem {
|
|
594
|
+
class,
|
|
595
|
+
kind: "injected",
|
|
596
|
+
},
|
|
597
|
+
..self
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
|
|
552
601
|
pub(crate) fn trust(&self) -> &crate::catalog::Trust {
|
|
553
602
|
&self.trust
|
|
554
603
|
}
|
|
@@ -867,12 +916,14 @@ impl Store {
|
|
|
867
916
|
child
|
|
868
917
|
.set_permissions(fs::Permissions::from_mode(0o755))
|
|
869
918
|
.map_err(|error| io("set public Store directory mode", error))?;
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
919
|
+
if self.flushes() {
|
|
920
|
+
child
|
|
921
|
+
.sync_all()
|
|
922
|
+
.map_err(|error| io("sync public Store directory", error))?;
|
|
923
|
+
parent
|
|
924
|
+
.sync_all()
|
|
925
|
+
.map_err(|error| io("sync public directory parent", error))?;
|
|
926
|
+
}
|
|
876
927
|
} else if metadata.uid() != owner.uid() || metadata.gid() != owner.gid() {
|
|
877
928
|
return refuse(
|
|
878
929
|
Code::STORE_ERA,
|
|
@@ -1072,8 +1123,10 @@ impl Store {
|
|
|
1072
1123
|
.map_err(|error| io("create manifest temp", error))?;
|
|
1073
1124
|
file.write_all(&bytes)
|
|
1074
1125
|
.map_err(|error| io("write manifest", error))?;
|
|
1075
|
-
|
|
1076
|
-
.
|
|
1126
|
+
if self.flushes() {
|
|
1127
|
+
file.sync_all()
|
|
1128
|
+
.map_err(|error| io("fsync manifest", error))?;
|
|
1129
|
+
}
|
|
1077
1130
|
fs::set_permissions(&temporary, fs::Permissions::from_mode(0o444))
|
|
1078
1131
|
.map_err(|error| io("chmod manifest", error))?;
|
|
1079
1132
|
let admitted = match fs::hard_link(&temporary, &path) {
|
|
@@ -1082,9 +1135,12 @@ impl Store {
|
|
|
1082
1135
|
Err(error) => return cleanup(&temporary, io("install manifest", error)),
|
|
1083
1136
|
};
|
|
1084
1137
|
let _ = fs::remove_file(&temporary);
|
|
1085
|
-
if admitted {
|
|
1138
|
+
if admitted && self.flushes() {
|
|
1086
1139
|
fsync_dir(path.parent().unwrap())?;
|
|
1087
1140
|
}
|
|
1141
|
+
if let (true, Some(epoch)) = (admitted, &self.epoch) {
|
|
1142
|
+
epoch.admitted(path.clone(), object.length)?;
|
|
1143
|
+
}
|
|
1088
1144
|
self.read_manifest(&object)?;
|
|
1089
1145
|
Ok(Put {
|
|
1090
1146
|
obj: object,
|
|
@@ -1404,8 +1460,10 @@ impl Store {
|
|
|
1404
1460
|
}
|
|
1405
1461
|
}
|
|
1406
1462
|
fault.hit("written");
|
|
1407
|
-
if
|
|
1408
|
-
|
|
1463
|
+
if self.flushes() {
|
|
1464
|
+
if let Err(e) = f.sync_all() {
|
|
1465
|
+
return cleanup(&tmp, io("fsync temp", e));
|
|
1466
|
+
}
|
|
1409
1467
|
}
|
|
1410
1468
|
fault.hit("fsync");
|
|
1411
1469
|
|
|
@@ -1483,13 +1541,18 @@ impl Store {
|
|
|
1483
1541
|
};
|
|
1484
1542
|
let publish = move || {
|
|
1485
1543
|
let installed = self.link_verified(&verified, fault)?;
|
|
1486
|
-
if let Some(dir) =
|
|
1544
|
+
if let Some(dir) = installed.as_ref().filter(|_| self.flushes()) {
|
|
1487
1545
|
if let Err(error) = fsync_dir(dir) {
|
|
1488
1546
|
let _ = fs::remove_file(&verified.tmp);
|
|
1489
1547
|
return Err(error);
|
|
1490
1548
|
}
|
|
1491
1549
|
}
|
|
1492
|
-
|
|
1550
|
+
let length = verified.length;
|
|
1551
|
+
let record = self.finish_linked(verified, installed.is_some())?;
|
|
1552
|
+
if let (Some(epoch), Some(record)) = (&self.epoch, &record) {
|
|
1553
|
+
epoch.admitted(self.object_path(&record.sha256), length)?;
|
|
1554
|
+
}
|
|
1555
|
+
Ok(record)
|
|
1493
1556
|
};
|
|
1494
1557
|
let admitted = match operation {
|
|
1495
1558
|
Some(operation) => catalog.admit_blob(Some(operation), &object, publish)?,
|
|
@@ -2143,15 +2206,20 @@ fn creation_lock(root: &Path) -> Result<File> {
|
|
|
2143
2206
|
Ok(directory)
|
|
2144
2207
|
}
|
|
2145
2208
|
|
|
2209
|
+
/// No handle is returned while an unclean shutdown's unsynced admissions are unverified.
|
|
2146
2210
|
fn open_validated(root: &Path) -> Result<Store> {
|
|
2147
2211
|
fs::create_dir_all(root.join("tmp")).map_err(|error| io("mkdir tmp", error))?;
|
|
2148
|
-
|
|
2212
|
+
let store = Store {
|
|
2149
2213
|
root: root.to_path_buf(),
|
|
2150
2214
|
progress: None,
|
|
2151
2215
|
cache: read_repo_cache_binding(root),
|
|
2152
2216
|
budget: read_disk_budget(root),
|
|
2153
2217
|
trust: crate::catalog::Trust::of(root),
|
|
2154
|
-
|
|
2218
|
+
disk: crate::disk::filesystem(root),
|
|
2219
|
+
epoch: None,
|
|
2220
|
+
};
|
|
2221
|
+
crate::unsynced::recover(&store)?;
|
|
2222
|
+
Ok(store)
|
|
2155
2223
|
}
|
|
2156
2224
|
|
|
2157
2225
|
fn open_real_directory(path: &Path) -> Result<File> {
|
|
@@ -1339,6 +1339,8 @@ pub struct PullReport {
|
|
|
1339
1339
|
pub bytes_total: u64,
|
|
1340
1340
|
pub presence: Vec<(String, &'static str)>,
|
|
1341
1341
|
pub stamps: Vec<(String, Vec<String>)>,
|
|
1342
|
+
/// The disk the Store lives on and what making this pull durable cost.
|
|
1343
|
+
pub durability: crate::unsynced::SyncCost,
|
|
1342
1344
|
}
|
|
1343
1345
|
|
|
1344
1346
|
/// Pull one hub checkpoint into a local store: closure, `FetchPlan::of` against the REAL
|
|
@@ -1437,6 +1439,10 @@ pub fn pull(request: &PullRequest<'_>) -> Result<PullReport> {
|
|
|
1437
1439
|
// planning pass condemn the very first mint that followed it.
|
|
1438
1440
|
ledger.transfer_begins();
|
|
1439
1441
|
if !plan.wanted.is_empty() {
|
|
1442
|
+
// On a persistent disk the walk admits without fsync and fsyncs its objects in
|
|
1443
|
+
// batches, the last one before anything records the checkpoint.
|
|
1444
|
+
let epoch = crate::unsynced::Epoch::begin(request.store)?;
|
|
1445
|
+
let store = request.store.syncing(epoch.clone());
|
|
1440
1446
|
let leases = Leases {
|
|
1441
1447
|
client,
|
|
1442
1448
|
base: request.base,
|
|
@@ -1452,7 +1458,7 @@ pub fn pull(request: &PullRequest<'_>) -> Result<PullReport> {
|
|
|
1452
1458
|
minting: Minting::default(),
|
|
1453
1459
|
};
|
|
1454
1460
|
let walked = fetch_wanted_bounded(
|
|
1455
|
-
|
|
1461
|
+
&store,
|
|
1456
1462
|
&plan,
|
|
1457
1463
|
&leases,
|
|
1458
1464
|
&policy,
|
|
@@ -1472,6 +1478,9 @@ pub fn pull(request: &PullRequest<'_>) -> Result<PullReport> {
|
|
|
1472
1478
|
report.bytes_moved = walked.bytes_moved;
|
|
1473
1479
|
report.cached = walked.cached;
|
|
1474
1480
|
report.bytes_cached = walked.bytes_cached;
|
|
1481
|
+
if let Some(epoch) = epoch {
|
|
1482
|
+
(report.durability.syncs, report.durability.seconds) = epoch.finish()?;
|
|
1483
|
+
}
|
|
1475
1484
|
}
|
|
1476
1485
|
ledger.check_running()?;
|
|
1477
1486
|
// Nothing moved and this Store already records the checkpoint, which it does only once
|
|
@@ -1522,6 +1531,11 @@ fn plan_against_store(
|
|
|
1522
1531
|
.map(|(object, presence)| (object.id(), presence.as_str()))
|
|
1523
1532
|
.collect(),
|
|
1524
1533
|
stamps: closure.stamps.clone(),
|
|
1534
|
+
durability: crate::unsynced::SyncCost {
|
|
1535
|
+
class: request.store.disk_class(),
|
|
1536
|
+
filesystem: request.store.disk().kind,
|
|
1537
|
+
..Default::default()
|
|
1538
|
+
},
|
|
1525
1539
|
..Default::default()
|
|
1526
1540
|
};
|
|
1527
1541
|
Ok((plan, report))
|