tensorfs 0.3.65__tar.gz → 0.3.66__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tensorfs-0.3.65 → tensorfs-0.3.66}/Cargo.lock +2 -2
- {tensorfs-0.3.65 → tensorfs-0.3.66}/Cargo.toml +1 -1
- {tensorfs-0.3.65 → tensorfs-0.3.66}/PKG-INFO +1 -1
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/bin/tfs/fetch.rs +7 -3
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/catalog.rs +36 -11
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ingest/preflight.rs +20 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ingest/source.rs +87 -17
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/store.rs +13 -11
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/tests.rs +122 -89
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/anima_hf_profile.rs +11 -0
- tensorfs-0.3.66/crates/tensorfs-core/tests/civitai_sdxl_profile.rs +84 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/fd_budget.rs +32 -19
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-py/src/ingest.rs +33 -9
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-py/src/lib.rs +1 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/pyproject.toml +1 -1
- {tensorfs-0.3.65 → tensorfs-0.3.66}/python/tensorfs/__init__.py +2 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/python/tensorfs/_ext.pyi +6 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/Cargo.toml +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/build.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/examples/h3-swiglu-layout.md +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/examples/optional-cache-backfill.md +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/examples/resident-conversion.md +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/examples/resident_conversion_profile.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/asset_update.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/b64.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/bin/tfs/ckpt.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/bin/tfs/conform.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/bin/tfs/ingest.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/bin/tfs/main.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/bin/tfs/rd.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/bin/tfs/repo.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/bin/tfs/snap.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/bin/tfs/watch.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/cache_roots.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/canon.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/capability.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/cbor.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/checkpoint.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/checkpoint_root.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/corpus.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived/adoption.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived/companion_tests.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived/estimate.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived/graft_tests.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived/part_rows_tests.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived/progress.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived/resume_tests.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived/retention.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived/roots.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/derived.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/descriptors.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/dtype.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/durability.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/err.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/fetch.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/fill.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/fit.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/gc.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/header.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/home.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ids.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ingest/carrier.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ingest/convert.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ingest/custody.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ingest/fingerprint.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ingest/gguf.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ingest/h3_lora.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ingest/journal.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ingest/mod.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ingest/stamp.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/ingest/transaction.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/jcs.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/lib.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/limits.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/machine.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/manifest.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/meta.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/project.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/providers.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/read.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/receipt.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/reclaim.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/registry.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/relation.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/repo_cache/identity.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/repo_cache.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/repository.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/sha256.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/source_artifact.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/spec.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/staging.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/stats.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/storage.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/api.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/http.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/http_reuse_tests.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/hub.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/ledger.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/memory.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/mod.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/pace.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/policy.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/prefix.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/pull.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/push.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/ranged.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/transport/speculate.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/src/vectors.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/additive_fields.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/checkpoint_roots.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/fd_headroom.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/fetch_url_progress.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/fit.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/fixtures/repo_apply/blobs.jsonl +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_list.jsonl +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_repository.json +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_rows.jsonl +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/fixtures/repo_apply/header.cbor +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/fixtures/repo_apply/manifest.json +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/fixtures/repo_apply/mutation.json +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/foreign_json.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/h3_lora.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/h3_swiglu.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/home.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/import_tree.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/instrument_profiles.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/liveness.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/qwen_image21_profile.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/read_plan.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/reclaim.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/reclaim_run.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/repo_cache.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/repo_store.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/resident_admission.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/sha256_backends.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/sharded_cas.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/source_artifacts.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/source_download.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/source_plan.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/source_prefixes.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/source_preflight.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/staging.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/store_boundaries.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/tensor_schema.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/transfer_memory.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/trust_index.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/version.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-core/tests/write_through.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-py/Cargo.toml +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-py/src/docs.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-py/src/errors.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/crates/tensorfs-py/src/store.rs +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/python/tensorfs/_derived_channel.py +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/python/tensorfs/_source_channel.py +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/python/tensorfs/_tensors.py +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/python/tensorfs/derived.py +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/python/tensorfs/errors.py +0 -0
- {tensorfs-0.3.65 → tensorfs-0.3.66}/python/tensorfs/py.typed +0 -0
|
@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
|
439
439
|
|
|
440
440
|
[[package]]
|
|
441
441
|
name = "tensorfs-core"
|
|
442
|
-
version = "0.3.
|
|
442
|
+
version = "0.3.66"
|
|
443
443
|
dependencies = [
|
|
444
444
|
"fs2",
|
|
445
445
|
"libc",
|
|
@@ -454,7 +454,7 @@ dependencies = [
|
|
|
454
454
|
|
|
455
455
|
[[package]]
|
|
456
456
|
name = "tensorfs-py"
|
|
457
|
-
version = "0.3.
|
|
457
|
+
version = "0.3.66"
|
|
458
458
|
dependencies = [
|
|
459
459
|
"pyo3",
|
|
460
460
|
"tensorfs-core",
|
|
@@ -1358,6 +1358,7 @@ struct PullProgress {
|
|
|
1358
1358
|
struct PullProgressState {
|
|
1359
1359
|
total_bytes: u64,
|
|
1360
1360
|
held_bytes: u64,
|
|
1361
|
+
wanted_objects: u64,
|
|
1361
1362
|
origin_bytes: u64,
|
|
1362
1363
|
cache_bytes: u64,
|
|
1363
1364
|
origin_objects: u64,
|
|
@@ -1371,6 +1372,7 @@ impl PullProgress {
|
|
|
1371
1372
|
state: Mutex::new(PullProgressState {
|
|
1372
1373
|
total_bytes: 0,
|
|
1373
1374
|
held_bytes: 0,
|
|
1375
|
+
wanted_objects: 0,
|
|
1374
1376
|
origin_bytes: 0,
|
|
1375
1377
|
cache_bytes: 0,
|
|
1376
1378
|
origin_objects: 0,
|
|
@@ -1385,6 +1387,7 @@ impl PullProgress {
|
|
|
1385
1387
|
let mut state = self.state.lock().unwrap();
|
|
1386
1388
|
state.total_bytes = plan.declared_bytes();
|
|
1387
1389
|
state.held_bytes = plan.held_bytes();
|
|
1390
|
+
state.wanted_objects = plan.wanted.len() as u64;
|
|
1388
1391
|
state.last = Some(Instant::now());
|
|
1389
1392
|
print_progress(&state);
|
|
1390
1393
|
}
|
|
@@ -1447,11 +1450,12 @@ impl PullProgress {
|
|
|
1447
1450
|
///
|
|
1448
1451
|
/// Every counter is always present, including a zero. A consumer that reads an absent key
|
|
1449
1452
|
/// as zero cannot tell a cache that answered nothing from a build that never reported the
|
|
1450
|
-
/// door at all, and those are the two states worth telling apart.
|
|
1453
|
+
/// door at all, and those are the two states worth telling apart. `wanted_objects` is the
|
|
1454
|
+
/// plan's count of objects to land; a pod host sizes this pull's storage reservation from it.
|
|
1451
1455
|
fn print_progress(state: &PullProgressState) {
|
|
1452
1456
|
eprintln!(
|
|
1453
|
-
"{{\"event\":\"pull.progress\",\"total_bytes\":{},\"held_bytes\":{},\"origin_bytes\":{},\"cache_bytes\":{},\"origin_objects\":{},\"cache_objects\":{}}}",
|
|
1454
|
-
state.total_bytes, state.held_bytes, state.origin_bytes, state.cache_bytes,
|
|
1457
|
+
"{{\"event\":\"pull.progress\",\"total_bytes\":{},\"held_bytes\":{},\"wanted_objects\":{},\"origin_bytes\":{},\"cache_bytes\":{},\"origin_objects\":{},\"cache_objects\":{}}}",
|
|
1458
|
+
state.total_bytes, state.held_bytes, state.wanted_objects, state.origin_bytes, state.cache_bytes,
|
|
1455
1459
|
state.origin_objects, state.cache_objects
|
|
1456
1460
|
);
|
|
1457
1461
|
}
|
|
@@ -62,6 +62,11 @@ fn sql(what: impl AsRef<str>, error: rusqlite::Error) -> Refusal {
|
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
+
/// Whether a catalog refusal is SQLite's busy timeout expiring behind another writer.
|
|
66
|
+
fn busy(refusal: &Refusal) -> bool {
|
|
67
|
+
refusal.code == Code::IO_FAILED && refusal.detail.ends_with("database is locked")
|
|
68
|
+
}
|
|
69
|
+
|
|
65
70
|
fn io(what: impl AsRef<str>, error: std::io::Error) -> Refusal {
|
|
66
71
|
crate::store::classify_io(what, error)
|
|
67
72
|
}
|
|
@@ -115,7 +120,20 @@ fn connect(path: &Path) -> Result<Connection> {
|
|
|
115
120
|
.entry(path.to_path_buf())
|
|
116
121
|
.or_default() += 1;
|
|
117
122
|
crate::stats::catalog_opened();
|
|
118
|
-
let connection = Connection::open(path).map_err(|error|
|
|
123
|
+
let connection = Connection::open(path).map_err(|error| {
|
|
124
|
+
// SQLite reports a full descriptor table only as "unable to open"; the table
|
|
125
|
+
// itself answers whether that is what happened.
|
|
126
|
+
if crate::descriptors::available().is_some_and(|free| free == 0) {
|
|
127
|
+
crate::descriptors::exhausted(&std::io::Error::from_raw_os_error(
|
|
128
|
+
rustix::io::Errno::MFILE.raw_os_error(),
|
|
129
|
+
));
|
|
130
|
+
return Refusal {
|
|
131
|
+
code: Code::FD_HEADROOM,
|
|
132
|
+
detail: format!("open catalog: no file descriptor ({error})"),
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
sql("open catalog", error)
|
|
136
|
+
})?;
|
|
119
137
|
connection
|
|
120
138
|
.busy_timeout(BUSY_TIMEOUT)
|
|
121
139
|
.map_err(|error| sql("set catalog busy timeout", error))?;
|
|
@@ -372,6 +390,22 @@ impl Catalog {
|
|
|
372
390
|
Ok(catalog)
|
|
373
391
|
}
|
|
374
392
|
|
|
393
|
+
/// `open` for an object admission. A catalog busy past SQLite's timeout while this
|
|
394
|
+
/// process's own admission batches keep committing is a queue, not a failure: the
|
|
395
|
+
/// verified bytes wait for their turn instead of being thrown away and fetched again.
|
|
396
|
+
/// Busy with no batch committed in the meantime is another writer's lock, and refuses.
|
|
397
|
+
pub(crate) fn open_admitting(root: &Path) -> Result<Self> {
|
|
398
|
+
loop {
|
|
399
|
+
let committed = crate::stats::snapshot().admission_batches;
|
|
400
|
+
match Self::open(root) {
|
|
401
|
+
Err(refusal)
|
|
402
|
+
if busy(&refusal) && crate::stats::snapshot().admission_batches > committed => {
|
|
403
|
+
}
|
|
404
|
+
opened => return opened,
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
375
409
|
pub fn open(root: &Path) -> Result<Self> {
|
|
376
410
|
let path = Self::path(root);
|
|
377
411
|
validated(&path)?;
|
|
@@ -1473,16 +1507,7 @@ impl WriterGuard {
|
|
|
1473
1507
|
.read(true)
|
|
1474
1508
|
.write(true)
|
|
1475
1509
|
.open(&path)
|
|
1476
|
-
.map_err(|error|
|
|
1477
|
-
if crate::descriptors::exhausted(&error) {
|
|
1478
|
-
Refusal {
|
|
1479
|
-
code: Code::FD_HEADROOM,
|
|
1480
|
-
detail: format!("create writer marker: {error}"),
|
|
1481
|
-
}
|
|
1482
|
-
} else {
|
|
1483
|
-
io("create writer marker", error)
|
|
1484
|
-
}
|
|
1485
|
-
})?;
|
|
1510
|
+
.map_err(|error| io("create writer marker", error))?;
|
|
1486
1511
|
file.try_lock_exclusive()
|
|
1487
1512
|
.map_err(|error| io("lock writer marker", error))?;
|
|
1488
1513
|
Ok(Self {
|
|
@@ -239,6 +239,26 @@ fn profile_plan(plan: &SourcePlan) -> ProfilePlan {
|
|
|
239
239
|
}
|
|
240
240
|
}
|
|
241
241
|
|
|
242
|
+
/// The one reviewed profile these member heads plan under, chosen from their headers alone:
|
|
243
|
+
/// several matching profiles that plan identically are one answer; different plans refuse
|
|
244
|
+
/// `AMBIGUOUS_CLASSIFICATION` naming them, and none refuses `UNREGISTERED_FINGERPRINT`.
|
|
245
|
+
pub fn select_profile(
|
|
246
|
+
registry_locator: &str,
|
|
247
|
+
registry_bytes: &[u8],
|
|
248
|
+
heads: &[MemberHead],
|
|
249
|
+
scratch: &Path,
|
|
250
|
+
) -> Result<String> {
|
|
251
|
+
let staged = stage(scratch, heads)?;
|
|
252
|
+
source::plan_source(
|
|
253
|
+
registry_locator,
|
|
254
|
+
registry_bytes,
|
|
255
|
+
staged.carriers(),
|
|
256
|
+
None,
|
|
257
|
+
None,
|
|
258
|
+
)
|
|
259
|
+
.map(|plan| plan.profile)
|
|
260
|
+
}
|
|
261
|
+
|
|
242
262
|
/// Decide the conversion plan for every requested profile, from headers alone.
|
|
243
263
|
///
|
|
244
264
|
/// One [`source::plan_source`] call per profile, over one carrier set — which is precisely
|
|
@@ -690,12 +690,6 @@ fn plan_source_prepared(
|
|
|
690
690
|
}
|
|
691
691
|
for assignments in profile_assignments(profile, ®istry, carriers, &headers) {
|
|
692
692
|
matches.push((profile, assignments));
|
|
693
|
-
if matches.len() >= 2 {
|
|
694
|
-
break;
|
|
695
|
-
}
|
|
696
|
-
}
|
|
697
|
-
if matches.len() >= 2 {
|
|
698
|
-
break;
|
|
699
693
|
}
|
|
700
694
|
}
|
|
701
695
|
if matches.is_empty() {
|
|
@@ -704,20 +698,61 @@ fn plan_source_prepared(
|
|
|
704
698
|
"carrier set matches no reviewed source profile",
|
|
705
699
|
);
|
|
706
700
|
}
|
|
707
|
-
|
|
708
|
-
|
|
701
|
+
// Every match is planned. Matches whose plans are identical (the same sources, order,
|
|
702
|
+
// target and session under different profile names) are one answer, named by the most
|
|
703
|
+
// specific profile; only genuinely different plans are ambiguous.
|
|
704
|
+
let mut planned = Vec::with_capacity(matches.len());
|
|
705
|
+
let mut first_error = None;
|
|
706
|
+
for (profile, assignments) in matches {
|
|
707
|
+
match plan_match(
|
|
708
|
+
profile,
|
|
709
|
+
assignments,
|
|
710
|
+
carriers,
|
|
711
|
+
&set,
|
|
712
|
+
®istry,
|
|
713
|
+
registry_locator,
|
|
714
|
+
registry_bytes,
|
|
715
|
+
spec_variant,
|
|
716
|
+
) {
|
|
717
|
+
Ok(result) => planned.push((profile, result)),
|
|
718
|
+
Err(error) => {
|
|
719
|
+
first_error.get_or_insert(error);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
if planned.is_empty() {
|
|
724
|
+
return Err(first_error.expect("every match was planned or refused"));
|
|
725
|
+
}
|
|
726
|
+
let same = |left: &SourcePlan, right: &SourcePlan| {
|
|
727
|
+
(
|
|
728
|
+
&left.session,
|
|
729
|
+
&left.construction_order,
|
|
730
|
+
&left.sources,
|
|
731
|
+
&left.target,
|
|
732
|
+
) == (
|
|
733
|
+
&right.session,
|
|
734
|
+
&right.construction_order,
|
|
735
|
+
&right.sources,
|
|
736
|
+
&right.target,
|
|
737
|
+
)
|
|
738
|
+
};
|
|
739
|
+
if planned
|
|
740
|
+
.iter()
|
|
741
|
+
.any(|(_, (plan, _))| !same(plan, &planned[0].1 .0))
|
|
742
|
+
{
|
|
743
|
+
let mut profiles: Vec<&str> = planned
|
|
709
744
|
.iter()
|
|
710
745
|
.map(|(profile, _)| profile.name.as_str())
|
|
711
746
|
.collect();
|
|
712
|
-
|
|
747
|
+
profiles.dedup();
|
|
748
|
+
let mut members: Vec<String> = planned
|
|
713
749
|
.iter()
|
|
714
|
-
.flat_map(|(_,
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
.member
|
|
750
|
+
.flat_map(|(_, (plan, _))| {
|
|
751
|
+
plan.sources.iter().map(|source| {
|
|
752
|
+
source
|
|
753
|
+
.source_member
|
|
719
754
|
.clone()
|
|
720
|
-
.unwrap_or_else(||
|
|
755
|
+
.unwrap_or_else(|| source.path.display().to_string())
|
|
721
756
|
})
|
|
722
757
|
})
|
|
723
758
|
.collect();
|
|
@@ -731,7 +766,42 @@ fn plan_source_prepared(
|
|
|
731
766
|
),
|
|
732
767
|
);
|
|
733
768
|
}
|
|
734
|
-
|
|
769
|
+
planned.sort_by_key(|(profile, _)| {
|
|
770
|
+
(
|
|
771
|
+
std::cmp::Reverse(specificity(profile)),
|
|
772
|
+
profile.name.clone(),
|
|
773
|
+
)
|
|
774
|
+
});
|
|
775
|
+
let (_, (plan, prepared)) = planned.swap_remove(0);
|
|
776
|
+
Ok((plan, prepared))
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
/// How exactly a profile binds its components: provider members, then member prefixes.
|
|
780
|
+
fn specificity(profile: &SourceProfile) -> (usize, usize) {
|
|
781
|
+
let exact = profile
|
|
782
|
+
.components
|
|
783
|
+
.iter()
|
|
784
|
+
.filter(|component| component.source_member.is_some())
|
|
785
|
+
.count();
|
|
786
|
+
let prefixed = profile
|
|
787
|
+
.components
|
|
788
|
+
.iter()
|
|
789
|
+
.filter(|component| component.source_member_prefix.is_some())
|
|
790
|
+
.count();
|
|
791
|
+
(exact, prefixed)
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
#[allow(clippy::too_many_arguments)]
|
|
795
|
+
fn plan_match(
|
|
796
|
+
profile: &SourceProfile,
|
|
797
|
+
mut assignments: Vec<SourceAssignment>,
|
|
798
|
+
carriers: &[CarrierInput],
|
|
799
|
+
set: &CarrierSet,
|
|
800
|
+
registry: &FingerprintRegistry,
|
|
801
|
+
registry_locator: &str,
|
|
802
|
+
registry_bytes: &[u8],
|
|
803
|
+
spec_variant: Option<usize>,
|
|
804
|
+
) -> Result<(SourcePlan, Prepared)> {
|
|
735
805
|
assignments.sort_by(|left, right| left.component.cmp(&right.component));
|
|
736
806
|
let sources: Vec<(String, CarrierInput)> = assignments
|
|
737
807
|
.iter()
|
|
@@ -762,7 +832,7 @@ fn plan_source_prepared(
|
|
|
762
832
|
.map(|key| (component.component.clone(), key.clone()))
|
|
763
833
|
})
|
|
764
834
|
.collect();
|
|
765
|
-
let mut prepared = prepare_with_registry(&target, &sources,
|
|
835
|
+
let mut prepared = prepare_with_registry(&target, &sources, set, registry, spec_variant)?;
|
|
766
836
|
prepared.plan.apply_order(&order)?;
|
|
767
837
|
prepared.plan.check_converts_something()?;
|
|
768
838
|
let plan = SourcePlan {
|
|
@@ -158,6 +158,14 @@ const EDQUOT: i32 = 69;
|
|
|
158
158
|
/// the owner re-issued the fetch, which re-planned, re-asked, and failed at the same byte.
|
|
159
159
|
/// On rented hardware, forever. Naming the condition is what lets a caller stop.
|
|
160
160
|
pub(crate) fn classify_io(what: impl AsRef<str>, e: std::io::Error) -> Refusal {
|
|
161
|
+
// A full descriptor table is the process's shortage, not the Store's: every open in
|
|
162
|
+
// the Store says so the same way, and a transfer treats it as backpressure.
|
|
163
|
+
if crate::descriptors::exhausted(&e) {
|
|
164
|
+
return Refusal {
|
|
165
|
+
code: Code::FD_HEADROOM,
|
|
166
|
+
detail: format!("{}: no file descriptor ({e})", what.as_ref()),
|
|
167
|
+
};
|
|
168
|
+
}
|
|
161
169
|
if matches!(e.raw_os_error(), Some(ENOSPC) | Some(EDQUOT)) {
|
|
162
170
|
return Refusal {
|
|
163
171
|
code: Code::CAPACITY_EXHAUSTED,
|
|
@@ -1362,16 +1370,7 @@ impl Store {
|
|
|
1362
1370
|
.create_new(true)
|
|
1363
1371
|
.mode(0o600)
|
|
1364
1372
|
.open(&tmp)
|
|
1365
|
-
.map_err(|e| {
|
|
1366
|
-
if crate::descriptors::exhausted(&e) {
|
|
1367
|
-
Refusal {
|
|
1368
|
-
code: Code::FD_HEADROOM,
|
|
1369
|
-
detail: format!("create temp {}: {e}", tmp.display()),
|
|
1370
|
-
}
|
|
1371
|
-
} else {
|
|
1372
|
-
io(format!("create temp {}", tmp.display()), e)
|
|
1373
|
-
}
|
|
1374
|
-
})?;
|
|
1373
|
+
.map_err(|e| io(format!("create temp {}", tmp.display()), e))?;
|
|
1375
1374
|
match f.try_lock_exclusive() {
|
|
1376
1375
|
Ok(()) => break (tmp, f),
|
|
1377
1376
|
// Only a reaper can hold the lock on a name this process just minted: it
|
|
@@ -1473,7 +1472,10 @@ impl Store {
|
|
|
1473
1472
|
fault: &Fault,
|
|
1474
1473
|
operation: Option<&str>,
|
|
1475
1474
|
) -> Result<Put> {
|
|
1476
|
-
let catalog = crate::catalog::Catalog::
|
|
1475
|
+
let catalog = match crate::catalog::Catalog::open_admitting(&self.root) {
|
|
1476
|
+
Ok(catalog) => catalog,
|
|
1477
|
+
Err(refusal) => return cleanup(&verified.tmp, refusal),
|
|
1478
|
+
};
|
|
1477
1479
|
let _writer = crate::catalog::WriterGuard::acquire(&self.root)?;
|
|
1478
1480
|
let object = ObjectRef {
|
|
1479
1481
|
sha256: verified.sha256.clone(),
|
|
@@ -2876,9 +2876,76 @@ fn paced_objects(size: usize, count: usize) -> (Vec<ObjectRef>, Bodies) {
|
|
|
2876
2876
|
(declared, Arc::new(serve))
|
|
2877
2877
|
}
|
|
2878
2878
|
|
|
2879
|
-
///
|
|
2880
|
-
|
|
2881
|
-
|
|
2879
|
+
/// How many GETs the origin itself was serving at once, counted where it serves them.
|
|
2880
|
+
#[derive(Default)]
|
|
2881
|
+
struct Concurrency {
|
|
2882
|
+
now: AtomicUsize,
|
|
2883
|
+
peak: AtomicUsize,
|
|
2884
|
+
}
|
|
2885
|
+
|
|
2886
|
+
impl Concurrency {
|
|
2887
|
+
fn enter(&self) {
|
|
2888
|
+
let now = self.now.fetch_add(1, Ordering::AcqRel) + 1;
|
|
2889
|
+
self.peak.fetch_max(now, Ordering::AcqRel);
|
|
2890
|
+
}
|
|
2891
|
+
fn leave(&self) {
|
|
2892
|
+
self.now.fetch_sub(1, Ordering::AcqRel);
|
|
2893
|
+
}
|
|
2894
|
+
fn reset(&self) -> usize {
|
|
2895
|
+
self.peak.swap(0, Ordering::AcqRel)
|
|
2896
|
+
}
|
|
2897
|
+
}
|
|
2898
|
+
|
|
2899
|
+
/// An origin that paces every body at `chunk` per `gap`. With `shared`, all connections
|
|
2900
|
+
/// draw from one schedule (the link is the limit); without, each has its own (one GET is
|
|
2901
|
+
/// the limit). It counts the GETs in flight on its own side of the socket.
|
|
2902
|
+
fn paced_origin(
|
|
2903
|
+
body: Bodies,
|
|
2904
|
+
chunk: usize,
|
|
2905
|
+
gap: Duration,
|
|
2906
|
+
shared: bool,
|
|
2907
|
+
) -> (Origin, Arc<Concurrency>) {
|
|
2908
|
+
let serving = Arc::new(Concurrency::default());
|
|
2909
|
+
let link = Arc::new(Mutex::new(std::time::Instant::now()));
|
|
2910
|
+
let counted = Arc::clone(&serving);
|
|
2911
|
+
let origin = origin(Arc::new(move |request: &Request| {
|
|
2912
|
+
let body = body(request.path.trim_start_matches("/obj/"));
|
|
2913
|
+
let (link, serving) = (Arc::clone(&link), Arc::clone(&counted));
|
|
2914
|
+
Serve::Script(Box::new(move |socket: &mut TcpStream| {
|
|
2915
|
+
serving.enter();
|
|
2916
|
+
let head = format!(
|
|
2917
|
+
"HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n",
|
|
2918
|
+
body.len()
|
|
2919
|
+
);
|
|
2920
|
+
let _ = socket.write_all(head.as_bytes());
|
|
2921
|
+
let mut own = std::time::Instant::now();
|
|
2922
|
+
for piece in body.chunks(chunk) {
|
|
2923
|
+
let due = if shared {
|
|
2924
|
+
let mut next = link.lock().unwrap();
|
|
2925
|
+
*next = (*next).max(std::time::Instant::now()) + gap;
|
|
2926
|
+
*next
|
|
2927
|
+
} else {
|
|
2928
|
+
own += gap;
|
|
2929
|
+
own
|
|
2930
|
+
};
|
|
2931
|
+
let now = std::time::Instant::now();
|
|
2932
|
+
if due > now {
|
|
2933
|
+
std::thread::sleep(due - now);
|
|
2934
|
+
}
|
|
2935
|
+
if socket.write_all(piece).is_err() {
|
|
2936
|
+
break;
|
|
2937
|
+
}
|
|
2938
|
+
}
|
|
2939
|
+
serving.leave();
|
|
2940
|
+
}))
|
|
2941
|
+
}));
|
|
2942
|
+
(origin, serving)
|
|
2943
|
+
}
|
|
2944
|
+
|
|
2945
|
+
/// Walk `subset` from `origin` into a fresh store and prove every object local and
|
|
2946
|
+
/// verified. The ledger samples each second: its stall floor (six samples) is far above
|
|
2947
|
+
/// any pause a loaded test box puts between two chunks, and nothing here is timed.
|
|
2948
|
+
fn paced_walk(origin: &Origin, streams: usize, subset: &[ObjectRef]) -> super::WalkOutcome {
|
|
2882
2949
|
let root = temporary(&format!("paced-{streams}-{}", subset.len()));
|
|
2883
2950
|
let store = Store::init(&root).unwrap();
|
|
2884
2951
|
let (plan, _) = FetchPlan::of_objects(&store, "paced", subset).unwrap();
|
|
@@ -2891,8 +2958,6 @@ fn paced_walk(origin: &Origin, streams: usize, subset: &[ObjectRef]) -> (f64, su
|
|
|
2891
2958
|
)
|
|
2892
2959
|
})
|
|
2893
2960
|
.collect();
|
|
2894
|
-
let ledger = Ledger::with_resolution(Duration::from_millis(200));
|
|
2895
|
-
let started = std::time::Instant::now();
|
|
2896
2961
|
let walked = super::pull::fetch_wanted(
|
|
2897
2962
|
&store,
|
|
2898
2963
|
&plan,
|
|
@@ -2900,12 +2965,11 @@ fn paced_walk(origin: &Origin, streams: usize, subset: &[ObjectRef]) -> (f64, su
|
|
|
2900
2965
|
&local_policy(),
|
|
2901
2966
|
&Anonymous,
|
|
2902
2967
|
Deadline::none(),
|
|
2903
|
-
&
|
|
2968
|
+
&Ledger::with_resolution(Duration::from_secs(1)),
|
|
2904
2969
|
streams,
|
|
2905
2970
|
None,
|
|
2906
2971
|
)
|
|
2907
2972
|
.unwrap();
|
|
2908
|
-
let seconds = started.elapsed().as_secs_f64();
|
|
2909
2973
|
plan.complete(&store).unwrap();
|
|
2910
2974
|
for object in subset {
|
|
2911
2975
|
let mut bytes = Vec::new();
|
|
@@ -2922,109 +2986,78 @@ fn paced_walk(origin: &Origin, streams: usize, subset: &[ObjectRef]) -> (f64, su
|
|
|
2922
2986
|
}
|
|
2923
2987
|
let _ = std::fs::remove_dir_all(root);
|
|
2924
2988
|
assert_eq!(walked.fetched as usize, subset.len());
|
|
2925
|
-
|
|
2989
|
+
let declared: u64 = subset.iter().map(|o| o.length).sum();
|
|
2990
|
+
assert_eq!(walked.bytes_moved, declared, "every declared byte, once");
|
|
2991
|
+
walked
|
|
2926
2992
|
}
|
|
2927
2993
|
|
|
2928
|
-
/// Run 1225's shape: one GET is the limit, not the link. Each connection
|
|
2929
|
-
///
|
|
2930
|
-
///
|
|
2931
|
-
///
|
|
2994
|
+
/// Run 1225's shape: one GET is the limit, not the link. Each connection is paced at
|
|
2995
|
+
/// ~0.2 MB/s on its own schedule, so every doubling of streams doubles what the origin
|
|
2996
|
+
/// can hand over. The walk must climb to its ceiling, and the ORIGIN must have served
|
|
2997
|
+
/// that many GETs at once. Nothing here asserts a rate or a duration.
|
|
2932
2998
|
#[test]
|
|
2933
2999
|
fn a_walk_grows_past_sixteen_while_one_get_is_the_limit() {
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
let gap = Duration::from_millis(20);
|
|
2937
|
-
let (declared, body) = paced_objects(SIZE, 384);
|
|
2938
|
-
let origin = origin(Arc::new(move |request: &Request| Serve::Drip {
|
|
2939
|
-
body: body(request.path.trim_start_matches("/obj/")),
|
|
2940
|
-
chunk: CHUNK,
|
|
2941
|
-
gap,
|
|
2942
|
-
ttfb: Duration::from_millis(20),
|
|
2943
|
-
}));
|
|
2944
|
-
let one_get = CHUNK as f64 / gap.as_secs_f64() / 1e6;
|
|
3000
|
+
let (declared, body) = paced_objects(1 << 20, 320);
|
|
3001
|
+
let (origin, serving) = paced_origin(body, 4 << 10, Duration::from_millis(20), false);
|
|
2945
3002
|
|
|
2946
|
-
let
|
|
2947
|
-
let (paced, grew) = paced_walk(&origin, super::PULL_STREAMS, &declared);
|
|
2948
|
-
println!(
|
|
2949
|
-
"one GET {one_get:.2} MB/s; held at 16: {held:.1} MB/s; paced: {paced:.1} MB/s, \
|
|
2950
|
-
ended at {} streams via {:?}",
|
|
2951
|
-
grew.streams, grew.levels
|
|
2952
|
-
);
|
|
2953
|
-
assert_eq!(fixed.streams, 16);
|
|
2954
|
-
assert!(fixed.levels.is_empty(), "an explicit 16 is a fixed walk");
|
|
3003
|
+
let fixed = paced_walk(&origin, 16, &declared[..32]);
|
|
2955
3004
|
assert!(
|
|
2956
|
-
|
|
2957
|
-
"
|
|
3005
|
+
fixed.streams == 16 && fixed.levels.is_empty(),
|
|
3006
|
+
"an explicit 16 is a fixed walk: {fixed:?}"
|
|
2958
3007
|
);
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
3008
|
+
// The origin counts a GET until its handler returns, which can overlap the next GET
|
|
3009
|
+
// by a scheduling instant; the fixed walk's peak is reported, not asserted.
|
|
3010
|
+
let held = serving.reset();
|
|
3011
|
+
let grew = paced_walk(&origin, super::PULL_STREAMS, &declared[32..]);
|
|
3012
|
+
let peak = serving.reset();
|
|
3013
|
+
println!(
|
|
3014
|
+
"fixed peak {held}; paced peak {peak}, ended at {} via {:?}",
|
|
3015
|
+
grew.streams, grew.levels
|
|
2962
3016
|
);
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
last.bytes_per_second >= 3.0 * first.bytes_per_second,
|
|
2968
|
-
"the top level moved {:.1} MB/s against {:.1} at 16",
|
|
2969
|
-
last.bytes_per_second / 1e6,
|
|
2970
|
-
first.bytes_per_second / 1e6
|
|
3017
|
+
assert_eq!(
|
|
3018
|
+
grew.streams,
|
|
3019
|
+
super::PULL_STREAMS,
|
|
3020
|
+
"growth stopped early: {grew:?}"
|
|
2971
3021
|
);
|
|
2972
3022
|
assert!(
|
|
2973
|
-
|
|
2974
|
-
"
|
|
3023
|
+
peak > 64,
|
|
3024
|
+
"the origin never served more than {peak} GETs at once: {grew:?}"
|
|
2975
3025
|
);
|
|
3026
|
+
for pair in grew.levels.windows(2) {
|
|
3027
|
+
assert_eq!(
|
|
3028
|
+
pair[1].streams,
|
|
3029
|
+
2 * pair[0].streams,
|
|
3030
|
+
"levels double: {grew:?}"
|
|
3031
|
+
);
|
|
3032
|
+
}
|
|
2976
3033
|
}
|
|
2977
3034
|
|
|
2978
|
-
/// The other half of the rule: when the LINK is the limit, a doubling buys nothing and
|
|
2979
|
-
/// walk stops growing instead of piling up partial objects. Every connection shares
|
|
2980
|
-
///
|
|
3035
|
+
/// The other half of the rule: when the LINK is the limit, a doubling buys nothing and
|
|
3036
|
+
/// the walk stops growing instead of piling up partial objects. Every connection shares
|
|
3037
|
+
/// one schedule here, which a handful of streams already fill.
|
|
2981
3038
|
#[test]
|
|
2982
3039
|
fn a_walk_does_not_grow_when_the_link_is_the_limit() {
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
let
|
|
2986
|
-
let
|
|
2987
|
-
let link = Arc::new(Mutex::new(std::time::Instant::now()));
|
|
2988
|
-
let origin = origin(Arc::new(move |request: &Request| {
|
|
2989
|
-
let body = body(request.path.trim_start_matches("/obj/"));
|
|
2990
|
-
let link = Arc::clone(&link);
|
|
2991
|
-
Serve::Script(Box::new(move |socket: &mut TcpStream| {
|
|
2992
|
-
let head = format!(
|
|
2993
|
-
"HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n",
|
|
2994
|
-
body.len()
|
|
2995
|
-
);
|
|
2996
|
-
let _ = socket.write_all(head.as_bytes());
|
|
2997
|
-
for piece in body.chunks(CHUNK) {
|
|
2998
|
-
let due = {
|
|
2999
|
-
let mut next = link.lock().unwrap();
|
|
3000
|
-
let now = std::time::Instant::now();
|
|
3001
|
-
*next = (*next).max(now) + slot;
|
|
3002
|
-
*next
|
|
3003
|
-
};
|
|
3004
|
-
let now = std::time::Instant::now();
|
|
3005
|
-
if due > now {
|
|
3006
|
-
std::thread::sleep(due - now);
|
|
3007
|
-
}
|
|
3008
|
-
if socket.write_all(piece).is_err() {
|
|
3009
|
-
return;
|
|
3010
|
-
}
|
|
3011
|
-
}
|
|
3012
|
-
}))
|
|
3013
|
-
}));
|
|
3014
|
-
let (held, _) = paced_walk(&origin, 16, &declared);
|
|
3015
|
-
let (rate, walked) = paced_walk(&origin, super::PULL_STREAMS, &declared);
|
|
3040
|
+
let (declared, body) = paced_objects(512 << 10, 160);
|
|
3041
|
+
let (origin, serving) = paced_origin(body, 16 << 10, Duration::from_micros(1250), true);
|
|
3042
|
+
let walked = paced_walk(&origin, super::PULL_STREAMS, &declared);
|
|
3043
|
+
let peak = serving.reset();
|
|
3016
3044
|
println!(
|
|
3017
|
-
"shared link:
|
|
3045
|
+
"shared link: peak {peak}, ended at {} via {:?}",
|
|
3018
3046
|
walked.streams, walked.levels
|
|
3019
3047
|
);
|
|
3020
3048
|
assert!(
|
|
3021
|
-
walked.streams
|
|
3022
|
-
"a full link
|
|
3049
|
+
walked.streams < super::PULL_STREAMS,
|
|
3050
|
+
"a full link grew the walk to its ceiling: peak {peak}, {walked:?}"
|
|
3023
3051
|
);
|
|
3024
|
-
|
|
3052
|
+
// It stopped because a doubling stopped paying, as it measured it.
|
|
3053
|
+
let last = walked
|
|
3054
|
+
.levels
|
|
3055
|
+
.windows(2)
|
|
3056
|
+
.last()
|
|
3057
|
+
.expect("at least one probe past 16");
|
|
3025
3058
|
assert!(
|
|
3026
|
-
|
|
3027
|
-
"the
|
|
3059
|
+
last[1].bytes_per_second < super::GAIN * last[0].bytes_per_second,
|
|
3060
|
+
"the walk stopped without a doubling that failed to pay: {walked:?}"
|
|
3028
3061
|
);
|
|
3029
3062
|
}
|
|
3030
3063
|
|
|
@@ -164,3 +164,14 @@ fn anima_hf_refuses_a_missing_component_before_payload_transfer() {
|
|
|
164
164
|
assert_ne!(error.code, Code::IO_FAILED, "{error:?}");
|
|
165
165
|
assert!(!path.exists());
|
|
166
166
|
}
|
|
167
|
+
|
|
168
|
+
#[test]
|
|
169
|
+
fn anima_hf_selects_its_profile_without_being_named() {
|
|
170
|
+
let path = scratch("select");
|
|
171
|
+
assert_eq!(
|
|
172
|
+
preflight::select_profile(BUILTIN_REGISTRY, BUILTIN_REGISTRY_BYTES, &heads(), &path)
|
|
173
|
+
.unwrap(),
|
|
174
|
+
PROFILE
|
|
175
|
+
);
|
|
176
|
+
assert!(!path.exists());
|
|
177
|
+
}
|