tensorfs 0.3.28__tar.gz → 0.3.30__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. {tensorfs-0.3.28 → tensorfs-0.3.30}/Cargo.lock +2 -2
  2. {tensorfs-0.3.28 → tensorfs-0.3.30}/Cargo.toml +1 -1
  3. {tensorfs-0.3.28 → tensorfs-0.3.30}/PKG-INFO +1 -1
  4. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/bin/tfs/fetch.rs +49 -6
  5. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/bin/tfs/main.rs +12 -1
  6. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/transport/api.rs +7 -1
  7. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/transport/http.rs +166 -16
  8. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/transport/hub.rs +16 -1
  9. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/transport/ledger.rs +104 -0
  10. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/transport/pull.rs +46 -1
  11. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/transport/push.rs +4 -1
  12. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/transport/tests.rs +261 -1
  13. {tensorfs-0.3.28 → tensorfs-0.3.30}/pyproject.toml +1 -1
  14. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/Cargo.toml +0 -0
  15. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/build.rs +0 -0
  16. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/examples/h3-swiglu-layout.md +0 -0
  17. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/examples/optional-cache-backfill.md +0 -0
  18. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/examples/resident-conversion.md +0 -0
  19. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/examples/resident_conversion_profile.rs +0 -0
  20. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/b64.rs +0 -0
  21. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/bin/tfs/ckpt.rs +0 -0
  22. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/bin/tfs/conform.rs +0 -0
  23. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/bin/tfs/ingest.rs +0 -0
  24. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/bin/tfs/rd.rs +0 -0
  25. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/bin/tfs/repo.rs +0 -0
  26. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/bin/tfs/snap.rs +0 -0
  27. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/bin/tfs/watch.rs +0 -0
  28. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/canon.rs +0 -0
  29. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/capability.rs +0 -0
  30. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/catalog.rs +0 -0
  31. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/cbor.rs +0 -0
  32. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/checkpoint.rs +0 -0
  33. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/corpus.rs +0 -0
  34. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/derived/adoption.rs +0 -0
  35. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/derived/graft_tests.rs +0 -0
  36. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/derived/progress.rs +0 -0
  37. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/derived/resume_tests.rs +0 -0
  38. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/derived/retention.rs +0 -0
  39. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/derived/roots.rs +0 -0
  40. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/derived.rs +0 -0
  41. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/dtype.rs +0 -0
  42. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/durability.rs +0 -0
  43. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/err.rs +0 -0
  44. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/fetch.rs +0 -0
  45. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/fill.rs +0 -0
  46. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/fit.rs +0 -0
  47. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/gc.rs +0 -0
  48. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/header.rs +0 -0
  49. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/home.rs +0 -0
  50. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/ids.rs +0 -0
  51. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/ingest/carrier.rs +0 -0
  52. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/ingest/convert.rs +0 -0
  53. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/ingest/fingerprint.rs +0 -0
  54. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/ingest/gguf.rs +0 -0
  55. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/ingest/journal.rs +0 -0
  56. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/ingest/mod.rs +0 -0
  57. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/ingest/preflight.rs +0 -0
  58. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/ingest/source.rs +0 -0
  59. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/ingest/stamp.rs +0 -0
  60. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/ingest/transaction.rs +0 -0
  61. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/jcs.rs +0 -0
  62. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/lib.rs +0 -0
  63. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/limits.rs +0 -0
  64. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/machine.rs +0 -0
  65. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/manifest.rs +0 -0
  66. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/meta.rs +0 -0
  67. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/project.rs +0 -0
  68. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/providers.rs +0 -0
  69. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/read.rs +0 -0
  70. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/receipt.rs +0 -0
  71. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/reclaim.rs +0 -0
  72. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/registry.rs +0 -0
  73. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/relation.rs +0 -0
  74. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/repo_cache.rs +0 -0
  75. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/repository.rs +0 -0
  76. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/sha256.rs +0 -0
  77. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/source_artifact.rs +0 -0
  78. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/spec.rs +0 -0
  79. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/staging.rs +0 -0
  80. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/storage.rs +0 -0
  81. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/store.rs +0 -0
  82. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/transport/mod.rs +0 -0
  83. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/transport/policy.rs +0 -0
  84. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/transport/prefix.rs +0 -0
  85. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/transport/ranged.rs +0 -0
  86. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/src/vectors.rs +0 -0
  87. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/fetch_url_progress.rs +0 -0
  88. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/fit.rs +0 -0
  89. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/fixtures/repo_apply/blobs.jsonl +0 -0
  90. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_list.jsonl +0 -0
  91. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_repository.json +0 -0
  92. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_rows.jsonl +0 -0
  93. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/fixtures/repo_apply/header.cbor +0 -0
  94. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/fixtures/repo_apply/manifest.json +0 -0
  95. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/fixtures/repo_apply/mutation.json +0 -0
  96. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/h3_swiglu.rs +0 -0
  97. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/home.rs +0 -0
  98. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/import_tree.rs +0 -0
  99. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/instrument_profiles.rs +0 -0
  100. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/liveness.rs +0 -0
  101. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/read_plan.rs +0 -0
  102. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/reclaim.rs +0 -0
  103. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/reclaim_run.rs +0 -0
  104. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/repo_cache.rs +0 -0
  105. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/repo_store.rs +0 -0
  106. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/resident_admission.rs +0 -0
  107. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/sha256_backends.rs +0 -0
  108. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/sharded_cas.rs +0 -0
  109. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/source_artifacts.rs +0 -0
  110. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/source_plan.rs +0 -0
  111. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/source_prefixes.rs +0 -0
  112. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/source_preflight.rs +0 -0
  113. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/staging.rs +0 -0
  114. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/store_boundaries.rs +0 -0
  115. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/tensor_schema.rs +0 -0
  116. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/version.rs +0 -0
  117. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-core/tests/write_through.rs +0 -0
  118. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-py/Cargo.toml +0 -0
  119. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-py/src/docs.rs +0 -0
  120. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-py/src/errors.rs +0 -0
  121. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-py/src/lib.rs +0 -0
  122. {tensorfs-0.3.28 → tensorfs-0.3.30}/crates/tensorfs-py/src/store.rs +0 -0
  123. {tensorfs-0.3.28 → tensorfs-0.3.30}/python/tensorfs/__init__.py +0 -0
  124. {tensorfs-0.3.28 → tensorfs-0.3.30}/python/tensorfs/_ext.pyi +0 -0
  125. {tensorfs-0.3.28 → tensorfs-0.3.30}/python/tensorfs/errors.py +0 -0
  126. {tensorfs-0.3.28 → tensorfs-0.3.30}/python/tensorfs/py.typed +0 -0
@@ -392,7 +392,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
392
392
 
393
393
  [[package]]
394
394
  name = "tensorfs-core"
395
- version = "0.3.28"
395
+ version = "0.3.30"
396
396
  dependencies = [
397
397
  "fs2",
398
398
  "minicbor",
@@ -404,7 +404,7 @@ dependencies = [
404
404
 
405
405
  [[package]]
406
406
  name = "tensorfs-py"
407
- version = "0.3.28"
407
+ version = "0.3.30"
408
408
  dependencies = [
409
409
  "pyo3",
410
410
  "tensorfs-core",
@@ -3,5 +3,5 @@ resolver = "2"
3
3
  members = ["crates/tensorfs-core", "crates/tensorfs-py"]
4
4
 
5
5
  [workspace.package]
6
- version = "0.3.28"
6
+ version = "0.3.30"
7
7
  edition = "2021"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tensorfs
3
- Version: 0.3.28
3
+ Version: 0.3.30
4
4
  Classifier: Programming Language :: Python :: 3
5
5
  Classifier: Programming Language :: Python :: Implementation :: CPython
6
6
  Classifier: Programming Language :: Rust
@@ -228,6 +228,34 @@ fn credential_of(
228
228
  /// unbound Store is a pull with no cache, and a cache that is missing, cold, corrupt,
229
229
  /// read-only or unmounted degrades to exactly that pull.
230
230
  pub fn cmd_pull(root: &Path, refspec: &str, flags: &Flags) -> ExitCode {
231
+ pull_with_budget(root, refspec, flags, None)
232
+ }
233
+
234
+ /// A distinct verb makes an older CLI refuse instead of ignoring new quota flags.
235
+ pub fn cmd_bounded_pull(root: &Path, refspec: &str, flags: &Flags) -> ExitCode {
236
+ let bound = |name: &str| {
237
+ let values = crate::flag_all(flags, name);
238
+ if values.len() != 1 {
239
+ return None;
240
+ }
241
+ values[0].parse::<u64>().ok()
242
+ };
243
+ let (Some(bytes), Some(objects)) = (bound("max-download-bytes"), bound("max-download-objects"))
244
+ else {
245
+ eprintln!(
246
+ "bounded fetch requires one unsigned --max-download-bytes and --max-download-objects"
247
+ );
248
+ return ExitCode::from(2);
249
+ };
250
+ pull_with_budget(root, refspec, flags, Some((bytes, objects)))
251
+ }
252
+
253
+ fn pull_with_budget(
254
+ root: &Path,
255
+ refspec: &str,
256
+ flags: &Flags,
257
+ budget: Option<(u64, u64)>,
258
+ ) -> ExitCode {
231
259
  let store = ok!(Store::open(root));
232
260
  let Some(base) = flag(flags, "hub") else {
233
261
  eprintln!("--hub <base-url> is required");
@@ -247,6 +275,7 @@ pub fn cmd_pull(root: &Path, refspec: &str, flags: &Flags) -> ExitCode {
247
275
  request.session = flag(flags, "session").unwrap_or("");
248
276
  request.streams = crate::flag_num(flags, "streams", transport::STREAMS);
249
277
  request.deadline = deadline_of(flags);
278
+ request.download_budget = budget;
250
279
  // THE WALK REPORTS WHILE IT WALKS.
251
280
  //
252
281
  // Until now `tfs fetch` printed one report at the end, the pod's supervisor buffered
@@ -258,17 +287,18 @@ pub fn cmd_pull(root: &Path, refspec: &str, flags: &Flags) -> ExitCode {
258
287
  // What is printed is what was MEASURED — bytes through each door and the instant they
259
288
  // landed — and nothing derived. A rate is the consumer's division of two of these
260
289
  // samples, which is the only place it can be computed honestly: this walk has sixteen
261
- // streams and no opinion about what interval anyone wants to average over. There is no
262
- // denominator here on purpose. `FetchPlan` knows it and `pull()` holds it, but a
263
- // caller that must not invent a percentage would rather be given none than one this
264
- // side guessed.
290
+ // streams and no opinion about what interval anyone wants to average over. The
291
+ // denominator is the native FetchPlan's declared byte count. The plan callback also
292
+ // supplies already-held bytes, so a resumed or warm pull starts at its real position.
265
293
  //
266
294
  // The two doors are separate counters because they cost different things and only one
267
295
  // of them is an invoice. A cache that has silently gone cold produces a complete
268
296
  // closure, correct totals, and a bill — and looks exactly like a warm one unless the
269
297
  // split is reported while it happens.
270
298
  let progress = PullProgress::new(request.sample_seconds);
299
+ let planned = |plan: &FetchPlan| progress.planned(plan);
271
300
  let observe = |_object: &ObjectRef, moved: u64, door: ObjectSource| progress.saw(moved, door);
301
+ request.on_plan = Some(&planned);
272
302
  request.on_object = Some(&observe);
273
303
  let report = ok!(transport::pull(&request));
274
304
  progress.settle();
@@ -1282,6 +1312,8 @@ struct PullProgress {
1282
1312
  }
1283
1313
 
1284
1314
  struct PullProgressState {
1315
+ total_bytes: u64,
1316
+ held_bytes: u64,
1285
1317
  origin_bytes: u64,
1286
1318
  cache_bytes: u64,
1287
1319
  origin_objects: u64,
@@ -1293,6 +1325,8 @@ impl PullProgress {
1293
1325
  fn new(sample_seconds: f64) -> PullProgress {
1294
1326
  PullProgress {
1295
1327
  state: Mutex::new(PullProgressState {
1328
+ total_bytes: 0,
1329
+ held_bytes: 0,
1296
1330
  origin_bytes: 0,
1297
1331
  cache_bytes: 0,
1298
1332
  origin_objects: 0,
@@ -1303,6 +1337,14 @@ impl PullProgress {
1303
1337
  }
1304
1338
  }
1305
1339
 
1340
+ fn planned(&self, plan: &FetchPlan) {
1341
+ let mut state = self.state.lock().unwrap();
1342
+ state.total_bytes = plan.declared_bytes();
1343
+ state.held_bytes = plan.held_bytes();
1344
+ state.last = Some(Instant::now());
1345
+ print_progress(&state);
1346
+ }
1347
+
1306
1348
  fn saw(&self, moved: u64, door: ObjectSource) {
1307
1349
  let mut state = self.state.lock().unwrap();
1308
1350
  match door {
@@ -1364,7 +1406,8 @@ impl PullProgress {
1364
1406
  /// door at all, and those are the two states worth telling apart.
1365
1407
  fn print_progress(state: &PullProgressState) {
1366
1408
  eprintln!(
1367
- "{{\"event\":\"pull.progress\",\"origin_bytes\":{},\"cache_bytes\":{},\"origin_objects\":{},\"cache_objects\":{}}}",
1368
- state.origin_bytes, state.cache_bytes, state.origin_objects, state.cache_objects
1409
+ "{{\"event\":\"pull.progress\",\"total_bytes\":{},\"held_bytes\":{},\"origin_bytes\":{},\"cache_bytes\":{},\"origin_objects\":{},\"cache_objects\":{}}}",
1410
+ state.total_bytes, state.held_bytes, state.origin_bytes, state.cache_bytes,
1411
+ state.origin_objects, state.cache_objects
1369
1412
  );
1370
1413
  }
@@ -47,6 +47,8 @@
47
47
  //! and the origin. It is bound by `tfs store ensure --repo-cache` and never by a flag here.
48
48
  //!
49
49
  //! tfs fetch <root> <ref> closure -> plan -> N-way walk (presign at use) -> proof
50
+ //! tfs fetch bounded <root> <ref> --max-download-bytes N --max-download-objects N
51
+ //! same pull, refused before body transfer if new objects exceed bounds
50
52
  //! tfs fetch plan-objects <root> <session> --refs <jsonl> HELD/WANTED, no manifest
51
53
  //! tfs fetch url <root> <sha256> <length> <url|-> [--progress] [--streams N]
52
54
  //! one granted object, in parallel ranges
@@ -351,6 +353,7 @@ fn main() -> ExitCode {
351
353
  ["fetch", "plan-objects", r, s] => fetch::cmd_plan_objects(Path::new(r), s, &flags),
352
354
  ["fetch", "admit", r, p, i] => fetch::cmd_admit(Path::new(r), Path::new(p), i, &flags),
353
355
  ["fetch", "complete", r, p] => fetch::cmd_complete(Path::new(r), Path::new(p)),
356
+ ["fetch", "bounded", r, refspec] => fetch::cmd_bounded_pull(Path::new(r), refspec, &flags),
354
357
  ["fetch", "url", r, i, l, u] => fetch::cmd_url(Path::new(r), i, l, u, &flags),
355
358
  ["staging", "fetch", r, i, l, u] => fetch::cmd_staging_fetch(Path::new(r), i, l, u, &flags),
356
359
  ["staging", "drop", r, i] => fetch::cmd_staging_drop(Path::new(r), i),
@@ -361,7 +364,15 @@ fn main() -> ExitCode {
361
364
  ["source", "plan", u] => fetch::cmd_source_plan(u, &flags),
362
365
  ["source", "pull", r, u] => fetch::cmd_source_pull(Path::new(r), u, &flags),
363
366
  ["fetch", sub, _t]
364
- if ["plan", "plan-objects", "admit", "complete", "url"].contains(sub) =>
367
+ if [
368
+ "plan",
369
+ "plan-objects",
370
+ "admit",
371
+ "complete",
372
+ "url",
373
+ "bounded",
374
+ ]
375
+ .contains(sub) =>
365
376
  {
366
377
  eprintln!("usage: see the header of {}", file!());
367
378
  ExitCode::from(2)
@@ -185,7 +185,13 @@ fn open_following_with<'l>(
185
185
  None,
186
186
  deadline,
187
187
  ledger,
188
- false,
188
+ // The one plane tfs-106 deliberately left as it was. A foreign source read is
189
+ // not serving a hub call and has the source lane's own per-part retry above it,
190
+ // so arming a plane here would change a path this lane did not measure. It is
191
+ // therefore still bounded by the caller's deadline alone, and a foreign source
192
+ // that accepts a connection and stops sending to a caller who set no deadline
193
+ // still waits. Named here so the next reader finds a decision, not an oversight.
194
+ http::Judge::Deadline,
189
195
  None,
190
196
  Code::TRANSFER_FAILED,
191
197
  )?;
@@ -8,8 +8,19 @@
8
8
  //! rebinding window `egress.py` could only name;
9
9
  //! - **every socket wait is bounded by the caller's deadline and judged by the liveness
10
10
  //! ledger, never by an invented constant.** The socket read timeout here is the ledger's
11
- //! sampling RESOLUTION; each empty tick re-consults the deadline and, on a judged body,
12
- //! the measured stall rule. Nothing in this file owns a number of seconds.
11
+ //! sampling RESOLUTION; each empty tick re-consults the deadline and the measured stall
12
+ //! rule. Nothing in this file owns a number of seconds.
13
+ //!
14
+ //! That second claim used to hold for the BYTE plane only, and tfs-106 is the repair. A
15
+ //! caller with no wall deadline — which is every pod fetch, deliberately, because a module
16
+ //! that substituted a number there would be deciding on the caller's behalf that a large
17
+ //! artifact on a slow link is a failure — could wait here forever: `read_head` and the TLS
18
+ //! handshake looped on an empty tick with nothing to consult, and a hub exchange left the
19
+ //! body rule disarmed. That is how a presign to a black-holed tunnel held `Leases::minting`
20
+ //! while sixteen streams parked behind it and two paid H100s billed fifteen minutes at
21
+ //! exactly zero bytes. A CONTROL-plane exchange now answers to `Judge::Transfer`: the
22
+ //! pull's own progress, which is a measurement, and never to this call's own duration,
23
+ //! which would be a constant.
13
24
 
14
25
  use crate::err::{refuse, Code, Refusal, Result};
15
26
  use crate::transport::ledger::{Deadline, Ledger};
@@ -27,6 +38,90 @@ fn would_block(error: &std::io::Error) -> bool {
27
38
  matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut)
28
39
  }
29
40
 
41
+ /// What an exchange's SILENCE answers to, beyond the caller's deadline.
42
+ ///
43
+ /// The two planes measure different things because they are different things, and the whole
44
+ /// defect tfs-106 repairs is that only one of them was measured at all.
45
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
46
+ pub enum Judge {
47
+ /// The BYTE plane. This stream's own chunks, against the pull's measured pace: an
48
+ /// object GET that accepted a connection and stopped sending is stalled, closed, and
49
+ /// re-asked, which the plan already made cost one object.
50
+ Stream,
51
+ /// The CONTROL plane. A hub call has no bytes of its own worth judging — a mint that
52
+ /// takes a minute while sixteen streams keep landing objects is a call, not a wedge —
53
+ /// so it is judged by the PULL'S OWN PROGRESS instead. It faults exactly when nothing
54
+ /// has landed anywhere in this pull for longer than the pull's measured pace allows,
55
+ /// which is the owner's rule read literally: the lack of measured progress, never a
56
+ /// number of seconds.
57
+ Transfer,
58
+ /// Neither. The caller's deadline is the only bound, which is what a foreign-source
59
+ /// read with its own per-part retry loop above it already had.
60
+ Deadline,
61
+ }
62
+
63
+ /// The CONTROL plane's verdict, quoting exactly the figures a reader needs and no others:
64
+ /// how long this call has been waiting, what the pull has actually landed, how long ago,
65
+ /// and the term the pull itself taught. Nobody has to read this file to learn whether the
66
+ /// number was earned.
67
+ ///
68
+ /// It refuses `TRANSFER_FAILED` and deliberately not the exchange's own `fail` code. The hub
69
+ /// may be perfectly healthy; what has been established is that the transfer this call serves
70
+ /// has stopped, and calling that "the hub did not answer" would send the next reader to the
71
+ /// wrong machine. It is also what makes the verdict terminal in `hub::call` instead of
72
+ /// re-asked three more times: a re-ask reaches the identical conclusion on its first tick and
73
+ /// would only overwrite this diagnosis with a duller one.
74
+ fn transfer_verdict(
75
+ judge: Judge,
76
+ ledger: &Ledger,
77
+ since: Instant,
78
+ waiting_for: &str,
79
+ ) -> Result<()> {
80
+ if judge != Judge::Transfer {
81
+ return Ok(());
82
+ }
83
+ let still = ledger.still();
84
+ // NOTHING MOVED IS NOT THE SAME AS STOPPED MOVING, and the owner's rule turns on exactly
85
+ // that distinction: kill or error on the lack of MEASURED progress. A pull that has not
86
+ // yet landed a byte has measured nothing about its own pace, so there is no term here
87
+ // that is not a number somebody made up — and a made-up number is the thing this whole
88
+ // module refuses. A hub call that wedges before the first byte of a pull is therefore
89
+ // still bounded by the caller's deadline alone. That is a real remaining silence and it
90
+ // is stated rather than papered over with a constant.
91
+ if still.moved == 0 {
92
+ return Ok(());
93
+ }
94
+ let patience = ledger.stillness();
95
+ // TWO CLOCKS, AND BOTH MUST BE PAST THE MEASURED TERM. `silent` is how long the pull has
96
+ // moved nothing; `outstanding` is how long THIS call has been the thing it waits on.
97
+ //
98
+ // A call must never inherit stillness that predates it. Between one object landing and
99
+ // the next mint going out, a pull is legitimately still — it is hashing, fsyncing and
100
+ // committing what just arrived — and a mint that opened into the tail of that would
101
+ // otherwise be condemned on its first tick for local work it had nothing to do with.
102
+ // Requiring the call to have been outstanding for the term as well gives every call a
103
+ // full measured window of its own, and leaves the verdict meaning what it says: this
104
+ // call has been waiting this long, and the transfer moved nothing in any of it.
105
+ let outstanding = since.elapsed();
106
+ if still.silent <= patience || outstanding <= patience {
107
+ return Ok(());
108
+ }
109
+ refuse(
110
+ Code::TRANSFER_FAILED,
111
+ format!(
112
+ "the transfer this call serves has stopped: {:.1} s waiting for {waiting_for}, \
113
+ {} B landed in this pull and none of them for {:.1} s, against the {:.1} s \
114
+ this pull's own measured pace allows ({} times the longest it has ever gone \
115
+ still and recovered, and never less than one stream's own term)",
116
+ outstanding.as_secs_f64(),
117
+ still.moved,
118
+ still.silent.as_secs_f64(),
119
+ patience.as_secs_f64(),
120
+ crate::transport::ledger::STALL_FACTOR,
121
+ ),
122
+ )
123
+ }
124
+
30
125
  pub struct Client {
31
126
  tls: Arc<rustls::ClientConfig>,
32
127
  }
@@ -182,10 +277,14 @@ fn tls_handshake(
182
277
  sock: &mut TcpStream,
183
278
  conn: &mut rustls::ClientConnection,
184
279
  deadline: Deadline,
185
- sample: Duration,
280
+ ledger: &Ledger,
281
+ judge: Judge,
282
+ since: Instant,
186
283
  fail: Code,
187
284
  ) -> Result<()> {
285
+ let sample = ledger.sample();
188
286
  while conn.is_handshaking() {
287
+ transfer_verdict(judge, ledger, since, "a TLS handshake")?;
189
288
  let tick = match deadline.remaining()? {
190
289
  Some(left) => left.min(sample),
191
290
  None => sample,
@@ -275,8 +374,11 @@ pub struct BodyReader<'l> {
275
374
  raw_at: usize,
276
375
  deadline: Deadline,
277
376
  ledger: &'l Ledger,
278
- judge: bool,
377
+ judge: Judge,
279
378
  advanced: Instant,
379
+ /// When this body's reader was built — the start of the wait a control-plane verdict
380
+ /// measures, so a body that opens into a pull's local pause is not condemned for it.
381
+ since: Instant,
280
382
  failure: Option<Refusal>,
281
383
  stalled: bool,
282
384
  fail: Code,
@@ -309,8 +411,9 @@ impl BodyReader<'_> {
309
411
  return Ok(n);
310
412
  }
311
413
  Err(e) if would_block(&e) => {
414
+ transfer_verdict(self.judge, self.ledger, self.since, "a response body")?;
312
415
  let silent = self.advanced.elapsed();
313
- if self.judge && silent > self.ledger.patience() {
416
+ if self.judge == Judge::Stream && silent > self.ledger.patience() {
314
417
  self.stalled = true;
315
418
  let _ = self.wire.sock().shutdown(std::net::Shutdown::Both);
316
419
  return refuse(
@@ -381,7 +484,30 @@ impl BodyReader<'_> {
381
484
  }
382
485
  }
383
486
 
487
+ /// One read of the framed body, and the ONE place the pull's own progress is counted.
488
+ ///
489
+ /// It is counted here and not at the socket because `read_head` fills its buffer with
490
+ /// the head AND whatever of the body arrived behind it, so a small object can land in
491
+ /// its entirety before this reader exists. Counting at the socket missed exactly those,
492
+ /// which left the control plane believing a busy pull had moved nothing — the one state
493
+ /// in which it must not judge at all.
494
+ ///
495
+ /// THE PULL'S PROGRESS IS ITS ARTIFACT BYTES. A judged body is an object crossing the
496
+ /// door and is counted; anything else — a hub's JSON answer, a foreign source read —
497
+ /// advances the pull without adding to the figure a refusal quotes, because a reader
498
+ /// checking that figure against the transfer's own report must find the same number.
384
499
  pub fn read_refusing(&mut self, out: &mut [u8]) -> Result<usize> {
500
+ let n = self.read_framed(out)?;
501
+ if n > 0 {
502
+ match self.judge {
503
+ Judge::Stream => self.ledger.moved(n as u64),
504
+ _ => self.ledger.answered(),
505
+ }
506
+ }
507
+ Ok(n)
508
+ }
509
+
510
+ fn read_framed(&mut self, out: &mut [u8]) -> Result<usize> {
385
511
  if out.is_empty() {
386
512
  return Ok(0);
387
513
  }
@@ -469,11 +595,13 @@ impl Read for BodyReader<'_> {
469
595
  }
470
596
 
471
597
  /// One exchange: connect to the CHECKED addresses, write the request, read the head, hand
472
- /// back the framed body. `judge_body` arms the ledger's stall verdict on body reads (the
473
- /// object plane); hub JSON exchanges leave it off and answer to the deadline alone, as
474
- /// `pull.py`'s `_json` did. `write_silence` bounds one WRITE operation that moves nothing
475
- /// — the push plane derives it from the ledger's floor, the port of egress.py's one
476
- /// derived silence bound; `None` answers to the deadline alone.
598
+ /// back the framed body. `judge` says which plane this exchange belongs to and therefore
599
+ /// what its silence answers to — see `Judge`. It governs EVERY wait in the exchange, not
600
+ /// only the body: the handshake, the request write and the response head are exactly where
601
+ /// a black-holed tunnel parks a hub call, and leaving them unjudged is what made a
602
+ /// deadline-free caller wait forever. `write_silence` additionally bounds one WRITE
603
+ /// operation that moves nothing — the push plane derives it from the ledger's floor, the
604
+ /// port of egress.py's one derived silence bound; `None` leaves the write to `judge`.
477
605
  #[allow(clippy::too_many_arguments)]
478
606
  pub fn request<'l>(
479
607
  client: &Client,
@@ -483,7 +611,7 @@ pub fn request<'l>(
483
611
  body: Option<&[u8]>,
484
612
  deadline: Deadline,
485
613
  ledger: &'l Ledger,
486
- judge_body: bool,
614
+ judge: Judge,
487
615
  write_silence: Option<Duration>,
488
616
  fail: Code,
489
617
  ) -> Result<Response<'l>> {
@@ -501,7 +629,7 @@ pub fn request<'l>(
501
629
  })?;
502
630
  let mut sock = sock;
503
631
  let mut conn = Box::new(conn);
504
- tls_handshake(&mut sock, &mut conn, deadline, ledger.sample(), fail)?;
632
+ tls_handshake(&mut sock, &mut conn, deadline, ledger, judge, opened, fail)?;
505
633
  Wire::Tls { sock, conn }
506
634
  } else {
507
635
  Wire::Plain(sock)
@@ -531,12 +659,19 @@ pub fn request<'l>(
531
659
  &request_bytes,
532
660
  deadline,
533
661
  ledger,
662
+ judge,
663
+ opened,
534
664
  write_silence,
535
665
  fail,
536
666
  )?;
537
667
 
538
- let (status, response_headers, spill) = read_head(&mut wire, deadline, ledger, fail)?;
668
+ let (status, response_headers, spill) =
669
+ read_head(&mut wire, deadline, ledger, judge, opened, fail)?;
539
670
  ledger.taught(opened.elapsed());
671
+ // A HEAD THAT ARRIVED IS PROGRESS. Without this, the second page of a paginated closure
672
+ // would be judged from the start of the first, and a pull whose hub is merely slow
673
+ // would condemn its own second call for the first call's honest latency.
674
+ ledger.answered();
540
675
 
541
676
  let framing = framing_of(status, &response_headers, fail)?;
542
677
  Ok(Response {
@@ -549,8 +684,9 @@ pub fn request<'l>(
549
684
  raw_at: 0,
550
685
  deadline,
551
686
  ledger,
552
- judge: judge_body,
687
+ judge,
553
688
  advanced: Instant::now(),
689
+ since: Instant::now(),
554
690
  failure: None,
555
691
  stalled: false,
556
692
  fail,
@@ -558,17 +694,21 @@ pub fn request<'l>(
558
694
  })
559
695
  }
560
696
 
697
+ #[allow(clippy::too_many_arguments)]
561
698
  fn write_all(
562
699
  wire: &mut Wire,
563
700
  bytes: &[u8],
564
701
  deadline: Deadline,
565
702
  ledger: &Ledger,
703
+ judge: Judge,
704
+ since: Instant,
566
705
  silence: Option<Duration>,
567
706
  fail: Code,
568
707
  ) -> Result<()> {
569
708
  let mut at = 0;
570
709
  let mut advanced = Instant::now();
571
710
  loop {
711
+ transfer_verdict(judge, ledger, since, "a request write")?;
572
712
  let tick = match deadline.remaining()? {
573
713
  Some(left) => left.min(ledger.sample()),
574
714
  None => ledger.sample(),
@@ -611,7 +751,14 @@ fn write_all(
611
751
  /// Status, headers (lowercased names), and body bytes read past the head.
612
752
  type Head = (u16, Vec<(String, String)>, Vec<u8>);
613
753
 
614
- fn read_head(wire: &mut Wire, deadline: Deadline, ledger: &Ledger, fail: Code) -> Result<Head> {
754
+ fn read_head(
755
+ wire: &mut Wire,
756
+ deadline: Deadline,
757
+ ledger: &Ledger,
758
+ judge: Judge,
759
+ since: Instant,
760
+ fail: Code,
761
+ ) -> Result<Head> {
615
762
  let mut collected: Vec<u8> = Vec::new();
616
763
  let mut buf = vec![0u8; 8 * 1024];
617
764
  let split = loop {
@@ -624,7 +771,7 @@ fn read_head(wire: &mut Wire, deadline: Deadline, ledger: &Ledger, fail: Code) -
624
771
  if collected.len() > HEAD_MAX_BYTES {
625
772
  return refuse(fail, "response head passed 64 KiB");
626
773
  }
627
- let n = read_step_ticking(wire, &mut buf, deadline, ledger, fail)?;
774
+ let n = read_step_ticking(wire, &mut buf, deadline, ledger, judge, since, fail)?;
628
775
  if n == 0 {
629
776
  return refuse(
630
777
  fail,
@@ -660,9 +807,12 @@ fn read_step_ticking(
660
807
  buf: &mut [u8],
661
808
  deadline: Deadline,
662
809
  ledger: &Ledger,
810
+ judge: Judge,
811
+ since: Instant,
663
812
  fail: Code,
664
813
  ) -> Result<usize> {
665
814
  loop {
815
+ transfer_verdict(judge, ledger, since, "a response head")?;
666
816
  let tick = match deadline.remaining()? {
667
817
  Some(left) => left.min(ledger.sample()),
668
818
  None => ledger.sample(),
@@ -505,6 +505,15 @@ fn call(
505
505
  client, base, path, body, credential, policy, deadline, ledger,
506
506
  ) {
507
507
  Ok(value) => return Ok(value),
508
+ // NOT WEATHER, AND NOT ABOUT THE HUB (tfs-106). `TRANSFER_FAILED` out of an
509
+ // exchange means the PULL has stopped moving while this call was outstanding.
510
+ // Re-asking reaches the identical verdict on the next attempt's first tick —
511
+ // nothing this loop does can restart a transfer — and the only thing three more
512
+ // asks would achieve is replacing a refusal that names the response head with
513
+ // one that names a request write. Answer with what was measured.
514
+ Err(Retryable { refusal, .. }) if refusal.code == Code::TRANSFER_FAILED => {
515
+ return Err(refusal)
516
+ }
508
517
  Err(Retryable { wait, refusal }) if !final_attempt => {
509
518
  // A wait is spent out of the caller's budget, never past it.
510
519
  let wait = match deadline.remaining()? {
@@ -561,7 +570,13 @@ fn call_once(
561
570
  Some(body),
562
571
  deadline,
563
572
  ledger,
564
- false,
573
+ // THE CONTROL PLANE (tfs-106). Every wait in this exchange answers to the pull's own
574
+ // progress. It is not a bound on how long a mint may take — a hub is entitled to be
575
+ // slow, and a mint outstanding while sixteen streams keep landing objects is left
576
+ // entirely alone. It is a bound on a call that is serving a transfer which has
577
+ // itself stopped, which is the only shape `Leases::minting` can wedge in, and the
578
+ // shape that cost `gayong` and `weed` fifteen billed minutes at zero bytes.
579
+ crate::transport::http::Judge::Transfer,
565
580
  None,
566
581
  Code::HUB_UNREACHABLE,
567
582
  )