covalence 0.1.0.dev4__tar.gz → 0.1.0.dev5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/Cargo.lock +19 -19
  2. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/Cargo.toml +17 -17
  3. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/PKG-INFO +25 -8
  4. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/README.md +24 -7
  5. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/src/lib.rs +106 -6
  6. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/src/write.rs +92 -8
  7. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/tests/kv.rs +391 -29
  8. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/README.md +24 -7
  9. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/src/lib.rs +19 -4
  10. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/tests/test_api.py +38 -0
  11. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/python/covalence/THIRD-PARTY-LICENSES.md +17 -17
  12. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/python/covalence/_native.pyi +14 -4
  13. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/LICENSE-APACHE +0 -0
  14. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/LICENSE-CC0 +0 -0
  15. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/LICENSE-MIT +0 -0
  16. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/Cargo.toml +0 -0
  17. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/LICENSE-APACHE +0 -0
  18. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/LICENSE-CC0 +0 -0
  19. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/LICENSE-MIT +0 -0
  20. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/README.md +0 -0
  21. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/src/lib.rs +0 -0
  22. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/tests/blob.rs +0 -0
  23. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/tests/common/mod.rs +0 -0
  24. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/tests/fact.rs +0 -0
  25. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/kv/Cargo.toml +0 -0
  26. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/kv/LICENSE-APACHE +0 -0
  27. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/kv/LICENSE-CC0 +0 -0
  28. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/kv/LICENSE-MIT +0 -0
  29. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/kv/src/lib.rs +0 -0
  30. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/kv/tests/kv.rs +0 -0
  31. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/memory/Cargo.toml +0 -0
  32. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/memory/LICENSE-APACHE +0 -0
  33. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/memory/LICENSE-CC0 +0 -0
  34. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/memory/LICENSE-MIT +0 -0
  35. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/memory/src/lib.rs +0 -0
  36. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/memory/tests/memory.rs +0 -0
  37. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/sqlite/Cargo.toml +0 -0
  38. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/sqlite/LICENSE-APACHE +0 -0
  39. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/sqlite/LICENSE-CC0 +0 -0
  40. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/sqlite/LICENSE-MIT +0 -0
  41. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/sqlite/src/lib.rs +0 -0
  42. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/sqlite/tests/sqlite.rs +0 -0
  43. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/trait/Cargo.toml +0 -0
  44. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/trait/LICENSE-APACHE +0 -0
  45. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/trait/LICENSE-CC0 +0 -0
  46. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/trait/LICENSE-MIT +0 -0
  47. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/trait/src/lib.rs +0 -0
  48. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/trait/tests/trait.rs +0 -0
  49. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/memory/Cargo.toml +0 -0
  50. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/memory/LICENSE-APACHE +0 -0
  51. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/memory/LICENSE-CC0 +0 -0
  52. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/memory/LICENSE-MIT +0 -0
  53. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/memory/src/lib.rs +0 -0
  54. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/memory/tests/memory.rs +0 -0
  55. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/Cargo.toml +0 -0
  56. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/LICENSE-APACHE +0 -0
  57. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/LICENSE-CC0 +0 -0
  58. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/LICENSE-MIT +0 -0
  59. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/src/read.rs +0 -0
  60. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/tests/aws.rs +0 -0
  61. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/trait/Cargo.toml +0 -0
  62. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/trait/LICENSE-APACHE +0 -0
  63. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/trait/LICENSE-CC0 +0 -0
  64. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/trait/LICENSE-MIT +0 -0
  65. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/trait/src/lib.rs +0 -0
  66. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/trait/tests/atomic.rs +0 -0
  67. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/Cargo.toml +0 -0
  68. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/LICENSE-APACHE +0 -0
  69. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/LICENSE-CC0 +0 -0
  70. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/LICENSE-MIT +0 -0
  71. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/README.md +0 -0
  72. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/dasl.rs +0 -0
  73. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/fact.rs +0 -0
  74. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/git.rs +0 -0
  75. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/lib.rs +0 -0
  76. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/multicodec.rs +0 -0
  77. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/o256.rs +0 -0
  78. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/storage.rs +0 -0
  79. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/cid.rs +0 -0
  80. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/fact.rs +0 -0
  81. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/git.rs +0 -0
  82. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/o256.rs +0 -0
  83. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/shattered/1.bin +0 -0
  84. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/shattered/2.bin +0 -0
  85. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/shattered/README.md +0 -0
  86. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/Cargo.toml +0 -0
  87. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/LICENSE-APACHE +0 -0
  88. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/LICENSE-CC0 +0 -0
  89. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/LICENSE-MIT +0 -0
  90. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/README.md +0 -0
  91. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/src/drisl.rs +0 -0
  92. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/src/event.rs +0 -0
  93. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/src/json.rs +0 -0
  94. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/src/lib.rs +0 -0
  95. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/codec.rs +0 -0
  96. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/events.rs +0 -0
  97. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fact.rs +0 -0
  98. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/LICENSE +0 -0
  99. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/README.md +0 -0
  100. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/cid.json +0 -0
  101. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/concat.json +0 -0
  102. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/floats.json +0 -0
  103. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/indefinite.json +0 -0
  104. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/integer_range.json +0 -0
  105. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/map_keys.json +0 -0
  106. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/numeric_reduction.json +0 -0
  107. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/short_form.json +0 -0
  108. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/simple.json +0 -0
  109. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/tags.json +0 -0
  110. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/utf8.json +0 -0
  111. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures.rs +0 -0
  112. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/json.rs +0 -0
  113. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/vectors/README.md +0 -0
  114. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/vectors/cid-vectors.jsonl +0 -0
  115. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/vectors/dasl-vectors.jsonl +0 -0
  116. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/vectors.rs +0 -0
  117. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/data/Cargo.toml +0 -0
  118. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/data/LICENSE-APACHE +0 -0
  119. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/data/LICENSE-CC0 +0 -0
  120. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/data/LICENSE-MIT +0 -0
  121. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/data/src/lib.rs +0 -0
  122. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/error/Cargo.toml +0 -0
  123. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/error/LICENSE-APACHE +0 -0
  124. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/error/LICENSE-CC0 +0 -0
  125. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/error/LICENSE-MIT +0 -0
  126. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/error/src/lib.rs +0 -0
  127. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/python/Cargo.toml +0 -0
  128. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/python/LICENSE-APACHE +0 -0
  129. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/python/LICENSE-CC0 +0 -0
  130. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/python/LICENSE-MIT +0 -0
  131. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/python/src/lib.rs +0 -0
  132. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/rand/Cargo.toml +0 -0
  133. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/rand/LICENSE-APACHE +0 -0
  134. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/rand/LICENSE-CC0 +0 -0
  135. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/rand/LICENSE-MIT +0 -0
  136. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/rand/src/lib.rs +0 -0
  137. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/serde/Cargo.toml +0 -0
  138. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/serde/LICENSE-APACHE +0 -0
  139. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/serde/LICENSE-CC0 +0 -0
  140. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/serde/LICENSE-MIT +0 -0
  141. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/serde/src/lib.rs +0 -0
  142. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/sqlite/Cargo.toml +0 -0
  143. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/sqlite/LICENSE-APACHE +0 -0
  144. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/sqlite/LICENSE-CC0 +0 -0
  145. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/sqlite/LICENSE-MIT +0 -0
  146. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/sqlite/src/lib.rs +0 -0
  147. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/Cargo.toml +0 -0
  148. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/LICENSE-APACHE +0 -0
  149. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/LICENSE-CC0 +0 -0
  150. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/LICENSE-MIT +0 -0
  151. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/about.md.hbs +0 -0
  152. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/about.toml +0 -0
  153. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/src/errors.rs +0 -0
  154. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/src/streams/download.rs +0 -0
  155. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/src/streams/mod.rs +0 -0
  156. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/src/streams/upload/tests.rs +0 -0
  157. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/src/streams/upload.rs +0 -0
  158. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/tests/test_bounded_io.py +0 -0
  159. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/tests/test_docs.py +0 -0
  160. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/tests/test_package.py +0 -0
  161. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/tests/test_stubs.py +0 -0
  162. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/tests/typed_usage.py +0 -0
  163. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/pyproject.toml +0 -0
  164. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/python/covalence/__init__.py +0 -0
  165. {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/python/covalence/py.typed +0 -0
@@ -779,7 +779,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
779
779
 
780
780
  [[package]]
781
781
  name = "covalence-blob-trait"
782
- version = "0.1.0-dev.4"
782
+ version = "0.1.0-dev.5"
783
783
  dependencies = [
784
784
  "covalence-lib-cid",
785
785
  "covalence-lib-data",
@@ -788,7 +788,7 @@ dependencies = [
788
788
 
789
789
  [[package]]
790
790
  name = "covalence-cas-kv"
791
- version = "0.1.0-dev.4"
791
+ version = "0.1.0-dev.5"
792
792
  dependencies = [
793
793
  "blake3",
794
794
  "covalence-cas-trait",
@@ -810,7 +810,7 @@ dependencies = [
810
810
 
811
811
  [[package]]
812
812
  name = "covalence-cas-memory"
813
- version = "0.1.0-dev.4"
813
+ version = "0.1.0-dev.5"
814
814
  dependencies = [
815
815
  "covalence-cas-trait",
816
816
  "covalence-lib-cid",
@@ -821,7 +821,7 @@ dependencies = [
821
821
 
822
822
  [[package]]
823
823
  name = "covalence-cas-sqlite"
824
- version = "0.1.0-dev.4"
824
+ version = "0.1.0-dev.5"
825
825
  dependencies = [
826
826
  "covalence-cas-trait",
827
827
  "covalence-lib-cid",
@@ -836,7 +836,7 @@ dependencies = [
836
836
 
837
837
  [[package]]
838
838
  name = "covalence-cas-trait"
839
- version = "0.1.0-dev.4"
839
+ version = "0.1.0-dev.5"
840
840
  dependencies = [
841
841
  "covalence-lib-cid",
842
842
  "covalence-lib-dasl",
@@ -847,7 +847,7 @@ dependencies = [
847
847
 
848
848
  [[package]]
849
849
  name = "covalence-kv-memory"
850
- version = "0.1.0-dev.4"
850
+ version = "0.1.0-dev.5"
851
851
  dependencies = [
852
852
  "covalence-kv-trait",
853
853
  "covalence-lib-dasl",
@@ -857,7 +857,7 @@ dependencies = [
857
857
 
858
858
  [[package]]
859
859
  name = "covalence-kv-s3"
860
- version = "0.1.0-dev.4"
860
+ version = "0.1.0-dev.5"
861
861
  dependencies = [
862
862
  "aws-config",
863
863
  "aws-credential-types",
@@ -881,7 +881,7 @@ dependencies = [
881
881
 
882
882
  [[package]]
883
883
  name = "covalence-kv-trait"
884
- version = "0.1.0-dev.4"
884
+ version = "0.1.0-dev.5"
885
885
  dependencies = [
886
886
  "covalence-lib-dasl",
887
887
  "covalence-lib-data",
@@ -890,7 +890,7 @@ dependencies = [
890
890
 
891
891
  [[package]]
892
892
  name = "covalence-lib-cid"
893
- version = "0.1.0-dev.4"
893
+ version = "0.1.0-dev.5"
894
894
  dependencies = [
895
895
  "blake3",
896
896
  "covalence-lib-data",
@@ -905,14 +905,14 @@ dependencies = [
905
905
 
906
906
  [[package]]
907
907
  name = "covalence-lib-cli"
908
- version = "0.1.0-dev.4"
908
+ version = "0.1.0-dev.5"
909
909
  dependencies = [
910
910
  "clap",
911
911
  ]
912
912
 
913
913
  [[package]]
914
914
  name = "covalence-lib-dasl"
915
- version = "0.1.0-dev.4"
915
+ version = "0.1.0-dev.5"
916
916
  dependencies = [
917
917
  "blake3",
918
918
  "covalence-lib-cid",
@@ -927,7 +927,7 @@ dependencies = [
927
927
 
928
928
  [[package]]
929
929
  name = "covalence-lib-data"
930
- version = "0.1.0-dev.4"
930
+ version = "0.1.0-dev.5"
931
931
  dependencies = [
932
932
  "arrayvec",
933
933
  "bytes",
@@ -939,7 +939,7 @@ dependencies = [
939
939
 
940
940
  [[package]]
941
941
  name = "covalence-lib-error"
942
- version = "0.1.0-dev.4"
942
+ version = "0.1.0-dev.5"
943
943
  dependencies = [
944
944
  "miette",
945
945
  "snafu",
@@ -947,14 +947,14 @@ dependencies = [
947
947
 
948
948
  [[package]]
949
949
  name = "covalence-lib-python"
950
- version = "0.1.0-dev.4"
950
+ version = "0.1.0-dev.5"
951
951
  dependencies = [
952
952
  "pyo3",
953
953
  ]
954
954
 
955
955
  [[package]]
956
956
  name = "covalence-lib-rand"
957
- version = "0.1.0-dev.4"
957
+ version = "0.1.0-dev.5"
958
958
  dependencies = [
959
959
  "getrandom 0.3.4",
960
960
  "rand",
@@ -962,21 +962,21 @@ dependencies = [
962
962
 
963
963
  [[package]]
964
964
  name = "covalence-lib-serde"
965
- version = "0.1.0-dev.4"
965
+ version = "0.1.0-dev.5"
966
966
  dependencies = [
967
967
  "serde",
968
968
  ]
969
969
 
970
970
  [[package]]
971
971
  name = "covalence-lib-sqlite"
972
- version = "0.1.0-dev.4"
972
+ version = "0.1.0-dev.5"
973
973
  dependencies = [
974
974
  "rusqlite",
975
975
  ]
976
976
 
977
977
  [[package]]
978
978
  name = "covalence-python"
979
- version = "0.1.0-dev.4"
979
+ version = "0.1.0-dev.5"
980
980
  dependencies = [
981
981
  "blake3",
982
982
  "covalence-blob-trait",
@@ -997,7 +997,7 @@ dependencies = [
997
997
 
998
998
  [[package]]
999
999
  name = "covalence-tools-glu"
1000
- version = "0.1.0-dev.4"
1000
+ version = "0.1.0-dev.5"
1001
1001
  dependencies = [
1002
1002
  "covalence-lib-cli",
1003
1003
  "covalence-lib-error",
@@ -3,7 +3,7 @@ members = ["crates/lib/error", "crates/lib/serde", "crates/lib/data", "crates/li
3
3
  resolver = "3"
4
4
 
5
5
  [workspace.package]
6
- version = "0.1.0-dev.4"
6
+ version = "0.1.0-dev.5"
7
7
  edition = "2024"
8
8
  rust-version = "1.98"
9
9
  license = "CC0-1.0 OR MIT OR Apache-2.0"
@@ -17,22 +17,22 @@ miette = "7.6.0"
17
17
  sha2 = "0.11.0"
18
18
  snafu = "0.9.2"
19
19
  # Published crates, at the workspace version so the set moves together.
20
- covalence-blob-trait = { path = "crates/blob/trait", version = "=0.1.0-dev.4" }
21
- covalence-cas-kv = { path = "crates/cas/kv", version = "=0.1.0-dev.4" }
22
- covalence-cas-memory = { path = "crates/cas/memory", version = "=0.1.0-dev.4" }
23
- covalence-cas-sqlite = { path = "crates/cas/sqlite", version = "=0.1.0-dev.4" }
24
- covalence-cas-trait = { path = "crates/cas/trait", version = "=0.1.0-dev.4" }
25
- covalence-kv-memory = { path = "crates/kv/memory", version = "=0.1.0-dev.4" }
26
- covalence-kv-s3 = { path = "crates/kv/s3", version = "=0.1.0-dev.4" }
27
- covalence-kv-trait = { path = "crates/kv/trait", version = "=0.1.0-dev.4" }
28
- covalence-lib-cid = { path = "crates/lib/cid", version = "=0.1.0-dev.4" }
29
- covalence-lib-cli = { path = "crates/lib/cli", version = "=0.1.0-dev.4" }
30
- covalence-lib-dasl = { path = "crates/lib/dasl", version = "=0.1.0-dev.4" }
31
- covalence-lib-data = { path = "crates/lib/data", version = "=0.1.0-dev.4" }
32
- covalence-lib-error = { path = "crates/lib/error", version = "=0.1.0-dev.4" }
33
- covalence-lib-rand = { path = "crates/lib/rand", version = "=0.1.0-dev.4" }
34
- covalence-lib-serde = { path = "crates/lib/serde", version = "=0.1.0-dev.4" }
35
- covalence-lib-sqlite = { path = "crates/lib/sqlite", version = "=0.1.0-dev.4" }
20
+ covalence-blob-trait = { path = "crates/blob/trait", version = "=0.1.0-dev.5" }
21
+ covalence-cas-kv = { path = "crates/cas/kv", version = "=0.1.0-dev.5" }
22
+ covalence-cas-memory = { path = "crates/cas/memory", version = "=0.1.0-dev.5" }
23
+ covalence-cas-sqlite = { path = "crates/cas/sqlite", version = "=0.1.0-dev.5" }
24
+ covalence-cas-trait = { path = "crates/cas/trait", version = "=0.1.0-dev.5" }
25
+ covalence-kv-memory = { path = "crates/kv/memory", version = "=0.1.0-dev.5" }
26
+ covalence-kv-s3 = { path = "crates/kv/s3", version = "=0.1.0-dev.5" }
27
+ covalence-kv-trait = { path = "crates/kv/trait", version = "=0.1.0-dev.5" }
28
+ covalence-lib-cid = { path = "crates/lib/cid", version = "=0.1.0-dev.5" }
29
+ covalence-lib-cli = { path = "crates/lib/cli", version = "=0.1.0-dev.5" }
30
+ covalence-lib-dasl = { path = "crates/lib/dasl", version = "=0.1.0-dev.5" }
31
+ covalence-lib-data = { path = "crates/lib/data", version = "=0.1.0-dev.5" }
32
+ covalence-lib-error = { path = "crates/lib/error", version = "=0.1.0-dev.5" }
33
+ covalence-lib-rand = { path = "crates/lib/rand", version = "=0.1.0-dev.5" }
34
+ covalence-lib-serde = { path = "crates/lib/serde", version = "=0.1.0-dev.5" }
35
+ covalence-lib-sqlite = { path = "crates/lib/sqlite", version = "=0.1.0-dev.5" }
36
36
 
37
37
  [workspace.lints.rust]
38
38
  unsafe_code = "forbid"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: covalence
3
- Version: 0.1.0.dev4
3
+ Version: 0.1.0.dev5
4
4
  License-File: LICENSE-APACHE
5
5
  License-File: LICENSE-CC0
6
6
  License-File: LICENSE-MIT
@@ -35,7 +35,7 @@ True
35
35
  ## Install
36
36
 
37
37
  ```sh
38
- pip install covalence==0.1.0.dev4
38
+ pip install covalence==0.1.0.dev5
39
39
  ```
40
40
 
41
41
  - Pre-releases need an exact pin like this one, or `pip install --pre`.
@@ -120,8 +120,23 @@ pip install covalence==0.1.0.dev4
120
120
  - S3 keys must be UTF-8 with non-empty `/`-separated segments other than `.`
121
121
  and `..`; other keys raise `StoreError`.
122
122
  - On S3, values over 8 MiB go up as multipart uploads, and values come down
123
- in ranged GETs of 8 MiB, 16 parts or ranges at a time. Covalence sets no
124
- size limit; S3's is 5 TiB.
123
+ in ranged GETs of 8 MiB, 16 parts or ranges at a time. Upload parts grow
124
+ when needed to stay within 10,000 parts. Covalence checks
125
+ [S3's limits](https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html)
126
+ before upload: at most 10,000 parts of 5 GiB each, subject to the platform's
127
+ allocation limit.
128
+ - `Kv.s3(..., upload_buffer_bytes=n)` caps aggregate upload payload
129
+ reservations across calls sharing that store. An upload reserves the smaller
130
+ of its length and its part size times its worker count before allocating
131
+ parts, reading its upload source or making requests. Insufficient capacity
132
+ raises `StoreError` immediately; zero admits only empty uploads. The default
133
+ is no budget. Reservations last through retries and cleanup.
134
+ - This budget excludes caller-owned sources, read buffers, allocator overhead,
135
+ HTTP/TLS buffers and request metadata; it is not an RSS limit. Owned byte
136
+ values still reserve their planned payload even though their parts share the
137
+ original allocation. File helpers may hash their source before upload
138
+ admission. A larger legal object can need a larger budget because its parts
139
+ grow; rejection does not consume its upload source or create an upload.
125
140
 
126
141
  ## Content-addressed stores
127
142
 
@@ -196,8 +211,10 @@ pip install covalence==0.1.0.dev4
196
211
  - These limits govern source length and output bytes, not the memory a backend
197
212
  uses before reading or writing. Python callback transfers are at most 1 MiB.
198
213
  - `put_file`, `put_fileobj`, `put_from`, `get_file` and `get_to` stream on `Cas.kv` over `Kv.s3`,
199
- holding up to 16 parts of 8 MiB (128 MiB) at a time, or as many as
214
+ holding up to 16 part payloads at a time, or as many as
200
215
  `Kv.s3(..., concurrency=n)` says; blocks over 78 GiB go up in larger parts.
216
+ Upload parts start at 8 MiB; read ranges stay at 8 MiB. Set
217
+ `upload_buffer_bytes=` to enforce a shared upload payload budget.
201
218
  Other stores, including `Cas.kv(Kv.memory())`, hold the block in memory.
202
219
  - A create-only write can reuse a block without checking its stored bytes.
203
220
  Returning a CID does not freshly verify an existing block: use a checked
@@ -305,16 +322,16 @@ stub; this package writes its stub by hand and checks it against `__doc__`:
305
322
  later) for manylinux_2_28 x86_64 and aarch64 and macOS arm64, and an sdist,
306
323
  and runs `tests/` against each wheel. Pull requests and pushes keep them as
307
324
  workflow artifacts. A release tag must match the workspace version, such as
308
- `0.1.0-dev.4`; the files carry its PEP 440 form, `0.1.0.dev4`:
325
+ `0.1.0-dev.5`; the files carry its PEP 440 form, `0.1.0.dev5`:
309
326
 
310
327
  - `v<version>` attaches them to that GitHub release. In the public
311
328
  `hiloopai/covalence` repository it also publishes them to PyPI, and
312
329
  `.github/workflows/crates.yml` publishes the crates to crates.io.
313
330
  - `python-v<version>` only attaches them to the GitHub release.
314
331
 
315
- To publish `0.1.0-dev.4`:
332
+ To publish `0.1.0-dev.5`:
316
333
 
317
334
  ```sh
318
- git tag v0.1.0-dev.4 && git push origin v0.1.0-dev.4
335
+ git tag v0.1.0-dev.5 && git push origin v0.1.0-dev.5
319
336
  ```
320
337
 
@@ -21,7 +21,7 @@ True
21
21
  ## Install
22
22
 
23
23
  ```sh
24
- pip install covalence==0.1.0.dev4
24
+ pip install covalence==0.1.0.dev5
25
25
  ```
26
26
 
27
27
  - Pre-releases need an exact pin like this one, or `pip install --pre`.
@@ -106,8 +106,23 @@ pip install covalence==0.1.0.dev4
106
106
  - S3 keys must be UTF-8 with non-empty `/`-separated segments other than `.`
107
107
  and `..`; other keys raise `StoreError`.
108
108
  - On S3, values over 8 MiB go up as multipart uploads, and values come down
109
- in ranged GETs of 8 MiB, 16 parts or ranges at a time. Covalence sets no
110
- size limit; S3's is 5 TiB.
109
+ in ranged GETs of 8 MiB, 16 parts or ranges at a time. Upload parts grow
110
+ when needed to stay within 10,000 parts. Covalence checks
111
+ [S3's limits](https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html)
112
+ before upload: at most 10,000 parts of 5 GiB each, subject to the platform's
113
+ allocation limit.
114
+ - `Kv.s3(..., upload_buffer_bytes=n)` caps aggregate upload payload
115
+ reservations across calls sharing that store. An upload reserves the smaller
116
+ of its length and its part size times its worker count before allocating
117
+ parts, reading its upload source or making requests. Insufficient capacity
118
+ raises `StoreError` immediately; zero admits only empty uploads. The default
119
+ is no budget. Reservations last through retries and cleanup.
120
+ - This budget excludes caller-owned sources, read buffers, allocator overhead,
121
+ HTTP/TLS buffers and request metadata; it is not an RSS limit. Owned byte
122
+ values still reserve their planned payload even though their parts share the
123
+ original allocation. File helpers may hash their source before upload
124
+ admission. A larger legal object can need a larger budget because its parts
125
+ grow; rejection does not consume its upload source or create an upload.
111
126
 
112
127
  ## Content-addressed stores
113
128
 
@@ -182,8 +197,10 @@ pip install covalence==0.1.0.dev4
182
197
  - These limits govern source length and output bytes, not the memory a backend
183
198
  uses before reading or writing. Python callback transfers are at most 1 MiB.
184
199
  - `put_file`, `put_fileobj`, `put_from`, `get_file` and `get_to` stream on `Cas.kv` over `Kv.s3`,
185
- holding up to 16 parts of 8 MiB (128 MiB) at a time, or as many as
200
+ holding up to 16 part payloads at a time, or as many as
186
201
  `Kv.s3(..., concurrency=n)` says; blocks over 78 GiB go up in larger parts.
202
+ Upload parts start at 8 MiB; read ranges stay at 8 MiB. Set
203
+ `upload_buffer_bytes=` to enforce a shared upload payload budget.
187
204
  Other stores, including `Cas.kv(Kv.memory())`, hold the block in memory.
188
205
  - A create-only write can reuse a block without checking its stored bytes.
189
206
  Returning a CID does not freshly verify an existing block: use a checked
@@ -291,15 +308,15 @@ stub; this package writes its stub by hand and checks it against `__doc__`:
291
308
  later) for manylinux_2_28 x86_64 and aarch64 and macOS arm64, and an sdist,
292
309
  and runs `tests/` against each wheel. Pull requests and pushes keep them as
293
310
  workflow artifacts. A release tag must match the workspace version, such as
294
- `0.1.0-dev.4`; the files carry its PEP 440 form, `0.1.0.dev4`:
311
+ `0.1.0-dev.5`; the files carry its PEP 440 form, `0.1.0.dev5`:
295
312
 
296
313
  - `v<version>` attaches them to that GitHub release. In the public
297
314
  `hiloopai/covalence` repository it also publishes them to PyPI, and
298
315
  `.github/workflows/crates.yml` publishes the crates to crates.io.
299
316
  - `python-v<version>` only attaches them to the GitHub release.
300
317
 
301
- To publish `0.1.0-dev.4`:
318
+ To publish `0.1.0-dev.5`:
302
319
 
303
320
  ```sh
304
- git tag v0.1.0-dev.4 && git push origin v0.1.0-dev.4
321
+ git tag v0.1.0-dev.5 && git push origin v0.1.0-dev.5
305
322
  ```
@@ -30,17 +30,18 @@
30
30
  mod read;
31
31
  mod write;
32
32
 
33
+ pub use write::UploadPlan;
33
34
  use write::Write;
34
35
 
35
36
  use std::{
36
- collections::BTreeMap,
37
+ collections::{BTreeMap, TryReserveError},
37
38
  fmt,
38
39
  io::{self, Read},
39
40
  panic::{self, AssertUnwindSafe},
40
41
  str::FromStr,
41
42
  sync::{
42
43
  Mutex, PoisonError,
43
- atomic::{AtomicBool, Ordering},
44
+ atomic::{AtomicBool, AtomicU64, Ordering},
44
45
  mpsc,
45
46
  },
46
47
  thread,
@@ -68,6 +69,34 @@ use rusty_s3::{Bucket, BucketError, UrlStyle};
68
69
  #[snafu(crate_root(covalence_lib_error::snafu))]
69
70
  #[non_exhaustive]
70
71
  pub enum Error {
72
+ /// An upload exceeds an S3 limit or the platform's allocation limit.
73
+ #[snafu(display("invalid upload {field} {value}: expected {minimum}..={maximum}"))]
74
+ UploadLimit {
75
+ /// The quantity checked.
76
+ field: &'static str,
77
+ /// The requested quantity.
78
+ value: u64,
79
+ /// Its smallest supported value.
80
+ minimum: u64,
81
+ /// Its largest supported value.
82
+ maximum: u64,
83
+ },
84
+ /// The store's upload payload budget cannot admit another upload.
85
+ #[snafu(display("upload needs {required} buffer bytes; {available} available"))]
86
+ UploadBuffer {
87
+ /// The payload reservation required for this upload.
88
+ required: u64,
89
+ /// The unreserved bytes when admission failed.
90
+ available: u64,
91
+ },
92
+ /// Allocating an upload part failed after admission.
93
+ #[snafu(display("could not allocate {bytes} upload bytes: {source}"))]
94
+ UploadAllocation {
95
+ /// The part's size in bytes.
96
+ bytes: u64,
97
+ /// Why the allocator refused it.
98
+ source: TryReserveError,
99
+ },
71
100
  /// The endpoint is not a URL.
72
101
  #[snafu(display("invalid endpoint {endpoint}: {source}"))]
73
102
  Endpoint {
@@ -254,10 +283,18 @@ const EXPIRY: Duration = Duration::from_secs(60);
254
283
  const TIMEOUT: Duration = Duration::from_secs(30);
255
284
  /// The slowest transfer, in bytes per second, a request attempt waits for.
256
285
  const MIN_RATE: u64 = 1 << 20;
257
- /// The largest S3 object, in bytes.
258
- const MAX_OBJECT: u64 = 5 << 40;
286
+ /// The body size above which request timeouts stop increasing.
287
+ const MAX_TIMEOUT_BYTES: u64 = 5 << 40;
259
288
  /// The most parts a multipart upload may have.
260
289
  const MAX_PARTS: u64 = 10_000;
290
+ /// S3's largest multipart part, in bytes.
291
+ const MAX_PART: u64 = 5 << 30;
292
+ /// S3's smallest non-final multipart part, in bytes.
293
+ const MIN_PART: u64 = 5 << 20;
294
+ /// S3's largest object: 10,000 parts of 5 GiB.
295
+ const MAX_OBJECT: u64 = MAX_PARTS * MAX_PART;
296
+ /// The largest body sent with one PUT, in bytes (5 GB).
297
+ const MAX_SINGLE_PUT: u64 = 5_000_000_000;
261
298
  /// The default part size, 8 MiB: values larger are uploaded in parts of at
262
299
  /// least this size, and values are read in ranges of it.
263
300
  pub const PART: u64 = 8 << 20;
@@ -283,7 +320,7 @@ pub struct AwsOptions {
283
320
 
284
321
  /// The limit on a request attempt moving `bytes` of body.
285
322
  fn timeout(bytes: u64) -> Duration {
286
- TIMEOUT + Duration::from_secs(bytes.min(MAX_OBJECT) / MIN_RATE)
323
+ TIMEOUT + Duration::from_secs(bytes.min(MAX_TIMEOUT_BYTES) / MIN_RATE)
287
324
  }
288
325
 
289
326
  /// Run `future` to completion on `runtime`, on a thread of its own so that
@@ -400,9 +437,12 @@ impl Cached {
400
437
  /// # Writes
401
438
  ///
402
439
  /// - A value of up to the part size ([`PART`] unless set with
403
- /// [`S3Kv::part_size`]) is written with one PUT.
440
+ /// [`S3Kv::part_size`]) and 5 GB is written with one PUT.
404
441
  /// - A larger value is written with a multipart upload, in parts of the part
405
442
  /// size, or larger parts if needed to keep to S3's limit of 10,000 parts.
443
+ /// - [`S3Kv::upload_plan`] checks S3 and allocation limits before any
444
+ /// upload request or source read. [`S3Kv::upload_buffer_bytes`] can bound
445
+ /// the aggregate upload payload held by calls sharing this store.
406
446
  /// - Each PUT or part sends the SHA-256 of its body as
407
447
  /// `x-amz-checksum-sha256`, which S3 checks against the bytes it receives.
408
448
  /// - A failed multipart upload is aborted once its parts in flight finish.
@@ -458,6 +498,43 @@ pub struct S3Kv {
458
498
  agent: ureq::Agent,
459
499
  part: u64,
460
500
  concurrency: usize,
501
+ upload_budget: Option<UploadBudget>,
502
+ }
503
+
504
+ /// Aggregate payload reservations for calls sharing one store.
505
+ struct UploadBudget {
506
+ limit: u64,
507
+ used: AtomicU64,
508
+ }
509
+
510
+ impl UploadBudget {
511
+ fn reserve(&self, required: u64) -> Result<UploadReservation<'_>, Error> {
512
+ self.used
513
+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |used| {
514
+ used.checked_add(required)
515
+ .filter(|next| *next <= self.limit)
516
+ })
517
+ .map_err(|used| Error::UploadBuffer {
518
+ required,
519
+ available: self.limit - used,
520
+ })?;
521
+ Ok(UploadReservation {
522
+ budget: self,
523
+ bytes: required,
524
+ })
525
+ }
526
+ }
527
+
528
+ /// Held until all upload requests, including cleanup, have finished.
529
+ struct UploadReservation<'a> {
530
+ budget: &'a UploadBudget,
531
+ bytes: u64,
532
+ }
533
+
534
+ impl Drop for UploadReservation<'_> {
535
+ fn drop(&mut self) {
536
+ self.budget.used.fetch_sub(self.bytes, Ordering::Relaxed);
537
+ }
461
538
  }
462
539
 
463
540
  impl fmt::Debug for S3Kv {
@@ -498,6 +575,7 @@ impl S3Kv {
498
575
  agent: agent(CONCURRENCY),
499
576
  part: PART,
500
577
  concurrency: CONCURRENCY,
578
+ upload_budget: None,
501
579
  })
502
580
  }
503
581
 
@@ -517,6 +595,28 @@ impl S3Kv {
517
595
  self
518
596
  }
519
597
 
598
+ /// Limit the aggregate upload payload reservation to `bytes`.
599
+ ///
600
+ /// Calls sharing this store (including through `Arc`) share the budget.
601
+ /// An upload reserves [`UploadPlan::buffer_bytes`] before reading its
602
+ /// source, allocating parts or sending requests, and keeps the reservation
603
+ /// until all its requests and cleanup finish. Insufficient capacity fails
604
+ /// immediately with [`Error::UploadBuffer`]; it does not wait. Zero admits
605
+ /// only empty uploads. No budget is set by default.
606
+ ///
607
+ /// This bounds upload part payloads, including parts held for retries.
608
+ /// Caller-owned sources, read buffers, allocator overhead, HTTP/TLS buffers
609
+ /// and request metadata are additional memory; this is not an RSS limit.
610
+ /// Already owned values still reserve their planned upload payload, even
611
+ /// though their parts share the caller's allocation.
612
+ pub fn upload_buffer_bytes(mut self, bytes: u64) -> Self {
613
+ self.upload_budget = Some(UploadBudget {
614
+ limit: bytes,
615
+ used: AtomicU64::new(0),
616
+ });
617
+ self
618
+ }
619
+
520
620
  /// A store for `location`, configured as the AWS SDKs are.
521
621
  ///
522
622
  /// - Profile: `options.profile`, else `AWS_PROFILE`, else `default`.
@@ -12,10 +12,43 @@ use rusty_s3::{S3Action, actions::CreateMultipartUpload};
12
12
  use sha2::{Digest, Sha256};
13
13
 
14
14
  use crate::{
15
- EXPIRY, Error, IoSnafu, MAX_ERROR, MAX_PARTS, MissingFieldSnafu, S3Kv, body, status_error,
16
- timeout,
15
+ EXPIRY, Error, IoSnafu, MAX_ERROR, MAX_OBJECT, MAX_PART, MAX_PARTS, MAX_SINGLE_PUT, MIN_PART,
16
+ MissingFieldSnafu, S3Kv, UploadAllocationSnafu, body, status_error, timeout,
17
17
  };
18
18
 
19
+ /// The checked geometry and upload payload reservation for a value.
20
+ ///
21
+ /// Produced by [`S3Kv::upload_plan`], which is also used to admit writes.
22
+ /// It does not allocate payloads, reserve budget or send requests.
23
+ #[derive(Clone, Copy, Debug, PartialEq, Eq)]
24
+ pub struct UploadPlan {
25
+ /// The largest part payload, or zero for an empty value.
26
+ pub part_bytes: u64,
27
+ /// The number of parts, including one for a single PUT.
28
+ pub parts: u16,
29
+ /// The maximum payload held by this upload's concurrent workers and reader.
30
+ ///
31
+ /// Excludes source, allocator, HTTP/TLS and metadata overhead; see
32
+ /// [`S3Kv::upload_buffer_bytes`].
33
+ pub buffer_bytes: u64,
34
+ /// Whether the value uses multipart upload, rather than one PUT.
35
+ pub multipart: bool,
36
+ }
37
+
38
+ /// Reject unsupported quantities before any allocation or upload request.
39
+ fn within(field: &'static str, value: u64, minimum: u64, maximum: u64) -> Result<(), Error> {
40
+ if (minimum..=maximum).contains(&value) {
41
+ Ok(())
42
+ } else {
43
+ Err(Error::UploadLimit {
44
+ field,
45
+ value,
46
+ minimum,
47
+ maximum,
48
+ })
49
+ }
50
+ }
51
+
19
52
  /// The header carrying the base64 SHA-256 of a PUT body, signed with the URL.
20
53
  const CHECKSUM: &str = "x-amz-checksum-sha256";
21
54
 
@@ -53,6 +86,46 @@ fn escape(text: &str) -> String {
53
86
  }
54
87
 
55
88
  impl S3Kv {
89
+ /// Plan a write of `len` bytes without reading, allocating or sending.
90
+ ///
91
+ /// Checks S3's 10,000-part limit, 5 GiB maximum part, 5 MiB minimum
92
+ /// non-final part, 5 GB single-PUT limit and this platform's allocation
93
+ /// limit. Parts grow beyond the configured size when needed. An invalid
94
+ /// length or part size returns [`Error::UploadLimit`].
95
+ ///
96
+ /// The reservation is the smaller of the value's length and the part
97
+ /// size times the number of workers. It covers the reader's next part
98
+ /// as well as in-flight parts: a worker slot is available before the
99
+ /// next part is read. This does not reserve capacity or check current
100
+ /// availability; the write does that atomically before doing any I/O.
101
+ pub fn upload_plan(&self, len: u64) -> Result<UploadPlan, Error> {
102
+ within("object length", len, 0, MAX_OBJECT)?;
103
+ // A small configured part remains useful for ranged reads and tiny
104
+ // single PUTs, but an oversized one is never a supported upload setting.
105
+ if self.part > MAX_PART {
106
+ within("part size", self.part, MIN_PART, MAX_PART)?;
107
+ }
108
+ let part = self.part_for(len);
109
+ let multipart = len > part || len > MAX_SINGLE_PUT;
110
+ if multipart {
111
+ within("part size", part, MIN_PART, MAX_PART)?;
112
+ }
113
+ let parts = if multipart { len.div_ceil(part) } else { 1 };
114
+ within("part count", parts, 1, MAX_PARTS)?;
115
+ let part_bytes = len.min(part);
116
+ // Vec allocations must fit isize, a stronger condition than usize.
117
+ within("part allocation", part_bytes, 0, isize::MAX as u64)?;
118
+ let workers = self.concurrency.min(parts as usize) as u64;
119
+ // Validated S3 geometry bounds this product by MAX_OBJECT.
120
+ let buffer_bytes = len.min(workers * part_bytes);
121
+ Ok(UploadPlan {
122
+ part_bytes,
123
+ parts: parts as u16,
124
+ buffer_bytes,
125
+ multipart,
126
+ })
127
+ }
128
+
56
129
  /// Store `value` under `key`, as `write` says; whether it stored.
57
130
  pub(crate) fn write(&self, key: &[u8], value: Bytes, write: Write) -> Result<bool, Error> {
58
131
  let mut offset = 0;
@@ -73,7 +146,10 @@ impl S3Kv {
73
146
  write: Write,
74
147
  ) -> Result<bool, Error> {
75
148
  self.upload(key, len, write, |len| {
76
- let mut part = vec![0; len];
149
+ let mut part = Vec::new();
150
+ part.try_reserve_exact(len)
151
+ .context(UploadAllocationSnafu { bytes: len as u64 })?;
152
+ part.resize(len, 0);
77
153
  reader.read_exact(&mut part).context(IoSnafu)?;
78
154
  Ok(part.into())
79
155
  })
@@ -91,14 +167,22 @@ impl S3Kv {
91
167
  write: Write,
92
168
  mut next: impl FnMut(usize) -> Result<Bytes, Error>,
93
169
  ) -> Result<bool, Error> {
170
+ let plan = self.upload_plan(len)?;
171
+ // This guard outlives in_order's scoped worker joins and the abort
172
+ // request, on success, error and unwinding alike.
173
+ let _reservation = self
174
+ .upload_budget
175
+ .as_ref()
176
+ .map(|budget| budget.reserve(plan.buffer_bytes))
177
+ .transpose()?;
94
178
  let object = self.object(key)?;
95
- let part = self.part_for(len);
96
- if len <= part {
97
- return self.put_object(&object, next(len as usize)?, write);
179
+ if !plan.multipart {
180
+ return self.put_object(&object, next(plan.part_bytes as usize)?, write);
98
181
  }
99
182
  let upload_id = self.create(&object)?;
100
- // At most MAX_PARTS, so the count and part numbers fit.
101
- let count = len.div_ceil(part);
183
+ // The planner checked every allocation and the part count above.
184
+ let part = plan.part_bytes;
185
+ let count = u64::from(plan.parts);
102
186
  let mut parts = Vec::with_capacity(count as usize);
103
187
  let jobs = (0..count).map(|index| {
104
188
  let size = part.min(len - index * part);