covalence 0.1.0.dev2__tar.gz → 0.1.0.dev4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/Cargo.lock +19 -19
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/Cargo.toml +17 -17
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/PKG-INFO +68 -10
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/README.md +67 -9
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/kv/src/lib.rs +28 -10
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/kv/tests/kv.rs +59 -6
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/trait/src/lib.rs +8 -4
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/memory/src/lib.rs +35 -2
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/memory/tests/memory.rs +30 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/s3/src/lib.rs +36 -2
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/s3/src/write.rs +83 -34
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/s3/tests/kv.rs +79 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/trait/src/lib.rs +40 -0
- covalence-0.1.0.dev4/crates/kv/trait/tests/atomic.rs +52 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/README.md +67 -9
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/src/lib.rs +243 -2
- covalence-0.1.0.dev4/crates/python/src/streams/download.rs +295 -0
- covalence-0.1.0.dev4/crates/python/src/streams/mod.rs +20 -0
- covalence-0.1.0.dev4/crates/python/src/streams/upload/tests.rs +241 -0
- covalence-0.1.0.dev4/crates/python/src/streams/upload.rs +215 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/tests/test_api.py +172 -4
- covalence-0.1.0.dev4/crates/python/tests/test_bounded_io.py +935 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/python/covalence/THIRD-PARTY-LICENSES.md +17 -17
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/python/covalence/_native.pyi +90 -3
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/blob/trait/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/blob/trait/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/blob/trait/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/blob/trait/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/blob/trait/README.md +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/blob/trait/src/lib.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/blob/trait/tests/blob.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/blob/trait/tests/common/mod.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/blob/trait/tests/fact.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/kv/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/kv/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/kv/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/kv/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/memory/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/memory/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/memory/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/memory/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/memory/src/lib.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/memory/tests/memory.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/sqlite/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/sqlite/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/sqlite/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/sqlite/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/sqlite/src/lib.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/sqlite/tests/sqlite.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/trait/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/trait/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/trait/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/trait/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/cas/trait/tests/trait.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/memory/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/memory/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/memory/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/memory/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/s3/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/s3/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/s3/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/s3/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/s3/src/read.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/s3/tests/aws.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/trait/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/trait/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/trait/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/kv/trait/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/README.md +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/src/dasl.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/src/fact.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/src/git.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/src/lib.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/src/multicodec.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/src/o256.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/src/storage.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/tests/cid.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/tests/fact.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/tests/git.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/tests/o256.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/tests/shattered/1.bin +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/tests/shattered/2.bin +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/cid/tests/shattered/README.md +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/README.md +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/src/drisl.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/src/event.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/src/json.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/src/lib.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/codec.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/events.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fact.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/LICENSE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/README.md +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/cid.json +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/concat.json +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/floats.json +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/indefinite.json +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/integer_range.json +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/map_keys.json +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/numeric_reduction.json +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/short_form.json +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/simple.json +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/tags.json +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures/utf8.json +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/fixtures.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/json.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/vectors/README.md +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/vectors/cid-vectors.jsonl +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/vectors/dasl-vectors.jsonl +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/dasl/tests/vectors.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/data/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/data/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/data/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/data/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/data/src/lib.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/error/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/error/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/error/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/error/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/error/src/lib.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/python/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/python/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/python/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/python/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/python/src/lib.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/rand/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/rand/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/rand/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/rand/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/rand/src/lib.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/serde/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/serde/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/serde/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/serde/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/serde/src/lib.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/sqlite/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/sqlite/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/sqlite/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/sqlite/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/lib/sqlite/src/lib.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/Cargo.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/about.md.hbs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/about.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/src/errors.rs +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/tests/test_docs.py +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/tests/test_package.py +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/tests/test_stubs.py +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/crates/python/tests/typed_usage.py +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/pyproject.toml +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/python/covalence/__init__.py +0 -0
- {covalence-0.1.0.dev2 → covalence-0.1.0.dev4}/python/covalence/py.typed +0 -0
|
@@ -779,7 +779,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
|
|
779
779
|
|
|
780
780
|
[[package]]
|
|
781
781
|
name = "covalence-blob-trait"
|
|
782
|
-
version = "0.1.0-dev.
|
|
782
|
+
version = "0.1.0-dev.4"
|
|
783
783
|
dependencies = [
|
|
784
784
|
"covalence-lib-cid",
|
|
785
785
|
"covalence-lib-data",
|
|
@@ -788,7 +788,7 @@ dependencies = [
|
|
|
788
788
|
|
|
789
789
|
[[package]]
|
|
790
790
|
name = "covalence-cas-kv"
|
|
791
|
-
version = "0.1.0-dev.
|
|
791
|
+
version = "0.1.0-dev.4"
|
|
792
792
|
dependencies = [
|
|
793
793
|
"blake3",
|
|
794
794
|
"covalence-cas-trait",
|
|
@@ -810,7 +810,7 @@ dependencies = [
|
|
|
810
810
|
|
|
811
811
|
[[package]]
|
|
812
812
|
name = "covalence-cas-memory"
|
|
813
|
-
version = "0.1.0-dev.
|
|
813
|
+
version = "0.1.0-dev.4"
|
|
814
814
|
dependencies = [
|
|
815
815
|
"covalence-cas-trait",
|
|
816
816
|
"covalence-lib-cid",
|
|
@@ -821,7 +821,7 @@ dependencies = [
|
|
|
821
821
|
|
|
822
822
|
[[package]]
|
|
823
823
|
name = "covalence-cas-sqlite"
|
|
824
|
-
version = "0.1.0-dev.
|
|
824
|
+
version = "0.1.0-dev.4"
|
|
825
825
|
dependencies = [
|
|
826
826
|
"covalence-cas-trait",
|
|
827
827
|
"covalence-lib-cid",
|
|
@@ -836,7 +836,7 @@ dependencies = [
|
|
|
836
836
|
|
|
837
837
|
[[package]]
|
|
838
838
|
name = "covalence-cas-trait"
|
|
839
|
-
version = "0.1.0-dev.
|
|
839
|
+
version = "0.1.0-dev.4"
|
|
840
840
|
dependencies = [
|
|
841
841
|
"covalence-lib-cid",
|
|
842
842
|
"covalence-lib-dasl",
|
|
@@ -847,7 +847,7 @@ dependencies = [
|
|
|
847
847
|
|
|
848
848
|
[[package]]
|
|
849
849
|
name = "covalence-kv-memory"
|
|
850
|
-
version = "0.1.0-dev.
|
|
850
|
+
version = "0.1.0-dev.4"
|
|
851
851
|
dependencies = [
|
|
852
852
|
"covalence-kv-trait",
|
|
853
853
|
"covalence-lib-dasl",
|
|
@@ -857,7 +857,7 @@ dependencies = [
|
|
|
857
857
|
|
|
858
858
|
[[package]]
|
|
859
859
|
name = "covalence-kv-s3"
|
|
860
|
-
version = "0.1.0-dev.
|
|
860
|
+
version = "0.1.0-dev.4"
|
|
861
861
|
dependencies = [
|
|
862
862
|
"aws-config",
|
|
863
863
|
"aws-credential-types",
|
|
@@ -881,7 +881,7 @@ dependencies = [
|
|
|
881
881
|
|
|
882
882
|
[[package]]
|
|
883
883
|
name = "covalence-kv-trait"
|
|
884
|
-
version = "0.1.0-dev.
|
|
884
|
+
version = "0.1.0-dev.4"
|
|
885
885
|
dependencies = [
|
|
886
886
|
"covalence-lib-dasl",
|
|
887
887
|
"covalence-lib-data",
|
|
@@ -890,7 +890,7 @@ dependencies = [
|
|
|
890
890
|
|
|
891
891
|
[[package]]
|
|
892
892
|
name = "covalence-lib-cid"
|
|
893
|
-
version = "0.1.0-dev.
|
|
893
|
+
version = "0.1.0-dev.4"
|
|
894
894
|
dependencies = [
|
|
895
895
|
"blake3",
|
|
896
896
|
"covalence-lib-data",
|
|
@@ -905,14 +905,14 @@ dependencies = [
|
|
|
905
905
|
|
|
906
906
|
[[package]]
|
|
907
907
|
name = "covalence-lib-cli"
|
|
908
|
-
version = "0.1.0-dev.
|
|
908
|
+
version = "0.1.0-dev.4"
|
|
909
909
|
dependencies = [
|
|
910
910
|
"clap",
|
|
911
911
|
]
|
|
912
912
|
|
|
913
913
|
[[package]]
|
|
914
914
|
name = "covalence-lib-dasl"
|
|
915
|
-
version = "0.1.0-dev.
|
|
915
|
+
version = "0.1.0-dev.4"
|
|
916
916
|
dependencies = [
|
|
917
917
|
"blake3",
|
|
918
918
|
"covalence-lib-cid",
|
|
@@ -927,7 +927,7 @@ dependencies = [
|
|
|
927
927
|
|
|
928
928
|
[[package]]
|
|
929
929
|
name = "covalence-lib-data"
|
|
930
|
-
version = "0.1.0-dev.
|
|
930
|
+
version = "0.1.0-dev.4"
|
|
931
931
|
dependencies = [
|
|
932
932
|
"arrayvec",
|
|
933
933
|
"bytes",
|
|
@@ -939,7 +939,7 @@ dependencies = [
|
|
|
939
939
|
|
|
940
940
|
[[package]]
|
|
941
941
|
name = "covalence-lib-error"
|
|
942
|
-
version = "0.1.0-dev.
|
|
942
|
+
version = "0.1.0-dev.4"
|
|
943
943
|
dependencies = [
|
|
944
944
|
"miette",
|
|
945
945
|
"snafu",
|
|
@@ -947,14 +947,14 @@ dependencies = [
|
|
|
947
947
|
|
|
948
948
|
[[package]]
|
|
949
949
|
name = "covalence-lib-python"
|
|
950
|
-
version = "0.1.0-dev.
|
|
950
|
+
version = "0.1.0-dev.4"
|
|
951
951
|
dependencies = [
|
|
952
952
|
"pyo3",
|
|
953
953
|
]
|
|
954
954
|
|
|
955
955
|
[[package]]
|
|
956
956
|
name = "covalence-lib-rand"
|
|
957
|
-
version = "0.1.0-dev.
|
|
957
|
+
version = "0.1.0-dev.4"
|
|
958
958
|
dependencies = [
|
|
959
959
|
"getrandom 0.3.4",
|
|
960
960
|
"rand",
|
|
@@ -962,21 +962,21 @@ dependencies = [
|
|
|
962
962
|
|
|
963
963
|
[[package]]
|
|
964
964
|
name = "covalence-lib-serde"
|
|
965
|
-
version = "0.1.0-dev.
|
|
965
|
+
version = "0.1.0-dev.4"
|
|
966
966
|
dependencies = [
|
|
967
967
|
"serde",
|
|
968
968
|
]
|
|
969
969
|
|
|
970
970
|
[[package]]
|
|
971
971
|
name = "covalence-lib-sqlite"
|
|
972
|
-
version = "0.1.0-dev.
|
|
972
|
+
version = "0.1.0-dev.4"
|
|
973
973
|
dependencies = [
|
|
974
974
|
"rusqlite",
|
|
975
975
|
]
|
|
976
976
|
|
|
977
977
|
[[package]]
|
|
978
978
|
name = "covalence-python"
|
|
979
|
-
version = "0.1.0-dev.
|
|
979
|
+
version = "0.1.0-dev.4"
|
|
980
980
|
dependencies = [
|
|
981
981
|
"blake3",
|
|
982
982
|
"covalence-blob-trait",
|
|
@@ -997,7 +997,7 @@ dependencies = [
|
|
|
997
997
|
|
|
998
998
|
[[package]]
|
|
999
999
|
name = "covalence-tools-glu"
|
|
1000
|
-
version = "0.1.0-dev.
|
|
1000
|
+
version = "0.1.0-dev.4"
|
|
1001
1001
|
dependencies = [
|
|
1002
1002
|
"covalence-lib-cli",
|
|
1003
1003
|
"covalence-lib-error",
|
|
@@ -3,7 +3,7 @@ members = ["crates/lib/error", "crates/lib/serde", "crates/lib/data", "crates/li
|
|
|
3
3
|
resolver = "3"
|
|
4
4
|
|
|
5
5
|
[workspace.package]
|
|
6
|
-
version = "0.1.0-dev.
|
|
6
|
+
version = "0.1.0-dev.4"
|
|
7
7
|
edition = "2024"
|
|
8
8
|
rust-version = "1.98"
|
|
9
9
|
license = "CC0-1.0 OR MIT OR Apache-2.0"
|
|
@@ -17,22 +17,22 @@ miette = "7.6.0"
|
|
|
17
17
|
sha2 = "0.11.0"
|
|
18
18
|
snafu = "0.9.2"
|
|
19
19
|
# Published crates, at the workspace version so the set moves together.
|
|
20
|
-
covalence-blob-trait = { path = "crates/blob/trait", version = "=0.1.0-dev.
|
|
21
|
-
covalence-cas-kv = { path = "crates/cas/kv", version = "=0.1.0-dev.
|
|
22
|
-
covalence-cas-memory = { path = "crates/cas/memory", version = "=0.1.0-dev.
|
|
23
|
-
covalence-cas-sqlite = { path = "crates/cas/sqlite", version = "=0.1.0-dev.
|
|
24
|
-
covalence-cas-trait = { path = "crates/cas/trait", version = "=0.1.0-dev.
|
|
25
|
-
covalence-kv-memory = { path = "crates/kv/memory", version = "=0.1.0-dev.
|
|
26
|
-
covalence-kv-s3 = { path = "crates/kv/s3", version = "=0.1.0-dev.
|
|
27
|
-
covalence-kv-trait = { path = "crates/kv/trait", version = "=0.1.0-dev.
|
|
28
|
-
covalence-lib-cid = { path = "crates/lib/cid", version = "=0.1.0-dev.
|
|
29
|
-
covalence-lib-cli = { path = "crates/lib/cli", version = "=0.1.0-dev.
|
|
30
|
-
covalence-lib-dasl = { path = "crates/lib/dasl", version = "=0.1.0-dev.
|
|
31
|
-
covalence-lib-data = { path = "crates/lib/data", version = "=0.1.0-dev.
|
|
32
|
-
covalence-lib-error = { path = "crates/lib/error", version = "=0.1.0-dev.
|
|
33
|
-
covalence-lib-rand = { path = "crates/lib/rand", version = "=0.1.0-dev.
|
|
34
|
-
covalence-lib-serde = { path = "crates/lib/serde", version = "=0.1.0-dev.
|
|
35
|
-
covalence-lib-sqlite = { path = "crates/lib/sqlite", version = "=0.1.0-dev.
|
|
20
|
+
covalence-blob-trait = { path = "crates/blob/trait", version = "=0.1.0-dev.4" }
|
|
21
|
+
covalence-cas-kv = { path = "crates/cas/kv", version = "=0.1.0-dev.4" }
|
|
22
|
+
covalence-cas-memory = { path = "crates/cas/memory", version = "=0.1.0-dev.4" }
|
|
23
|
+
covalence-cas-sqlite = { path = "crates/cas/sqlite", version = "=0.1.0-dev.4" }
|
|
24
|
+
covalence-cas-trait = { path = "crates/cas/trait", version = "=0.1.0-dev.4" }
|
|
25
|
+
covalence-kv-memory = { path = "crates/kv/memory", version = "=0.1.0-dev.4" }
|
|
26
|
+
covalence-kv-s3 = { path = "crates/kv/s3", version = "=0.1.0-dev.4" }
|
|
27
|
+
covalence-kv-trait = { path = "crates/kv/trait", version = "=0.1.0-dev.4" }
|
|
28
|
+
covalence-lib-cid = { path = "crates/lib/cid", version = "=0.1.0-dev.4" }
|
|
29
|
+
covalence-lib-cli = { path = "crates/lib/cli", version = "=0.1.0-dev.4" }
|
|
30
|
+
covalence-lib-dasl = { path = "crates/lib/dasl", version = "=0.1.0-dev.4" }
|
|
31
|
+
covalence-lib-data = { path = "crates/lib/data", version = "=0.1.0-dev.4" }
|
|
32
|
+
covalence-lib-error = { path = "crates/lib/error", version = "=0.1.0-dev.4" }
|
|
33
|
+
covalence-lib-rand = { path = "crates/lib/rand", version = "=0.1.0-dev.4" }
|
|
34
|
+
covalence-lib-serde = { path = "crates/lib/serde", version = "=0.1.0-dev.4" }
|
|
35
|
+
covalence-lib-sqlite = { path = "crates/lib/sqlite", version = "=0.1.0-dev.4" }
|
|
36
36
|
|
|
37
37
|
[workspace.lints.rust]
|
|
38
38
|
unsafe_code = "forbid"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: covalence
|
|
3
|
-
Version: 0.1.0.
|
|
3
|
+
Version: 0.1.0.dev4
|
|
4
4
|
License-File: LICENSE-APACHE
|
|
5
5
|
License-File: LICENSE-CC0
|
|
6
6
|
License-File: LICENSE-MIT
|
|
@@ -35,7 +35,7 @@ True
|
|
|
35
35
|
## Install
|
|
36
36
|
|
|
37
37
|
```sh
|
|
38
|
-
pip install covalence==0.1.0.
|
|
38
|
+
pip install covalence==0.1.0.dev4
|
|
39
39
|
```
|
|
40
40
|
|
|
41
41
|
- Pre-releases need an exact pin like this one, or `pip install --pre`.
|
|
@@ -72,6 +72,11 @@ pip install covalence==0.1.0.dev2
|
|
|
72
72
|
- `Cid.of` hashes with `"blake3"` and `"raw"` by default.
|
|
73
73
|
- `Cid.of_file(path)` is the raw BLAKE3 `Cid` of a file, as `Cas.put_file`
|
|
74
74
|
returns it, without storing it or reading the file into memory.
|
|
75
|
+
- `Cid.of_fileobj(file, max_bytes=..., expected_length=...)` hashes an
|
|
76
|
+
already-open, seekable binary file object once from offset zero, without
|
|
77
|
+
storing it or holding it in memory. It uses the same bounded reader as
|
|
78
|
+
`Cas.put_fileobj`, and leaves the caller's source open at EOF on success.
|
|
79
|
+
Keep that source unchanged and exclusively owned until it returns.
|
|
75
80
|
|
|
76
81
|
## Values
|
|
77
82
|
|
|
@@ -138,6 +143,9 @@ pip install covalence==0.1.0.dev2
|
|
|
138
143
|
`prefix/blake3/<digest>/bytes`.
|
|
139
144
|
- `put` returns the raw BLAKE3 `Cid`, after checking that the store kept the
|
|
140
145
|
data under it.
|
|
146
|
+
- `Cas.kv` never replaces a block its `Kv` already holds: on `Kv.s3` each
|
|
147
|
+
write is conditional (`If-None-Match: *`), so concurrent writers of one
|
|
148
|
+
block race safely and the first one's bytes stay.
|
|
141
149
|
- `put_value(value)` stores the DRISL encoding of a `Value` (or what
|
|
142
150
|
`Value(...)` takes) and returns its DRISL CID, which is `value.cid()`.
|
|
143
151
|
- `get` returns the bytes and `get_fact` a `BlobFact`, both checked against
|
|
@@ -156,16 +164,44 @@ pip install covalence==0.1.0.dev2
|
|
|
156
164
|
- Ranges are clipped at the end.
|
|
157
165
|
- Stores without native ranges fetch the whole value and slice it.
|
|
158
166
|
|
|
159
|
-
### Files
|
|
167
|
+
### Files and streams
|
|
160
168
|
|
|
161
169
|
- `Cas.put_file(path)` stores a file and returns its raw BLAKE3 `Cid`, as
|
|
162
170
|
`put` does.
|
|
171
|
+
- `Cas.put_from(cid, chunks, length)` stores a block from an iterable of
|
|
172
|
+
bytes, such as a request body, and returns its raw BLAKE3 `Cid`. Before a
|
|
173
|
+
new block is stored, its bytes must match `length` and the BLAKE3 `cid`,
|
|
174
|
+
and `chunks` must end there. A store may return an existing block without
|
|
175
|
+
consuming all the input; unread bytes and source errors are not checked.
|
|
176
|
+
While consuming input, a hash mismatch (`IntegrityError`), too few or too
|
|
177
|
+
many bytes (`ValueError`), or a source exception publish no new block.
|
|
178
|
+
Source exceptions propagate unchanged unless cleanup also fails, in which
|
|
179
|
+
case `StoreError` reports that failure with the source as its `__cause__`.
|
|
163
180
|
- `Cas.get_file(cid, path)` writes the block to a file and returns its
|
|
164
181
|
length, or `None` if missing.
|
|
165
|
-
- `
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
182
|
+
- `Cas.put_fileobj(file, max_bytes=..., expected_length=...)` stores a
|
|
183
|
+
caller-owned, seekable binary file object. It reads from offset zero twice,
|
|
184
|
+
checks the complete source on both passes even when deduplicating, and
|
|
185
|
+
leaves it open at EOF on success. The caller must keep the source unchanged
|
|
186
|
+
and exclusively owned until it returns; the method creates no snapshot.
|
|
187
|
+
`max_bytes` caps the accepted source length; `expected_length`, if given,
|
|
188
|
+
must match it exactly. A length violation raises `ValueError` before a new
|
|
189
|
+
block is published. A pass may read one extra byte to check the end.
|
|
190
|
+
- `Cas.get_to(cid, writer, max_bytes=..., expected_length=...)` writes to a
|
|
191
|
+
caller-owned binary file object and returns the length, or `None` if missing.
|
|
192
|
+
It checks all bytes against the CID, caps the bytes the writer accepts, and
|
|
193
|
+
checks the optional exact length. It handles short writes and never seeks,
|
|
194
|
+
flushes or closes the writer. A failure may leave partial or unchecked bytes
|
|
195
|
+
there: write to private staging and publish only after a successful return.
|
|
196
|
+
- These limits govern source length and output bytes, not the memory a backend
|
|
197
|
+
uses before reading or writing. Python callback transfers are at most 1 MiB.
|
|
198
|
+
- `put_file`, `put_fileobj`, `put_from`, `get_file` and `get_to` stream on `Cas.kv` over `Kv.s3`,
|
|
199
|
+
holding up to 16 parts of 8 MiB (128 MiB) at a time, or as many as
|
|
200
|
+
`Kv.s3(..., concurrency=n)` says; blocks over 78 GiB go up in larger parts.
|
|
201
|
+
Other stores, including `Cas.kv(Kv.memory())`, hold the block in memory.
|
|
202
|
+
- A create-only write can reuse a block without checking its stored bytes.
|
|
203
|
+
Returning a CID does not freshly verify an existing block: use a checked
|
|
204
|
+
read, such as `get_to`, when that guarantee is needed.
|
|
169
205
|
- `put_file` reads the file twice, to hash it and to store it, and stores the
|
|
170
206
|
bytes it hashed. If they changed in between it raises `IntegrityError`,
|
|
171
207
|
and if the file got shorter `OSError`, storing nothing either way. Bytes
|
|
@@ -176,6 +212,28 @@ pip install covalence==0.1.0.dev2
|
|
|
176
212
|
was. The new file has the permissions of the file it replaces, or those
|
|
177
213
|
the umask gives a new file.
|
|
178
214
|
|
|
215
|
+
Use an already-open file without exposing its path:
|
|
216
|
+
|
|
217
|
+
```pycon
|
|
218
|
+
>>> import io
|
|
219
|
+
>>> source = io.BytesIO(b"hello")
|
|
220
|
+
>>> cid = cas.put_fileobj(source, max_bytes=5, expected_length=5)
|
|
221
|
+
>>> sink = io.BytesIO()
|
|
222
|
+
>>> cas.get_to(cid, sink, max_bytes=5, expected_length=5)
|
|
223
|
+
5
|
|
224
|
+
>>> sink.getvalue()
|
|
225
|
+
b'hello'
|
|
226
|
+
>>> Cid.of_fileobj(sink, max_bytes=5, expected_length=5) == cid
|
|
227
|
+
True
|
|
228
|
+
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
An integer file descriptor can be wrapped with `os.fdopen(os.dup(fd), "wb",
|
|
232
|
+
buffering=0)` for downloads, or `"rb"` for uploads. Close that wrapper when
|
|
233
|
+
finished; the original descriptor remains open. Duplicates share a file
|
|
234
|
+
position, so the caller must also keep the original descriptor exclusively
|
|
235
|
+
owned during the operation.
|
|
236
|
+
|
|
179
237
|
## Blobs and facts
|
|
180
238
|
|
|
181
239
|
- `Blob` is bytes read whole or by range:
|
|
@@ -247,16 +305,16 @@ stub; this package writes its stub by hand and checks it against `__doc__`:
|
|
|
247
305
|
later) for manylinux_2_28 x86_64 and aarch64 and macOS arm64, and an sdist,
|
|
248
306
|
and runs `tests/` against each wheel. Pull requests and pushes keep them as
|
|
249
307
|
workflow artifacts. A release tag must match the workspace version, such as
|
|
250
|
-
`0.1.0-dev.
|
|
308
|
+
`0.1.0-dev.4`; the files carry its PEP 440 form, `0.1.0.dev4`:
|
|
251
309
|
|
|
252
310
|
- `v<version>` attaches them to that GitHub release. In the public
|
|
253
311
|
`hiloopai/covalence` repository it also publishes them to PyPI, and
|
|
254
312
|
`.github/workflows/crates.yml` publishes the crates to crates.io.
|
|
255
313
|
- `python-v<version>` only attaches them to the GitHub release.
|
|
256
314
|
|
|
257
|
-
To publish `0.1.0-dev.
|
|
315
|
+
To publish `0.1.0-dev.4`:
|
|
258
316
|
|
|
259
317
|
```sh
|
|
260
|
-
git tag v0.1.0-dev.
|
|
318
|
+
git tag v0.1.0-dev.4 && git push origin v0.1.0-dev.4
|
|
261
319
|
```
|
|
262
320
|
|
|
@@ -21,7 +21,7 @@ True
|
|
|
21
21
|
## Install
|
|
22
22
|
|
|
23
23
|
```sh
|
|
24
|
-
pip install covalence==0.1.0.
|
|
24
|
+
pip install covalence==0.1.0.dev4
|
|
25
25
|
```
|
|
26
26
|
|
|
27
27
|
- Pre-releases need an exact pin like this one, or `pip install --pre`.
|
|
@@ -58,6 +58,11 @@ pip install covalence==0.1.0.dev2
|
|
|
58
58
|
- `Cid.of` hashes with `"blake3"` and `"raw"` by default.
|
|
59
59
|
- `Cid.of_file(path)` is the raw BLAKE3 `Cid` of a file, as `Cas.put_file`
|
|
60
60
|
returns it, without storing it or reading the file into memory.
|
|
61
|
+
- `Cid.of_fileobj(file, max_bytes=..., expected_length=...)` hashes an
|
|
62
|
+
already-open, seekable binary file object once from offset zero, without
|
|
63
|
+
storing it or holding it in memory. It uses the same bounded reader as
|
|
64
|
+
`Cas.put_fileobj`, and leaves the caller's source open at EOF on success.
|
|
65
|
+
Keep that source unchanged and exclusively owned until it returns.
|
|
61
66
|
|
|
62
67
|
## Values
|
|
63
68
|
|
|
@@ -124,6 +129,9 @@ pip install covalence==0.1.0.dev2
|
|
|
124
129
|
`prefix/blake3/<digest>/bytes`.
|
|
125
130
|
- `put` returns the raw BLAKE3 `Cid`, after checking that the store kept the
|
|
126
131
|
data under it.
|
|
132
|
+
- `Cas.kv` never replaces a block its `Kv` already holds: on `Kv.s3` each
|
|
133
|
+
write is conditional (`If-None-Match: *`), so concurrent writers of one
|
|
134
|
+
block race safely and the first one's bytes stay.
|
|
127
135
|
- `put_value(value)` stores the DRISL encoding of a `Value` (or what
|
|
128
136
|
`Value(...)` takes) and returns its DRISL CID, which is `value.cid()`.
|
|
129
137
|
- `get` returns the bytes and `get_fact` a `BlobFact`, both checked against
|
|
@@ -142,16 +150,44 @@ pip install covalence==0.1.0.dev2
|
|
|
142
150
|
- Ranges are clipped at the end.
|
|
143
151
|
- Stores without native ranges fetch the whole value and slice it.
|
|
144
152
|
|
|
145
|
-
### Files
|
|
153
|
+
### Files and streams
|
|
146
154
|
|
|
147
155
|
- `Cas.put_file(path)` stores a file and returns its raw BLAKE3 `Cid`, as
|
|
148
156
|
`put` does.
|
|
157
|
+
- `Cas.put_from(cid, chunks, length)` stores a block from an iterable of
|
|
158
|
+
bytes, such as a request body, and returns its raw BLAKE3 `Cid`. Before a
|
|
159
|
+
new block is stored, its bytes must match `length` and the BLAKE3 `cid`,
|
|
160
|
+
and `chunks` must end there. A store may return an existing block without
|
|
161
|
+
consuming all the input; unread bytes and source errors are not checked.
|
|
162
|
+
While consuming input, a hash mismatch (`IntegrityError`), too few or too
|
|
163
|
+
many bytes (`ValueError`), or a source exception publish no new block.
|
|
164
|
+
Source exceptions propagate unchanged unless cleanup also fails, in which
|
|
165
|
+
case `StoreError` reports that failure with the source as its `__cause__`.
|
|
149
166
|
- `Cas.get_file(cid, path)` writes the block to a file and returns its
|
|
150
167
|
length, or `None` if missing.
|
|
151
|
-
- `
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
168
|
+
- `Cas.put_fileobj(file, max_bytes=..., expected_length=...)` stores a
|
|
169
|
+
caller-owned, seekable binary file object. It reads from offset zero twice,
|
|
170
|
+
checks the complete source on both passes even when deduplicating, and
|
|
171
|
+
leaves it open at EOF on success. The caller must keep the source unchanged
|
|
172
|
+
and exclusively owned until it returns; the method creates no snapshot.
|
|
173
|
+
`max_bytes` caps the accepted source length; `expected_length`, if given,
|
|
174
|
+
must match it exactly. A length violation raises `ValueError` before a new
|
|
175
|
+
block is published. A pass may read one extra byte to check the end.
|
|
176
|
+
- `Cas.get_to(cid, writer, max_bytes=..., expected_length=...)` writes to a
|
|
177
|
+
caller-owned binary file object and returns the length, or `None` if missing.
|
|
178
|
+
It checks all bytes against the CID, caps the bytes the writer accepts, and
|
|
179
|
+
checks the optional exact length. It handles short writes and never seeks,
|
|
180
|
+
flushes or closes the writer. A failure may leave partial or unchecked bytes
|
|
181
|
+
there: write to private staging and publish only after a successful return.
|
|
182
|
+
- These limits govern source length and output bytes, not the memory a backend
|
|
183
|
+
uses before reading or writing. Python callback transfers are at most 1 MiB.
|
|
184
|
+
- `put_file`, `put_fileobj`, `put_from`, `get_file` and `get_to` stream on `Cas.kv` over `Kv.s3`,
|
|
185
|
+
holding up to 16 parts of 8 MiB (128 MiB) at a time, or as many as
|
|
186
|
+
`Kv.s3(..., concurrency=n)` says; blocks over 78 GiB go up in larger parts.
|
|
187
|
+
Other stores, including `Cas.kv(Kv.memory())`, hold the block in memory.
|
|
188
|
+
- A create-only write can reuse a block without checking its stored bytes.
|
|
189
|
+
Returning a CID does not freshly verify an existing block: use a checked
|
|
190
|
+
read, such as `get_to`, when that guarantee is needed.
|
|
155
191
|
- `put_file` reads the file twice, to hash it and to store it, and stores the
|
|
156
192
|
bytes it hashed. If they changed in between it raises `IntegrityError`,
|
|
157
193
|
and if the file got shorter `OSError`, storing nothing either way. Bytes
|
|
@@ -162,6 +198,28 @@ pip install covalence==0.1.0.dev2
|
|
|
162
198
|
was. The new file has the permissions of the file it replaces, or those
|
|
163
199
|
the umask gives a new file.
|
|
164
200
|
|
|
201
|
+
Use an already-open file without exposing its path:
|
|
202
|
+
|
|
203
|
+
```pycon
|
|
204
|
+
>>> import io
|
|
205
|
+
>>> source = io.BytesIO(b"hello")
|
|
206
|
+
>>> cid = cas.put_fileobj(source, max_bytes=5, expected_length=5)
|
|
207
|
+
>>> sink = io.BytesIO()
|
|
208
|
+
>>> cas.get_to(cid, sink, max_bytes=5, expected_length=5)
|
|
209
|
+
5
|
|
210
|
+
>>> sink.getvalue()
|
|
211
|
+
b'hello'
|
|
212
|
+
>>> Cid.of_fileobj(sink, max_bytes=5, expected_length=5) == cid
|
|
213
|
+
True
|
|
214
|
+
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
An integer file descriptor can be wrapped with `os.fdopen(os.dup(fd), "wb",
|
|
218
|
+
buffering=0)` for downloads, or `"rb"` for uploads. Close that wrapper when
|
|
219
|
+
finished; the original descriptor remains open. Duplicates share a file
|
|
220
|
+
position, so the caller must also keep the original descriptor exclusively
|
|
221
|
+
owned during the operation.
|
|
222
|
+
|
|
165
223
|
## Blobs and facts
|
|
166
224
|
|
|
167
225
|
- `Blob` is bytes read whole or by range:
|
|
@@ -233,15 +291,15 @@ stub; this package writes its stub by hand and checks it against `__doc__`:
|
|
|
233
291
|
later) for manylinux_2_28 x86_64 and aarch64 and macOS arm64, and an sdist,
|
|
234
292
|
and runs `tests/` against each wheel. Pull requests and pushes keep them as
|
|
235
293
|
workflow artifacts. A release tag must match the workspace version, such as
|
|
236
|
-
`0.1.0-dev.
|
|
294
|
+
`0.1.0-dev.4`; the files carry its PEP 440 form, `0.1.0.dev4`:
|
|
237
295
|
|
|
238
296
|
- `v<version>` attaches them to that GitHub release. In the public
|
|
239
297
|
`hiloopai/covalence` repository it also publishes them to PyPI, and
|
|
240
298
|
`.github/workflows/crates.yml` publishes the crates to crates.io.
|
|
241
299
|
- `python-v<version>` only attaches them to the GitHub release.
|
|
242
300
|
|
|
243
|
-
To publish `0.1.0-dev.
|
|
301
|
+
To publish `0.1.0-dev.4`:
|
|
244
302
|
|
|
245
303
|
```sh
|
|
246
|
-
git tag v0.1.0-dev.
|
|
304
|
+
git tag v0.1.0-dev.4 && git push origin v0.1.0-dev.4
|
|
247
305
|
```
|
|
@@ -46,13 +46,16 @@ use data_encoding::BASE32_NOPAD;
|
|
|
46
46
|
/// `get_fact` and `aget_fact` check the bytes against the CID and fail with
|
|
47
47
|
/// [`Error::Check`] if they do not match. `get_unchecked`, sizes and ranges
|
|
48
48
|
/// are not checked.
|
|
49
|
-
/// - `has` asks the key-value store whether the key exists
|
|
50
|
-
///
|
|
51
|
-
/// - `put`
|
|
52
|
-
///
|
|
53
|
-
///
|
|
49
|
+
/// - `has` asks the key-value store whether the key exists, without reading
|
|
50
|
+
/// the bytes.
|
|
51
|
+
/// - `put`, `aput` and `put_from` never replace a block: they store through
|
|
52
|
+
/// the key-value store's [`Kv::put_new`], [`Kv::aput_new`] and
|
|
53
|
+
/// [`Kv::put_new_from`], so concurrent writers of one block race safely.
|
|
54
|
+
/// Writes fail if the backend does not implement atomic create-only writes.
|
|
54
55
|
/// - `put_from` and `get_to` stream through the key-value store's own
|
|
55
|
-
/// `
|
|
56
|
+
/// `put_new_from` and `get_to`, hashing the bytes as they pass.
|
|
57
|
+
/// If the backend skips an existing block, `put_from` returns without
|
|
58
|
+
/// consuming or checking any remaining input.
|
|
56
59
|
/// - If the bytes `put_from` reads do not hash to the CID, the read that would
|
|
57
60
|
/// hand over the last byte fails instead. The key-value store's `put_from`
|
|
58
61
|
/// then fails as [`Kv::put_from`] says, storing nothing, and `put_from`
|
|
@@ -60,6 +63,18 @@ use data_encoding::BASE32_NOPAD;
|
|
|
60
63
|
/// store reports a further failure, such as failing to clean up.
|
|
61
64
|
/// - [`Cas::info`] is `{"name": "kv", "prefix": <prefix>, "kv": <the
|
|
62
65
|
/// key-value store's info>}`; [`Cas::stats`] is the key-value store's.
|
|
66
|
+
///
|
|
67
|
+
/// # S3
|
|
68
|
+
///
|
|
69
|
+
/// Writes use conditional requests to preserve existing blocks. They do not
|
|
70
|
+
/// make a preliminary HEAD request, so they need only write access.
|
|
71
|
+
///
|
|
72
|
+
/// `has` uses a HEAD request. S3 answers a HEAD for a missing key with 403 to
|
|
73
|
+
/// a client that may not list the bucket.
|
|
74
|
+
///
|
|
75
|
+
/// S3 may consume the source before discovering that a block already exists.
|
|
76
|
+
/// Once the backend skips insertion, `put_from` leaves any remaining input
|
|
77
|
+
/// unread.
|
|
63
78
|
pub struct KvCas {
|
|
64
79
|
kv: Arc<dyn Kv>,
|
|
65
80
|
prefix: String,
|
|
@@ -277,7 +292,7 @@ impl Put<Blake3> for KvCas {
|
|
|
277
292
|
fn put(&self, bytes: Bytes) -> Result<Blake3, Error> {
|
|
278
293
|
let cid = Blake3::of(&bytes);
|
|
279
294
|
self.kv
|
|
280
|
-
.
|
|
295
|
+
.put_new(self.key(&cid).as_bytes(), bytes)
|
|
281
296
|
.map_err(backend)?;
|
|
282
297
|
Ok(cid)
|
|
283
298
|
}
|
|
@@ -294,7 +309,10 @@ impl AsyncPut for KvCas {
|
|
|
294
309
|
Box::pin(async move {
|
|
295
310
|
let cid = Blake3::of(&bytes);
|
|
296
311
|
let key = self.key(&cid);
|
|
297
|
-
self.kv
|
|
312
|
+
self.kv
|
|
313
|
+
.aput_new(key.as_bytes(), bytes)
|
|
314
|
+
.await
|
|
315
|
+
.map_err(backend)?;
|
|
298
316
|
Ok(cid.into())
|
|
299
317
|
})
|
|
300
318
|
}
|
|
@@ -334,8 +352,8 @@ impl Cas for KvCas {
|
|
|
334
352
|
mismatch: false,
|
|
335
353
|
};
|
|
336
354
|
let key = self.key(&cid);
|
|
337
|
-
match self.kv.
|
|
338
|
-
Ok(
|
|
355
|
+
match self.kv.put_new_from(key.as_bytes(), &mut checked, len) {
|
|
356
|
+
Ok(_) => Ok(cid.into()),
|
|
339
357
|
// The store failed as asked; a store error beyond that, such as a
|
|
340
358
|
// failed cleanup, is the one to report.
|
|
341
359
|
Err(covalence_kv_trait::Error::Io { .. }) if checked.mismatch => {
|
|
@@ -25,7 +25,7 @@ use tokio::sync::oneshot;
|
|
|
25
25
|
const HELLO: &str = "blake3/5khrmpntq2bjexseshc6ldklwnig56gbj23yvbxjbdcwestheahq/bytes";
|
|
26
26
|
|
|
27
27
|
/// Store "hello" and check reads, then `corrupt` it and check that checked
|
|
28
|
-
/// reads notice, and that `put`
|
|
28
|
+
/// reads notice, and that `put` leaves the block it finds: it never replaces.
|
|
29
29
|
fn check(store: &KvCas, corrupt: impl FnOnce()) {
|
|
30
30
|
let cid: Cid = store.put("hello".into()).unwrap();
|
|
31
31
|
assert_eq!(cid, Blake3::of(b"hello").into());
|
|
@@ -59,7 +59,12 @@ fn check(store: &KvCas, corrupt: impl FnOnce()) {
|
|
|
59
59
|
));
|
|
60
60
|
assert!(matches!(store.get_fact(&cid), Err(Error::Check { .. })));
|
|
61
61
|
let _: Cid = store.put("hello".into()).unwrap();
|
|
62
|
-
assert_eq!(store.
|
|
62
|
+
assert_eq!(store.get_unchecked(&cid).unwrap().unwrap(), "wrong");
|
|
63
|
+
let _ = block_on(store.aput("hello".into())).unwrap();
|
|
64
|
+
assert_eq!(store.get_unchecked(&cid).unwrap().unwrap(), "wrong");
|
|
65
|
+
let put_from = store.put_from(Blake3::of(b"hello"), &mut &b"hello"[..], 5);
|
|
66
|
+
assert_eq!(put_from.unwrap(), cid);
|
|
67
|
+
assert_eq!(store.get_unchecked(&cid).unwrap().unwrap(), "wrong");
|
|
63
68
|
}
|
|
64
69
|
|
|
65
70
|
#[test]
|
|
@@ -72,7 +77,7 @@ fn memory() {
|
|
|
72
77
|
}
|
|
73
78
|
|
|
74
79
|
/// A key-value store that cannot read values, only whether they exist.
|
|
75
|
-
struct Blind(MemoryKv);
|
|
80
|
+
struct Blind(MemoryKv, usize);
|
|
76
81
|
|
|
77
82
|
impl Kv for Blind {
|
|
78
83
|
fn get(&self, _: &[u8]) -> Result<Option<Bytes>, covalence_kv_trait::Error> {
|
|
@@ -90,13 +95,61 @@ impl Kv for Blind {
|
|
|
90
95
|
fn exists(&self, key: &[u8]) -> Result<bool, covalence_kv_trait::Error> {
|
|
91
96
|
self.0.exists(key)
|
|
92
97
|
}
|
|
98
|
+
fn put_new(&self, key: &[u8], value: Bytes) -> Result<bool, covalence_kv_trait::Error> {
|
|
99
|
+
self.0.put_new(key, value)
|
|
100
|
+
}
|
|
101
|
+
fn aput_new<'a>(&'a self, key: &'a [u8], value: Bytes) -> KvFuture<'a, bool> {
|
|
102
|
+
self.0.aput_new(key, value)
|
|
103
|
+
}
|
|
104
|
+
fn put_new_from(
|
|
105
|
+
&self,
|
|
106
|
+
key: &[u8],
|
|
107
|
+
reader: &mut dyn std::io::Read,
|
|
108
|
+
len: u64,
|
|
109
|
+
) -> Result<bool, covalence_kv_trait::Error> {
|
|
110
|
+
if self.0.exists(key)? {
|
|
111
|
+
reader
|
|
112
|
+
.read_exact(&mut vec![0; self.1])
|
|
113
|
+
.map_err(|source| covalence_kv_trait::Error::Io { source })?;
|
|
114
|
+
return Ok(false);
|
|
115
|
+
}
|
|
116
|
+
self.0.put_new_from(key, reader, len)
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/// An existing block may succeed with invalid or failing input left unread.
|
|
121
|
+
#[test]
|
|
122
|
+
fn existing_blocks_leave_unused_stream_bytes_unchecked() {
|
|
123
|
+
for consumed in [0, 2] {
|
|
124
|
+
// A backend may skip immediately or after reading part of the source.
|
|
125
|
+
let store = KvCas::new(Blind(MemoryKv::default(), consumed).into_dyn());
|
|
126
|
+
let cid: Cid = store.put("hello".into()).unwrap();
|
|
127
|
+
let claim = Blake3::of(b"hello");
|
|
128
|
+
for data in [&b"wrong"[..], &b"bad"[..]] {
|
|
129
|
+
let mut source = std::io::Cursor::new(data);
|
|
130
|
+
assert_eq!(store.put_from(claim, &mut source, 5).unwrap(), cid);
|
|
131
|
+
assert_eq!(source.position(), consumed as u64);
|
|
132
|
+
}
|
|
133
|
+
// An error beyond the consumed prefix must not be observed either.
|
|
134
|
+
let mut source = std::io::Read::chain(std::io::Cursor::new(b"he"), FailingReader);
|
|
135
|
+
assert_eq!(store.put_from(claim, &mut source, 5).unwrap(), cid);
|
|
136
|
+
assert_eq!(source.get_ref().0.position(), consumed as u64);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
struct FailingReader;
|
|
141
|
+
|
|
142
|
+
impl std::io::Read for FailingReader {
|
|
143
|
+
fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
|
|
144
|
+
Err(std::io::Error::other("source failed"))
|
|
145
|
+
}
|
|
93
146
|
}
|
|
94
147
|
|
|
95
148
|
/// `has` and `missing` ask whether keys exist rather than reading values,
|
|
96
149
|
/// through `Arc<dyn Cas>` too.
|
|
97
150
|
#[test]
|
|
98
151
|
fn existence_without_reads() {
|
|
99
|
-
let store = KvCas::new(Blind(MemoryKv::default()).into_dyn()).into_dyn();
|
|
152
|
+
let store = KvCas::new(Blind(MemoryKv::default(), 0).into_dyn()).into_dyn();
|
|
100
153
|
let cid = store.put("hello".into()).unwrap();
|
|
101
154
|
assert_eq!(store.put("hello".into()).unwrap(), cid);
|
|
102
155
|
let missing: Cid = Blake3::of(b"missing").into();
|
|
@@ -352,12 +405,12 @@ impl Kv for Leaky {
|
|
|
352
405
|
fn put(&self, _: &[u8], _: Bytes) -> Result<(), covalence_kv_trait::Error> {
|
|
353
406
|
Ok(())
|
|
354
407
|
}
|
|
355
|
-
fn
|
|
408
|
+
fn put_new_from(
|
|
356
409
|
&self,
|
|
357
410
|
_: &[u8],
|
|
358
411
|
reader: &mut dyn std::io::Read,
|
|
359
412
|
_: u64,
|
|
360
|
-
) -> Result<
|
|
413
|
+
) -> Result<bool, covalence_kv_trait::Error> {
|
|
361
414
|
let _ = reader.read_to_end(&mut Vec::new());
|
|
362
415
|
Err(covalence_kv_trait::Error::Backend {
|
|
363
416
|
source: "abort failed".into(),
|