object-storage-client 0.4.1__tar.gz → 0.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/.github/workflows/tests.yaml +3 -2
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/Cargo.lock +19 -1
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/Cargo.toml +4 -3
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/PKG-INFO +134 -109
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/README.md +133 -108
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/docs/GUI.md +39 -10
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/docs/UPGRADING.md +15 -0
- object_storage_client-0.4.3/docs/images/osc-gui.png +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/backend/aws_profile.rs +1 -1
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/backend/azure.rs +92 -1
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/backend/connector.rs +46 -1
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/backend/gcs.rs +113 -19
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/backend/mod.rs +216 -9
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/backend/s3.rs +113 -3
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/backend/stub.rs +11 -3
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/bin/osc-gui.rs +1 -1
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/client.rs +400 -18
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/codec.rs +42 -11
- object_storage_client-0.4.3/src/connection/aws.rs +384 -0
- object_storage_client-0.4.3/src/connection/azure.rs +629 -0
- object_storage_client-0.4.3/src/connection/gcp.rs +290 -0
- object_storage_client-0.4.3/src/connection/mod.rs +509 -0
- object_storage_client-0.4.3/src/connection/sso.rs +552 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/envelope/tests.rs +5 -4
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/envelope.rs +1 -1
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/action/close.rs +1 -0
- object_storage_client-0.4.3/src/gui/action/conn.rs +120 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/action/job.rs +64 -22
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/action/nav.rs +18 -3
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/action/tests/close.rs +1 -1
- object_storage_client-0.4.3/src/gui/action/tests/conn.rs +504 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/action/tests/jobs.rs +19 -2
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/action/tests.rs +53 -22
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/action.rs +94 -14
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/app.rs +4 -3
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/args.rs +8 -25
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/keys.rs +5 -5
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/location.rs +60 -15
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/model.rs +26 -2
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/ops/tests.rs +169 -18
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/ops.rs +37 -24
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/startup.rs +13 -23
- object_storage_client-0.4.3/src/gui/view/address_bar.rs +68 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/view/dialogs.rs +27 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/view/panel.rs +35 -5
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/view/tests.rs +313 -1
- object_storage_client-0.4.3/src/gui/worker/tests.rs +559 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/worker.rs +201 -23
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui.rs +22 -38
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/lib.rs +2 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/registry.rs +94 -8
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration/azure_storage_ops.rs +14 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration/common/mod.rs +31 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration/gcs_storage_ops.rs +12 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration/gui_ops.rs +2 -1
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration/s3_storage_ops.rs +14 -0
- object_storage_client-0.4.1/docs/images/osc-gui.png +0 -0
- object_storage_client-0.4.1/src/gui/view/address_bar.rs +0 -41
- object_storage_client-0.4.1/src/gui/worker/tests.rs +0 -275
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/.cargo/config.toml +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/.docker/garage.toml +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/.gitattributes +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/.github/dependabot.yml +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/.github/workflows/ci.yaml +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/.github/workflows/perf.yaml +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/.gitignore +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/.python-version +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/LICENSE +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/benches/alloc-counter/Cargo.toml +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/benches/alloc-counter/src/lib.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/benches/common/mod.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/benches/throughput.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/compose.yml +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/deny.toml +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/docs/BEHAVIOR.md +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/docs/CONFIGURATION.md +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/docs/DEVELOPMENT.md +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/docs/ENCRYPTION.md +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/docs/TESTING.md +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/proptest-regressions/encryption/context.txt +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/pyproject.toml +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/python/object_storage_client/__init__.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/python/object_storage_client/__init__.pyi +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/python/object_storage_client/_stream.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/python/object_storage_client/_types.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/python/object_storage_client/py.typed +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/scripts/esdk_oracle.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/scripts/hostile-env.sh +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/scripts/perf_smoke.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/backend/body.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/backend/http.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/backend/local.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/cli/args.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/cli/enc/tests.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/cli/enc.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/cli/harden.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/cli/keyring.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/cli/output.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/cli/tests.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/cli/transfer/tests.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/cli/transfer.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/cli/tree/tests.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/cli/tree.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/cli.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/client/local_delete.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/context.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/fixtures.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/frame.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/fuzzing.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/header.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/keys/keyring/tests.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/keys/keyring.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/keys/mod.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/keys/raw_aes.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/mod.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/suite.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/encryption/tamper_tests.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/forward.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/fsmode.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/action/tests/nav.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/display.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/view/startup_error.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/view/status.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/view/toolbar.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/gui/view.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/hex.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/main.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/python/client.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/python/encryption.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/python/mod.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/python/stream.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/sign.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/text.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/src/upload.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/functional/aws_profile.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/functional/cli.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/functional/client.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/functional/encryption.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/functional/golden.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/functional/sign.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/functional/suite.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/functional.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration/common/azurite.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration/common/gcs.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration/common/provider.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration/encryption.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration/file_roundtrip.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration/reencrypt.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration/storage_ops.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/integration.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/perf_guards.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/conftest.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_abort_at_exit.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_encrypted_client.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_encryption_runtime.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_free_threading.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_gil_contention.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_list_stream.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_pipe_native.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_plain_client.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_profile.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_roundtrip.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_shared_client.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_stream_leak.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_stream_sticky.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_stub_conformance.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_subinterpreter.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/python/test_worker_threads.py +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/support/mini_http.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/support/mock.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/support/mod.rs +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/README.md +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/ctx_long.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/ctx_min.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/ctx_utf8.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/manifest.json +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p0_f4096.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p0_f65536.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p1_f4096.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p1_f65536.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p200000_f4096.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p200000_f65536.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p4095_f4096.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p4095_f65536.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p4096_f4096.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p4096_f65536.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p4097_f4096.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p4097_f65536.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p65535_f4096.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p65535_f65536.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p65536_f4096.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p65536_f65536.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p65537_f4096.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/esdk/p65537_f65536.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/golden/README.md +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/golden/v1/context.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/golden/v1/keyring +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/golden/v1/manifest.json +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/golden/v1/old_key.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/golden/v1/size_0.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/golden/v1/size_1.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/golden/v1/size_200000.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/golden/v1/size_65536.bin +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/keyrings/README.md +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/keyrings/one.keyring +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/keyrings/other.keyring +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/tests/vectors/keyrings/two.keyring +0 -0
- {object_storage_client-0.4.1 → object_storage_client-0.4.3}/uv.lock +0 -0
|
@@ -139,8 +139,9 @@ jobs:
|
|
|
139
139
|
- run: cargo check --locked --no-default-features --features encryption --lib
|
|
140
140
|
- run: cargo check --locked --no-default-features --features gui --bins
|
|
141
141
|
# Keeps `encryption::fuzzing` and the fuzz targets compiling, against the
|
|
142
|
-
# committed fuzz/Cargo.lock.
|
|
143
|
-
|
|
142
|
+
# committed fuzz/Cargo.lock. Not --locked: CI's cargo fails on a lock
|
|
143
|
+
# update instead of applying it.
|
|
144
|
+
- run: cargo check --manifest-path fuzz/Cargo.toml --target-dir target
|
|
144
145
|
|
|
145
146
|
python:
|
|
146
147
|
name: python tests (${{ matrix.python }})
|
|
@@ -1556,6 +1556,7 @@ dependencies = [
|
|
|
1556
1556
|
"raw-window-handle",
|
|
1557
1557
|
"smithay-clipboard",
|
|
1558
1558
|
"web-time",
|
|
1559
|
+
"webbrowser",
|
|
1559
1560
|
"windows-sys 0.61.2",
|
|
1560
1561
|
"winit",
|
|
1561
1562
|
]
|
|
@@ -3345,7 +3346,7 @@ dependencies = [
|
|
|
3345
3346
|
|
|
3346
3347
|
[[package]]
|
|
3347
3348
|
name = "object-storage-client"
|
|
3348
|
-
version = "0.4.
|
|
3349
|
+
version = "0.4.3"
|
|
3349
3350
|
dependencies = [
|
|
3350
3351
|
"alloc-counter",
|
|
3351
3352
|
"async-trait",
|
|
@@ -3353,6 +3354,7 @@ dependencies = [
|
|
|
3353
3354
|
"aws-credential-types",
|
|
3354
3355
|
"aws-lc-rs",
|
|
3355
3356
|
"aws-runtime",
|
|
3357
|
+
"aws-sdk-ssooidc",
|
|
3356
3358
|
"aws-sigv4",
|
|
3357
3359
|
"aws-smithy-runtime-api",
|
|
3358
3360
|
"aws-smithy-types",
|
|
@@ -5503,6 +5505,22 @@ dependencies = [
|
|
|
5503
5505
|
"wasm-bindgen",
|
|
5504
5506
|
]
|
|
5505
5507
|
|
|
5508
|
+
[[package]]
|
|
5509
|
+
name = "webbrowser"
|
|
5510
|
+
version = "1.2.6"
|
|
5511
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
5512
|
+
checksum = "fe8d20d0214027baf11b8b0cc307c1da7657d8f4db9aa956f7de27a6fb1acbd5"
|
|
5513
|
+
dependencies = [
|
|
5514
|
+
"jni",
|
|
5515
|
+
"log",
|
|
5516
|
+
"ndk-context",
|
|
5517
|
+
"objc2 0.6.5",
|
|
5518
|
+
"objc2-app-kit 0.3.2",
|
|
5519
|
+
"objc2-foundation 0.3.2",
|
|
5520
|
+
"url",
|
|
5521
|
+
"web-sys",
|
|
5522
|
+
]
|
|
5523
|
+
|
|
5506
5524
|
[[package]]
|
|
5507
5525
|
name = "webpki-root-certs"
|
|
5508
5526
|
version = "1.0.11"
|
|
@@ -9,7 +9,7 @@ license = "GPL-3.0-only"
|
|
|
9
9
|
name = "object-storage-client"
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
repository = "https://github.com/bixority/object-storage-client"
|
|
12
|
-
version = "0.4.
|
|
12
|
+
version = "0.4.3"
|
|
13
13
|
|
|
14
14
|
[lib]
|
|
15
15
|
# maturin adds the cdylib crate type itself; do not list it here.
|
|
@@ -22,12 +22,13 @@ aws-config = { version = "1.13", default-features = false, features = ["rt-tokio
|
|
|
22
22
|
aws-credential-types = "1.3"
|
|
23
23
|
aws-lc-rs = { version = "1.19", default-features = false, features = ["aws-lc-sys", "prebuilt-nasm"] }
|
|
24
24
|
aws-runtime = "1.10"
|
|
25
|
+
aws-sdk-ssooidc = { version = "1.121", default-features = false, optional = true }
|
|
25
26
|
aws-sigv4 = { version = "1.6", default-features = false, features = ["sign-http", "http1"] }
|
|
26
27
|
aws-types = "1.7"
|
|
27
28
|
bytes = "1.12"
|
|
28
29
|
chrono = "0.4"
|
|
29
30
|
clap = { version = "4.6", features = ["derive", "env"], optional = true }
|
|
30
|
-
eframe = { version = "0.37.0", default-features = false, features = ["glow", "default_fonts", "wayland", "clipboard"], optional = true }
|
|
31
|
+
eframe = { version = "0.37.0", default-features = false, features = ["glow", "default_fonts", "wayland", "clipboard", "links"], optional = true }
|
|
31
32
|
egui_extras = { version = "0.37.0", default-features = false, optional = true }
|
|
32
33
|
# Only for its bundled icon and emoji fonts (folder/file/arrow glyphs); eframe
|
|
33
34
|
# brings everything else.
|
|
@@ -59,7 +60,7 @@ default = ["encryption", "cli"]
|
|
|
59
60
|
cli = ["dep:clap", "dep:tokio-util", "tokio/signal", "tokio/io-std", "tokio/macros", "tokio/process", "encryption"]
|
|
60
61
|
encryption = ["dep:zeroize"]
|
|
61
62
|
# The `osc-gui` binary and its `gui` module (a two-pane desktop browser).
|
|
62
|
-
gui = ["cli", "dep:eframe", "dep:egui", "dep:egui_extras"]
|
|
63
|
+
gui = ["cli", "dep:aws-sdk-ssooidc", "dep:eframe", "dep:egui", "dep:egui_extras"]
|
|
63
64
|
# Exposes `encryption::fuzzing` harness entry points for the `fuzz/` crate.
|
|
64
65
|
fuzzing = ["encryption"]
|
|
65
66
|
python = ["dep:pyo3", "dep:pyo3-async-runtimes", "encryption"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: object-storage-client
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.3
|
|
4
4
|
Classifier: Programming Language :: Rust
|
|
5
5
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -21,12 +21,13 @@ Project-URL: Repository, https://github.com/bixority/object-storage-client
|
|
|
21
21
|
# Object Storage Client
|
|
22
22
|
|
|
23
23
|
One URL-based API for AWS S3, Google Cloud Storage, Azure Blob Storage,
|
|
24
|
-
HTTP(S) and the local filesystem, as a Rust library, Python bindings
|
|
25
|
-
`osc` command-line tool. The scheme of each
|
|
26
|
-
client can copy and move objects between
|
|
27
|
-
bounded memory (see
|
|
28
|
-
for the limits) and
|
|
29
|
-
|
|
24
|
+
HTTP(S) and the local filesystem, as a Rust library, Python bindings, the
|
|
25
|
+
`osc` command-line tool and the `osc-gui` desktop browser. The scheme of each
|
|
26
|
+
URL picks the backend, so a single client can copy and move objects between
|
|
27
|
+
providers, stream large objects in bounded memory (see
|
|
28
|
+
[Behaviour](docs/BEHAVIOR.md#large-objects-and-streaming) for the limits) and
|
|
29
|
+
generate pre-signed URLs. Optional client-side encryption keeps object contents
|
|
30
|
+
hidden from the storage provider.
|
|
30
31
|
|
|
31
32
|
- **One API for every backend**, including cross-provider copy and move.
|
|
32
33
|
- **Streaming** uploads and downloads, with multipart uploads for large objects.
|
|
@@ -55,6 +56,120 @@ itself: list it, create it or check that it exists. Credentials come from
|
|
|
55
56
|
environment variables or, for `s3://`, from a named AWS profile; see
|
|
56
57
|
[Configuration](docs/CONFIGURATION.md).
|
|
57
58
|
|
|
59
|
+
## Desktop browser (`osc-gui`)
|
|
60
|
+
|
|
61
|
+
`osc-gui` is a two-panel desktop browser for the same stores as `osc`. Each
|
|
62
|
+
panel shows one folder on its own connection, and you copy, move, rename and
|
|
63
|
+
delete objects and whole folders within a panel or between the two, also across
|
|
64
|
+
accounts and providers. It finds the connections already set up on the machine
|
|
65
|
+
and takes the same encryption options as `osc`.
|
|
66
|
+
|
|
67
|
+

|
|
68
|
+
|
|
69
|
+
### Install
|
|
70
|
+
|
|
71
|
+
The browser is behind the optional `gui` feature, so `osc` alone is unaffected.
|
|
72
|
+
Install both binaries from the registry:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
export CARGO_REGISTRIES_BIXORITY_INDEX="sparse+https://codeberg.org/api/packages/bixority/cargo/"
|
|
76
|
+
cargo install object-storage-client --registry bixority --features gui
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Or from a checkout:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
cargo install --path . --features gui
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
On Linux, `osc-gui` needs a Wayland session; X11 is not supported.
|
|
86
|
+
|
|
87
|
+
### Run
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
osc-gui # both panels on Local, the current directory
|
|
91
|
+
osc-gui --keyring keyring # copies encrypt and decrypt
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Each panel has a connection selector for Local and what is found on the
|
|
95
|
+
machine: AWS profiles, provider environment variables, gcloud configurations and
|
|
96
|
+
Azure CLI storage accounts; a cloud connection lists its buckets. Type any
|
|
97
|
+
matching URL or path into a panel's address bar and press Enter. If a
|
|
98
|
+
connection cannot authenticate, an AWS SSO profile can log in from the panel's
|
|
99
|
+
error banner; for gcloud and Azure the banner shows the command to run. The
|
|
100
|
+
keys and shortcuts are listed in [Desktop browser](docs/GUI.md), which also
|
|
101
|
+
covers limits and closing behaviour.
|
|
102
|
+
|
|
103
|
+
## CLI (`osc`)
|
|
104
|
+
|
|
105
|
+
### Install
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
export CARGO_REGISTRIES_BIXORITY_INDEX="sparse+https://codeberg.org/api/packages/bixority/cargo/"
|
|
109
|
+
cargo install object-storage-client --registry bixority
|
|
110
|
+
# or: cargo install --git https://codeberg.org/bixority/object-storage-client
|
|
111
|
+
# or, from a checkout: cargo install --path .
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### Run
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
osc put my_file.txt s3://my-bucket/remote_file.txt # upload a file
|
|
118
|
+
osc put --recursive my_dir s3://my-bucket/remote_dir/ # upload a directory
|
|
119
|
+
osc get gs://my-bucket/data.json ./local_data.json # download
|
|
120
|
+
osc get-stream gs://my-bucket/large_file.bin > large_file.bin # stream to stdout; progress on stderr
|
|
121
|
+
osc --profile dev ls s3://my-bucket/ # credentials from the AWS profile "dev"
|
|
122
|
+
osc cp s3://source-bucket/image.png az://dest-container/image.png # copy between providers
|
|
123
|
+
osc mv s3://my-bucket/old_name.txt s3://my-bucket/new_name.txt
|
|
124
|
+
osc ls s3://my-bucket/logs/ # names; --url prints full object URLs
|
|
125
|
+
osc rm s3://my-bucket/temp_file.tmp
|
|
126
|
+
osc exists s3://my-bucket/report.pdf # prints true or false
|
|
127
|
+
osc mb s3://my-new-bucket # create a bucket (a directory for local paths)
|
|
128
|
+
osc bucket-exists s3://my-bucket # prints true or false
|
|
129
|
+
|
|
130
|
+
# Pre-signed URLs (S3, GCS, Azure): a download link valid for the default
|
|
131
|
+
# 1 hour, and an upload (PUT) link valid for 15 minutes. With temporary
|
|
132
|
+
# credentials (SSO, an assumed role) an s3:// link stops working when that
|
|
133
|
+
# session ends, whatever the expiry
|
|
134
|
+
osc sign s3://my-bucket/report.pdf
|
|
135
|
+
osc sign --method PUT --expires-in 900 s3://my-bucket/upload.bin
|
|
136
|
+
# S3 only: the upload is rejected unless Content-Length and Content-Type match
|
|
137
|
+
osc sign --method PUT --content-length 1048576 \
|
|
138
|
+
--content-type application/pdf s3://my-bucket/upload.pdf
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
`get-stream` writes each chunk to standard output as it arrives, so its output
|
|
142
|
+
is complete only if `osc` exits 0. A failed read, or a truncated or tampered
|
|
143
|
+
encrypted object, stops the output part way: every byte written was
|
|
144
|
+
authenticated, but the object may be cut short. In a pipeline use
|
|
145
|
+
`set -o pipefail`, or give a FILE for an all-or-nothing download.
|
|
146
|
+
|
|
147
|
+
### Encryption
|
|
148
|
+
|
|
149
|
+
With a keyring, `put`, `get`, `get-stream`, `cp` and `mv` encrypt and decrypt
|
|
150
|
+
transparently:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
umask 077 # keyring files must not be readable by others
|
|
154
|
+
osc keyring new > keyring # a keyring with one fresh key
|
|
155
|
+
osc --keyring keyring put report.csv s3://my-bucket/report.csv
|
|
156
|
+
osc --keyring keyring get s3://my-bucket/report.csv report.csv
|
|
157
|
+
osc enc inspect s3://my-bucket/report.csv # header details; no keys needed
|
|
158
|
+
|
|
159
|
+
# Rotation: add a key, wait until every reader has it, activate it, then move
|
|
160
|
+
# objects off the old one (a single reader may use `osc keyring rotate`)
|
|
161
|
+
osc keyring add < keyring > keyring.new && mv keyring.new keyring
|
|
162
|
+
osc keyring activate <new key ID> < keyring > keyring.new && mv keyring.new keyring
|
|
163
|
+
osc --keyring keyring enc reencrypt s3://my-bucket/ --from <old key ID>
|
|
164
|
+
osc enc scan s3://my-bucket/ # objects per key ID; retire the old key only when it has none and the scan exits 0
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
`--keyring` can also come from `OSC_KEYRING_FILE`, or from a command that
|
|
168
|
+
prints the keyring (`OSC_KEYRING_COMMAND`), so the keyring never touches the
|
|
169
|
+
disk. With `--require-encryption` (or `OSC_REQUIRE_ENCRYPTION=1`), `put`, `get`,
|
|
170
|
+
`get-stream`, `cp`, `mv` and `sign` fail when no keyring is configured, instead
|
|
171
|
+
of sending or reading plaintext.
|
|
172
|
+
|
|
58
173
|
## Rust
|
|
59
174
|
|
|
60
175
|
The crate is published to the
|
|
@@ -96,6 +211,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
|
96
211
|
let retrieved = client.get("s3://my-bucket/hello.txt").await?;
|
|
97
212
|
println!("Retrieved: {}", String::from_utf8_lossy(&retrieved));
|
|
98
213
|
|
|
214
|
+
// Stream a large object in bounded memory: the download feeds the upload
|
|
215
|
+
let stream = client.get_stream("s3://my-bucket/hello.txt").await?;
|
|
216
|
+
client.put_stream("file:///tmp/hello_copy.txt", stream).await?;
|
|
217
|
+
|
|
99
218
|
// List the whole bucket from a bucket-root URL (flat, recursive: every key
|
|
100
219
|
// is returned). Pass a prefix such as "s3://my-bucket/logs/" to narrow it.
|
|
101
220
|
let keys = client.list("s3://my-bucket").await?;
|
|
@@ -176,7 +295,7 @@ let health = keyring.status(); // active key, key ages, last reload error
|
|
|
176
295
|
## Python
|
|
177
296
|
|
|
178
297
|
The package is published on [PyPI](https://pypi.org/project/object-storage-client/)
|
|
179
|
-
and needs Python 3.13
|
|
298
|
+
and needs Python 3.13.9 to 3.15:
|
|
180
299
|
|
|
181
300
|
```bash
|
|
182
301
|
pip install object-storage-client
|
|
@@ -231,6 +350,13 @@ async def main():
|
|
|
231
350
|
async for chunk in stream:
|
|
232
351
|
print(f"Chunk size: {len(chunk)}")
|
|
233
352
|
|
|
353
|
+
# Stream an upload from any async iterable of bytes (size_hint= is optional);
|
|
354
|
+
# a ByteStream from get_object_stream is piped natively
|
|
355
|
+
await client.put_object_stream(
|
|
356
|
+
"s3://my-bucket/hello-copy.txt",
|
|
357
|
+
await client.get_object_stream("s3://my-bucket/hello.txt"),
|
|
358
|
+
)
|
|
359
|
+
|
|
234
360
|
# --- Move data across providers with one client ---
|
|
235
361
|
|
|
236
362
|
# Copy S3 -> Google Cloud Storage (source is left in place)
|
|
@@ -274,107 +400,6 @@ data = await secure.get_object("s3://my-bucket/payroll.csv") # authentic
|
|
|
274
400
|
health = keyring.status() # active key, key ages, last reload error
|
|
275
401
|
```
|
|
276
402
|
|
|
277
|
-
## CLI (`osc`)
|
|
278
|
-
|
|
279
|
-
```bash
|
|
280
|
-
export CARGO_REGISTRIES_BIXORITY_INDEX="sparse+https://codeberg.org/api/packages/bixority/cargo/"
|
|
281
|
-
cargo install object-storage-client --registry bixority
|
|
282
|
-
# or: cargo install --git https://codeberg.org/bixority/object-storage-client
|
|
283
|
-
# or, from a checkout: cargo install --path .
|
|
284
|
-
```
|
|
285
|
-
|
|
286
|
-
```bash
|
|
287
|
-
osc put my_file.txt s3://my-bucket/remote_file.txt # upload a file
|
|
288
|
-
osc put --recursive my_dir s3://my-bucket/remote_dir/ # upload a directory
|
|
289
|
-
osc get gs://my-bucket/data.json ./local_data.json # download
|
|
290
|
-
osc get-stream gs://my-bucket/large_file.bin > large_file.bin # stream to stdout; progress on stderr
|
|
291
|
-
osc --profile dev ls s3://my-bucket/ # credentials from the AWS profile "dev"
|
|
292
|
-
osc cp s3://source-bucket/image.png az://dest-container/image.png # copy between providers
|
|
293
|
-
osc mv s3://my-bucket/old_name.txt s3://my-bucket/new_name.txt
|
|
294
|
-
osc ls s3://my-bucket/logs/ # names; --url prints full object URLs
|
|
295
|
-
osc rm s3://my-bucket/temp_file.tmp
|
|
296
|
-
osc exists s3://my-bucket/report.pdf # prints true or false
|
|
297
|
-
osc mb s3://my-new-bucket # create a bucket (a directory for local paths)
|
|
298
|
-
osc bucket-exists s3://my-bucket # prints true or false
|
|
299
|
-
|
|
300
|
-
# Pre-signed URLs (S3, GCS, Azure): a download link valid for the default
|
|
301
|
-
# 1 hour, and an upload (PUT) link valid for 15 minutes. With temporary
|
|
302
|
-
# credentials (SSO, an assumed role) an s3:// link stops working when that
|
|
303
|
-
# session ends, whatever the expiry
|
|
304
|
-
osc sign s3://my-bucket/report.pdf
|
|
305
|
-
osc sign --method PUT --expires-in 900 s3://my-bucket/upload.bin
|
|
306
|
-
# S3 only: the upload is rejected unless Content-Length and Content-Type match
|
|
307
|
-
osc sign --method PUT --content-length 1048576 \
|
|
308
|
-
--content-type application/pdf s3://my-bucket/upload.pdf
|
|
309
|
-
```
|
|
310
|
-
|
|
311
|
-
`get-stream` writes each chunk to standard output as it arrives, so its output
|
|
312
|
-
is complete only if `osc` exits 0. A failed read, or a truncated or tampered
|
|
313
|
-
encrypted object, stops the output part way: every byte written was
|
|
314
|
-
authenticated, but the object may be cut short. In a pipeline use
|
|
315
|
-
`set -o pipefail`, or give a FILE for an all-or-nothing download.
|
|
316
|
-
|
|
317
|
-
With a keyring, `put`, `get`, `get-stream`, `cp` and `mv` encrypt and decrypt
|
|
318
|
-
transparently:
|
|
319
|
-
|
|
320
|
-
```bash
|
|
321
|
-
umask 077 # keyring files must not be readable by others
|
|
322
|
-
osc keyring new > keyring # a keyring with one fresh key
|
|
323
|
-
osc --keyring keyring put report.csv s3://my-bucket/report.csv
|
|
324
|
-
osc --keyring keyring get s3://my-bucket/report.csv report.csv
|
|
325
|
-
osc enc inspect s3://my-bucket/report.csv # header details; no keys needed
|
|
326
|
-
|
|
327
|
-
# Rotation: add a key, wait until every reader has it, activate it, then move
|
|
328
|
-
# objects off the old one (a single reader may use `osc keyring rotate`)
|
|
329
|
-
osc keyring add < keyring > keyring.new && mv keyring.new keyring
|
|
330
|
-
osc keyring activate <new key ID> < keyring > keyring.new && mv keyring.new keyring
|
|
331
|
-
osc --keyring keyring enc reencrypt s3://my-bucket/ --from <old key ID>
|
|
332
|
-
```
|
|
333
|
-
|
|
334
|
-
`--keyring` can also come from `OSC_KEYRING_FILE`, or from a command that
|
|
335
|
-
prints the keyring (`OSC_KEYRING_COMMAND`), so the keyring never touches the
|
|
336
|
-
disk.
|
|
337
|
-
|
|
338
|
-
## Desktop browser (`osc-gui`)
|
|
339
|
-
|
|
340
|
-
`osc-gui` is a two-panel desktop browser for the same stores as `osc`. Each
|
|
341
|
-
panel shows one folder, and you copy, move, rename and delete objects and whole
|
|
342
|
-
folders within a panel or between the two. It uses the same credentials and
|
|
343
|
-
encryption options as `osc`.
|
|
344
|
-
|
|
345
|
-

|
|
346
|
-
|
|
347
|
-
### Install
|
|
348
|
-
|
|
349
|
-
The browser is behind the optional `gui` feature, so `osc` alone is unaffected.
|
|
350
|
-
Install both binaries from the registry:
|
|
351
|
-
|
|
352
|
-
```bash
|
|
353
|
-
export CARGO_REGISTRIES_BIXORITY_INDEX="sparse+https://codeberg.org/api/packages/bixority/cargo/"
|
|
354
|
-
cargo install object-storage-client --registry bixority --features gui
|
|
355
|
-
```
|
|
356
|
-
|
|
357
|
-
Or from a checkout:
|
|
358
|
-
|
|
359
|
-
```bash
|
|
360
|
-
cargo install --path . --features gui
|
|
361
|
-
```
|
|
362
|
-
|
|
363
|
-
On Linux, `osc-gui` needs a Wayland session; X11 is not supported.
|
|
364
|
-
|
|
365
|
-
### Run
|
|
366
|
-
|
|
367
|
-
```bash
|
|
368
|
-
osc-gui # both panels on the current directory
|
|
369
|
-
osc-gui s3://my-bucket/photos/ ~/Pictures # left and right panel
|
|
370
|
-
osc-gui --profile dev s3://my-bucket/ # credentials from the AWS profile "dev"
|
|
371
|
-
osc-gui --keyring keyring s3://my-bucket/ . # copies encrypt and decrypt
|
|
372
|
-
```
|
|
373
|
-
|
|
374
|
-
Type any URL or path into a panel's address bar and press Enter. Buckets are not
|
|
375
|
-
listed, so type the bucket's URL. The keys and shortcuts are listed in
|
|
376
|
-
[Desktop browser](docs/GUI.md), which also covers limits and closing behaviour.
|
|
377
|
-
|
|
378
403
|
## Documentation
|
|
379
404
|
|
|
380
405
|
- [Configuration](docs/CONFIGURATION.md): credentials and environment variables
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
# Object Storage Client
|
|
2
2
|
|
|
3
3
|
One URL-based API for AWS S3, Google Cloud Storage, Azure Blob Storage,
|
|
4
|
-
HTTP(S) and the local filesystem, as a Rust library, Python bindings
|
|
5
|
-
`osc` command-line tool. The scheme of each
|
|
6
|
-
client can copy and move objects between
|
|
7
|
-
bounded memory (see
|
|
8
|
-
for the limits) and
|
|
9
|
-
|
|
4
|
+
HTTP(S) and the local filesystem, as a Rust library, Python bindings, the
|
|
5
|
+
`osc` command-line tool and the `osc-gui` desktop browser. The scheme of each
|
|
6
|
+
URL picks the backend, so a single client can copy and move objects between
|
|
7
|
+
providers, stream large objects in bounded memory (see
|
|
8
|
+
[Behaviour](docs/BEHAVIOR.md#large-objects-and-streaming) for the limits) and
|
|
9
|
+
generate pre-signed URLs. Optional client-side encryption keeps object contents
|
|
10
|
+
hidden from the storage provider.
|
|
10
11
|
|
|
11
12
|
- **One API for every backend**, including cross-provider copy and move.
|
|
12
13
|
- **Streaming** uploads and downloads, with multipart uploads for large objects.
|
|
@@ -35,6 +36,120 @@ itself: list it, create it or check that it exists. Credentials come from
|
|
|
35
36
|
environment variables or, for `s3://`, from a named AWS profile; see
|
|
36
37
|
[Configuration](docs/CONFIGURATION.md).
|
|
37
38
|
|
|
39
|
+
## Desktop browser (`osc-gui`)
|
|
40
|
+
|
|
41
|
+
`osc-gui` is a two-panel desktop browser for the same stores as `osc`. Each
|
|
42
|
+
panel shows one folder on its own connection, and you copy, move, rename and
|
|
43
|
+
delete objects and whole folders within a panel or between the two, also across
|
|
44
|
+
accounts and providers. It finds the connections already set up on the machine
|
|
45
|
+
and takes the same encryption options as `osc`.
|
|
46
|
+
|
|
47
|
+

|
|
48
|
+
|
|
49
|
+
### Install
|
|
50
|
+
|
|
51
|
+
The browser is behind the optional `gui` feature, so `osc` alone is unaffected.
|
|
52
|
+
Install both binaries from the registry:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
export CARGO_REGISTRIES_BIXORITY_INDEX="sparse+https://codeberg.org/api/packages/bixority/cargo/"
|
|
56
|
+
cargo install object-storage-client --registry bixority --features gui
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Or from a checkout:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
cargo install --path . --features gui
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
On Linux, `osc-gui` needs a Wayland session; X11 is not supported.
|
|
66
|
+
|
|
67
|
+
### Run
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
osc-gui # both panels on Local, the current directory
|
|
71
|
+
osc-gui --keyring keyring # copies encrypt and decrypt
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Each panel has a connection selector for Local and what is found on the
|
|
75
|
+
machine: AWS profiles, provider environment variables, gcloud configurations and
|
|
76
|
+
Azure CLI storage accounts; a cloud connection lists its buckets. Type any
|
|
77
|
+
matching URL or path into a panel's address bar and press Enter. If a
|
|
78
|
+
connection cannot authenticate, an AWS SSO profile can log in from the panel's
|
|
79
|
+
error banner; for gcloud and Azure the banner shows the command to run. The
|
|
80
|
+
keys and shortcuts are listed in [Desktop browser](docs/GUI.md), which also
|
|
81
|
+
covers limits and closing behaviour.
|
|
82
|
+
|
|
83
|
+
## CLI (`osc`)
|
|
84
|
+
|
|
85
|
+
### Install
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
export CARGO_REGISTRIES_BIXORITY_INDEX="sparse+https://codeberg.org/api/packages/bixority/cargo/"
|
|
89
|
+
cargo install object-storage-client --registry bixority
|
|
90
|
+
# or: cargo install --git https://codeberg.org/bixority/object-storage-client
|
|
91
|
+
# or, from a checkout: cargo install --path .
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### Run
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
osc put my_file.txt s3://my-bucket/remote_file.txt # upload a file
|
|
98
|
+
osc put --recursive my_dir s3://my-bucket/remote_dir/ # upload a directory
|
|
99
|
+
osc get gs://my-bucket/data.json ./local_data.json # download
|
|
100
|
+
osc get-stream gs://my-bucket/large_file.bin > large_file.bin # stream to stdout; progress on stderr
|
|
101
|
+
osc --profile dev ls s3://my-bucket/ # credentials from the AWS profile "dev"
|
|
102
|
+
osc cp s3://source-bucket/image.png az://dest-container/image.png # copy between providers
|
|
103
|
+
osc mv s3://my-bucket/old_name.txt s3://my-bucket/new_name.txt
|
|
104
|
+
osc ls s3://my-bucket/logs/ # names; --url prints full object URLs
|
|
105
|
+
osc rm s3://my-bucket/temp_file.tmp
|
|
106
|
+
osc exists s3://my-bucket/report.pdf # prints true or false
|
|
107
|
+
osc mb s3://my-new-bucket # create a bucket (a directory for local paths)
|
|
108
|
+
osc bucket-exists s3://my-bucket # prints true or false
|
|
109
|
+
|
|
110
|
+
# Pre-signed URLs (S3, GCS, Azure): a download link valid for the default
|
|
111
|
+
# 1 hour, and an upload (PUT) link valid for 15 minutes. With temporary
|
|
112
|
+
# credentials (SSO, an assumed role) an s3:// link stops working when that
|
|
113
|
+
# session ends, whatever the expiry
|
|
114
|
+
osc sign s3://my-bucket/report.pdf
|
|
115
|
+
osc sign --method PUT --expires-in 900 s3://my-bucket/upload.bin
|
|
116
|
+
# S3 only: the upload is rejected unless Content-Length and Content-Type match
|
|
117
|
+
osc sign --method PUT --content-length 1048576 \
|
|
118
|
+
--content-type application/pdf s3://my-bucket/upload.pdf
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
`get-stream` writes each chunk to standard output as it arrives, so its output
|
|
122
|
+
is complete only if `osc` exits 0. A failed read, or a truncated or tampered
|
|
123
|
+
encrypted object, stops the output part way: every byte written was
|
|
124
|
+
authenticated, but the object may be cut short. In a pipeline use
|
|
125
|
+
`set -o pipefail`, or give a FILE for an all-or-nothing download.
|
|
126
|
+
|
|
127
|
+
### Encryption
|
|
128
|
+
|
|
129
|
+
With a keyring, `put`, `get`, `get-stream`, `cp` and `mv` encrypt and decrypt
|
|
130
|
+
transparently:
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
umask 077 # keyring files must not be readable by others
|
|
134
|
+
osc keyring new > keyring # a keyring with one fresh key
|
|
135
|
+
osc --keyring keyring put report.csv s3://my-bucket/report.csv
|
|
136
|
+
osc --keyring keyring get s3://my-bucket/report.csv report.csv
|
|
137
|
+
osc enc inspect s3://my-bucket/report.csv # header details; no keys needed
|
|
138
|
+
|
|
139
|
+
# Rotation: add a key, wait until every reader has it, activate it, then move
|
|
140
|
+
# objects off the old one (a single reader may use `osc keyring rotate`)
|
|
141
|
+
osc keyring add < keyring > keyring.new && mv keyring.new keyring
|
|
142
|
+
osc keyring activate <new key ID> < keyring > keyring.new && mv keyring.new keyring
|
|
143
|
+
osc --keyring keyring enc reencrypt s3://my-bucket/ --from <old key ID>
|
|
144
|
+
osc enc scan s3://my-bucket/ # objects per key ID; retire the old key only when it has none and the scan exits 0
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
`--keyring` can also come from `OSC_KEYRING_FILE`, or from a command that
|
|
148
|
+
prints the keyring (`OSC_KEYRING_COMMAND`), so the keyring never touches the
|
|
149
|
+
disk. With `--require-encryption` (or `OSC_REQUIRE_ENCRYPTION=1`), `put`, `get`,
|
|
150
|
+
`get-stream`, `cp`, `mv` and `sign` fail when no keyring is configured, instead
|
|
151
|
+
of sending or reading plaintext.
|
|
152
|
+
|
|
38
153
|
## Rust
|
|
39
154
|
|
|
40
155
|
The crate is published to the
|
|
@@ -76,6 +191,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
|
76
191
|
let retrieved = client.get("s3://my-bucket/hello.txt").await?;
|
|
77
192
|
println!("Retrieved: {}", String::from_utf8_lossy(&retrieved));
|
|
78
193
|
|
|
194
|
+
// Stream a large object in bounded memory: the download feeds the upload
|
|
195
|
+
let stream = client.get_stream("s3://my-bucket/hello.txt").await?;
|
|
196
|
+
client.put_stream("file:///tmp/hello_copy.txt", stream).await?;
|
|
197
|
+
|
|
79
198
|
// List the whole bucket from a bucket-root URL (flat, recursive: every key
|
|
80
199
|
// is returned). Pass a prefix such as "s3://my-bucket/logs/" to narrow it.
|
|
81
200
|
let keys = client.list("s3://my-bucket").await?;
|
|
@@ -156,7 +275,7 @@ let health = keyring.status(); // active key, key ages, last reload error
|
|
|
156
275
|
## Python
|
|
157
276
|
|
|
158
277
|
The package is published on [PyPI](https://pypi.org/project/object-storage-client/)
|
|
159
|
-
and needs Python 3.13
|
|
278
|
+
and needs Python 3.13.9 to 3.15:
|
|
160
279
|
|
|
161
280
|
```bash
|
|
162
281
|
pip install object-storage-client
|
|
@@ -211,6 +330,13 @@ async def main():
|
|
|
211
330
|
async for chunk in stream:
|
|
212
331
|
print(f"Chunk size: {len(chunk)}")
|
|
213
332
|
|
|
333
|
+
# Stream an upload from any async iterable of bytes (size_hint= is optional);
|
|
334
|
+
# a ByteStream from get_object_stream is piped natively
|
|
335
|
+
await client.put_object_stream(
|
|
336
|
+
"s3://my-bucket/hello-copy.txt",
|
|
337
|
+
await client.get_object_stream("s3://my-bucket/hello.txt"),
|
|
338
|
+
)
|
|
339
|
+
|
|
214
340
|
# --- Move data across providers with one client ---
|
|
215
341
|
|
|
216
342
|
# Copy S3 -> Google Cloud Storage (source is left in place)
|
|
@@ -254,107 +380,6 @@ data = await secure.get_object("s3://my-bucket/payroll.csv") # authentic
|
|
|
254
380
|
health = keyring.status() # active key, key ages, last reload error
|
|
255
381
|
```
|
|
256
382
|
|
|
257
|
-
## CLI (`osc`)
|
|
258
|
-
|
|
259
|
-
```bash
|
|
260
|
-
export CARGO_REGISTRIES_BIXORITY_INDEX="sparse+https://codeberg.org/api/packages/bixority/cargo/"
|
|
261
|
-
cargo install object-storage-client --registry bixority
|
|
262
|
-
# or: cargo install --git https://codeberg.org/bixority/object-storage-client
|
|
263
|
-
# or, from a checkout: cargo install --path .
|
|
264
|
-
```
|
|
265
|
-
|
|
266
|
-
```bash
|
|
267
|
-
osc put my_file.txt s3://my-bucket/remote_file.txt # upload a file
|
|
268
|
-
osc put --recursive my_dir s3://my-bucket/remote_dir/ # upload a directory
|
|
269
|
-
osc get gs://my-bucket/data.json ./local_data.json # download
|
|
270
|
-
osc get-stream gs://my-bucket/large_file.bin > large_file.bin # stream to stdout; progress on stderr
|
|
271
|
-
osc --profile dev ls s3://my-bucket/ # credentials from the AWS profile "dev"
|
|
272
|
-
osc cp s3://source-bucket/image.png az://dest-container/image.png # copy between providers
|
|
273
|
-
osc mv s3://my-bucket/old_name.txt s3://my-bucket/new_name.txt
|
|
274
|
-
osc ls s3://my-bucket/logs/ # names; --url prints full object URLs
|
|
275
|
-
osc rm s3://my-bucket/temp_file.tmp
|
|
276
|
-
osc exists s3://my-bucket/report.pdf # prints true or false
|
|
277
|
-
osc mb s3://my-new-bucket # create a bucket (a directory for local paths)
|
|
278
|
-
osc bucket-exists s3://my-bucket # prints true or false
|
|
279
|
-
|
|
280
|
-
# Pre-signed URLs (S3, GCS, Azure): a download link valid for the default
|
|
281
|
-
# 1 hour, and an upload (PUT) link valid for 15 minutes. With temporary
|
|
282
|
-
# credentials (SSO, an assumed role) an s3:// link stops working when that
|
|
283
|
-
# session ends, whatever the expiry
|
|
284
|
-
osc sign s3://my-bucket/report.pdf
|
|
285
|
-
osc sign --method PUT --expires-in 900 s3://my-bucket/upload.bin
|
|
286
|
-
# S3 only: the upload is rejected unless Content-Length and Content-Type match
|
|
287
|
-
osc sign --method PUT --content-length 1048576 \
|
|
288
|
-
--content-type application/pdf s3://my-bucket/upload.pdf
|
|
289
|
-
```
|
|
290
|
-
|
|
291
|
-
`get-stream` writes each chunk to standard output as it arrives, so its output
|
|
292
|
-
is complete only if `osc` exits 0. A failed read, or a truncated or tampered
|
|
293
|
-
encrypted object, stops the output part way: every byte written was
|
|
294
|
-
authenticated, but the object may be cut short. In a pipeline use
|
|
295
|
-
`set -o pipefail`, or give a FILE for an all-or-nothing download.
|
|
296
|
-
|
|
297
|
-
With a keyring, `put`, `get`, `get-stream`, `cp` and `mv` encrypt and decrypt
|
|
298
|
-
transparently:
|
|
299
|
-
|
|
300
|
-
```bash
|
|
301
|
-
umask 077 # keyring files must not be readable by others
|
|
302
|
-
osc keyring new > keyring # a keyring with one fresh key
|
|
303
|
-
osc --keyring keyring put report.csv s3://my-bucket/report.csv
|
|
304
|
-
osc --keyring keyring get s3://my-bucket/report.csv report.csv
|
|
305
|
-
osc enc inspect s3://my-bucket/report.csv # header details; no keys needed
|
|
306
|
-
|
|
307
|
-
# Rotation: add a key, wait until every reader has it, activate it, then move
|
|
308
|
-
# objects off the old one (a single reader may use `osc keyring rotate`)
|
|
309
|
-
osc keyring add < keyring > keyring.new && mv keyring.new keyring
|
|
310
|
-
osc keyring activate <new key ID> < keyring > keyring.new && mv keyring.new keyring
|
|
311
|
-
osc --keyring keyring enc reencrypt s3://my-bucket/ --from <old key ID>
|
|
312
|
-
```
|
|
313
|
-
|
|
314
|
-
`--keyring` can also come from `OSC_KEYRING_FILE`, or from a command that
|
|
315
|
-
prints the keyring (`OSC_KEYRING_COMMAND`), so the keyring never touches the
|
|
316
|
-
disk.
|
|
317
|
-
|
|
318
|
-
## Desktop browser (`osc-gui`)
|
|
319
|
-
|
|
320
|
-
`osc-gui` is a two-panel desktop browser for the same stores as `osc`. Each
|
|
321
|
-
panel shows one folder, and you copy, move, rename and delete objects and whole
|
|
322
|
-
folders within a panel or between the two. It uses the same credentials and
|
|
323
|
-
encryption options as `osc`.
|
|
324
|
-
|
|
325
|
-

|
|
326
|
-
|
|
327
|
-
### Install
|
|
328
|
-
|
|
329
|
-
The browser is behind the optional `gui` feature, so `osc` alone is unaffected.
|
|
330
|
-
Install both binaries from the registry:
|
|
331
|
-
|
|
332
|
-
```bash
|
|
333
|
-
export CARGO_REGISTRIES_BIXORITY_INDEX="sparse+https://codeberg.org/api/packages/bixority/cargo/"
|
|
334
|
-
cargo install object-storage-client --registry bixority --features gui
|
|
335
|
-
```
|
|
336
|
-
|
|
337
|
-
Or from a checkout:
|
|
338
|
-
|
|
339
|
-
```bash
|
|
340
|
-
cargo install --path . --features gui
|
|
341
|
-
```
|
|
342
|
-
|
|
343
|
-
On Linux, `osc-gui` needs a Wayland session; X11 is not supported.
|
|
344
|
-
|
|
345
|
-
### Run
|
|
346
|
-
|
|
347
|
-
```bash
|
|
348
|
-
osc-gui # both panels on the current directory
|
|
349
|
-
osc-gui s3://my-bucket/photos/ ~/Pictures # left and right panel
|
|
350
|
-
osc-gui --profile dev s3://my-bucket/ # credentials from the AWS profile "dev"
|
|
351
|
-
osc-gui --keyring keyring s3://my-bucket/ . # copies encrypt and decrypt
|
|
352
|
-
```
|
|
353
|
-
|
|
354
|
-
Type any URL or path into a panel's address bar and press Enter. Buckets are not
|
|
355
|
-
listed, so type the bucket's URL. The keys and shortcuts are listed in
|
|
356
|
-
[Desktop browser](docs/GUI.md), which also covers limits and closing behaviour.
|
|
357
|
-
|
|
358
383
|
## Documentation
|
|
359
384
|
|
|
360
385
|
- [Configuration](docs/CONFIGURATION.md): credentials and environment variables
|