icechunk 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {icechunk-0.2.1 → icechunk-0.2.2}/PKG-INFO +1 -1
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/Cargo.toml +1 -1
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/repository.rs +28 -7
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/storage/mod.rs +5 -1
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/storage/s3.rs +9 -4
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/Cargo.toml +2 -2
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/notebooks/demo-dummy-data.ipynb +2 -2
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/notebooks/version-control.ipynb +2 -2
- {icechunk-0.2.1 → icechunk-0.2.2/icechunk-python}/python/icechunk/__init__.py +2 -0
- {icechunk-0.2.1 → icechunk-0.2.2/icechunk-python}/python/icechunk/_icechunk_python.pyi +5 -4
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/python/icechunk/repository.py +25 -15
- {icechunk-0.2.1 → icechunk-0.2.2/icechunk-python}/python/icechunk/storage.py +5 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/python/icechunk/store.py +1 -1
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/src/repository.rs +25 -13
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_can_read_old.py +2 -2
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_credentials.py +1 -1
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_stateful_repo_ops.py +1 -1
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_timetravel.py +42 -7
- {icechunk-0.2.1/icechunk-python → icechunk-0.2.2}/python/icechunk/__init__.py +2 -0
- {icechunk-0.2.1/icechunk-python → icechunk-0.2.2}/python/icechunk/_icechunk_python.pyi +5 -4
- {icechunk-0.2.1 → icechunk-0.2.2}/python/icechunk/repository.py +25 -15
- {icechunk-0.2.1/icechunk-python → icechunk-0.2.2}/python/icechunk/storage.py +5 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/python/icechunk/store.py +1 -1
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/examples/low_level_dataset.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/examples/multithreaded_get_chunk_refs.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/examples/multithreaded_store.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/flatbuffers/all.fbs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/flatbuffers/manifest.fbs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/flatbuffers/object_ids.fbs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/flatbuffers/snapshot.fbs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/flatbuffers/transaction_log.fbs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/proptest-regressions/dataset.txt +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/proptest-regressions/format/manifest.txt +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/proptest-regressions/manifest.txt +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/proptest-regressions/session.txt +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/proptest-regressions/storage/mod.txt +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/proptest-regressions/storage/s3.txt +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/proptest-regressions/storage/virtual_ref.txt +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/asset_manager.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/change_set.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/config.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/conflicts/basic_solver.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/conflicts/detector.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/conflicts/mod.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/error.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/format/attributes.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/format/flatbuffers/all_generated.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/format/manifest.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/format/mod.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/format/serializers/mod.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/format/snapshot.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/format/transaction_log.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/lib.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/ops/gc.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/ops/mod.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/refs.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/session.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/storage/logging.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/storage/object_store.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/store.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/strategies.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/src/virtual_chunks.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/tests/test_concurrency.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/tests/test_distributed_writes.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/tests/test_gc.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/tests/test_storage.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk/tests/test_virtual_refs.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/.gitignore +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/Cargo.lock +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/__init__.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/coiled_runner.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/conftest.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/create_era5.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/datasets.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/envs/icechunk-alpha-12.txt +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/envs/icechunk-alpha-release.txt +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/helpers.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/lib.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/most_recent.sh +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/runner.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/tasks.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/test_benchmark_reads.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/benchmarks/test_benchmark_writes.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/examples/dask_write.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/notebooks/conflict.ipynb +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/notebooks/demo-azure-blob.ipynb +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/notebooks/demo-gcs.ipynb +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/notebooks/demo-s3.ipynb +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/notebooks/environment.yml +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/notebooks/memorystore.ipynb +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/notebooks/performance/era5_xarray-Icechunk.ipynb +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/notebooks/performance/era5_xarray-zarr2.ipynb +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/notebooks/performance/era5_xarray-zarr3.ipynb +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/python/icechunk/credentials.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/python/icechunk/dask.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/python/icechunk/distributed.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/python/icechunk/py.typed +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/python/icechunk/session.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/python/icechunk/vendor/__init__.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/python/icechunk/vendor/xarray.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/python/icechunk/xarray.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/src/config.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/src/conflicts.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/src/errors.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/src/lib.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/src/session.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/src/store.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/src/streams.rs +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/__init__.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/conftest.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/chunks/09HEW2P03CSMHFAZY7DG +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/chunks/52H0E4NSPN8SVRK9EVGG +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/chunks/DWQ75SDC624XF9H326RG +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/chunks/RW938N1KP2R4BHMW62QG +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/config.yaml +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/manifests/CMYVHDWMSTG9R25780YG +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/manifests/G3W2W8V6ZG09J6C21WE0 +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/manifests/Q04J7QW5RQ8D17TPA10G +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/manifests/SHTEAP8C784YMZSJKBM0 +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/refs/branch.main/ref.json +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ref.json +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/refs/tag.deleted/ref.json +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/refs/tag.deleted/ref.json.deleted +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/refs/tag.it also works!/ref.json +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/refs/tag.it works!/ref.json +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/snapshots/4QF8JA0YPDN51MHSSYVG +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/snapshots/7XAF0Q905SH4938DN9CG +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/snapshots/GC4YVH5SKBPEZCENYQE0 +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/snapshots/NXH3M0HJ7EEJ0699DPP0 +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/snapshots/P874YS3J196959RDHX7G +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/snapshots/XDZ162T1TYBEJMK99NPG +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/transactions/4QF8JA0YPDN51MHSSYVG +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/transactions/7XAF0Q905SH4938DN9CG +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/transactions/GC4YVH5SKBPEZCENYQE0 +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/transactions/NXH3M0HJ7EEJ0699DPP0 +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/data/test-repo/transactions/XDZ162T1TYBEJMK99NPG +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/run_xarray_backends_tests.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_commit_properties.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_concurrency.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_config.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_conflicts.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_dask.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_distributed_writers.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_error.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_gc.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_pickle.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_regressions.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_store.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_virtual_ref.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_xarray.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_zarr/__init__.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_zarr/test_array.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_zarr/test_group.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_zarr/test_properties.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_zarr/test_stateful.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_zarr/test_store/__init__.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_zarr/test_store/test_core.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/icechunk-python/tests/test_zarr/test_store/test_icechunk_store.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/pyproject.toml +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/python/icechunk/credentials.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/python/icechunk/dask.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/python/icechunk/distributed.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/python/icechunk/py.typed +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/python/icechunk/session.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/python/icechunk/vendor/__init__.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/python/icechunk/vendor/xarray.py +0 -0
- {icechunk-0.2.1 → icechunk-0.2.2}/python/icechunk/xarray.py +0 -0
|
@@ -5,7 +5,9 @@ use std::{
|
|
|
5
5
|
sync::Arc,
|
|
6
6
|
};
|
|
7
7
|
|
|
8
|
+
use async_recursion::async_recursion;
|
|
8
9
|
use bytes::Bytes;
|
|
10
|
+
use chrono::{DateTime, Utc};
|
|
9
11
|
use err_into::ErrorInto as _;
|
|
10
12
|
use futures::{
|
|
11
13
|
stream::{FuturesOrdered, FuturesUnordered},
|
|
@@ -37,15 +39,13 @@ use crate::{
|
|
|
37
39
|
Storage, StorageError,
|
|
38
40
|
};
|
|
39
41
|
|
|
40
|
-
#[derive(Debug, Clone,
|
|
42
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
41
43
|
#[non_exhaustive]
|
|
42
44
|
pub enum VersionInfo {
|
|
43
|
-
#[serde(rename = "snapshot_id")]
|
|
44
45
|
SnapshotId(SnapshotId),
|
|
45
|
-
#[serde(rename = "tag")]
|
|
46
46
|
TagRef(String),
|
|
47
|
-
#[serde(rename = "branch")]
|
|
48
47
|
BranchTipRef(String),
|
|
48
|
+
AsOf { branch: String, at: DateTime<Utc> },
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
#[derive(Debug, Error)]
|
|
@@ -60,6 +60,8 @@ pub enum RepositoryErrorKind {
|
|
|
60
60
|
|
|
61
61
|
#[error("snapshot not found: `{id}`")]
|
|
62
62
|
SnapshotNotFound { id: SnapshotId },
|
|
63
|
+
#[error("branch {branch} does not have a snapshots before or at {at}")]
|
|
64
|
+
InvalidAsOfSpec { branch: String, at: DateTime<Utc> },
|
|
63
65
|
#[error("invalid snapshot id: `{0}`")]
|
|
64
66
|
InvalidSnapshotId(String),
|
|
65
67
|
#[error("tag error: `{0}`")]
|
|
@@ -404,11 +406,12 @@ impl Repository {
|
|
|
404
406
|
}
|
|
405
407
|
|
|
406
408
|
/// Returns the sequence of parents of the snapshot pointed by the given version
|
|
409
|
+
#[async_recursion(?Send)]
|
|
407
410
|
#[instrument(skip(self))]
|
|
408
|
-
pub async fn ancestry(
|
|
409
|
-
&self,
|
|
411
|
+
pub async fn ancestry<'a>(
|
|
412
|
+
&'a self,
|
|
410
413
|
version: &VersionInfo,
|
|
411
|
-
) -> RepositoryResult<impl Stream<Item = RepositoryResult<SnapshotInfo>> + '
|
|
414
|
+
) -> RepositoryResult<impl Stream<Item = RepositoryResult<SnapshotInfo>> + 'a> {
|
|
412
415
|
let snapshot_id = self.resolve_version(version).await?;
|
|
413
416
|
self.snapshot_ancestry(&snapshot_id).await
|
|
414
417
|
}
|
|
@@ -572,6 +575,24 @@ impl Repository {
|
|
|
572
575
|
.await?;
|
|
573
576
|
Ok(ref_data.snapshot)
|
|
574
577
|
}
|
|
578
|
+
VersionInfo::AsOf { branch, at } => {
|
|
579
|
+
let tip = VersionInfo::BranchTipRef(branch.clone());
|
|
580
|
+
let snap = self
|
|
581
|
+
.ancestry(&tip)
|
|
582
|
+
.await?
|
|
583
|
+
.try_skip_while(|parent| ready(Ok(&parent.flushed_at > at)))
|
|
584
|
+
.take(1)
|
|
585
|
+
.try_collect::<Vec<_>>()
|
|
586
|
+
.await?;
|
|
587
|
+
match snap.into_iter().next() {
|
|
588
|
+
Some(snap) => Ok(snap.id),
|
|
589
|
+
None => Err(RepositoryErrorKind::InvalidAsOfSpec {
|
|
590
|
+
branch: branch.clone(),
|
|
591
|
+
at: *at,
|
|
592
|
+
}
|
|
593
|
+
.into()),
|
|
594
|
+
}
|
|
595
|
+
}
|
|
575
596
|
}
|
|
576
597
|
}
|
|
577
598
|
|
|
@@ -406,7 +406,11 @@ pub trait Storage: fmt::Debug + private::Sealed + Sync + Send {
|
|
|
406
406
|
) -> StorageResult<DateTime<Utc>>;
|
|
407
407
|
|
|
408
408
|
async fn root_is_clean(&self) -> StorageResult<bool> {
|
|
409
|
-
|
|
409
|
+
match self.list_objects(&Settings::default(), "").await?.next().await {
|
|
410
|
+
None => Ok(true),
|
|
411
|
+
Some(Ok(_)) => Ok(false),
|
|
412
|
+
Some(Err(err)) => Err(err),
|
|
413
|
+
}
|
|
410
414
|
}
|
|
411
415
|
|
|
412
416
|
async fn list_chunks(
|
|
@@ -205,9 +205,11 @@ impl S3Storage {
|
|
|
205
205
|
let mut b =
|
|
206
206
|
self.get_client().await.put_object().bucket(self.bucket.clone()).key(key);
|
|
207
207
|
|
|
208
|
-
if
|
|
209
|
-
|
|
210
|
-
|
|
208
|
+
if settings.unsafe_use_metadata() {
|
|
209
|
+
if let Some(ct) = content_type {
|
|
210
|
+
b = b.content_type(ct)
|
|
211
|
+
};
|
|
212
|
+
}
|
|
211
213
|
|
|
212
214
|
if settings.unsafe_use_metadata() {
|
|
213
215
|
for (k, v) in metadata {
|
|
@@ -312,9 +314,12 @@ impl Storage for S3Storage {
|
|
|
312
314
|
.put_object()
|
|
313
315
|
.bucket(self.bucket.clone())
|
|
314
316
|
.key(key)
|
|
315
|
-
.content_type("application/yaml")
|
|
316
317
|
.body(config.into());
|
|
317
318
|
|
|
319
|
+
if settings.unsafe_use_metadata() {
|
|
320
|
+
req = req.content_type("application/yaml")
|
|
321
|
+
}
|
|
322
|
+
|
|
318
323
|
match (
|
|
319
324
|
previous_version.etag(),
|
|
320
325
|
settings.unsafe_use_conditional_create(),
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "icechunk-python"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.2"
|
|
4
4
|
description = "Transactional storage engine for Zarr designed for use on cloud object storage"
|
|
5
5
|
readme = "../README.md"
|
|
6
6
|
repository = "https://github.com/earth-mover/icechunk"
|
|
@@ -21,7 +21,7 @@ crate-type = ["cdylib"]
|
|
|
21
21
|
bytes = "1.9.0"
|
|
22
22
|
chrono = { version = "0.4.39" }
|
|
23
23
|
futures = "0.3.31"
|
|
24
|
-
icechunk = { path = "../icechunk", version = "0.2.
|
|
24
|
+
icechunk = { path = "../icechunk", version = "0.2.2", features = ["logs"] }
|
|
25
25
|
itertools = "0.14.0"
|
|
26
26
|
pyo3 = { version = "0.23", features = [
|
|
27
27
|
"chrono",
|
|
@@ -335,7 +335,7 @@
|
|
|
335
335
|
},
|
|
336
336
|
{
|
|
337
337
|
"cell_type": "code",
|
|
338
|
-
"execution_count":
|
|
338
|
+
"execution_count": null,
|
|
339
339
|
"id": "d904f719-98cf-4f51-8e9a-1631dcb3fcba",
|
|
340
340
|
"metadata": {},
|
|
341
341
|
"outputs": [
|
|
@@ -348,7 +348,7 @@
|
|
|
348
348
|
}
|
|
349
349
|
],
|
|
350
350
|
"source": [
|
|
351
|
-
"session = repo.readonly_session(
|
|
351
|
+
"session = repo.readonly_session(snapshot_id=first_commit)\n",
|
|
352
352
|
"root_group = zarr.open_group(session.store, mode=\"r\")\n",
|
|
353
353
|
"\n",
|
|
354
354
|
"try:\n",
|
|
@@ -242,7 +242,7 @@
|
|
|
242
242
|
},
|
|
243
243
|
{
|
|
244
244
|
"cell_type": "code",
|
|
245
|
-
"execution_count":
|
|
245
|
+
"execution_count": null,
|
|
246
246
|
"id": "e785d9a1-36ec-4207-b334-20e0a68e3ac8",
|
|
247
247
|
"metadata": {},
|
|
248
248
|
"outputs": [
|
|
@@ -258,7 +258,7 @@
|
|
|
258
258
|
}
|
|
259
259
|
],
|
|
260
260
|
"source": [
|
|
261
|
-
"session = repo.readonly_session(
|
|
261
|
+
"session = repo.readonly_session(snapshot_id=first_commit)\n",
|
|
262
262
|
"root_group = zarr.open_group(store=session.store, mode=\"r\")\n",
|
|
263
263
|
"dict(root_group.attrs)"
|
|
264
264
|
]
|
|
@@ -14,6 +14,7 @@ from icechunk._icechunk_python import (
|
|
|
14
14
|
ConflictType,
|
|
15
15
|
Credentials,
|
|
16
16
|
Diff,
|
|
17
|
+
GcsBearerCredential,
|
|
17
18
|
GcsCredentials,
|
|
18
19
|
GcsStaticCredentials,
|
|
19
20
|
GCSummary,
|
|
@@ -95,6 +96,7 @@ __all__ = [
|
|
|
95
96
|
"Credentials",
|
|
96
97
|
"Diff",
|
|
97
98
|
"GCSummary",
|
|
99
|
+
"GcsBearerCredential",
|
|
98
100
|
"GcsCredentials",
|
|
99
101
|
"GcsStaticCredentials",
|
|
100
102
|
"IcechunkError",
|
|
@@ -957,7 +957,7 @@ class PyRepository:
|
|
|
957
957
|
*,
|
|
958
958
|
branch: str | None = None,
|
|
959
959
|
tag: str | None = None,
|
|
960
|
-
|
|
960
|
+
snapshot_id: str | None = None,
|
|
961
961
|
) -> AsyncIterator[SnapshotInfo]: ...
|
|
962
962
|
def create_branch(self, branch: str, snapshot_id: str) -> None: ...
|
|
963
963
|
def list_branches(self) -> set[str]: ...
|
|
@@ -972,17 +972,18 @@ class PyRepository:
|
|
|
972
972
|
self,
|
|
973
973
|
from_branch: str | None = None,
|
|
974
974
|
from_tag: str | None = None,
|
|
975
|
-
|
|
975
|
+
from_snapshot_id: str | None = None,
|
|
976
976
|
to_branch: str | None = None,
|
|
977
977
|
to_tag: str | None = None,
|
|
978
|
-
|
|
978
|
+
to_snapshot_id: str | None = None,
|
|
979
979
|
) -> Diff: ...
|
|
980
980
|
def readonly_session(
|
|
981
981
|
self,
|
|
982
982
|
branch: str | None = None,
|
|
983
983
|
*,
|
|
984
984
|
tag: str | None = None,
|
|
985
|
-
|
|
985
|
+
snapshot_id: str | None = None,
|
|
986
|
+
as_of: datetime.datetime | None = None,
|
|
986
987
|
) -> PySession: ...
|
|
987
988
|
def writable_session(self, branch: str) -> PySession: ...
|
|
988
989
|
def expire_snapshots(
|
|
@@ -196,7 +196,7 @@ class Repository:
|
|
|
196
196
|
*,
|
|
197
197
|
branch: str | None = None,
|
|
198
198
|
tag: str | None = None,
|
|
199
|
-
|
|
199
|
+
snapshot_id: str | None = None,
|
|
200
200
|
) -> Iterator[SnapshotInfo]:
|
|
201
201
|
"""
|
|
202
202
|
Get the ancestry of a snapshot.
|
|
@@ -207,7 +207,7 @@ class Repository:
|
|
|
207
207
|
The branch to get the ancestry of.
|
|
208
208
|
tag : str, optional
|
|
209
209
|
The tag to get the ancestry of.
|
|
210
|
-
|
|
210
|
+
snapshot_id : str, optional
|
|
211
211
|
The snapshot ID to get the ancestry of.
|
|
212
212
|
|
|
213
213
|
Returns
|
|
@@ -223,7 +223,9 @@ class Repository:
|
|
|
223
223
|
# the returned object is both an Async and Sync iterator
|
|
224
224
|
res = cast(
|
|
225
225
|
Iterator[SnapshotInfo],
|
|
226
|
-
self._repository.async_ancestry(
|
|
226
|
+
self._repository.async_ancestry(
|
|
227
|
+
branch=branch, tag=tag, snapshot_id=snapshot_id
|
|
228
|
+
),
|
|
227
229
|
)
|
|
228
230
|
return res
|
|
229
231
|
|
|
@@ -232,7 +234,7 @@ class Repository:
|
|
|
232
234
|
*,
|
|
233
235
|
branch: str | None = None,
|
|
234
236
|
tag: str | None = None,
|
|
235
|
-
|
|
237
|
+
snapshot_id: str | None = None,
|
|
236
238
|
) -> AsyncIterator[SnapshotInfo]:
|
|
237
239
|
"""
|
|
238
240
|
Get the ancestry of a snapshot.
|
|
@@ -243,7 +245,7 @@ class Repository:
|
|
|
243
245
|
The branch to get the ancestry of.
|
|
244
246
|
tag : str, optional
|
|
245
247
|
The tag to get the ancestry of.
|
|
246
|
-
|
|
248
|
+
snapshot_id : str, optional
|
|
247
249
|
The snapshot ID to get the ancestry of.
|
|
248
250
|
|
|
249
251
|
Returns
|
|
@@ -255,7 +257,9 @@ class Repository:
|
|
|
255
257
|
-----
|
|
256
258
|
Only one of the arguments can be specified.
|
|
257
259
|
"""
|
|
258
|
-
return self._repository.async_ancestry(
|
|
260
|
+
return self._repository.async_ancestry(
|
|
261
|
+
branch=branch, tag=tag, snapshot_id=snapshot_id
|
|
262
|
+
)
|
|
259
263
|
|
|
260
264
|
def create_branch(self, branch: str, snapshot_id: str) -> None:
|
|
261
265
|
"""
|
|
@@ -336,7 +340,7 @@ class Repository:
|
|
|
336
340
|
"""
|
|
337
341
|
self._repository.delete_branch(branch)
|
|
338
342
|
|
|
339
|
-
def delete_tag(self,
|
|
343
|
+
def delete_tag(self, tag: str) -> None:
|
|
340
344
|
"""
|
|
341
345
|
Delete a tag.
|
|
342
346
|
|
|
@@ -349,7 +353,7 @@ class Repository:
|
|
|
349
353
|
-------
|
|
350
354
|
None
|
|
351
355
|
"""
|
|
352
|
-
self._repository.delete_tag(
|
|
356
|
+
self._repository.delete_tag(tag)
|
|
353
357
|
|
|
354
358
|
def create_tag(self, tag: str, snapshot_id: str) -> None:
|
|
355
359
|
"""
|
|
@@ -400,10 +404,10 @@ class Repository:
|
|
|
400
404
|
*,
|
|
401
405
|
from_branch: str | None = None,
|
|
402
406
|
from_tag: str | None = None,
|
|
403
|
-
|
|
407
|
+
from_snapshot_id: str | None = None,
|
|
404
408
|
to_branch: str | None = None,
|
|
405
409
|
to_tag: str | None = None,
|
|
406
|
-
|
|
410
|
+
to_snapshot_id: str | None = None,
|
|
407
411
|
) -> Diff:
|
|
408
412
|
"""
|
|
409
413
|
Compute an overview of the operations executed from version `from` to version `to`.
|
|
@@ -421,10 +425,10 @@ class Repository:
|
|
|
421
425
|
return self._repository.diff(
|
|
422
426
|
from_branch=from_branch,
|
|
423
427
|
from_tag=from_tag,
|
|
424
|
-
|
|
428
|
+
from_snapshot_id=from_snapshot_id,
|
|
425
429
|
to_branch=to_branch,
|
|
426
430
|
to_tag=to_tag,
|
|
427
|
-
|
|
431
|
+
to_snapshot_id=to_snapshot_id,
|
|
428
432
|
)
|
|
429
433
|
|
|
430
434
|
def readonly_session(
|
|
@@ -432,7 +436,8 @@ class Repository:
|
|
|
432
436
|
branch: str | None = None,
|
|
433
437
|
*,
|
|
434
438
|
tag: str | None = None,
|
|
435
|
-
|
|
439
|
+
snapshot_id: str | None = None,
|
|
440
|
+
as_of: datetime.datetime | None = None,
|
|
436
441
|
) -> Session:
|
|
437
442
|
"""
|
|
438
443
|
Create a read-only session.
|
|
@@ -447,8 +452,11 @@ class Repository:
|
|
|
447
452
|
If provided, the branch to create the session on.
|
|
448
453
|
tag : str, optional
|
|
449
454
|
If provided, the tag to create the session on.
|
|
450
|
-
|
|
455
|
+
snapshot_id : str, optional
|
|
451
456
|
If provided, the snapshot ID to create the session on.
|
|
457
|
+
as_of: datetime.datetime, optional
|
|
458
|
+
When combined with the branch argument, it will open the session at the last
|
|
459
|
+
snapshot that is at or before this datetime
|
|
452
460
|
|
|
453
461
|
Returns
|
|
454
462
|
-------
|
|
@@ -460,7 +468,9 @@ class Repository:
|
|
|
460
468
|
Only one of the arguments can be specified.
|
|
461
469
|
"""
|
|
462
470
|
return Session(
|
|
463
|
-
self._repository.readonly_session(
|
|
471
|
+
self._repository.readonly_session(
|
|
472
|
+
branch=branch, tag=tag, snapshot_id=snapshot_id, as_of=as_of
|
|
473
|
+
)
|
|
464
474
|
)
|
|
465
475
|
|
|
466
476
|
def writable_session(self, branch: str) -> Session:
|
|
@@ -2,6 +2,7 @@ from collections.abc import Callable
|
|
|
2
2
|
from datetime import datetime
|
|
3
3
|
|
|
4
4
|
from icechunk._icechunk_python import (
|
|
5
|
+
GcsBearerCredential,
|
|
5
6
|
ObjectStoreConfig,
|
|
6
7
|
S3Options,
|
|
7
8
|
S3StaticCredentials,
|
|
@@ -222,6 +223,7 @@ def gcs_storage(
|
|
|
222
223
|
bearer_token: str | None = None,
|
|
223
224
|
from_env: bool | None = None,
|
|
224
225
|
config: dict[str, str] | None = None,
|
|
226
|
+
get_credentials: Callable[[], GcsBearerCredential] | None = None,
|
|
225
227
|
) -> Storage:
|
|
226
228
|
"""Create a Storage instance that saves data in Google Cloud Storage object store.
|
|
227
229
|
|
|
@@ -235,6 +237,8 @@ def gcs_storage(
|
|
|
235
237
|
Fetch credentials from the operative system environment
|
|
236
238
|
bearer_token: str | None
|
|
237
239
|
The bearer token to use for the object store
|
|
240
|
+
get_credentials: Callable[[], GcsBearerCredential] | None
|
|
241
|
+
Use this function to get and refresh object store credentials
|
|
238
242
|
"""
|
|
239
243
|
credentials = gcs_credentials(
|
|
240
244
|
service_account_file=service_account_file,
|
|
@@ -242,6 +246,7 @@ def gcs_storage(
|
|
|
242
246
|
application_credentials=application_credentials,
|
|
243
247
|
bearer_token=bearer_token,
|
|
244
248
|
from_env=from_env,
|
|
249
|
+
get_credentials=get_credentials,
|
|
245
250
|
)
|
|
246
251
|
return Storage.new_gcs(
|
|
247
252
|
bucket=bucket,
|
|
@@ -506,18 +506,18 @@ impl PyRepository {
|
|
|
506
506
|
}
|
|
507
507
|
|
|
508
508
|
/// Returns an object that is both a sync and an async iterator
|
|
509
|
-
#[pyo3(signature = (*, branch = None, tag = None,
|
|
509
|
+
#[pyo3(signature = (*, branch = None, tag = None, snapshot_id = None))]
|
|
510
510
|
pub fn async_ancestry(
|
|
511
511
|
&self,
|
|
512
512
|
py: Python<'_>,
|
|
513
513
|
branch: Option<String>,
|
|
514
514
|
tag: Option<String>,
|
|
515
|
-
|
|
515
|
+
snapshot_id: Option<String>,
|
|
516
516
|
) -> PyResult<PyAsyncGenerator> {
|
|
517
517
|
let repo = Arc::clone(&self.0);
|
|
518
518
|
// This function calls block_on, so we need to allow other thread python to make progress
|
|
519
519
|
py.allow_threads(move || {
|
|
520
|
-
let version = args_to_version_info(branch, tag,
|
|
520
|
+
let version = args_to_version_info(branch, tag, snapshot_id, None)?;
|
|
521
521
|
let ancestry = pyo3_async_runtimes::tokio::get_runtime()
|
|
522
522
|
.block_on(async move { repo.ancestry_arc(&version).await })
|
|
523
523
|
.map_err(PyIcechunkStoreError::RepositoryError)?
|
|
@@ -689,20 +689,20 @@ impl PyRepository {
|
|
|
689
689
|
})
|
|
690
690
|
}
|
|
691
691
|
|
|
692
|
-
#[pyo3(signature = (*, from_branch=None, from_tag=None,
|
|
692
|
+
#[pyo3(signature = (*, from_branch=None, from_tag=None, from_snapshot_id=None, to_branch=None, to_tag=None, to_snapshot_id=None))]
|
|
693
693
|
#[allow(clippy::too_many_arguments)]
|
|
694
694
|
pub fn diff(
|
|
695
695
|
&self,
|
|
696
696
|
py: Python<'_>,
|
|
697
697
|
from_branch: Option<String>,
|
|
698
698
|
from_tag: Option<String>,
|
|
699
|
-
|
|
699
|
+
from_snapshot_id: Option<String>,
|
|
700
700
|
to_branch: Option<String>,
|
|
701
701
|
to_tag: Option<String>,
|
|
702
|
-
|
|
702
|
+
to_snapshot_id: Option<String>,
|
|
703
703
|
) -> PyResult<PyDiff> {
|
|
704
|
-
let from = args_to_version_info(from_branch, from_tag,
|
|
705
|
-
let to = args_to_version_info(to_branch, to_tag,
|
|
704
|
+
let from = args_to_version_info(from_branch, from_tag, from_snapshot_id, None)?;
|
|
705
|
+
let to = args_to_version_info(to_branch, to_tag, to_snapshot_id, None)?;
|
|
706
706
|
|
|
707
707
|
// This function calls block_on, so we need to allow other thread python to make progress
|
|
708
708
|
py.allow_threads(move || {
|
|
@@ -717,17 +717,18 @@ impl PyRepository {
|
|
|
717
717
|
})
|
|
718
718
|
}
|
|
719
719
|
|
|
720
|
-
#[pyo3(signature = (*, branch = None, tag = None,
|
|
720
|
+
#[pyo3(signature = (*, branch = None, tag = None, snapshot_id = None, as_of = None))]
|
|
721
721
|
pub fn readonly_session(
|
|
722
722
|
&self,
|
|
723
723
|
py: Python<'_>,
|
|
724
724
|
branch: Option<String>,
|
|
725
725
|
tag: Option<String>,
|
|
726
|
-
|
|
726
|
+
snapshot_id: Option<String>,
|
|
727
|
+
as_of: Option<DateTime<Utc>>,
|
|
727
728
|
) -> PyResult<PySession> {
|
|
728
729
|
// This function calls block_on, so we need to allow other thread python to make progress
|
|
729
730
|
py.allow_threads(move || {
|
|
730
|
-
let version = args_to_version_info(branch, tag,
|
|
731
|
+
let version = args_to_version_info(branch, tag, snapshot_id, as_of)?;
|
|
731
732
|
let session =
|
|
732
733
|
pyo3_async_runtimes::tokio::get_runtime().block_on(async move {
|
|
733
734
|
self.0
|
|
@@ -841,6 +842,7 @@ fn args_to_version_info(
|
|
|
841
842
|
branch: Option<String>,
|
|
842
843
|
tag: Option<String>,
|
|
843
844
|
snapshot: Option<String>,
|
|
845
|
+
as_of: Option<DateTime<Utc>>,
|
|
844
846
|
) -> PyResult<VersionInfo> {
|
|
845
847
|
let n = [&branch, &tag, &snapshot].iter().filter(|r| !r.is_none()).count();
|
|
846
848
|
if n > 1 {
|
|
@@ -849,8 +851,18 @@ fn args_to_version_info(
|
|
|
849
851
|
));
|
|
850
852
|
}
|
|
851
853
|
|
|
852
|
-
if
|
|
853
|
-
|
|
854
|
+
if as_of.is_some() && branch.is_none() {
|
|
855
|
+
return Err(PyValueError::new_err(
|
|
856
|
+
"as_of argument must be provided together with a branch name",
|
|
857
|
+
));
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
if let Some(branch) = branch {
|
|
861
|
+
if let Some(at) = as_of {
|
|
862
|
+
Ok(VersionInfo::AsOf { branch, at })
|
|
863
|
+
} else {
|
|
864
|
+
Ok(VersionInfo::BranchTipRef(branch))
|
|
865
|
+
}
|
|
854
866
|
} else if let Some(tag_name) = tag {
|
|
855
867
|
Ok(VersionInfo::TagRef(tag_name))
|
|
856
868
|
} else if let Some(snapshot_id) = snapshot {
|
|
@@ -177,7 +177,7 @@ async def test_icechunk_can_read_old_repo() -> None:
|
|
|
177
177
|
"Repository initialized",
|
|
178
178
|
]
|
|
179
179
|
assert [
|
|
180
|
-
p.message for p in repo.ancestry(
|
|
180
|
+
p.message for p in repo.ancestry(snapshot_id=main_snapshot)
|
|
181
181
|
] == expected_main_history
|
|
182
182
|
|
|
183
183
|
expected_branch_history = [
|
|
@@ -256,7 +256,7 @@ async def test_icechunk_can_read_old_repo() -> None:
|
|
|
256
256
|
assert_array_equal(big_chunks[:], 42.0)
|
|
257
257
|
|
|
258
258
|
parents = list(repo.ancestry(branch="main"))
|
|
259
|
-
diff = repo.diff(to_branch="main",
|
|
259
|
+
diff = repo.diff(to_branch="main", from_snapshot_id=parents[-2].id)
|
|
260
260
|
assert diff.new_groups == set()
|
|
261
261
|
assert diff.new_arrays == set()
|
|
262
262
|
assert set(diff.updated_chunks.keys()) == {
|
|
@@ -113,7 +113,7 @@ class ExpirableCredentials:
|
|
|
113
113
|
)
|
|
114
114
|
|
|
115
115
|
|
|
116
|
-
def
|
|
116
|
+
def test_s3_refreshable_credentials_refresh(tmp_path: Path) -> None:
|
|
117
117
|
path = tmp_path / "calls.txt"
|
|
118
118
|
creds_obj = ExpirableCredentials(path)
|
|
119
119
|
|
|
@@ -271,7 +271,7 @@ class VersionControlStateMachine(RuleBasedStateMachine):
|
|
|
271
271
|
@rule(ref=commits)
|
|
272
272
|
def checkout_commit(self, ref):
|
|
273
273
|
note(f"Checking out commit {ref}")
|
|
274
|
-
self.session = self.repo.readonly_session(
|
|
274
|
+
self.session = self.repo.readonly_session(snapshot_id=ref)
|
|
275
275
|
assert self.session.read_only
|
|
276
276
|
self.model.checkout_commit(ref)
|
|
277
277
|
|
|
@@ -58,14 +58,14 @@ def test_timetravel() -> None:
|
|
|
58
58
|
|
|
59
59
|
new_snapshot_id = session.commit("commit 2")
|
|
60
60
|
|
|
61
|
-
session = repo.readonly_session(
|
|
61
|
+
session = repo.readonly_session(snapshot_id=first_snapshot_id)
|
|
62
62
|
store = session.store
|
|
63
63
|
group = zarr.open_group(store=store, mode="r")
|
|
64
64
|
air_temp = cast(zarr.core.array.Array, group["air_temp"])
|
|
65
65
|
assert store.read_only
|
|
66
66
|
assert air_temp[200, 6] == 42
|
|
67
67
|
|
|
68
|
-
session = repo.readonly_session(
|
|
68
|
+
session = repo.readonly_session(snapshot_id=new_snapshot_id)
|
|
69
69
|
store = session.store
|
|
70
70
|
group = zarr.open_group(store=store, mode="r")
|
|
71
71
|
air_temp = cast(zarr.core.array.Array, group["air_temp"])
|
|
@@ -116,7 +116,7 @@ def test_timetravel() -> None:
|
|
|
116
116
|
air_temp = cast(zarr.core.array.Array, group["air_temp"])
|
|
117
117
|
assert air_temp[200, 6] == 90
|
|
118
118
|
|
|
119
|
-
parents = list(repo.ancestry(
|
|
119
|
+
parents = list(repo.ancestry(snapshot_id=feature_snapshot_id))
|
|
120
120
|
assert [snap.message for snap in parents] == [
|
|
121
121
|
"commit 3",
|
|
122
122
|
"commit 2",
|
|
@@ -128,7 +128,7 @@ def test_timetravel() -> None:
|
|
|
128
128
|
assert list(repo.ancestry(tag="v1.0")) == parents
|
|
129
129
|
assert list(repo.ancestry(branch="feature-not-dead")) == parents
|
|
130
130
|
|
|
131
|
-
diff = repo.diff(to_tag="v1.0",
|
|
131
|
+
diff = repo.diff(to_tag="v1.0", from_snapshot_id=parents[-1].id)
|
|
132
132
|
assert diff.new_groups == {"/"}
|
|
133
133
|
assert diff.new_arrays == {"/air_temp"}
|
|
134
134
|
assert list(diff.updated_chunks.keys()) == ["/air_temp"]
|
|
@@ -185,11 +185,11 @@ Arrays deleted:
|
|
|
185
185
|
|
|
186
186
|
with pytest.raises(ValueError, match="doesn't include"):
|
|
187
187
|
# if we call diff in the wrong order it fails with a message
|
|
188
|
-
repo.diff(from_tag="v1.0",
|
|
188
|
+
repo.diff(from_tag="v1.0", to_snapshot_id=parents[-1].id)
|
|
189
189
|
|
|
190
190
|
# check async ancestry works
|
|
191
|
-
assert list(repo.ancestry(
|
|
192
|
-
async_ancestry(repo,
|
|
191
|
+
assert list(repo.ancestry(snapshot_id=feature_snapshot_id)) == asyncio.run(
|
|
192
|
+
async_ancestry(repo, snapshot_id=feature_snapshot_id)
|
|
193
193
|
)
|
|
194
194
|
assert list(repo.ancestry(tag="v1.0")) == asyncio.run(
|
|
195
195
|
async_ancestry(repo, tag="v1.0")
|
|
@@ -258,3 +258,38 @@ async def test_tag_delete() -> None:
|
|
|
258
258
|
|
|
259
259
|
with pytest.raises(ValueError):
|
|
260
260
|
repo.create_tag("tag", snap)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
async def test_session_with_as_of() -> None:
|
|
264
|
+
repo = ic.Repository.create(
|
|
265
|
+
storage=ic.in_memory_storage(),
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
session = repo.writable_session("main")
|
|
269
|
+
store = session.store
|
|
270
|
+
|
|
271
|
+
times = []
|
|
272
|
+
group = zarr.group(store=store, overwrite=True)
|
|
273
|
+
sid = session.commit("root")
|
|
274
|
+
times.append(next(repo.ancestry(snapshot_id=sid)).written_at)
|
|
275
|
+
|
|
276
|
+
for i in range(5):
|
|
277
|
+
session = repo.writable_session("main")
|
|
278
|
+
store = session.store
|
|
279
|
+
group = zarr.open_group(store=store)
|
|
280
|
+
group.create_group(f"child {i}")
|
|
281
|
+
sid = session.commit(f"child {i}")
|
|
282
|
+
times.append(next(repo.ancestry(snapshot_id=sid)).written_at)
|
|
283
|
+
|
|
284
|
+
ancestry = list(p for p in repo.ancestry(branch="main"))
|
|
285
|
+
assert len(ancestry) == 7 # initial + root + 5 children
|
|
286
|
+
|
|
287
|
+
store = repo.readonly_session("main", as_of=times[-1]).store
|
|
288
|
+
group = zarr.open_group(store=store, mode="r")
|
|
289
|
+
|
|
290
|
+
for i, time in enumerate(times):
|
|
291
|
+
store = repo.readonly_session("main", as_of=time).store
|
|
292
|
+
group = zarr.open_group(store=store, mode="r")
|
|
293
|
+
expected_children = {f"child {j}" for j in range(i)}
|
|
294
|
+
actual_children = {g[0] for g in group.members()}
|
|
295
|
+
assert expected_children == actual_children
|
|
@@ -14,6 +14,7 @@ from icechunk._icechunk_python import (
|
|
|
14
14
|
ConflictType,
|
|
15
15
|
Credentials,
|
|
16
16
|
Diff,
|
|
17
|
+
GcsBearerCredential,
|
|
17
18
|
GcsCredentials,
|
|
18
19
|
GcsStaticCredentials,
|
|
19
20
|
GCSummary,
|
|
@@ -95,6 +96,7 @@ __all__ = [
|
|
|
95
96
|
"Credentials",
|
|
96
97
|
"Diff",
|
|
97
98
|
"GCSummary",
|
|
99
|
+
"GcsBearerCredential",
|
|
98
100
|
"GcsCredentials",
|
|
99
101
|
"GcsStaticCredentials",
|
|
100
102
|
"IcechunkError",
|