rustytree-xarray 0.2.1__tar.gz → 0.3.0__tar.gz

{rustytree_xarray-0.2.1 → rustytree_xarray-0.3.0}/.github/workflows/ci.yml

@@ -15,7 +15,22 @@ env:
   RUSTFLAGS: -D warnings
 
 jobs:
-  # Cargo gates run once on stable Rust. Cheap; no Python needed.
+  # Python lint + format gate. Fast and dependency-free (ruff is a standalone
+  # binary; no extension build needed), so it runs in parallel with the cargo
+  # and pytest jobs and fails fast on style drift. The pinned version must
+  # match the `ruff` pin in pyproject's `dev` extra so local `ruff format`
+  # and this check agree.
+  lint:
+    name: ruff (lint + format)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+      - run: uvx 'ruff@0.15.8' check .
+      - run: uvx 'ruff@0.15.8' format --check .
+
+  # Cargo gates run once on stable Rust. `cargo test` links libpython (see the
+  # test step), so Python is set up here too.
   cargo:
     name: cargo (fmt, clippy, test)
     runs-on: ubuntu-latest
@@ -24,10 +39,21 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy, rustfmt
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
       - uses: Swatinem/rust-cache@v2
       - run: cargo fmt --check
       - run: cargo clippy --all-targets --all-features --locked -- -D warnings
-      - run: cargo test --locked
+      # Build tests WITHOUT `extension-module` so PyO3 links libpython: the
+      # credential-fetcher shim's `#[typetag::serde]` impls are force-retained
+      # by `inventory` and call `Python::attach`, so the test binary references
+      # libpython symbols that can't be dead-code-eliminated (they can under
+      # `extension-module`, which suppresses libpython linking for the cdylib).
+      - name: cargo test
+        run: cargo test --locked --no-default-features
+        env:
+          LD_LIBRARY_PATH: ${{ env.pythonLocation }}/lib
 
   # Python integration: build the cdylib via maturin and run pytest against
   # the supported interpreter range. Each matrix cell rebuilds the extension
@@ -59,8 +85,9 @@ jobs:
         # Install the [dev] extras directly rather than `pip install -e ".[dev]"`,
         # which would re-trigger a full cdylib rebuild on top of maturin develop
         # below (we'd build the wheel twice — saves ~80s per matrix cell).
-        # Keep this list in sync with the `dev` extra in pyproject.toml.
-        run: uv pip install --python .venv/bin/python pytest pytest-cov 'zarr>=3.0' 'icechunk>=2.0.5' ruff
+        # These are the deps pytest needs; ruff is not among them — linting runs
+        # in the dedicated `lint` job above.
+        run: uv pip install --python .venv/bin/python pytest pytest-cov 'zarr>=3.0' 'icechunk>=2.1.0,<2.2' 'dask[distributed]>=2024.1'
       - name: Build extension (maturin develop)
         run: .venv/bin/maturin develop
       - name: pytest

{rustytree_xarray-0.2.1 → rustytree_xarray-0.3.0}/CHANGELOG.md

@@ -11,6 +11,180 @@ release, that section is renamed to `[x.y.z] - YYYY-MM-DD` and a fresh
 
 ## [Unreleased]
 
+## [0.3.0] - 2026-07-04
+
+### Added
+
+- `group_filter` keyword for `open_datatree` ([#49]). An
+  `fnmatch`-style glob matched against every group path — only matching groups
+  (plus their ancestors and the root, so the tree stays connected) are loaded.
+  Mirrors xarray PR [pydata/xarray#11302](https://github.com/pydata/xarray/pull/11302):
+  matching follows `PurePosixPath.match` (right-anchored), so
+  `group_filter="*/sweep_0"` opens the lowest sweep of every volume, and group
+  names that literally contain glob metacharacters are reachable via character-
+  class escapes (`[*]`, `[?]`, `[[]`). Mutually exclusive with `group`. The Rust
+  walk prunes non-matching subtrees up front (conservative prefix predicate);
+  Python's `_filter_by_glob` remains the source of truth. `open_dataset` rejects
+  `group_filter` (a single Dataset has no defined target for a multi-match glob).
+
+- Docs: a "Distributed compute (`dask.distributed` / Coiled)" section in
+  `docs/usage.md` ([#46]) covering the requirements for a remote cluster
+  (`spawn` workers, matching `rustytree` + `icechunk` in the worker environment)
+  and — mirroring icechunk's own guidance — how to avoid shipping secrets to
+  workers: use `from_env` / `refreshable` / `anonymous` credentials (secret-free
+  in the task graph) rather than `static` keys. Verified end-to-end that a
+  `from_env` session opened via rustytree pickles with no secret in the graph and
+  still computes on a distributed cluster.
+
+- Picklable array handles for `dask.distributed` ([#44], fixes #44).
+  DataArrays opened with `engine="rustytree"` could not be computed under a
+  `dask.distributed` cluster — `.compute()` failed while distributed pickled
+  the task graph, because each lazy chunk held a native
+  `ZarrsArrayHandle` that raised `TypeError: cannot pickle
+  'rustytree._rustytree.ZarrsArrayHandle' object`. `ZarrsArrayHandle` now
+  implements `__reduce__`: for stores opened via an icechunk `Session` (both
+  local-filesystem and remote S3, the cases in #44) the handle carries the
+  session's own `as_bytes()` msgpack plus the array path, and a worker revives
+  it via icechunk's `Session::from_bytes` + `Array::async_open`. This mirrors
+  icechunk exactly — rustytree adds **no** credential handling of its own, so
+  credential exposure is precisely icechunk's: `from_env` / `anonymous` sessions
+  carry no secret into the task graph, while **static** credentials (and a
+  refreshable session's scattered `initial` creds) are embedded in the session
+  bytes and travel to every worker in the pickled graph — the same as
+  distributing an icechunk Session directly. Opening through an icechunk Session
+  with `from_env` / `anonymous` / `refreshable` credentials is the recommended
+  distributed pattern. Workers must use the `spawn` start-method (the
+  `dask.distributed` default); rustytree's tokio runtime is not fork-safe, so
+  fork-based multiprocessing after opening a store is unsupported. Vanilla
+  `s3://` / local Zarr stores are not yet picklable (no icechunk Session to
+  mirror) and now raise a clear, actionable error at pickle time instead of the
+  opaque default. New `tests/test_pickle.py` covers the pickle round-trip
+  (including deeply-nested arrays and multi-hop re-pickling), the not-picklable
+  and corrupt-state error paths, and an opt-in `distributed`-marked
+  `LocalCluster` compute.
+
+- `numcodecs.zlib` codec support ([#41], fixes #42). Enables the `zlib`
+  feature on the `zarrs` dependency so rustytree can decode arrays whose
+  codec pipeline uses the non-standard `numcodecs.`-namespace codecs
+  zarr-python writes — e.g. the public `earthmover-public/goes-16`
+  arraylake dataset (and the ismip6 icechunk store reported in #42, where
+  enabling it gave a ~10× speedup over `engine="zarr"`), which chain
+  `bytes → numcodecs.shuffle → numcodecs.zlib`. zarrs's `zlib` codec is
+  documented byte-compatible
+  with zarr-python's; `numcodecs.shuffle` is always compiled, and
+  `gzip` / `blosc` / `zstd` / `crc32c` already come from zarrs's
+  default features. Verified bit-for-bit against zarr-python on a
+  GOES-16 `CMI_C01` slice. New non-gated `tests/test_codecs.py` writes
+  the shuffle+zlib pipeline to a local store and checks the decode
+  round-trips.
+
+### Changed
+
+- CI now runs `ruff check` and `ruff format --check` in a dedicated `lint` job,
+  and `ruff` is pinned (`>=0.15.8,<0.16`) in the `dev` extra so local and CI
+  formatting agree ([#50], fixes #48). Previously CI installed `ruff` but never
+  invoked it, so lint/format drift accumulated silently on `main`; this normalises
+  the drifted files (`backend.py`, `conftest.py`, `test_backend_entrypoint.py`,
+  `test_codecs.py`, `klot_demo.ipynb`) in one no-logic-change pass and gates
+  against recurrence. No runtime behaviour change.
+
+- **Breaking:** `group=` is now exact-path only and no longer auto-detects glob
+  patterns ([#49]). Previously a `group=` value containing `*`, `?`, or `[`
+  silently switched to glob filtering; that overloaded behaviour is removed in
+  favour of the explicit `group_filter=` (see Added). This aligns with the final
+  design of xarray PR [pydata/xarray#11302](https://github.com/pydata/xarray/pull/11302),
+  which rejected the ambiguous overloaded form because group names may legitimately
+  contain glob metacharacters. Migrate `open_datatree(group="*/sweep_0")` to
+  `open_datatree(group_filter="*/sweep_0")`; `group` and `group_filter` are
+  mutually exclusive (passing both raises `ValueError`). A `group=` value with
+  glob characters is now looked up as a literal path.
+
+- Docs: tidy the `notebooks/klot_demo.ipynb` demo — trim WHAT-narrating comments
+  and add an `xradar` prerequisite note ([#47]). No code change.
+
+- Bump the pinned `icechunk` from 2.0.5 to 2.1.0 ([#46]). rustytree links the
+  `icechunk` Rust crate and round-trips sessions through
+  `Session::{as,from}_bytes`, so it can only open stores whose on-disk format
+  matches its pinned icechunk — 2.0.5 could not open repositories written by
+  icechunk 2.0.6+, failing in `Repository::open` with "the repository doesn't
+  exist" even though the store was present and the credentials valid. Bumping to
+  2.1.0 restores access to current icechunk stores; `typetag` moves to `=0.2.22`
+  to keep sharing icechunk's `inventory` credential-fetcher registry (the #41
+  `py_credentials` shim). The `dev` extra and CI now require
+  `icechunk>=2.1.0,<2.2` so the Python and Rust icechunk versions share a minor —
+  the msgpack session-bytes format is coupled across the FFI boundary, and a skew
+  is exactly what produced the "repository doesn't exist" failure. No rustytree
+  source changes were needed (the API surface is unchanged). Verified end-to-end
+  against a real credentialed AWS S3 icechunk 2.0.6 store: parity with
+  `engine="zarr"`, pickle round-trip, and a distributed `LocalCluster` compute
+  whose workers reopen the store from the pickled session.
+
+### Fixed
+
+- Unified the missing-group error type across backends ([#52], fixes #51).
+  A literal `group=` path that doesn't exist now raises `KeyError` on both
+  vanilla Zarr and icechunk stores — previously vanilla raised `RuntimeError`
+  (from the Rust walk) while icechunk raised `KeyError`, so callers couldn't
+  `except` one type. The Rust walk now maps zarrs' `GroupCreateError::MissingMetadata`
+  to `RustytreeError::NotFound` (→ `PyKeyError`); genuine storage/IO/corruption
+  errors still surface as `RuntimeError`.
+
+- `open_dataset` and `open_datatree` now validate the `group`/`group_filter`
+  pair consistently ([#52]). Passing both to `open_dataset` raises the same
+  `ValueError` (mutually exclusive) as `open_datatree` instead of a
+  `NotImplementedError`; the shared validator (renamed
+  `_check_group_filter_mutex` → `_validate_group_filter` since it also rejects
+  an empty `group_filter`) runs first on both entry points.
+
+- Docs and the `klot_demo` notebook still used the removed glob-`group=`
+  form (`group="*/sweep_0"`) that [#49] turned into a literal-path lookup —
+  the examples would now raise instead of filtering. Migrated `README.md`,
+  `docs/usage.md`, and `notebooks/klot_demo.ipynb` to `group_filter=`
+  ([#50]).
+
+- Spurious `SerializationWarning: variable '...' has multiple fill values`
+  and silently-broken masking for stores whose `_FillValue` is written in
+  raw Zarr wire form ([#43]). Some virtual/icechunk stores (e.g. the
+  source.coop ISMIP6 dataset) carry `_FillValue` as a base64-encoded
+  string (`'AAAAgB2vFUQ='`) alongside a numeric `missing_value`; xarray's
+  CF mask coder saw two distinct sentinels for the same value and warned,
+  and the base64 string masked nothing (a float array never equals a
+  `str`), so masking quietly fell back to `missing_value` alone.
+  `_RustyDataStore.get_variables` now mirrors xarray's zarr backend
+  (`ZarrStore.open_store_variable`): a base64 `str`/`bytes` `_FillValue`
+  is decoded to a numeric sentinel via `FillValueCoder.decode`, while an
+  already-numeric fill is left untouched. Verified byte-for-byte against
+  `engine="zarr"` on the ISMIP6 store (warnings 28 → 0, identical
+  `_FillValue` encoding and masking). Non-numeric list/tuple wire forms
+  (e.g. a complex `[real, imag]` fill) have no decoder branch and pass
+  through unchanged rather than raising. Network-free regression tests
+  cover the base64 decode, numeric pass-through, and list pass-through.
+
+- Open arraylake / Earthmover icechunk sessions ([#41], fixes #40).
+  `xr.open_datatree(session.store, engine="rustytree")` raised
+  `ValueError: icechunk session: unknown error: unknown variant
+  PythonCredentialsFetcher, there are no variants` for sessions
+  created via arraylake. Such sessions store S3 credentials as
+  `Refreshable(Arc<dyn S3CredentialsFetcher>)` whose concrete fetcher
+  is `#[typetag::serde]`-registered only inside icechunk-python's
+  cdylib; rustytree links the vanilla `icechunk` crate, whose fetcher
+  registry is empty, so `Session::from_bytes` couldn't resolve the tag.
+  `xr.open_zarr` is unaffected because it uses the live in-process
+  store and never round-trips through bytes. New `src/py_credentials.rs`
+  re-registers `PythonCredentialsFetcher` (S3 / GCS / Azure) inside
+  rustytree's cdylib, mirroring icechunk-python's `get()`: serve the
+  scattered `initial` static credentials while fresh, else acquire the
+  GIL and re-run the embedded pickled credential callable to refresh —
+  so the common case and `scatter_initial_credentials=False` /
+  mid-walk credential expiry all keep working. `typetag` is pinned
+  `=0.2.21` to share icechunk's `inventory` registry. A Python-side
+  friendly error (`_rust_open_or_explain` in `backend.py`) is a safety
+  net for a future icechunk-python that renames the fetcher. CI now
+  builds tests with `--no-default-features` so PyO3 links libpython:
+  the inventory-retained fetcher impls reference `Python::attach`, so
+  their libpython symbols can no longer be dead-code-eliminated from
+  the test binary.
+
 ## [0.2.1] - 2026-05-23
 
 ### Fixed
@@ -530,7 +704,9 @@ below.
   intentionally not normalized — relative-vs-absolute glob
   semantics differ in `PurePosixPath.match`.
 
-[Unreleased]: https://github.com/aladinor/rustytree/compare/v0.2.0...HEAD
+[Unreleased]: https://github.com/aladinor/rustytree/compare/v0.3.0...HEAD
+[0.3.0]: https://github.com/aladinor/rustytree/compare/v0.2.1...v0.3.0
+[0.2.1]: https://github.com/aladinor/rustytree/compare/v0.2.0...v0.2.1
 [0.2.0]: https://github.com/aladinor/rustytree/compare/v0.1.0...v0.2.0
 [0.1.0]: https://github.com/aladinor/rustytree/releases/tag/v0.1.0
 [#1]: https://github.com/aladinor/rustytree/pull/1
@@ -558,3 +734,11 @@ below.
 [#25]: https://github.com/aladinor/rustytree/pull/25
 [#26]: https://github.com/aladinor/rustytree/pull/26
 [#27]: https://github.com/aladinor/rustytree/pull/27
+[#41]: https://github.com/aladinor/rustytree/pull/41
+[#43]: https://github.com/aladinor/rustytree/pull/43
+[#44]: https://github.com/aladinor/rustytree/pull/44
+[#46]: https://github.com/aladinor/rustytree/pull/46
+[#47]: https://github.com/aladinor/rustytree/pull/47
+[#49]: https://github.com/aladinor/rustytree/pull/49
+[#50]: https://github.com/aladinor/rustytree/pull/50
+[#52]: https://github.com/aladinor/rustytree/pull/52

{rustytree_xarray-0.2.1 → rustytree_xarray-0.3.0}/Cargo.lock

@@ -505,15 +505,16 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-runtime"
-version = "1.11.1"
+version = "1.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0504b1ab12debb5959e5165ee5fe97dd387e7aa7ea6a477bfd7635dfe769a4f5"
+checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-http",
  "aws-smithy-http-client",
  "aws-smithy-observability",
  "aws-smithy-runtime-api",
+ "aws-smithy-schema",
  "aws-smithy-types",
  "bytes",
  "fastrand",
@@ -530,9 +531,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-runtime-api"
-version = "1.12.0"
+version = "1.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b71a13df6ada0aafbf21a73bdfcdf9324cfa9df77d96b8446045be3cde61b42e"
+checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-runtime-api-macros",
@@ -557,11 +558,22 @@ dependencies = [
  "syn 2.0.117",
 ]
 
+[[package]]
+name = "aws-smithy-schema"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5"
+dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "http 1.4.0",
+]
+
 [[package]]
 name = "aws-smithy-types"
-version = "1.4.7"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c"
+checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85"
 dependencies = [
  "base64-simd",
  "bytes",
@@ -706,6 +718,15 @@ dependencies = [
  "zstd-sys",
 ]
 
+[[package]]
+name = "bs58"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4"
+dependencies = [
+ "tinyvec",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.20.2"
@@ -1846,9 +1867,9 @@ dependencies = [
 
 [[package]]
 name = "icechunk"
-version = "2.0.5"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7af416f5b28f3bea35f2d48dd018b7cba711c35a1c965311782222ac337bbccd"
+checksum = "08a02fb61448609885f19dd8e9ffb0909792a59f1b36599ddb3c0d35f7090fef"
 dependencies = [
  "async-compression",
  "async-recursion",
@@ -1864,7 +1885,7 @@ dependencies = [
  "icechunk-s3",
  "icechunk-storage",
  "icechunk-types",
- "itertools",
+ "itertools 0.15.0",
  "quick_cache",
  "rand 0.10.1",
  "regex",
@@ -1887,14 +1908,15 @@ dependencies = [
 
 [[package]]
 name = "icechunk-arrow-object-store"
-version = "2.0.5"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e7610964a5f7859731c95eaec642fcf839e5467137da10d9a2a477c47c1ca2b"
+checksum = "ceb90dfc6ec56c155ec98338c726660d5398fa5fda450bde7f63145a9e1e22ac"
 dependencies = [
  "async-trait",
  "bytes",
  "chrono",
  "futures",
+ "http 1.4.0",
  "icechunk-storage",
  "icechunk-types",
  "object_store",
@@ -1909,9 +1931,9 @@ dependencies = [
 
 [[package]]
 name = "icechunk-format"
-version = "2.0.5"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "258c49d6a1b89fab075fa8b9c31829861ec52f92e5984ab924f4ad834ede13a6"
+checksum = "d1e8bbd643e33147b3850ba9044981990998cb59c9f0bc5d274220c526c15536"
 dependencies = [
  "base32",
  "bytes",
@@ -1920,7 +1942,7 @@ dependencies = [
  "flexbuffers",
  "futures",
  "icechunk-types",
- "itertools",
+ "itertools 0.15.0",
  "quick_cache",
  "rand 0.10.1",
  "rmp-serde",
@@ -1935,9 +1957,9 @@ dependencies = [
 
 [[package]]
 name = "icechunk-s3"
-version = "2.0.5"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79e08bc924fcc375bf6a2214cc897c09d46b242821999f6c6e3eb87adb0fb0a4"
+checksum = "e381c4d1f80a1f89e5d3175120700ed72a11a238a62a14e41b259ba38e169353"
 dependencies = [
  "async-trait",
  "aws-config",
@@ -1960,16 +1982,16 @@ dependencies = [
 
 [[package]]
 name = "icechunk-storage"
-version = "2.0.5"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8341b2ce1729d586b9460af93c5c0549d2937df48ad7407101498f56ce99d75"
+checksum = "8f1e267f2a4d7749bf37766764b8b68a098079609aed98296efa2b3255fb577c"
 dependencies = [
  "async-trait",
  "bytes",
  "chrono",
  "futures",
  "icechunk-types",
- "itertools",
+ "itertools 0.15.0",
  "serde",
  "thiserror",
  "tokio",
@@ -1981,9 +2003,9 @@ dependencies = [
 
 [[package]]
 name = "icechunk-types"
-version = "2.0.5"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ae8062e3d19227364ba07bd13f2b3bfbb33beaf46883b72b26f36d0ae359449"
+checksum = "3fd025f9b9e34ec544f50bf2bbb0b9ea5bd2b9b90f165e473c57c4f6227cfc6c"
 dependencies = [
  "serde",
  "serde_with",
@@ -2164,6 +2186,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.18"
@@ -2582,7 +2613,7 @@ dependencies = [
  "httparse",
  "humantime",
  "hyper 1.9.0",
- "itertools",
+ "itertools 0.14.0",
  "md-5 0.10.6",
  "parking_lot",
  "percent-encoding",
@@ -2796,6 +2827,7 @@ version = "0.28.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12"
 dependencies = [
+ "chrono",
  "libc",
  "once_cell",
  "portable-atomic",
@@ -2860,9 +2892,9 @@ dependencies = [
 
 [[package]]
 name = "quick_cache"
-version = "0.6.21"
+version = "0.6.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a70b1b8b47e31d0498ecbc3c5470bb931399a8bfed1fd79d1717a61ce7f96e3"
+checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9"
 dependencies = [
  "ahash",
  "equivalent",
@@ -3346,15 +3378,20 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
 
 [[package]]
 name = "rustytree"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
+ "async-trait",
+ "chrono",
  "futures",
  "icechunk",
  "numpy",
  "pyo3",
+ "rmp-serde",
+ "serde",
  "serde_json",
  "thiserror",
  "tokio",
+ "typetag",
  "zarrs",
  "zarrs_icechunk",
  "zarrs_object_store",
@@ -3510,9 +3547,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.149"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
+checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
 dependencies = [
  "indexmap 2.14.0",
  "itoa",
@@ -3547,11 +3584,12 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.19.0"
+version = "3.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f05839ce67618e14a09b286535c0d9c94e85ef25469b0e13cb4f844e5593eb19"
+checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
 dependencies = [
  "base64",
+ "bs58",
  "chrono",
  "hex",
  "indexmap 1.9.3",
@@ -3566,9 +3604,9 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.19.0"
+version = "3.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf2ebbe86054f9b45bc3881e865683ccfaccce97b9b4cb53f3039d67f355a334"
+checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
 dependencies = [
  "darling",
  "proc-macro2",
@@ -3928,9 +3966,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.52.2"
+version = "1.52.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386"
+checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
 dependencies = [
  "bytes",
  "libc",
@@ -4126,9 +4164,9 @@ checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
 
 [[package]]
 name = "typetag"
-version = "0.2.21"
+version = "0.2.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be2212c8a9b9bcfca32024de14998494cf9a5dfa59ea1b829de98bac374b86bf"
+checksum = "c5a897b12c6c1151ad0b138b8db50252dc301f93bc3b027db05eec82aeed298c"
 dependencies = [
  "erased-serde",
  "inventory",
@@ -4139,9 +4177,9 @@ dependencies = [
 
 [[package]]
 name = "typetag-impl"
-version = "0.2.21"
+version = "0.2.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846"
+checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4827,7 +4865,7 @@ dependencies = [
  "getrandom 0.3.4",
  "half",
  "inventory",
- "itertools",
+ "itertools 0.14.0",
  "itoa",
  "log",
  "lru",
@@ -4877,7 +4915,7 @@ checksum = "270efeb0181651aee5460b3232f2fc83e91bd646cefe75001d1c8f9a4f3abf81"
 dependencies = [
  "bytes",
  "derive_more",
- "itertools",
+ "itertools 0.14.0",
  "libc",
  "page_size",
  "pathdiff",
@@ -4974,7 +5012,7 @@ dependencies = [
  "bytes",
  "derive_more",
  "futures",
- "itertools",
+ "itertools 0.14.0",
  "thiserror",
  "unsafe_cell_slice",
 ]

{rustytree_xarray-0.2.1 → rustytree_xarray-0.3.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rustytree"
-version = "0.2.1"
+version = "0.3.0"
 edition = "2024"
 rust-version = "1.91.1"
 license = "Apache-2.0"
@@ -22,12 +22,27 @@ default = ["extension-module"]
 extension-module = ["pyo3/extension-module"]
 
 [dependencies]
-pyo3 = { version = "0.28" }
+# `chrono` feature lets us extract credential `expires_after` (a Python
+# `datetime`) straight into `DateTime<Utc>` in the credential-fetcher shim
+# (`src/py_credentials.rs`).
+pyo3 = { version = "0.28", features = ["chrono"] }
 numpy = "0.28"
 tokio = { version = "1", features = ["rt-multi-thread", "sync"] }
 futures = "0.3"
 thiserror = "2"
 
+# Credential-fetcher shim (`src/py_credentials.rs`): re-register icechunk-python's
+# `PythonCredentialsFetcher` typetag inside our cdylib so `Session::from_bytes`
+# can deserialize arraylake/Earthmover sessions. `typetag` MUST match the version
+# icechunk links (`=0.2.22` for icechunk 2.1.x) so the `inventory` registry is
+# shared and our impls are found by icechunk's `Session` deserializer; a skew
+# silently splits the registry. `chrono` unifies to icechunk's 0.4.x so
+# `DateTime<Utc>` types match.
+typetag = "=0.2.22"
+async-trait = "0.1"
+chrono = "0.4"
+serde = { version = "1", features = ["derive"] }
+
 # Zarr v3 reader stack. Remote object_store backends (S3/GCS/Azure/HTTP)
 # land in a follow-up; this PR ships local-filesystem only for both the
 # vanilla and icechunk paths.
@@ -36,7 +51,15 @@ thiserror = "2"
 # exported) so we don't have to track its semver-incompatible version
 # bumps separately. The `fs` feature on `zarrs_object_store` enables
 # `LocalFileSystem` in the underlying object_store.
-zarrs = { version = "0.22", features = ["async"] }
+#
+# `zlib` enables zarrs's `numcodecs.zlib` codec — a non-standard but common
+# numcodecs-namespace codec that zarr-python writes (e.g. the GOES-16 arraylake
+# dataset uses `bytes -> numcodecs.shuffle -> numcodecs.zlib`). zarrs documents
+# its impl as byte-compatible with zarr-python's, mirroring how icechunk reads
+# such data by delegating to zarr-python's numcodecs shim. `numcodecs.shuffle`
+# is always compiled by zarrs; the other common compressors (gzip, blosc, zstd,
+# crc32c) come from zarrs's default features, which we keep enabled.
+zarrs = { version = "0.22", features = ["async", "zlib"] }
 zarrs_storage = { version = "0.4", features = ["async"] }
 zarrs_object_store = { version = "0.6", features = ["fs", "aws"] }
 serde_json = "1"
@@ -45,9 +68,15 @@ serde_json = "1"
 # icechunk `Session` as `AsyncReadableListableStorageTraits` so the walk
 # code stays polymorphic: the same `open_single` runs against either an
 # icechunk session or a vanilla object_store-backed store.
-icechunk = "=2.0.5"
+icechunk = "=2.1.0"
 zarrs_icechunk = "0.5"
 
+# msgpack codec — the same one icechunk's `Session::{as,from}_bytes` use. We use
+# it to serialise the `ZarrsArrayHandle` pickle state (`ReopenSpec` + array path)
+# for `dask.distributed` (issue #44), and `src/py_credentials.rs` tests reuse it
+# to round-trip `S3Credentials`.
+rmp-serde = "1"
+
 [lints.clippy]
 pedantic = { level = "warn", priority = -1 }
 perf = { level = "deny", priority = -1 }