zarrs 0.1.5__tar.gz → 0.2.0__tar.gz

{zarrs-0.1.5 → zarrs-0.2.0}/Cargo.lock

@@ -244,9 +244,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.28"
+version = "1.2.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ad45f4f74e4e20eaa392913b7b33a7091c87e59628f4dd27888205ad888843c"
+checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362"
 dependencies = [
  "jobserver",
  "libc",
@@ -305,9 +305,9 @@ dependencies = [
 
 [[package]]
 name = "crc32fast"
-version = "1.4.2"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
 dependencies = [
  "cfg-if",
 ]
@@ -759,9 +759,9 @@ dependencies = [
 
 [[package]]
 name = "hyper-util"
-version = "0.1.14"
+version = "0.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb"
+checksum = "7f66d5bd4c6f02bf0542fad85d626775bab9258cf795a4256dcaf3161114d1df"
 dependencies = [
  "base64",
  "bytes",
@@ -1940,9 +1940,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.28"
+version = "0.23.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643"
+checksum = "2491382039b29b9b11ff08b76ff6c97cf287671dbb74f0be44bda389fffe9bd1"
 dependencies = [
  "once_cell",
  "ring",
@@ -1964,9 +1964,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.103.3"
+version = "0.103.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435"
+checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc"
 dependencies = [
  "ring",
  "rustls-pki-types",
@@ -2258,9 +2258,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.46.0"
+version = "1.46.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1140bb80481756a8cbe10541f37433b459c5aa1e727b4c020fbfebdc25bf3ec4"
+checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17"
 dependencies = [
  "backtrace",
  "bytes",
@@ -2893,9 +2893,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
-version = "0.7.11"
+version = "0.7.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd"
+checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95"
 dependencies = [
  "memchr",
 ]
@@ -2984,7 +2984,7 @@ dependencies = [
 
 [[package]]
 name = "zarrs-python"
-version = "0.1.5"
+version = "0.2.0"
 dependencies = [
  "itertools 0.14.0",
  "numpy",

{zarrs-0.1.5 → zarrs-0.2.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "zarrs-python"
-version = "0.1.5"
+version = "0.2.0"
 edition = "2021"
 publish = false
 

{zarrs-0.1.5 → zarrs-0.2.0}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.3
 Name: zarrs
-Version: 0.1.5
+Version: 0.2.0
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Typing :: Typed
 Requires-Dist: numpy >=1.24
-Requires-Dist: zarr >=3.0.3, <3.1
+Requires-Dist: zarr >=3.1
 License-File: LICENSE
 Author: Ilan Gold, Lachlan Deakin, Philipp Angerer
 License: MIT
@@ -73,14 +73,13 @@ zarr.config.set({
     "codec_pipeline": {
         "path": "zarrs.ZarrsCodecPipeline",
         "validate_checksums": True,
-        "store_empty_chunks": False,
         "chunk_concurrent_maximum": None,
         "chunk_concurrent_minimum": 4,
     }
 })
 ```
 
-If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `store_empty_chunks`, `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value.  However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed.  This may change in the future as guidance from the `zarr` community becomes clear.
+If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value.  However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed.  This may change in the future as guidance from the `zarr` community becomes clear.
 
 ## Concurrency
 

{zarrs-0.1.5 → zarrs-0.2.0}/README.md

@@ -58,14 +58,13 @@ zarr.config.set({
     "codec_pipeline": {
         "path": "zarrs.ZarrsCodecPipeline",
         "validate_checksums": True,
-        "store_empty_chunks": False,
         "chunk_concurrent_maximum": None,
         "chunk_concurrent_minimum": 4,
     }
 })
 ```
 
-If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `store_empty_chunks`, `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value.  However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed.  This may change in the future as guidance from the `zarr` community becomes clear.
+If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value.  However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed.  This may change in the future as guidance from the `zarr` community becomes clear.
 
 ## Concurrency
 

{zarrs-0.1.5 → zarrs-0.2.0}/pyproject.toml

@@ -20,7 +20,7 @@ classifiers = [
 dynamic = ["version"]
 dependencies = [
     "numpy>=1.24",
-    "zarr>=3.0.3,<3.1",
+    "zarr>=3.1",
 ]
 
 [dependency-groups]
@@ -31,6 +31,7 @@ test = [
     "pytest",
     "pytest-asyncio",
     "pytest-xdist",
+    "pytest-mock",
 ]
 doc = ["sphinx>=7.4.6", "myst-parser"]
 dev = [

{zarrs-0.1.5 → zarrs-0.2.0}/python/zarrs/_internal.pyi

@@ -13,10 +13,10 @@ class Basic:
 class CodecPipelineImpl:
     def __new__(
         cls,
-        metadata: builtins.str,
+        array_metadata: builtins.str,
+        store_config: StoreConfig,
         *,
         validate_checksums: builtins.bool | None = None,
-        store_empty_chunks: builtins.bool | None = None,
         chunk_concurrent_minimum: builtins.int | None = None,
         chunk_concurrent_maximum: builtins.int | None = None,
         num_threads: builtins.int | None = None,
@@ -30,6 +30,7 @@ class CodecPipelineImpl:
         self,
         chunk_descriptions: typing.Sequence[WithSubset],
         value: numpy.typing.NDArray[typing.Any],
+        write_empty_chunks: builtins.bool,
     ) -> None: ...
 
 class FilesystemStoreConfig:

{zarrs-0.1.5 → zarrs-0.2.0}/python/zarrs/pipeline.py

@@ -2,25 +2,28 @@ from __future__ import annotations
 
 import asyncio
 import json
-import re
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, TypedDict
+from warnings import warn
 
 import numpy as np
 from zarr.abc.codec import Codec, CodecPipeline
+from zarr.codecs._v2 import V2Codec
 from zarr.core import BatchedCodecPipeline
 from zarr.core.config import config
+from zarr.core.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata
 
 if TYPE_CHECKING:
     from collections.abc import Generator, Iterable, Iterator
     from typing import Any, Self
 
-    from zarr.abc.store import ByteGetter, ByteSetter
+    from zarr.abc.store import ByteGetter, ByteSetter, Store
     from zarr.core.array_spec import ArraySpec
     from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer
     from zarr.core.chunk_grids import ChunkGrid
     from zarr.core.common import ChunkCoords
     from zarr.core.indexing import SelectorTuple
+    from zarr.dtype import ZDType
 
 from ._internal import CodecPipelineImpl, codec_metadata_v2_to_v3
 from .utils import (
@@ -39,12 +42,15 @@ class UnsupportedMetadataError(Exception):
     pass
 
 
-def get_codec_pipeline_impl(codec_metadata_json: str) -> CodecPipelineImpl | None:
+def get_codec_pipeline_impl(
+    metadata: ArrayMetadata, store: Store
+) -> CodecPipelineImpl | None:
     try:
+        array_metadata_json = json.dumps(metadata.to_dict())
         return CodecPipelineImpl(
-            codec_metadata_json,
+            array_metadata_json,
+            store_config=store,
             validate_checksums=config.get("codec_pipeline.validate_checksums", None),
-            store_empty_chunks=config.get("array.write_empty_chunks", None),
             chunk_concurrent_minimum=config.get(
                 "codec_pipeline.chunk_concurrent_minimum", None
             ),
@@ -54,10 +60,11 @@ def get_codec_pipeline_impl(codec_metadata_json: str) -> CodecPipelineImpl | Non
             num_threads=config.get("threading.max_workers", None),
         )
     except TypeError as e:
-        if re.match(r"codec (delta|zlib) is not supported", str(e)):
-            return None
-        else:
-            raise e
+        warn(
+            f"Array is unsupported by ZarrsCodecPipeline: {e}",
+            category=UserWarning,
+        )
+        return None
 
 
 def codecs_to_dict(codecs: Iterable[Codec]) -> Generator[dict[str, Any], None, None]:
@@ -88,37 +95,47 @@ class ZarrsCodecPipelineState(TypedDict):
     codecs: tuple[Codec, ...]
 
 
+def array_metadata_to_codecs(metadata: ArrayMetadata) -> list[Codec]:
+    if isinstance(metadata, ArrayV3Metadata):
+        return metadata.codecs
+    elif isinstance(metadata, ArrayV2Metadata):
+        v2_codec = V2Codec(filters=metadata.filters, compressor=metadata.compressor)
+        return [v2_codec]
+
+
 @dataclass
 class ZarrsCodecPipeline(CodecPipeline):
-    codecs: tuple[Codec, ...]
+    metadata: ArrayMetadata
+    store: Store
     impl: CodecPipelineImpl | None
-    codec_metadata_json: str
     python_impl: BatchedCodecPipeline
 
     def __getstate__(self) -> ZarrsCodecPipelineState:
-        return {"codec_metadata_json": self.codec_metadata_json, "codecs": self.codecs}
+        return {"metadata": self.metadata, "store": self.store}
 
     def __setstate__(self, state: ZarrsCodecPipelineState):
-        self.codecs = state["codecs"]
-        self.codec_metadata_json = state["codec_metadata_json"]
-        self.impl = get_codec_pipeline_impl(self.codec_metadata_json)
-        self.python_impl = BatchedCodecPipeline.from_codecs(self.codecs)
+        self.metadata = state["metadata"]
+        self.store = state["store"]
+        self.impl = get_codec_pipeline_impl(self.metadata, self.store)
+        codecs = array_metadata_to_codecs(self.metadata)
+        self.python_impl = BatchedCodecPipeline.from_codecs(codecs)
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
-        raise NotImplementedError("evolve_from_array_spec")
+        return self
 
     @classmethod
     def from_codecs(cls, codecs: Iterable[Codec]) -> Self:
-        codec_metadata = list(codecs_to_dict(codecs))
-        codec_metadata_json = json.dumps(codec_metadata)
-        # TODO: upstream zarr-python has not settled on how to deal with configs yet
-        # Should they be checked when an array is created, or when an operation is performed?
-        # https://github.com/zarr-developers/zarr-python/issues/2409
-        # https://github.com/zarr-developers/zarr-python/pull/2429#issuecomment-2566976567
+        return BatchedCodecPipeline.from_codecs(codecs)
+
+    @classmethod
+    def from_array_metadata_and_store(
+        cls, array_metadata: ArrayMetadata, store: Store
+    ) -> Self:
+        codecs = array_metadata_to_codecs(array_metadata)
         return cls(
-            codec_metadata_json=codec_metadata_json,
-            codecs=tuple(codecs),
-            impl=get_codec_pipeline_impl(codec_metadata_json),
+            metadata=array_metadata,
+            store=store,
+            impl=get_codec_pipeline_impl(array_metadata, store),
             python_impl=BatchedCodecPipeline.from_codecs(codecs),
         )
 
@@ -134,7 +151,7 @@ class ZarrsCodecPipeline(CodecPipeline):
         yield from self.codecs
 
     def validate(
-        self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid
+        self, *, shape: ChunkCoords, dtype: ZDType, chunk_grid: ChunkGrid
     ) -> None:
         raise NotImplementedError("validate")
 
@@ -184,7 +201,7 @@ class ZarrsCodecPipeline(CodecPipeline):
             out: NDArrayLike = out.as_ndarray_like()
             await asyncio.to_thread(
                 self.impl.retrieve_chunks_and_apply_index,
-                chunks_desc,
+                chunks_desc.chunk_info_with_indices,
                 out,
             )
             return None
@@ -223,7 +240,10 @@ class ZarrsCodecPipeline(CodecPipeline):
             elif not value_np.flags.c_contiguous:
                 value_np = np.ascontiguousarray(value_np)
             await asyncio.to_thread(
-                self.impl.store_chunks_with_indices, chunks_desc, value_np
+                self.impl.store_chunks_with_indices,
+                chunks_desc.chunk_info_with_indices,
+                value_np,
+                chunks_desc.write_empty_chunks,
             )
             return None
 
@@ -236,7 +256,7 @@ class ZarrsCodecPipeline(CodecPipeline):
         # https://github.com/LDeakin/zarrs/blob/0532fe983b7b42b59dbf84e50a2fe5e6f7bad4ce/zarrs_metadata/src/v2_to_v3.rs#L289-L293 for VSUMm
         # Further, our pipeline does not support variable-length objects due to limitations on decode_into, so object/np.dtypes.StringDType is also out
         if any(
-            info.dtype.kind in {"V", "S", "U", "M", "m", "O", "T"}
+            info.dtype.to_native_dtype().kind in {"V", "S", "U", "M", "m", "O", "T"}
             for (_, info, _, _, _) in batch_info
         ):
             raise UnsupportedDataTypeError()

{zarrs-0.1.5 → zarrs-0.2.0}/python/zarrs/utils.py

@@ -2,13 +2,13 @@ from __future__ import annotations
 
 import operator
 import os
+from dataclasses import dataclass
 from functools import reduce
 from typing import TYPE_CHECKING, Any
 
 import numpy as np
 from zarr.core.array_spec import ArraySpec
 from zarr.core.indexing import SelectorTuple, is_integer
-from zarr.core.metadata.v2 import _default_fill_value
 
 from zarrs._internal import Basic, WithSubset
 
@@ -17,6 +17,7 @@ if TYPE_CHECKING:
     from types import EllipsisType
 
     from zarr.abc.store import ByteGetter, ByteSetter
+    from zarr.dtype import ZDType
 
 
 # adapted from https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
@@ -139,12 +140,18 @@ def get_shape_for_selector(
     return resulting_shape_from_index(shape, selector_tuple, drop_axes, pad=pad)
 
 
-def get_implicit_fill_value(dtype: np.dtype, fill_value: Any) -> Any:
+def get_implicit_fill_value(dtype: ZDType, fill_value: Any) -> Any:
     if fill_value is None:
-        fill_value = _default_fill_value(dtype)
+        fill_value = dtype.default_scalar()
     return fill_value
 
 
+@dataclass(frozen=True)
+class RustChunkInfo:
+    chunk_info_with_indices: list[WithSubset]
+    write_empty_chunks: bool
+
+
 def make_chunk_info_for_rust_with_indices(
     batch_info: Iterable[
         tuple[ByteGetter | ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
@@ -154,6 +161,7 @@ def make_chunk_info_for_rust_with_indices(
 ) -> list[WithSubset]:
     shape = shape if shape else (1,)  # constant array
     chunk_info_with_indices: list[WithSubset] = []
+    write_empty_chunks: bool = True
     for (
         byte_getter,
         chunk_spec,
@@ -161,6 +169,7 @@ def make_chunk_info_for_rust_with_indices(
         out_selection,
         _,
     ) in batch_info:
+        write_empty_chunks = chunk_spec.config.write_empty_chunks
         if chunk_spec.fill_value is None:
             chunk_spec = ArraySpec(
                 chunk_spec.shape,
@@ -193,4 +202,4 @@ def make_chunk_info_for_rust_with_indices(
                 shape=shape,
             )
         )
-    return chunk_info_with_indices
+    return RustChunkInfo(chunk_info_with_indices, write_empty_chunks)

{zarrs-0.1.5 → zarrs-0.2.0}/src/chunk_item.rs

@@ -14,10 +14,9 @@ use zarrs::{
     storage::StoreKey,
 };
 
-use crate::{store::StoreConfig, utils::PyErrExt};
+use crate::utils::PyErrExt;
 
 pub(crate) trait ChunksItem {
-    fn store_config(&self) -> StoreConfig;
     fn key(&self) -> &StoreKey;
     fn representation(&self) -> &ChunkRepresentation;
 }
@@ -26,7 +25,6 @@ pub(crate) trait ChunksItem {
 #[gen_stub_pyclass]
 #[pyclass]
 pub(crate) struct Basic {
-    store: StoreConfig,
     key: StoreKey,
     representation: ChunkRepresentation,
 }
@@ -62,12 +60,12 @@ fn fill_value_to_bytes(dtype: &str, fill_value: &Bound<'_, PyAny>) -> PyResult<V
 impl Basic {
     #[new]
     fn new(byte_interface: &Bound<'_, PyAny>, chunk_spec: &Bound<'_, PyAny>) -> PyResult<Self> {
-        let store: StoreConfig = byte_interface.getattr("store")?.extract()?;
         let path: String = byte_interface.getattr("path")?.extract()?;
 
         let chunk_shape = chunk_spec.getattr("shape")?.extract()?;
         let mut dtype: String = chunk_spec
             .getattr("dtype")?
+            .call_method0("to_native_dtype")?
             .call_method0("__str__")?
             .extract()?;
         if dtype == "object" {
@@ -78,7 +76,6 @@ impl Basic {
         let fill_value: Bound<'_, PyAny> = chunk_spec.getattr("fill_value")?;
         let fill_value_bytes = fill_value_to_bytes(&dtype, &fill_value)?;
         Ok(Self {
-            store,
             key: StoreKey::new(path).map_py_err::<PyValueError>()?,
             representation: get_chunk_representation(chunk_shape, &dtype, fill_value_bytes)?,
         })
@@ -117,9 +114,6 @@ impl WithSubset {
 }
 
 impl ChunksItem for Basic {
-    fn store_config(&self) -> StoreConfig {
-        self.store.clone()
-    }
     fn key(&self) -> &StoreKey {
         &self.key
     }
@@ -129,9 +123,6 @@ impl ChunksItem for Basic {
 }
 
 impl ChunksItem for WithSubset {
-    fn store_config(&self) -> StoreConfig {
-        self.item.store.clone()
-    }
     fn key(&self) -> &StoreKey {
         &self.item.key
     }

{zarrs-0.1.5 → zarrs-0.2.0}/src/lib.rs

@@ -20,14 +20,15 @@ use unsafe_cell_slice::UnsafeCellSlice;
 use utils::is_whole_chunk;
 use zarrs::array::codec::{
     ArrayPartialDecoderTraits, ArrayToBytesCodecTraits, CodecOptions, CodecOptionsBuilder,
+    StoragePartialDecoder,
 };
 use zarrs::array::{
-    copy_fill_value_into, update_array_bytes, ArrayBytes, ArrayBytesFixedDisjointView, ArraySize,
-    CodecChain, FillValue,
+    copy_fill_value_into, update_array_bytes, Array, ArrayBytes, ArrayBytesFixedDisjointView,
+    ArrayMetadata, ArraySize, CodecChain, FillValue,
 };
 use zarrs::array_subset::ArraySubset;
-use zarrs::metadata::v3::MetadataV3;
-use zarrs::storage::StoreKey;
+use zarrs::storage::store::MemoryStore;
+use zarrs::storage::{ReadableWritableListableStorage, StorageHandle, StoreKey};
 
 mod chunk_item;
 mod concurrency;
@@ -41,14 +42,14 @@ mod utils;
 use crate::chunk_item::ChunksItem;
 use crate::concurrency::ChunkConcurrentLimitAndCodecOptions;
 use crate::metadata_v2::codec_metadata_v2_to_v3;
-use crate::store::StoreManager;
+use crate::store::StoreConfig;
 use crate::utils::{PyErrExt as _, PyUntypedArrayExt as _};
 
 // TODO: Use a OnceLock for store with get_or_try_init when stabilised?
 #[gen_stub_pyclass]
 #[pyclass]
 pub struct CodecPipelineImpl {
-    pub(crate) stores: StoreManager,
+    pub(crate) store: ReadableWritableListableStorage,
     pub(crate) codec_chain: Arc<CodecChain>,
     pub(crate) codec_options: CodecOptions,
     pub(crate) chunk_concurrent_minimum: usize,
@@ -63,7 +64,7 @@ impl CodecPipelineImpl {
         codec_chain: &CodecChain,
         codec_options: &CodecOptions,
     ) -> PyResult<ArrayBytes<'a>> {
-        let value_encoded = self.stores.get(item)?;
+        let value_encoded = self.store.get(item.key()).map_py_err::<PyRuntimeError>()?;
         let value_decoded = if let Some(value_encoded) = value_encoded {
             let value_encoded: Vec<u8> = value_encoded.into(); // zero-copy in this case
             codec_chain
@@ -94,7 +95,7 @@ impl CodecPipelineImpl {
             .map_py_err::<PyValueError>()?;
 
         if value_decoded.is_fill_value(item.representation().fill_value()) {
-            self.stores.erase(item)
+            self.store.erase(item.key()).map_py_err::<PyRuntimeError>()
         } else {
             let value_encoded = codec_chain
                 .encode(value_decoded, item.representation(), codec_options)
@@ -102,7 +103,9 @@ impl CodecPipelineImpl {
                 .map_py_err::<PyRuntimeError>()?;
 
             // Store the encoded chunk
-            self.stores.set(item, value_encoded.into())
+            self.store
+                .set(item.key(), value_encoded.into())
+                .map_py_err::<PyRuntimeError>()
         }
     }
 
@@ -204,34 +207,35 @@ impl CodecPipelineImpl {
 #[pymethods]
 impl CodecPipelineImpl {
     #[pyo3(signature = (
-        metadata,
+        array_metadata,
+        store_config,
         *,
         validate_checksums=None,
-        store_empty_chunks=None,
         chunk_concurrent_minimum=None,
         chunk_concurrent_maximum=None,
         num_threads=None,
     ))]
     #[new]
     fn new(
-        metadata: &str,
+        array_metadata: &str,
+        store_config: StoreConfig,
         validate_checksums: Option<bool>,
-        store_empty_chunks: Option<bool>,
         chunk_concurrent_minimum: Option<usize>,
         chunk_concurrent_maximum: Option<usize>,
         num_threads: Option<usize>,
     ) -> PyResult<Self> {
-        let metadata: Vec<MetadataV3> =
-            serde_json::from_str(metadata).map_py_err::<PyTypeError>()?;
-        let codec_chain =
-            Arc::new(CodecChain::from_metadata(&metadata).map_py_err::<PyTypeError>()?);
+        let metadata: ArrayMetadata =
+            serde_json::from_str(array_metadata).map_py_err::<PyTypeError>()?;
+
+        // TODO: Add a direct metadata -> codec chain method to zarrs
+        let store = Arc::new(MemoryStore::new());
+        let array = Array::new_with_metadata(store, "/", metadata).map_py_err::<PyTypeError>()?;
+        let codec_chain = Arc::new(array.codecs().clone());
+
         let mut codec_options = CodecOptionsBuilder::new();
         if let Some(validate_checksums) = validate_checksums {
             codec_options = codec_options.validate_checksums(validate_checksums);
         }
-        if let Some(store_empty_chunks) = store_empty_chunks {
-            codec_options = codec_options.store_empty_chunks(store_empty_chunks);
-        }
         let codec_options = codec_options.build();
 
         let chunk_concurrent_minimum = chunk_concurrent_minimum
@@ -240,8 +244,11 @@ impl CodecPipelineImpl {
             chunk_concurrent_maximum.unwrap_or(rayon::current_num_threads());
         let num_threads = num_threads.unwrap_or(rayon::current_num_threads());
 
+        let store: ReadableWritableListableStorage =
+            (&store_config).try_into().map_py_err::<PyTypeError>()?;
+
         Ok(Self {
-            stores: StoreManager::default(),
+            store,
             codec_chain,
             codec_options,
             chunk_concurrent_minimum,
@@ -281,7 +288,9 @@ impl CodecPipelineImpl {
                 partial_chunk_descriptions,
                 map,
                 |item| {
-                    let input_handle = self.stores.decoder(item)?;
+                    let storage_handle = Arc::new(StorageHandle::new(self.store.clone()));
+                    let input_handle =
+                        StoragePartialDecoder::new(storage_handle, item.key().clone());
                     let partial_decoder = self
                         .codec_chain
                         .clone()
@@ -331,7 +340,9 @@ impl CodecPipelineImpl {
                     && chunk_subset.shape() == item.representation().shape_u64()
                 {
                     // See zarrs::array::Array::retrieve_chunk_into
-                    if let Some(chunk_encoded) = self.stores.get(&item)? {
+                    if let Some(chunk_encoded) =
+                        self.store.get(item.key()).map_py_err::<PyRuntimeError>()?
+                    {
                         // Decode the encoded data into the output buffer
                         let chunk_encoded: Vec<u8> = chunk_encoded.into();
                         self.codec_chain.decode_into(
@@ -378,6 +389,7 @@ impl CodecPipelineImpl {
         py: Python,
         chunk_descriptions: Vec<chunk_item::WithSubset>,
         value: &Bound<'_, PyUntypedArray>,
+        write_empty_chunks: bool,
     ) -> PyResult<()> {
         enum InputValue<'a> {
             Array(ArrayBytes<'a>),
@@ -395,11 +407,12 @@ impl CodecPipelineImpl {
         let input_shape: Vec<u64> = value.shape_zarr()?;
 
         // Adjust the concurrency based on the codec chain and the first chunk description
-        let Some((chunk_concurrent_limit, codec_options)) =
+        let Some((chunk_concurrent_limit, mut codec_options)) =
             chunk_descriptions.get_chunk_concurrent_limit_and_codec_options(self)?
         else {
             return Ok(());
         };
+        codec_options.set_store_empty_chunks(write_empty_chunks);
 
         py.allow_threads(move || {
             let store_chunk = |item: chunk_item::WithSubset| match &input {

{zarrs-0.1.5 → zarrs-0.2.0}/src/store/filesystem.rs

@@ -6,7 +6,7 @@ use zarrs::{filesystem::FilesystemStore, storage::ReadableWritableListableStorag
 
 use crate::utils::PyErrExt;
 
-#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Clone)]
 #[gen_stub_pyclass]
 #[pyclass]
 pub struct FilesystemStoreConfig {

{zarrs-0.1.5 → zarrs-0.2.0}/src/store/http.rs

@@ -6,7 +6,7 @@ use zarrs::storage::ReadableWritableListableStorage;
 
 use super::opendal_builder_to_sync_store;
 
-#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Clone)]
 #[gen_stub_pyclass]
 #[pyclass]
 pub struct HttpStoreConfig {