zarrs 0.1.5__cp311-abi3-musllinux_1_2_x86_64.whl → 0.2.0__cp311-abi3-musllinux_1_2_x86_64.whl

zarrs/_internal.pyi

@@ -13,10 +13,10 @@ class Basic:
 class CodecPipelineImpl:
     def __new__(
         cls,
-        metadata: builtins.str,
+        array_metadata: builtins.str,
+        store_config: StoreConfig,
         *,
         validate_checksums: builtins.bool | None = None,
-        store_empty_chunks: builtins.bool | None = None,
         chunk_concurrent_minimum: builtins.int | None = None,
         chunk_concurrent_maximum: builtins.int | None = None,
         num_threads: builtins.int | None = None,
@@ -30,6 +30,7 @@ class CodecPipelineImpl:
         self,
         chunk_descriptions: typing.Sequence[WithSubset],
         value: numpy.typing.NDArray[typing.Any],
+        write_empty_chunks: builtins.bool,
     ) -> None: ...
 
 class FilesystemStoreConfig:

zarrs/pipeline.py

@@ -2,25 +2,28 @@ from __future__ import annotations
 
 import asyncio
 import json
-import re
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, TypedDict
+from warnings import warn
 
 import numpy as np
 from zarr.abc.codec import Codec, CodecPipeline
+from zarr.codecs._v2 import V2Codec
 from zarr.core import BatchedCodecPipeline
 from zarr.core.config import config
+from zarr.core.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata
 
 if TYPE_CHECKING:
     from collections.abc import Generator, Iterable, Iterator
     from typing import Any, Self
 
-    from zarr.abc.store import ByteGetter, ByteSetter
+    from zarr.abc.store import ByteGetter, ByteSetter, Store
     from zarr.core.array_spec import ArraySpec
     from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer
     from zarr.core.chunk_grids import ChunkGrid
     from zarr.core.common import ChunkCoords
     from zarr.core.indexing import SelectorTuple
+    from zarr.dtype import ZDType
 
 from ._internal import CodecPipelineImpl, codec_metadata_v2_to_v3
 from .utils import (
@@ -39,12 +42,15 @@ class UnsupportedMetadataError(Exception):
     pass
 
 
-def get_codec_pipeline_impl(codec_metadata_json: str) -> CodecPipelineImpl | None:
+def get_codec_pipeline_impl(
+    metadata: ArrayMetadata, store: Store
+) -> CodecPipelineImpl | None:
     try:
+        array_metadata_json = json.dumps(metadata.to_dict())
         return CodecPipelineImpl(
-            codec_metadata_json,
+            array_metadata_json,
+            store_config=store,
             validate_checksums=config.get("codec_pipeline.validate_checksums", None),
-            store_empty_chunks=config.get("array.write_empty_chunks", None),
             chunk_concurrent_minimum=config.get(
                 "codec_pipeline.chunk_concurrent_minimum", None
             ),
@@ -54,10 +60,11 @@ def get_codec_pipeline_impl(codec_metadata_json: str) -> CodecPipelineImpl | Non
             num_threads=config.get("threading.max_workers", None),
         )
     except TypeError as e:
-        if re.match(r"codec (delta|zlib) is not supported", str(e)):
-            return None
-        else:
-            raise e
+        warn(
+            f"Array is unsupported by ZarrsCodecPipeline: {e}",
+            category=UserWarning,
+        )
+        return None
 
 
 def codecs_to_dict(codecs: Iterable[Codec]) -> Generator[dict[str, Any], None, None]:
@@ -88,37 +95,47 @@ class ZarrsCodecPipelineState(TypedDict):
     codecs: tuple[Codec, ...]
 
 
+def array_metadata_to_codecs(metadata: ArrayMetadata) -> list[Codec]:
+    if isinstance(metadata, ArrayV3Metadata):
+        return metadata.codecs
+    elif isinstance(metadata, ArrayV2Metadata):
+        v2_codec = V2Codec(filters=metadata.filters, compressor=metadata.compressor)
+        return [v2_codec]
+
+
 @dataclass
 class ZarrsCodecPipeline(CodecPipeline):
-    codecs: tuple[Codec, ...]
+    metadata: ArrayMetadata
+    store: Store
     impl: CodecPipelineImpl | None
-    codec_metadata_json: str
     python_impl: BatchedCodecPipeline
 
     def __getstate__(self) -> ZarrsCodecPipelineState:
-        return {"codec_metadata_json": self.codec_metadata_json, "codecs": self.codecs}
+        return {"metadata": self.metadata, "store": self.store}
 
     def __setstate__(self, state: ZarrsCodecPipelineState):
-        self.codecs = state["codecs"]
-        self.codec_metadata_json = state["codec_metadata_json"]
-        self.impl = get_codec_pipeline_impl(self.codec_metadata_json)
-        self.python_impl = BatchedCodecPipeline.from_codecs(self.codecs)
+        self.metadata = state["metadata"]
+        self.store = state["store"]
+        self.impl = get_codec_pipeline_impl(self.metadata, self.store)
+        codecs = array_metadata_to_codecs(self.metadata)
+        self.python_impl = BatchedCodecPipeline.from_codecs(codecs)
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
-        raise NotImplementedError("evolve_from_array_spec")
+        return self
 
     @classmethod
     def from_codecs(cls, codecs: Iterable[Codec]) -> Self:
-        codec_metadata = list(codecs_to_dict(codecs))
-        codec_metadata_json = json.dumps(codec_metadata)
-        # TODO: upstream zarr-python has not settled on how to deal with configs yet
-        # Should they be checked when an array is created, or when an operation is performed?
-        # https://github.com/zarr-developers/zarr-python/issues/2409
-        # https://github.com/zarr-developers/zarr-python/pull/2429#issuecomment-2566976567
+        return BatchedCodecPipeline.from_codecs(codecs)
+
+    @classmethod
+    def from_array_metadata_and_store(
+        cls, array_metadata: ArrayMetadata, store: Store
+    ) -> Self:
+        codecs = array_metadata_to_codecs(array_metadata)
         return cls(
-            codec_metadata_json=codec_metadata_json,
-            codecs=tuple(codecs),
-            impl=get_codec_pipeline_impl(codec_metadata_json),
+            metadata=array_metadata,
+            store=store,
+            impl=get_codec_pipeline_impl(array_metadata, store),
             python_impl=BatchedCodecPipeline.from_codecs(codecs),
         )
 
@@ -134,7 +151,7 @@ class ZarrsCodecPipeline(CodecPipeline):
         yield from self.codecs
 
     def validate(
-        self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid
+        self, *, shape: ChunkCoords, dtype: ZDType, chunk_grid: ChunkGrid
     ) -> None:
         raise NotImplementedError("validate")
 
@@ -184,7 +201,7 @@ class ZarrsCodecPipeline(CodecPipeline):
             out: NDArrayLike = out.as_ndarray_like()
             await asyncio.to_thread(
                 self.impl.retrieve_chunks_and_apply_index,
-                chunks_desc,
+                chunks_desc.chunk_info_with_indices,
                 out,
             )
             return None
@@ -223,7 +240,10 @@ class ZarrsCodecPipeline(CodecPipeline):
             elif not value_np.flags.c_contiguous:
                 value_np = np.ascontiguousarray(value_np)
             await asyncio.to_thread(
-                self.impl.store_chunks_with_indices, chunks_desc, value_np
+                self.impl.store_chunks_with_indices,
+                chunks_desc.chunk_info_with_indices,
+                value_np,
+                chunks_desc.write_empty_chunks,
             )
             return None
 
@@ -236,7 +256,7 @@ class ZarrsCodecPipeline(CodecPipeline):
         # https://github.com/LDeakin/zarrs/blob/0532fe983b7b42b59dbf84e50a2fe5e6f7bad4ce/zarrs_metadata/src/v2_to_v3.rs#L289-L293 for VSUMm
         # Further, our pipeline does not support variable-length objects due to limitations on decode_into, so object/np.dtypes.StringDType is also out
         if any(
-            info.dtype.kind in {"V", "S", "U", "M", "m", "O", "T"}
+            info.dtype.to_native_dtype().kind in {"V", "S", "U", "M", "m", "O", "T"}
             for (_, info, _, _, _) in batch_info
         ):
             raise UnsupportedDataTypeError()

zarrs/utils.py

@@ -2,13 +2,13 @@ from __future__ import annotations
 
 import operator
 import os
+from dataclasses import dataclass
 from functools import reduce
 from typing import TYPE_CHECKING, Any
 
 import numpy as np
 from zarr.core.array_spec import ArraySpec
 from zarr.core.indexing import SelectorTuple, is_integer
-from zarr.core.metadata.v2 import _default_fill_value
 
 from zarrs._internal import Basic, WithSubset
 
@@ -17,6 +17,7 @@ if TYPE_CHECKING:
     from types import EllipsisType
 
     from zarr.abc.store import ByteGetter, ByteSetter
+    from zarr.dtype import ZDType
 
 
 # adapted from https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
@@ -139,12 +140,18 @@ def get_shape_for_selector(
     return resulting_shape_from_index(shape, selector_tuple, drop_axes, pad=pad)
 
 
-def get_implicit_fill_value(dtype: np.dtype, fill_value: Any) -> Any:
+def get_implicit_fill_value(dtype: ZDType, fill_value: Any) -> Any:
     if fill_value is None:
-        fill_value = _default_fill_value(dtype)
+        fill_value = dtype.default_scalar()
     return fill_value
 
 
+@dataclass(frozen=True)
+class RustChunkInfo:
+    chunk_info_with_indices: list[WithSubset]
+    write_empty_chunks: bool
+
+
 def make_chunk_info_for_rust_with_indices(
     batch_info: Iterable[
         tuple[ByteGetter | ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
@@ -154,6 +161,7 @@ def make_chunk_info_for_rust_with_indices(
 ) -> list[WithSubset]:
     shape = shape if shape else (1,)  # constant array
     chunk_info_with_indices: list[WithSubset] = []
+    write_empty_chunks: bool = True
     for (
         byte_getter,
         chunk_spec,
@@ -161,6 +169,7 @@ def make_chunk_info_for_rust_with_indices(
         out_selection,
         _,
     ) in batch_info:
+        write_empty_chunks = chunk_spec.config.write_empty_chunks
         if chunk_spec.fill_value is None:
             chunk_spec = ArraySpec(
                 chunk_spec.shape,
@@ -193,4 +202,4 @@ def make_chunk_info_for_rust_with_indices(
                 shape=shape,
             )
         )
-    return chunk_info_with_indices
+    return RustChunkInfo(chunk_info_with_indices, write_empty_chunks)

{zarrs-0.1.5.dist-info → zarrs-0.2.0.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.3
 Name: zarrs
-Version: 0.1.5
+Version: 0.2.0
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Typing :: Typed
 Requires-Dist: numpy >=1.24
-Requires-Dist: zarr >=3.0.3, <3.1
+Requires-Dist: zarr >=3.1
 License-File: LICENSE
 Author: Ilan Gold, Lachlan Deakin, Philipp Angerer
 License: MIT
@@ -73,14 +73,13 @@ zarr.config.set({
     "codec_pipeline": {
         "path": "zarrs.ZarrsCodecPipeline",
         "validate_checksums": True,
-        "store_empty_chunks": False,
         "chunk_concurrent_maximum": None,
         "chunk_concurrent_minimum": 4,
     }
 })
 ```
 
-If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `store_empty_chunks`, `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value.  However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed.  This may change in the future as guidance from the `zarr` community becomes clear.
+If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value.  However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed.  This may change in the future as guidance from the `zarr` community becomes clear.
 
 ## Concurrency
 

zarrs-0.2.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+zarrs-0.2.0.dist-info/METADATA,sha256=XxZ7DwO-fwgs0UKqvISPYRv2C58rtWNZTFnUlQ4GN0M,7626
+zarrs-0.2.0.dist-info/WHEEL,sha256=8tKmwb3aeXaLJBYgSl6ahUJGBpB605-iMTJthul8CFo,106
+zarrs-0.2.0.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
+zarrs.libs/libgcc_s-1e52349c.so.1,sha256=Ani0u_W_tB8ESsFePO4D2mn2lrgWLhyFdw6RETxBTYM,437537
+zarrs.libs/libstdc++-e3b7606c.so.6,sha256=0od4MdFOKqq1GTnbD4qV7uMNWb0Hte3YHycydGf5nOU,18138121
+zarrs/__init__.py,sha256=4oWtWDZO8r7z4Uh7Fy_brmkxXDpULQdgjlA0iFw98eA,573
+zarrs/pipeline.py,sha256=9cog6mm1BMwxgRsKyO67mn_DyRdqsDrxykrTpyMyclo,9099
+zarrs/_internal.pyi,sha256=4WAVy2Upg3y_DX0j8AvdAr0_FAMeReDkS57XFc3Y9xE,1448
+zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+zarrs/utils.py,sha256=My01lFsg9ZFxkMajbTKHz7audaSxHJW6CviDjqo2tDs,7030
+zarrs/_internal.abi3.so,sha256=RVAg8MpchfGy666qPMe3pJrJxkaz1dHzHQoHbzcLJFw,10793089
+zarrs-0.2.0.dist-info/RECORD,,

zarrs-0.1.5.dist-info/RECORD

@@ -1,12 +0,0 @@
-zarrs-0.1.5.dist-info/METADATA,sha256=MraaZvJnr2BaC4xLsvHjNIWkaE_KMYsHfDRJ5CJ4nsY,7693
-zarrs-0.1.5.dist-info/WHEEL,sha256=8tKmwb3aeXaLJBYgSl6ahUJGBpB605-iMTJthul8CFo,106
-zarrs-0.1.5.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
-zarrs.libs/libstdc++-e3b7606c.so.6,sha256=0od4MdFOKqq1GTnbD4qV7uMNWb0Hte3YHycydGf5nOU,18138121
-zarrs.libs/libgcc_s-1e52349c.so.1,sha256=Ani0u_W_tB8ESsFePO4D2mn2lrgWLhyFdw6RETxBTYM,437537
-zarrs/__init__.py,sha256=4oWtWDZO8r7z4Uh7Fy_brmkxXDpULQdgjlA0iFw98eA,573
-zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-zarrs/pipeline.py,sha256=GV6Z1xoMdLQpnp4iX4nCuIYZTK1bPDG9nO0-t23F0_I,8708
-zarrs/_internal.pyi,sha256=H2afxhX5LDKOSR4bOBoH3mXoVqdM24Rc-xdjqMiOE5Y,1421
-zarrs/utils.py,sha256=uKIDi2EKRJRFdlyCElEvZrooVXkbmO4ZoE5fTHFNHGo,6763
-zarrs/_internal.abi3.so,sha256=67nE-R-OG-_yBFVb46tl5UWPDLYxGaq187-tKyJJk94,10498177
-zarrs-0.1.5.dist-info/RECORD,,