zarrs 0.1.5__cp311-abi3-macosx_10_12_x86_64.whl → 0.2.1__cp311-abi3-macosx_10_12_x86_64.whl

zarrs/__init__.py

@@ -1,5 +1,3 @@
-from zarr.registry import register_pipeline
-
 from ._internal import __version__
 from .pipeline import ZarrsCodecPipeline as _ZarrsCodecPipeline
 from .utils import CollapsedDimensionError, DiscontiguousArrayError
@@ -10,8 +8,6 @@ class ZarrsCodecPipeline(_ZarrsCodecPipeline):
     pass
 
 
-register_pipeline(ZarrsCodecPipeline)
-
 __all__ = [
     "ZarrsCodecPipeline",
     "DiscontiguousArrayError",

zarrs/_internal.pyi

@@ -3,9 +3,9 @@
 
 import builtins
 import typing
-from enum import Enum
 
 import numpy.typing
+import zarr.abc.store
 
 class Basic:
     def __new__(cls, byte_interface: typing.Any, chunk_spec: typing.Any) -> Basic: ...
@@ -13,13 +13,14 @@ class Basic:
 class CodecPipelineImpl:
     def __new__(
         cls,
-        metadata: builtins.str,
+        array_metadata: builtins.str,
+        store_config: zarr.abc.store.Store,
         *,
-        validate_checksums: builtins.bool | None = None,
-        store_empty_chunks: builtins.bool | None = None,
+        validate_checksums: builtins.bool = False,
         chunk_concurrent_minimum: builtins.int | None = None,
         chunk_concurrent_maximum: builtins.int | None = None,
         num_threads: builtins.int | None = None,
+        direct_io: builtins.bool = False,
     ) -> CodecPipelineImpl: ...
     def retrieve_chunks_and_apply_index(
         self,
@@ -30,14 +31,9 @@ class CodecPipelineImpl:
         self,
         chunk_descriptions: typing.Sequence[WithSubset],
         value: numpy.typing.NDArray[typing.Any],
+        write_empty_chunks: builtins.bool,
     ) -> None: ...
 
-class FilesystemStoreConfig:
-    root: builtins.str
-
-class HttpStoreConfig:
-    endpoint: builtins.str
-
 class WithSubset:
     def __new__(
         cls,
@@ -46,7 +42,3 @@ class WithSubset:
         subset: typing.Sequence[slice],
         shape: typing.Sequence[builtins.int],
     ) -> WithSubset: ...
-
-class StoreConfig(Enum):
-    Filesystem = ...
-    Http = ...

zarrs/pipeline.py

@@ -2,27 +2,29 @@ from __future__ import annotations
 
 import asyncio
 import json
-import re
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, TypedDict
+from warnings import warn
 
 import numpy as np
 from zarr.abc.codec import Codec, CodecPipeline
+from zarr.codecs._v2 import V2Codec
 from zarr.core import BatchedCodecPipeline
 from zarr.core.config import config
+from zarr.core.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata
 
 if TYPE_CHECKING:
-    from collections.abc import Generator, Iterable, Iterator
-    from typing import Any, Self
+    from collections.abc import Iterable, Iterator
+    from typing import Self
 
-    from zarr.abc.store import ByteGetter, ByteSetter
+    from zarr.abc.store import ByteGetter, ByteSetter, Store
     from zarr.core.array_spec import ArraySpec
     from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer
     from zarr.core.chunk_grids import ChunkGrid
-    from zarr.core.common import ChunkCoords
     from zarr.core.indexing import SelectorTuple
+    from zarr.dtype import ZDType
 
-from ._internal import CodecPipelineImpl, codec_metadata_v2_to_v3
+from ._internal import CodecPipelineImpl
 from .utils import (
     CollapsedDimensionError,
     DiscontiguousArrayError,
@@ -39,12 +41,19 @@ class UnsupportedMetadataError(Exception):
     pass
 
 
-def get_codec_pipeline_impl(codec_metadata_json: str) -> CodecPipelineImpl | None:
+def get_codec_pipeline_impl(
+    metadata: ArrayMetadata, store: Store
+) -> CodecPipelineImpl | None:
     try:
+        array_metadata_json = json.dumps(metadata.to_dict())
+        # Maintain old behavior: https://github.com/zarrs/zarrs-python/tree/b36ba797cafec77f5f41a25316be02c718a2b4f8?tab=readme-ov-file#configuration
+        validate_checksums = config.get("codec_pipeline.validate_checksums", True)
+        if validate_checksums is None:
+            validate_checksums = True
         return CodecPipelineImpl(
-            codec_metadata_json,
-            validate_checksums=config.get("codec_pipeline.validate_checksums", None),
-            store_empty_chunks=config.get("array.write_empty_chunks", None),
+            array_metadata_json,
+            store_config=store,
+            validate_checksums=validate_checksums,
             chunk_concurrent_minimum=config.get(
                 "codec_pipeline.chunk_concurrent_minimum", None
             ),
@@ -52,35 +61,14 @@ def get_codec_pipeline_impl(codec_metadata_json: str) -> CodecPipelineImpl | Non
                 "codec_pipeline.chunk_concurrent_maximum", None
             ),
             num_threads=config.get("threading.max_workers", None),
+            direct_io=config.get("codec_pipeline.direct_io", False),
         )
     except TypeError as e:
-        if re.match(r"codec (delta|zlib) is not supported", str(e)):
-            return None
-        else:
-            raise e
-
-
-def codecs_to_dict(codecs: Iterable[Codec]) -> Generator[dict[str, Any], None, None]:
-    for codec in codecs:
-        if codec.__class__.__name__ == "V2Codec":
-            codec_dict = codec.to_dict()
-            if codec_dict.get("filters", None) is not None:
-                filters = [
-                    json.dumps(filter.get_config())
-                    for filter in codec_dict.get("filters")
-                ]
-            else:
-                filters = None
-            if codec_dict.get("compressor", None) is not None:
-                compressor_json = codec_dict.get("compressor").get_config()
-                compressor = json.dumps(compressor_json)
-            else:
-                compressor = None
-            codecs_v3 = codec_metadata_v2_to_v3(filters, compressor)
-            for codec in codecs_v3:
-                yield json.loads(codec)
-        else:
-            yield codec.to_dict()
+        warn(
+            f"Array is unsupported by ZarrsCodecPipeline: {e}",
+            category=UserWarning,
+        )
+        return None
 
 
 class ZarrsCodecPipelineState(TypedDict):
@@ -88,37 +76,47 @@ class ZarrsCodecPipelineState(TypedDict):
     codecs: tuple[Codec, ...]
 
 
+def array_metadata_to_codecs(metadata: ArrayMetadata) -> list[Codec]:
+    if isinstance(metadata, ArrayV3Metadata):
+        return metadata.codecs
+    elif isinstance(metadata, ArrayV2Metadata):
+        v2_codec = V2Codec(filters=metadata.filters, compressor=metadata.compressor)
+        return [v2_codec]
+
+
 @dataclass
 class ZarrsCodecPipeline(CodecPipeline):
-    codecs: tuple[Codec, ...]
+    metadata: ArrayMetadata
+    store: Store
     impl: CodecPipelineImpl | None
-    codec_metadata_json: str
     python_impl: BatchedCodecPipeline
 
     def __getstate__(self) -> ZarrsCodecPipelineState:
-        return {"codec_metadata_json": self.codec_metadata_json, "codecs": self.codecs}
+        return {"metadata": self.metadata, "store": self.store}
 
     def __setstate__(self, state: ZarrsCodecPipelineState):
-        self.codecs = state["codecs"]
-        self.codec_metadata_json = state["codec_metadata_json"]
-        self.impl = get_codec_pipeline_impl(self.codec_metadata_json)
-        self.python_impl = BatchedCodecPipeline.from_codecs(self.codecs)
+        self.metadata = state["metadata"]
+        self.store = state["store"]
+        self.impl = get_codec_pipeline_impl(self.metadata, self.store)
+        codecs = array_metadata_to_codecs(self.metadata)
+        self.python_impl = BatchedCodecPipeline.from_codecs(codecs)
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
-        raise NotImplementedError("evolve_from_array_spec")
+        return self
 
     @classmethod
     def from_codecs(cls, codecs: Iterable[Codec]) -> Self:
-        codec_metadata = list(codecs_to_dict(codecs))
-        codec_metadata_json = json.dumps(codec_metadata)
-        # TODO: upstream zarr-python has not settled on how to deal with configs yet
-        # Should they be checked when an array is created, or when an operation is performed?
-        # https://github.com/zarr-developers/zarr-python/issues/2409
-        # https://github.com/zarr-developers/zarr-python/pull/2429#issuecomment-2566976567
+        return BatchedCodecPipeline.from_codecs(codecs)
+
+    @classmethod
+    def from_array_metadata_and_store(
+        cls, array_metadata: ArrayMetadata, store: Store
+    ) -> Self:
+        codecs = array_metadata_to_codecs(array_metadata)
         return cls(
-            codec_metadata_json=codec_metadata_json,
-            codecs=tuple(codecs),
-            impl=get_codec_pipeline_impl(codec_metadata_json),
+            metadata=array_metadata,
+            store=store,
+            impl=get_codec_pipeline_impl(array_metadata, store),
             python_impl=BatchedCodecPipeline.from_codecs(codecs),
         )
 
@@ -134,7 +132,7 @@ class ZarrsCodecPipeline(CodecPipeline):
         yield from self.codecs
 
     def validate(
-        self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid
+        self, *, shape: tuple[int, ...], dtype: ZDType, chunk_grid: ChunkGrid
     ) -> None:
         raise NotImplementedError("validate")
 
@@ -184,7 +182,7 @@ class ZarrsCodecPipeline(CodecPipeline):
             out: NDArrayLike = out.as_ndarray_like()
             await asyncio.to_thread(
                 self.impl.retrieve_chunks_and_apply_index,
-                chunks_desc,
+                chunks_desc.chunk_info_with_indices,
                 out,
             )
             return None
@@ -223,7 +221,10 @@ class ZarrsCodecPipeline(CodecPipeline):
             elif not value_np.flags.c_contiguous:
                 value_np = np.ascontiguousarray(value_np)
             await asyncio.to_thread(
-                self.impl.store_chunks_with_indices, chunks_desc, value_np
+                self.impl.store_chunks_with_indices,
+                chunks_desc.chunk_info_with_indices,
+                value_np,
+                chunks_desc.write_empty_chunks,
             )
             return None
 
@@ -236,7 +237,7 @@ class ZarrsCodecPipeline(CodecPipeline):
         # https://github.com/LDeakin/zarrs/blob/0532fe983b7b42b59dbf84e50a2fe5e6f7bad4ce/zarrs_metadata/src/v2_to_v3.rs#L289-L293 for VSUMm
         # Further, our pipeline does not support variable-length objects due to limitations on decode_into, so object/np.dtypes.StringDType is also out
         if any(
-            info.dtype.kind in {"V", "S", "U", "M", "m", "O", "T"}
+            info.dtype.to_native_dtype().kind in {"V", "S", "U", "M", "m", "O", "T"}
             for (_, info, _, _, _) in batch_info
         ):
             raise UnsupportedDataTypeError()

zarrs/utils.py

@@ -2,13 +2,13 @@ from __future__ import annotations
 
 import operator
 import os
+from dataclasses import dataclass
 from functools import reduce
 from typing import TYPE_CHECKING, Any
 
 import numpy as np
 from zarr.core.array_spec import ArraySpec
 from zarr.core.indexing import SelectorTuple, is_integer
-from zarr.core.metadata.v2 import _default_fill_value
 
 from zarrs._internal import Basic, WithSubset
 
@@ -17,6 +17,7 @@ if TYPE_CHECKING:
     from types import EllipsisType
 
     from zarr.abc.store import ByteGetter, ByteSetter
+    from zarr.dtype import ZDType
 
 
 # adapted from https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
@@ -139,21 +140,28 @@ def get_shape_for_selector(
     return resulting_shape_from_index(shape, selector_tuple, drop_axes, pad=pad)
 
 
-def get_implicit_fill_value(dtype: np.dtype, fill_value: Any) -> Any:
+def get_implicit_fill_value(dtype: ZDType, fill_value: Any) -> Any:
     if fill_value is None:
-        fill_value = _default_fill_value(dtype)
+        fill_value = dtype.default_scalar()
     return fill_value
 
 
+@dataclass(frozen=True)
+class RustChunkInfo:
+    chunk_info_with_indices: list[WithSubset]
+    write_empty_chunks: bool
+
+
 def make_chunk_info_for_rust_with_indices(
     batch_info: Iterable[
         tuple[ByteGetter | ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
     ],
     drop_axes: tuple[int, ...],
     shape: tuple[int, ...],
-) -> list[WithSubset]:
+) -> RustChunkInfo:
     shape = shape if shape else (1,)  # constant array
     chunk_info_with_indices: list[WithSubset] = []
+    write_empty_chunks: bool = True
     for (
         byte_getter,
         chunk_spec,
@@ -161,6 +169,7 @@ def make_chunk_info_for_rust_with_indices(
         out_selection,
         _,
     ) in batch_info:
+        write_empty_chunks = chunk_spec.config.write_empty_chunks
         if chunk_spec.fill_value is None:
             chunk_spec = ArraySpec(
                 chunk_spec.shape,
@@ -193,4 +202,4 @@ def make_chunk_info_for_rust_with_indices(
                 shape=shape,
             )
         )
-    return chunk_info_with_indices
+    return RustChunkInfo(chunk_info_with_indices, write_empty_chunks)

{zarrs-0.1.5.dist-info → zarrs-0.2.1.dist-info}/METADATA

@@ -1,15 +1,16 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: zarrs
-Version: 0.1.5
+Version: 0.2.1
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Typing :: Typed
-Requires-Dist: numpy >=1.24
-Requires-Dist: zarr >=3.0.3, <3.1
+Requires-Dist: numpy>=1.24
+Requires-Dist: zarr>=3.1
 License-File: LICENSE
+Summary: A CodecPipeline for zarr-python backed by the zarrs Rust crate
 Author: Ilan Gold, Lachlan Deakin, Philipp Angerer
-License: MIT
+License-Expression: MIT
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 
@@ -28,7 +29,6 @@ To use the project, simply install our package (which depends on `zarr-python>=3
 
 ```python
 import zarr
-import zarrs
 zarr.config.set({"codec_pipeline.path": "zarrs.ZarrsCodecPipeline"})
 ```
 
@@ -63,7 +63,9 @@ The `ZarrsCodecPipeline` specific options are:
 - `codec_pipeline.chunk_concurrent_minimum`: the minimum number of chunks retrieved/stored concurrently when balancing chunk/codec concurrency.
   - Defaults to 4 if `None`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#chunk-concurrent-minimum) for more info.
 - `codec_pipeline.validate_checksums`: enable checksum validation (e.g. with the CRC32C codec).
-  - Defaults to true if `None`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#validate-checksums) for more info.
+  - Defaults to `True`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#validate-checksums) for more info.
+- `codec_pipeline.direct_io`: enable `O_DIRECT` read/write, needs support from the operating system (currently only Linux) and file system.
+  - Defaults to `False`.
 
 For example:
 ```python
@@ -73,14 +75,14 @@ zarr.config.set({
     "codec_pipeline": {
         "path": "zarrs.ZarrsCodecPipeline",
         "validate_checksums": True,
-        "store_empty_chunks": False,
         "chunk_concurrent_maximum": None,
         "chunk_concurrent_minimum": 4,
+        "direct_io": False
     }
 })
 ```
 
-If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `store_empty_chunks`, `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value.  However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed.  This may change in the future as guidance from the `zarr` community becomes clear.
+If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value.  However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed.  This may change in the future as guidance from the `zarr` community becomes clear.
 
 ## Concurrency
 

zarrs-0.2.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+zarrs-0.2.1.dist-info/METADATA,sha256=8Y8S6VPAKwydDwl2oSRsMQlD_VSVUCtbfdm78FM4LZs,7878
+zarrs-0.2.1.dist-info/WHEEL,sha256=N8W3-0eDM6igWj-H12r7VkxoMaJIqJLxUyWCFstEaGg,105
+zarrs-0.2.1.dist-info/entry_points.txt,sha256=EzI6yCIUPDHBHzjDdexuGGYbOLXf8x2ICokOJXnuX3k,68
+zarrs-0.2.1.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
+zarrs/__init__.py,sha256=lRVtAPzCzJkGs4vQrW4UgANq-pC-khS0ZF7HTj4__Hg,489
+zarrs/_internal.abi3.so,sha256=zasHnt8Mym7fh5MkSq0_houhMcW-oYZm_zR1j31eJ5I,10334504
+zarrs/_internal.pyi,sha256=D_sHSSWGoVWLBS3q0dPE3pPJULZZ7cxV-9CErqGp8Z8,1327
+zarrs/pipeline.py,sha256=Ihc-RkEOqDTQvUtuwSJf8TBDVZMn2kYYH2_yATjydTI,8449
+zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+zarrs/utils.py,sha256=mGFsSnYU_jt4QJtlvr2JfxoBDZLe8V3Y8DbVfZSUpmA,7027
+zarrs-0.2.1.dist-info/RECORD,,

{zarrs-0.1.5.dist-info → zarrs-0.2.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: maturin (1.7.4)
+Generator: maturin (1.9.4)
 Root-Is-Purelib: false
 Tag: cp311-abi3-macosx_10_12_x86_64

zarrs-0.2.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[zarr.codec_pipeline]
+zarrs.codec_pipeline=zarrs:ZarrsCodecPipeline

zarrs-0.1.5.dist-info/RECORD

@@ -1,10 +0,0 @@
-zarrs-0.1.5.dist-info/METADATA,sha256=MraaZvJnr2BaC4xLsvHjNIWkaE_KMYsHfDRJ5CJ4nsY,7693
-zarrs-0.1.5.dist-info/WHEEL,sha256=LZygbeT1PTQw7a9tONPp78bbG4FZc86U59Z0RFJcoR8,105
-zarrs-0.1.5.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
-zarrs/__init__.py,sha256=4oWtWDZO8r7z4Uh7Fy_brmkxXDpULQdgjlA0iFw98eA,573
-zarrs/_internal.pyi,sha256=H2afxhX5LDKOSR4bOBoH3mXoVqdM24Rc-xdjqMiOE5Y,1421
-zarrs/utils.py,sha256=uKIDi2EKRJRFdlyCElEvZrooVXkbmO4ZoE5fTHFNHGo,6763
-zarrs/pipeline.py,sha256=GV6Z1xoMdLQpnp4iX4nCuIYZTK1bPDG9nO0-t23F0_I,8708
-zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-zarrs/_internal.abi3.so,sha256=rOHAgLFXNSc3DqJsmPOWY31M02VTg2vpNMhLovY5-TI,9573408
-zarrs-0.1.5.dist-info/RECORD,,