zarrs 0.2.0__cp311-abi3-manylinux_2_28_aarch64.whl → 0.2.2__cp311-abi3-manylinux_2_28_aarch64.whl

zarrs/__init__.py

@@ -1,5 +1,3 @@
-from zarr.registry import register_pipeline
-
 from ._internal import __version__
 from .pipeline import ZarrsCodecPipeline as _ZarrsCodecPipeline
 from .utils import CollapsedDimensionError, DiscontiguousArrayError
@@ -10,8 +8,6 @@ class ZarrsCodecPipeline(_ZarrsCodecPipeline):
     pass
 
 
-register_pipeline(ZarrsCodecPipeline)
-
 __all__ = [
     "ZarrsCodecPipeline",
     "DiscontiguousArrayError",

zarrs/_internal.pyi

@@ -3,51 +3,42 @@
 
 import builtins
 import typing
-from enum import Enum
 
 import numpy.typing
+import zarr.abc.store
 
-class Basic:
-    def __new__(cls, byte_interface: typing.Any, chunk_spec: typing.Any) -> Basic: ...
+@typing.final
+class ChunkItem:
+    def __new__(
+        cls,
+        key: builtins.str,
+        chunk_subset: typing.Sequence[slice],
+        chunk_shape: typing.Sequence[builtins.int],
+        subset: typing.Sequence[slice],
+        shape: typing.Sequence[builtins.int],
+    ) -> ChunkItem: ...
 
+@typing.final
 class CodecPipelineImpl:
     def __new__(
         cls,
         array_metadata: builtins.str,
-        store_config: StoreConfig,
+        store_config: zarr.abc.store.Store,
         *,
-        validate_checksums: builtins.bool | None = None,
+        validate_checksums: builtins.bool = False,
         chunk_concurrent_minimum: builtins.int | None = None,
         chunk_concurrent_maximum: builtins.int | None = None,
         num_threads: builtins.int | None = None,
+        direct_io: builtins.bool = False,
     ) -> CodecPipelineImpl: ...
     def retrieve_chunks_and_apply_index(
         self,
-        chunk_descriptions: typing.Sequence[WithSubset],
+        chunk_descriptions: typing.Sequence[ChunkItem],
         value: numpy.typing.NDArray[typing.Any],
     ) -> None: ...
     def store_chunks_with_indices(
         self,
-        chunk_descriptions: typing.Sequence[WithSubset],
+        chunk_descriptions: typing.Sequence[ChunkItem],
         value: numpy.typing.NDArray[typing.Any],
         write_empty_chunks: builtins.bool,
     ) -> None: ...
-
-class FilesystemStoreConfig:
-    root: builtins.str
-
-class HttpStoreConfig:
-    endpoint: builtins.str
-
-class WithSubset:
-    def __new__(
-        cls,
-        item: Basic,
-        chunk_subset: typing.Sequence[slice],
-        subset: typing.Sequence[slice],
-        shape: typing.Sequence[builtins.int],
-    ) -> WithSubset: ...
-
-class StoreConfig(Enum):
-    Filesystem = ...
-    Http = ...

zarrs/pipeline.py

@@ -14,18 +14,17 @@ from zarr.core.config import config
 from zarr.core.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata
 
 if TYPE_CHECKING:
-    from collections.abc import Generator, Iterable, Iterator
-    from typing import Any, Self
+    from collections.abc import Iterable, Iterator
+    from typing import Self
 
     from zarr.abc.store import ByteGetter, ByteSetter, Store
     from zarr.core.array_spec import ArraySpec
     from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer
     from zarr.core.chunk_grids import ChunkGrid
-    from zarr.core.common import ChunkCoords
     from zarr.core.indexing import SelectorTuple
     from zarr.dtype import ZDType
 
-from ._internal import CodecPipelineImpl, codec_metadata_v2_to_v3
+from ._internal import CodecPipelineImpl
 from .utils import (
     CollapsedDimensionError,
     DiscontiguousArrayError,
@@ -43,14 +42,18 @@ class UnsupportedMetadataError(Exception):
 
 
 def get_codec_pipeline_impl(
-    metadata: ArrayMetadata, store: Store
+    metadata: ArrayMetadata, store: Store, *, strict: bool
 ) -> CodecPipelineImpl | None:
     try:
         array_metadata_json = json.dumps(metadata.to_dict())
+        # Maintain old behavior: https://github.com/zarrs/zarrs-python/tree/b36ba797cafec77f5f41a25316be02c718a2b4f8?tab=readme-ov-file#configuration
+        validate_checksums = config.get("codec_pipeline.validate_checksums", True)
+        if validate_checksums is None:
+            validate_checksums = True
         return CodecPipelineImpl(
             array_metadata_json,
             store_config=store,
-            validate_checksums=config.get("codec_pipeline.validate_checksums", None),
+            validate_checksums=validate_checksums,
             chunk_concurrent_minimum=config.get(
                 "codec_pipeline.chunk_concurrent_minimum", None
             ),
@@ -58,8 +61,12 @@ def get_codec_pipeline_impl(
                 "codec_pipeline.chunk_concurrent_maximum", None
             ),
             num_threads=config.get("threading.max_workers", None),
+            direct_io=config.get("codec_pipeline.direct_io", False),
         )
     except TypeError as e:
+        if strict:
+            raise UnsupportedMetadataError() from e
+
         warn(
             f"Array is unsupported by ZarrsCodecPipeline: {e}",
             category=UserWarning,
@@ -67,27 +74,14 @@ def get_codec_pipeline_impl(
         return None
 
 
-def codecs_to_dict(codecs: Iterable[Codec]) -> Generator[dict[str, Any], None, None]:
-    for codec in codecs:
-        if codec.__class__.__name__ == "V2Codec":
-            codec_dict = codec.to_dict()
-            if codec_dict.get("filters", None) is not None:
-                filters = [
-                    json.dumps(filter.get_config())
-                    for filter in codec_dict.get("filters")
-                ]
-            else:
-                filters = None
-            if codec_dict.get("compressor", None) is not None:
-                compressor_json = codec_dict.get("compressor").get_config()
-                compressor = json.dumps(compressor_json)
-            else:
-                compressor = None
-            codecs_v3 = codec_metadata_v2_to_v3(filters, compressor)
-            for codec in codecs_v3:
-                yield json.loads(codec)
-        else:
-            yield codec.to_dict()
+def get_codec_pipeline_fallback(
+    metadata: ArrayMetadata, *, strict: bool
+) -> BatchedCodecPipeline | None:
+    if strict:
+        return None
+    else:
+        codecs = array_metadata_to_codecs(metadata)
+        return BatchedCodecPipeline.from_codecs(codecs)
 
 
 class ZarrsCodecPipelineState(TypedDict):
@@ -108,7 +102,7 @@ class ZarrsCodecPipeline(CodecPipeline):
     metadata: ArrayMetadata
     store: Store
     impl: CodecPipelineImpl | None
-    python_impl: BatchedCodecPipeline
+    python_impl: BatchedCodecPipeline | None
 
     def __getstate__(self) -> ZarrsCodecPipelineState:
         return {"metadata": self.metadata, "store": self.store}
@@ -116,9 +110,9 @@ class ZarrsCodecPipeline(CodecPipeline):
     def __setstate__(self, state: ZarrsCodecPipelineState):
         self.metadata = state["metadata"]
         self.store = state["store"]
-        self.impl = get_codec_pipeline_impl(self.metadata, self.store)
-        codecs = array_metadata_to_codecs(self.metadata)
-        self.python_impl = BatchedCodecPipeline.from_codecs(codecs)
+        strict = config.get("codec_pipeline.strict", False)
+        self.impl = get_codec_pipeline_impl(self.metadata, self.store, strict=strict)
+        self.python_impl = get_codec_pipeline_fallback(self.metadata, strict=strict)
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
         return self
@@ -131,12 +125,12 @@ class ZarrsCodecPipeline(CodecPipeline):
     def from_array_metadata_and_store(
         cls, array_metadata: ArrayMetadata, store: Store
     ) -> Self:
-        codecs = array_metadata_to_codecs(array_metadata)
+        strict = config.get("codec_pipeline.strict", False)
         return cls(
             metadata=array_metadata,
             store=store,
-            impl=get_codec_pipeline_impl(array_metadata, store),
-            python_impl=BatchedCodecPipeline.from_codecs(codecs),
+            impl=get_codec_pipeline_impl(array_metadata, store, strict=strict),
+            python_impl=get_codec_pipeline_fallback(array_metadata, strict=strict),
         )
 
     @property
@@ -151,7 +145,7 @@ class ZarrsCodecPipeline(CodecPipeline):
         yield from self.codecs
 
     def validate(
-        self, *, shape: ChunkCoords, dtype: ZDType, chunk_grid: ChunkGrid
+        self, *, shape: tuple[int, ...], dtype: ZDType, chunk_grid: ChunkGrid
     ) -> None:
         raise NotImplementedError("validate")
 
@@ -195,6 +189,8 @@ class ZarrsCodecPipeline(CodecPipeline):
             UnsupportedDataTypeError,
             FillValueNoneError,
         ):
+            if self.python_impl is None:
+                raise
             await self.python_impl.read(batch_info, out, drop_axes)
             return None
         else:
@@ -228,6 +224,8 @@ class ZarrsCodecPipeline(CodecPipeline):
             UnsupportedDataTypeError,
             FillValueNoneError,
         ):
+            if self.python_impl is None:
+                raise
             await self.python_impl.write(batch_info, value, drop_axes)
             return None
         else:

zarrs/utils.py

@@ -10,7 +10,7 @@ import numpy as np
 from zarr.core.array_spec import ArraySpec
 from zarr.core.indexing import SelectorTuple, is_integer
 
-from zarrs._internal import Basic, WithSubset
+from zarrs._internal import ChunkItem
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
@@ -148,7 +148,7 @@ def get_implicit_fill_value(dtype: ZDType, fill_value: Any) -> Any:
 
 @dataclass(frozen=True)
 class RustChunkInfo:
-    chunk_info_with_indices: list[WithSubset]
+    chunk_info_with_indices: list[ChunkItem]
     write_empty_chunks: bool
 
 
@@ -158,9 +158,9 @@ def make_chunk_info_for_rust_with_indices(
     ],
     drop_axes: tuple[int, ...],
     shape: tuple[int, ...],
-) -> list[WithSubset]:
+) -> RustChunkInfo:
     shape = shape if shape else (1,)  # constant array
-    chunk_info_with_indices: list[WithSubset] = []
+    chunk_info_with_indices: list[ChunkItem] = []
     write_empty_chunks: bool = True
     for (
         byte_getter,
@@ -178,7 +178,6 @@ def make_chunk_info_for_rust_with_indices(
                 chunk_spec.config,
                 chunk_spec.prototype,
             )
-        chunk_info = Basic(byte_getter, chunk_spec)
         out_selection_as_slices = selector_tuple_to_slice_selection(out_selection)
         chunk_selection_as_slices = selector_tuple_to_slice_selection(chunk_selection)
         shape_chunk_selection_slices = get_shape_for_selector(
@@ -195,9 +194,10 @@ def make_chunk_info_for_rust_with_indices(
                 f"{shape_chunk_selection} != {shape_chunk_selection_slices}"
             )
         chunk_info_with_indices.append(
-            WithSubset(
-                chunk_info,
+            ChunkItem(
+                key=byte_getter.path,
                 chunk_subset=chunk_selection_as_slices,
+                chunk_shape=chunk_spec.shape,
                 subset=out_selection_as_slices,
                 shape=shape,
             )

{zarrs-0.2.0.dist-info → zarrs-0.2.2.dist-info}/METADATA

@@ -1,15 +1,16 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: zarrs
-Version: 0.2.0
+Version: 0.2.2
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Typing :: Typed
-Requires-Dist: numpy >=1.24
-Requires-Dist: zarr >=3.1
+Requires-Dist: numpy>=1.24
+Requires-Dist: zarr>=3.1
 License-File: LICENSE
+Summary: A CodecPipeline for zarr-python backed by the zarrs Rust crate
 Author: Ilan Gold, Lachlan Deakin, Philipp Angerer
-License: MIT
+License-Expression: MIT
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 
@@ -28,7 +29,6 @@ To use the project, simply install our package (which depends on `zarr-python>=3
 
 ```python
 import zarr
-import zarrs
 zarr.config.set({"codec_pipeline.path": "zarrs.ZarrsCodecPipeline"})
 ```
 
@@ -40,12 +40,11 @@ We export a `ZarrsCodecPipeline` class so that `zarr-python` can use the class b
 
 At the moment, we only support a subset of the `zarr-python` stores:
 
-- [x] [LocalStore](https://zarr.readthedocs.io/en/latest/_autoapi/zarr/storage/index.html#zarr.storage.LocalStore) (FileSystem)
-- [FsspecStore](https://zarr.readthedocs.io/en/latest/_autoapi/zarr/storage/index.html#zarr.storage.FsspecStore)
-  - [x] [HTTPFileSystem](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.implementations.http.HTTPFileSystem)
+- [`LocalStore`](https://zarr.readthedocs.io/en/latest/api/zarr/storage/#zarr.storage.LocalStore) (local filesystem)
+- [`ObjectStore`](https://zarr.readthedocs.io/en/latest/user-guide/storage/#object-store) (cloud storage)
+- [`HTTPFileSystem`](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.implementations.http.HTTPFileSystem) via [`FsspecStore`](https://zarr.readthedocs.io/en/latest/api/zarr/storage/#zarr.storage.FsspecStore)
 
 A `NotImplementedError` will be raised if a store is not supported.
-We intend to support more stores in the future: https://github.com/zarrs/zarrs-python/issues/44.
 
 ### Configuration
 
@@ -63,7 +62,11 @@ The `ZarrsCodecPipeline` specific options are:
 - `codec_pipeline.chunk_concurrent_minimum`: the minimum number of chunks retrieved/stored concurrently when balancing chunk/codec concurrency.
   - Defaults to 4 if `None`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#chunk-concurrent-minimum) for more info.
 - `codec_pipeline.validate_checksums`: enable checksum validation (e.g. with the CRC32C codec).
-  - Defaults to true if `None`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#validate-checksums) for more info.
+  - Defaults to `True`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#validate-checksums) for more info.
+- `codec_pipeline.direct_io`: enable `O_DIRECT` read/write, needs support from the operating system (currently only Linux) and file system.
+  - Defaults to `False`.
+- `codec_pipeline.strict`: raise exceptions for unsupported operations instead of falling back to the default codec pipeline of `zarr-python`.
+  - Defaults to `False`.
 
 For example:
 ```python
@@ -75,6 +78,8 @@ zarr.config.set({
         "validate_checksums": True,
         "chunk_concurrent_maximum": None,
         "chunk_concurrent_minimum": 4,
+        "direct_io": False,
+        "strict": False
     }
 })
 ```

zarrs-0.2.2.dist-info/RECORD

@@ -0,0 +1,11 @@
+zarrs-0.2.2.dist-info/METADATA,sha256=8K1AOS_SVQgRLzQ2rWtNbJxCAb12XmTLRrT6QklQgOI,8054
+zarrs-0.2.2.dist-info/WHEEL,sha256=pQhpX1zEYym7lHVMSudaqVXS44siTf8XBrjmsFdZ39M,108
+zarrs-0.2.2.dist-info/entry_points.txt,sha256=EzI6yCIUPDHBHzjDdexuGGYbOLXf8x2ICokOJXnuX3k,68
+zarrs-0.2.2.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
+zarrs/__init__.py,sha256=lRVtAPzCzJkGs4vQrW4UgANq-pC-khS0ZF7HTj4__Hg,489
+zarrs/_internal.abi3.so,sha256=2oa6VVFlcQ-C3qa6UK_LKKzRFc36GhmZfzKXqiJnvLA,15343720
+zarrs/_internal.pyi,sha256=a_D4yx99r4xeQX1ntY_A_Q4wVmLeLwJZHWAQV_mVu9A,1308
+zarrs/pipeline.py,sha256=YfB13GWNfxELerXVtJ_ipFwSL7bN-YuPys6jCB9lnms,9008
+zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+zarrs/utils.py,sha256=W2XCjJDVrdHYJgtVaRKN533Ljw1MF7o0YwXuz5ZAk2g,7020
+zarrs-0.2.2.dist-info/RECORD,,

{zarrs-0.2.0.dist-info → zarrs-0.2.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: maturin (1.7.4)
+Generator: maturin (1.9.4)
 Root-Is-Purelib: false
 Tag: cp311-abi3-manylinux_2_28_aarch64

zarrs-0.2.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[zarr.codec_pipeline]
+zarrs.codec_pipeline=zarrs:ZarrsCodecPipeline

zarrs-0.2.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-zarrs-0.2.0.dist-info/METADATA,sha256=XxZ7DwO-fwgs0UKqvISPYRv2C58rtWNZTFnUlQ4GN0M,7626
-zarrs-0.2.0.dist-info/WHEEL,sha256=vlVK9XRfjbZsffx4VpMMyFV477nGomrs3TkCmK8HoZM,108
-zarrs-0.2.0.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
-zarrs/__init__.py,sha256=4oWtWDZO8r7z4Uh7Fy_brmkxXDpULQdgjlA0iFw98eA,573
-zarrs/pipeline.py,sha256=9cog6mm1BMwxgRsKyO67mn_DyRdqsDrxykrTpyMyclo,9099
-zarrs/_internal.pyi,sha256=4WAVy2Upg3y_DX0j8AvdAr0_FAMeReDkS57XFc3Y9xE,1448
-zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-zarrs/utils.py,sha256=My01lFsg9ZFxkMajbTKHz7audaSxHJW6CviDjqo2tDs,7030
-zarrs/_internal.abi3.so,sha256=J5mngjmu3CE_IywczCJY9MM0NYh3VLXyOqlXN8As41o,9929656
-zarrs-0.2.0.dist-info/RECORD,,