zarrs 0.1.0__cp311-abi3-macosx_10_12_x86_64.whl → 0.2.2__cp311-abi3-macosx_10_12_x86_64.whl

zarrs/__init__.py

@@ -1,5 +1,3 @@
-from zarr.registry import register_pipeline
-
 from ._internal import __version__
 from .pipeline import ZarrsCodecPipeline as _ZarrsCodecPipeline
 from .utils import CollapsedDimensionError, DiscontiguousArrayError
@@ -10,8 +8,6 @@ class ZarrsCodecPipeline(_ZarrsCodecPipeline):
     pass
 
 
-register_pipeline(ZarrsCodecPipeline)
-
 __all__ = [
     "ZarrsCodecPipeline",
     "DiscontiguousArrayError",

zarrs/_internal.pyi

@@ -1,47 +1,44 @@
 # This file is automatically generated by pyo3_stub_gen
 # ruff: noqa: E501, F401
 
+import builtins
 import typing
 
-import numpy
 import numpy.typing
+import zarr.abc.store
 
+@typing.final
+class ChunkItem:
+    def __new__(
+        cls,
+        key: builtins.str,
+        chunk_subset: typing.Sequence[slice],
+        chunk_shape: typing.Sequence[builtins.int],
+        subset: typing.Sequence[slice],
+        shape: typing.Sequence[builtins.int],
+    ) -> ChunkItem: ...
+
+@typing.final
 class CodecPipelineImpl:
     def __new__(
         cls,
-        metadata,
+        array_metadata: builtins.str,
+        store_config: zarr.abc.store.Store,
         *,
-        validate_checksums=...,
-        store_empty_chunks=...,
-        chunk_concurrent_minimum=...,
-        chunk_concurrent_maximum=...,
-        num_threads=...,
-    ): ...
+        validate_checksums: builtins.bool = False,
+        chunk_concurrent_minimum: builtins.int | None = None,
+        chunk_concurrent_maximum: builtins.int | None = None,
+        num_threads: builtins.int | None = None,
+        direct_io: builtins.bool = False,
+    ) -> CodecPipelineImpl: ...
     def retrieve_chunks_and_apply_index(
         self,
-        chunk_descriptions: typing.Sequence[
-            tuple[
-                tuple[str, typing.Sequence[int], str, typing.Sequence[int]],
-                typing.Sequence[slice],
-                typing.Sequence[slice],
-            ]
-        ],
-        value: numpy.NDArray[typing.Any],
+        chunk_descriptions: typing.Sequence[ChunkItem],
+        value: numpy.typing.NDArray[typing.Any],
     ) -> None: ...
-    def retrieve_chunks(
-        self,
-        chunk_descriptions: typing.Sequence[
-            tuple[str, typing.Sequence[int], str, typing.Sequence[int]]
-        ],
-    ) -> list[numpy.typing.NDArray[numpy.uint8]]: ...
     def store_chunks_with_indices(
         self,
-        chunk_descriptions: typing.Sequence[
-            tuple[
-                tuple[str, typing.Sequence[int], str, typing.Sequence[int]],
-                typing.Sequence[slice],
-                typing.Sequence[slice],
-            ]
-        ],
-        value: numpy.NDArray[typing.Any],
+        chunk_descriptions: typing.Sequence[ChunkItem],
+        value: numpy.typing.NDArray[typing.Any],
+        write_empty_chunks: builtins.bool,
     ) -> None: ...

zarrs/pipeline.py

@@ -3,68 +3,134 @@ from __future__ import annotations
 import asyncio
 import json
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, TypedDict
+from warnings import warn
 
 import numpy as np
-from zarr.abc.codec import (
-    Codec,
-    CodecPipeline,
-)
+from zarr.abc.codec import Codec, CodecPipeline
+from zarr.codecs._v2 import V2Codec
+from zarr.core import BatchedCodecPipeline
 from zarr.core.config import config
+from zarr.core.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Iterator
     from typing import Self
 
-    from zarr.abc.store import ByteGetter, ByteSetter
+    from zarr.abc.store import ByteGetter, ByteSetter, Store
     from zarr.core.array_spec import ArraySpec
-    from zarr.core.buffer import Buffer, NDBuffer
+    from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer
     from zarr.core.chunk_grids import ChunkGrid
-    from zarr.core.common import ChunkCoords
     from zarr.core.indexing import SelectorTuple
+    from zarr.dtype import ZDType
 
 from ._internal import CodecPipelineImpl
 from .utils import (
     CollapsedDimensionError,
     DiscontiguousArrayError,
-    make_chunk_info_for_rust,
+    FillValueNoneError,
     make_chunk_info_for_rust_with_indices,
 )
 
 
-@dataclass(frozen=True)
-class ZarrsCodecPipeline(CodecPipeline):
+class UnsupportedDataTypeError(Exception):
+    pass
+
+
+class UnsupportedMetadataError(Exception):
+    pass
+
+
+def get_codec_pipeline_impl(
+    metadata: ArrayMetadata, store: Store, *, strict: bool
+) -> CodecPipelineImpl | None:
+    try:
+        array_metadata_json = json.dumps(metadata.to_dict())
+        # Maintain old behavior: https://github.com/zarrs/zarrs-python/tree/b36ba797cafec77f5f41a25316be02c718a2b4f8?tab=readme-ov-file#configuration
+        validate_checksums = config.get("codec_pipeline.validate_checksums", True)
+        if validate_checksums is None:
+            validate_checksums = True
+        return CodecPipelineImpl(
+            array_metadata_json,
+            store_config=store,
+            validate_checksums=validate_checksums,
+            chunk_concurrent_minimum=config.get(
+                "codec_pipeline.chunk_concurrent_minimum", None
+            ),
+            chunk_concurrent_maximum=config.get(
+                "codec_pipeline.chunk_concurrent_maximum", None
+            ),
+            num_threads=config.get("threading.max_workers", None),
+            direct_io=config.get("codec_pipeline.direct_io", False),
+        )
+    except TypeError as e:
+        if strict:
+            raise UnsupportedMetadataError() from e
+
+        warn(
+            f"Array is unsupported by ZarrsCodecPipeline: {e}",
+            category=UserWarning,
+        )
+        return None
+
+
+def get_codec_pipeline_fallback(
+    metadata: ArrayMetadata, *, strict: bool
+) -> BatchedCodecPipeline | None:
+    if strict:
+        return None
+    else:
+        codecs = array_metadata_to_codecs(metadata)
+        return BatchedCodecPipeline.from_codecs(codecs)
+
+
+class ZarrsCodecPipelineState(TypedDict):
+    codec_metadata_json: str
     codecs: tuple[Codec, ...]
-    impl: CodecPipelineImpl
+
+
+def array_metadata_to_codecs(metadata: ArrayMetadata) -> list[Codec]:
+    if isinstance(metadata, ArrayV3Metadata):
+        return metadata.codecs
+    elif isinstance(metadata, ArrayV2Metadata):
+        v2_codec = V2Codec(filters=metadata.filters, compressor=metadata.compressor)
+        return [v2_codec]
+
+
+@dataclass
+class ZarrsCodecPipeline(CodecPipeline):
+    metadata: ArrayMetadata
+    store: Store
+    impl: CodecPipelineImpl | None
+    python_impl: BatchedCodecPipeline | None
+
+    def __getstate__(self) -> ZarrsCodecPipelineState:
+        return {"metadata": self.metadata, "store": self.store}
+
+    def __setstate__(self, state: ZarrsCodecPipelineState):
+        self.metadata = state["metadata"]
+        self.store = state["store"]
+        strict = config.get("codec_pipeline.strict", False)
+        self.impl = get_codec_pipeline_impl(self.metadata, self.store, strict=strict)
+        self.python_impl = get_codec_pipeline_fallback(self.metadata, strict=strict)
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
-        raise NotImplementedError("evolve_from_array_spec")
+        return self
 
     @classmethod
     def from_codecs(cls, codecs: Iterable[Codec]) -> Self:
-        codec_metadata = [codec.to_dict() for codec in codecs]
-        codec_metadata_json = json.dumps(codec_metadata)
-        # TODO: upstream zarr-python has not settled on how to deal with configs yet
-        # Should they be checked when an array is created, or when an operation is performed?
-        # https://github.com/zarr-developers/zarr-python/issues/2409
-        # https://github.com/zarr-developers/zarr-python/pull/2429
+        return BatchedCodecPipeline.from_codecs(codecs)
+
+    @classmethod
+    def from_array_metadata_and_store(
+        cls, array_metadata: ArrayMetadata, store: Store
+    ) -> Self:
+        strict = config.get("codec_pipeline.strict", False)
         return cls(
-            codecs=tuple(codecs),
-            impl=CodecPipelineImpl(
-                codec_metadata_json,
-                validate_checksums=config.get(
-                    "codec_pipeline.validate_checksums", None
-                ),
-                # TODO: upstream zarr-python array.write_empty_chunks is not merged yet #2429
-                store_empty_chunks=config.get("array.write_empty_chunks", None),
-                chunk_concurrent_minimum=config.get(
-                    "codec_pipeline.chunk_concurrent_minimum", None
-                ),
-                chunk_concurrent_maximum=config.get(
-                    "codec_pipeline.chunk_concurrent_maximum", None
-                ),
-                num_threads=config.get("threading.max_workers", None),
-            ),
+            metadata=array_metadata,
+            store=store,
+            impl=get_codec_pipeline_impl(array_metadata, store, strict=strict),
+            python_impl=get_codec_pipeline_fallback(array_metadata, strict=strict),
         )
 
     @property
@@ -79,7 +145,7 @@ class ZarrsCodecPipeline(CodecPipeline):
         yield from self.codecs
 
     def validate(
-        self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid
+        self, *, shape: tuple[int, ...], dtype: ZDType, chunk_grid: ChunkGrid
     ) -> None:
         raise NotImplementedError("validate")
 
@@ -101,55 +167,94 @@ class ZarrsCodecPipeline(CodecPipeline):
     async def read(
         self,
         batch_info: Iterable[
-            tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]
+            tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
         ],
-        out: NDBuffer,
+        out: NDBuffer,  # type: ignore
         drop_axes: tuple[int, ...] = (),  # FIXME: unused
     ) -> None:
-        out = out.as_ndarray_like()  # FIXME: Error if array is not in host memory
+        # FIXME: Error if array is not in host memory
         if not out.dtype.isnative:
             raise RuntimeError("Non-native byte order not supported")
         try:
-            chunks_desc = make_chunk_info_for_rust_with_indices(batch_info, drop_axes)
-            index_in_rust = True
-        except (DiscontiguousArrayError, CollapsedDimensionError):
-            chunks_desc = make_chunk_info_for_rust(batch_info)
-            index_in_rust = False
-        if index_in_rust:
+            if self.impl is None:
+                raise UnsupportedMetadataError()
+            self._raise_error_on_unsupported_batch_dtype(batch_info)
+            chunks_desc = make_chunk_info_for_rust_with_indices(
+                batch_info, drop_axes, out.shape
+            )
+        except (
+            UnsupportedMetadataError,
+            DiscontiguousArrayError,
+            CollapsedDimensionError,
+            UnsupportedDataTypeError,
+            FillValueNoneError,
+        ):
+            if self.python_impl is None:
+                raise
+            await self.python_impl.read(batch_info, out, drop_axes)
+            return None
+        else:
+            out: NDArrayLike = out.as_ndarray_like()
             await asyncio.to_thread(
                 self.impl.retrieve_chunks_and_apply_index,
-                chunks_desc,
+                chunks_desc.chunk_info_with_indices,
                 out,
             )
             return None
-        chunks = await asyncio.to_thread(self.impl.retrieve_chunks, chunks_desc)
-        for chunk, chunk_info in zip(chunks, batch_info):
-            out_selection = chunk_info[3]
-            selection = chunk_info[2]
-            spec = chunk_info[1]
-            chunk_reshaped = chunk.view(spec.dtype).reshape(spec.shape)
-            chunk_selected = chunk_reshaped[selection]
-            if drop_axes:
-                chunk_selected = np.squeeze(chunk_selected, axis=drop_axes)
-            out[out_selection] = chunk_selected
 
     async def write(
         self,
         batch_info: Iterable[
-            tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]
+            tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
         ],
-        value: NDBuffer,
+        value: NDBuffer,  # type: ignore
         drop_axes: tuple[int, ...] = (),
     ) -> None:
-        value = value.as_ndarray_like()  # FIXME: Error if array is not in host memory
-        if not value.dtype.isnative:
-            value = np.ascontiguousarray(value, dtype=value.dtype.newbyteorder("="))
-        elif not value.flags.c_contiguous:
-            value = np.ascontiguousarray(value)
-        chunks_desc = make_chunk_info_for_rust_with_indices(batch_info, drop_axes)
-        await asyncio.to_thread(
-            self.impl.store_chunks_with_indices,
-            chunks_desc,
-            value,
-        )
-        return None
+        try:
+            if self.impl is None:
+                raise UnsupportedMetadataError()
+            self._raise_error_on_unsupported_batch_dtype(batch_info)
+            chunks_desc = make_chunk_info_for_rust_with_indices(
+                batch_info, drop_axes, value.shape
+            )
+        except (
+            UnsupportedMetadataError,
+            DiscontiguousArrayError,
+            CollapsedDimensionError,
+            UnsupportedDataTypeError,
+            FillValueNoneError,
+        ):
+            if self.python_impl is None:
+                raise
+            await self.python_impl.write(batch_info, value, drop_axes)
+            return None
+        else:
+            # FIXME: Error if array is not in host memory
+            value_np: NDArrayLike | np.ndarray = value.as_ndarray_like()
+            if not value_np.dtype.isnative:
+                value_np = np.ascontiguousarray(
+                    value_np, dtype=value_np.dtype.newbyteorder("=")
+                )
+            elif not value_np.flags.c_contiguous:
+                value_np = np.ascontiguousarray(value_np)
+            await asyncio.to_thread(
+                self.impl.store_chunks_with_indices,
+                chunks_desc.chunk_info_with_indices,
+                value_np,
+                chunks_desc.write_empty_chunks,
+            )
+            return None
+
+    def _raise_error_on_unsupported_batch_dtype(
+        self,
+        batch_info: Iterable[
+            tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
+        ],
+    ):
+        # https://github.com/LDeakin/zarrs/blob/0532fe983b7b42b59dbf84e50a2fe5e6f7bad4ce/zarrs_metadata/src/v2_to_v3.rs#L289-L293 for VSUMm
+        # Further, our pipeline does not support variable-length objects due to limitations on decode_into, so object/np.dtypes.StringDType is also out
+        if any(
+            info.dtype.to_native_dtype().kind in {"V", "S", "U", "M", "m", "O", "T"}
+            for (_, info, _, _, _) in batch_info
+        ):
+            raise UnsupportedDataTypeError()

zarrs/utils.py

@@ -2,19 +2,22 @@ from __future__ import annotations
 
 import operator
 import os
+from dataclasses import dataclass
 from functools import reduce
 from typing import TYPE_CHECKING, Any
 
 import numpy as np
+from zarr.core.array_spec import ArraySpec
 from zarr.core.indexing import SelectorTuple, is_integer
 
+from zarrs._internal import ChunkItem
+
 if TYPE_CHECKING:
     from collections.abc import Iterable
     from types import EllipsisType
 
     from zarr.abc.store import ByteGetter, ByteSetter
-    from zarr.core.array_spec import ArraySpec
-    from zarr.core.common import ChunkCoords
+    from zarr.dtype import ZDType
 
 
 # adapted from https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
@@ -30,6 +33,10 @@ class CollapsedDimensionError(Exception):
     pass
 
 
+class FillValueNoneError(Exception):
+    pass
+
+
 # This is a (mostly) copy of the function from zarr.core.indexing that fixes:
 #   DeprecationWarning: Conversion of an array with ndim > 0 to a scalar is deprecated
 # TODO: Upstream this fix
@@ -62,17 +69,6 @@ def selector_tuple_to_slice_selection(selector_tuple: SelectorTuple) -> list[sli
     return make_slice_selection(selector_tuple)
 
 
-def convert_chunk_to_primitive(
-    byte_getter: ByteGetter | ByteSetter, chunk_spec: ArraySpec
-) -> tuple[str, ChunkCoords, str, Any]:
-    return (
-        str(byte_getter),
-        chunk_spec.shape,
-        str(chunk_spec.dtype),
-        chunk_spec.fill_value.tobytes(),
-    )
-
-
 def resulting_shape_from_index(
     array_shape: tuple[int, ...],
     index_tuple: tuple[int | slice | EllipsisType | np.ndarray],
@@ -144,15 +140,44 @@ def get_shape_for_selector(
     return resulting_shape_from_index(shape, selector_tuple, drop_axes, pad=pad)
 
 
+def get_implicit_fill_value(dtype: ZDType, fill_value: Any) -> Any:
+    if fill_value is None:
+        fill_value = dtype.default_scalar()
+    return fill_value
+
+
+@dataclass(frozen=True)
+class RustChunkInfo:
+    chunk_info_with_indices: list[ChunkItem]
+    write_empty_chunks: bool
+
+
 def make_chunk_info_for_rust_with_indices(
     batch_info: Iterable[
-        tuple[ByteGetter | ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]
+        tuple[ByteGetter | ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
     ],
     drop_axes: tuple[int, ...],
-) -> list[tuple[tuple[str, ChunkCoords, str, Any], list[slice], list[slice]]]:
-    chunk_info_with_indices = []
-    for byte_getter, chunk_spec, chunk_selection, out_selection in batch_info:
-        chunk_info = convert_chunk_to_primitive(byte_getter, chunk_spec)
+    shape: tuple[int, ...],
+) -> RustChunkInfo:
+    shape = shape if shape else (1,)  # constant array
+    chunk_info_with_indices: list[ChunkItem] = []
+    write_empty_chunks: bool = True
+    for (
+        byte_getter,
+        chunk_spec,
+        chunk_selection,
+        out_selection,
+        _,
+    ) in batch_info:
+        write_empty_chunks = chunk_spec.config.write_empty_chunks
+        if chunk_spec.fill_value is None:
+            chunk_spec = ArraySpec(
+                chunk_spec.shape,
+                chunk_spec.dtype,
+                get_implicit_fill_value(chunk_spec.dtype, chunk_spec.fill_value),
+                chunk_spec.config,
+                chunk_spec.prototype,
+            )
         out_selection_as_slices = selector_tuple_to_slice_selection(out_selection)
         chunk_selection_as_slices = selector_tuple_to_slice_selection(chunk_selection)
         shape_chunk_selection_slices = get_shape_for_selector(
@@ -169,17 +194,12 @@ def make_chunk_info_for_rust_with_indices(
                 f"{shape_chunk_selection} != {shape_chunk_selection_slices}"
             )
         chunk_info_with_indices.append(
-            (chunk_info, out_selection_as_slices, chunk_selection_as_slices)
+            ChunkItem(
+                key=byte_getter.path,
+                chunk_subset=chunk_selection_as_slices,
+                chunk_shape=chunk_spec.shape,
+                subset=out_selection_as_slices,
+                shape=shape,
+            )
         )
-    return chunk_info_with_indices
-
-
-def make_chunk_info_for_rust(
-    batch_info: Iterable[
-        tuple[ByteGetter | ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]
-    ],
-) -> list[tuple[str, ChunkCoords, str, Any]]:
-    return list(
-        convert_chunk_to_primitive(byte_getter, chunk_spec)
-        for (byte_getter, chunk_spec, _, _) in batch_info
-    )
+    return RustChunkInfo(chunk_info_with_indices, write_empty_chunks)

{zarrs-0.1.0.dist-info → zarrs-0.2.2.dist-info}/METADATA

@@ -1,66 +1,34 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: zarrs
-Version: 0.1.0
+Version: 0.2.2
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Typing :: Typed
-Requires-Dist: asciitree
-Requires-Dist: numpy >=1.24
-Requires-Dist: fasteners
-Requires-Dist: numcodecs[msgpack] >=0.10.0
-Requires-Dist: fsspec >2024
-Requires-Dist: crc32c
-Requires-Dist: zstandard
-Requires-Dist: typing-extensions
-Requires-Dist: donfig
-Requires-Dist: pytest
-Requires-Dist: universal-pathlib >=0.2.0
-Requires-Dist: zarr >=3.0.0b2
-Requires-Dist: coverage ; extra == 'test'
-Requires-Dist: pytest ; extra == 'test'
-Requires-Dist: pytest-cov ; extra == 'test'
-Requires-Dist: msgpack ; extra == 'test'
-Requires-Dist: lmdb ; extra == 'test'
-Requires-Dist: s3fs ; extra == 'test'
-Requires-Dist: pytest-asyncio ; extra == 'test'
-Requires-Dist: moto[s3] ; extra == 'test'
-Requires-Dist: flask-cors ; extra == 'test'
-Requires-Dist: flask ; extra == 'test'
-Requires-Dist: requests ; extra == 'test'
-Requires-Dist: mypy ; extra == 'test'
-Requires-Dist: hypothesis ; extra == 'test'
-Requires-Dist: pytest-xdist ; extra == 'test'
-Requires-Dist: maturin ; extra == 'dev'
-Requires-Dist: pip ; extra == 'dev'
-Requires-Dist: pre-commit ; extra == 'dev'
-Requires-Dist: sphinx >=7.4.6 ; extra == 'doc'
-Requires-Dist: myst-parser ; extra == 'doc'
-Provides-Extra: test
-Provides-Extra: dev
-Provides-Extra: doc
+Requires-Dist: numpy>=1.24
+Requires-Dist: zarr>=3.1
 License-File: LICENSE
+Summary: A CodecPipeline for zarr-python backed by the zarrs Rust crate
 Author: Ilan Gold, Lachlan Deakin, Philipp Angerer
-License: MIT
+License-Expression: MIT
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 
 # zarrs-python
 
-```{warning}
-⚠️ The version of `zarr-python` we currently depend on is still in pre-release and this
-package is accordingly extremely experimental.
-We cannot guarantee any stability or correctness at the moment, although we have
-tried to do extensive testing and make clear what we think we support and do not.
-```
+[![PyPI](https://img.shields.io/pypi/v/zarrs.svg)](https://pypi.org/project/zarrs)
+[![Downloads](https://static.pepy.tech/badge/zarrs/month)](https://pepy.tech/project/zarrs)
+[![Downloads](https://static.pepy.tech/badge/zarrs)](https://pepy.tech/project/zarrs)
+[![Stars](https://img.shields.io/github/stars/zarrs/zarrs-python?style=flat&logo=github&color=yellow)](https://github.com/zarrs/zarrs-python/stargazers)
+![CI](https://github.com/zarrs/zarrs-python/actions/workflows/ci.yml/badge.svg)
+![CD](https://github.com/zarrs/zarrs-python/actions/workflows/cd.yml/badge.svg)
 
-This project serves as a bridge between [`zarrs`](https://docs.rs/zarrs/latest/zarrs/) and [`zarr`](https://zarr.readthedocs.io/en/latest/index.html) via [`PyO3`](https://pyo3.rs/v0.22.3/).  The main goal of the project is to speed up i/o.
+This project serves as a bridge between [`zarrs`](https://docs.rs/zarrs/latest/zarrs/) (Rust) and [`zarr`](https://zarr.readthedocs.io/en/latest/index.html) (`zarr-python`) via [`PyO3`](https://pyo3.rs/v0.22.3/).  The main goal of the project is to speed up i/o (see [`zarr_benchmarks`](https://github.com/LDeakin/zarr_benchmarks)).
 
-To use the project, simply install our package (which depends on `zarr-python>3.0.0b0`), and run:
+To use the project, simply install our package (which depends on `zarr-python>=3.0.0`), and run:
 
 ```python
 import zarr
-import zarrs
 zarr.config.set({"codec_pipeline.path": "zarrs.ZarrsCodecPipeline"})
 ```
 
@@ -68,7 +36,15 @@ You can then use your `zarr` as normal (with some caveats)!
 
 ## API
 
-We export a `ZarrsCodecPipeline` class so that `zarr-python` can use the class but it is not meant to be instantiated and we do not guarantee the stability of its API beyond what is required so that `zarr-python` can use it.  Therefore, it is not documented here.  We also export two errors, `DiscontiguousArrayError` and `CollapsedDimensionError` that can be thrown in the process of converting to indexers that `zarrs` can understand (see below for more details).
+We export a `ZarrsCodecPipeline` class so that `zarr-python` can use the class but it is not meant to be instantiated and we do not guarantee the stability of its API beyond what is required so that `zarr-python` can use it.  Therefore, it is not documented here.
+
+At the moment, we only support a subset of the `zarr-python` stores:
+
+- [`LocalStore`](https://zarr.readthedocs.io/en/latest/api/zarr/storage/#zarr.storage.LocalStore) (local filesystem)
+- [`ObjectStore`](https://zarr.readthedocs.io/en/latest/user-guide/storage/#object-store) (cloud storage)
+- [`HTTPFileSystem`](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.implementations.http.HTTPFileSystem) via [`FsspecStore`](https://zarr.readthedocs.io/en/latest/api/zarr/storage/#zarr.storage.FsspecStore)
+
+A `NotImplementedError` will be raised if a store is not supported.
 
 ### Configuration
 
@@ -79,7 +55,6 @@ Standard `zarr.config` options control some functionality (see the defaults in t
   - Defaults to the number of threads in the global `rayon` thread pool if set to `None`, which is [typically the number of logical CPUs](https://docs.rs/rayon/latest/rayon/struct.ThreadPoolBuilder.html#method.num_threads).
 - `array.write_empty_chunks`: whether or not to store empty chunks.
   - Defaults to false if `None`. Note that checking for emptiness has some overhead, see [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#store-empty-chunks) for more info.
-  - This option name is proposed in [zarr-python #2429](https://github.com/zarr-developers/zarr-python/pull/2429)
 
 The `ZarrsCodecPipeline` specific options are:
 - `codec_pipeline.chunk_concurrent_maximum`: the maximum number of chunks stored/retrieved concurrently.
@@ -87,7 +62,11 @@ The `ZarrsCodecPipeline` specific options are:
 - `codec_pipeline.chunk_concurrent_minimum`: the minimum number of chunks retrieved/stored concurrently when balancing chunk/codec concurrency.
   - Defaults to 4 if `None`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#chunk-concurrent-minimum) for more info.
 - `codec_pipeline.validate_checksums`: enable checksum validation (e.g. with the CRC32C codec).
-  - Defaults to true if `None`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#validate-checksums) for more info.
+  - Defaults to `True`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#validate-checksums) for more info.
+- `codec_pipeline.direct_io`: enable `O_DIRECT` read/write, needs support from the operating system (currently only Linux) and file system.
+  - Defaults to `False`.
+- `codec_pipeline.strict`: raise exceptions for unsupported operations instead of falling back to the default codec pipeline of `zarr-python`.
+  - Defaults to `False`.
 
 For example:
 ```python
@@ -97,13 +76,16 @@ zarr.config.set({
     "codec_pipeline": {
         "path": "zarrs.ZarrsCodecPipeline",
         "validate_checksums": True,
-        "store_empty_chunks": False,
         "chunk_concurrent_maximum": None,
         "chunk_concurrent_minimum": 4,
+        "direct_io": False,
+        "strict": False
     }
 })
 ```
 
+If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value.  However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed.  This may change in the future as guidance from the `zarr` community becomes clear.
+
 ## Concurrency
 
 Concurrency can be classified into two types:
@@ -123,7 +105,7 @@ Chunk concurrency is typically favored because:
 
 ## Supported Indexing Methods
 
-We **do not** officially support the following indexing methods.  Some of these methods may error out, others may not:
+The following methods will trigger use with the old zarr-python pipeline:
 
 1. Any `oindex` or `vindex` integer `np.ndarray` indexing with dimensionality >=3 i.e.,
 
@@ -153,7 +135,10 @@ We **do not** officially support the following indexing methods.  Some of these
    arr[0:10, ..., 0:5]
    ```
 
-Otherwise, we believe that we support your indexing case: slices, ints, and all integer `np.ndarray` indices in 2D for reading, contiguous integer `np.ndarray` indices along one axis for writing etc.  Please file an issue if you believe we have more holes in our coverage than we are aware of or you wish to contribute!  For example, we have an [issue in zarrs for integer-array indexing](https://github.com/LDeakin/zarrs/issues/52) that would unblock a lot of these issues!
 
-That being said, using non-contiguous integer `np.ndarray` indexing for reads may not be as fast as expected given the performance of other supported methods.  Until `zarrs` supports integer indexing, only fetching chunks is done in `rust` while indexing then occurs in `python`.
+Furthermore, using anything except contiguous (i.e., slices or consecutive integer) `np.ndarray` for numeric data will fall back to the default `zarr-python` implementation.
+
+Please file an issue if you believe we have more holes in our coverage than we are aware of or you wish to contribute!  For example, we have an [issue in zarrs for integer-array indexing](https://github.com/LDeakin/zarrs/issues/52) that would unblock a lot the use of the rust pipeline for that use-case (very useful for mini-batch training perhaps!).
+
+Further, any codecs not supported by `zarrs` will also automatically fall back to the python implementation.
 

zarrs-0.2.2.dist-info/RECORD

@@ -0,0 +1,11 @@
+zarrs-0.2.2.dist-info/METADATA,sha256=8K1AOS_SVQgRLzQ2rWtNbJxCAb12XmTLRrT6QklQgOI,8054
+zarrs-0.2.2.dist-info/WHEEL,sha256=N8W3-0eDM6igWj-H12r7VkxoMaJIqJLxUyWCFstEaGg,105
+zarrs-0.2.2.dist-info/entry_points.txt,sha256=EzI6yCIUPDHBHzjDdexuGGYbOLXf8x2ICokOJXnuX3k,68
+zarrs-0.2.2.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
+zarrs/__init__.py,sha256=lRVtAPzCzJkGs4vQrW4UgANq-pC-khS0ZF7HTj4__Hg,489
+zarrs/_internal.abi3.so,sha256=tnP5IiuDmhfwB15cX4yTyu1mVZgShtJy1lnS87TzK1o,14928348
+zarrs/_internal.pyi,sha256=a_D4yx99r4xeQX1ntY_A_Q4wVmLeLwJZHWAQV_mVu9A,1308
+zarrs/pipeline.py,sha256=YfB13GWNfxELerXVtJ_ipFwSL7bN-YuPys6jCB9lnms,9008
+zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+zarrs/utils.py,sha256=W2XCjJDVrdHYJgtVaRKN533Ljw1MF7o0YwXuz5ZAk2g,7020
+zarrs-0.2.2.dist-info/RECORD,,

{zarrs-0.1.0.dist-info → zarrs-0.2.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: maturin (1.7.4)
+Generator: maturin (1.9.4)
 Root-Is-Purelib: false
 Tag: cp311-abi3-macosx_10_12_x86_64

zarrs-0.2.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[zarr.codec_pipeline]
+zarrs.codec_pipeline=zarrs:ZarrsCodecPipeline

zarrs-0.1.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-zarrs-0.1.0.dist-info/METADATA,sha256=Qf4O1LEJrZrrnWtO4vqOn4gF3cf882aX0kIf3TMhUMY,7923
-zarrs-0.1.0.dist-info/WHEEL,sha256=LZygbeT1PTQw7a9tONPp78bbG4FZc86U59Z0RFJcoR8,105
-zarrs-0.1.0.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
-zarrs/__init__.py,sha256=4oWtWDZO8r7z4Uh7Fy_brmkxXDpULQdgjlA0iFw98eA,573
-zarrs/_internal.pyi,sha256=revBHMbEur_WKTDRtyJqah0e-D6CPy58sIIZLpicRgA,1330
-zarrs/utils.py,sha256=qV-__rjVNs7bhvnyY4U2eOtEFESGm-XQUv6t9ECvjcc,6588
-zarrs/pipeline.py,sha256=pcZ56LVQ131e5iq_HpEZWOyo-HzDi6GUOSulDHPWyLQ,5576
-zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-zarrs/_internal.abi3.so,sha256=uc2IGucmaDNNx0t5K5lETXYmckCIJedoPcPLJhHCfOs,3193056
-zarrs-0.1.0.dist-info/RECORD,,