zarrs 0.1.1__cp311-abi3-macosx_10_12_x86_64.whl → 0.2.2__cp311-abi3-macosx_10_12_x86_64.whl

zarrs/__init__.py

@@ -1,5 +1,3 @@
-from zarr.registry import register_pipeline
-
 from ._internal import __version__
 from .pipeline import ZarrsCodecPipeline as _ZarrsCodecPipeline
 from .utils import CollapsedDimensionError, DiscontiguousArrayError
@@ -10,8 +8,6 @@ class ZarrsCodecPipeline(_ZarrsCodecPipeline):
     pass
 
 
-register_pipeline(ZarrsCodecPipeline)
-
 __all__ = [
     "ZarrsCodecPipeline",
     "DiscontiguousArrayError",

zarrs/_internal.pyi

@@ -1,57 +1,44 @@
 # This file is automatically generated by pyo3_stub_gen
 # ruff: noqa: E501, F401
 
+import builtins
 import typing
-from enum import Enum, auto
 
-import numpy
 import numpy.typing
+import zarr.abc.store
 
-class Basic:
-    def __new__(cls, byte_interface: typing.Any, chunk_spec: typing.Any): ...
-    ...
+@typing.final
+class ChunkItem:
+    def __new__(
+        cls,
+        key: builtins.str,
+        chunk_subset: typing.Sequence[slice],
+        chunk_shape: typing.Sequence[builtins.int],
+        subset: typing.Sequence[slice],
+        shape: typing.Sequence[builtins.int],
+    ) -> ChunkItem: ...
 
+@typing.final
 class CodecPipelineImpl:
     def __new__(
         cls,
-        metadata,
+        array_metadata: builtins.str,
+        store_config: zarr.abc.store.Store,
         *,
-        validate_checksums=...,
-        store_empty_chunks=...,
-        chunk_concurrent_minimum=...,
-        chunk_concurrent_maximum=...,
-        num_threads=...,
-    ): ...
+        validate_checksums: builtins.bool = False,
+        chunk_concurrent_minimum: builtins.int | None = None,
+        chunk_concurrent_maximum: builtins.int | None = None,
+        num_threads: builtins.int | None = None,
+        direct_io: builtins.bool = False,
+    ) -> CodecPipelineImpl: ...
     def retrieve_chunks_and_apply_index(
         self,
-        chunk_descriptions: typing.Sequence[WithSubset],
+        chunk_descriptions: typing.Sequence[ChunkItem],
         value: numpy.typing.NDArray[typing.Any],
     ) -> None: ...
-    def retrieve_chunks(
-        self, chunk_descriptions: typing.Sequence[Basic]
-    ) -> list[numpy.typing.NDArray[numpy.uint8]]: ...
     def store_chunks_with_indices(
         self,
-        chunk_descriptions: typing.Sequence[WithSubset],
+        chunk_descriptions: typing.Sequence[ChunkItem],
         value: numpy.typing.NDArray[typing.Any],
+        write_empty_chunks: builtins.bool,
     ) -> None: ...
-
-class FilesystemStoreConfig:
-    root: str
-
-class HttpStoreConfig:
-    endpoint: str
-
-class WithSubset:
-    def __new__(
-        cls,
-        item: Basic,
-        chunk_subset: typing.Sequence[slice],
-        subset: typing.Sequence[slice],
-        shape: typing.Sequence[int],
-    ): ...
-    ...
-
-class StoreConfig(Enum):
-    Filesystem = auto()
-    Http = auto()

zarrs/pipeline.py

@@ -4,44 +4,84 @@ import asyncio
 import json
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, TypedDict
+from warnings import warn
 
 import numpy as np
 from zarr.abc.codec import Codec, CodecPipeline
+from zarr.codecs._v2 import V2Codec
+from zarr.core import BatchedCodecPipeline
 from zarr.core.config import config
+from zarr.core.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Iterator
-    from typing import Any, Self
+    from typing import Self
 
-    from zarr.abc.store import ByteGetter, ByteSetter
+    from zarr.abc.store import ByteGetter, ByteSetter, Store
     from zarr.core.array_spec import ArraySpec
     from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer
     from zarr.core.chunk_grids import ChunkGrid
-    from zarr.core.common import ChunkCoords
     from zarr.core.indexing import SelectorTuple
+    from zarr.dtype import ZDType
 
 from ._internal import CodecPipelineImpl
 from .utils import (
     CollapsedDimensionError,
     DiscontiguousArrayError,
-    make_chunk_info_for_rust,
+    FillValueNoneError,
     make_chunk_info_for_rust_with_indices,
 )
 
 
-def get_codec_pipeline_impl(codec_metadata_json: str) -> CodecPipelineImpl:
-    return CodecPipelineImpl(
-        codec_metadata_json,
-        validate_checksums=config.get("codec_pipeline.validate_checksums", None),
-        store_empty_chunks=config.get("array.write_empty_chunks", None),
-        chunk_concurrent_minimum=config.get(
-            "codec_pipeline.chunk_concurrent_minimum", None
-        ),
-        chunk_concurrent_maximum=config.get(
-            "codec_pipeline.chunk_concurrent_maximum", None
-        ),
-        num_threads=config.get("threading.max_workers", None),
-    )
+class UnsupportedDataTypeError(Exception):
+    pass
+
+
+class UnsupportedMetadataError(Exception):
+    pass
+
+
+def get_codec_pipeline_impl(
+    metadata: ArrayMetadata, store: Store, *, strict: bool
+) -> CodecPipelineImpl | None:
+    try:
+        array_metadata_json = json.dumps(metadata.to_dict())
+        # Maintain old behavior: https://github.com/zarrs/zarrs-python/tree/b36ba797cafec77f5f41a25316be02c718a2b4f8?tab=readme-ov-file#configuration
+        validate_checksums = config.get("codec_pipeline.validate_checksums", True)
+        if validate_checksums is None:
+            validate_checksums = True
+        return CodecPipelineImpl(
+            array_metadata_json,
+            store_config=store,
+            validate_checksums=validate_checksums,
+            chunk_concurrent_minimum=config.get(
+                "codec_pipeline.chunk_concurrent_minimum", None
+            ),
+            chunk_concurrent_maximum=config.get(
+                "codec_pipeline.chunk_concurrent_maximum", None
+            ),
+            num_threads=config.get("threading.max_workers", None),
+            direct_io=config.get("codec_pipeline.direct_io", False),
+        )
+    except TypeError as e:
+        if strict:
+            raise UnsupportedMetadataError() from e
+
+        warn(
+            f"Array is unsupported by ZarrsCodecPipeline: {e}",
+            category=UserWarning,
+        )
+        return None
+
+
+def get_codec_pipeline_fallback(
+    metadata: ArrayMetadata, *, strict: bool
+) -> BatchedCodecPipeline | None:
+    if strict:
+        return None
+    else:
+        codecs = array_metadata_to_codecs(metadata)
+        return BatchedCodecPipeline.from_codecs(codecs)
 
 
 class ZarrsCodecPipelineState(TypedDict):
@@ -49,35 +89,48 @@ class ZarrsCodecPipelineState(TypedDict):
     codecs: tuple[Codec, ...]
 
 
+def array_metadata_to_codecs(metadata: ArrayMetadata) -> list[Codec]:
+    if isinstance(metadata, ArrayV3Metadata):
+        return metadata.codecs
+    elif isinstance(metadata, ArrayV2Metadata):
+        v2_codec = V2Codec(filters=metadata.filters, compressor=metadata.compressor)
+        return [v2_codec]
+
+
 @dataclass
 class ZarrsCodecPipeline(CodecPipeline):
-    codecs: tuple[Codec, ...]
-    impl: CodecPipelineImpl
-    codec_metadata_json: str
+    metadata: ArrayMetadata
+    store: Store
+    impl: CodecPipelineImpl | None
+    python_impl: BatchedCodecPipeline | None
 
     def __getstate__(self) -> ZarrsCodecPipelineState:
-        return {"codec_metadata_json": self.codec_metadata_json, "codecs": self.codecs}
+        return {"metadata": self.metadata, "store": self.store}
 
     def __setstate__(self, state: ZarrsCodecPipelineState):
-        self.codecs = state["codecs"]
-        self.codec_metadata_json = state["codec_metadata_json"]
-        self.impl = get_codec_pipeline_impl(self.codec_metadata_json)
+        self.metadata = state["metadata"]
+        self.store = state["store"]
+        strict = config.get("codec_pipeline.strict", False)
+        self.impl = get_codec_pipeline_impl(self.metadata, self.store, strict=strict)
+        self.python_impl = get_codec_pipeline_fallback(self.metadata, strict=strict)
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
-        raise NotImplementedError("evolve_from_array_spec")
+        return self
 
     @classmethod
     def from_codecs(cls, codecs: Iterable[Codec]) -> Self:
-        codec_metadata = [codec.to_dict() for codec in codecs]
-        codec_metadata_json = json.dumps(codec_metadata)
-        # TODO: upstream zarr-python has not settled on how to deal with configs yet
-        # Should they be checked when an array is created, or when an operation is performed?
-        # https://github.com/zarr-developers/zarr-python/issues/2409
-        # https://github.com/zarr-developers/zarr-python/pull/2429#issuecomment-2566976567
+        return BatchedCodecPipeline.from_codecs(codecs)
+
+    @classmethod
+    def from_array_metadata_and_store(
+        cls, array_metadata: ArrayMetadata, store: Store
+    ) -> Self:
+        strict = config.get("codec_pipeline.strict", False)
         return cls(
-            codec_metadata_json=codec_metadata_json,
-            codecs=tuple(codecs),
-            impl=get_codec_pipeline_impl(codec_metadata_json),
+            metadata=array_metadata,
+            store=store,
+            impl=get_codec_pipeline_impl(array_metadata, store, strict=strict),
+            python_impl=get_codec_pipeline_fallback(array_metadata, strict=strict),
         )
 
     @property
@@ -92,7 +145,7 @@ class ZarrsCodecPipeline(CodecPipeline):
         yield from self.codecs
 
     def validate(
-        self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid
+        self, *, shape: tuple[int, ...], dtype: ZDType, chunk_grid: ChunkGrid
     ) -> None:
         raise NotImplementedError("validate")
 
@@ -114,52 +167,94 @@ class ZarrsCodecPipeline(CodecPipeline):
     async def read(
         self,
         batch_info: Iterable[
-            tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]
+            tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
         ],
         out: NDBuffer,  # type: ignore
         drop_axes: tuple[int, ...] = (),  # FIXME: unused
     ) -> None:
         # FIXME: Error if array is not in host memory
-        out: NDArrayLike = out.as_ndarray_like()
         if not out.dtype.isnative:
             raise RuntimeError("Non-native byte order not supported")
         try:
+            if self.impl is None:
+                raise UnsupportedMetadataError()
+            self._raise_error_on_unsupported_batch_dtype(batch_info)
             chunks_desc = make_chunk_info_for_rust_with_indices(
                 batch_info, drop_axes, out.shape
             )
-        except (DiscontiguousArrayError, CollapsedDimensionError):
-            chunks_desc = make_chunk_info_for_rust(batch_info)
+        except (
+            UnsupportedMetadataError,
+            DiscontiguousArrayError,
+            CollapsedDimensionError,
+            UnsupportedDataTypeError,
+            FillValueNoneError,
+        ):
+            if self.python_impl is None:
+                raise
+            await self.python_impl.read(batch_info, out, drop_axes)
+            return None
         else:
+            out: NDArrayLike = out.as_ndarray_like()
             await asyncio.to_thread(
                 self.impl.retrieve_chunks_and_apply_index,
-                chunks_desc,
+                chunks_desc.chunk_info_with_indices,
                 out,
             )
             return None
-        chunks = await asyncio.to_thread(self.impl.retrieve_chunks, chunks_desc)
-        for chunk, (_, spec, selection, out_selection) in zip(chunks, batch_info):
-            chunk_reshaped = chunk.view(spec.dtype).reshape(spec.shape)
-            chunk_selected = chunk_reshaped[selection]
-            if drop_axes:
-                chunk_selected = np.squeeze(chunk_selected, axis=drop_axes)
-            out[out_selection] = chunk_selected
 
     async def write(
         self,
         batch_info: Iterable[
-            tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]
+            tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
         ],
         value: NDBuffer,  # type: ignore
         drop_axes: tuple[int, ...] = (),
     ) -> None:
-        # FIXME: Error if array is not in host memory
-        value: NDArrayLike | np.ndarray = value.as_ndarray_like()
-        if not value.dtype.isnative:
-            value = np.ascontiguousarray(value, dtype=value.dtype.newbyteorder("="))
-        elif not value.flags.c_contiguous:
-            value = np.ascontiguousarray(value)
-        chunks_desc = make_chunk_info_for_rust_with_indices(
-            batch_info, drop_axes, value.shape
-        )
-        await asyncio.to_thread(self.impl.store_chunks_with_indices, chunks_desc, value)
-        return None
+        try:
+            if self.impl is None:
+                raise UnsupportedMetadataError()
+            self._raise_error_on_unsupported_batch_dtype(batch_info)
+            chunks_desc = make_chunk_info_for_rust_with_indices(
+                batch_info, drop_axes, value.shape
+            )
+        except (
+            UnsupportedMetadataError,
+            DiscontiguousArrayError,
+            CollapsedDimensionError,
+            UnsupportedDataTypeError,
+            FillValueNoneError,
+        ):
+            if self.python_impl is None:
+                raise
+            await self.python_impl.write(batch_info, value, drop_axes)
+            return None
+        else:
+            # FIXME: Error if array is not in host memory
+            value_np: NDArrayLike | np.ndarray = value.as_ndarray_like()
+            if not value_np.dtype.isnative:
+                value_np = np.ascontiguousarray(
+                    value_np, dtype=value_np.dtype.newbyteorder("=")
+                )
+            elif not value_np.flags.c_contiguous:
+                value_np = np.ascontiguousarray(value_np)
+            await asyncio.to_thread(
+                self.impl.store_chunks_with_indices,
+                chunks_desc.chunk_info_with_indices,
+                value_np,
+                chunks_desc.write_empty_chunks,
+            )
+            return None
+
+    def _raise_error_on_unsupported_batch_dtype(
+        self,
+        batch_info: Iterable[
+            tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
+        ],
+    ):
+        # https://github.com/LDeakin/zarrs/blob/0532fe983b7b42b59dbf84e50a2fe5e6f7bad4ce/zarrs_metadata/src/v2_to_v3.rs#L289-L293 for VSUMm
+        # Further, our pipeline does not support variable-length objects due to limitations on decode_into, so object/np.dtypes.StringDType is also out
+        if any(
+            info.dtype.to_native_dtype().kind in {"V", "S", "U", "M", "m", "O", "T"}
+            for (_, info, _, _, _) in batch_info
+        ):
+            raise UnsupportedDataTypeError()

zarrs/utils.py

@@ -2,20 +2,22 @@ from __future__ import annotations
 
 import operator
 import os
+from dataclasses import dataclass
 from functools import reduce
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
+from zarr.core.array_spec import ArraySpec
 from zarr.core.indexing import SelectorTuple, is_integer
 
-from zarrs._internal import Basic, WithSubset
+from zarrs._internal import ChunkItem
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
     from types import EllipsisType
 
     from zarr.abc.store import ByteGetter, ByteSetter
-    from zarr.core.array_spec import ArraySpec
+    from zarr.dtype import ZDType
 
 
 # adapted from https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
@@ -31,6 +33,10 @@ class CollapsedDimensionError(Exception):
     pass
 
 
+class FillValueNoneError(Exception):
+    pass
+
+
 # This is a (mostly) copy of the function from zarr.core.indexing that fixes:
 #   DeprecationWarning: Conversion of an array with ndim > 0 to a scalar is deprecated
 # TODO: Upstream this fix
@@ -134,17 +140,44 @@ def get_shape_for_selector(
     return resulting_shape_from_index(shape, selector_tuple, drop_axes, pad=pad)
 
 
+def get_implicit_fill_value(dtype: ZDType, fill_value: Any) -> Any:
+    if fill_value is None:
+        fill_value = dtype.default_scalar()
+    return fill_value
+
+
+@dataclass(frozen=True)
+class RustChunkInfo:
+    chunk_info_with_indices: list[ChunkItem]
+    write_empty_chunks: bool
+
+
 def make_chunk_info_for_rust_with_indices(
     batch_info: Iterable[
-        tuple[ByteGetter | ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]
+        tuple[ByteGetter | ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
     ],
     drop_axes: tuple[int, ...],
     shape: tuple[int, ...],
-) -> list[WithSubset]:
+) -> RustChunkInfo:
     shape = shape if shape else (1,)  # constant array
-    chunk_info_with_indices: list[WithSubset] = []
-    for byte_getter, chunk_spec, chunk_selection, out_selection in batch_info:
-        chunk_info = Basic(byte_getter, chunk_spec)
+    chunk_info_with_indices: list[ChunkItem] = []
+    write_empty_chunks: bool = True
+    for (
+        byte_getter,
+        chunk_spec,
+        chunk_selection,
+        out_selection,
+        _,
+    ) in batch_info:
+        write_empty_chunks = chunk_spec.config.write_empty_chunks
+        if chunk_spec.fill_value is None:
+            chunk_spec = ArraySpec(
+                chunk_spec.shape,
+                chunk_spec.dtype,
+                get_implicit_fill_value(chunk_spec.dtype, chunk_spec.fill_value),
+                chunk_spec.config,
+                chunk_spec.prototype,
+            )
         out_selection_as_slices = selector_tuple_to_slice_selection(out_selection)
         chunk_selection_as_slices = selector_tuple_to_slice_selection(chunk_selection)
         shape_chunk_selection_slices = get_shape_for_selector(
@@ -161,22 +194,12 @@ def make_chunk_info_for_rust_with_indices(
                 f"{shape_chunk_selection} != {shape_chunk_selection_slices}"
             )
         chunk_info_with_indices.append(
-            WithSubset(
-                chunk_info,
+            ChunkItem(
+                key=byte_getter.path,
                 chunk_subset=chunk_selection_as_slices,
+                chunk_shape=chunk_spec.shape,
                 subset=out_selection_as_slices,
                 shape=shape,
             )
         )
-    return chunk_info_with_indices
-
-
-def make_chunk_info_for_rust(
-    batch_info: Iterable[
-        tuple[ByteGetter | ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]
-    ],
-) -> list[Basic]:
-    return [
-        Basic(byte_interface, chunk_spec)
-        for (byte_interface, chunk_spec, _, _) in batch_info
-    ]
+    return RustChunkInfo(chunk_info_with_indices, write_empty_chunks)

{zarrs-0.1.1.dist-info → zarrs-0.2.2.dist-info}/METADATA

@@ -1,47 +1,16 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: zarrs
-Version: 0.1.1
+Version: 0.2.2
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Typing :: Typed
-Requires-Dist: asciitree
-Requires-Dist: numpy >=1.24
-Requires-Dist: fasteners
-Requires-Dist: numcodecs[msgpack] >=0.10.0
-Requires-Dist: fsspec >2024
-Requires-Dist: crc32c
-Requires-Dist: zstandard
-Requires-Dist: typing-extensions
-Requires-Dist: donfig
-Requires-Dist: pytest
-Requires-Dist: universal-pathlib >=0.2.0
-Requires-Dist: zarr >=3.0.0
-Requires-Dist: coverage ; extra == 'test'
-Requires-Dist: pytest ; extra == 'test'
-Requires-Dist: pytest-cov ; extra == 'test'
-Requires-Dist: msgpack ; extra == 'test'
-Requires-Dist: lmdb ; extra == 'test'
-Requires-Dist: s3fs ; extra == 'test'
-Requires-Dist: pytest-asyncio ; extra == 'test'
-Requires-Dist: moto[s3] ; extra == 'test'
-Requires-Dist: flask-cors ; extra == 'test'
-Requires-Dist: flask ; extra == 'test'
-Requires-Dist: requests ; extra == 'test'
-Requires-Dist: mypy ; extra == 'test'
-Requires-Dist: hypothesis ; extra == 'test'
-Requires-Dist: pytest-xdist ; extra == 'test'
-Requires-Dist: maturin ; extra == 'dev'
-Requires-Dist: pip ; extra == 'dev'
-Requires-Dist: pre-commit ; extra == 'dev'
-Requires-Dist: sphinx >=7.4.6 ; extra == 'doc'
-Requires-Dist: myst-parser ; extra == 'doc'
-Provides-Extra: test
-Provides-Extra: dev
-Provides-Extra: doc
+Requires-Dist: numpy>=1.24
+Requires-Dist: zarr>=3.1
 License-File: LICENSE
+Summary: A CodecPipeline for zarr-python backed by the zarrs Rust crate
 Author: Ilan Gold, Lachlan Deakin, Philipp Angerer
-License: MIT
+License-Expression: MIT
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 
@@ -50,9 +19,9 @@ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 [![PyPI](https://img.shields.io/pypi/v/zarrs.svg)](https://pypi.org/project/zarrs)
 [![Downloads](https://static.pepy.tech/badge/zarrs/month)](https://pepy.tech/project/zarrs)
 [![Downloads](https://static.pepy.tech/badge/zarrs)](https://pepy.tech/project/zarrs)
-[![Stars](https://img.shields.io/github/stars/ilan-gold/zarrs-python?style=flat&logo=github&color=yellow)](https://github.com/ilan-gold/zarrs-python/stargazers)
-![CI](https://github.com/ilan-gold/zarrs-python/actions/workflows/ci.yml/badge.svg)
-![CD](https://github.com/ilan-gold/zarrs-python/actions/workflows/cd.yml/badge.svg)
+[![Stars](https://img.shields.io/github/stars/zarrs/zarrs-python?style=flat&logo=github&color=yellow)](https://github.com/zarrs/zarrs-python/stargazers)
+![CI](https://github.com/zarrs/zarrs-python/actions/workflows/ci.yml/badge.svg)
+![CD](https://github.com/zarrs/zarrs-python/actions/workflows/cd.yml/badge.svg)
 
 This project serves as a bridge between [`zarrs`](https://docs.rs/zarrs/latest/zarrs/) (Rust) and [`zarr`](https://zarr.readthedocs.io/en/latest/index.html) (`zarr-python`) via [`PyO3`](https://pyo3.rs/v0.22.3/).  The main goal of the project is to speed up i/o (see [`zarr_benchmarks`](https://github.com/LDeakin/zarr_benchmarks)).
 
@@ -60,7 +29,6 @@ To use the project, simply install our package (which depends on `zarr-python>=3
 
 ```python
 import zarr
-import zarrs
 zarr.config.set({"codec_pipeline.path": "zarrs.ZarrsCodecPipeline"})
 ```
 
@@ -68,16 +36,15 @@ You can then use your `zarr` as normal (with some caveats)!
 
 ## API
 
-We export a `ZarrsCodecPipeline` class so that `zarr-python` can use the class but it is not meant to be instantiated and we do not guarantee the stability of its API beyond what is required so that `zarr-python` can use it.  Therefore, it is not documented here.  We also export two errors, `DiscontiguousArrayError` and `CollapsedDimensionError` that can be thrown in the process of converting to indexers that `zarrs` can understand (see below for more details).
+We export a `ZarrsCodecPipeline` class so that `zarr-python` can use the class but it is not meant to be instantiated and we do not guarantee the stability of its API beyond what is required so that `zarr-python` can use it.  Therefore, it is not documented here.
 
 At the moment, we only support a subset of the `zarr-python` stores:
 
-- [x] [LocalStore](https://zarr.readthedocs.io/en/latest/_autoapi/zarr/storage/index.html#zarr.storage.LocalStore) (FileSystem)
-- [FsspecStore](https://zarr.readthedocs.io/en/latest/_autoapi/zarr/storage/index.html#zarr.storage.FsspecStore)
-  - [x] [HTTPFileSystem](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.implementations.http.HTTPFileSystem)
+- [`LocalStore`](https://zarr.readthedocs.io/en/latest/api/zarr/storage/#zarr.storage.LocalStore) (local filesystem)
+- [`ObjectStore`](https://zarr.readthedocs.io/en/latest/user-guide/storage/#object-store) (cloud storage)
+- [`HTTPFileSystem`](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.implementations.http.HTTPFileSystem) via [`FsspecStore`](https://zarr.readthedocs.io/en/latest/api/zarr/storage/#zarr.storage.FsspecStore)
 
 A `NotImplementedError` will be raised if a store is not supported.
-We intend to support more stores in the future: https://github.com/ilan-gold/zarrs-python/issues/44.
 
 ### Configuration
 
@@ -95,7 +62,11 @@ The `ZarrsCodecPipeline` specific options are:
 - `codec_pipeline.chunk_concurrent_minimum`: the minimum number of chunks retrieved/stored concurrently when balancing chunk/codec concurrency.
   - Defaults to 4 if `None`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#chunk-concurrent-minimum) for more info.
 - `codec_pipeline.validate_checksums`: enable checksum validation (e.g. with the CRC32C codec).
-  - Defaults to true if `None`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#validate-checksums) for more info.
+  - Defaults to `True`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#validate-checksums) for more info.
+- `codec_pipeline.direct_io`: enable `O_DIRECT` read/write, needs support from the operating system (currently only Linux) and file system.
+  - Defaults to `False`.
+- `codec_pipeline.strict`: raise exceptions for unsupported operations instead of falling back to the default codec pipeline of `zarr-python`.
+  - Defaults to `False`.
 
 For example:
 ```python
@@ -105,14 +76,15 @@ zarr.config.set({
     "codec_pipeline": {
         "path": "zarrs.ZarrsCodecPipeline",
         "validate_checksums": True,
-        "store_empty_chunks": False,
         "chunk_concurrent_maximum": None,
         "chunk_concurrent_minimum": 4,
+        "direct_io": False,
+        "strict": False
     }
 })
 ```
 
-If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `store_empty_chunks`, `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value.  However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed.  This may change in the future as guidance from the `zarr` community becomes clear.
+If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value.  However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed.  This may change in the future as guidance from the `zarr` community becomes clear.
 
 ## Concurrency
 
@@ -133,7 +105,7 @@ Chunk concurrency is typically favored because:
 
 ## Supported Indexing Methods
 
-We **do not** officially support the following indexing methods.  Some of these methods may error out, others may not:
+The following methods will trigger use with the old zarr-python pipeline:
 
 1. Any `oindex` or `vindex` integer `np.ndarray` indexing with dimensionality >=3 i.e.,
 
@@ -163,7 +135,10 @@ We **do not** officially support the following indexing methods.  Some of these
    arr[0:10, ..., 0:5]
    ```
 
-Otherwise, we believe that we support your indexing case: slices, ints, and all integer `np.ndarray` indices in 2D for reading, contiguous integer `np.ndarray` indices along one axis for writing etc.  Please file an issue if you believe we have more holes in our coverage than we are aware of or you wish to contribute!  For example, we have an [issue in zarrs for integer-array indexing](https://github.com/LDeakin/zarrs/issues/52) that would unblock a lot of these issues!
 
-That being said, using non-contiguous integer `np.ndarray` indexing for reads may not be as fast as expected given the performance of other supported methods.  Until `zarrs` supports integer indexing, only fetching chunks is done in `rust` while indexing then occurs in `python`.
+Furthermore, using anything except contiguous (i.e., slices or consecutive integer) `np.ndarray` for numeric data will fall back to the default `zarr-python` implementation.
+
+Please file an issue if you believe we have more holes in our coverage than we are aware of or you wish to contribute!  For example, we have an [issue in zarrs for integer-array indexing](https://github.com/LDeakin/zarrs/issues/52) that would unblock a lot the use of the rust pipeline for that use-case (very useful for mini-batch training perhaps!).
+
+Further, any codecs not supported by `zarrs` will also automatically fall back to the python implementation.
 

zarrs-0.2.2.dist-info/RECORD

@@ -0,0 +1,11 @@
+zarrs-0.2.2.dist-info/METADATA,sha256=8K1AOS_SVQgRLzQ2rWtNbJxCAb12XmTLRrT6QklQgOI,8054
+zarrs-0.2.2.dist-info/WHEEL,sha256=N8W3-0eDM6igWj-H12r7VkxoMaJIqJLxUyWCFstEaGg,105
+zarrs-0.2.2.dist-info/entry_points.txt,sha256=EzI6yCIUPDHBHzjDdexuGGYbOLXf8x2ICokOJXnuX3k,68
+zarrs-0.2.2.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
+zarrs/__init__.py,sha256=lRVtAPzCzJkGs4vQrW4UgANq-pC-khS0ZF7HTj4__Hg,489
+zarrs/_internal.abi3.so,sha256=tnP5IiuDmhfwB15cX4yTyu1mVZgShtJy1lnS87TzK1o,14928348
+zarrs/_internal.pyi,sha256=a_D4yx99r4xeQX1ntY_A_Q4wVmLeLwJZHWAQV_mVu9A,1308
+zarrs/pipeline.py,sha256=YfB13GWNfxELerXVtJ_ipFwSL7bN-YuPys6jCB9lnms,9008
+zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+zarrs/utils.py,sha256=W2XCjJDVrdHYJgtVaRKN533Ljw1MF7o0YwXuz5ZAk2g,7020
+zarrs-0.2.2.dist-info/RECORD,,

{zarrs-0.1.1.dist-info → zarrs-0.2.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: maturin (1.7.4)
+Generator: maturin (1.9.4)
 Root-Is-Purelib: false
 Tag: cp311-abi3-macosx_10_12_x86_64

zarrs-0.2.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[zarr.codec_pipeline]
+zarrs.codec_pipeline=zarrs:ZarrsCodecPipeline

zarrs-0.1.1.dist-info/RECORD

@@ -1,10 +0,0 @@
-zarrs-0.1.1.dist-info/METADATA,sha256=jwXKTiA6MIVeg8i_zrb3I4H7GTsqG5lA-H8GicZUDTE,9215
-zarrs-0.1.1.dist-info/WHEEL,sha256=LZygbeT1PTQw7a9tONPp78bbG4FZc86U59Z0RFJcoR8,105
-zarrs-0.1.1.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
-zarrs/__init__.py,sha256=4oWtWDZO8r7z4Uh7Fy_brmkxXDpULQdgjlA0iFw98eA,573
-zarrs/_internal.pyi,sha256=fWWKwTSf8bHFlK6XusyHlDzahTlcjcfth6c1nJyrYdE,1375
-zarrs/utils.py,sha256=PNZWAjU46s2_ZjWN4cxr2JsZHl88H2NAiaPzFf7SPUE,6403
-zarrs/pipeline.py,sha256=M2r7QIauWIyQrNkd722osMZOeQ1LqYP1aX8SypplaAY,6021
-zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-zarrs/_internal.abi3.so,sha256=fUxAxUdypeC3THpaKKO1nppCP0traxV7bFsmGB63oLc,6804296
-zarrs-0.1.1.dist-info/RECORD,,