zarrs 0.2.2__cp311-abi3-manylinux_2_28_ppc64le.whl

zarrs/__init__.py

@@ -0,0 +1,16 @@
+from ._internal import __version__
+from .pipeline import ZarrsCodecPipeline as _ZarrsCodecPipeline
+from .utils import CollapsedDimensionError, DiscontiguousArrayError
+
+
+# Need to do this redirection so people can access the pipeline as `zarrs.ZarrsCodecPipeline` instead of `zarrs.pipeline.ZarrsCodecPipeline`
+class ZarrsCodecPipeline(_ZarrsCodecPipeline):
+    pass
+
+
+__all__ = [
+    "ZarrsCodecPipeline",
+    "DiscontiguousArrayError",
+    "CollapsedDimensionError",
+    "__version__",
+]

zarrs/_internal.pyi

@@ -0,0 +1,44 @@
+# This file is automatically generated by pyo3_stub_gen
+# ruff: noqa: E501, F401
+
+import builtins
+import typing
+
+import numpy.typing
+import zarr.abc.store
+
+@typing.final
+class ChunkItem:
+    def __new__(
+        cls,
+        key: builtins.str,
+        chunk_subset: typing.Sequence[slice],
+        chunk_shape: typing.Sequence[builtins.int],
+        subset: typing.Sequence[slice],
+        shape: typing.Sequence[builtins.int],
+    ) -> ChunkItem: ...
+
+@typing.final
+class CodecPipelineImpl:
+    def __new__(
+        cls,
+        array_metadata: builtins.str,
+        store_config: zarr.abc.store.Store,
+        *,
+        validate_checksums: builtins.bool = False,
+        chunk_concurrent_minimum: builtins.int | None = None,
+        chunk_concurrent_maximum: builtins.int | None = None,
+        num_threads: builtins.int | None = None,
+        direct_io: builtins.bool = False,
+    ) -> CodecPipelineImpl: ...
+    def retrieve_chunks_and_apply_index(
+        self,
+        chunk_descriptions: typing.Sequence[ChunkItem],
+        value: numpy.typing.NDArray[typing.Any],
+    ) -> None: ...
+    def store_chunks_with_indices(
+        self,
+        chunk_descriptions: typing.Sequence[ChunkItem],
+        value: numpy.typing.NDArray[typing.Any],
+        write_empty_chunks: builtins.bool,
+    ) -> None: ...

zarrs/pipeline.py

@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+from warnings import warn
+
+import numpy as np
+from zarr.abc.codec import Codec, CodecPipeline
+from zarr.codecs._v2 import V2Codec
+from zarr.core import BatchedCodecPipeline
+from zarr.core.config import config
+from zarr.core.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Iterator
+    from typing import Self
+
+    from zarr.abc.store import ByteGetter, ByteSetter, Store
+    from zarr.core.array_spec import ArraySpec
+    from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer
+    from zarr.core.chunk_grids import ChunkGrid
+    from zarr.core.indexing import SelectorTuple
+    from zarr.dtype import ZDType
+
+from ._internal import CodecPipelineImpl
+from .utils import (
+    CollapsedDimensionError,
+    DiscontiguousArrayError,
+    FillValueNoneError,
+    make_chunk_info_for_rust_with_indices,
+)
+
+
+class UnsupportedDataTypeError(Exception):
+    pass
+
+
+class UnsupportedMetadataError(Exception):
+    pass
+
+
+def get_codec_pipeline_impl(
+    metadata: ArrayMetadata, store: Store, *, strict: bool
+) -> CodecPipelineImpl | None:
+    try:
+        array_metadata_json = json.dumps(metadata.to_dict())
+        # Maintain old behavior: https://github.com/zarrs/zarrs-python/tree/b36ba797cafec77f5f41a25316be02c718a2b4f8?tab=readme-ov-file#configuration
+        validate_checksums = config.get("codec_pipeline.validate_checksums", True)
+        if validate_checksums is None:
+            validate_checksums = True
+        return CodecPipelineImpl(
+            array_metadata_json,
+            store_config=store,
+            validate_checksums=validate_checksums,
+            chunk_concurrent_minimum=config.get(
+                "codec_pipeline.chunk_concurrent_minimum", None
+            ),
+            chunk_concurrent_maximum=config.get(
+                "codec_pipeline.chunk_concurrent_maximum", None
+            ),
+            num_threads=config.get("threading.max_workers", None),
+            direct_io=config.get("codec_pipeline.direct_io", False),
+        )
+    except TypeError as e:
+        if strict:
+            raise UnsupportedMetadataError() from e
+
+        warn(
+            f"Array is unsupported by ZarrsCodecPipeline: {e}",
+            category=UserWarning,
+        )
+        return None
+
+
+def get_codec_pipeline_fallback(
+    metadata: ArrayMetadata, *, strict: bool
+) -> BatchedCodecPipeline | None:
+    if strict:
+        return None
+    else:
+        codecs = array_metadata_to_codecs(metadata)
+        return BatchedCodecPipeline.from_codecs(codecs)
+
+
+class ZarrsCodecPipelineState(TypedDict):
+    codec_metadata_json: str
+    codecs: tuple[Codec, ...]
+
+
+def array_metadata_to_codecs(metadata: ArrayMetadata) -> list[Codec]:
+    if isinstance(metadata, ArrayV3Metadata):
+        return metadata.codecs
+    elif isinstance(metadata, ArrayV2Metadata):
+        v2_codec = V2Codec(filters=metadata.filters, compressor=metadata.compressor)
+        return [v2_codec]
+
+
+@dataclass
+class ZarrsCodecPipeline(CodecPipeline):
+    metadata: ArrayMetadata
+    store: Store
+    impl: CodecPipelineImpl | None
+    python_impl: BatchedCodecPipeline | None
+
+    def __getstate__(self) -> ZarrsCodecPipelineState:
+        return {"metadata": self.metadata, "store": self.store}
+
+    def __setstate__(self, state: ZarrsCodecPipelineState):
+        self.metadata = state["metadata"]
+        self.store = state["store"]
+        strict = config.get("codec_pipeline.strict", False)
+        self.impl = get_codec_pipeline_impl(self.metadata, self.store, strict=strict)
+        self.python_impl = get_codec_pipeline_fallback(self.metadata, strict=strict)
+
+    def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
+        return self
+
+    @classmethod
+    def from_codecs(cls, codecs: Iterable[Codec]) -> Self:
+        return BatchedCodecPipeline.from_codecs(codecs)
+
+    @classmethod
+    def from_array_metadata_and_store(
+        cls, array_metadata: ArrayMetadata, store: Store
+    ) -> Self:
+        strict = config.get("codec_pipeline.strict", False)
+        return cls(
+            metadata=array_metadata,
+            store=store,
+            impl=get_codec_pipeline_impl(array_metadata, store, strict=strict),
+            python_impl=get_codec_pipeline_fallback(array_metadata, strict=strict),
+        )
+
+    @property
+    def supports_partial_decode(self) -> bool:
+        return False
+
+    @property
+    def supports_partial_encode(self) -> bool:
+        return False
+
+    def __iter__(self) -> Iterator[Codec]:
+        yield from self.codecs
+
+    def validate(
+        self, *, shape: tuple[int, ...], dtype: ZDType, chunk_grid: ChunkGrid
+    ) -> None:
+        raise NotImplementedError("validate")
+
+    def compute_encoded_size(self, byte_length: int, array_spec: ArraySpec) -> int:
+        raise NotImplementedError("compute_encoded_size")
+
+    async def decode(
+        self,
+        chunk_bytes_and_specs: Iterable[tuple[Buffer | None, ArraySpec]],
+    ) -> Iterable[NDBuffer | None]:
+        raise NotImplementedError("decode")
+
+    async def encode(
+        self,
+        chunk_arrays_and_specs: Iterable[tuple[NDBuffer | None, ArraySpec]],
+    ) -> Iterable[Buffer | None]:
+        raise NotImplementedError("encode")
+
+    async def read(
+        self,
+        batch_info: Iterable[
+            tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
+        ],
+        out: NDBuffer,  # type: ignore
+        drop_axes: tuple[int, ...] = (),  # FIXME: unused
+    ) -> None:
+        # FIXME: Error if array is not in host memory
+        if not out.dtype.isnative:
+            raise RuntimeError("Non-native byte order not supported")
+        try:
+            if self.impl is None:
+                raise UnsupportedMetadataError()
+            self._raise_error_on_unsupported_batch_dtype(batch_info)
+            chunks_desc = make_chunk_info_for_rust_with_indices(
+                batch_info, drop_axes, out.shape
+            )
+        except (
+            UnsupportedMetadataError,
+            DiscontiguousArrayError,
+            CollapsedDimensionError,
+            UnsupportedDataTypeError,
+            FillValueNoneError,
+        ):
+            if self.python_impl is None:
+                raise
+            await self.python_impl.read(batch_info, out, drop_axes)
+            return None
+        else:
+            out: NDArrayLike = out.as_ndarray_like()
+            await asyncio.to_thread(
+                self.impl.retrieve_chunks_and_apply_index,
+                chunks_desc.chunk_info_with_indices,
+                out,
+            )
+            return None
+
+    async def write(
+        self,
+        batch_info: Iterable[
+            tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
+        ],
+        value: NDBuffer,  # type: ignore
+        drop_axes: tuple[int, ...] = (),
+    ) -> None:
+        try:
+            if self.impl is None:
+                raise UnsupportedMetadataError()
+            self._raise_error_on_unsupported_batch_dtype(batch_info)
+            chunks_desc = make_chunk_info_for_rust_with_indices(
+                batch_info, drop_axes, value.shape
+            )
+        except (
+            UnsupportedMetadataError,
+            DiscontiguousArrayError,
+            CollapsedDimensionError,
+            UnsupportedDataTypeError,
+            FillValueNoneError,
+        ):
+            if self.python_impl is None:
+                raise
+            await self.python_impl.write(batch_info, value, drop_axes)
+            return None
+        else:
+            # FIXME: Error if array is not in host memory
+            value_np: NDArrayLike | np.ndarray = value.as_ndarray_like()
+            if not value_np.dtype.isnative:
+                value_np = np.ascontiguousarray(
+                    value_np, dtype=value_np.dtype.newbyteorder("=")
+                )
+            elif not value_np.flags.c_contiguous:
+                value_np = np.ascontiguousarray(value_np)
+            await asyncio.to_thread(
+                self.impl.store_chunks_with_indices,
+                chunks_desc.chunk_info_with_indices,
+                value_np,
+                chunks_desc.write_empty_chunks,
+            )
+            return None
+
+    def _raise_error_on_unsupported_batch_dtype(
+        self,
+        batch_info: Iterable[
+            tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
+        ],
+    ):
+        # https://github.com/LDeakin/zarrs/blob/0532fe983b7b42b59dbf84e50a2fe5e6f7bad4ce/zarrs_metadata/src/v2_to_v3.rs#L289-L293 for VSUMm
+        # Further, our pipeline does not support variable-length objects due to limitations on decode_into, so object/np.dtypes.StringDType is also out
+        if any(
+            info.dtype.to_native_dtype().kind in {"V", "S", "U", "M", "m", "O", "T"}
+            for (_, info, _, _, _) in batch_info
+        ):
+            raise UnsupportedDataTypeError()

zarrs/utils.py

@@ -0,0 +1,205 @@
+from __future__ import annotations
+
+import operator
+import os
+from dataclasses import dataclass
+from functools import reduce
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+from zarr.core.array_spec import ArraySpec
+from zarr.core.indexing import SelectorTuple, is_integer
+
+from zarrs._internal import ChunkItem
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+    from types import EllipsisType
+
+    from zarr.abc.store import ByteGetter, ByteSetter
+    from zarr.dtype import ZDType
+
+
+# adapted from https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
+def get_max_threads() -> int:
+    return (os.cpu_count() or 1) + 4
+
+
+class DiscontiguousArrayError(Exception):
+    pass
+
+
+class CollapsedDimensionError(Exception):
+    pass
+
+
+class FillValueNoneError(Exception):
+    pass
+
+
+# This is a (mostly) copy of the function from zarr.core.indexing that fixes:
+#   DeprecationWarning: Conversion of an array with ndim > 0 to a scalar is deprecated
+# TODO: Upstream this fix
+def make_slice_selection(selection: tuple[np.ndarray | float]) -> list[slice]:
+    ls: list[slice] = []
+    for dim_selection in selection:
+        if is_integer(dim_selection):
+            ls.append(slice(int(dim_selection), int(dim_selection) + 1, 1))
+        elif isinstance(dim_selection, np.ndarray):
+            dim_selection = dim_selection.ravel()
+            if len(dim_selection) == 1:
+                ls.append(
+                    slice(int(dim_selection.item()), int(dim_selection.item()) + 1, 1)
+                )
+            else:
+                diff = np.diff(dim_selection)
+                if (diff != 1).any() and (diff != 0).any():
+                    raise DiscontiguousArrayError(diff)
+                ls.append(slice(dim_selection[0], dim_selection[-1] + 1, 1))
+        else:
+            ls.append(dim_selection)
+    return ls
+
+
+def selector_tuple_to_slice_selection(selector_tuple: SelectorTuple) -> list[slice]:
+    if isinstance(selector_tuple, slice):
+        return [selector_tuple]
+    if all(isinstance(s, slice) for s in selector_tuple):
+        return list(selector_tuple)
+    return make_slice_selection(selector_tuple)
+
+
+def resulting_shape_from_index(
+    array_shape: tuple[int, ...],
+    index_tuple: tuple[int | slice | EllipsisType | np.ndarray],
+    drop_axes: tuple[int, ...],
+    *,
+    pad: bool,
+) -> tuple[int, ...]:
+    result_shape = []
+    advanced_index_shapes = [
+        idx.shape for idx in index_tuple if isinstance(idx, np.ndarray)
+    ]
+    basic_shape_index = 0
+
+    # Broadcast all advanced indices, if any
+    if advanced_index_shapes:
+        result_shape += np.broadcast_shapes(*advanced_index_shapes)
+        # Consume dimensions from array_shape
+        basic_shape_index += len(advanced_index_shapes)
+
+    # Process each remaining index in index_tuple
+    for idx in index_tuple:
+        if isinstance(idx, int):
+            # Integer index reduces dimension, so skip this dimension in array_shape
+            basic_shape_index += 1
+        elif isinstance(idx, slice):
+            if idx.step is not None and idx.step > 1:
+                raise DiscontiguousArrayError(
+                    "Step size greater than 1 is not supported"
+                )
+            # Slice keeps dimension, adjust size accordingly
+            start, stop, _ = idx.indices(array_shape[basic_shape_index])
+            result_shape.append(stop - start)
+            basic_shape_index += 1
+        elif idx is Ellipsis:
+            # Calculate number of dimensions that Ellipsis should fill
+            num_to_fill = len(array_shape) - len(index_tuple) + 1
+            result_shape += array_shape[
+                basic_shape_index : basic_shape_index + num_to_fill
+            ]
+            basic_shape_index += num_to_fill
+        elif not isinstance(idx, np.ndarray):
+            raise ValueError(f"Invalid index type: {type(idx)}")
+
+    # Step 4: Append remaining dimensions from array_shape if fewer indices were used
+    if basic_shape_index < len(array_shape) and pad:
+        result_shape += array_shape[basic_shape_index:]
+
+    return tuple(size for idx, size in enumerate(result_shape) if idx not in drop_axes)
+
+
+def prod_op(x: Iterable[int]) -> int:
+    return reduce(operator.mul, x, 1)
+
+
+def get_shape_for_selector(
+    selector_tuple: SelectorTuple,
+    shape: tuple[int, ...],
+    *,
+    pad: bool,
+    drop_axes: tuple[int, ...] = (),
+) -> tuple[int, ...]:
+    if isinstance(selector_tuple, slice | np.ndarray):
+        return resulting_shape_from_index(
+            shape,
+            (selector_tuple,),
+            drop_axes,
+            pad=pad,
+        )
+    return resulting_shape_from_index(shape, selector_tuple, drop_axes, pad=pad)
+
+
+def get_implicit_fill_value(dtype: ZDType, fill_value: Any) -> Any:
+    if fill_value is None:
+        fill_value = dtype.default_scalar()
+    return fill_value
+
+
+@dataclass(frozen=True)
+class RustChunkInfo:
+    chunk_info_with_indices: list[ChunkItem]
+    write_empty_chunks: bool
+
+
+def make_chunk_info_for_rust_with_indices(
+    batch_info: Iterable[
+        tuple[ByteGetter | ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
+    ],
+    drop_axes: tuple[int, ...],
+    shape: tuple[int, ...],
+) -> RustChunkInfo:
+    shape = shape if shape else (1,)  # constant array
+    chunk_info_with_indices: list[ChunkItem] = []
+    write_empty_chunks: bool = True
+    for (
+        byte_getter,
+        chunk_spec,
+        chunk_selection,
+        out_selection,
+        _,
+    ) in batch_info:
+        write_empty_chunks = chunk_spec.config.write_empty_chunks
+        if chunk_spec.fill_value is None:
+            chunk_spec = ArraySpec(
+                chunk_spec.shape,
+                chunk_spec.dtype,
+                get_implicit_fill_value(chunk_spec.dtype, chunk_spec.fill_value),
+                chunk_spec.config,
+                chunk_spec.prototype,
+            )
+        out_selection_as_slices = selector_tuple_to_slice_selection(out_selection)
+        chunk_selection_as_slices = selector_tuple_to_slice_selection(chunk_selection)
+        shape_chunk_selection_slices = get_shape_for_selector(
+            tuple(chunk_selection_as_slices),
+            chunk_spec.shape,
+            pad=True,
+            drop_axes=drop_axes,
+        )
+        shape_chunk_selection = get_shape_for_selector(
+            chunk_selection, chunk_spec.shape, pad=True, drop_axes=drop_axes
+        )
+        if prod_op(shape_chunk_selection) != prod_op(shape_chunk_selection_slices):
+            raise CollapsedDimensionError(
+                f"{shape_chunk_selection} != {shape_chunk_selection_slices}"
+            )
+        chunk_info_with_indices.append(
+            ChunkItem(
+                key=byte_getter.path,
+                chunk_subset=chunk_selection_as_slices,
+                chunk_shape=chunk_spec.shape,
+                subset=out_selection_as_slices,
+                shape=shape,
+            )
+        )
+    return RustChunkInfo(chunk_info_with_indices, write_empty_chunks)

zarrs-0.2.2.dist-info/METADATA

@@ -0,0 +1,144 @@
+Metadata-Version: 2.4
+Name: zarrs
+Version: 0.2.2
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Typing :: Typed
+Requires-Dist: numpy>=1.24
+Requires-Dist: zarr>=3.1
+License-File: LICENSE
+Summary: A CodecPipeline for zarr-python backed by the zarrs Rust crate
+Author: Ilan Gold, Lachlan Deakin, Philipp Angerer
+License-Expression: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+
+# zarrs-python
+
+[![PyPI](https://img.shields.io/pypi/v/zarrs.svg)](https://pypi.org/project/zarrs)
+[![Downloads](https://static.pepy.tech/badge/zarrs/month)](https://pepy.tech/project/zarrs)
+[![Downloads](https://static.pepy.tech/badge/zarrs)](https://pepy.tech/project/zarrs)
+[![Stars](https://img.shields.io/github/stars/zarrs/zarrs-python?style=flat&logo=github&color=yellow)](https://github.com/zarrs/zarrs-python/stargazers)
+![CI](https://github.com/zarrs/zarrs-python/actions/workflows/ci.yml/badge.svg)
+![CD](https://github.com/zarrs/zarrs-python/actions/workflows/cd.yml/badge.svg)
+
+This project serves as a bridge between [`zarrs`](https://docs.rs/zarrs/latest/zarrs/) (Rust) and [`zarr`](https://zarr.readthedocs.io/en/latest/index.html) (`zarr-python`) via [`PyO3`](https://pyo3.rs/v0.22.3/).  The main goal of the project is to speed up i/o (see [`zarr_benchmarks`](https://github.com/LDeakin/zarr_benchmarks)).
+
+To use the project, simply install our package (which depends on `zarr-python>=3.0.0`), and run:
+
+```python
+import zarr
+zarr.config.set({"codec_pipeline.path": "zarrs.ZarrsCodecPipeline"})
+```
+
+You can then use your `zarr` as normal (with some caveats)!
+
+## API
+
+We export a `ZarrsCodecPipeline` class so that `zarr-python` can use the class but it is not meant to be instantiated and we do not guarantee the stability of its API beyond what is required so that `zarr-python` can use it.  Therefore, it is not documented here.
+
+At the moment, we only support a subset of the `zarr-python` stores:
+
+- [`LocalStore`](https://zarr.readthedocs.io/en/latest/api/zarr/storage/#zarr.storage.LocalStore) (local filesystem)
+- [`ObjectStore`](https://zarr.readthedocs.io/en/latest/user-guide/storage/#object-store) (cloud storage)
+- [`HTTPFileSystem`](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.implementations.http.HTTPFileSystem) via [`FsspecStore`](https://zarr.readthedocs.io/en/latest/api/zarr/storage/#zarr.storage.FsspecStore)
+
+A `NotImplementedError` will be raised if a store is not supported.
+
+### Configuration
+
+`ZarrsCodecPipeline` options are exposed through `zarr.config`.
+
+Standard `zarr.config` options control some functionality (see the defaults in the [config.py](https://github.com/zarr-developers/zarr-python/blob/main/src/zarr/core/config.py) of `zarr-python`):
+- `threading.max_workers`: the maximum number of threads used internally by the `ZarrsCodecPipeline` on the Rust side.
+  - Defaults to the number of threads in the global `rayon` thread pool if set to `None`, which is [typically the number of logical CPUs](https://docs.rs/rayon/latest/rayon/struct.ThreadPoolBuilder.html#method.num_threads).
+- `array.write_empty_chunks`: whether or not to store empty chunks.
+  - Defaults to false if `None`. Note that checking for emptiness has some overhead, see [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#store-empty-chunks) for more info.
+
+The `ZarrsCodecPipeline` specific options are:
+- `codec_pipeline.chunk_concurrent_maximum`: the maximum number of chunks stored/retrieved concurrently.
+  - Defaults to the number of logical CPUs if `None`. It is constrained by `threading.max_workers` as well.
+- `codec_pipeline.chunk_concurrent_minimum`: the minimum number of chunks retrieved/stored concurrently when balancing chunk/codec concurrency.
+  - Defaults to 4 if `None`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#chunk-concurrent-minimum) for more info.
+- `codec_pipeline.validate_checksums`: enable checksum validation (e.g. with the CRC32C codec).
+  - Defaults to `True`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#validate-checksums) for more info.
+- `codec_pipeline.direct_io`: enable `O_DIRECT` read/write, needs support from the operating system (currently only Linux) and file system.
+  - Defaults to `False`.
+- `codec_pipeline.strict`: raise exceptions for unsupported operations instead of falling back to the default codec pipeline of `zarr-python`.
+  - Defaults to `False`.
+
+For example:
+```python
+zarr.config.set({
+    "threading.max_workers": None,
+    "array.write_empty_chunks": False,
+    "codec_pipeline": {
+        "path": "zarrs.ZarrsCodecPipeline",
+        "validate_checksums": True,
+        "chunk_concurrent_maximum": None,
+        "chunk_concurrent_minimum": 4,
+        "direct_io": False,
+        "strict": False
+    }
+})
+```
+
+If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value.  However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed.  This may change in the future as guidance from the `zarr` community becomes clear.
+
+## Concurrency
+
+Concurrency can be classified into two types:
+- chunk (outer) concurrency: the number of chunks retrieved/stored concurrently.
+  - This is chosen automatically based on various factors, such as the chunk size and codecs.
+  - It is constrained between `codec_pipeline.chunk_concurrent_minimum` and `codec_pipeline.chunk_concurrent_maximum` for operations involving multiple chunks.
+- codec (inner) concurrency: the number of threads encoding/decoding a chunk.
+  - This is chosen automatically in combination with the chunk concurrency.
+
+The product of the chunk and codec concurrency will approximately match `threading.max_workers`.
+
+Chunk concurrency is typically favored because:
+- parallel encoding/decoding can have a high overhead with some codecs, especially with small chunks, and
+- it is advantageous to retrieve/store multiple chunks concurrently, especially with high latency stores.
+
+`zarrs-python` will often favor codec concurrency with sharded arrays, as they are well suited to codec concurrency.
+
+## Supported Indexing Methods
+
+The following methods will trigger use with the old zarr-python pipeline:
+
+1. Any `oindex` or `vindex` integer `np.ndarray` indexing with dimensionality >=3 i.e.,
+
+   ```python
+   arr[np.array([...]), :, np.array([...])]
+   arr[np.array([...]), np.array([...]), np.array([...])]
+   arr[np.array([...]), np.array([...]), np.array([...])] = ...
+   arr.oindex[np.array([...]), np.array([...]), np.array([...])] = ...
+   ```
+
+2. Any `vindex` or `oindex` discontinuous integer `np.ndarray` indexing for writes in 2D
+
+   ```python
+   arr[np.array([0, 5]), :] = ...
+   arr.oindex[np.array([0, 5]), :] = ...
+   ```
+
+3. `vindex` writes in 2D where both indexers are integer `np.ndarray` indices i.e.,
+
+   ```python
+   arr[np.array([...]), np.array([...])] = ...
+   ```
+
+4. Ellipsis indexing.  We have tested some, but others fail even with `zarr-python`'s default codec pipeline.  Thus for now we advise proceeding with caution here.
+
+   ```python
+   arr[0:10, ..., 0:5]
+   ```
+
+
+Furthermore, using anything except contiguous (i.e., slices or consecutive integer) `np.ndarray` for numeric data will fall back to the default `zarr-python` implementation.
+
+Please file an issue if you believe we have more holes in our coverage than we are aware of or you wish to contribute!  For example, we have an [issue in zarrs for integer-array indexing](https://github.com/LDeakin/zarrs/issues/52) that would unblock a lot the use of the rust pipeline for that use-case (very useful for mini-batch training perhaps!).
+
+Further, any codecs not supported by `zarrs` will also automatically fall back to the python implementation.
+

zarrs-0.2.2.dist-info/RECORD

@@ -0,0 +1,11 @@
+zarrs-0.2.2.dist-info/METADATA,sha256=8K1AOS_SVQgRLzQ2rWtNbJxCAb12XmTLRrT6QklQgOI,8054
+zarrs-0.2.2.dist-info/WHEEL,sha256=LyKrmraG_uSXY5vhpX0CJs5HieuGNOxWFsl0o305mFk,108
+zarrs-0.2.2.dist-info/entry_points.txt,sha256=EzI6yCIUPDHBHzjDdexuGGYbOLXf8x2ICokOJXnuX3k,68
+zarrs-0.2.2.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
+zarrs/__init__.py,sha256=lRVtAPzCzJkGs4vQrW4UgANq-pC-khS0ZF7HTj4__Hg,489
+zarrs/_internal.abi3.so,sha256=GCAoGz9VUm-OVKCf_MRcPJwsZxtNI0sdsYYJUTr1vRk,17179376
+zarrs/_internal.pyi,sha256=a_D4yx99r4xeQX1ntY_A_Q4wVmLeLwJZHWAQV_mVu9A,1308
+zarrs/pipeline.py,sha256=YfB13GWNfxELerXVtJ_ipFwSL7bN-YuPys6jCB9lnms,9008
+zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+zarrs/utils.py,sha256=W2XCjJDVrdHYJgtVaRKN533Ljw1MF7o0YwXuz5ZAk2g,7020
+zarrs-0.2.2.dist-info/RECORD,,

zarrs-0.2.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.9.4)
+Root-Is-Purelib: false
+Tag: cp311-abi3-manylinux_2_28_ppc64le

zarrs-0.2.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[zarr.codec_pipeline]
+zarrs.codec_pipeline=zarrs:ZarrsCodecPipeline

zarrs-0.2.2.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Ilan Gold, Lachlan Deakin, Philipp Angerer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.