numcodecs-combinators 0.2.6__tar.gz → 0.2.8__tar.gz

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.8}/.github/workflows/ci.yml

@@ -6,7 +6,10 @@ on:
       - main
   pull_request:
     branches:
-      - '*'
+      - "*"
+
+env:
+  CLICOLOR: 1
 
 jobs:
   check:
@@ -21,6 +24,16 @@ jobs:
         with:
           args: check
 
+  spelling:
+    name: Spellcheck
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout the Repository
+      uses: actions/checkout@v4
+
+    - name: Spellcheck repo
+      uses: crate-ci/typos@v1.32.0
+
   fmt:
     name: Formatting
     runs-on: ubuntu-latest
@@ -50,7 +63,7 @@ jobs:
         run: uv sync --all-extras --dev && uv pip install .
 
       - name: Run tests
-        run: uv run pytest
+        run: uv run pytest -v -W error
 
   mypy:
     name: Typecheck

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: numcodecs-combinators
-Version: 0.2.6
+Version: 0.2.8
 Summary: Combinator codecs for the `numcodecs` buffer compression API
 License: Copyright (c) 2024, Juniper Tyree
         
@@ -382,6 +382,7 @@ Requires-Python: >=3.10
 Requires-Dist: numcodecs<0.17,>=0.13.0
 Requires-Dist: numpy~=2.0
 Requires-Dist: typing-extensions~=4.6
+Requires-Dist: varint~=1.0
 Provides-Extra: xarray
 Requires-Dist: dask>=2024.6; extra == 'xarray'
 Requires-Dist: xarray>=2024.06; extra == 'xarray'
@@ -400,6 +401,8 @@ Combinator codecs for the [`numcodecs`] buffer compression API.
 The following combinators, implementing the `CodecCombinatorMixin` are provided:
 
 - `CodecStack`: a stack of codecs
+- `FramedCodecStack`: a stack of codecs that is framed with array data type and shape information
+- `PickBestCodec`: pick the best codec to encode the data
 
 [`numcodecs`]: https://numcodecs.readthedocs.io/en/stable/
 

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.8}/README.md

@@ -11,6 +11,8 @@ Combinator codecs for the [`numcodecs`] buffer compression API.
 The following combinators, implementing the `CodecCombinatorMixin` are provided:
 
 - `CodecStack`: a stack of codecs
+- `FramedCodecStack`: a stack of codecs that is framed with array data type and shape information
+- `PickBestCodec`: pick the best codec to encode the data
 
 [`numcodecs`]: https://numcodecs.readthedocs.io/en/stable/
 

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.8}/docs/index.md

@@ -11,6 +11,8 @@ Combinator codecs for the [`numcodecs`][numcodecs] buffer compression API.
 The following combinators, implementing the [`CodecCombinatorMixin`][numcodecs_combinators.abc.CodecCombinatorMixin] are provided:
 
 - [`CodecStack`][numcodecs_combinators.stack.CodecStack]: a stack of codecs
+- [`FramedCodecStack`][numcodecs_combinators.framed.FramedCodecStack]: a stack of codecs that is framed with array data type and shape information
+- [`PickBestCodec`][numcodecs_combinators.best.PickBestCodec]: pick the best codec to encode the data
 
 ## Funding
 

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.8}/mkdocs.yml

@@ -54,5 +54,9 @@ plugins:
                   modules: true
           inventories:
             - https://docs.python.org/3.12/objects.inv
+            - https://numpy.org/doc/2.2/objects.inv
             - https://numcodecs.readthedocs.io/en/v0.15.0/objects.inv
             - https://docs.xarray.dev/en/v2025.01.0/objects.inv
+
+watch:
+  - src

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "numcodecs-combinators"
-version = "0.2.6"
+version = "0.2.8"
 description = "Combinator codecs for the `numcodecs` buffer compression API"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -13,6 +13,7 @@ dependencies = [
     "numcodecs>=0.13.0,<0.17",
     "numpy~=2.0",
     "typing-extensions~=4.6",
+    "varint~=1.0",
 ]
 optional-dependencies.xarray = [ "xarray>=2024.06", "dask>=2024.6" ]
 
@@ -20,6 +21,8 @@ optional-dependencies.xarray = [ "xarray>=2024.06", "dask>=2024.6" ]
 dev = ["mypy~=1.14", "pytest~=8.3"]
 
 [project.entry-points."numcodecs.codecs"]
+"combinators.best" = "numcodecs_combinators.best:PickBestCodec"
+"combinators.framed" = "numcodecs_combinators.framed:FramedCodecStack"
 "combinators.stack" = "numcodecs_combinators.stack:CodecStack"
 
 [tool.setuptools.packages.find]
@@ -28,6 +31,13 @@ where = ["src"]
 [tool.setuptools.package-data]
 "numcodecs_combinators" = ["py.typed"]
 
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "I"]
+
+[tool.pytest.ini_options]
+addopts = ["--import-mode=importlib"]
+xfail_strict = true
+
 [[tool.mypy.overrides]]
-module = ["numcodecs.*"]
+module = ["numcodecs.*", "varint.*"]
 follow_untyped_imports = true

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.8}/src/numcodecs_combinators/__init__.py

@@ -1,9 +1,15 @@
 """
 Combinator codecs for the [`numcodecs`][numcodecs] buffer compression API.
 
-The following combinators, implementing the [`CodecCombinatorMixin`][numcodecs_combinators.abc.CodecCombinatorMixin] are provided:
+The following combinators, implementing the
+[`CodecCombinatorMixin`][numcodecs_combinators.abc.CodecCombinatorMixin] are
+provided:
 
 - [`CodecStack`][numcodecs_combinators.stack.CodecStack]: a stack of codecs
+- [`FramedCodecStack`][numcodecs_combinators.framed.FramedCodecStack]: a stack
+  of codecs that is framed with array data type and shape information
+- [`PickBestCodec`][numcodecs_combinators.best.PickBestCodec]: pick the best
+  codec to encode the data
 """
 
 __all__ = ["map_codec"]
@@ -14,6 +20,8 @@ from typing import Callable
 from numcodecs.abc import Codec
 
 from . import abc as abc
+from . import best as best
+from . import framed as framed
 from . import stack as stack
 
 

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.8}/src/numcodecs_combinators/abc.py

@@ -4,12 +4,11 @@ This module defines the [`CodecCombinatorMixin`][numcodecs_combinators.abc.Codec
 
 __all__ = ["CodecCombinatorMixin"]
 
+from abc import ABC, abstractmethod
 from typing import Callable
 
 from numcodecs.abc import Codec
 
-from abc import ABC, abstractmethod
-
 
 class CodecCombinatorMixin(ABC):
     """

numcodecs_combinators-0.2.8/src/numcodecs_combinators/best.py

@@ -0,0 +1,212 @@
+"""
+This module defines the [`PickBestCodec`][numcodecs_combinators.best.PickBestCodec] class, which picks the codec that encoded the data best.
+"""
+
+__all__ = ["PickBestCodec"]
+
+from io import BytesIO
+from typing import Callable, Optional
+
+import numcodecs
+import numcodecs.compat
+import numcodecs.registry
+import numpy as np
+import varint
+from numcodecs.abc import Codec
+from typing_extensions import Buffer, Self  # MSPV 3.12
+
+from .abc import CodecCombinatorMixin
+
+
+class PickBestCodec(Codec, CodecCombinatorMixin, tuple[Codec]):
+    """
+    A codec that tries encoding with all combined codecs and then picks the one with the fewest bytes.
+
+    The inner codecs must all encode to 1D byte arrays. To use a codec not
+    encoding to bytes with this combinator, you can wrap it using
+    [`FramedCodecStack(codec)`][numcodecs_combinators.framed.FramedCodecStack]
+    combinator.
+
+    This combinator uses the ULEB128 variable length integer encoding to encode
+    the index of the codec that was chosen to encode and uses this index as a
+    header before the encoded bytes. The header index is only included if this
+    combinator wraps at least two codecs. If this combinator wraps zero codecs,
+    it passes the original data through unchanged.
+    """
+
+    __slots__ = ()
+
+    codec_id: str = "combinators.best"  # type: ignore
+
+    def __init__(self, *args: dict | Codec):
+        pass
+
+    def __new__(cls, *args: dict | Codec) -> Self:
+        return super(PickBestCodec, cls).__new__(
+            cls,
+            tuple(
+                codec
+                if isinstance(codec, Codec)
+                else numcodecs.registry.get_codec(codec)
+                for codec in args
+            ),
+        )
+
+    def encode(self, buf: Buffer) -> bytes:
+        """Encode the data in `buf`.
+
+        Parameters
+        ----------
+        buf : Buffer
+            Data to be encoded. May be any object supporting the new-style
+            buffer protocol.
+
+        Returns
+        -------
+        enc : bytes
+            Encoded and data as a bytestring.
+        """
+
+        if len(self) == 0:
+            return buf
+
+        data = numcodecs.compat.ensure_ndarray(buf)
+
+        best_size = np.inf
+        best_index = None
+        best_encoded = None
+
+        for i, codec in enumerate(self):
+            encoded = numcodecs.compat.ensure_ndarray(codec.encode(np.copy(data)))
+            assert encoded.dtype == np.dtype("uint8"), (
+                f"codec best[{i}] must encode to bytes"
+            )
+            assert encoded.ndim <= 1, f"codec best[{i}] must encode to 1D bytes"
+
+            if encoded.nbytes < best_size:
+                best_size = encoded.nbytes
+                best_index = i
+                best_encoded = encoded
+
+        encoded_index = varint.encode(best_index)
+        encoded_bytes = numcodecs.compat.ensure_bytes(best_encoded)
+
+        if len(self) == 1:
+            return encoded_bytes
+
+        return encoded_index + encoded_bytes
+
+    def decode(self, buf: Buffer, out: Optional[Buffer] = None) -> Buffer:
+        """Decode the data in `buf`.
+
+        Parameters
+        ----------
+        buf : Buffer
+            Encoded data. Must be an object representing a bytestring, e.g.
+            [`bytes`][bytes] or a 1D array of [`np.uint8`][numpy.uint8]s etc.
+        out : Buffer, optional
+            Writeable buffer to store decoded data. N.B. if provided, this buffer must
+            be exactly the right size to store the decoded data.
+
+        Returns
+        -------
+        dec : Buffer
+            Decoded data. May be any object supporting the new-style
+            buffer protocol.
+        """
+
+        if len(self) == 0:
+            return numcodecs.compat.ndarray_copy(buf, out)
+
+        b = numcodecs.compat.ensure_bytes(buf)
+        b_io = BytesIO(b)
+
+        if len(self) == 1:
+            best_index = 0
+        else:
+            best_index = varint.decode_stream(b_io)
+
+        return self[best_index].decode(b_io.read(), out=out)
+
+    def get_config(self) -> dict:
+        """
+        Returns the configuration of the best codec combinator.
+
+        [`numcodecs.registry.get_codec(config)`][numcodecs.registry.get_codec]
+        can be used to reconstruct this combinator from the returned config.
+
+        Returns
+        -------
+        config : dict
+            Configuration of the best codec combinator.
+        """
+
+        return dict(
+            id=type(self).codec_id,
+            codecs=tuple(codec.get_config() for codec in self),
+        )
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Instantiate the best codec combinator from a configuration [`dict`][dict].
+
+        Parameters
+        ----------
+        config : dict
+            Configuration of the best codec combinator.
+
+        Returns
+        -------
+        best : PickBestCodec
+            Instantiated best codec combinator.
+        """
+
+        return cls(*config["codecs"])
+
+    def __repr__(self) -> str:
+        repr = ", ".join(f"{codec!r}" for codec in self)
+
+        return f"{type(self).__name__}({repr})"
+
+    def map(self, mapper: Callable[[Codec], Codec]) -> "PickBestCodec":
+        """
+        Apply the `mapper` to all codecs that are in this combinator.
+        In the returned combinator, each codec is replaced by its mapped codec.
+
+        The `mapper` should recursively apply itself to any inner codecs that
+        also implement the [`CodecCombinatorMixin`][numcodecs_combinators.abc.CodecCombinatorMixin]
+        mixin.
+
+        To automatically handle the recursive application as a caller, you can
+        use
+        ```python
+        numcodecs_combinators.map_codec(best, mapper)
+        ```
+        instead.
+
+        Parameters
+        ----------
+        mapper : Callable[[Codec], Codec]
+            The callable that should be applied to each codec to map over this
+            best codec combinator.
+
+        Returns
+        -------
+        mapped : PickBestCodec
+            The mapped best codec combinator.
+        """
+
+        return PickBestCodec(*map(mapper, self))
+
+    def __add__(self, other) -> "PickBestCodec":
+        return PickBestCodec(*tuple.__add__(self, other))
+
+    def __mul__(self, other) -> "PickBestCodec":
+        return PickBestCodec(*tuple.__mul__(self, other))
+
+    def __rmul__(self, other) -> "PickBestCodec":
+        return PickBestCodec(*tuple.__rmul__(self, other))
+
+
+numcodecs.registry.register_codec(PickBestCodec)

numcodecs_combinators-0.2.8/src/numcodecs_combinators/framed.py

@@ -0,0 +1,260 @@
+"""
+This module defines the [`FramedCodecStack`][numcodecs_combinators.framed.FramedCodecStack] class, which exposes a framed stack of codecs as a combined codec.
+"""
+
+__all__ = ["FramedCodecStack"]
+
+from io import BytesIO
+from sys import byteorder
+from typing import Callable, Optional
+
+import numcodecs
+import numcodecs.compat
+import numcodecs.registry
+import numpy as np
+import varint
+from numcodecs.abc import Codec
+from typing_extensions import Buffer, Self  # MSPV 3.12
+
+from .abc import CodecCombinatorMixin
+
+
+class FramedCodecStack(Codec, CodecCombinatorMixin, tuple[Codec]):
+    """
+    A framed stack of codecs, which makes up a combined codec.
+
+    On encoding, the result of applying the codecs from left to right to encode
+    is framed s.t. the data types and shapes of all arrays (input,
+    intermediary, encoded) are stored as part of the encoding, which is output
+    as a bytestring.
+
+    On decoding, this framing information is used to apply the codecs from
+    right to left to decode into known output data types and shapes.
+
+    Therefore, the [`FramedCodecStack`][numcodecs_combinators.framed.FramedCodecStack]
+    can be used to combine codecs which require knowing the output data type
+    and shape during decoding. It can also be used to encode arrays into
+    bytestrings.
+
+    Unlike the [`CodecStack`][numcodecs_combinators.stack.CodecStack], this
+    class does *not* provide an additional `encode_decode(buf)` method, since
+    it is equivalent to `framed.decode(stack.encode(buf))` due to the framing.
+    """
+
+    __slots__ = ()
+
+    codec_id: str = "combinators.framed"  # type: ignore
+
+    def __init__(self, *args: dict | Codec):
+        pass
+
+    def __new__(cls, *args: dict | Codec) -> Self:
+        return super(FramedCodecStack, cls).__new__(
+            cls,
+            tuple(
+                codec
+                if isinstance(codec, Codec)
+                else numcodecs.registry.get_codec(codec)
+                for codec in args
+            ),
+        )
+
+    def encode(self, buf: Buffer) -> bytes:
+        """Encode the data in `buf`.
+
+        Parameters
+        ----------
+        buf : Buffer
+            Data to be encoded. May be any object supporting the new-style
+            buffer protocol.
+
+        Returns
+        -------
+        enc : bytes
+            Encoded and framed data as a bytestring.
+        """
+
+        encoded = buf
+        encoded_ndarray = np.asarray(
+            numcodecs.compat.ensure_contiguous_ndarray_like(encoded, flatten=False)
+        )
+
+        frames = [(encoded_ndarray.dtype, encoded_ndarray.shape)]
+
+        for codec in self:
+            encoded = codec.encode(encoded_ndarray)
+            encoded_ndarray = np.asarray(
+                numcodecs.compat.ensure_contiguous_ndarray_like(encoded, flatten=False)
+            )
+            frames.append((encoded_ndarray.dtype, encoded_ndarray.shape))
+
+        # convert the encoded array to little endian bytes
+        encoded_byteorder = encoded_ndarray.dtype.byteorder
+        encoded_byteorder = (
+            encoded_byteorder
+            if encoded_byteorder in ("<", ">")
+            else ("<" if (byteorder == "little") else ">")
+        )
+        if encoded_byteorder != "<":
+            encoded_ndarray = encoded_ndarray.byteswap()
+        encoded_bytes = encoded_ndarray.tobytes()
+
+        message = [varint.encode(len(frames))]
+
+        for dtype, shape in frames:
+            message.append(varint.encode(len(dtype.str)))
+            message.append(dtype.str.encode("ascii"))
+
+            message.append(varint.encode(len(shape)))
+            for s in shape:
+                message.append(varint.encode(s))
+
+        message.append(encoded_bytes)
+
+        return b"".join(message)
+
+    def decode(self, buf: Buffer, out: Optional[Buffer] = None) -> Buffer:
+        """Decode the data in `buf`.
+
+        Parameters
+        ----------
+        buf : Buffer
+            Encoded data. Must be an object representing a bytestring, e.g.
+            [`bytes`][bytes] or a 1D array of [`np.uint8`][numpy.uint8]s etc.
+        out : Buffer, optional
+            Writeable buffer to store decoded data. N.B. if provided, this buffer must
+            be exactly the right size to store the decoded data.
+
+        Returns
+        -------
+        dec : Buffer
+            Decoded data. May be any object supporting the new-style
+            buffer protocol.
+        """
+
+        b = numcodecs.compat.ensure_bytes(buf)
+
+        b_io = BytesIO(b)
+
+        n_frames = varint.decode_stream(b_io)
+        assert n_frames == len(self) + 1, (
+            f"encoded data must contain {len(self) + 1} frames, found {n_frames}"
+        )
+
+        frames = []
+        for _ in range(n_frames):
+            dtype = np.dtype(b_io.read(varint.decode_stream(b_io)).decode("ascii"))
+            shape = tuple(
+                varint.decode_stream(b_io) for _ in range(varint.decode_stream(b_io))
+            )
+            frames.append((dtype, shape))
+
+        # read the decoded array from the little endian bytes
+        decoded = np.frombuffer(
+            b_io.read(np.prod(shape, dtype=int) * dtype.itemsize),
+            dtype=dtype.newbyteorder("<"),
+            count=np.prod(shape, dtype=int),
+        ).reshape(shape)
+        dtype_byteorder = dtype.byteorder
+        dtype_byteorder = (
+            dtype_byteorder
+            if dtype_byteorder in ("<", ">")
+            else ("<" if (byteorder == "little") else ">")
+        )
+        if dtype_byteorder != "<":
+            decoded = decoded.byteswap()
+
+        for codec, (dtype, shape) in zip(reversed(self), frames[:-1][::-1]):
+            decoded = (
+                codec.decode(
+                    decoded,
+                    out=np.empty(shape, dtype),
+                )
+                .view(dtype)
+                .reshape(shape)
+            )
+
+        return numcodecs.compat.ndarray_copy(decoded, out)  # type: ignore
+
+    def get_config(self) -> dict:
+        """
+        Returns the configuration of the framed codec stack.
+
+        [`numcodecs.registry.get_codec(config)`][numcodecs.registry.get_codec]
+        can be used to reconstruct this stack from the returned config.
+
+        Returns
+        -------
+        config : dict
+            Configuration of the framed codec stack.
+        """
+
+        return dict(
+            id=type(self).codec_id,
+            codecs=tuple(codec.get_config() for codec in self),
+        )
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Instantiate the framed codec stack from a configuration [`dict`][dict].
+
+        Parameters
+        ----------
+        config : dict
+            Configuration of the framed codec stack.
+
+        Returns
+        -------
+        stack : FramedCodecStack
+            Instantiated framed codec stack.
+        """
+
+        return cls(*config["codecs"])
+
+    def __repr__(self) -> str:
+        repr = ", ".join(f"{codec!r}" for codec in self)
+
+        return f"{type(self).__name__}({repr})"
+
+    def map(self, mapper: Callable[[Codec], Codec]) -> "FramedCodecStack":
+        """
+        Apply the `mapper` to all codecs that are in this framed stack.
+        In the returned stack, each codec is replaced by its mapped codec.
+
+        The `mapper` should recursively apply itself to any inner codecs that
+        also implement the [`CodecCombinatorMixin`][numcodecs_combinators.abc.CodecCombinatorMixin]
+        mixin.
+
+        To automatically handle the recursive application as a caller, you can
+        use
+        ```python
+        numcodecs_combinators.map_codec(stack, mapper)
+        ```
+        instead.
+
+        Parameters
+        ----------
+        mapper : Callable[[Codec], Codec]
+            The callable that should be applied to each codec to map over this
+            framed codec stack.
+
+        Returns
+        -------
+        mapped : FramedCodecStack
+            The mapped framed codec stack.
+        """
+
+        return FramedCodecStack(*map(mapper, self))
+
+    def __add__(self, other) -> "FramedCodecStack":
+        return FramedCodecStack(*tuple.__add__(self, other))
+
+    def __mul__(self, other) -> "FramedCodecStack":
+        return FramedCodecStack(*tuple.__mul__(self, other))
+
+    def __rmul__(self, other) -> "FramedCodecStack":
+        return FramedCodecStack(*tuple.__rmul__(self, other))
+
+
+numcodecs.registry.register_codec(FramedCodecStack)

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.8}/src/numcodecs_combinators/stack.py

@@ -4,13 +4,13 @@ This module defines the [`CodecStack`][numcodecs_combinators.stack.CodecStack] c
 
 __all__ = ["CodecStack"]
 
-from typing import Optional, Callable
-from typing_extensions import Buffer, Self  # MSPV 3.12
+from typing import Callable, Optional
 
 import numcodecs
 import numcodecs.compat
 import numcodecs.registry
 import numpy as np
+from typing_extensions import Buffer, Self  # MSPV 3.12
 
 try:
     import xarray as xr
@@ -140,13 +140,17 @@ class CodecStack(Codec, CodecCombinatorMixin, tuple[Codec]):
             buffer protocol.
         """
 
-        encoded = numcodecs.compat.ensure_contiguous_ndarray_like(buf, flatten=False)
+        encoded = np.asarray(
+            numcodecs.compat.ensure_contiguous_ndarray_like(buf, flatten=False)
+        )
         silhouettes = []
 
         for codec in self:
-            silhouettes.append((encoded.shape, np.dtype(encoded.dtype.name)))
-            encoded = numcodecs.compat.ensure_contiguous_ndarray_like(
-                codec.encode((encoded)), flatten=False
+            silhouettes.append((encoded.shape, encoded.dtype))
+            encoded = np.asarray(
+                numcodecs.compat.ensure_contiguous_ndarray_like(
+                    codec.encode((encoded)), flatten=False
+                )
             )
 
         decoded = encoded
@@ -154,7 +158,7 @@ class CodecStack(Codec, CodecCombinatorMixin, tuple[Codec]):
         for codec in reversed(self):
             shape, dtype = silhouettes.pop()
             out = np.empty(shape=shape, dtype=dtype)
-            decoded = codec.decode(decoded, out).reshape(shape)
+            decoded = codec.decode(decoded, out).view(dtype).reshape(shape)
 
         if isinstance(decoded, type(buf)):
             return decoded
@@ -163,7 +167,14 @@ class CodecStack(Codec, CodecCombinatorMixin, tuple[Codec]):
 
     def encode_decode_data_array(self, da: "xr.DataArray") -> "xr.DataArray":
         """
-        Encode, then decode the data array in `da`.
+        Encode, then decode each chunk (independently) in the data array `da`.
+
+        Since each chunk is encoded *independently*, this method may cause
+        chunk boundary artifacts. Do *not* use this method if the codec
+        requires access to the entire data at once or if it needs to access
+        a neighbourhood of points across the chunk boundary. In these cases,
+        it is preferable to use
+        `da.copy(data=stack.encode_decode(da.values))` instead.
 
         The encode-decode computation may be deferred until the
         [`compute`][xarray.DataArray.compute] method is called on the result.

numcodecs_combinators-0.2.8/tests/test_best.py

@@ -0,0 +1,85 @@
+import numcodecs
+import numcodecs.compat
+import numpy as np
+
+import numcodecs_combinators
+from numcodecs_combinators.best import PickBestCodec
+from numcodecs_combinators.framed import FramedCodecStack
+
+
+def assert_config_roundtrip(codec: numcodecs.abc.Codec):
+    config = codec.get_config()
+    codec2 = numcodecs.get_codec(config)
+    assert codec2 == codec
+
+
+def test_init_config():
+    best = PickBestCodec()
+    assert len(best) == 0
+    assert_config_roundtrip(best)
+
+    best = PickBestCodec(dict(id="zlib", level=9))
+    assert len(best) == 1
+    assert_config_roundtrip(best)
+
+    best = PickBestCodec(dict(id="zlib", level=9), numcodecs.CRC32())
+    assert len(best) == 2
+    assert_config_roundtrip(best)
+
+
+def test_encode_decode():
+    for best in [
+        PickBestCodec(),
+        PickBestCodec(dict(id="combinators.framed", codecs=[dict(id="zlib", level=9)])),
+        PickBestCodec(
+            FramedCodecStack(numcodecs.Zlib(level=9)),
+            FramedCodecStack(numcodecs.CRC32()),
+        ),
+        PickBestCodec(
+            FramedCodecStack(numcodecs.Zlib(level=9)),
+            FramedCodecStack(numcodecs.CRC32()),
+            FramedCodecStack(numcodecs.Zstd(level=20)),
+        ),
+    ]:
+        for data in [
+            np.zeros(shape=(0,)),
+            np.array(3),
+            np.array([97, 98, 99], dtype=np.uint8),
+            np.linspace(1, 100, 100).reshape(10, 10),
+            np.linspace(1, 100, 100).reshape(10, 10).byteswap(),
+        ]:
+            encoded = best.encode(data)
+            if len(best) > 0:
+                assert isinstance(encoded, bytes)
+            decoded = best.decode(encoded)
+            print(best)
+            assert np.all(decoded == data)
+
+
+def test_map():
+    best = PickBestCodec(numcodecs.Zlib(level=9), numcodecs.CRC32())
+
+    mapped = numcodecs_combinators.map_codec(best, lambda c: c)
+    assert mapped == best
+
+    mapped = numcodecs_combinators.map_codec(best, lambda c: PickBestCodec(c))
+    assert mapped == PickBestCodec(
+        PickBestCodec(
+            PickBestCodec(numcodecs.Zlib(level=9)),
+            PickBestCodec(numcodecs.CRC32()),
+        )
+    )
+
+    mapped = numcodecs_combinators.map_codec(mapped, lambda c: PickBestCodec(c))
+    assert mapped == PickBestCodec(
+        PickBestCodec(
+            PickBestCodec(
+                PickBestCodec(
+                    PickBestCodec(
+                        PickBestCodec(PickBestCodec(numcodecs.Zlib(level=9)))
+                    ),
+                    PickBestCodec(PickBestCodec(PickBestCodec(numcodecs.CRC32()))),
+                )
+            )
+        )
+    )

numcodecs_combinators-0.2.8/tests/test_framed.py

@@ -0,0 +1,75 @@
+import numcodecs
+import numcodecs.compat
+import numpy as np
+
+import numcodecs_combinators
+from numcodecs_combinators.framed import FramedCodecStack
+
+
+def assert_config_roundtrip(codec: numcodecs.abc.Codec):
+    config = codec.get_config()
+    codec2 = numcodecs.get_codec(config)
+    assert codec2 == codec
+
+
+def test_init_config():
+    stack = FramedCodecStack()
+    assert len(stack) == 0
+    assert_config_roundtrip(stack)
+
+    stack = FramedCodecStack(dict(id="zlib", level=9))
+    assert len(stack) == 1
+    assert_config_roundtrip(stack)
+
+    stack = FramedCodecStack(dict(id="zlib", level=9), numcodecs.CRC32())
+    assert len(stack) == 2
+    assert_config_roundtrip(stack)
+
+
+def test_encode_decode():
+    for stack in [
+        FramedCodecStack(),
+        FramedCodecStack(dict(id="zlib", level=9)),
+        FramedCodecStack(numcodecs.Zlib(level=9), numcodecs.CRC32()),
+    ]:
+        for data in [
+            b"abc",
+            np.array(3),
+            np.linspace(1, 100, 100).reshape(10, 10),
+            np.linspace(1, 100, 100).reshape(10, 10).byteswap(),
+        ]:
+            encoded = stack.encode(data)
+            assert isinstance(encoded, bytes)
+            decoded = stack.decode(encoded)
+            assert np.all(decoded == numcodecs.compat.ensure_ndarray_like(data))
+
+
+def test_map():
+    stack = FramedCodecStack(numcodecs.Zlib(level=9), numcodecs.CRC32())
+
+    mapped = numcodecs_combinators.map_codec(stack, lambda c: c)
+    assert mapped == stack
+
+    mapped = numcodecs_combinators.map_codec(stack, lambda c: FramedCodecStack(c))
+    assert mapped == FramedCodecStack(
+        FramedCodecStack(
+            FramedCodecStack(numcodecs.Zlib(level=9)),
+            FramedCodecStack(numcodecs.CRC32()),
+        )
+    )
+
+    mapped = numcodecs_combinators.map_codec(mapped, lambda c: FramedCodecStack(c))
+    assert mapped == FramedCodecStack(
+        FramedCodecStack(
+            FramedCodecStack(
+                FramedCodecStack(
+                    FramedCodecStack(
+                        FramedCodecStack(FramedCodecStack(numcodecs.Zlib(level=9)))
+                    ),
+                    FramedCodecStack(
+                        FramedCodecStack(FramedCodecStack(numcodecs.CRC32()))
+                    ),
+                )
+            )
+        )
+    )