numcodecs-combinators 0.2.6__tar.gz → 0.2.7__tar.gz

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: numcodecs-combinators
-Version: 0.2.6
+Version: 0.2.7
 Summary: Combinator codecs for the `numcodecs` buffer compression API
 License: Copyright (c) 2024, Juniper Tyree
         
@@ -382,6 +382,7 @@ Requires-Python: >=3.10
 Requires-Dist: numcodecs<0.17,>=0.13.0
 Requires-Dist: numpy~=2.0
 Requires-Dist: typing-extensions~=4.6
+Requires-Dist: varint~=1.0
 Provides-Extra: xarray
 Requires-Dist: dask>=2024.6; extra == 'xarray'
 Requires-Dist: xarray>=2024.06; extra == 'xarray'
@@ -400,6 +401,7 @@ Combinator codecs for the [`numcodecs`] buffer compression API.
 The following combinators, implementing the `CodecCombinatorMixin` are provided:
 
 - `CodecStack`: a stack of codecs
+- `FramedCodecStack`: a stack of codecs that is framed with array data type and shape information
 
 [`numcodecs`]: https://numcodecs.readthedocs.io/en/stable/
 

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.7}/README.md

@@ -11,6 +11,7 @@ Combinator codecs for the [`numcodecs`] buffer compression API.
 The following combinators, implementing the `CodecCombinatorMixin` are provided:
 
 - `CodecStack`: a stack of codecs
+- `FramedCodecStack`: a stack of codecs that is framed with array data type and shape information
 
 [`numcodecs`]: https://numcodecs.readthedocs.io/en/stable/
 

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.7}/docs/index.md

@@ -11,6 +11,7 @@ Combinator codecs for the [`numcodecs`][numcodecs] buffer compression API.
 The following combinators, implementing the [`CodecCombinatorMixin`][numcodecs_combinators.abc.CodecCombinatorMixin] are provided:
 
 - [`CodecStack`][numcodecs_combinators.stack.CodecStack]: a stack of codecs
+- [`FramedCodecStack`][numcodecs_combinators.framed.FramedCodecStack]: a stack of codecs that is framed with array data type and shape information
 
 ## Funding
 

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.7}/mkdocs.yml

@@ -54,5 +54,9 @@ plugins:
                   modules: true
           inventories:
             - https://docs.python.org/3.12/objects.inv
+            - https://numpy.org/doc/2.2/objects.inv
             - https://numcodecs.readthedocs.io/en/v0.15.0/objects.inv
             - https://docs.xarray.dev/en/v2025.01.0/objects.inv
+
+watch:
+  - src

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "numcodecs-combinators"
-version = "0.2.6"
+version = "0.2.7"
 description = "Combinator codecs for the `numcodecs` buffer compression API"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -13,6 +13,7 @@ dependencies = [
     "numcodecs>=0.13.0,<0.17",
     "numpy~=2.0",
     "typing-extensions~=4.6",
+    "varint~=1.0",
 ]
 optional-dependencies.xarray = [ "xarray>=2024.06", "dask>=2024.6" ]
 
@@ -21,6 +22,7 @@ dev = ["mypy~=1.14", "pytest~=8.3"]
 
 [project.entry-points."numcodecs.codecs"]
 "combinators.stack" = "numcodecs_combinators.stack:CodecStack"
+"combinators.framed" = "numcodecs_combinators.framed:FramedCodecStack"
 
 [tool.setuptools.packages.find]
 where = ["src"]
@@ -28,6 +30,13 @@ where = ["src"]
 [tool.setuptools.package-data]
 "numcodecs_combinators" = ["py.typed"]
 
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "I"]
+
+[tool.pytest.ini_options]
+addopts = ["--import-mode=importlib"]
+xfail_strict = true
+
 [[tool.mypy.overrides]]
-module = ["numcodecs.*"]
+module = ["numcodecs.*", "varint.*"]
 follow_untyped_imports = true

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.7}/src/numcodecs_combinators/__init__.py

@@ -1,9 +1,13 @@
 """
 Combinator codecs for the [`numcodecs`][numcodecs] buffer compression API.
 
-The following combinators, implementing the [`CodecCombinatorMixin`][numcodecs_combinators.abc.CodecCombinatorMixin] are provided:
+The following combinators, implementing the
+[`CodecCombinatorMixin`][numcodecs_combinators.abc.CodecCombinatorMixin] are
+provided:
 
 - [`CodecStack`][numcodecs_combinators.stack.CodecStack]: a stack of codecs
+- [`FramedCodecStack`][numcodecs_combinators.framed.FramedCodecStack]: a stack
+  of codecs that is framed with array data type and shape information
 """
 
 __all__ = ["map_codec"]
@@ -14,6 +18,7 @@ from typing import Callable
 from numcodecs.abc import Codec
 
 from . import abc as abc
+from . import framed as framed
 from . import stack as stack
 
 

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.7}/src/numcodecs_combinators/abc.py

@@ -4,12 +4,11 @@ This module defines the [`CodecCombinatorMixin`][numcodecs_combinators.abc.Codec
 
 __all__ = ["CodecCombinatorMixin"]
 
+from abc import ABC, abstractmethod
 from typing import Callable
 
 from numcodecs.abc import Codec
 
-from abc import ABC, abstractmethod
-
 
 class CodecCombinatorMixin(ABC):
     """

numcodecs_combinators-0.2.7/src/numcodecs_combinators/framed.py

@@ -0,0 +1,260 @@
+"""
+This module defines the [`FramedCodecStack`][numcodecs_combinators.framed.FramedCodecStack] class, which exposes a framed stack of codecs as a combined codec.
+"""
+
+__all__ = ["FramedCodecStack"]
+
+from io import BytesIO
+from sys import byteorder
+from typing import Callable, Optional
+
+import numcodecs
+import numcodecs.compat
+import numcodecs.registry
+import numpy as np
+import varint
+from numcodecs.abc import Codec
+from typing_extensions import Buffer, Self  # MSPV 3.12
+
+from .abc import CodecCombinatorMixin
+
+
+class FramedCodecStack(Codec, CodecCombinatorMixin, tuple[Codec]):
+    """
+    A framed stack of codecs, which makes up a combined codec.
+
+    On encoding, the result of applying the codecs from left to right to encode
+    is framed s.t. the data types and shapes of all arrays (input,
+    intermediary, encoded) are stored as part of the encoding, which is output
+    as a bytestring.
+
+    On deconding, this framing information is used to apply the codecs from
+    right to left to decode into known ouput data types and shapes.
+
+    Therefore, the [`FramedCodecStack`][numcodecs_combinators.framed.FramedCodecStack]
+    can be used to combine codecs which require knowing the output data type
+    and shape during decoding. It can also be used to encode arrays into
+    bytestrings.
+
+    Unlike the [`CodecStack`][numcodecs_combinators.stack.CodecStack], this
+    class does *not* provide an additional `encode_decode(buf)` method, since
+    it is equivalent to `framed.decode(stack.encode(buf))` due to the framing.
+    """
+
+    __slots__ = ()
+
+    codec_id: str = "combinators.framed"  # type: ignore
+
+    def __init__(self, *args: dict | Codec):
+        pass
+
+    def __new__(cls, *args: dict | Codec) -> Self:
+        return super(FramedCodecStack, cls).__new__(
+            cls,
+            tuple(
+                codec
+                if isinstance(codec, Codec)
+                else numcodecs.registry.get_codec(codec)
+                for codec in args
+            ),
+        )
+
+    def encode(self, buf: Buffer) -> bytes:
+        """Encode the data in `buf`.
+
+        Parameters
+        ----------
+        buf : Buffer
+            Data to be encoded. May be any object supporting the new-style
+            buffer protocol.
+
+        Returns
+        -------
+        enc : bytes
+            Encoded and framed data as a bytestring.
+        """
+
+        encoded = buf
+        encoded_ndarray = np.asarray(
+            numcodecs.compat.ensure_contiguous_ndarray_like(encoded, flatten=False)
+        )
+
+        frames = [(encoded_ndarray.dtype, encoded_ndarray.shape)]
+
+        for codec in self:
+            encoded = codec.encode(encoded_ndarray)
+            encoded_ndarray = np.asarray(
+                numcodecs.compat.ensure_contiguous_ndarray_like(encoded, flatten=False)
+            )
+            frames.append((encoded_ndarray.dtype, encoded_ndarray.shape))
+
+        # convert the encoded array to little endian bytes
+        encoded_byteorder = encoded_ndarray.dtype.byteorder
+        encoded_byteorder = (
+            encoded_byteorder
+            if encoded_byteorder in ("<", ">")
+            else ("<" if (byteorder == "little") else ">")
+        )
+        if encoded_byteorder != "<":
+            encoded_ndarray = encoded_ndarray.byteswap()
+        encoded_bytes = encoded_ndarray.tobytes()
+
+        message = [varint.encode(len(frames))]
+
+        for dtype, shape in frames:
+            message.append(varint.encode(len(dtype.str)))
+            message.append(dtype.str.encode("ascii"))
+
+            message.append(varint.encode(len(shape)))
+            for s in shape:
+                message.append(varint.encode(s))
+
+        message.append(encoded_bytes)
+
+        return b"".join(message)
+
+    def decode(self, buf: Buffer, out: Optional[Buffer] = None) -> Buffer:
+        """Decode the data in `buf`.
+
+        Parameters
+        ----------
+        buf : Buffer
+            Encoded data. Must be an object representing a bytestring, e.g.
+            [`bytes`][bytes] or a 1D array of [`np.uint8`][numpy.uint8]s etc.
+        out : Buffer, optional
+            Writeable buffer to store decoded data. N.B. if provided, this buffer must
+            be exactly the right size to store the decoded data.
+
+        Returns
+        -------
+        dec : Buffer
+            Decoded data. May be any object supporting the new-style
+            buffer protocol.
+        """
+
+        b = numcodecs.compat.ensure_bytes(buf)
+
+        b_io = BytesIO(b)
+
+        n_frames = varint.decode_stream(b_io)
+        assert n_frames == len(self) + 1, (
+            f"encoded data must contain {len(self) + 1} frames, found {n_frames}"
+        )
+
+        frames = []
+        for _ in range(n_frames):
+            dtype = np.dtype(b_io.read(varint.decode_stream(b_io)).decode("ascii"))
+            shape = tuple(
+                varint.decode_stream(b_io) for _ in range(varint.decode_stream(b_io))
+            )
+            frames.append((dtype, shape))
+
+        # read the decoded array from the little endian bytes
+        decoded = np.frombuffer(
+            b_io.read(np.prod(shape, dtype=int) * dtype.itemsize),
+            dtype=dtype.newbyteorder("<"),
+            count=np.prod(shape, dtype=int),
+        ).reshape(shape)
+        dtype_byteorder = dtype.byteorder
+        dtype_byteorder = (
+            dtype_byteorder
+            if dtype_byteorder in ("<", ">")
+            else ("<" if (byteorder == "little") else ">")
+        )
+        if dtype_byteorder != "<":
+            decoded = decoded.byteswap()
+
+        for codec, (dtype, shape) in zip(reversed(self), frames[:-1][::-1]):
+            decoded = (
+                codec.decode(
+                    decoded,
+                    out=np.empty(shape, dtype),
+                )
+                .view(dtype)
+                .reshape(shape)
+            )
+
+        return numcodecs.compat.ndarray_copy(decoded, out)  # type: ignore
+
+    def get_config(self) -> dict:
+        """
+        Returns the configuration of the framed codec stack.
+
+        [`numcodecs.registry.get_codec(config)`][numcodecs.registry.get_codec]
+        can be used to reconstruct this stack from the returned config.
+
+        Returns
+        -------
+        config : dict
+            Configuration of the framed codec stack.
+        """
+
+        return dict(
+            id=type(self).codec_id,
+            codecs=tuple(codec.get_config() for codec in self),
+        )
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Instantiate the framed codec stack from a configuration [`dict`][dict].
+
+        Parameters
+        ----------
+        config : dict
+            Configuration of the framed codec stack.
+
+        Returns
+        -------
+        stack : FramedCodecStack
+            Instantiated framed codec stack.
+        """
+
+        return cls(*config["codecs"])
+
+    def __repr__(self) -> str:
+        repr = ", ".join(f"{codec!r}" for codec in self)
+
+        return f"{type(self).__name__}({repr})"
+
+    def map(self, mapper: Callable[[Codec], Codec]) -> "FramedCodecStack":
+        """
+        Apply the `mapper` to all codecs that are in this framed stack.
+        In the returned stack, each codec is replaced by its mapped codec.
+
+        The `mapper` should recursively apply itself to any inner codecs that
+        also implement the [`CodecCombinatorMixin`][numcodecs_combinators.abc.CodecCombinatorMixin]
+        mixin.
+
+        To automatically handle the recursive application as a caller, you can
+        use
+        ```python
+        numcodecs_combinators.map_codec(stack, mapper)
+        ```
+        instead.
+
+        Parameters
+        ----------
+        mapper : Callable[[Codec], Codec]
+            The callable that should be applied to each codec to map over this
+            framed codec stack.
+
+        Returns
+        -------
+        mapped : FramedCodecStack
+            The mapped framed codec stack.
+        """
+
+        return FramedCodecStack(*map(mapper, self))
+
+    def __add__(self, other) -> "FramedCodecStack":
+        return FramedCodecStack(*tuple.__add__(self, other))
+
+    def __mul__(self, other) -> "FramedCodecStack":
+        return FramedCodecStack(*tuple.__mul__(self, other))
+
+    def __rmul__(self, other) -> "FramedCodecStack":
+        return FramedCodecStack(*tuple.__rmul__(self, other))
+
+
+numcodecs.registry.register_codec(FramedCodecStack)

{numcodecs_combinators-0.2.6 → numcodecs_combinators-0.2.7}/src/numcodecs_combinators/stack.py

@@ -4,13 +4,13 @@ This module defines the [`CodecStack`][numcodecs_combinators.stack.CodecStack] c
 
 __all__ = ["CodecStack"]
 
-from typing import Optional, Callable
-from typing_extensions import Buffer, Self  # MSPV 3.12
+from typing import Callable, Optional
 
 import numcodecs
 import numcodecs.compat
 import numcodecs.registry
 import numpy as np
+from typing_extensions import Buffer, Self  # MSPV 3.12
 
 try:
     import xarray as xr
@@ -140,13 +140,17 @@ class CodecStack(Codec, CodecCombinatorMixin, tuple[Codec]):
             buffer protocol.
         """
 
-        encoded = numcodecs.compat.ensure_contiguous_ndarray_like(buf, flatten=False)
+        encoded = np.asarray(
+            numcodecs.compat.ensure_contiguous_ndarray_like(buf, flatten=False)
+        )
         silhouettes = []
 
         for codec in self:
-            silhouettes.append((encoded.shape, np.dtype(encoded.dtype.name)))
-            encoded = numcodecs.compat.ensure_contiguous_ndarray_like(
-                codec.encode((encoded)), flatten=False
+            silhouettes.append((encoded.shape, encoded.dtype))
+            encoded = np.asarray(
+                numcodecs.compat.ensure_contiguous_ndarray_like(
+                    codec.encode((encoded)), flatten=False
+                )
             )
 
         decoded = encoded
@@ -154,7 +158,7 @@ class CodecStack(Codec, CodecCombinatorMixin, tuple[Codec]):
         for codec in reversed(self):
             shape, dtype = silhouettes.pop()
             out = np.empty(shape=shape, dtype=dtype)
-            decoded = codec.decode(decoded, out).reshape(shape)
+            decoded = codec.decode(decoded, out).view(dtype).reshape(shape)
 
         if isinstance(decoded, type(buf)):
             return decoded

numcodecs_combinators-0.2.7/tests/test_framed.py

@@ -0,0 +1,75 @@
+import numcodecs
+import numcodecs.compat
+import numpy as np
+
+import numcodecs_combinators
+from numcodecs_combinators.framed import FramedCodecStack
+
+
+def assert_config_roundtrip(codec: numcodecs.abc.Codec):
+    config = codec.get_config()
+    codec2 = numcodecs.get_codec(config)
+    assert codec2 == codec
+
+
+def test_init_config():
+    stack = FramedCodecStack()
+    assert len(stack) == 0
+    assert_config_roundtrip(stack)
+
+    stack = FramedCodecStack(dict(id="zlib", level=9))
+    assert len(stack) == 1
+    assert_config_roundtrip(stack)
+
+    stack = FramedCodecStack(dict(id="zlib", level=9), numcodecs.CRC32())
+    assert len(stack) == 2
+    assert_config_roundtrip(stack)
+
+
+def test_encode_decode():
+    for stack in [
+        FramedCodecStack(),
+        FramedCodecStack(dict(id="zlib", level=9)),
+        FramedCodecStack(numcodecs.Zlib(level=9), numcodecs.CRC32()),
+    ]:
+        for data in [
+            b"abc",
+            np.array(3),
+            np.linspace(1, 100, 100).reshape(10, 10),
+            np.linspace(1, 100, 100).reshape(10, 10).byteswap(),
+        ]:
+            encoded = stack.encode(data)
+            assert isinstance(encoded, bytes)
+            decoded = stack.decode(encoded)
+            assert np.all(decoded == numcodecs.compat.ensure_ndarray_like(data))
+
+
+def test_map():
+    stack = FramedCodecStack(numcodecs.Zlib(level=9), numcodecs.CRC32())
+
+    mapped = numcodecs_combinators.map_codec(stack, lambda c: c)
+    assert mapped == stack
+
+    mapped = numcodecs_combinators.map_codec(stack, lambda c: FramedCodecStack(c))
+    assert mapped == FramedCodecStack(
+        FramedCodecStack(
+            FramedCodecStack(numcodecs.Zlib(level=9)),
+            FramedCodecStack(numcodecs.CRC32()),
+        )
+    )
+
+    mapped = numcodecs_combinators.map_codec(mapped, lambda c: FramedCodecStack(c))
+    assert mapped == FramedCodecStack(
+        FramedCodecStack(
+            FramedCodecStack(
+                FramedCodecStack(
+                    FramedCodecStack(
+                        FramedCodecStack(FramedCodecStack(numcodecs.Zlib(level=9)))
+                    ),
+                    FramedCodecStack(
+                        FramedCodecStack(FramedCodecStack(numcodecs.CRC32()))
+                    ),
+                )
+            )
+        )
+    )