numcodecs 0.16.4__cp313-cp313-macosx_11_0_arm64.whl

numcodecs/tests/test_zfpy.py

@@ -0,0 +1,104 @@
+from types import ModuleType
+from typing import cast
+
+import numpy as np
+import pytest
+
+try:
+    # noinspection PyProtectedMember
+    from numcodecs.zfpy import ZFPY, _zfpy
+except ImportError:  # pragma: no cover
+    pytest.skip("ZFPY not available", allow_module_level=True)
+
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+
+_zfpy = cast(ModuleType, _zfpy)
+
+
+codecs = [
+    ZFPY(mode=_zfpy.mode_fixed_rate, rate=-1),
+    ZFPY(),
+    ZFPY(mode=_zfpy.mode_fixed_accuracy, tolerance=-1),
+    ZFPY(mode=_zfpy.mode_fixed_precision, precision=-1),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.linspace(1000, 1001, 1000, dtype="f4"),
+    np.linspace(1000, 1001, 1000, dtype="f8"),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.normal(loc=1000, scale=1, size=(10, 10, 10)),
+    np.random.normal(loc=1000, scale=1, size=(2, 5, 10, 10)),
+    np.random.randint(-(2**31), -(2**31) + 20, size=1000, dtype="i4").reshape(100, 10),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype="i8").reshape(10, 10, 10),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        if arr.dtype in (np.int32, np.int64):
+            codec = [codecs[-1]]
+        else:
+            codec = codecs
+        for code in codec:
+            check_encode_decode_array(arr, code)
+
+
+def test_config():
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("ZFPY(mode=4, tolerance=0.001, rate=-1, precision=-1)")
+
+
+def test_backwards_compatibility():
+    for code in codecs:
+        if code.mode == _zfpy.mode_fixed_rate:
+            codec = [code]
+            check_backwards_compatibility(ZFPY.codec_id, arrays, codec)
+        else:
+            check_backwards_compatibility(ZFPY.codec_id, arrays[: len(arrays) - 2], codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(ZFPY())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(ZFPY())
+
+
+def test_err_encode_list():
+    data = ['foo', 'bar', 'baz']
+    for codec in codecs:
+        with pytest.raises(TypeError):
+            codec.encode(data)
+
+
+def test_err_encode_non_contiguous():
+    # non-contiguous memory
+    arr = np.arange(1000, dtype='i4')[::2]
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)
+
+
+def test_err_encode_fortran_array():
+    # fortran array
+    arr = np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(5, 10, 20)))
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)

numcodecs/tests/test_zlib.py

@@ -0,0 +1,94 @@
+import itertools
+
+import numpy as np
+import pytest
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+from numcodecs.zlib import Zlib
+
+codecs = [
+    Zlib(),
+    Zlib(level=-1),
+    Zlib(level=0),
+    Zlib(level=1),
+    Zlib(level=5),
+    Zlib(level=9),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype='i4'),
+    np.linspace(1000, 1001, 1000, dtype='f8'),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode(arr, codec)
+
+
+def test_config():
+    codec = Zlib(level=3)
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("Zlib(level=3)")
+
+
+def test_eq():
+    assert Zlib() == Zlib()
+    assert not Zlib() != Zlib()
+    assert Zlib(1) == Zlib(1)
+    assert Zlib(1) != Zlib(9)
+    assert Zlib() != 'foo'
+    assert 'foo' != Zlib()
+    assert not Zlib() == 'foo'
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(Zlib.codec_id, arrays, codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(Zlib())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(Zlib())
+
+
+def test_err_encode_list():
+    data = ['foo', 'bar', 'baz']
+    for codec in codecs:
+        with pytest.raises(TypeError):
+            codec.encode(data)
+
+
+def test_err_encode_non_contiguous():
+    # non-contiguous memory
+    arr = np.arange(1000, dtype='i4')[::2]
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)

numcodecs/tests/test_zstd.py

@@ -0,0 +1,189 @@
+import itertools
+import subprocess
+
+import numpy as np
+import pytest
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+from numcodecs.zstd import Zstd
+
+codecs = [
+    Zstd(),
+    Zstd(level=-1),
+    Zstd(level=0),
+    Zstd(level=1),
+    Zstd(level=10),
+    Zstd(level=22),
+    Zstd(level=100),
+    Zstd(checksum=True),
+    Zstd(level=0, checksum=True),
+    Zstd(level=22, checksum=True),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype="i4"),
+    np.linspace(1000, 1001, 1000, dtype="f8"),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode(arr, codec)
+
+
+def test_config():
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("Zstd(level=3)")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(Zstd.codec_id, arrays, codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(Zstd())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(Zstd())
+
+
+def test_checksum():
+    data = np.arange(0, 64, dtype="uint8")
+    assert len(Zstd(level=0, checksum=False).encode(data)) + 4 == len(
+        Zstd(level=0, checksum=True).encode(data)
+    )
+
+
+def test_native_functions():
+    # Note, these assertions might need to be changed for new versions of zstd
+    assert Zstd.default_level() == 3
+    assert Zstd.min_level() == -131072
+    assert Zstd.max_level() == 22
+
+
+def test_streaming_decompression():
+    # Test input frames with unknown frame content size
+    codec = Zstd()
+
+    # If the zstd command line interface is available, check the bytes
+    cli = zstd_cli_available()
+    if cli:
+        view_zstd_streaming_bytes()
+
+    # Encode bytes directly that were the result of streaming compression
+    bytes_val = b'(\xb5/\xfd\x00Xa\x00\x00Hello World!'
+    dec = codec.decode(bytes_val)
+    dec_expected = b'Hello World!'
+    assert dec == dec_expected
+    if cli:
+        assert bytes_val == generate_zstd_streaming_bytes(dec_expected)
+        assert dec_expected == generate_zstd_streaming_bytes(bytes_val, decompress=True)
+
+    # Two consecutive frames given as input
+    bytes2 = bytes(bytearray(bytes_val * 2))
+    dec2 = codec.decode(bytes2)
+    dec2_expected = b'Hello World!Hello World!'
+    assert dec2 == dec2_expected
+    if cli:
+        assert dec2_expected == generate_zstd_streaming_bytes(bytes2, decompress=True)
+
+    # Single long frame that decompresses to a large output
+    bytes3 = b'(\xb5/\xfd\x00X$\x02\x00\xa4\x03ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz\x01\x00:\xfc\xdfs\x05\x05L\x00\x00\x08s\x01\x00\xfc\xff9\x10\x02L\x00\x00\x08k\x01\x00\xfc\xff9\x10\x02L\x00\x00\x08c\x01\x00\xfc\xff9\x10\x02L\x00\x00\x08[\x01\x00\xfc\xff9\x10\x02L\x00\x00\x08S\x01\x00\xfc\xff9\x10\x02L\x00\x00\x08K\x01\x00\xfc\xff9\x10\x02L\x00\x00\x08C\x01\x00\xfc\xff9\x10\x02L\x00\x00\x08u\x01\x00\xfc\xff9\x10\x02L\x00\x00\x08m\x01\x00\xfc\xff9\x10\x02L\x00\x00\x08e\x01\x00\xfc\xff9\x10\x02L\x00\x00\x08]\x01\x00\xfc\xff9\x10\x02L\x00\x00\x08U\x01\x00\xfc\xff9\x10\x02L\x00\x00\x08M\x01\x00\xfc\xff9\x10\x02M\x00\x00\x08E\x01\x00\xfc\x7f\x1d\x08\x01'
+    dec3 = codec.decode(bytes3)
+    dec3_expected = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz' * 1024 * 32
+    assert dec3 == dec3_expected
+    if cli:
+        assert bytes3 == generate_zstd_streaming_bytes(dec3_expected)
+        assert dec3_expected == generate_zstd_streaming_bytes(bytes3, decompress=True)
+
+    # Garbage input results in an error
+    bytes4 = bytes(bytearray([0, 0, 0, 0, 0, 0, 0, 0]))
+    with pytest.raises(RuntimeError, match='Zstd decompression error: invalid input data'):
+        codec.decode(bytes4)
+
+
+def generate_zstd_streaming_bytes(input: bytes, *, decompress: bool = False) -> bytes:
+    """
+    Use the zstd command line interface to compress or decompress bytes in streaming mode.
+    """
+    if decompress:
+        args = ["-d"]
+    else:
+        args = []
+
+    p = subprocess.run(["zstd", "--no-check", *args], input=input, capture_output=True)
+    return p.stdout
+
+
+def view_zstd_streaming_bytes():
+    bytes_val = generate_zstd_streaming_bytes(b"Hello world!")
+    print(f"    bytes_val = {bytes_val}")
+
+    bytes3 = generate_zstd_streaming_bytes(
+        b"ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz" * 1024 * 32
+    )
+    print(f"    bytes3 = {bytes3}")
+
+
+def zstd_cli_available() -> bool:
+    return not subprocess.run(
+        ["zstd", "-V"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+    ).returncode
+
+
+def test_multi_frame():
+    codec = Zstd()
+
+    hello_world = codec.encode(b"Hello world!")
+    assert codec.decode(hello_world) == b"Hello world!"
+    assert codec.decode(hello_world * 2) == b"Hello world!Hello world!"
+
+    hola = codec.encode(b"Hola ")
+    mundo = codec.encode(b"Mundo!")
+    assert codec.decode(hola) == b"Hola "
+    assert codec.decode(mundo) == b"Mundo!"
+    assert codec.decode(hola + mundo) == b"Hola Mundo!"
+
+    bytes_val = b'(\xb5/\xfd\x00Xa\x00\x00Hello World!'
+    dec = codec.decode(bytes_val)
+    dec_expected = b'Hello World!'
+    assert dec == dec_expected
+    cli = zstd_cli_available()
+    if cli:
+        assert bytes_val == generate_zstd_streaming_bytes(dec_expected)
+        assert dec_expected == generate_zstd_streaming_bytes(bytes_val, decompress=True)
+
+    # Concatenate frames of known sizes and unknown sizes
+    # unknown size frame at the end
+    assert codec.decode(hola + mundo + bytes_val) == b"Hola Mundo!Hello World!"
+    # unknown size frame at the beginning
+    assert codec.decode(bytes_val + hola + mundo) == b"Hello World!Hola Mundo!"
+    # unknown size frame in the middle
+    assert codec.decode(hola + bytes_val + mundo) == b"Hola Hello World!Mundo!"

numcodecs/version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.16.4'
+__version_tuple__ = version_tuple = (0, 16, 4)
+
+__commit_id__ = commit_id = 'g91dfe4296'

numcodecs/zarr3.py

@@ -0,0 +1,67 @@
+"""
+This module is DEPRECATED. It will may be removed entirely in a future release of Numcodecs.
+The codecs exported here are available in Zarr Python >= 3.1.3
+"""
+
+from __future__ import annotations
+
+import importlib
+import warnings
+from importlib.metadata import version
+from typing import Any
+
+from packaging.version import Version
+
+
+def __getattr__(name: str) -> Any:
+    """
+    Emit a warning when someone imports from this module
+    """
+    if name in __all__:
+        msg = (
+            "The numcodecs.zarr3 module is deprecated and will be removed in a future release of numcodecs. "
+            f"Import {name} via zarr.codecs.numcodecs.{name} instead. This requires Zarr Python >= 3.1.3. "
+        )
+
+        warnings.warn(msg, DeprecationWarning, stacklevel=2)
+        module = importlib.import_module("zarr.codecs.numcodecs")
+        obj = getattr(module, name)
+        globals()[name] = obj  # cache so subsequent lookups skip __getattr__
+        return obj
+    raise AttributeError(f"module {__name__} has no attribute {name}")
+
+
+try:
+    import zarr  # noqa: F401
+
+    zarr_version = version('zarr')
+    if Version(zarr_version) < Version("3.1.3"):  # pragma: no cover
+        msg = f"Zarr 3.1.3 or later is required to use the numcodecs zarr integration. Got {zarr_version}."
+        raise ImportError(msg)
+except ImportError as e:  # pragma: no cover
+    msg = "zarr could not be imported. Zarr 3.1.3 or later is required to use the numcodecs zarr integration."
+    raise ImportError(msg) from e
+
+__all__ = [
+    "BZ2",  # noqa: F822
+    "CRC32",  # noqa: F822
+    "CRC32C",  # noqa: F822
+    "LZ4",  # noqa: F822
+    "LZMA",  # noqa: F822
+    "ZFPY",  # noqa: F822
+    "Adler32",  # noqa: F822
+    "AsType",  # noqa: F822
+    "BitRound",  # noqa: F822
+    "Blosc",  # noqa: F822
+    "Delta",  # noqa: F822
+    "FixedScaleOffset",  # noqa: F822
+    "Fletcher32",  # noqa: F822
+    "GZip",  # noqa: F822
+    "JenkinsLookup3",  # noqa: F822
+    "PCodec",  # noqa: F822
+    "PackBits",  # noqa: F822
+    "Quantize",  # noqa: F822
+    "Shuffle",  # noqa: F822
+    "Zlib",  # noqa: F822
+    "Zstd",  # noqa: F822
+]

numcodecs/zfpy.py

@@ -0,0 +1,112 @@
+import warnings
+from contextlib import suppress
+from importlib.metadata import PackageNotFoundError, version
+from types import ModuleType
+
+_zfpy: ModuleType | None = None
+
+_zfpy_version: tuple = ()
+with suppress(PackageNotFoundError):
+    _zfpy_version = tuple(map(int, version("zfpy").split(".")))
+
+if _zfpy_version:
+    # Check NumPy version
+    _numpy_version: tuple = tuple(map(int, version("numpy").split('.')))
+    if _numpy_version >= (2, 0, 0) and _zfpy_version < (1, 0, 1):  # pragma: no cover
+        _zfpy_version = ()
+        warnings.warn(
+            "NumPy version >= 2.0.0 detected. The zfpy library is incompatible with this version of NumPy. "
+            "Please downgrade to NumPy < 2.0.0 or wait for an update from zfpy.",
+            UserWarning,
+            stacklevel=2,
+        )
+    else:
+        with suppress(ImportError):
+            import zfpy as _zfpy  # type: ignore[no-redef]
+
+if _zfpy:
+    import numpy as np
+
+    from .abc import Codec
+    from .compat import ensure_bytes, ensure_contiguous_ndarray, ndarray_copy
+
+    # noinspection PyShadowingBuiltins
+    class ZFPY(Codec):
+        """Codec providing compression using zfpy via the Python standard
+        library.
+
+        Parameters
+        ----------
+        mode : integer
+            One of the zfpy mode choice, e.g., ``zfpy.mode_fixed_accuracy``.
+        tolerance : double, optional
+            A double-precision number, specifying the compression accuracy needed.
+        rate : double, optional
+            A double-precision number, specifying the compression rate needed.
+        precision : int, optional
+            A integer number, specifying the compression precision needed.
+
+        """
+
+        codec_id = "zfpy"
+
+        def __init__(
+            self,
+            mode=_zfpy.mode_fixed_accuracy,
+            tolerance=-1,
+            rate=-1,
+            precision=-1,
+            compression_kwargs=None,
+        ):
+            self.mode = mode
+            if mode == _zfpy.mode_fixed_accuracy:
+                self.compression_kwargs = {"tolerance": tolerance}
+            elif mode == _zfpy.mode_fixed_rate:
+                self.compression_kwargs = {"rate": rate}
+            elif mode == _zfpy.mode_fixed_precision:
+                self.compression_kwargs = {"precision": precision}
+
+            self.tolerance = tolerance
+            self.rate = rate
+            self.precision = precision
+
+        def encode(self, buf):
+            # not flatten c-order array and raise exception for f-order array
+            if not isinstance(buf, np.ndarray):
+                raise TypeError(
+                    "The zfp codec does not support none numpy arrays."
+                    f" Your buffers were {type(buf)}."
+                )
+            if buf.flags.c_contiguous:
+                flatten = False
+            else:
+                raise ValueError(
+                    "The zfp codec does not support F order arrays. "
+                    f"Your arrays flags were {buf.flags}."
+                )
+            buf = ensure_contiguous_ndarray(buf, flatten=flatten)
+
+            # do compression
+            return _zfpy.compress_numpy(buf, write_header=True, **self.compression_kwargs)
+
+        def decode(self, buf, out=None):
+            # normalise inputs
+            buf = ensure_bytes(buf)
+            if out is not None:
+                out = ensure_contiguous_ndarray(out)
+
+            # do decompression
+            dec = _zfpy.decompress_numpy(buf)
+
+            # handle destination
+            if out is not None:
+                return ndarray_copy(dec, out)
+            else:
+                return dec
+
+        def __repr__(self):
+            return (
+                f"{type(self).__name__}(mode={self.mode!r}, "
+                f"tolerance={self.tolerance}, rate={self.rate}, "
+                f"precision={self.precision})"
+            )

numcodecs/zlib.py

@@ -0,0 +1,42 @@
+import zlib as _zlib
+
+from .abc import Codec
+from .compat import ensure_contiguous_ndarray, ndarray_copy
+
+
+class Zlib(Codec):
+    """Codec providing compression using zlib via the Python standard library.
+
+    Parameters
+    ----------
+    level : int
+        Compression level.
+
+    """
+
+    codec_id = 'zlib'
+
+    def __init__(self, level=1):
+        self.level = level
+
+    def encode(self, buf):
+        # normalise inputs
+        buf = ensure_contiguous_ndarray(buf)
+
+        # do compression
+        return _zlib.compress(buf, self.level)
+
+    # noinspection PyMethodMayBeStatic
+    def decode(self, buf, out=None):
+        # normalise inputs
+        buf = ensure_contiguous_ndarray(buf)
+        if out is not None:
+            out = ensure_contiguous_ndarray(out)
+
+        # do decompression
+        dec = _zlib.decompress(buf)
+
+        # handle destination - Python standard library zlib module does not
+        # support direct decompression into buffer, so we have to copy into
+        # out if given
+        return ndarray_copy(dec, out)