numcodecs 0.16.0__cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

numcodecs/tests/test_zarr3.py

@@ -0,0 +1,279 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pytest
+
+if TYPE_CHECKING:  # pragma: no cover
+    import zarr
+else:
+    zarr = pytest.importorskip("zarr")
+
+import zarr.storage
+from zarr.core.common import JSON
+
+import numcodecs.zarr3
+
+pytestmark = [
+    pytest.mark.skipif(zarr.__version__ < "3.0.0", reason="zarr 3.0.0 or later is required"),
+    pytest.mark.filterwarnings("ignore:Codec 'numcodecs.*' not configured in config.*:UserWarning"),
+    pytest.mark.filterwarnings(
+        "ignore:Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations."
+    ),
+]
+
+get_codec_class = zarr.registry.get_codec_class
+Array = zarr.Array
+BytesCodec = zarr.codecs.BytesCodec
+Store = zarr.abc.store.Store
+MemoryStore = zarr.storage.MemoryStore
+StorePath = zarr.storage.StorePath
+
+
+EXPECTED_WARNING_STR = "Numcodecs codecs are not in the Zarr version 3.*"
+
+
+@pytest.fixture
+def store() -> StorePath:
+    return StorePath(MemoryStore(read_only=False))
+
+
+ALL_CODECS = [getattr(numcodecs.zarr3, cls_name) for cls_name in numcodecs.zarr3.__all__]
+
+
+@pytest.mark.parametrize("codec_class", ALL_CODECS)
+def test_entry_points(codec_class: type[numcodecs.zarr3._NumcodecsCodec]):
+    codec_name = codec_class.codec_name
+    assert get_codec_class(codec_name) == codec_class
+
+
+@pytest.mark.parametrize("codec_class", ALL_CODECS)
+def test_docstring(codec_class: type[numcodecs.zarr3._NumcodecsCodec]):
+    assert "See :class:`numcodecs." in codec_class.__doc__  # type: ignore[operator]
+
+
+@pytest.mark.parametrize(
+    "codec_class",
+    [
+        numcodecs.zarr3.Blosc,
+        numcodecs.zarr3.LZ4,
+        numcodecs.zarr3.Zstd,
+        numcodecs.zarr3.Zlib,
+        numcodecs.zarr3.GZip,
+        numcodecs.zarr3.BZ2,
+        numcodecs.zarr3.LZMA,
+        numcodecs.zarr3.Shuffle,
+    ],
+)
+def test_generic_compressor(
+    store: StorePath, codec_class: type[numcodecs.zarr3._NumcodecsBytesBytesCodec]
+):
+    data = np.arange(0, 256, dtype="uint16").reshape((16, 16))
+
+    with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
+        a = zarr.create_array(
+            store / "generic",
+            shape=data.shape,
+            chunks=(16, 16),
+            dtype=data.dtype,
+            fill_value=0,
+            compressors=[codec_class()],
+        )
+
+    a[:, :] = data.copy()
+    np.testing.assert_array_equal(data, a[:, :])
+
+
+@pytest.mark.parametrize(
+    ("codec_class", "codec_config"),
+    [
+        (numcodecs.zarr3.Delta, {"dtype": "float32"}),
+        (numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 25.5}),
+        (numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 51, "astype": "uint16"}),
+        (numcodecs.zarr3.AsType, {"encode_dtype": "float32", "decode_dtype": "float32"}),
+    ],
+    ids=[
+        "delta",
+        "fixedscaleoffset",
+        "fixedscaleoffset2",
+        "astype",
+    ],
+)
+def test_generic_filter(
+    store: StorePath,
+    codec_class: type[numcodecs.zarr3._NumcodecsArrayArrayCodec],
+    codec_config: dict[str, JSON],
+):
+    data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
+
+    with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
+        a = zarr.create_array(
+            store / "generic",
+            shape=data.shape,
+            chunks=(16, 16),
+            dtype=data.dtype,
+            fill_value=0,
+            filters=[
+                codec_class(**codec_config),
+            ],
+        )
+
+        a[:, :] = data.copy()
+        a = zarr.open_array(store / "generic", mode="r")
+    np.testing.assert_array_equal(data, a[:, :])
+
+
+def test_generic_filter_bitround(store: StorePath):
+    data = np.linspace(0, 1, 256, dtype="float32").reshape((16, 16))
+
+    with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
+        a = zarr.create_array(
+            store / "generic_bitround",
+            shape=data.shape,
+            chunks=(16, 16),
+            dtype=data.dtype,
+            fill_value=0,
+            filters=[numcodecs.zarr3.BitRound(keepbits=3)],
+        )
+
+        a[:, :] = data.copy()
+        a = zarr.open_array(store / "generic_bitround", mode="r")
+    assert np.allclose(data, a[:, :], atol=0.1)
+
+
+def test_generic_filter_quantize(store: StorePath):
+    data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
+
+    with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
+        a = zarr.create_array(
+            store / "generic_quantize",
+            shape=data.shape,
+            chunks=(16, 16),
+            dtype=data.dtype,
+            fill_value=0,
+            filters=[numcodecs.zarr3.Quantize(digits=3)],
+        )
+
+        a[:, :] = data.copy()
+        a = zarr.open_array(store / "generic_quantize", mode="r")
+    assert np.allclose(data, a[:, :], atol=0.001)
+
+
+def test_generic_filter_packbits(store: StorePath):
+    data = np.zeros((16, 16), dtype="bool")
+    data[0:4, :] = True
+
+    with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
+        a = zarr.create_array(
+            store / "generic_packbits",
+            shape=data.shape,
+            chunks=(16, 16),
+            dtype=data.dtype,
+            fill_value=0,
+            filters=[numcodecs.zarr3.PackBits()],
+        )
+
+        a[:, :] = data.copy()
+        a = zarr.open_array(store / "generic_packbits", mode="r")
+    np.testing.assert_array_equal(data, a[:, :])
+
+    with pytest.raises(ValueError, match=".*requires bool dtype.*"):
+        zarr.create_array(
+            store / "generic_packbits_err",
+            shape=data.shape,
+            chunks=(16, 16),
+            dtype="uint32",
+            fill_value=0,
+            filters=[numcodecs.zarr3.PackBits()],
+        )
+
+
+@pytest.mark.parametrize(
+    "codec_class",
+    [
+        numcodecs.zarr3.CRC32,
+        numcodecs.zarr3.CRC32C,
+        numcodecs.zarr3.Adler32,
+        numcodecs.zarr3.Fletcher32,
+        numcodecs.zarr3.JenkinsLookup3,
+    ],
+)
+def test_generic_checksum(
+    store: StorePath, codec_class: type[numcodecs.zarr3._NumcodecsBytesBytesCodec]
+):
+    data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
+
+    with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
+        a = zarr.create_array(
+            store / "generic_checksum",
+            shape=data.shape,
+            chunks=(16, 16),
+            dtype=data.dtype,
+            fill_value=0,
+            compressors=[codec_class()],
+        )
+
+        a[:, :] = data.copy()
+        a = zarr.open_array(store / "generic_checksum", mode="r")
+    np.testing.assert_array_equal(data, a[:, :])
+
+
+@pytest.mark.parametrize("codec_class", [numcodecs.zarr3.PCodec, numcodecs.zarr3.ZFPY])
+def test_generic_bytes_codec(
+    store: StorePath, codec_class: type[numcodecs.zarr3._NumcodecsArrayBytesCodec]
+):
+    try:
+        codec_class()._codec  # noqa: B018
+    except ValueError as e:  # pragma: no cover
+        if "codec not available" in str(e):
+            pytest.xfail(f"{codec_class.codec_name} is not available: {e}")
+        else:
+            raise
+    except ImportError as e:  # pragma: no cover
+        pytest.xfail(f"{codec_class.codec_name} is not available: {e}")
+
+    data = np.arange(0, 256, dtype="float32").reshape((16, 16))
+
+    with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
+        a = zarr.create_array(
+            store / "generic",
+            shape=data.shape,
+            chunks=(16, 16),
+            dtype=data.dtype,
+            fill_value=0,
+            serializer=codec_class(),
+        )
+
+    a[:, :] = data.copy()
+    np.testing.assert_array_equal(data, a[:, :])
+
+
+def test_delta_astype(store: StorePath):
+    data = np.linspace(0, 10, 256, dtype="i8").reshape((16, 16))
+
+    with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
+        a = zarr.create_array(
+            store / "generic",
+            shape=data.shape,
+            chunks=(16, 16),
+            dtype=data.dtype,
+            fill_value=0,
+            filters=[
+                numcodecs.zarr3.Delta(dtype="i8", astype="i2"),  # type: ignore[arg-type]
+            ],
+        )
+
+        a[:, :] = data.copy()
+        a = zarr.open_array(store / "generic", mode="r")
+    np.testing.assert_array_equal(data, a[:, :])
+
+
+def test_repr():
+    codec = numcodecs.zarr3.LZ4(level=5)
+    assert repr(codec) == "LZ4(codec_name='numcodecs.lz4', codec_config={'level': 5})"
+
+
+def test_to_dict():
+    codec = numcodecs.zarr3.LZ4(level=5)
+    assert codec.to_dict() == {"name": "numcodecs.lz4", "configuration": {"level": 5}}

numcodecs/tests/test_zarr3_import.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+import pytest
+
+
+def test_zarr3_import():
+    ERROR_MESSAGE_MATCH = "zarr 3.0.0 or later.*"
+
+    try:
+        import zarr  # noqa: F401
+    except ImportError:  # pragma: no cover
+        with pytest.raises(ImportError, match=ERROR_MESSAGE_MATCH):
+            import numcodecs.zarr3  # noqa: F401

numcodecs/tests/test_zfpy.py

@@ -0,0 +1,104 @@
+from types import ModuleType
+from typing import cast
+
+import numpy as np
+import pytest
+
+try:
+    # noinspection PyProtectedMember
+    from numcodecs.zfpy import ZFPY, _zfpy
+except ImportError:  # pragma: no cover
+    pytest.skip("ZFPY not available", allow_module_level=True)
+
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+
+_zfpy = cast(ModuleType, _zfpy)
+
+
+codecs = [
+    ZFPY(mode=_zfpy.mode_fixed_rate, rate=-1),
+    ZFPY(),
+    ZFPY(mode=_zfpy.mode_fixed_accuracy, tolerance=-1),
+    ZFPY(mode=_zfpy.mode_fixed_precision, precision=-1),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.linspace(1000, 1001, 1000, dtype="f4"),
+    np.linspace(1000, 1001, 1000, dtype="f8"),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.normal(loc=1000, scale=1, size=(10, 10, 10)),
+    np.random.normal(loc=1000, scale=1, size=(2, 5, 10, 10)),
+    np.random.randint(-(2**31), -(2**31) + 20, size=1000, dtype="i4").reshape(100, 10),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype="i8").reshape(10, 10, 10),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        if arr.dtype in (np.int32, np.int64):
+            codec = [codecs[-1]]
+        else:
+            codec = codecs
+        for code in codec:
+            check_encode_decode_array(arr, code)
+
+
+def test_config():
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("ZFPY(mode=4, tolerance=0.001, rate=-1, precision=-1)")
+
+
+def test_backwards_compatibility():
+    for code in codecs:
+        if code.mode == _zfpy.mode_fixed_rate:
+            codec = [code]
+            check_backwards_compatibility(ZFPY.codec_id, arrays, codec)
+        else:
+            check_backwards_compatibility(ZFPY.codec_id, arrays[: len(arrays) - 2], codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(ZFPY())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(ZFPY())
+
+
+def test_err_encode_list():
+    data = ['foo', 'bar', 'baz']
+    for codec in codecs:
+        with pytest.raises(TypeError):
+            codec.encode(data)
+
+
+def test_err_encode_non_contiguous():
+    # non-contiguous memory
+    arr = np.arange(1000, dtype='i4')[::2]
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)
+
+
+def test_err_encode_fortran_array():
+    # fortran array
+    arr = np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(5, 10, 20)))
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)

numcodecs/tests/test_zlib.py

@@ -0,0 +1,94 @@
+import itertools
+
+import numpy as np
+import pytest
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+from numcodecs.zlib import Zlib
+
+codecs = [
+    Zlib(),
+    Zlib(level=-1),
+    Zlib(level=0),
+    Zlib(level=1),
+    Zlib(level=5),
+    Zlib(level=9),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype='i4'),
+    np.linspace(1000, 1001, 1000, dtype='f8'),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode(arr, codec)
+
+
+def test_config():
+    codec = Zlib(level=3)
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("Zlib(level=3)")
+
+
+def test_eq():
+    assert Zlib() == Zlib()
+    assert not Zlib() != Zlib()
+    assert Zlib(1) == Zlib(1)
+    assert Zlib(1) != Zlib(9)
+    assert Zlib() != 'foo'
+    assert 'foo' != Zlib()
+    assert not Zlib() == 'foo'
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(Zlib.codec_id, arrays, codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(Zlib())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(Zlib())
+
+
+def test_err_encode_list():
+    data = ['foo', 'bar', 'baz']
+    for codec in codecs:
+        with pytest.raises(TypeError):
+            codec.encode(data)
+
+
+def test_err_encode_non_contiguous():
+    # non-contiguous memory
+    arr = np.arange(1000, dtype='i4')[::2]
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)

numcodecs/tests/test_zstd.py

@@ -0,0 +1,92 @@
+import itertools
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.zstd import Zstd
+except ImportError:  # pragma: no cover
+    pytest.skip("numcodecs.zstd not available", allow_module_level=True)
+
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+
+codecs = [
+    Zstd(),
+    Zstd(level=-1),
+    Zstd(level=0),
+    Zstd(level=1),
+    Zstd(level=10),
+    Zstd(level=22),
+    Zstd(level=100),
+    Zstd(checksum=True),
+    Zstd(level=0, checksum=True),
+    Zstd(level=22, checksum=True),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype="i4"),
+    np.linspace(1000, 1001, 1000, dtype="f8"),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode(arr, codec)
+
+
+def test_config():
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("Zstd(level=3)")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(Zstd.codec_id, arrays, codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(Zstd())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(Zstd())
+
+
+def test_checksum():
+    data = np.arange(0, 64, dtype="uint8")
+    assert len(Zstd(level=0, checksum=False).encode(data)) + 4 == len(
+        Zstd(level=0, checksum=True).encode(data)
+    )
+
+
+def test_native_functions():
+    # Note, these assertions might need to be changed for new versions of zstd
+    assert Zstd.default_level() == 3
+    assert Zstd.min_level() == -131072
+    assert Zstd.max_level() == 22

numcodecs/version.py

@@ -0,0 +1,21 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.16.0'
+__version_tuple__ = version_tuple = (0, 16, 0)