numcodecs 0.16.4__cp313-cp313-macosx_11_0_arm64.whl

numcodecs/tests/test_pyzstd.py

@@ -0,0 +1,76 @@
+# Check Zstd against pyzstd package
+
+import numpy as np
+import pytest
+import pyzstd
+
+from numcodecs.zstd import Zstd
+
+test_data = [
+    b"Hello World!",
+    np.arange(113).tobytes(),
+    np.arange(10, 15).tobytes(),
+    np.random.randint(3, 50, size=(53,), dtype=np.uint16).tobytes(),
+]
+
+
+@pytest.mark.parametrize("input", test_data)
+def test_pyzstd_simple(input):
+    """
+    Test if Zstd.[decode, encode] can perform the inverse operation to
+    pyzstd.[compress, decompress] in the simple case.
+    """
+    z = Zstd()
+    assert z.decode(pyzstd.compress(input)) == input
+    assert pyzstd.decompress(z.encode(input)) == input
+
+
+@pytest.mark.parametrize("input", test_data)
+def test_pyzstd_simple_multiple_frames_decode(input):
+    """
+    Test decompression of two concatenated frames of known sizes
+
+    numcodecs.zstd.Zstd currently fails because it only assesses the size of the
+    first frame. Rather, it should keep iterating through all the frames until
+    the end of the input buffer.
+    """
+    z = Zstd()
+    assert pyzstd.decompress(pyzstd.compress(input) * 2) == input * 2
+    assert z.decode(pyzstd.compress(input) * 2) == input * 2
+
+
+@pytest.mark.parametrize("input", test_data)
+def test_pyzstd_simple_multiple_frames_encode(input):
+    """
+    Test if pyzstd can decompress two concatenated frames from Zstd.encode
+    """
+    z = Zstd()
+    assert pyzstd.decompress(z.encode(input) * 2) == input * 2
+
+
+@pytest.mark.parametrize("input", test_data)
+def test_pyzstd_streaming(input):
+    """
+    Test if Zstd can decode a single frame and concatenated frames in streaming
+    mode where the decompressed size is not recorded in the frame header.
+    """
+    pyzstd_c = pyzstd.ZstdCompressor()
+    pyzstd_d = pyzstd.ZstdDecompressor()
+    pyzstd_e = pyzstd.EndlessZstdDecompressor()
+    z = Zstd()
+
+    d_bytes = input
+    pyzstd_c.compress(d_bytes)
+    c_bytes = pyzstd_c.flush()
+    assert z.decode(c_bytes) == d_bytes
+    assert pyzstd_d.decompress(z.encode(d_bytes)) == d_bytes
+
+    # Test multiple streaming frames
+    assert z.decode(c_bytes * 2) == pyzstd_e.decompress(c_bytes * 2)
+    assert z.decode(c_bytes * 3) == pyzstd_e.decompress(c_bytes * 3)
+    assert z.decode(c_bytes * 4) == pyzstd_e.decompress(c_bytes * 4)
+    assert z.decode(c_bytes * 5) == pyzstd_e.decompress(c_bytes * 5)
+    assert z.decode(c_bytes * 7) == pyzstd_e.decompress(c_bytes * 7)
+    assert z.decode(c_bytes * 11) == pyzstd_e.decompress(c_bytes * 11)
+    assert z.decode(c_bytes * 13) == pyzstd_e.decompress(c_bytes * 13)
+    assert z.decode(c_bytes * 99) == pyzstd_e.decompress(c_bytes * 99)

numcodecs/tests/test_quantize.py

@@ -0,0 +1,76 @@
+import itertools
+
+import numpy as np
+import pytest
+from numpy.testing import assert_array_almost_equal, assert_array_equal
+
+from numcodecs.quantize import Quantize
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_repr,
+)
+
+arrays = [
+    np.linspace(100, 200, 1000, dtype='<f8'),
+    np.random.normal(loc=0, scale=1, size=1000).astype('<f8'),
+    np.linspace(100, 200, 1000, dtype='<f8').reshape(100, 10),
+    np.linspace(100, 200, 1000, dtype='<f8').reshape(100, 10, order='F'),
+    np.linspace(100, 200, 1000, dtype='<f8').reshape(10, 10, 10),
+]
+
+
+codecs = [
+    Quantize(digits=-1, dtype='<f8', astype='<f2'),
+    Quantize(digits=0, dtype='<f8', astype='<f2'),
+    Quantize(digits=1, dtype='<f8', astype='<f2'),
+    Quantize(digits=5, dtype='<f8', astype='<f4'),
+    Quantize(digits=12, dtype='<f8', astype='<f8'),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode(arr, codec, precision=codec.digits)
+
+
+def test_encode():
+    for arr, codec in itertools.product(arrays, codecs):
+        if arr.flags.f_contiguous:
+            order = 'F'
+        else:
+            order = 'C'
+        enc = codec.encode(arr).reshape(arr.shape, order=order)
+        assert_array_almost_equal(arr, enc, decimal=codec.digits)
+
+
+def test_decode():
+    # decode is a no-op
+    for arr, codec in itertools.product(arrays, codecs):
+        enc = codec.encode(arr)
+        dec = codec.decode(enc)
+        assert_array_equal(enc, dec)
+
+
+def test_config():
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("Quantize(digits=2, dtype='<f8', astype='<f2')")
+
+
+def test_errors():
+    with pytest.raises(ValueError):
+        Quantize(digits=2, dtype='i4')
+    with pytest.raises(ValueError):
+        Quantize(digits=2, dtype=object)
+    with pytest.raises(ValueError):
+        Quantize(digits=2, dtype='f8', astype=object)
+
+
+def test_backwards_compatibility():
+    precision = [codec.digits for codec in codecs]
+    check_backwards_compatibility(Quantize.codec_id, arrays, codecs, precision=precision)

numcodecs/tests/test_registry.py

@@ -0,0 +1,43 @@
+import inspect
+
+import pytest
+
+import numcodecs
+from numcodecs.errors import UnknownCodecError
+from numcodecs.registry import get_codec
+
+
+def test_registry_errors():
+    with pytest.raises(UnknownCodecError, match='foo'):
+        get_codec({'id': 'foo'})
+
+
+def test_get_codec_argument():
+    # Check that get_codec doesn't modify its argument.
+    arg = {"id": "json2"}
+    before = dict(arg)
+    get_codec(arg)
+    assert before == arg
+
+
+def test_all_classes_registered():
+    """
+    find all Codec subclasses in this repository and check that they
+    have been registered.
+
+    see #346 for more info
+    """
+    missing = {
+        obj.codec_id
+        for _, submod in inspect.getmembers(numcodecs, inspect.ismodule)
+        for _, obj in inspect.getmembers(submod)
+        if (
+            inspect.isclass(obj)
+            and issubclass(obj, numcodecs.abc.Codec)
+            and obj.codec_id not in numcodecs.registry.codec_registry
+            and obj.codec_id is not None  # remove `None`
+        )
+    }
+
+    if missing:
+        raise Exception(f"these codecs are missing: {missing}")  # pragma: no cover

numcodecs/tests/test_shuffle.py

@@ -0,0 +1,166 @@
+from multiprocessing import Pool
+from multiprocessing.pool import ThreadPool
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.shuffle import Shuffle
+except ImportError:  # pragma: no cover
+    pytest.skip("numcodecs.shuffle not available", allow_module_level=True)
+
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+)
+
+codecs = [
+    Shuffle(),
+    Shuffle(elementsize=0),
+    Shuffle(elementsize=4),
+    Shuffle(elementsize=8),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype='i4'),
+    np.linspace(1000, 1001, 1000, dtype='f8'),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
+]
+
+
+@pytest.mark.parametrize('array', arrays)
+@pytest.mark.parametrize('codec', codecs)
+def test_encode_decode(array, codec):
+    check_encode_decode(array, codec)
+
+
+def test_config():
+    codec = Shuffle()
+    check_config(codec)
+    codec = Shuffle(elementsize=8)
+    check_config(codec)
+
+
+def test_repr():
+    expect = "Shuffle(elementsize=0)"
+    actual = repr(Shuffle(elementsize=0))
+    assert expect == actual
+    expect = "Shuffle(elementsize=4)"
+    actual = repr(Shuffle(elementsize=4))
+    assert expect == actual
+    expect = "Shuffle(elementsize=8)"
+    actual = repr(Shuffle(elementsize=8))
+    assert expect == actual
+    expect = "Shuffle(elementsize=16)"
+    actual = repr(Shuffle(elementsize=16))
+    assert expect == actual
+
+
+def test_eq():
+    assert Shuffle() == Shuffle()
+    assert Shuffle(elementsize=16) != Shuffle()
+
+
+def _encode_worker(data):
+    compressor = Shuffle()
+    return compressor.encode(data)
+
+
+def _decode_worker(enc):
+    compressor = Shuffle()
+    return compressor.decode(enc)
+
+
+@pytest.mark.parametrize('pool', [Pool, ThreadPool])
+def test_multiprocessing(pool):
+    data = np.arange(1000000)
+    enc = _encode_worker(data)
+
+    pool = pool(5)
+
+    # test with process pool and thread pool
+
+    # test encoding
+    enc_results = pool.map(_encode_worker, [data] * 5)
+    assert all(len(enc) == len(e) for e in enc_results)
+
+    # test decoding
+    dec_results = pool.map(_decode_worker, [enc] * 5)
+    assert all(data.nbytes == len(d) for d in dec_results)
+
+    # tidy up
+    pool.close()
+    pool.join()
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(Shuffle.codec_id, arrays, codecs)
+
+
+# def test_err_decode_object_buffer():
+#     check_err_decode_object_buffer(Shuffle())
+
+
+# def test_err_encode_object_buffer():
+#     check_err_encode_object_buffer(Shuffle())
+
+# def test_decompression_error_handling():
+#     for codec in codecs:
+#         with pytest.raises(RuntimeError):
+#             codec.decode(bytearray())
+#         with pytest.raises(RuntimeError):
+#             codec.decode(bytearray(0))
+
+
+def test_expected_result():
+    # If the input is treated as a 2D byte array, with shape (size of element, number of elements),
+    # the shuffle is essentially a transpose. This can be made more apparent by using an array of
+    # big-endian integers, as below.
+    arr = np.array(
+        [
+            0x0001020304050607,
+            0x08090A0B0C0D0E0F,
+            0x1011121314151617,
+            0x18191A1B1C1D1E1F,
+        ],
+        dtype='>u8',
+    )
+    expected = np.array(
+        [
+            0x00081018,
+            0x01091119,
+            0x020A121A,
+            0x030B131B,
+            0x040C141C,
+            0x050D151D,
+            0x060E161E,
+            0x070F171F,
+        ],
+        dtype='u4',
+    )
+    codec = Shuffle(elementsize=arr.data.itemsize)
+    enc = codec.encode(arr)
+    np.testing.assert_array_equal(np.frombuffer(enc.data, '>u4'), expected)
+
+
+def test_incompatible_elementsize():
+    arr = np.arange(1001, dtype='u1')
+    codec = Shuffle(elementsize=4)
+    with pytest.raises(ValueError):
+        codec.encode(arr)

numcodecs/tests/test_vlen_array.py

@@ -0,0 +1,97 @@
+import unittest
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.vlen import VLenArray
+except ImportError as e:  # pragma: no cover
+    raise unittest.SkipTest("vlen-array not available") from e
+from numcodecs.tests.common import (
+    assert_array_items_equal,
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+)
+
+arrays = [
+    np.array([np.array([1, 2, 3]), np.array([4]), np.array([5, 6])] * 300, dtype=object),
+    np.array([np.array([1, 2, 3]), np.array([4]), np.array([5, 6])] * 300, dtype=object).reshape(
+        90, 10
+    ),
+]
+
+
+codecs = [
+    VLenArray('<i1'),
+    VLenArray('<i2'),
+    VLenArray('<i4'),
+    VLenArray('<i8'),
+    VLenArray('<u1'),
+    VLenArray('<u2'),
+    VLenArray('<u4'),
+    VLenArray('<u8'),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        for codec in codecs:
+            check_encode_decode_array(arr, codec)
+
+
+def test_config():
+    codec = VLenArray('<i8')
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("VLenArray(dtype='<i8')")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(VLenArray.codec_id, arrays, codecs)
+
+
+def test_encode_errors():
+    codec = VLenArray('<i8')
+    with pytest.raises(ValueError):
+        codec.encode('foo')
+    with pytest.raises(ValueError):
+        codec.encode(['foo', 'bar'])
+
+
+def test_decode_errors():
+    codec = VLenArray('<i8')
+    with pytest.raises(TypeError):
+        codec.decode(1234)
+    # these should look like corrupt data
+    with pytest.raises(ValueError):
+        codec.decode(b'foo')
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(2, 3, dtype='i4'))
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(10, 20, dtype='i4'))
+    with pytest.raises(TypeError):
+        codec.decode('foo')
+
+    # test out parameter
+    enc = codec.encode(arrays[0])
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=b'foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out='foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=123)
+    with pytest.raises(ValueError):
+        codec.decode(enc, out=np.zeros(10, dtype='i4'))
+
+
+def test_encode_none():
+    a = np.array([[1, 3], None, [4, 7]], dtype=object)
+    codec = VLenArray(int)
+    enc = codec.encode(a)
+    dec = codec.decode(enc)
+    expect = np.array([np.array([1, 3]), np.array([]), np.array([4, 7])], dtype=object)
+    assert_array_items_equal(expect, dec)

numcodecs/tests/test_vlen_bytes.py

@@ -0,0 +1,93 @@
+import unittest
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.vlen import VLenBytes
+except ImportError as e:  # pragma: no cover
+    raise unittest.SkipTest("vlen-bytes not available") from e
+from numcodecs.tests.common import (
+    assert_array_items_equal,
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+    greetings,
+)
+
+greetings_bytes = [g.encode('utf-8') for g in greetings]
+
+
+arrays = [
+    np.array([b'foo', b'bar', b'baz'] * 300, dtype=object),
+    np.array(greetings_bytes * 100, dtype=object),
+    np.array([b'foo', b'bar', b'baz'] * 300, dtype=object).reshape(90, 10),
+    np.array(greetings_bytes * 1000, dtype=object).reshape(
+        len(greetings_bytes), 100, 10, order='F'
+    ),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        codec = VLenBytes()
+        check_encode_decode_array(arr, codec)
+
+
+def test_config():
+    codec = VLenBytes()
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("VLenBytes()")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(VLenBytes.codec_id, arrays, [VLenBytes()])
+
+
+def test_encode_errors():
+    codec = VLenBytes()
+    with pytest.raises(TypeError):
+        codec.encode(1234)
+    with pytest.raises(TypeError):
+        codec.encode([1234, 5678])
+    with pytest.raises(TypeError):
+        codec.encode(np.ones(10, dtype='i4'))
+
+
+def test_decode_errors():
+    codec = VLenBytes()
+    with pytest.raises(TypeError):
+        codec.decode(1234)
+    # these should look like corrupt data
+    with pytest.raises(ValueError):
+        codec.decode(b'foo')
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(2, 3, dtype='i4'))
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(10, 20, dtype='i4'))
+    with pytest.raises(TypeError):
+        codec.decode('foo')
+
+    # test out parameter
+    enc = codec.encode(arrays[0])
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=b'foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out='foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=123)
+    with pytest.raises(ValueError):
+        codec.decode(enc, out=np.zeros(10, dtype='i4'))
+
+
+def test_encode_none():
+    a = np.array([b'foo', None, b'bar'], dtype=object)
+    codec = VLenBytes()
+    enc = codec.encode(a)
+    dec = codec.decode(enc)
+    expect = np.array([b'foo', b'', b'bar'], dtype=object)
+    assert_array_items_equal(expect, dec)

numcodecs/tests/test_vlen_utf8.py

@@ -0,0 +1,91 @@
+import unittest
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.vlen import VLenUTF8
+except ImportError as e:  # pragma: no cover
+    raise unittest.SkipTest("vlen-utf8 not available") from e
+from numcodecs.tests.common import (
+    assert_array_items_equal,
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+    greetings,
+)
+
+arrays = [
+    np.array(['foo', 'bar', 'baz'] * 300, dtype=object),
+    np.array(greetings * 100, dtype=object),
+    np.array(['foo', 'bar', 'baz'] * 300, dtype=object).reshape(90, 10),
+    np.array(greetings * 1000, dtype=object).reshape(len(greetings), 100, 10, order='F'),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        codec = VLenUTF8()
+        check_encode_decode_array(arr, codec)
+
+
+def test_config():
+    codec = VLenUTF8()
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("VLenUTF8()")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(VLenUTF8.codec_id, arrays, [VLenUTF8()])
+
+
+def test_encode_errors():
+    codec = VLenUTF8()
+    with pytest.raises(TypeError):
+        codec.encode(1234)
+    with pytest.raises(TypeError):
+        codec.encode([1234, 5678])
+    with pytest.raises(TypeError):
+        codec.encode(np.ones(10, dtype='i4'))
+
+
+def test_decode_errors():
+    codec = VLenUTF8()
+    with pytest.raises(TypeError):
+        codec.decode(1234)
+    # these should look like corrupt data
+    with pytest.raises(ValueError):
+        codec.decode(b'foo')
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(2, 3, dtype='i4'))
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(10, 20, dtype='i4'))
+    with pytest.raises(TypeError):
+        codec.decode('foo')
+
+    # test out parameter
+    enc = codec.encode(arrays[0])
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=b'foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out='foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=123)
+    with pytest.raises(ValueError):
+        codec.decode(enc, out=np.zeros(10, dtype='i4'))
+
+
+@pytest.mark.parametrize("writable", [True, False])
+def test_encode_utf8(writable):
+    a = np.array(['foo', None, 'bar'], dtype=object)
+    if not writable:
+        a.setflags(write=False)
+    codec = VLenUTF8()
+    enc = codec.encode(a)
+    dec = codec.decode(enc)
+    expect = np.array(['foo', '', 'bar'], dtype=object)
+    assert_array_items_equal(expect, dec)

numcodecs/tests/test_zarr3.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:  # pragma: no cover
+    import zarr
+else:
+    zarr = pytest.importorskip("zarr", "3.1.3")
+
+import numcodecs.zarr3 as zarr3
+
+codec_names = [
+    "BZ2",
+    "CRC32",
+    "CRC32C",
+    "LZ4",
+    "LZMA",
+    "ZFPY",
+    "Adler32",
+    "AsType",
+    "BitRound",
+    "Blosc",
+    "Delta",
+    "FixedScaleOffset",
+    "Fletcher32",
+    "GZip",
+    "JenkinsLookup3",
+    "PCodec",
+    "PackBits",
+    "Quantize",
+    "Shuffle",
+    "Zlib",
+    "Zstd",
+]
+
+
+@pytest.mark.parametrize('codec_name', codec_names)
+def test_export(codec_name: str) -> None:
+    """
+    Ensure that numcodecs.zarr3 re-exports codecs defined in zarr.codecs.numcodecs
+    """
+    with pytest.warns(
+        DeprecationWarning,
+        match="The numcodecs.zarr3 module is deprecated and will be removed in a future release of numcodecs. ",
+    ):
+        assert getattr(zarr3, codec_name) == getattr(zarr.codecs.numcodecs, codec_name)

numcodecs/tests/test_zarr3_import.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+import pytest
+
+
+def test_zarr3_import():
+    ERROR_MESSAGE_MATCH = "Zarr 3.1.3 or later*"
+
+    try:
+        import zarr  # noqa: F401
+    except ImportError:  # pragma: no cover
+        with pytest.raises(ImportError, match=ERROR_MESSAGE_MATCH):
+            import numcodecs.zarr3  # noqa: F401