numcodecs 0.13.1__cp313-cp313-macosx_10_13_x86_64.whl

numcodecs/tests/test_msgpacks.py

@@ -0,0 +1,123 @@
+import unittest
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.msgpacks import MsgPack
+except ImportError as e:  # pragma: no cover
+    raise unittest.SkipTest("msgpack not available") from e
+
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+    greetings,
+)
+
+# object array with strings
+# object array with mix strings / nans
+# object array with mix of string, int, float
+# ...
+arrays = [
+    np.array(['foo', 'bar', 'baz'] * 300, dtype=object),
+    np.array([['foo', 'bar', np.nan]] * 300, dtype=object),
+    np.array(['foo', 1.0, 2] * 300, dtype=object),
+    np.arange(1000, dtype='i4'),
+    np.array(['foo', 'bar', 'baz'] * 300),
+    np.array(['foo', ['bar', 1.0, 2], {'a': 'b', 'c': 42}] * 300, dtype=object),
+    np.array(greetings * 100),
+    np.array(greetings * 100, dtype=object),
+    np.array([b'foo', b'bar', b'baz'] * 300, dtype=object),
+    np.array([g.encode('utf-8') for g in greetings] * 100, dtype=object),
+    np.array([[0, 1], [2, 3]], dtype=object),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        check_encode_decode_array(arr, MsgPack())
+
+
+def test_config():
+    check_config(MsgPack())
+
+
+def test_repr():
+    check_repr("MsgPack(raw=False, use_bin_type=True, use_single_float=False)")
+    check_repr("MsgPack(raw=True, use_bin_type=False, use_single_float=True)")
+
+
+def test_backwards_compatibility():
+    codec = MsgPack()
+    check_backwards_compatibility(codec.codec_id, arrays, [codec])
+
+
+@pytest.mark.parametrize(
+    "input_data, dtype",
+    [
+        ([0, 1], None),
+        ([[0, 1], [2, 3]], None),
+        ([[0], [1], [2, 3]], object),
+        ([[[0, 0]], [[1, 1]], [[2, 3]]], None),
+        (["1"], None),
+        (["11", "11"], None),
+        (["11", "1", "1"], None),
+        ([{}], None),
+        ([{"key": "value"}, ["list", "of", "strings"]], object),
+        ([b"1"], None),
+        ([b"11", b"11"], None),
+        ([b"11", b"1", b"1"], None),
+        ([{b"key": b"value"}, [b"list", b"of", b"strings"]], object),
+    ],
+)
+def test_non_numpy_inputs(input_data, dtype):
+    codec = MsgPack()
+    # numpy will infer a range of different shapes and dtypes for these inputs.
+    # Make sure that round-tripping through encode preserves this.
+    actual = codec.decode(codec.encode(input_data))
+    expect = np.array(input_data, dtype=dtype)
+    assert expect.shape == actual.shape
+    assert np.array_equal(expect, actual)
+
+
+def test_encode_decode_shape_dtype_preserved():
+    codec = MsgPack()
+    for arr in arrays:
+        actual = codec.decode(codec.encode(arr))
+        assert arr.shape == actual.shape
+        assert arr.dtype == actual.dtype
+
+
+def test_bytes():
+    # test msgpack behaviour with bytes and str (unicode)
+    bytes_arr = np.array([b'foo', b'bar', b'baz'], dtype=object)
+    unicode_arr = np.array(['foo', 'bar', 'baz'], dtype=object)
+
+    # raw=False (default)
+    codec = MsgPack()
+    # works for bytes array, round-trips bytes to bytes
+    b = codec.decode(codec.encode(bytes_arr))
+    assert np.array_equal(bytes_arr, b)
+    assert isinstance(b[0], bytes)
+    assert b[0] == b'foo'
+    # works for unicode array, round-trips unicode to unicode
+    b = codec.decode(codec.encode(unicode_arr))
+    assert np.array_equal(unicode_arr, b)
+    assert isinstance(b[0], str)
+    assert b[0] == 'foo'
+
+    # raw=True
+    codec = MsgPack(raw=True)
+    # works for bytes array, round-trips bytes to bytes
+    b = codec.decode(codec.encode(bytes_arr))
+    assert np.array_equal(bytes_arr, b)
+    assert isinstance(b[0], bytes)
+    assert b[0] == b'foo'
+    # broken for unicode array, round-trips unicode to bytes
+    b = codec.decode(codec.encode(unicode_arr))
+    assert not np.array_equal(unicode_arr, b)
+    assert isinstance(b[0], bytes)
+    assert b[0] == b'foo'

numcodecs/tests/test_ndarray_like.py

@@ -0,0 +1,48 @@
+import pytest
+
+from numcodecs.ndarray_like import DType, FlagsObj, NDArrayLike
+
+
+@pytest.mark.parametrize("module", ["numpy", "cupy"])
+def test_is_ndarray_like(module):
+    m = pytest.importorskip(module)
+    a = m.arange(10)
+    assert isinstance(a, NDArrayLike)
+
+
+def test_is_not_ndarray_like():
+    assert not isinstance([1, 2, 3], NDArrayLike)
+    assert not isinstance(b"1,2,3", NDArrayLike)
+
+
+@pytest.mark.parametrize("module", ["numpy", "cupy"])
+def test_is_dtype_like(module):
+    m = pytest.importorskip(module)
+    d = m.dtype("u8")
+    assert isinstance(d, DType)
+
+
+def test_is_not_dtype_like():
+    assert not isinstance([1, 2, 3], DType)
+    assert not isinstance(b"1,2,3", DType)
+
+
+@pytest.mark.parametrize("module", ["numpy", "cupy"])
+def test_is_flags_like(module):
+    m = pytest.importorskip(module)
+    d = m.arange(10).flags
+    assert isinstance(d, FlagsObj)
+
+
+def test_is_not_flags_like():
+    assert not isinstance([1, 2, 3], FlagsObj)
+    assert not isinstance(b"1,2,3", FlagsObj)
+
+
+@pytest.mark.parametrize("module", ["numpy", "cupy"])
+def test_cached_isinstance_check(module):
+    m = pytest.importorskip(module)
+    a = m.arange(10)
+    assert isinstance(a, NDArrayLike)
+    assert not isinstance(a, DType)
+    assert not isinstance(a, FlagsObj)

numcodecs/tests/test_packbits.py

@@ -0,0 +1,39 @@
+import numpy as np
+
+from numcodecs.packbits import PackBits
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_repr,
+)
+
+arrays = [
+    np.random.randint(0, 2, size=1000, dtype=bool),
+    np.random.randint(0, 2, size=(100, 10), dtype=bool),
+    np.random.randint(0, 2, size=(10, 10, 10), dtype=bool),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(10, 10, 10, order='F'),
+]
+
+
+def test_encode_decode():
+    codec = PackBits()
+    for arr in arrays:
+        check_encode_decode(arr, codec)
+    # check different number of left-over bits
+    arr = np.random.randint(0, 2, size=1000, dtype=bool)
+    for size in list(range(1, 17)):
+        check_encode_decode(arr[:size], codec)
+
+
+def test_config():
+    codec = PackBits()
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("PackBits()")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(PackBits.codec_id, arrays, [PackBits()])

numcodecs/tests/test_pcodec.py

@@ -0,0 +1,80 @@
+import numpy as np
+import pytest
+
+from numcodecs.pcodec import PCodec
+
+try:
+    # initializing codec triggers ImportError
+    PCodec()
+except ImportError:  # pragma: no cover
+    pytest.skip("pcodec not available", allow_module_level=True)
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array_to_bytes,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+
+codecs = [
+    PCodec(),
+    PCodec(level=1),
+    PCodec(level=5),
+    PCodec(level=9),
+    PCodec(mode_spec='classic'),
+    PCodec(equal_pages_up_to=300),
+]
+
+
+# mix of dtypes: integer, float
+# mix of shapes: 1D, 2D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype="u4"),
+    np.arange(1000, dtype="u8"),
+    np.arange(1000, dtype="i4"),
+    np.arange(1000, dtype="i8"),
+    np.linspace(1000, 1001, 1000, dtype="f4"),
+    np.linspace(1000, 1001, 1000, dtype="f8"),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(100, 10))),
+    np.random.randint(0, 2**60, size=1000, dtype="u8"),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype="i8"),
+]
+
+
+@pytest.mark.parametrize("arr", arrays)
+@pytest.mark.parametrize("codec", codecs)
+def test_encode_decode(arr, codec):
+    check_encode_decode_array_to_bytes(arr, codec)
+
+
+def test_config():
+    codec = PCodec(level=3)
+    check_config(codec)
+
+
+def test_invalid_config_error():
+    with pytest.raises(ValueError):
+        codec = PCodec(mode_spec='bogus')
+        check_encode_decode_array_to_bytes(arrays[0], codec)
+
+
+def test_repr():
+    check_repr(
+        "PCodec(delta_encoding_order=None, equal_pages_up_to=262144, level=3, mode_spec='auto')"
+    )
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(PCodec.codec_id, arrays, codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(PCodec())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(PCodec())

numcodecs/tests/test_pickles.py

@@ -0,0 +1,61 @@
+import itertools
+import sys
+
+import numpy as np
+import pytest
+
+from numcodecs.pickles import Pickle
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+    greetings,
+)
+
+codecs = [Pickle(protocol=i) for i in range(5)]
+
+# object array with strings
+# object array with mix strings / nans
+# object array with mix of string, int, float
+# ...
+arrays = [
+    np.array(['foo', 'bar', 'baz'] * 300, dtype=object),
+    np.array([['foo', 'bar', np.nan]] * 300, dtype=object),
+    np.array(['foo', 1.0, 2] * 300, dtype=object),
+    np.arange(1000, dtype='i4'),
+    np.array(['foo', 'bar', 'baz'] * 300),
+    np.array(['foo', ['bar', 1.0, 2], {'a': 'b', 'c': 42}] * 300, dtype=object),
+    np.array(greetings * 100),
+    np.array(greetings * 100, dtype=object),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode_array(arr, codec)
+
+
+def test_config():
+    codec = Pickle(protocol=-1)
+    check_config(codec)
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("Pickle(protocol=-1)")
+
+
+# Details on xfail
+# https://stackoverflow.com/questions/58194852/modulenotfounderror-no-module-named-numpy-core-multiarray-r
+@pytest.mark.skipif(sys.byteorder != "little", reason="Pickle does not restore byte orders")
+@pytest.mark.xfail(
+    sys.platform == "win32",
+    reason=(
+        "Pickle fails to read w/ Windows carriage return "
+        "( https://github.com/zarr-developers/numcodecs/issues/271 )"
+    ),
+)
+def test_backwards_compatibility():
+    check_backwards_compatibility(Pickle.codec_id, arrays, codecs)

numcodecs/tests/test_quantize.py

@@ -0,0 +1,76 @@
+import itertools
+
+import numpy as np
+import pytest
+from numpy.testing import assert_array_almost_equal, assert_array_equal
+
+from numcodecs.quantize import Quantize
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_repr,
+)
+
+arrays = [
+    np.linspace(100, 200, 1000, dtype='<f8'),
+    np.random.normal(loc=0, scale=1, size=1000).astype('<f8'),
+    np.linspace(100, 200, 1000, dtype='<f8').reshape(100, 10),
+    np.linspace(100, 200, 1000, dtype='<f8').reshape(100, 10, order='F'),
+    np.linspace(100, 200, 1000, dtype='<f8').reshape(10, 10, 10),
+]
+
+
+codecs = [
+    Quantize(digits=-1, dtype='<f8', astype='<f2'),
+    Quantize(digits=0, dtype='<f8', astype='<f2'),
+    Quantize(digits=1, dtype='<f8', astype='<f2'),
+    Quantize(digits=5, dtype='<f8', astype='<f4'),
+    Quantize(digits=12, dtype='<f8', astype='<f8'),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode(arr, codec, precision=codec.digits)
+
+
+def test_encode():
+    for arr, codec in itertools.product(arrays, codecs):
+        if arr.flags.f_contiguous:
+            order = 'F'
+        else:
+            order = 'C'
+        enc = codec.encode(arr).reshape(arr.shape, order=order)
+        assert_array_almost_equal(arr, enc, decimal=codec.digits)
+
+
+def test_decode():
+    # decode is a no-op
+    for arr, codec in itertools.product(arrays, codecs):
+        enc = codec.encode(arr)
+        dec = codec.decode(enc)
+        assert_array_equal(enc, dec)
+
+
+def test_config():
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("Quantize(digits=2, dtype='<f8', astype='<f2')")
+
+
+def test_errors():
+    with pytest.raises(ValueError):
+        Quantize(digits=2, dtype='i4')
+    with pytest.raises(ValueError):
+        Quantize(digits=2, dtype=object)
+    with pytest.raises(ValueError):
+        Quantize(digits=2, dtype='f8', astype=object)
+
+
+def test_backwards_compatibility():
+    precision = [codec.digits for codec in codecs]
+    check_backwards_compatibility(Quantize.codec_id, arrays, codecs, precision=precision)

numcodecs/tests/test_registry.py

@@ -0,0 +1,40 @@
+import inspect
+
+import pytest
+
+import numcodecs
+from numcodecs.registry import get_codec
+
+
+def test_registry_errors():
+    with pytest.raises(ValueError):
+        get_codec({'id': 'foo'})
+
+
+def test_get_codec_argument():
+    # Check that get_codec doesn't modify its argument.
+    arg = {"id": "json2"}
+    before = dict(arg)
+    get_codec(arg)
+    assert before == arg
+
+
+def test_all_classes_registered():
+    """
+    find all Codec subclasses in this repository and check that they
+    have been registered.
+
+    see #346 for more info
+    """
+    missing = set()
+    for _name, submod in inspect.getmembers(numcodecs, inspect.ismodule):
+        for _name, obj in inspect.getmembers(submod):
+            if inspect.isclass(obj):
+                if issubclass(obj, numcodecs.abc.Codec):
+                    if obj.codec_id not in numcodecs.registry.codec_registry:
+                        missing.add(obj.codec_id)
+
+    # remove `None``
+    missing.remove(None)
+    if missing:
+        raise Exception(f"these codecs are missing: {missing}")  # pragma: no cover

numcodecs/tests/test_shuffle.py

@@ -0,0 +1,168 @@
+from multiprocessing import Pool
+from multiprocessing.pool import ThreadPool
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.shuffle import Shuffle
+except ImportError:  # pragma: no cover
+    pytest.skip("numcodecs.shuffle not available", allow_module_level=True)
+
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+)
+
+codecs = [
+    Shuffle(),
+    Shuffle(elementsize=0),
+    Shuffle(elementsize=4),
+    Shuffle(elementsize=8),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype='i4'),
+    np.linspace(1000, 1001, 1000, dtype='f8'),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
+]
+
+
+@pytest.mark.parametrize('array', arrays)
+@pytest.mark.parametrize('codec', codecs)
+def test_encode_decode(array, codec):
+    check_encode_decode(array, codec)
+
+
+def test_config():
+    codec = Shuffle()
+    check_config(codec)
+    codec = Shuffle(elementsize=8)
+    check_config(codec)
+
+
+def test_repr():
+    expect = "Shuffle(elementsize=0)"
+    actual = repr(Shuffle(elementsize=0))
+    assert expect == actual
+    expect = "Shuffle(elementsize=4)"
+    actual = repr(Shuffle(elementsize=4))
+    assert expect == actual
+    expect = "Shuffle(elementsize=8)"
+    actual = repr(Shuffle(elementsize=8))
+    assert expect == actual
+    expect = "Shuffle(elementsize=16)"
+    actual = repr(Shuffle(elementsize=16))
+    assert expect == actual
+
+
+def test_eq():
+    assert Shuffle() == Shuffle()
+    assert Shuffle(elementsize=16) != Shuffle()
+
+
+def _encode_worker(data):
+    compressor = Shuffle()
+    enc = compressor.encode(data)
+    return enc
+
+
+def _decode_worker(enc):
+    compressor = Shuffle()
+    data = compressor.decode(enc)
+    return data
+
+
+@pytest.mark.parametrize('pool', (Pool, ThreadPool))
+def test_multiprocessing(pool):
+    data = np.arange(1000000)
+    enc = _encode_worker(data)
+
+    pool = pool(5)
+
+    # test with process pool and thread pool
+
+    # test encoding
+    enc_results = pool.map(_encode_worker, [data] * 5)
+    assert all(len(enc) == len(e) for e in enc_results)
+
+    # test decoding
+    dec_results = pool.map(_decode_worker, [enc] * 5)
+    assert all(data.nbytes == len(d) for d in dec_results)
+
+    # tidy up
+    pool.close()
+    pool.join()
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(Shuffle.codec_id, arrays, codecs)
+
+
+# def test_err_decode_object_buffer():
+#     check_err_decode_object_buffer(Shuffle())
+
+
+# def test_err_encode_object_buffer():
+#     check_err_encode_object_buffer(Shuffle())
+
+# def test_decompression_error_handling():
+#     for codec in codecs:
+#         with pytest.raises(RuntimeError):
+#             codec.decode(bytearray())
+#         with pytest.raises(RuntimeError):
+#             codec.decode(bytearray(0))
+
+
+def test_expected_result():
+    # If the input is treated as a 2D byte array, with shape (size of element, number of elements),
+    # the shuffle is essentially a transpose. This can be made more apparent by using an array of
+    # big-endian integers, as below.
+    arr = np.array(
+        [
+            0x0001020304050607,
+            0x08090A0B0C0D0E0F,
+            0x1011121314151617,
+            0x18191A1B1C1D1E1F,
+        ],
+        dtype='>u8',
+    )
+    expected = np.array(
+        [
+            0x00081018,
+            0x01091119,
+            0x020A121A,
+            0x030B131B,
+            0x040C141C,
+            0x050D151D,
+            0x060E161E,
+            0x070F171F,
+        ],
+        dtype='u4',
+    )
+    codec = Shuffle(elementsize=arr.data.itemsize)
+    enc = codec.encode(arr)
+    np.testing.assert_array_equal(np.frombuffer(enc.data, '>u4'), expected)
+
+
+def test_incompatible_elementsize():
+    with pytest.raises(ValueError):
+        arr = np.arange(1001, dtype='u1')
+        codec = Shuffle(elementsize=4)
+        codec.encode(arr)