numcodecs 0.16.0__cp311-cp311-macosx_10_13_x86_64.whl

numcodecs/tests/test_shuffle.py

@@ -0,0 +1,166 @@
+from multiprocessing import Pool
+from multiprocessing.pool import ThreadPool
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.shuffle import Shuffle
+except ImportError:  # pragma: no cover
+    pytest.skip("numcodecs.shuffle not available", allow_module_level=True)
+
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+)
+
+codecs = [
+    Shuffle(),
+    Shuffle(elementsize=0),
+    Shuffle(elementsize=4),
+    Shuffle(elementsize=8),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype='i4'),
+    np.linspace(1000, 1001, 1000, dtype='f8'),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
+]
+
+
+@pytest.mark.parametrize('array', arrays)
+@pytest.mark.parametrize('codec', codecs)
+def test_encode_decode(array, codec):
+    check_encode_decode(array, codec)
+
+
+def test_config():
+    codec = Shuffle()
+    check_config(codec)
+    codec = Shuffle(elementsize=8)
+    check_config(codec)
+
+
+def test_repr():
+    expect = "Shuffle(elementsize=0)"
+    actual = repr(Shuffle(elementsize=0))
+    assert expect == actual
+    expect = "Shuffle(elementsize=4)"
+    actual = repr(Shuffle(elementsize=4))
+    assert expect == actual
+    expect = "Shuffle(elementsize=8)"
+    actual = repr(Shuffle(elementsize=8))
+    assert expect == actual
+    expect = "Shuffle(elementsize=16)"
+    actual = repr(Shuffle(elementsize=16))
+    assert expect == actual
+
+
+def test_eq():
+    assert Shuffle() == Shuffle()
+    assert Shuffle(elementsize=16) != Shuffle()
+
+
+def _encode_worker(data):
+    compressor = Shuffle()
+    return compressor.encode(data)
+
+
+def _decode_worker(enc):
+    compressor = Shuffle()
+    return compressor.decode(enc)
+
+
+@pytest.mark.parametrize('pool', [Pool, ThreadPool])
+def test_multiprocessing(pool):
+    data = np.arange(1000000)
+    enc = _encode_worker(data)
+
+    pool = pool(5)
+
+    # test with process pool and thread pool
+
+    # test encoding
+    enc_results = pool.map(_encode_worker, [data] * 5)
+    assert all(len(enc) == len(e) for e in enc_results)
+
+    # test decoding
+    dec_results = pool.map(_decode_worker, [enc] * 5)
+    assert all(data.nbytes == len(d) for d in dec_results)
+
+    # tidy up
+    pool.close()
+    pool.join()
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(Shuffle.codec_id, arrays, codecs)
+
+
+# def test_err_decode_object_buffer():
+#     check_err_decode_object_buffer(Shuffle())
+
+
+# def test_err_encode_object_buffer():
+#     check_err_encode_object_buffer(Shuffle())
+
+# def test_decompression_error_handling():
+#     for codec in codecs:
+#         with pytest.raises(RuntimeError):
+#             codec.decode(bytearray())
+#         with pytest.raises(RuntimeError):
+#             codec.decode(bytearray(0))
+
+
+def test_expected_result():
+    # If the input is treated as a 2D byte array, with shape (size of element, number of elements),
+    # the shuffle is essentially a transpose. This can be made more apparent by using an array of
+    # big-endian integers, as below.
+    arr = np.array(
+        [
+            0x0001020304050607,
+            0x08090A0B0C0D0E0F,
+            0x1011121314151617,
+            0x18191A1B1C1D1E1F,
+        ],
+        dtype='>u8',
+    )
+    expected = np.array(
+        [
+            0x00081018,
+            0x01091119,
+            0x020A121A,
+            0x030B131B,
+            0x040C141C,
+            0x050D151D,
+            0x060E161E,
+            0x070F171F,
+        ],
+        dtype='u4',
+    )
+    codec = Shuffle(elementsize=arr.data.itemsize)
+    enc = codec.encode(arr)
+    np.testing.assert_array_equal(np.frombuffer(enc.data, '>u4'), expected)
+
+
+def test_incompatible_elementsize():
+    arr = np.arange(1001, dtype='u1')
+    codec = Shuffle(elementsize=4)
+    with pytest.raises(ValueError):
+        codec.encode(arr)

numcodecs/tests/test_vlen_array.py

@@ -0,0 +1,97 @@
+import unittest
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.vlen import VLenArray
+except ImportError as e:  # pragma: no cover
+    raise unittest.SkipTest("vlen-array not available") from e
+from numcodecs.tests.common import (
+    assert_array_items_equal,
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+)
+
+arrays = [
+    np.array([np.array([1, 2, 3]), np.array([4]), np.array([5, 6])] * 300, dtype=object),
+    np.array([np.array([1, 2, 3]), np.array([4]), np.array([5, 6])] * 300, dtype=object).reshape(
+        90, 10
+    ),
+]
+
+
+codecs = [
+    VLenArray('<i1'),
+    VLenArray('<i2'),
+    VLenArray('<i4'),
+    VLenArray('<i8'),
+    VLenArray('<u1'),
+    VLenArray('<u2'),
+    VLenArray('<u4'),
+    VLenArray('<u8'),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        for codec in codecs:
+            check_encode_decode_array(arr, codec)
+
+
+def test_config():
+    codec = VLenArray('<i8')
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("VLenArray(dtype='<i8')")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(VLenArray.codec_id, arrays, codecs)
+
+
+def test_encode_errors():
+    codec = VLenArray('<i8')
+    with pytest.raises(ValueError):
+        codec.encode('foo')
+    with pytest.raises(ValueError):
+        codec.encode(['foo', 'bar'])
+
+
+def test_decode_errors():
+    codec = VLenArray('<i8')
+    with pytest.raises(TypeError):
+        codec.decode(1234)
+    # these should look like corrupt data
+    with pytest.raises(ValueError):
+        codec.decode(b'foo')
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(2, 3, dtype='i4'))
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(10, 20, dtype='i4'))
+    with pytest.raises(TypeError):
+        codec.decode('foo')
+
+    # test out parameter
+    enc = codec.encode(arrays[0])
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=b'foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out='foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=123)
+    with pytest.raises(ValueError):
+        codec.decode(enc, out=np.zeros(10, dtype='i4'))
+
+
+def test_encode_none():
+    a = np.array([[1, 3], None, [4, 7]], dtype=object)
+    codec = VLenArray(int)
+    enc = codec.encode(a)
+    dec = codec.decode(enc)
+    expect = np.array([np.array([1, 3]), np.array([]), np.array([4, 7])], dtype=object)
+    assert_array_items_equal(expect, dec)

numcodecs/tests/test_vlen_bytes.py

@@ -0,0 +1,97 @@
+import sys
+import unittest
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.vlen import VLenBytes
+except ImportError as e:  # pragma: no cover
+    raise unittest.SkipTest("vlen-bytes not available") from e
+from numcodecs.tests.common import (
+    assert_array_items_equal,
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+    greetings,
+)
+
+greetings_bytes = [g.encode('utf-8') for g in greetings]
+
+
+arrays = [
+    np.array([b'foo', b'bar', b'baz'] * 300, dtype=object),
+    np.array(greetings_bytes * 100, dtype=object),
+    np.array([b'foo', b'bar', b'baz'] * 300, dtype=object).reshape(90, 10),
+    np.array(greetings_bytes * 1000, dtype=object).reshape(
+        len(greetings_bytes), 100, 10, order='F'
+    ),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        codec = VLenBytes()
+        check_encode_decode_array(arr, codec)
+
+
+def test_config():
+    codec = VLenBytes()
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("VLenBytes()")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(VLenBytes.codec_id, arrays, [VLenBytes()])
+
+
+def test_encode_errors():
+    codec = VLenBytes()
+    with pytest.raises(TypeError):
+        codec.encode(1234)
+    with pytest.raises(TypeError):
+        codec.encode([1234, 5678])
+    with pytest.raises(TypeError):
+        codec.encode(np.ones(10, dtype='i4'))
+
+
+def test_decode_errors():
+    codec = VLenBytes()
+    with pytest.raises(TypeError):
+        codec.decode(1234)
+    # these should look like corrupt data
+    with pytest.raises(ValueError):
+        codec.decode(b'foo')
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(2, 3, dtype='i4'))
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(10, 20, dtype='i4'))
+    with pytest.raises(TypeError):
+        codec.decode('foo')
+
+    # test out parameter
+    enc = codec.encode(arrays[0])
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=b'foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out='foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=123)
+    with pytest.raises(ValueError):
+        codec.decode(enc, out=np.zeros(10, dtype='i4'))
+
+
+# TODO: fix this test on GitHub actions somehow...
+# See https://github.com/zarr-developers/numcodecs/issues/683
+@pytest.mark.skipif(sys.platform == "darwin", reason="Test is failing on macOS on GitHub actions.")
+def test_encode_none():
+    a = np.array([b'foo', None, b'bar'], dtype=object)
+    codec = VLenBytes()
+    enc = codec.encode(a)
+    dec = codec.decode(enc)
+    expect = np.array([b'foo', b'', b'bar'], dtype=object)
+    assert_array_items_equal(expect, dec)

numcodecs/tests/test_vlen_utf8.py

@@ -0,0 +1,91 @@
+import unittest
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.vlen import VLenUTF8
+except ImportError as e:  # pragma: no cover
+    raise unittest.SkipTest("vlen-utf8 not available") from e
+from numcodecs.tests.common import (
+    assert_array_items_equal,
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+    greetings,
+)
+
+arrays = [
+    np.array(['foo', 'bar', 'baz'] * 300, dtype=object),
+    np.array(greetings * 100, dtype=object),
+    np.array(['foo', 'bar', 'baz'] * 300, dtype=object).reshape(90, 10),
+    np.array(greetings * 1000, dtype=object).reshape(len(greetings), 100, 10, order='F'),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        codec = VLenUTF8()
+        check_encode_decode_array(arr, codec)
+
+
+def test_config():
+    codec = VLenUTF8()
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("VLenUTF8()")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(VLenUTF8.codec_id, arrays, [VLenUTF8()])
+
+
+def test_encode_errors():
+    codec = VLenUTF8()
+    with pytest.raises(TypeError):
+        codec.encode(1234)
+    with pytest.raises(TypeError):
+        codec.encode([1234, 5678])
+    with pytest.raises(TypeError):
+        codec.encode(np.ones(10, dtype='i4'))
+
+
+def test_decode_errors():
+    codec = VLenUTF8()
+    with pytest.raises(TypeError):
+        codec.decode(1234)
+    # these should look like corrupt data
+    with pytest.raises(ValueError):
+        codec.decode(b'foo')
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(2, 3, dtype='i4'))
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(10, 20, dtype='i4'))
+    with pytest.raises(TypeError):
+        codec.decode('foo')
+
+    # test out parameter
+    enc = codec.encode(arrays[0])
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=b'foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out='foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=123)
+    with pytest.raises(ValueError):
+        codec.decode(enc, out=np.zeros(10, dtype='i4'))
+
+
+@pytest.mark.parametrize("writable", [True, False])
+def test_encode_utf8(writable):
+    a = np.array(['foo', None, 'bar'], dtype=object)
+    if not writable:
+        a.setflags(write=False)
+    codec = VLenUTF8()
+    enc = codec.encode(a)
+    dec = codec.decode(enc)
+    expect = np.array(['foo', '', 'bar'], dtype=object)
+    assert_array_items_equal(expect, dec)