numcodecs 0.16.4__cp313-cp313-macosx_11_0_arm64.whl

numcodecs/tests/test_compat.py

@@ -0,0 +1,111 @@
+import array
+import mmap
+
+import numpy as np
+import pytest
+
+from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray, ensure_text
+
+
+def test_ensure_text():
+    bufs = [
+        b'adsdasdas',
+        'adsdasdas',
+        np.asarray(memoryview(b'adsdasdas')),
+        array.array('B', b'qwertyuiqwertyui'),
+    ]
+    for buf in bufs:
+        b = ensure_text(buf)
+        assert isinstance(b, str)
+
+
+def test_ensure_bytes():
+    bufs = [
+        b'adsdasdas',
+        bytes(20),
+        np.arange(100),
+        array.array('l', b'qwertyuiqwertyui'),
+    ]
+    for buf in bufs:
+        b = ensure_bytes(buf)
+        assert isinstance(b, bytes)
+
+
+def test_ensure_contiguous_ndarray_shares_memory():
+    typed_bufs = [
+        ('u', 1, b'adsdasdas'),
+        ('u', 1, bytes(20)),
+        ('i', 8, np.arange(100, dtype=np.int64)),
+        ('f', 8, np.linspace(0, 1, 100, dtype=np.float64)),
+        ('i', 4, array.array('i', b'qwertyuiqwertyui')),
+        ('u', 4, array.array('I', b'qwertyuiqwertyui')),
+        ('f', 4, array.array('f', b'qwertyuiqwertyui')),
+        ('f', 8, array.array('d', b'qwertyuiqwertyui')),
+        ('i', 1, array.array('b', b'qwertyuiqwertyui')),
+        ('u', 1, array.array('B', b'qwertyuiqwertyui')),
+        ('u', 1, mmap.mmap(-1, 10)),
+    ]
+    for expected_kind, expected_itemsize, buf in typed_bufs:
+        a = ensure_contiguous_ndarray(buf)
+        assert isinstance(a, np.ndarray)
+        assert expected_kind == a.dtype.kind
+        if isinstance(buf, array.array):
+            assert buf.itemsize == a.dtype.itemsize
+        else:
+            assert expected_itemsize == a.dtype.itemsize
+        assert np.shares_memory(a, memoryview(buf))
+
+
+def test_ensure_bytes_invalid_inputs():
+    # object array not allowed
+    a = np.array(['Xin chào thế giới'], dtype=object)
+    for e in (a, memoryview(a)):
+        with pytest.raises(TypeError):
+            ensure_bytes(e)
+
+
+@pytest.mark.filterwarnings(
+    "ignore:The 'u' type code is deprecated and will be removed in Python 3.16"
+)
+def test_ensure_contiguous_ndarray_invalid_inputs():
+    # object array not allowed
+    a = np.array(['Xin chào thế giới'], dtype=object)
+    for e in (a, memoryview(a)):
+        with pytest.raises(TypeError):
+            ensure_contiguous_ndarray(e)
+
+    # non-contiguous arrays not allowed
+    with pytest.raises(ValueError):
+        ensure_contiguous_ndarray(np.arange(100)[::2])
+
+    # unicode array.array not allowed
+    a = array.array('u', 'qwertyuiqwertyui')
+    with pytest.raises(TypeError):
+        ensure_contiguous_ndarray(a)
+
+
+def test_ensure_contiguous_ndarray_writeable():
+    # check that the writeability of the underlying buffer is preserved
+    for writeable in (False, True):
+        a = np.arange(100)
+        a.setflags(write=writeable)
+        m = ensure_contiguous_ndarray(a)
+        assert m.flags.writeable == writeable
+        m = ensure_contiguous_ndarray(memoryview(a))
+        assert m.flags.writeable == writeable
+
+
+def test_ensure_contiguous_ndarray_max_buffer_size():
+    for max_buffer_size in (4, 64, 1024):
+        ensure_contiguous_ndarray(np.zeros(max_buffer_size - 1, dtype=np.int8), max_buffer_size)
+        ensure_contiguous_ndarray(np.zeros(max_buffer_size, dtype=np.int8), max_buffer_size)
+        buffers = [
+            bytes(b"x" * (max_buffer_size + 1)),
+            np.zeros(max_buffer_size + 1, dtype=np.int8),
+            np.zeros(max_buffer_size + 2, dtype=np.int8),
+            np.zeros(max_buffer_size, dtype=np.int16),
+            np.zeros(max_buffer_size, dtype=np.int32),
+        ]
+        for buf in buffers:
+            with pytest.raises(ValueError):
+                ensure_contiguous_ndarray(buf, max_buffer_size=max_buffer_size)

numcodecs/tests/test_delta.py

@@ -0,0 +1,61 @@
+import numpy as np
+import pytest
+from numpy.testing import assert_array_equal
+
+from numcodecs.delta import Delta
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_repr,
+)
+
+# mix of dtypes: integer, float
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.random.randint(0, 1, size=110, dtype='?').reshape(10, 11),
+    np.arange(1000, dtype='<i4'),
+    np.linspace(1000, 1001, 1000, dtype='<f4').reshape(100, 10),
+    np.random.normal(loc=1000, scale=1, size=(10, 10, 10)).astype('<f8'),
+    np.random.randint(0, 200, size=1000, dtype='u2').astype('<u2').reshape(100, 10, order='F'),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        codec = Delta(dtype=arr.dtype)
+        check_encode_decode(arr, codec)
+
+
+def test_encode():
+    dtype = 'i8'
+    astype = 'i4'
+    codec = Delta(dtype=dtype, astype=astype)
+    arr = np.arange(10, 20, 1, dtype=dtype)
+    expect = np.array([10] + ([1] * 9), dtype=astype)
+    actual = codec.encode(arr)
+    assert_array_equal(expect, actual)
+    assert np.dtype(astype) == actual.dtype
+
+
+def test_config():
+    codec = Delta(dtype='<i4', astype='<i2')
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("Delta(dtype='<i4', astype='<i2')")
+
+
+def test_backwards_compatibility():
+    for arr in arrays:
+        codec = Delta(dtype=arr.dtype)
+        check_backwards_compatibility(Delta.codec_id, [arr], [codec], prefix=str(arr.dtype))
+
+
+def test_errors():
+    with pytest.raises(ValueError):
+        Delta(dtype=object)
+    with pytest.raises(ValueError):
+        Delta(dtype='i8', astype=object)

numcodecs/tests/test_entrypoints.py

@@ -0,0 +1,24 @@
+import os.path
+import sys
+
+import pytest
+
+import numcodecs.registry
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+@pytest.fixture
+def set_path():
+    sys.path.append(here)
+    numcodecs.registry.run_entrypoints()
+    yield
+    sys.path.remove(here)
+    numcodecs.registry.run_entrypoints()
+    numcodecs.registry.codec_registry.pop("test")
+
+
+@pytest.mark.usefixtures("set_path")
+def test_entrypoint_codec():
+    cls = numcodecs.registry.get_codec({"id": "test"})
+    assert cls.codec_id == "test"

numcodecs/tests/test_entrypoints_backport.py

@@ -0,0 +1,36 @@
+import importlib.util
+import os.path
+import sys
+from multiprocessing import Process
+
+import pytest
+
+import numcodecs.registry
+
+importlib_spec = importlib.util.find_spec("importlib_metadata")
+if importlib_spec is None or importlib_spec.loader is None:  # pragma: no cover
+    pytest.skip(
+        "This test module requires importlib_metadata to be installed",
+        allow_module_level=True,
+    )
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+def get_entrypoints_with_importlib_metadata_loaded():
+    # importlib_metadata patches importlib.metadata, which can lead to breaking changes
+    # to the APIs of EntryPoint objects used when registering entrypoints. Attempt to
+    # isolate those changes to just this test.
+    import importlib_metadata  # noqa: F401
+
+    sys.path.append(here)
+    numcodecs.registry.run_entrypoints()
+    cls = numcodecs.registry.get_codec({"id": "test"})
+    assert cls.codec_id == "test"
+
+
+def test_entrypoint_codec_with_importlib_metadata():
+    p = Process(target=get_entrypoints_with_importlib_metadata_loaded)
+    p.start()
+    p.join()
+    assert p.exitcode == 0

numcodecs/tests/test_fixedscaleoffset.py

@@ -0,0 +1,77 @@
+import itertools
+
+import numpy as np
+import pytest
+from numpy.testing import assert_array_equal
+
+from numcodecs.fixedscaleoffset import FixedScaleOffset
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_repr,
+)
+
+arrays = [
+    np.linspace(1000, 1001, 1000, dtype='<f8'),
+    np.random.normal(loc=1000, scale=1, size=1000).astype('<f8'),
+    np.linspace(1000, 1001, 1000, dtype='<f8').reshape(100, 10),
+    np.linspace(1000, 1001, 1000, dtype='<f8').reshape(100, 10, order='F'),
+    np.linspace(1000, 1001, 1000, dtype='<f8').reshape(10, 10, 10),
+]
+
+
+codecs = [
+    FixedScaleOffset(offset=1000, scale=10, dtype='<f8', astype='<i1'),
+    FixedScaleOffset(offset=1000, scale=10**2, dtype='<f8', astype='<i2'),
+    FixedScaleOffset(offset=1000, scale=10**6, dtype='<f8', astype='<i4'),
+    FixedScaleOffset(offset=1000, scale=10**12, dtype='<f8', astype='<i8'),
+    FixedScaleOffset(offset=1000, scale=10**12, dtype='<f8'),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        precision = int(np.log10(codec.scale))
+        check_encode_decode(arr, codec, precision=precision)
+
+
+@pytest.mark.parametrize(
+    ("offset", "scale", "expected"),
+    [
+        (1000, 10, [0, 6, 11, 17, 22, 28, 33, 39, 44, 50]),
+        (1002.5, 10, [-25, -19, -14, -8, -3, 3, 8, 14, 19, 25]),
+        (1000, 0.5, [0, 0, 1, 1, 1, 1, 2, 2, 2, 2]),
+    ],
+)
+def test_encode(offset: float, scale: float, expected: list[int]):
+    dtype = '<f8'
+    astype = np.int16
+    codec = FixedScaleOffset(scale=scale, offset=offset, dtype=dtype, astype=astype)
+    arr = np.linspace(1000, 1005, 10, dtype=dtype)
+    expect = np.array(expected, dtype=astype)
+    actual = codec.encode(arr)
+    assert_array_equal(expect, actual)
+    assert np.dtype(astype) == actual.dtype
+
+
+def test_config():
+    codec = FixedScaleOffset(dtype='<f8', astype='<i4', scale=10, offset=100)
+    check_config(codec)
+
+
+def test_repr():
+    stmt = "FixedScaleOffset(scale=10, offset=100, dtype='<f8', astype='<i4')"
+    check_repr(stmt)
+
+
+def test_backwards_compatibility():
+    precision = [int(np.log10(codec.scale)) for codec in codecs]
+    check_backwards_compatibility(FixedScaleOffset.codec_id, arrays, codecs, precision=precision)
+
+
+def test_errors():
+    with pytest.raises(ValueError):
+        FixedScaleOffset(dtype=object, astype='i4', scale=10, offset=100)
+    with pytest.raises(ValueError):
+        FixedScaleOffset(dtype='f8', astype=object, scale=10, offset=100)

numcodecs/tests/test_fletcher32.py

@@ -0,0 +1,56 @@
+import numpy as np
+import pytest
+
+from numcodecs.fletcher32 import Fletcher32
+
+
+@pytest.mark.parametrize("dtype", ["uint8", "int32", "float32"])
+def test_with_data(dtype):
+    data = np.arange(100, dtype=dtype)
+    f = Fletcher32()
+    arr = np.frombuffer(f.decode(f.encode(data)), dtype=dtype)
+    assert (arr == data).all()
+
+
+def test_error():
+    data = np.arange(100)
+    f = Fletcher32()
+    enc = f.encode(data)
+    enc2 = bytearray(enc)
+    enc2[0] += 1
+    with pytest.raises(RuntimeError) as e:
+        f.decode(enc2)
+    assert "fletcher32 checksum" in str(e.value)
+
+
+def test_known():
+    data = (
+        b'w\x07\x00\x00\x00\x00\x00\x00\x85\xf6\xff\xff\xff\xff\xff\xff'
+        b'i\x07\x00\x00\x00\x00\x00\x00\x94\xf6\xff\xff\xff\xff\xff\xff'
+        b'\x88\t\x00\x00\x00\x00\x00\x00i\x03\x00\x00\x00\x00\x00\x00'
+        b'\x93\xfd\xff\xff\xff\xff\xff\xff\xc3\xfc\xff\xff\xff\xff\xff\xff'
+        b"'\x02\x00\x00\x00\x00\x00\x00\xba\xf7\xff\xff\xff\xff\xff\xff"
+        b'\xfd%\x86d'
+    )
+    data3 = Fletcher32().decode(data)
+    outarr = np.frombuffer(data3, dtype="<i8")
+    expected = [
+        1911,
+        -2427,
+        1897,
+        -2412,
+        2440,
+        873,
+        -621,
+        -829,
+        551,
+        -2118,
+    ]
+    assert outarr.tolist() == expected
+
+
+def test_out():
+    data = np.frombuffer(bytearray(b"Hello World"), dtype="uint8")
+    f = Fletcher32()
+    result = f.encode(data)
+    f.decode(result, out=data)

numcodecs/tests/test_gzip.py

@@ -0,0 +1,110 @@
+import itertools
+
+import numpy as np
+import pytest
+
+from numcodecs.gzip import GZip
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+
+codecs = [
+    GZip(),
+    GZip(level=-1),
+    GZip(level=0),
+    GZip(level=1),
+    GZip(level=5),
+    GZip(level=9),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype='i4'),
+    np.linspace(1000, 1001, 1000, dtype='f8'),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode(arr, codec)
+
+
+def test_config():
+    codec = GZip(level=3)
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("GZip(level=3)")
+
+
+def test_eq():
+    assert GZip() == GZip()
+    assert not GZip() != GZip()
+    assert GZip(1) == GZip(1)
+    assert GZip(1) != GZip(9)
+    assert GZip() != 'foo'
+    assert 'foo' != GZip()
+    assert not GZip() == 'foo'
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(GZip.codec_id, arrays, codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(GZip())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(GZip())
+
+
+def test_err_encode_list():
+    data = ['foo', 'bar', 'baz']
+    for codec in codecs:
+        with pytest.raises(TypeError):
+            codec.encode(data)
+
+
+def test_err_encode_non_contiguous():
+    # non-contiguous memory
+    arr = np.arange(1000, dtype='i4')[::2]
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)
+
+
+def test_err_out_too_small():
+    arr = np.arange(10, dtype='i4')
+    out = np.empty_like(arr)[:-1]
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.decode(codec.encode(arr), out)
+
+
+def test_out_too_large():
+    out = np.empty((10,), dtype='i4')
+    arr = out[:-1]
+    arr[:] = 5
+    for codec in codecs:
+        codec.decode(codec.encode(arr), out)

numcodecs/tests/test_jenkins.py

@@ -0,0 +1,150 @@
+import numpy as np
+import pytest
+
+from numcodecs.checksum32 import JenkinsLookup3
+from numcodecs.jenkins import jenkins_lookup3
+
+
+def test_jenkins_lookup3():
+    h = jenkins_lookup3(b"", 0)
+    assert h == 0xDEADBEEF
+    h = jenkins_lookup3(b"", 0xDEADBEEF)
+    assert h == 0xBD5B7DDE
+    h = jenkins_lookup3(b"Four score and seven years ago", 0)
+    assert h == 0x17770551
+    h = jenkins_lookup3(b"Four score and seven years ago", 1)
+    assert h == 0xCD628161
+
+    # jenkins-cffi example
+    h = jenkins_lookup3(b"jenkins", 0)
+    assert h == 202276345
+
+    h_last = [0]
+
+    h = jenkins_lookup3(b"", h_last[-1])
+    assert h not in h_last
+    h_last.append(h)
+
+    h = jenkins_lookup3(b"", h_last[-1])
+    assert h not in h_last
+    h_last.append(h)
+
+    h = jenkins_lookup3(b"", h_last[-1])
+    assert h not in h_last
+    h_last.append(h)
+
+    h = jenkins_lookup3(b"", h_last[-1])
+    assert h not in h_last
+    h_last.append(h)
+
+    h = jenkins_lookup3(b"", h_last[-1])
+    assert h not in h_last
+    h_last.append(h)
+
+    h = jenkins_lookup3(b"", h_last[-1])
+    assert h not in h_last
+    h_last.append(h)
+
+    h = jenkins_lookup3(b"", h_last[-1])
+    assert h not in h_last
+    h_last.append(h)
+
+    h = jenkins_lookup3(b"", h_last[-1])
+    assert h not in h_last
+    h_last.append(h)
+
+    h = jenkins_lookup3(b"", h_last[-1])
+    assert h not in h_last
+    h_last.append(h)
+
+    a = np.frombuffer(b"Four score and seven years ago", dtype="uint8")
+    h = jenkins_lookup3(a, 0)
+    assert h == 0x17770551
+
+
+def test_jenkins_lookup3_codec():
+    s = b"Four score and seven years ago"
+    j = JenkinsLookup3()
+    result = j.encode(s)
+    assert result[-4:] == b'\x51\x05\x77\x17'
+    assert bytes(j.decode(result)) == s
+
+    j = JenkinsLookup3(initval=0xDEADBEEF)
+    result = j.encode(s)
+    assert bytes(j.decode(result)) == s
+
+    j = JenkinsLookup3(initval=1230)
+    result = j.encode(s)
+    assert result[-4:] == b'\xd7Z\xe2\x0e'
+    assert bytes(j.decode(result)) == s
+
+    j = JenkinsLookup3(initval=1230, prefix=b"Hello world")
+    result = j.encode(s)
+    assert bytes(j.decode(result)) == s
+
+    chunk_index = (
+        b"\x00\x08\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x17\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\xee'\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\xe57\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\xdcG\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\xd3W\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\xcag\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\xc1w\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\xb8\x87\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\xaf\x97\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\xa6\xa7\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\x9d\xb7\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\x94\xc7\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\x8b\xd7\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"\x82\xe7\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"y\xf7\x00\x00\x00\x00\x00\x00"
+        b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
+        b"n\x96\x07\x85"
+    )
+    hdf5_fadb_prefix = b'FADB\x00\x01\xcf\x01\x00\x00\x00\x00\x00\x00'
+    j = JenkinsLookup3(prefix=hdf5_fadb_prefix)
+    result = j.encode(chunk_index[:-4])
+    j.decode(result)
+    assert result == chunk_index
+
+
+@pytest.mark.parametrize("dtype", ["uint8", "int32", "float32"])
+def test_with_data(dtype):
+    data = np.arange(100, dtype=dtype)
+    j = JenkinsLookup3()
+    arr = np.frombuffer(j.decode(j.encode(data)), dtype=dtype)
+    assert (arr == data).all()
+
+
+def test_error():
+    data = np.arange(100)
+    j = JenkinsLookup3()
+    enc = j.encode(data)
+    enc2 = bytearray(enc)
+    enc2[0] += 1
+    with pytest.raises(RuntimeError) as e:
+        j.decode(enc2)
+    assert "Bob Jenkin's lookup3 checksum" in str(e.value)
+
+
+def test_out():
+    data = np.frombuffer(bytearray(b"Hello World"), dtype="uint8")
+    j = JenkinsLookup3()
+    result = j.encode(data)
+    j.decode(result, out=data)