numcodecs 0.16.0__cp311-cp311-macosx_10_13_x86_64.whl

numcodecs/tests/test_lzma.py

@@ -0,0 +1,94 @@
+import itertools
+import unittest
+from types import ModuleType
+from typing import cast
+
+import numpy as np
+import pytest
+
+try:
+    # noinspection PyProtectedMember
+    from numcodecs.lzma import LZMA, _lzma
+except ImportError as e:  # pragma: no cover
+    raise unittest.SkipTest("LZMA not available") from e
+
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+
+_lzma = cast(ModuleType, _lzma)
+
+codecs = [
+    LZMA(),
+    LZMA(preset=1),
+    LZMA(preset=5),
+    LZMA(preset=9),
+    LZMA(format=_lzma.FORMAT_RAW, filters=[{"id": _lzma.FILTER_LZMA2, "preset": 1}]),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype='i4'),
+    np.linspace(1000, 1001, 1000, dtype='f8'),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode(arr, codec)
+
+
+def test_config():
+    codec = LZMA(preset=1, format=_lzma.FORMAT_XZ, check=_lzma.CHECK_NONE, filters=None)
+    check_config(codec)
+
+
+def test_repr():
+    check_repr('LZMA(format=1, check=0, preset=1, filters=None)')
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(LZMA.codec_id, arrays, codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(LZMA())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(LZMA())
+
+
+def test_err_encode_list():
+    data = ['foo', 'bar', 'baz']
+    for codec in codecs:
+        with pytest.raises(TypeError):
+            codec.encode(data)
+
+
+def test_err_encode_non_contiguous():
+    # non-contiguous memory
+    arr = np.arange(1000, dtype='i4')[::2]
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)

numcodecs/tests/test_msgpacks.py

@@ -0,0 +1,126 @@
+import unittest
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.msgpacks import MsgPack
+except ImportError as e:  # pragma: no cover
+    raise unittest.SkipTest("msgpack not available") from e
+
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+    greetings,
+)
+
+# object array with strings
+# object array with mix strings / nans
+# object array with mix of string, int, float
+# ...
+arrays = [
+    np.array(['foo', 'bar', 'baz'] * 300, dtype=object),
+    np.array([['foo', 'bar', np.nan]] * 300, dtype=object),
+    np.array(['foo', 1.0, 2] * 300, dtype=object),
+    np.arange(1000, dtype='i4'),
+    np.array(['foo', 'bar', 'baz'] * 300),
+    np.array(['foo', ['bar', 1.0, 2], {'a': 'b', 'c': 42}] * 300, dtype=object),
+    np.array(greetings * 100),
+    np.array(greetings * 100, dtype=object),
+    np.array([b'foo', b'bar', b'baz'] * 300, dtype=object),
+    np.array([g.encode('utf-8') for g in greetings] * 100, dtype=object),
+    np.array([[0, 1], [2, 3]], dtype=object),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        check_encode_decode_array(arr, MsgPack())
+
+
+def test_config():
+    check_config(MsgPack())
+
+
+def test_repr():
+    check_repr("MsgPack(raw=False, use_bin_type=True, use_single_float=False)")
+    check_repr("MsgPack(raw=True, use_bin_type=False, use_single_float=True)")
+
+
+def test_backwards_compatibility():
+    codec = MsgPack()
+    check_backwards_compatibility(codec.codec_id, arrays, [codec])
+
+
+@pytest.mark.filterwarnings(
+    "ignore:Creating an ndarray from ragged nested sequences .* is deprecated.*"
+)
+@pytest.mark.parametrize(
+    ("input_data", "dtype"),
+    [
+        ([0, 1], None),
+        ([[0, 1], [2, 3]], None),
+        ([[0], [1], [2, 3]], object),
+        ([[[0, 0]], [[1, 1]], [[2, 3]]], None),
+        (["1"], None),
+        (["11", "11"], None),
+        (["11", "1", "1"], None),
+        ([{}], None),
+        ([{"key": "value"}, ["list", "of", "strings"]], object),
+        ([b"1"], None),
+        ([b"11", b"11"], None),
+        ([b"11", b"1", b"1"], None),
+        ([{b"key": b"value"}, [b"list", b"of", b"strings"]], object),
+    ],
+)
+def test_non_numpy_inputs(input_data, dtype):
+    codec = MsgPack()
+    # numpy will infer a range of different shapes and dtypes for these inputs.
+    # Make sure that round-tripping through encode preserves this.
+    actual = codec.decode(codec.encode(input_data))
+    expect = np.array(input_data, dtype=dtype)
+    assert expect.shape == actual.shape
+    assert np.array_equal(expect, actual)
+
+
+def test_encode_decode_shape_dtype_preserved():
+    codec = MsgPack()
+    for arr in arrays:
+        actual = codec.decode(codec.encode(arr))
+        assert arr.shape == actual.shape
+        assert arr.dtype == actual.dtype
+
+
+def test_bytes():
+    # test msgpack behaviour with bytes and str (unicode)
+    bytes_arr = np.array([b'foo', b'bar', b'baz'], dtype=object)
+    unicode_arr = np.array(['foo', 'bar', 'baz'], dtype=object)
+
+    # raw=False (default)
+    codec = MsgPack()
+    # works for bytes array, round-trips bytes to bytes
+    b = codec.decode(codec.encode(bytes_arr))
+    assert np.array_equal(bytes_arr, b)
+    assert isinstance(b[0], bytes)
+    assert b[0] == b'foo'
+    # works for unicode array, round-trips unicode to unicode
+    b = codec.decode(codec.encode(unicode_arr))
+    assert np.array_equal(unicode_arr, b)
+    assert isinstance(b[0], str)
+    assert b[0] == 'foo'
+
+    # raw=True
+    codec = MsgPack(raw=True)
+    # works for bytes array, round-trips bytes to bytes
+    b = codec.decode(codec.encode(bytes_arr))
+    assert np.array_equal(bytes_arr, b)
+    assert isinstance(b[0], bytes)
+    assert b[0] == b'foo'
+    # broken for unicode array, round-trips unicode to bytes
+    b = codec.decode(codec.encode(unicode_arr))
+    assert not np.array_equal(unicode_arr, b)
+    assert isinstance(b[0], bytes)
+    assert b[0] == b'foo'

numcodecs/tests/test_ndarray_like.py

@@ -0,0 +1,48 @@
+import pytest
+
+from numcodecs.ndarray_like import DType, FlagsObj, NDArrayLike
+
+
+@pytest.mark.parametrize("module", ["numpy", "cupy"])
+def test_is_ndarray_like(module):
+    m = pytest.importorskip(module)
+    a = m.arange(10)
+    assert isinstance(a, NDArrayLike)
+
+
+def test_is_not_ndarray_like():
+    assert not isinstance([1, 2, 3], NDArrayLike)
+    assert not isinstance(b"1,2,3", NDArrayLike)
+
+
+@pytest.mark.parametrize("module", ["numpy", "cupy"])
+def test_is_dtype_like(module):
+    m = pytest.importorskip(module)
+    d = m.dtype("u8")
+    assert isinstance(d, DType)
+
+
+def test_is_not_dtype_like():
+    assert not isinstance([1, 2, 3], DType)
+    assert not isinstance(b"1,2,3", DType)
+
+
+@pytest.mark.parametrize("module", ["numpy", "cupy"])
+def test_is_flags_like(module):
+    m = pytest.importorskip(module)
+    d = m.arange(10).flags
+    assert isinstance(d, FlagsObj)
+
+
+def test_is_not_flags_like():
+    assert not isinstance([1, 2, 3], FlagsObj)
+    assert not isinstance(b"1,2,3", FlagsObj)
+
+
+@pytest.mark.parametrize("module", ["numpy", "cupy"])
+def test_cached_isinstance_check(module):
+    m = pytest.importorskip(module)
+    a = m.arange(10)
+    assert isinstance(a, NDArrayLike)
+    assert not isinstance(a, DType)
+    assert not isinstance(a, FlagsObj)

numcodecs/tests/test_packbits.py

@@ -0,0 +1,39 @@
+import numpy as np
+
+from numcodecs.packbits import PackBits
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_repr,
+)
+
+arrays = [
+    np.random.randint(0, 2, size=1000, dtype=bool),
+    np.random.randint(0, 2, size=(100, 10), dtype=bool),
+    np.random.randint(0, 2, size=(10, 10, 10), dtype=bool),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(10, 10, 10, order='F'),
+]
+
+
+def test_encode_decode():
+    codec = PackBits()
+    for arr in arrays:
+        check_encode_decode(arr, codec)
+    # check different number of left-over bits
+    arr = np.random.randint(0, 2, size=1000, dtype=bool)
+    for size in list(range(1, 17)):
+        check_encode_decode(arr[:size], codec)
+
+
+def test_config():
+    codec = PackBits()
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("PackBits()")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(PackBits.codec_id, arrays, [PackBits()])

numcodecs/tests/test_pcodec.py

@@ -0,0 +1,90 @@
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.pcodec import PCodec
+except ImportError:  # pragma: no cover
+    pytest.skip("pcodec not available", allow_module_level=True)
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array_to_bytes,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+
+codecs = [
+    PCodec(),
+    PCodec(level=1),
+    PCodec(level=5),
+    PCodec(level=9),
+    PCodec(mode_spec="classic"),
+    PCodec(equal_pages_up_to=300),
+    PCodec(delta_encoding_order=2),
+    PCodec(delta_spec="try_lookback"),
+    PCodec(delta_spec="none"),
+    PCodec(delta_spec="try_consecutive", delta_encoding_order=1),
+]
+
+
+# mix of dtypes: integer, float
+# mix of shapes: 1D, 2D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype="u4"),
+    np.arange(1000, dtype="u8"),
+    np.arange(1000, dtype="i4"),
+    np.arange(1000, dtype="i8"),
+    np.linspace(1000, 1001, 1000, dtype="f4"),
+    np.linspace(1000, 1001, 1000, dtype="f8"),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(100, 10))),
+    np.random.randint(0, 2**60, size=1000, dtype="u8"),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype="i8"),
+]
+
+
+@pytest.mark.parametrize("arr", arrays)
+@pytest.mark.parametrize("codec", codecs)
+def test_encode_decode(arr, codec):
+    check_encode_decode_array_to_bytes(arr, codec)
+
+
+def test_config():
+    codec = PCodec(level=3)
+    check_config(codec)
+
+
+@pytest.mark.parametrize("param", ["mode_spec", "delta_spec", "paging_spec"])
+def test_invalid_config_error(param):
+    codec = PCodec(**{param: "bogus"})
+    with pytest.raises(ValueError):
+        check_encode_decode_array_to_bytes(arrays[0], codec)
+
+
+def test_invalid_delta_encoding_combo():
+    codec = PCodec(delta_encoding_order=2, delta_spec="none")
+    with pytest.raises(ValueError):
+        check_encode_decode_array_to_bytes(arrays[0], codec)
+
+
+def test_repr():
+    check_repr(
+        "PCodec(delta_encoding_order=None, delta_spec='auto',"
+        " equal_pages_up_to=262144, level=3, mode_spec='auto',"
+        " paging_spec='equal_pages_up_to')"
+    )
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(PCodec.codec_id, arrays, codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(PCodec())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(PCodec())

numcodecs/tests/test_pickles.py

@@ -0,0 +1,61 @@
+import itertools
+import sys
+
+import numpy as np
+import pytest
+
+from numcodecs.pickles import Pickle
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+    greetings,
+)
+
+codecs = [Pickle(protocol=i) for i in range(5)]
+
+# object array with strings
+# object array with mix strings / nans
+# object array with mix of string, int, float
+# ...
+arrays = [
+    np.array(['foo', 'bar', 'baz'] * 300, dtype=object),
+    np.array([['foo', 'bar', np.nan]] * 300, dtype=object),
+    np.array(['foo', 1.0, 2] * 300, dtype=object),
+    np.arange(1000, dtype='i4'),
+    np.array(['foo', 'bar', 'baz'] * 300),
+    np.array(['foo', ['bar', 1.0, 2], {'a': 'b', 'c': 42}] * 300, dtype=object),
+    np.array(greetings * 100),
+    np.array(greetings * 100, dtype=object),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode_array(arr, codec)
+
+
+def test_config():
+    codec = Pickle(protocol=-1)
+    check_config(codec)
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("Pickle(protocol=-1)")
+
+
+# Details on xfail
+# https://stackoverflow.com/questions/58194852/modulenotfounderror-no-module-named-numpy-core-multiarray-r
+@pytest.mark.skipif(sys.byteorder != "little", reason="Pickle does not restore byte orders")
+@pytest.mark.xfail(
+    sys.platform == "win32",
+    reason=(
+        "Pickle fails to read w/ Windows carriage return "
+        "( https://github.com/zarr-developers/numcodecs/issues/271 )"
+    ),
+)
+def test_backwards_compatibility():
+    check_backwards_compatibility(Pickle.codec_id, arrays, codecs)

numcodecs/tests/test_quantize.py

@@ -0,0 +1,76 @@
+import itertools
+
+import numpy as np
+import pytest
+from numpy.testing import assert_array_almost_equal, assert_array_equal
+
+from numcodecs.quantize import Quantize
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_repr,
+)
+
+arrays = [
+    np.linspace(100, 200, 1000, dtype='<f8'),
+    np.random.normal(loc=0, scale=1, size=1000).astype('<f8'),
+    np.linspace(100, 200, 1000, dtype='<f8').reshape(100, 10),
+    np.linspace(100, 200, 1000, dtype='<f8').reshape(100, 10, order='F'),
+    np.linspace(100, 200, 1000, dtype='<f8').reshape(10, 10, 10),
+]
+
+
+codecs = [
+    Quantize(digits=-1, dtype='<f8', astype='<f2'),
+    Quantize(digits=0, dtype='<f8', astype='<f2'),
+    Quantize(digits=1, dtype='<f8', astype='<f2'),
+    Quantize(digits=5, dtype='<f8', astype='<f4'),
+    Quantize(digits=12, dtype='<f8', astype='<f8'),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode(arr, codec, precision=codec.digits)
+
+
+def test_encode():
+    for arr, codec in itertools.product(arrays, codecs):
+        if arr.flags.f_contiguous:
+            order = 'F'
+        else:
+            order = 'C'
+        enc = codec.encode(arr).reshape(arr.shape, order=order)
+        assert_array_almost_equal(arr, enc, decimal=codec.digits)
+
+
+def test_decode():
+    # decode is a no-op
+    for arr, codec in itertools.product(arrays, codecs):
+        enc = codec.encode(arr)
+        dec = codec.decode(enc)
+        assert_array_equal(enc, dec)
+
+
+def test_config():
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("Quantize(digits=2, dtype='<f8', astype='<f2')")
+
+
+def test_errors():
+    with pytest.raises(ValueError):
+        Quantize(digits=2, dtype='i4')
+    with pytest.raises(ValueError):
+        Quantize(digits=2, dtype=object)
+    with pytest.raises(ValueError):
+        Quantize(digits=2, dtype='f8', astype=object)
+
+
+def test_backwards_compatibility():
+    precision = [codec.digits for codec in codecs]
+    check_backwards_compatibility(Quantize.codec_id, arrays, codecs, precision=precision)

numcodecs/tests/test_registry.py

@@ -0,0 +1,43 @@
+import inspect
+
+import pytest
+
+import numcodecs
+from numcodecs.errors import UnknownCodecError
+from numcodecs.registry import get_codec
+
+
+def test_registry_errors():
+    with pytest.raises(UnknownCodecError, match='foo'):
+        get_codec({'id': 'foo'})
+
+
+def test_get_codec_argument():
+    # Check that get_codec doesn't modify its argument.
+    arg = {"id": "json2"}
+    before = dict(arg)
+    get_codec(arg)
+    assert before == arg
+
+
+def test_all_classes_registered():
+    """
+    find all Codec subclasses in this repository and check that they
+    have been registered.
+
+    see #346 for more info
+    """
+    missing = {
+        obj.codec_id
+        for _, submod in inspect.getmembers(numcodecs, inspect.ismodule)
+        for _, obj in inspect.getmembers(submod)
+        if (
+            inspect.isclass(obj)
+            and issubclass(obj, numcodecs.abc.Codec)
+            and obj.codec_id not in numcodecs.registry.codec_registry
+            and obj.codec_id is not None  # remove `None`
+        )
+    }
+
+    if missing:
+        raise Exception(f"these codecs are missing: {missing}")  # pragma: no cover