numcodecs 0.13.1__cp313-cp313-win_amd64.whl

numcodecs/tests/test_vlen_array.py

@@ -0,0 +1,97 @@
+import unittest
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.vlen import VLenArray
+except ImportError as e:  # pragma: no cover
+    raise unittest.SkipTest("vlen-array not available") from e
+from numcodecs.tests.common import (
+    assert_array_items_equal,
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+)
+
+arrays = [
+    np.array([np.array([1, 2, 3]), np.array([4]), np.array([5, 6])] * 300, dtype=object),
+    np.array([np.array([1, 2, 3]), np.array([4]), np.array([5, 6])] * 300, dtype=object).reshape(
+        90, 10
+    ),
+]
+
+
+codecs = [
+    VLenArray('<i1'),
+    VLenArray('<i2'),
+    VLenArray('<i4'),
+    VLenArray('<i8'),
+    VLenArray('<u1'),
+    VLenArray('<u2'),
+    VLenArray('<u4'),
+    VLenArray('<u8'),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        for codec in codecs:
+            check_encode_decode_array(arr, codec)
+
+
+def test_config():
+    codec = VLenArray('<i8')
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("VLenArray(dtype='<i8')")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(VLenArray.codec_id, arrays, codecs)
+
+
+def test_encode_errors():
+    codec = VLenArray('<i8')
+    with pytest.raises(ValueError):
+        codec.encode('foo')
+    with pytest.raises(ValueError):
+        codec.encode(['foo', 'bar'])
+
+
+def test_decode_errors():
+    codec = VLenArray('<i8')
+    with pytest.raises(TypeError):
+        codec.decode(1234)
+    # these should look like corrupt data
+    with pytest.raises(ValueError):
+        codec.decode(b'foo')
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(2, 3, dtype='i4'))
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(10, 20, dtype='i4'))
+    with pytest.raises(TypeError):
+        codec.decode('foo')
+
+    # test out parameter
+    enc = codec.encode(arrays[0])
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=b'foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out='foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=123)
+    with pytest.raises(ValueError):
+        codec.decode(enc, out=np.zeros(10, dtype='i4'))
+
+
+def test_encode_none():
+    a = np.array([[1, 3], None, [4, 7]], dtype=object)
+    codec = VLenArray(int)
+    enc = codec.encode(a)
+    dec = codec.decode(enc)
+    expect = np.array([np.array([1, 3]), np.array([]), np.array([4, 7])], dtype=object)
+    assert_array_items_equal(expect, dec)

numcodecs/tests/test_vlen_bytes.py

@@ -0,0 +1,93 @@
+import unittest
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.vlen import VLenBytes
+except ImportError as e:  # pragma: no cover
+    raise unittest.SkipTest("vlen-bytes not available") from e
+from numcodecs.tests.common import (
+    assert_array_items_equal,
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+    greetings,
+)
+
+greetings_bytes = [g.encode('utf-8') for g in greetings]
+
+
+arrays = [
+    np.array([b'foo', b'bar', b'baz'] * 300, dtype=object),
+    np.array(greetings_bytes * 100, dtype=object),
+    np.array([b'foo', b'bar', b'baz'] * 300, dtype=object).reshape(90, 10),
+    np.array(greetings_bytes * 1000, dtype=object).reshape(
+        len(greetings_bytes), 100, 10, order='F'
+    ),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        codec = VLenBytes()
+        check_encode_decode_array(arr, codec)
+
+
+def test_config():
+    codec = VLenBytes()
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("VLenBytes()")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(VLenBytes.codec_id, arrays, [VLenBytes()])
+
+
+def test_encode_errors():
+    codec = VLenBytes()
+    with pytest.raises(TypeError):
+        codec.encode(1234)
+    with pytest.raises(TypeError):
+        codec.encode([1234, 5678])
+    with pytest.raises(TypeError):
+        codec.encode(np.ones(10, dtype='i4'))
+
+
+def test_decode_errors():
+    codec = VLenBytes()
+    with pytest.raises(TypeError):
+        codec.decode(1234)
+    # these should look like corrupt data
+    with pytest.raises(ValueError):
+        codec.decode(b'foo')
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(2, 3, dtype='i4'))
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(10, 20, dtype='i4'))
+    with pytest.raises(TypeError):
+        codec.decode('foo')
+
+    # test out parameter
+    enc = codec.encode(arrays[0])
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=b'foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out='foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=123)
+    with pytest.raises(ValueError):
+        codec.decode(enc, out=np.zeros(10, dtype='i4'))
+
+
+def test_encode_none():
+    a = np.array([b'foo', None, b'bar'], dtype=object)
+    codec = VLenBytes()
+    enc = codec.encode(a)
+    dec = codec.decode(enc)
+    expect = np.array([b'foo', b'', b'bar'], dtype=object)
+    assert_array_items_equal(expect, dec)

numcodecs/tests/test_vlen_utf8.py

@@ -0,0 +1,91 @@
+import unittest
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.vlen import VLenUTF8
+except ImportError as e:  # pragma: no cover
+    raise unittest.SkipTest("vlen-utf8 not available") from e
+from numcodecs.tests.common import (
+    assert_array_items_equal,
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_repr,
+    greetings,
+)
+
+arrays = [
+    np.array(['foo', 'bar', 'baz'] * 300, dtype=object),
+    np.array(greetings * 100, dtype=object),
+    np.array(['foo', 'bar', 'baz'] * 300, dtype=object).reshape(90, 10),
+    np.array(greetings * 1000, dtype=object).reshape(len(greetings), 100, 10, order='F'),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        codec = VLenUTF8()
+        check_encode_decode_array(arr, codec)
+
+
+def test_config():
+    codec = VLenUTF8()
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("VLenUTF8()")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(VLenUTF8.codec_id, arrays, [VLenUTF8()])
+
+
+def test_encode_errors():
+    codec = VLenUTF8()
+    with pytest.raises(TypeError):
+        codec.encode(1234)
+    with pytest.raises(TypeError):
+        codec.encode([1234, 5678])
+    with pytest.raises(TypeError):
+        codec.encode(np.ones(10, dtype='i4'))
+
+
+def test_decode_errors():
+    codec = VLenUTF8()
+    with pytest.raises(TypeError):
+        codec.decode(1234)
+    # these should look like corrupt data
+    with pytest.raises(ValueError):
+        codec.decode(b'foo')
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(2, 3, dtype='i4'))
+    with pytest.raises(ValueError):
+        codec.decode(np.arange(10, 20, dtype='i4'))
+    with pytest.raises(TypeError):
+        codec.decode('foo')
+
+    # test out parameter
+    enc = codec.encode(arrays[0])
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=b'foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out='foo')
+    with pytest.raises(TypeError):
+        codec.decode(enc, out=123)
+    with pytest.raises(ValueError):
+        codec.decode(enc, out=np.zeros(10, dtype='i4'))
+
+
+@pytest.mark.parametrize("writable", [True, False])
+def test_encode_utf8(writable):
+    a = np.array(['foo', None, 'bar'], dtype=object)
+    if not writable:
+        a.setflags(write=False)
+    codec = VLenUTF8()
+    enc = codec.encode(a)
+    dec = codec.decode(enc)
+    expect = np.array(['foo', '', 'bar'], dtype=object)
+    assert_array_items_equal(expect, dec)

numcodecs/tests/test_zfpy.py

@@ -0,0 +1,98 @@
+import numpy as np
+import pytest
+
+try:
+    # noinspection PyProtectedMember
+    from numcodecs.zfpy import ZFPY, _zfpy
+except ImportError:  # pragma: no cover
+    pytest.skip("ZFPY not available", allow_module_level=True)
+
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode_array,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+
+codecs = [
+    ZFPY(mode=_zfpy.mode_fixed_rate, rate=-1),
+    ZFPY(),
+    ZFPY(mode=_zfpy.mode_fixed_accuracy, tolerance=-1),
+    ZFPY(mode=_zfpy.mode_fixed_precision, precision=-1),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.linspace(1000, 1001, 1000, dtype="f4"),
+    np.linspace(1000, 1001, 1000, dtype="f8"),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.normal(loc=1000, scale=1, size=(10, 10, 10)),
+    np.random.normal(loc=1000, scale=1, size=(2, 5, 10, 10)),
+    np.random.randint(-(2**31), -(2**31) + 20, size=1000, dtype="i4").reshape(100, 10),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype="i8").reshape(10, 10, 10),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        if arr.dtype in (np.int32, np.int64):
+            codec = [codecs[-1]]
+        else:
+            codec = codecs
+        for code in codec:
+            check_encode_decode_array(arr, code)
+
+
+def test_config():
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("ZFPY(mode=4, tolerance=0.001, rate=-1, precision=-1)")
+
+
+def test_backwards_compatibility():
+    for code in codecs:
+        if code.mode == _zfpy.mode_fixed_rate:
+            codec = [code]
+            check_backwards_compatibility(ZFPY.codec_id, arrays, codec)
+        else:
+            check_backwards_compatibility(ZFPY.codec_id, arrays[: len(arrays) - 2], codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(ZFPY())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(ZFPY())
+
+
+def test_err_encode_list():
+    data = ['foo', 'bar', 'baz']
+    for codec in codecs:
+        with pytest.raises(TypeError):
+            codec.encode(data)
+
+
+def test_err_encode_non_contiguous():
+    # non-contiguous memory
+    arr = np.arange(1000, dtype='i4')[::2]
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)
+
+
+def test_err_encode_fortran_array():
+    # fortran array
+    arr = np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(5, 10, 20)))
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)

numcodecs/tests/test_zlib.py

@@ -0,0 +1,94 @@
+import itertools
+
+import numpy as np
+import pytest
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+from numcodecs.zlib import Zlib
+
+codecs = [
+    Zlib(),
+    Zlib(level=-1),
+    Zlib(level=0),
+    Zlib(level=1),
+    Zlib(level=5),
+    Zlib(level=9),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype='i4'),
+    np.linspace(1000, 1001, 1000, dtype='f8'),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode(arr, codec)
+
+
+def test_config():
+    codec = Zlib(level=3)
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("Zlib(level=3)")
+
+
+def test_eq():
+    assert Zlib() == Zlib()
+    assert not Zlib() != Zlib()
+    assert Zlib(1) == Zlib(1)
+    assert Zlib(1) != Zlib(9)
+    assert Zlib() != 'foo'
+    assert 'foo' != Zlib()
+    assert not Zlib() == 'foo'
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(Zlib.codec_id, arrays, codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(Zlib())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(Zlib())
+
+
+def test_err_encode_list():
+    data = ['foo', 'bar', 'baz']
+    for codec in codecs:
+        with pytest.raises(TypeError):
+            codec.encode(data)
+
+
+def test_err_encode_non_contiguous():
+    # non-contiguous memory
+    arr = np.arange(1000, dtype='i4')[::2]
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)

numcodecs/tests/test_zstd.py

@@ -0,0 +1,92 @@
+import itertools
+
+import numpy as np
+import pytest
+
+try:
+    from numcodecs.zstd import Zstd
+except ImportError:  # pragma: no cover
+    pytest.skip("numcodecs.zstd not available", allow_module_level=True)
+
+
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+
+codecs = [
+    Zstd(),
+    Zstd(level=-1),
+    Zstd(level=0),
+    Zstd(level=1),
+    Zstd(level=10),
+    Zstd(level=22),
+    Zstd(level=100),
+    Zstd(checksum=True),
+    Zstd(level=0, checksum=True),
+    Zstd(level=22, checksum=True),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype="i4"),
+    np.linspace(1000, 1001, 1000, dtype="f8"),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode(arr, codec)
+
+
+def test_config():
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("Zstd(level=3)")
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(Zstd.codec_id, arrays, codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(Zstd())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(Zstd())
+
+
+def test_checksum():
+    data = np.arange(0, 64, dtype="uint8")
+    assert len(Zstd(level=0, checksum=False).encode(data)) + 4 == len(
+        Zstd(level=0, checksum=True).encode(data)
+    )
+
+
+def test_native_functions():
+    # Note, these assertions might need to be changed for new versions of zstd
+    assert Zstd.default_level() == 3
+    assert Zstd.min_level() == -131072
+    assert Zstd.max_level() == 22

numcodecs/version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.13.1'
+__version_tuple__ = version_tuple = (0, 13, 1)

numcodecs/zfpy.py

@@ -0,0 +1,111 @@
+import warnings
+from contextlib import suppress
+from importlib.metadata import PackageNotFoundError, version
+
+_zfpy = None
+
+_zfpy_version: tuple = ()
+with suppress(PackageNotFoundError):
+    _zfpy_version = tuple(map(int, version("zfpy").split(".")))
+
+if _zfpy_version:
+    # Check NumPy version
+    _numpy_version: tuple = tuple(map(int, version("numpy").split('.')))
+    if _numpy_version >= (2, 0, 0) and _zfpy_version <= (1, 0, 1):  # pragma: no cover
+        _zfpy_version = ()
+        warnings.warn(
+            "NumPy version >= 2.0.0 detected. The zfpy library is incompatible with this version of NumPy. "
+            "Please downgrade to NumPy < 2.0.0 or wait for an update from zfpy.",
+            UserWarning,
+            stacklevel=2,
+        )
+    else:
+        with suppress(ImportError):
+            import zfpy as _zfpy
+
+if _zfpy:
+    import numpy as np
+
+    from .abc import Codec
+    from .compat import ensure_bytes, ensure_contiguous_ndarray, ndarray_copy
+
+    # noinspection PyShadowingBuiltins
+    class ZFPY(Codec):
+        """Codec providing compression using zfpy via the Python standard
+        library.
+
+        Parameters
+        ----------
+        mode : integer
+            One of the zfpy mode choice, e.g., ``zfpy.mode_fixed_accuracy``.
+        tolerance : double, optional
+            A double-precision number, specifying the compression accuracy needed.
+        rate : double, optional
+            A double-precision number, specifying the compression rate needed.
+        precision : int, optional
+            A integer number, specifying the compression precision needed.
+
+        """
+
+        codec_id = "zfpy"
+
+        def __init__(
+            self,
+            mode=_zfpy.mode_fixed_accuracy,
+            tolerance=-1,
+            rate=-1,
+            precision=-1,
+            compression_kwargs=None,
+        ):
+            self.mode = mode
+            if mode == _zfpy.mode_fixed_accuracy:
+                self.compression_kwargs = {"tolerance": tolerance}
+            elif mode == _zfpy.mode_fixed_rate:
+                self.compression_kwargs = {"rate": rate}
+            elif mode == _zfpy.mode_fixed_precision:
+                self.compression_kwargs = {"precision": precision}
+
+            self.tolerance = tolerance
+            self.rate = rate
+            self.precision = precision
+
+        def encode(self, buf):
+            # not flatten c-order array and raise exception for f-order array
+            if not isinstance(buf, np.ndarray):
+                raise TypeError(
+                    "The zfp codec does not support none numpy arrays."
+                    f" Your buffers were {type(buf)}."
+                )
+            if buf.flags.c_contiguous:
+                flatten = False
+            else:
+                raise ValueError(
+                    "The zfp codec does not support F order arrays. "
+                    f"Your arrays flags were {buf.flags}."
+                )
+            buf = ensure_contiguous_ndarray(buf, flatten=flatten)
+
+            # do compression
+            return _zfpy.compress_numpy(buf, write_header=True, **self.compression_kwargs)
+
+        def decode(self, buf, out=None):
+            # normalise inputs
+            buf = ensure_bytes(buf)
+            if out is not None:
+                out = ensure_contiguous_ndarray(out)
+
+            # do decompression
+            dec = _zfpy.decompress_numpy(buf)
+
+            # handle destination
+            if out is not None:
+                return ndarray_copy(dec, out)
+            else:
+                return dec
+
+        def __repr__(self):
+            return (
+                f"{type(self).__name__}(mode={self.mode!r}, "
+                f"tolerance={self.tolerance}, rate={self.rate}, "
+                f"precision={self.precision})"
+            )