numcodecs 0.16.0__cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

numcodecs/registry.py

@@ -0,0 +1,74 @@
+"""The registry module provides some simple convenience functions to enable
+applications to dynamically register and look-up codec classes."""
+
+import logging
+from importlib.metadata import EntryPoints, entry_points
+
+from numcodecs.abc import Codec
+from numcodecs.errors import UnknownCodecError
+
+logger = logging.getLogger("numcodecs")
+codec_registry: dict[str, Codec] = {}
+entries: dict[str, EntryPoints] = {}
+
+
+def run_entrypoints():
+    entries.clear()
+    eps = entry_points()
+    entries.update({e.name: e for e in eps.select(group="numcodecs.codecs")})
+
+
+run_entrypoints()
+
+
+def get_codec(config):
+    """Obtain a codec for the given configuration.
+
+    Parameters
+    ----------
+    config : dict-like
+        Configuration object.
+
+    Returns
+    -------
+    codec : Codec
+
+    Examples
+    --------
+
+    >>> import numcodecs as codecs
+    >>> codec = codecs.get_codec(dict(id='zlib', level=1))
+    >>> codec
+    Zlib(level=1)
+
+    """
+    config = dict(config)
+    codec_id = config.pop('id', None)
+    cls = codec_registry.get(codec_id)
+    if cls is None and codec_id in entries:
+        logger.debug("Auto loading codec '%s' from entrypoint", codec_id)
+        cls = entries[codec_id].load()
+        register_codec(cls, codec_id=codec_id)
+    if cls:
+        return cls.from_config(config)
+    raise UnknownCodecError(f"{codec_id!r}")
+
+
+def register_codec(cls, codec_id=None):
+    """Register a codec class.
+
+    Parameters
+    ----------
+    cls : Codec class
+
+    Notes
+    -----
+    This function maintains a mapping from codec identifiers to codec
+    classes. When a codec class is registered, it will replace any class
+    previously registered under the same codec identifier, if present.
+
+    """
+    if codec_id is None:
+        codec_id = cls.codec_id
+    logger.debug("Registering codec '%s'", codec_id)
+    codec_registry[codec_id] = cls

numcodecs/shuffle.py

@@ -0,0 +1,61 @@
+import numpy as np
+
+from ._shuffle import _doShuffle, _doUnshuffle
+from .abc import Codec
+from .compat import ensure_contiguous_ndarray
+
+
+class Shuffle(Codec):
+    """Codec providing shuffle
+
+    Parameters
+    ----------
+    elementsize : int
+        Size in bytes of the array elements.  Default = 4
+
+    """
+
+    codec_id = 'shuffle'
+
+    def __init__(self, elementsize=4):
+        self.elementsize = elementsize
+
+    def _prepare_arrays(self, buf, out):
+        buf = ensure_contiguous_ndarray(buf)
+
+        if out is None:
+            out = np.zeros(buf.nbytes, dtype='uint8')
+        else:
+            out = ensure_contiguous_ndarray(out)
+
+        if self.elementsize <= 1:
+            out.view(buf.dtype)[: len(buf)] = buf[:]  # no shuffling needed
+            return buf, out
+
+        if buf.nbytes % self.elementsize != 0:
+            raise ValueError("Shuffle buffer is not an integer multiple of elementsize")
+
+        return buf, out
+
+    def encode(self, buf, out=None):
+        buf, out = self._prepare_arrays(buf, out)
+
+        if self.elementsize <= 1:
+            return out  # no shuffling needed
+
+        _doShuffle(buf.view("uint8"), out.view("uint8"), self.elementsize)
+
+        return out
+
+    def decode(self, buf, out=None):
+        buf, out = self._prepare_arrays(buf, out)
+
+        if self.elementsize <= 1:
+            return out  # no shuffling needed
+
+        _doUnshuffle(buf.view("uint8"), out.view("uint8"), self.elementsize)
+
+        return out
+
+    def __repr__(self):
+        return f'{type(self).__name__}(elementsize={self.elementsize})'

numcodecs/tests/__init__.py

@@ -0,0 +1,3 @@
+import pytest
+
+pytest.register_assert_rewrite('numcodecs.tests.common')

numcodecs/tests/common.py

@@ -0,0 +1,285 @@
+import array
+import json as _json
+import os
+from glob import glob
+
+import numpy as np
+import pytest
+from numpy.testing import assert_array_almost_equal, assert_array_equal
+
+from numcodecs import *  # noqa: F403  # for eval to find names in repr tests
+from numcodecs.compat import ensure_bytes, ensure_ndarray
+from numcodecs.registry import get_codec
+
+greetings = [
+    '¡Hola mundo!',
+    'Hej Världen!',
+    'Servus Woid!',
+    'Hei maailma!',
+    'Xin chào thế giới',
+    'Njatjeta Botë!',
+    'Γεια σου κόσμε!',  # noqa: RUF001
+    'こんにちは世界',
+    '世界，你好！',  # noqa: RUF001
+    'Helló, világ!',
+    'Zdravo svete!',
+    'เฮลโลเวิลด์',
+]
+
+
+def compare_arrays(arr, res, precision=None):
+    # ensure numpy array with matching dtype
+    res = ensure_ndarray(res).view(arr.dtype)
+
+    # convert to correct shape
+    if arr.flags.f_contiguous:
+        order = 'F'
+    else:
+        order = 'C'
+    res = res.reshape(arr.shape, order=order)
+
+    # exact compare
+    if precision is None:
+        assert_array_equal(arr, res)
+
+    # fuzzy compare
+    else:
+        assert_array_almost_equal(arr, res, decimal=precision)
+
+
+def check_encode_decode(arr, codec, precision=None):
+    # N.B., watch out here with blosc compressor, if the itemsize of
+    # the source buffer is different then the results of encoding
+    # (i.e., compression) may be different. Hence we *do not* require that
+    # the results of encoding be identical for all possible inputs, rather
+    # we just require that the results of the encode/decode round-trip can
+    # be compared to the original array.
+
+    # encoding should support any object exporting the buffer protocol
+
+    # test encoding of numpy array
+    enc = codec.encode(arr)
+    dec = codec.decode(enc)
+    compare_arrays(arr, dec, precision=precision)
+
+    # test encoding of bytes
+    buf = arr.tobytes(order='A')
+    enc = codec.encode(buf)
+    dec = codec.decode(enc)
+    compare_arrays(arr, dec, precision=precision)
+
+    # test encoding of bytearray
+    buf = bytearray(arr.tobytes(order='A'))
+    enc = codec.encode(buf)
+    dec = codec.decode(enc)
+    compare_arrays(arr, dec, precision=precision)
+
+    # test encoding of array.array
+    buf = array.array('b', arr.tobytes(order='A'))
+    enc = codec.encode(buf)
+    dec = codec.decode(enc)
+    compare_arrays(arr, dec, precision=precision)
+
+    # decoding should support any object exporting the buffer protocol,
+
+    # setup
+    enc_bytes = ensure_bytes(enc)
+
+    # test decoding of raw bytes
+    dec = codec.decode(enc_bytes)
+    compare_arrays(arr, dec, precision=precision)
+
+    # test decoding of bytearray
+    dec = codec.decode(bytearray(enc_bytes))
+    compare_arrays(arr, dec, precision=precision)
+
+    # test decoding of array.array
+    buf = array.array('b', enc_bytes)
+    dec = codec.decode(buf)
+    compare_arrays(arr, dec, precision=precision)
+
+    # test decoding of numpy array
+    buf = np.frombuffer(enc_bytes, dtype='u1')
+    dec = codec.decode(buf)
+    compare_arrays(arr, dec, precision=precision)
+
+    # test decoding directly into numpy array
+    out = np.empty_like(arr)
+    codec.decode(enc_bytes, out=out)
+    compare_arrays(arr, out, precision=precision)
+
+    # test decoding directly into bytearray
+    out = bytearray(arr.nbytes)
+    codec.decode(enc_bytes, out=out)
+    # noinspection PyTypeChecker
+    compare_arrays(arr, out, precision=precision)
+
+
+def assert_array_items_equal(res, arr):
+    assert isinstance(res, np.ndarray)
+    res = res.reshape(-1, order='A')
+    arr = arr.reshape(-1, order='A')
+    assert res.shape == arr.shape
+    assert res.dtype == arr.dtype
+
+    # numpy asserts don't compare object arrays
+    # properly; assert that we have the same nans
+    # and values
+    arr = arr.ravel().tolist()
+    res = res.ravel().tolist()
+    for a, r in zip(arr, res, strict=True):
+        if isinstance(a, np.ndarray):
+            assert_array_equal(a, r)
+        elif a != a:
+            assert r != r
+        else:
+            assert a == r
+
+
+def check_encode_decode_array(arr, codec):
+    enc = codec.encode(arr)
+    dec = codec.decode(enc)
+    assert_array_items_equal(arr, dec)
+
+    out = np.empty_like(arr)
+    codec.decode(enc, out=out)
+    assert_array_items_equal(arr, out)
+
+    enc = codec.encode(arr)
+    dec = codec.decode(ensure_ndarray(enc))
+    assert_array_items_equal(arr, dec)
+
+
+def check_encode_decode_array_to_bytes(arr, codec):
+    enc = codec.encode(arr)
+    dec = codec.decode(enc)
+    assert_array_items_equal(arr, dec)
+
+    out = np.empty_like(arr)
+    codec.decode(enc, out=out)
+    assert_array_items_equal(arr, out)
+
+
+def check_config(codec):
+    config = codec.get_config()
+    # round-trip through JSON to check serialization
+    config = _json.loads(_json.dumps(config))
+    assert codec == get_codec(config)
+
+
+def check_repr(stmt):
+    # check repr matches instantiation statement
+    codec = eval(stmt)
+    actual = repr(codec)
+    assert stmt == actual
+
+
+def check_backwards_compatibility(codec_id, arrays, codecs, precision=None, prefix=None):
+    # setup directory to hold data fixture
+    if prefix:
+        fixture_dir = os.path.join('fixture', codec_id, prefix)
+    else:
+        fixture_dir = os.path.join('fixture', codec_id)
+    if not os.path.exists(fixture_dir):  # pragma: no cover
+        os.makedirs(fixture_dir)
+
+    # save fixture data
+    for i, arr in enumerate(arrays):
+        arr_fn = os.path.join(fixture_dir, f'array.{i:02d}.npy')
+        if not os.path.exists(arr_fn):  # pragma: no cover
+            np.save(arr_fn, arr)
+
+    # load fixture data
+    for arr_fn in glob(os.path.join(fixture_dir, 'array.*.npy')):
+        # setup
+        i = int(arr_fn.split('.')[-2])
+        arr = np.load(arr_fn, allow_pickle=True)
+        arr_bytes = arr.tobytes(order='A')
+        if arr.flags.f_contiguous:
+            order = 'F'
+        else:
+            order = 'C'
+
+        for j, codec in enumerate(codecs):
+            if codec is None:
+                pytest.skip("codec has been removed")
+
+            # setup a directory to hold encoded data
+            codec_dir = os.path.join(fixture_dir, f'codec.{j:02d}')
+            if not os.path.exists(codec_dir):  # pragma: no cover
+                os.makedirs(codec_dir)
+
+            # file with codec configuration information
+            codec_fn = os.path.join(codec_dir, 'config.json')
+            # one time save config
+            if not os.path.exists(codec_fn):  # pragma: no cover
+                with open(codec_fn, mode='w') as cf:
+                    _json.dump(codec.get_config(), cf, sort_keys=True, indent=4)
+            # load config and compare with expectation
+            with open(codec_fn) as cf:
+                config = _json.load(cf)
+                assert codec == get_codec(config)
+
+            enc_fn = os.path.join(codec_dir, f'encoded.{i:02d}.dat')
+
+            # one time encode and save array
+            if not os.path.exists(enc_fn):  # pragma: no cover
+                enc = codec.encode(arr)
+                enc = ensure_bytes(enc)
+                with open(enc_fn, mode='wb') as ef:
+                    ef.write(enc)
+
+            # load and decode data
+            with open(enc_fn, mode='rb') as ef:
+                enc = ef.read()
+                dec = codec.decode(enc)
+                dec_arr = ensure_ndarray(dec).reshape(-1, order='A')
+                dec_arr = dec_arr.view(dtype=arr.dtype).reshape(arr.shape, order=order)
+                if precision and precision[j] is not None:
+                    assert_array_almost_equal(arr, dec_arr, decimal=precision[j])
+                elif arr.dtype == 'object':
+                    assert_array_items_equal(arr, dec_arr)
+                else:
+                    assert_array_equal(arr, dec_arr)
+                    assert arr_bytes == ensure_bytes(dec)
+
+
+def check_err_decode_object_buffer(compressor):
+    # cannot decode directly into object array, leads to segfaults
+    a = np.arange(10)
+    enc = compressor.encode(a)
+    out = np.empty(10, dtype=object)
+    with pytest.raises(TypeError):
+        compressor.decode(enc, out=out)
+
+
+def check_err_encode_object_buffer(compressor):
+    # compressors cannot encode object array
+    a = np.array(['foo', 'bar', 'baz'], dtype=object)
+    with pytest.raises(TypeError):
+        compressor.encode(a)
+
+
+def check_max_buffer_size(codec):
+    for max_buffer_size in (4, 64, 1024):
+        old_max_buffer_size = codec.max_buffer_size
+        try:
+            codec.max_buffer_size = max_buffer_size
+            # Just up the max_buffer_size is fine.
+            codec.encode(np.zeros(max_buffer_size - 1, dtype=np.int8))
+            codec.encode(np.zeros(max_buffer_size, dtype=np.int8))
+
+            buffers = [
+                bytes(b"x" * (max_buffer_size + 1)),
+                np.zeros(max_buffer_size + 1, dtype=np.int8),
+                np.zeros(max_buffer_size + 2, dtype=np.int8),
+                np.zeros(max_buffer_size, dtype=np.int16),
+                np.zeros(max_buffer_size, dtype=np.int32),
+            ]
+            for buf in buffers:
+                with pytest.raises(ValueError):
+                    codec.encode(buf)
+                with pytest.raises(ValueError):
+                    codec.decode(buf)
+        finally:
+            codec.max_buffer_size = old_max_buffer_size

numcodecs/tests/package_with_entrypoint/__init__.py

@@ -0,0 +1,11 @@
+from numcodecs.abc import Codec
+
+
+class TestCodec(Codec):
+    codec_id = "test"
+
+    def encode(self, buf):  # pragma: no cover
+        pass
+
+    def decode(self, buf, out=None):  # pragma: no cover
+        pass

numcodecs/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[numcodecs.codecs]
+test = package_with_entrypoint:TestCodec

numcodecs/tests/test_astype.py

@@ -0,0 +1,74 @@
+import numpy as np
+from numpy.testing import assert_array_equal
+
+from numcodecs.astype import AsType
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_config,
+    check_encode_decode,
+    check_repr,
+)
+
+# mix of dtypes: integer, float
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype='i4'),
+    np.linspace(1000, 1001, 1000, dtype='f8').reshape(100, 10),
+    np.random.normal(loc=1000, scale=1, size=(10, 10, 10)),
+    np.random.randint(0, 200, size=1000, dtype='u2').reshape(100, 10, order='F'),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        codec = AsType(encode_dtype=arr.dtype, decode_dtype=arr.dtype)
+        check_encode_decode(arr, codec)
+
+
+def test_decode():
+    encode_dtype, decode_dtype = '<i4', '<i8'
+    codec = AsType(encode_dtype=encode_dtype, decode_dtype=decode_dtype)
+    arr = np.arange(10, 20, 1, dtype=encode_dtype)
+    expect = arr.astype(decode_dtype)
+    actual = codec.decode(arr)
+    assert_array_equal(expect, actual)
+    assert np.dtype(decode_dtype) == actual.dtype
+
+
+def test_encode():
+    encode_dtype, decode_dtype = '<i4', '<i8'
+    codec = AsType(encode_dtype=encode_dtype, decode_dtype=decode_dtype)
+    arr = np.arange(10, 20, 1, dtype=decode_dtype)
+    expect = arr.astype(encode_dtype)
+    actual = codec.encode(arr)
+    assert_array_equal(expect, actual)
+    assert np.dtype(encode_dtype) == actual.dtype
+
+
+def test_config():
+    encode_dtype, decode_dtype = '<i4', '<i8'
+    codec = AsType(encode_dtype=encode_dtype, decode_dtype=decode_dtype)
+    check_config(codec)
+
+
+def test_repr():
+    check_repr("AsType(encode_dtype='<i4', decode_dtype='<i2')")
+
+
+def test_backwards_compatibility():
+    # integers
+    arrs = [
+        np.arange(1000, dtype='<i4'),
+        np.random.randint(0, 200, size=1000, dtype='i4').astype('<i4').reshape(100, 10, order='F'),
+    ]
+    codec = AsType(encode_dtype='<i2', decode_dtype='<i4')
+    check_backwards_compatibility(AsType.codec_id, arrs, [codec], prefix='i')
+
+    # floats
+    arrs = [
+        np.linspace(1000, 1001, 1000, dtype='<f8').reshape(100, 10, order='F'),
+        np.random.normal(loc=1000, scale=1, size=(10, 10, 10)).astype('<f8'),
+    ]
+    codec = AsType(encode_dtype='<f4', decode_dtype='<f8')
+    check_backwards_compatibility(AsType.codec_id, arrs, [codec], precision=[3], prefix='f')

numcodecs/tests/test_base64.py

@@ -0,0 +1,81 @@
+import itertools
+
+import numpy as np
+import pytest
+
+from numcodecs.base64 import Base64
+from numcodecs.tests.common import (
+    check_backwards_compatibility,
+    check_encode_decode,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+
+codecs = [
+    Base64(),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype="i4"),
+    np.linspace(1000, 1001, 1000, dtype="f8"),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order="F"),
+    np.random.choice([b"a", b"bb", b"ccc"], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype="u8").view("M8[ns]"),
+    np.random.randint(0, 2**60, size=1000, dtype="u8").view("m8[ns]"),
+    np.random.randint(0, 2**25, size=1000, dtype="u8").view("M8[m]"),
+    np.random.randint(0, 2**25, size=1000, dtype="u8").view("m8[m]"),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype="i8").view("M8[ns]"),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype="i8").view("m8[ns]"),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype="i8").view("M8[m]"),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype="i8").view("m8[m]"),
+]
+
+
+def test_encode_decode():
+    for arr, codec in itertools.product(arrays, codecs):
+        check_encode_decode(arr, codec)
+
+
+def test_repr():
+    check_repr("Base64()")
+
+
+def test_eq():
+    assert Base64() == Base64()
+    assert not Base64() != Base64()
+    assert Base64() != "foo"
+    assert "foo" != Base64()
+    assert not Base64() == "foo"
+
+
+def test_backwards_compatibility():
+    check_backwards_compatibility(Base64.codec_id, arrays, codecs)
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(Base64())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(Base64())
+
+
+def test_err_encode_list():
+    data = ["foo", "bar", "baz"]
+    for codec in codecs:
+        with pytest.raises(TypeError):
+            codec.encode(data)
+
+
+def test_err_encode_non_contiguous():
+    # non-contiguous memory
+    arr = np.arange(1000, dtype="i4")[::2]
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)

numcodecs/tests/test_bitround.py

@@ -0,0 +1,81 @@
+import numpy as np
+import pytest
+
+from numcodecs.bitround import BitRound, max_bits
+
+# adapted from https://github.com/milankl/BitInformation.jl/blob/main/test/round_nearest.jl
+
+
+# TODO: add other dtypes
+@pytest.fixture(params=["float32", "float64"])
+def dtype(request):
+    return request.param
+
+
+def round(data, keepbits):
+    codec = BitRound(keepbits=keepbits)
+    data = data.copy()  # otherwise overwrites the input
+    encoded = codec.encode(data)
+    return codec.decode(encoded)
+
+
+def test_round_zero_to_zero(dtype):
+    a = np.zeros((3, 2), dtype=dtype)
+    # Don't understand Milan's original test:
+    # How is it possible to have negative keepbits?
+    # for k in range(-5, 50):
+    for k in range(max_bits[dtype]):
+        ar = round(a, k)
+        np.testing.assert_equal(a, ar)
+
+
+def test_round_one_to_one(dtype):
+    a = np.ones((3, 2), dtype=dtype)
+    for k in range(max_bits[dtype]):
+        ar = round(a, k)
+        np.testing.assert_equal(a, ar)
+
+
+def test_round_minus_one_to_minus_one(dtype):
+    a = -np.ones((3, 2), dtype=dtype)
+    for k in range(max_bits[dtype]):
+        ar = round(a, k)
+        np.testing.assert_equal(a, ar)
+
+
+def test_no_rounding(dtype):
+    a = np.random.random_sample((300, 200)).astype(dtype)
+    keepbits = max_bits[dtype]
+    ar = round(a, keepbits)
+    np.testing.assert_equal(a, ar)
+
+
+APPROX_KEEPBITS = {"float32": 11, "float64": 18}
+
+
+def test_approx_equal(dtype):
+    a = np.random.random_sample((300, 200)).astype(dtype)
+    ar = round(a, APPROX_KEEPBITS[dtype])
+    # Mimic julia behavior - https://docs.julialang.org/en/v1/base/math/#Base.isapprox
+    rtol = np.sqrt(np.finfo(np.float32).eps)
+    # This gets us much closer but still failing for ~6% of the array
+    # It does pass if we add 1 to keepbits (11 instead of 10)
+    # Is there an off-by-one issue here?
+    np.testing.assert_allclose(a, ar, rtol=rtol)
+
+
+def test_idempotence(dtype):
+    a = np.random.random_sample((300, 200)).astype(dtype)
+    for k in range(20):
+        ar = round(a, k)
+        ar2 = round(a, k)
+        np.testing.assert_equal(ar, ar2)
+
+
+def test_errors():
+    with pytest.raises(ValueError):
+        BitRound(keepbits=99).encode(np.array([0], dtype="float32"))
+    with pytest.raises(TypeError):
+        BitRound(keepbits=10).encode(np.array([0]))
+    with pytest.raises(ValueError):
+        BitRound(-1)