numcodecs 0.13.1__cp312-cp312-win_amd64.whl → 0.14.0__cp312-cp312-win_amd64.whl

numcodecs/__init__.py

@@ -1,4 +1,4 @@
-# ruff: noqa: E402,F401
+# ruff: noqa: E402
 """Numcodecs is a Python package providing buffer compression and
 transformation codecs for use in data storage and communication
 applications. These include:
@@ -21,8 +21,9 @@ import atexit
 import multiprocessing
 from contextlib import suppress
 
-from numcodecs.registry import get_codec, register_codec
-from numcodecs.version import version as __version__
+from numcodecs.registry import get_codec as get_codec
+from numcodecs.registry import register_codec
+from numcodecs.version import version as __version__  # noqa: F401
 from numcodecs.zlib import Zlib
 
 register_codec(Zlib)
@@ -55,13 +56,13 @@ with suppress(ImportError):
     atexit.register(blosc.destroy)
 
 with suppress(ImportError):
-    from numcodecs import zstd
+    from numcodecs import zstd as zstd
     from numcodecs.zstd import Zstd
 
     register_codec(Zstd)
 
 with suppress(ImportError):
-    from numcodecs import lz4
+    from numcodecs import lz4 as lz4
     from numcodecs.lz4 import LZ4
 
     register_codec(LZ4)
@@ -116,9 +117,10 @@ with suppress(ImportError):
 
     register_codec(MsgPack)
 
-from numcodecs.checksum32 import CRC32, Adler32, JenkinsLookup3
+from numcodecs.checksum32 import CRC32, CRC32C, Adler32, JenkinsLookup3
 
 register_codec(CRC32)
+register_codec(CRC32C)
 register_codec(Adler32)
 register_codec(JenkinsLookup3)
 
@@ -127,7 +129,7 @@ from numcodecs.json import JSON
 register_codec(JSON)
 
 with suppress(ImportError):
-    from numcodecs import vlen
+    from numcodecs import vlen as vlen
     from numcodecs.vlen import VLenArray, VLenBytes, VLenUTF8
 
     register_codec(VLenUTF8)

numcodecs/abc.py

@@ -29,13 +29,14 @@ other and vice versa.
 """
 
 from abc import ABC, abstractmethod
+from typing import Optional
 
 
 class Codec(ABC):
     """Codec abstract base class."""
 
     # override in sub-class
-    codec_id = None
+    codec_id: Optional[str] = None
     """Codec identifier."""
 
     @abstractmethod

numcodecs/bitround.py

@@ -59,7 +59,8 @@ class BitRound(Codec):
             return a
         if self.keepbits > bits:
             raise ValueError("Keepbits too large for given dtype")
-        b = a.view(a_int_dtype)
+        b = a.copy()
+        b = b.view(a_int_dtype)
         maskbits = bits - self.keepbits
         mask = (all_set >> maskbits) << maskbits
         half_quantum1 = (1 << (maskbits - 1)) - 1

numcodecs/checksum32.py

@@ -1,5 +1,7 @@
 import struct
 import zlib
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Literal
 
 import numpy as np
 
@@ -7,36 +9,107 @@ from .abc import Codec
 from .compat import ensure_contiguous_ndarray, ndarray_copy
 from .jenkins import jenkins_lookup3
 
+if TYPE_CHECKING:
+    from typing_extensions import Buffer
+
+CHECKSUM_LOCATION = Literal['start', 'end']
+
 
 class Checksum32(Codec):
     # override in sub-class
-    checksum = None
+    checksum: Callable[["Buffer", int], int] | None = None
+    location: CHECKSUM_LOCATION = 'start'
+
+    def __init__(self, location: CHECKSUM_LOCATION | None = None):
+        if location is not None:
+            self.location = location
+        if self.location not in ['start', 'end']:
+            raise ValueError(f"Invalid checksum location: {self.location}")
 
     def encode(self, buf):
         arr = ensure_contiguous_ndarray(buf).view('u1')
         checksum = self.checksum(arr) & 0xFFFFFFFF
         enc = np.empty(arr.nbytes + 4, dtype='u1')
-        enc[:4].view('<u4')[0] = checksum
-        ndarray_copy(arr, enc[4:])
+        if self.location == 'start':
+            checksum_view = enc[:4]
+            payload_view = enc[4:]
+        else:
+            checksum_view = enc[-4:]
+            payload_view = enc[:-4]
+        checksum_view.view('<u4')[0] = checksum
+        ndarray_copy(arr, payload_view)
         return enc
 
     def decode(self, buf, out=None):
+        if len(buf) < 4:
+            raise ValueError("Input buffer is too short to contain a 32-bit checksum.")
+        if out is not None:
+            ensure_contiguous_ndarray(out)  # check that out is a valid ndarray
+
         arr = ensure_contiguous_ndarray(buf).view('u1')
-        expect = arr[:4].view('<u4')[0]
-        checksum = self.checksum(arr[4:]) & 0xFFFFFFFF
+        if self.location == 'start':
+            checksum_view = arr[:4]
+            payload_view = arr[4:]
+        else:
+            checksum_view = arr[-4:]
+            payload_view = arr[:-4]
+        expect = checksum_view.view('<u4')[0]
+        checksum = self.checksum(payload_view) & 0xFFFFFFFF
         if expect != checksum:
-            raise RuntimeError('checksum failed')
-        return ndarray_copy(arr[4:], out)
+            raise RuntimeError(
+                f"Stored and computed {self.codec_id} checksum do not match. Stored: {expect}. Computed: {checksum}."
+            )
+        return ndarray_copy(payload_view, out)
 
 
 class CRC32(Checksum32):
+    """Codec add a crc32 checksum to the buffer.
+
+    Parameters
+    ----------
+    location : 'start' or 'end'
+        Where to place the checksum in the buffer.
+    """
+
     codec_id = 'crc32'
     checksum = zlib.crc32
+    location = 'start'
+
+
+class CRC32C(Checksum32):
+    """Codec add a crc32c checksum to the buffer.
+
+    Parameters
+    ----------
+    location : 'start' or 'end'
+        Where to place the checksum in the buffer.
+    """
+
+    codec_id = 'crc32c'
+
+    def checksum(self, buf):
+        try:
+            from crc32c import crc32c as crc32c_
+
+            return crc32c_(buf)
+        except ImportError:  # pragma: no cover
+            raise ImportError("crc32c must be installed to use the CRC32C checksum codec.")
+
+    location = 'end'
 
 
 class Adler32(Checksum32):
+    """Codec add a adler32 checksum to the buffer.
+
+    Parameters
+    ----------
+    location : 'start' or 'end'
+        Where to place the checksum in the buffer.
+    """
+
     codec_id = 'adler32'
     checksum = zlib.adler32
+    location = 'start'
 
 
 class JenkinsLookup3(Checksum32):
@@ -50,9 +123,12 @@ class JenkinsLookup3(Checksum32):
     the data portion and compared with the four-byte checksum, raising
     RuntimeError if inconsistent.
 
-    Attributes:
-       initval: initial seed passed to the hash algorithm, default: 0
-       prefix: bytes prepended to the buffer before evaluating the hash, default: None
+    Parameters
+    ----------
+    initval : int
+        initial seed passed to the hash algorithm, default: 0
+    prefix : int
+        bytes prepended to the buffer before evaluating the hash, default: None
     """
 
     checksum = jenkins_lookup3

numcodecs/compat.py

@@ -1,7 +1,5 @@
-# ruff: noqa: F401
 import array
 import codecs
-import functools
 
 import numpy as np
 
@@ -102,7 +100,7 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N
 
     # check for datetime or timedelta ndarray, the buffer interface doesn't support those
     if arr.dtype.kind in "Mm":
-        arr = arr.view(np.int64)
+        arr = arr.view(np.int64)  # type: ignore[arg-type]
 
     # check memory is contiguous, if so flatten
     if arr.flags.c_contiguous or arr.flags.f_contiguous:
@@ -119,7 +117,7 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N
     return arr
 
 
-def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.array:
+def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.ndarray:
     """Convenience function to coerce `buf` to a numpy array, if it is not already a
     numpy array. Also ensures that the returned value exports fully contiguous memory,
     and supports the new-style buffer interface. If the optional max_buffer_size is

numcodecs/delta.py

@@ -63,7 +63,11 @@ class Delta(Codec):
         enc[0] = arr[0]
 
         # compute differences
-        enc[1:] = np.diff(arr)
+        # using np.subtract for in-place operations
+        if arr.dtype == bool:
+            np.not_equal(arr[1:], arr[:-1], out=enc[1:])
+        else:
+            np.subtract(arr[1:], arr[:-1], out=enc[1:])
 
         return enc
 

numcodecs/lzma.py

@@ -1,9 +1,12 @@
-_lzma = None
+from types import ModuleType
+from typing import Optional
+
+_lzma: Optional[ModuleType] = None
 try:
     import lzma as _lzma
 except ImportError:  # pragma: no cover
     try:
-        from backports import lzma as _lzma
+        from backports import lzma as _lzma  # type: ignore[no-redef]
     except ImportError:
         pass
 

numcodecs/ndarray_like.py

@@ -1,7 +1,7 @@
 from typing import Any, ClassVar, Optional, Protocol, runtime_checkable
 
 
-class _CachedProtocolMeta(Protocol.__class__):
+class _CachedProtocolMeta(Protocol.__class__):  # type: ignore[name-defined]
     """Custom implementation of @runtime_checkable
 
     The native implementation of @runtime_checkable is slow,

numcodecs/packbits.py

@@ -29,9 +29,6 @@ class PackBits(Codec):
 
     codec_id = 'packbits'
 
-    def __init__(self):
-        pass
-
     def encode(self, buf):
         # normalise input
         arr = ensure_ndarray(buf).view(bool)

numcodecs/registry.py

@@ -2,11 +2,13 @@
 applications to dynamically register and look-up codec classes."""
 
 import logging
-from importlib.metadata import entry_points
+from importlib.metadata import EntryPoints, entry_points
+
+from numcodecs.abc import Codec
 
 logger = logging.getLogger("numcodecs")
-codec_registry = {}
-entries = {}
+codec_registry: dict[str, Codec] = {}
+entries: dict[str, "EntryPoints"] = {}
 
 
 def run_entrypoints():
@@ -42,11 +44,10 @@ def get_codec(config):
     config = dict(config)
     codec_id = config.pop('id', None)
     cls = codec_registry.get(codec_id)
-    if cls is None:
-        if codec_id in entries:
-            logger.debug("Auto loading codec '%s' from entrypoint", codec_id)
-            cls = entries[codec_id].load()
-            register_codec(cls, codec_id=codec_id)
+    if cls is None and codec_id in entries:
+        logger.debug("Auto loading codec '%s' from entrypoint", codec_id)
+        cls = entries[codec_id].load()
+        register_codec(cls, codec_id=codec_id)
     if cls:
         return cls.from_config(config)
     raise ValueError(f'codec not available: {codec_id!r}')

numcodecs/tests/test_bitround.py

@@ -24,21 +24,21 @@ def test_round_zero_to_zero(dtype):
     # Don't understand Milan's original test:
     # How is it possible to have negative keepbits?
     # for k in range(-5, 50):
-    for k in range(0, max_bits[dtype]):
+    for k in range(max_bits[dtype]):
         ar = round(a, k)
         np.testing.assert_equal(a, ar)
 
 
 def test_round_one_to_one(dtype):
     a = np.ones((3, 2), dtype=dtype)
-    for k in range(0, max_bits[dtype]):
+    for k in range(max_bits[dtype]):
         ar = round(a, k)
         np.testing.assert_equal(a, ar)
 
 
 def test_round_minus_one_to_minus_one(dtype):
     a = -np.ones((3, 2), dtype=dtype)
-    for k in range(0, max_bits[dtype]):
+    for k in range(max_bits[dtype]):
         ar = round(a, k)
         np.testing.assert_equal(a, ar)
 

numcodecs/tests/test_blosc.py

@@ -133,7 +133,7 @@ def test_compress_blocksize_default(use_threads):
     assert blocksize > 0
 
 
-@pytest.mark.parametrize('bs', (2**7, 2**8))
+@pytest.mark.parametrize('bs', [2**7, 2**8])
 def test_compress_blocksize(use_threads, bs):
     arr = np.arange(1000, dtype='i4')
 
@@ -225,7 +225,7 @@ def _decode_worker(enc):
     return data
 
 
-@pytest.mark.parametrize('pool', (Pool, ThreadPool))
+@pytest.mark.parametrize('pool', [Pool, ThreadPool])
 def test_multiprocessing(use_threads, pool):
     data = np.arange(1000000)
     enc = _encode_worker(data)

numcodecs/tests/test_checksum32.py

@@ -3,11 +3,12 @@ import itertools
 import numpy as np
 import pytest
 
-from numcodecs.checksum32 import CRC32, Adler32
+from numcodecs.checksum32 import CRC32, CRC32C, Adler32
 from numcodecs.tests.common import (
     check_backwards_compatibility,
     check_config,
     check_encode_decode,
+    check_err_decode_object_buffer,
     check_err_encode_object_buffer,
     check_repr,
 )
@@ -21,38 +22,117 @@ arrays = [
     np.random.normal(loc=1000, scale=1, size=(100, 10)),
     np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
     np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
 ]
 
-codecs = [CRC32(), Adler32()]
+codecs = [
+    CRC32(),
+    CRC32(location="end"),
+    CRC32C(location="start"),
+    CRC32C(),
+    Adler32(),
+    Adler32(location="end"),
+]
+
+
+@pytest.mark.parametrize(("codec", "arr"), itertools.product(codecs, arrays))
+def test_encode_decode(codec, arr):
+    check_encode_decode(arr, codec)
+
+
+@pytest.mark.parametrize(("codec", "arr"), itertools.product(codecs, arrays))
+def test_errors(codec, arr):
+    enc = codec.encode(arr)
+    with pytest.raises(RuntimeError):
+        codec.decode(enc[:-1])
+
+
+@pytest.mark.parametrize("codec", codecs)
+def test_config(codec):
+    check_config(codec)
+
+
+@pytest.mark.parametrize("codec", codecs)
+def test_err_input_too_small(codec):
+    buf = b'000'  # 3 bytes are too little for a 32-bit checksum
+    with pytest.raises(ValueError):
+        codec.decode(buf)
 
 
-def test_encode_decode():
-    for codec, arr in itertools.product(codecs, arrays):
-        check_encode_decode(arr, codec)
+@pytest.mark.parametrize("codec", codecs)
+def test_err_encode_non_contiguous(codec):
+    # non-contiguous memory
+    arr = np.arange(1000, dtype='i4')[::2]
+    with pytest.raises(ValueError):
+        codec.encode(arr)
 
 
-def test_errors():
-    for codec, arr in itertools.product(codecs, arrays):
-        enc = codec.encode(arr)
-        with pytest.raises(RuntimeError):
-            codec.decode(enc[:-1])
+@pytest.mark.parametrize("codec", codecs)
+def test_err_encode_list(codec):
+    data = ['foo', 'bar', 'baz']
+    with pytest.raises(TypeError):
+        codec.encode(data)
 
 
-def test_config():
-    for codec in codecs:
-        check_config(codec)
+def test_err_location():
+    with pytest.raises(ValueError):
+        CRC32(location="foo")
+    with pytest.raises(ValueError):
+        CRC32C(location="foo")
+    with pytest.raises(ValueError):
+        Adler32(location="foo")
 
 
 def test_repr():
-    check_repr("CRC32()")
-    check_repr("Adler32()")
+    check_repr("CRC32(location='start')")
+    check_repr("CRC32C(location='start')")
+    check_repr("Adler32(location='start')")
+    check_repr("CRC32(location='end')")
+    check_repr("CRC32C(location='end')")
+    check_repr("Adler32(location='end')")
 
 
 def test_backwards_compatibility():
     check_backwards_compatibility(CRC32.codec_id, arrays, [CRC32()])
     check_backwards_compatibility(Adler32.codec_id, arrays, [Adler32()])
+    check_backwards_compatibility(CRC32C.codec_id, arrays, [CRC32C()])
+
+
+@pytest.mark.parametrize("codec", codecs)
+def test_err_encode_object_buffer(codec):
+    check_err_encode_object_buffer(codec)
+
+
+@pytest.mark.parametrize("codec", codecs)
+def test_err_decode_object_buffer(codec):
+    check_err_decode_object_buffer(codec)
+
+
+@pytest.mark.parametrize("codec", codecs)
+def test_err_out_too_small(codec):
+    arr = np.arange(10, dtype='i4')
+    out = np.empty_like(arr)[:-1]
+    with pytest.raises(ValueError):
+        codec.decode(codec.encode(arr), out)
+
+
+def test_crc32c_checksum():
+    arr = np.arange(0, 64, dtype="uint8")
+    buf = CRC32C(location="end").encode(arr)
+    assert np.frombuffer(buf, dtype="<u4", offset=(len(buf) - 4))[0] == np.uint32(4218238699)
 
 
-def test_err_encode_object_buffer():
-    check_err_encode_object_buffer(CRC32())
-    check_err_encode_object_buffer(Adler32())
+@pytest.mark.parametrize("codec", codecs)
+def test_err_checksum(codec):
+    arr = np.arange(0, 64, dtype="uint8")
+    buf = bytearray(codec.encode(arr))
+    buf[-1] = 0  # corrupt the checksum
+    with pytest.raises(RuntimeError):
+        codec.decode(buf)

numcodecs/tests/test_compat.py

@@ -64,6 +64,9 @@ def test_ensure_bytes_invalid_inputs():
             ensure_bytes(e)
 
 
+@pytest.mark.filterwarnings(
+    "ignore:The 'u' type code is deprecated and will be removed in Python 3.16"
+)
 def test_ensure_contiguous_ndarray_invalid_inputs():
     # object array not allowed
     a = np.array(['Xin chào thế giới'], dtype=object)

numcodecs/tests/test_delta.py

@@ -14,6 +14,7 @@ from numcodecs.tests.common import (
 # mix of shapes: 1D, 2D, 3D
 # mix of orders: C, F
 arrays = [
+    np.random.randint(0, 1, size=110, dtype='?').reshape(10, 11),
     np.arange(1000, dtype='<i4'),
     np.linspace(1000, 1001, 1000, dtype='<f4').reshape(100, 10),
     np.random.normal(loc=1000, scale=1, size=(10, 10, 10)).astype('<f8'),

numcodecs/tests/test_entrypoints.py

@@ -8,7 +8,7 @@ import numcodecs.registry
 here = os.path.abspath(os.path.dirname(__file__))
 
 
-@pytest.fixture()
+@pytest.fixture
 def set_path():
     sys.path.append(here)
     numcodecs.registry.run_entrypoints()

numcodecs/tests/test_entrypoints_backport.py

@@ -1,4 +1,4 @@
-import importlib
+import importlib.util
 import os.path
 import sys
 from multiprocessing import Process
@@ -7,7 +7,8 @@ import pytest
 
 import numcodecs.registry
 
-if not importlib.util.find_spec("importlib_metadata").loader:  # pragma: no cover
+importlib_spec = importlib.util.find_spec("importlib_metadata")
+if importlib_spec is None or importlib_spec.loader is None:  # pragma: no cover
     pytest.skip(
         "This test module requires importlib_metadata to be installed",
         allow_module_level=True,

numcodecs/tests/test_fixedscaleoffset.py

@@ -36,12 +36,20 @@ def test_encode_decode():
         check_encode_decode(arr, codec, precision=precision)
 
 
-def test_encode():
+@pytest.mark.parametrize(
+    ("offset", "scale", "expected"),
+    [
+        (1000, 10, [0, 6, 11, 17, 22, 28, 33, 39, 44, 50]),
+        (1002.5, 10, [-25, -19, -14, -8, -3, 3, 8, 14, 19, 25]),
+        (1000, 0.5, [0, 0, 1, 1, 1, 1, 2, 2, 2, 2]),
+    ],
+)
+def test_encode(offset: float, scale: float, expected: list[int]):
     dtype = '<f8'
-    astype = '|u1'
-    codec = FixedScaleOffset(scale=10, offset=1000, dtype=dtype, astype=astype)
-    arr = np.linspace(1000, 1001, 10, dtype=dtype)
-    expect = np.array([0, 1, 2, 3, 4, 6, 7, 8, 9, 10], dtype=astype)
+    astype = np.int16
+    codec = FixedScaleOffset(scale=scale, offset=offset, dtype=dtype, astype=astype)
+    arr = np.linspace(1000, 1005, 10, dtype=dtype)
+    expect = np.array(expected, dtype=astype)
     actual = codec.encode(arr)
     assert_array_equal(expect, actual)
     assert np.dtype(astype) == actual.dtype

numcodecs/tests/test_json.py

@@ -59,7 +59,7 @@ def test_backwards_compatibility():
 
 
 @pytest.mark.parametrize(
-    "input_data, dtype",
+    ('input_data', 'dtype'),
     [
         ([0, 1], None),
         ([[0, 1], [2, 3]], None),

numcodecs/tests/test_lzma.py

@@ -1,5 +1,7 @@
 import itertools
 import unittest
+from types import ModuleType
+from typing import cast
 
 import numpy as np
 import pytest
@@ -20,6 +22,8 @@ from numcodecs.tests.common import (
     check_repr,
 )
 
+_lzma = cast(ModuleType, _lzma)
+
 codecs = [
     LZMA(),
     LZMA(preset=1),

numcodecs/tests/test_msgpacks.py

@@ -55,8 +55,11 @@ def test_backwards_compatibility():
     check_backwards_compatibility(codec.codec_id, arrays, [codec])
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Creating an ndarray from ragged nested sequences .* is deprecated.*"
+)
 @pytest.mark.parametrize(
-    "input_data, dtype",
+    ("input_data", "dtype"),
     [
         ([0, 1], None),
         ([[0, 1], [2, 3]], None),

numcodecs/tests/test_pcodec.py

@@ -57,8 +57,8 @@ def test_config():
 
 
 def test_invalid_config_error():
+    codec = PCodec(mode_spec='bogus')
     with pytest.raises(ValueError):
-        codec = PCodec(mode_spec='bogus')
         check_encode_decode_array_to_bytes(arrays[0], codec)