numcodecs 0.16.0__cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

numcodecs/checksum32.py

@@ -0,0 +1,183 @@
+import abc
+import struct
+import zlib
+from contextlib import suppress
+from types import ModuleType
+from typing import Literal, Optional
+
+import numpy as np
+from typing_extensions import Buffer
+
+from .abc import Codec
+from .compat import ensure_contiguous_ndarray, ndarray_copy
+from .jenkins import jenkins_lookup3
+
+_crc32c: Optional[ModuleType] = None
+with suppress(ImportError):
+    import crc32c as _crc32c  # type: ignore[no-redef, unused-ignore]
+
+CHECKSUM_LOCATION = Literal['start', 'end']
+
+
+class Checksum32(Codec, abc.ABC):
+    # override in sub-class
+    location: CHECKSUM_LOCATION = 'start'
+
+    def __init__(self, location: CHECKSUM_LOCATION | None = None):
+        if location is not None:
+            self.location = location
+        if self.location not in ['start', 'end']:
+            raise ValueError(f"Invalid checksum location: {self.location}")
+
+    def encode(self, buf):
+        arr = ensure_contiguous_ndarray(buf).view('u1')
+        checksum = self.checksum(arr) & 0xFFFFFFFF
+        enc = np.empty(arr.nbytes + 4, dtype='u1')
+        if self.location == 'start':
+            checksum_view = enc[:4]
+            payload_view = enc[4:]
+        else:
+            checksum_view = enc[-4:]
+            payload_view = enc[:-4]
+        checksum_view.view('<u4')[0] = checksum
+        ndarray_copy(arr, payload_view)
+        return enc
+
+    def decode(self, buf, out=None):
+        if len(buf) < 4:
+            raise ValueError("Input buffer is too short to contain a 32-bit checksum.")
+        if out is not None:
+            ensure_contiguous_ndarray(out)  # check that out is a valid ndarray
+
+        arr = ensure_contiguous_ndarray(buf).view('u1')
+        if self.location == 'start':
+            checksum_view = arr[:4]
+            payload_view = arr[4:]
+        else:
+            checksum_view = arr[-4:]
+            payload_view = arr[:-4]
+        expect = checksum_view.view('<u4')[0]
+        checksum = self.checksum(payload_view) & 0xFFFFFFFF
+        if expect != checksum:
+            raise RuntimeError(
+                f"Stored and computed {self.codec_id} checksum do not match. Stored: {expect}. Computed: {checksum}."
+            )
+        return ndarray_copy(payload_view, out)
+
+    @staticmethod
+    @abc.abstractmethod
+    def checksum(data: Buffer, value: int) -> int: ...
+
+
+class CRC32(Checksum32):
+    """Codec add a crc32 checksum to the buffer.
+
+    Parameters
+    ----------
+    location : 'start' or 'end'
+        Where to place the checksum in the buffer.
+    """
+
+    codec_id = 'crc32'
+    location = 'start'
+
+    @staticmethod
+    def checksum(data: Buffer, value: int = 0) -> int:
+        """
+        Thin wrapper around ``zlib.crc32``.
+        """
+        return zlib.crc32(data, value)
+
+
+class Adler32(Checksum32):
+    """Codec add a adler32 checksum to the buffer.
+
+    Parameters
+    ----------
+    location : 'start' or 'end'
+        Where to place the checksum in the buffer.
+    """
+
+    codec_id = 'adler32'
+    location = 'start'
+
+    @staticmethod
+    def checksum(data: Buffer, value: int = 1) -> int:
+        """
+        Thin wrapper around ``zlib.adler32``.
+        """
+        return zlib.adler32(data, value)
+
+
+class JenkinsLookup3(Checksum32):
+    """Bob Jenkin's lookup3 checksum with 32-bit output
+
+    This is the HDF5 implementation.
+    https://github.com/HDFGroup/hdf5/blob/577c192518598c7e2945683655feffcdbdf5a91b/src/H5checksum.c#L378-L472
+
+    With this codec, the checksum is concatenated on the end of the data
+    bytes when encoded. At decode time, the checksum is performed on
+    the data portion and compared with the four-byte checksum, raising
+    RuntimeError if inconsistent.
+
+    Parameters
+    ----------
+    initval : int
+        initial seed passed to the hash algorithm, default: 0
+    prefix : int
+        bytes prepended to the buffer before evaluating the hash, default: None
+    """
+
+    checksum = jenkins_lookup3
+    codec_id = "jenkins_lookup3"
+
+    def __init__(self, initval: int = 0, prefix=None):
+        self.initval = initval
+        if prefix is None:
+            self.prefix = None
+        else:
+            self.prefix = np.frombuffer(prefix, dtype='uint8')
+
+    def encode(self, buf):
+        """Return buffer plus 4-byte Bob Jenkin's lookup3 checksum"""
+        buf = ensure_contiguous_ndarray(buf).ravel().view('uint8')
+        if self.prefix is None:
+            val = jenkins_lookup3(buf, self.initval)
+        else:
+            val = jenkins_lookup3(np.hstack((self.prefix, buf)), self.initval)
+        return buf.tobytes() + struct.pack("<I", val)
+
+    def decode(self, buf, out=None):
+        """Check Bob Jenkin's lookup3 checksum, and return buffer without it"""
+        b = ensure_contiguous_ndarray(buf).view('uint8')
+        if self.prefix is None:
+            val = jenkins_lookup3(b[:-4], self.initval)
+        else:
+            val = jenkins_lookup3(np.hstack((self.prefix, b[:-4])), self.initval)
+        found = b[-4:].view("<u4")[0]
+        if val != found:
+            raise RuntimeError(
+                f"The Bob Jenkin's lookup3 checksum of the data ({val}) did not"
+                f" match the expected checksum ({found}).\n"
+                "This could be a sign that the data has been corrupted."
+            )
+        if out is not None:
+            out.view("uint8")[:] = b[:-4]
+            return out
+        return memoryview(b[:-4])
+
+
+if _crc32c:
+
+    class CRC32C(Checksum32):
+        """Codec add a crc32c checksum to the buffer.
+
+        Parameters
+        ----------
+        location : 'start' or 'end'
+            Where to place the checksum in the buffer.
+        """
+
+        codec_id = 'crc32c'
+        checksum = _crc32c.crc32c  # type: ignore[union-attr]
+        location = 'end'

numcodecs/compat.py

@@ -0,0 +1,206 @@
+import array
+import codecs
+
+import numpy as np
+
+from .ndarray_like import NDArrayLike, is_ndarray_like
+
+
+def ensure_ndarray_like(buf) -> NDArrayLike:
+    """Convenience function to coerce `buf` to ndarray-like array.
+
+    Parameters
+    ----------
+    buf : ndarray-like, array-like, or bytes-like
+        A numpy array like object such as numpy.ndarray, cupy.ndarray, or
+        any object exporting a buffer interface.
+
+    Returns
+    -------
+    arr : NDArrayLike
+        A ndarray-like, sharing memory with `buf`.
+
+    Notes
+    -----
+    This function will not create a copy under any circumstances, it is guaranteed to
+    return a view on memory exported by `buf`.
+    """
+
+    if not is_ndarray_like(buf):
+        if isinstance(buf, array.array) and buf.typecode in "cu":
+            # Guard condition, do not support array.array with unicode type, this is
+            # problematic because numpy does not support it on all platforms. Also do not
+            # support char as it was removed in Python 3.
+            raise TypeError("array.array with char or unicode type is not supported")
+        else:
+            # N.B., first take a memoryview to make sure that we subsequently create a
+            # numpy array from a memory buffer with no copy
+            mem = memoryview(buf)
+            # instantiate array from memoryview, ensures no copy
+            buf = np.array(mem, copy=False)
+    return buf
+
+
+def ensure_ndarray(buf) -> np.ndarray:
+    """Convenience function to coerce `buf` to a numpy array, if it is not already a
+    numpy array.
+
+    Parameters
+    ----------
+    buf : array-like or bytes-like
+        A numpy array or any object exporting a buffer interface.
+
+    Returns
+    -------
+    arr : ndarray
+        A numpy array, sharing memory with `buf`.
+
+    Notes
+    -----
+    This function will not create a copy under any circumstances, it is guaranteed to
+    return a view on memory exported by `buf`.
+    """
+    return np.array(ensure_ndarray_like(buf), copy=False)
+
+
+def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> NDArrayLike:
+    """Convenience function to coerce `buf` to ndarray-like array.
+    Also ensures that the returned value exports fully contiguous memory,
+    and supports the new-style buffer interface. If the optional max_buffer_size is
+    provided, raise a ValueError if the number of bytes consumed by the returned
+    array exceeds this value.
+
+    Parameters
+    ----------
+    buf : ndarray-like, array-like, or bytes-like
+        A numpy array like object such as numpy.ndarray, cupy.ndarray, or
+        any object exporting a buffer interface.
+    max_buffer_size : int
+        If specified, the largest allowable value of arr.nbytes, where arr
+        is the returned array.
+    flatten : bool
+        If True, the array are flatten.
+
+    Returns
+    -------
+    arr : NDArrayLike
+        A ndarray-like, sharing memory with `buf`.
+
+    Notes
+    -----
+    This function will not create a copy under any circumstances, it is guaranteed to
+    return a view on memory exported by `buf`.
+    """
+    arr = ensure_ndarray_like(buf)
+
+    # check for object arrays, these are just memory pointers, actual memory holding
+    # item data is scattered elsewhere
+    if arr.dtype == object:
+        raise TypeError("object arrays are not supported")
+
+    # check for datetime or timedelta ndarray, the buffer interface doesn't support those
+    if arr.dtype.kind in "Mm":
+        arr = arr.view(np.int64)  # type: ignore[arg-type]
+
+    # check memory is contiguous, if so flatten
+    if arr.flags.c_contiguous or arr.flags.f_contiguous:
+        if flatten:
+            # can flatten without copy
+            arr = arr.reshape(-1, order="A")
+    else:
+        raise ValueError("an array with contiguous memory is required")
+
+    if max_buffer_size is not None and arr.nbytes > max_buffer_size:
+        msg = f"Codec does not support buffers of > {max_buffer_size} bytes"
+        raise ValueError(msg)
+
+    return arr
+
+
+def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.ndarray:
+    """Convenience function to coerce `buf` to a numpy array, if it is not already a
+    numpy array. Also ensures that the returned value exports fully contiguous memory,
+    and supports the new-style buffer interface. If the optional max_buffer_size is
+    provided, raise a ValueError if the number of bytes consumed by the returned
+    array exceeds this value.
+
+    Parameters
+    ----------
+    buf : array-like or bytes-like
+        A numpy array or any object exporting a buffer interface.
+    max_buffer_size : int
+        If specified, the largest allowable value of arr.nbytes, where arr
+        is the returned array.
+    flatten : bool
+        If True, the array are flatten.
+
+    Returns
+    -------
+    arr : ndarray
+        A numpy array, sharing memory with `buf`.
+
+    Notes
+    -----
+    This function will not create a copy under any circumstances, it is guaranteed to
+    return a view on memory exported by `buf`.
+    """
+
+    return ensure_ndarray(
+        ensure_contiguous_ndarray_like(buf, max_buffer_size=max_buffer_size, flatten=flatten)
+    )
+
+
+def ensure_bytes(buf) -> bytes:
+    """Obtain a bytes object from memory exposed by `buf`."""
+
+    if not isinstance(buf, bytes):
+        arr = ensure_ndarray_like(buf)
+
+        # check for object arrays, these are just memory pointers,
+        # actual memory holding item data is scattered elsewhere
+        if arr.dtype == object:
+            raise TypeError("object arrays are not supported")
+
+        # create bytes
+        buf = arr.tobytes(order="A")
+
+    return buf
+
+
+def ensure_text(s, encoding="utf-8"):
+    if not isinstance(s, str):
+        s = ensure_contiguous_ndarray(s)
+        s = codecs.decode(s, encoding)
+    return s
+
+
+def ndarray_copy(src, dst) -> NDArrayLike:
+    """Copy the contents of the array from `src` to `dst`."""
+
+    if dst is None:
+        # no-op
+        return src
+
+    # ensure ndarray like
+    src = ensure_ndarray_like(src)
+    dst = ensure_ndarray_like(dst)
+
+    # flatten source array
+    src = src.reshape(-1, order="A")
+
+    # ensure same data type
+    if dst.dtype != object:
+        src = src.view(dst.dtype)
+
+    # reshape source to match destination
+    if src.shape != dst.shape:
+        if dst.flags.f_contiguous:
+            order = "F"
+        else:
+            order = "C"
+        src = src.reshape(dst.shape, order=order)
+
+    # copy via numpy
+    np.copyto(dst, src)
+
+    return dst

numcodecs/delta.py

@@ -0,0 +1,94 @@
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_ndarray, ndarray_copy
+
+
+class Delta(Codec):
+    """Codec to encode data as the difference between adjacent values.
+
+    Parameters
+    ----------
+    dtype : dtype
+        Data type to use for decoded data.
+    astype : dtype, optional
+        Data type to use for encoded data.
+
+    Notes
+    -----
+    If `astype` is an integer data type, please ensure that it is
+    sufficiently large to store encoded values. No checks are made and data
+    may become corrupted due to integer overflow if `astype` is too small.
+    Note also that the encoded data for each chunk includes the absolute
+    value of the first element in the chunk, and so the encoded data type in
+    general needs to be large enough to store absolute values from the array.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> x = np.arange(100, 120, 2, dtype='i2')
+    >>> codec = numcodecs.Delta(dtype='i2', astype='i1')
+    >>> y = codec.encode(x)
+    >>> y
+    array([100,   2,   2,   2,   2,   2,   2,   2,   2,   2], dtype=int8)
+    >>> z = codec.decode(y)
+    >>> z
+    array([100, 102, 104, 106, 108, 110, 112, 114, 116, 118], dtype=int16)
+
+    """
+
+    codec_id = 'delta'
+
+    def __init__(self, dtype, astype=None):
+        self.dtype = np.dtype(dtype)
+        if astype is None:
+            self.astype = self.dtype
+        else:
+            self.astype = np.dtype(astype)
+        if self.dtype == np.dtype(object) or self.astype == np.dtype(object):
+            raise ValueError('object arrays are not supported')
+
+    def encode(self, buf):
+        # normalise input
+        arr = ensure_ndarray(buf).view(self.dtype)
+
+        # flatten to simplify implementation
+        arr = arr.reshape(-1, order='A')
+
+        # setup encoded output
+        enc = np.empty_like(arr, dtype=self.astype)
+
+        # set first element
+        enc[0] = arr[0]
+
+        # compute differences
+        enc[1:] = np.diff(arr)
+        return enc
+
+    def decode(self, buf, out=None):
+        # normalise input
+        enc = ensure_ndarray(buf).view(self.astype)
+
+        # flatten to simplify implementation
+        enc = enc.reshape(-1, order='A')
+
+        # setup decoded output
+        dec = np.empty_like(enc, dtype=self.dtype)
+
+        # decode differences
+        np.cumsum(enc, out=dec)
+
+        # handle output
+        return ndarray_copy(dec, out)
+
+    def get_config(self):
+        # override to handle encoding dtypes
+        return {'id': self.codec_id, 'dtype': self.dtype.str, 'astype': self.astype.str}
+
+    def __repr__(self):
+        r = f'{type(self).__name__}(dtype={self.dtype.str!r}'
+        if self.astype != self.dtype:
+            r += f', astype={self.astype.str!r}'
+        r += ')'
+        return r

numcodecs/errors.py

@@ -0,0 +1,26 @@
+"""
+This module defines custom exceptions that are raised in the `numcodecs` codebase.
+"""
+
+
+class UnknownCodecError(ValueError):
+    """
+    An exception that is raised when trying to receive a codec that has not been registered.
+
+    Parameters
+    ----------
+    codec_id : str
+        Codec identifier.
+
+    Examples
+    ----------
+    >>> import numcodecs
+    >>> numcodecs.get_codec({"codec_id": "unknown"})
+    Traceback (most recent call last):
+        ...
+    UnknownCodecError: codec not available: 'unknown'
+    """
+
+    def __init__(self, codec_id: str):
+        self.codec_id = codec_id
+        super().__init__(f"codec not available: '{codec_id}'")

numcodecs/fixedscaleoffset.py

@@ -0,0 +1,130 @@
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_ndarray, ndarray_copy
+
+
+class FixedScaleOffset(Codec):
+    """Simplified version of the scale-offset filter available in HDF5.
+    Applies the transformation `(x - offset) * scale` to all chunks. Results
+    are rounded to the nearest integer but are not packed according to the
+    minimum number of bits.
+
+    Parameters
+    ----------
+    offset : float
+        Value to subtract from data.
+    scale : float
+        Value to multiply by data.
+    dtype : dtype
+        Data type to use for decoded data.
+    astype : dtype, optional
+        Data type to use for encoded data.
+
+    Notes
+    -----
+    If `astype` is an integer data type, please ensure that it is
+    sufficiently large to store encoded values. No checks are made and data
+    may become corrupted due to integer overflow if `astype` is too small.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> x = np.linspace(1000, 1001, 10, dtype='f8')
+    >>> x
+    array([1000.        , 1000.11111111, 1000.22222222, 1000.33333333,
+           1000.44444444, 1000.55555556, 1000.66666667, 1000.77777778,
+           1000.88888889, 1001.        ])
+    >>> codec = numcodecs.FixedScaleOffset(offset=1000, scale=10, dtype='f8', astype='u1')
+    >>> y1 = codec.encode(x)
+    >>> y1
+    array([ 0,  1,  2,  3,  4,  6,  7,  8,  9, 10], dtype=uint8)
+    >>> z1 = codec.decode(y1)
+    >>> z1
+    array([1000. , 1000.1, 1000.2, 1000.3, 1000.4, 1000.6, 1000.7,
+           1000.8, 1000.9, 1001. ])
+    >>> codec = numcodecs.FixedScaleOffset(offset=1000, scale=10**2, dtype='f8', astype='u1')
+    >>> y2 = codec.encode(x)
+    >>> y2
+    array([ 0,  11,  22,  33,  44,  56,  67,  78,  89, 100], dtype=uint8)
+    >>> z2 = codec.decode(y2)
+    >>> z2
+    array([1000.  , 1000.11, 1000.22, 1000.33, 1000.44, 1000.56,
+           1000.67, 1000.78, 1000.89, 1001.  ])
+    >>> codec = numcodecs.FixedScaleOffset(offset=1000, scale=10**3, dtype='f8', astype='u2')
+    >>> y3 = codec.encode(x)
+    >>> y3
+    array([ 0,  111,  222,  333,  444,  556,  667,  778,  889, 1000], dtype=uint16)
+    >>> z3 = codec.decode(y3)
+    >>> z3
+    array([1000.   , 1000.111, 1000.222, 1000.333, 1000.444, 1000.556,
+           1000.667, 1000.778, 1000.889, 1001.   ])
+
+    See Also
+    --------
+    numcodecs.quantize.Quantize
+
+    """
+
+    codec_id = 'fixedscaleoffset'
+
+    def __init__(self, offset, scale, dtype, astype=None):
+        self.offset = offset
+        self.scale = scale
+        self.dtype = np.dtype(dtype)
+        if astype is None:
+            self.astype = self.dtype
+        else:
+            self.astype = np.dtype(astype)
+        if self.dtype == np.dtype(object) or self.astype == np.dtype(object):
+            raise ValueError('object arrays are not supported')
+
+    def encode(self, buf):
+        # normalise input
+        arr = ensure_ndarray(buf).view(self.dtype)
+
+        # flatten to simplify implementation
+        arr = arr.reshape(-1, order='A')
+
+        # compute scale offset
+        enc = (arr - self.offset) * self.scale
+
+        # round to nearest integer
+        enc = np.around(enc)
+
+        # convert dtype
+        return enc.astype(self.astype, copy=False)
+
+    def decode(self, buf, out=None):
+        # interpret buffer as numpy array
+        enc = ensure_ndarray(buf).view(self.astype)
+
+        # flatten to simplify implementation
+        enc = enc.reshape(-1, order='A')
+
+        # decode scale offset
+        dec = (enc / self.scale) + self.offset
+
+        # convert dtype
+        dec = dec.astype(self.dtype, copy=False)
+
+        # handle output
+        return ndarray_copy(dec, out)
+
+    def get_config(self):
+        # override to handle encoding dtypes
+        return {
+            'id': self.codec_id,
+            'scale': self.scale,
+            'offset': self.offset,
+            'dtype': self.dtype.str,
+            'astype': self.astype.str,
+        }
+
+    def __repr__(self):
+        r = f'{type(self).__name__}(scale={self.scale}, offset={self.offset}, dtype={self.dtype.str!r}'
+        if self.astype != self.dtype:
+            r += f', astype={self.astype.str!r}'
+        r += ')'
+        return r

numcodecs/gzip.py

@@ -0,0 +1,50 @@
+import gzip as _gzip
+import io
+
+from .abc import Codec
+from .compat import ensure_bytes, ensure_contiguous_ndarray
+
+
+class GZip(Codec):
+    """Codec providing gzip compression using zlib via the Python standard library.
+
+    Parameters
+    ----------
+    level : int
+        Compression level.
+
+    """
+
+    codec_id = 'gzip'
+
+    def __init__(self, level=1):
+        self.level = level
+
+    def encode(self, buf):
+        # normalise inputs
+        buf = ensure_contiguous_ndarray(buf)
+
+        # do compression
+        compressed = io.BytesIO()
+        with _gzip.GzipFile(fileobj=compressed, mode='wb', compresslevel=self.level) as compressor:
+            compressor.write(buf)
+        return compressed.getvalue()
+
+    # noinspection PyMethodMayBeStatic
+    def decode(self, buf, out=None):
+        # normalise inputs
+        # BytesIO only copies if the data is not of `bytes` type.
+        # This allows `bytes` objects to pass through without copying.
+        buf = io.BytesIO(ensure_bytes(buf))
+
+        # do decompression
+        with _gzip.GzipFile(fileobj=buf, mode='rb') as decompressor:
+            if out is not None:
+                out_view = ensure_contiguous_ndarray(out)
+                decompressor.readinto(out_view)
+                if decompressor.read(1) != b'':
+                    raise ValueError("Unable to fit data into `out`")
+            else:
+                out = decompressor.read()
+
+        return out