numcodecs 0.13.1__cp312-cp312-macosx_10_13_x86_64.whl

numcodecs/__init__.py

@@ -0,0 +1,143 @@
+# ruff: noqa: E402,F401
+"""Numcodecs is a Python package providing buffer compression and
+transformation codecs for use in data storage and communication
+applications. These include:
+
+* Compression codecs, e.g., Zlib, BZ2, LZMA, ZFPY and Blosc.
+* Pre-compression filters, e.g., Delta, Quantize, FixedScaleOffset,
+  PackBits, Categorize.
+* Integrity checks, e.g., CRC32, Adler32.
+
+All codecs implement the same API, allowing codecs to be organized into
+pipelines in a variety of ways.
+
+If you have a question, find a bug, would like to make a suggestion or
+contribute code, please `raise an issue on GitHub
+<https://github.com/zarr-developers/numcodecs/issues>`_.
+
+"""
+
+import atexit
+import multiprocessing
+from contextlib import suppress
+
+from numcodecs.registry import get_codec, register_codec
+from numcodecs.version import version as __version__
+from numcodecs.zlib import Zlib
+
+register_codec(Zlib)
+
+from numcodecs.gzip import GZip
+
+register_codec(GZip)
+
+from numcodecs.bz2 import BZ2
+
+register_codec(BZ2)
+
+with suppress(ImportError):
+    from numcodecs.lzma import LZMA
+
+    register_codec(LZMA)
+
+with suppress(ImportError):
+    from numcodecs import blosc
+    from numcodecs.blosc import Blosc
+
+    register_codec(Blosc)
+    # initialize blosc
+    try:
+        ncores = multiprocessing.cpu_count()
+    except OSError:  # pragma: no cover
+        ncores = 1
+    blosc.init()
+    blosc.set_nthreads(min(8, ncores))
+    atexit.register(blosc.destroy)
+
+with suppress(ImportError):
+    from numcodecs import zstd
+    from numcodecs.zstd import Zstd
+
+    register_codec(Zstd)
+
+with suppress(ImportError):
+    from numcodecs import lz4
+    from numcodecs.lz4 import LZ4
+
+    register_codec(LZ4)
+
+with suppress(ImportError):
+    from numcodecs.zfpy import ZFPY
+
+    register_codec(ZFPY)
+
+from numcodecs.astype import AsType
+
+register_codec(AsType)
+
+from numcodecs.delta import Delta
+
+register_codec(Delta)
+
+from numcodecs.quantize import Quantize
+
+register_codec(Quantize)
+
+from numcodecs.fixedscaleoffset import FixedScaleOffset
+
+register_codec(FixedScaleOffset)
+
+from numcodecs.packbits import PackBits
+
+register_codec(PackBits)
+
+from numcodecs.categorize import Categorize
+
+register_codec(Categorize)
+
+from numcodecs.pickles import Pickle
+
+register_codec(Pickle)
+
+from numcodecs.base64 import Base64
+
+register_codec(Base64)
+
+from numcodecs.shuffle import Shuffle
+
+register_codec(Shuffle)
+
+from numcodecs.bitround import BitRound
+
+register_codec(BitRound)
+
+with suppress(ImportError):
+    from numcodecs.msgpacks import MsgPack
+
+    register_codec(MsgPack)
+
+from numcodecs.checksum32 import CRC32, Adler32, JenkinsLookup3
+
+register_codec(CRC32)
+register_codec(Adler32)
+register_codec(JenkinsLookup3)
+
+from numcodecs.json import JSON
+
+register_codec(JSON)
+
+with suppress(ImportError):
+    from numcodecs import vlen
+    from numcodecs.vlen import VLenArray, VLenBytes, VLenUTF8
+
+    register_codec(VLenUTF8)
+    register_codec(VLenBytes)
+    register_codec(VLenArray)
+
+from numcodecs.fletcher32 import Fletcher32
+
+register_codec(Fletcher32)
+
+from numcodecs.pcodec import PCodec
+
+register_codec(PCodec)

numcodecs/abc.py

@@ -0,0 +1,126 @@
+"""This module defines the :class:`Codec` base class, a common interface for
+all codec classes.
+
+Codec classes must implement :func:`Codec.encode` and :func:`Codec.decode`
+methods. Inputs to and outputs from these methods may be any Python object
+exporting a contiguous buffer via the new-style Python protocol.
+
+Codec classes must implement a :func:`Codec.get_config` method,
+which must return a dictionary holding all configuration parameters
+required to enable encoding and decoding of data. The expectation is that
+these configuration parameters will be stored or communicated separately
+from encoded data, and thus the codecs do not need to store all encoding
+parameters within the encoded data. For broad compatibility,
+the configuration object must contain only JSON-serializable values. The
+configuration object must also contain an 'id' field storing the codec
+identifier (see below).
+
+Codec classes must implement a :func:`Codec.from_config` class method,
+which will return an instance of the class initialized from a configuration
+object.
+
+Finally, codec classes must set a `codec_id` class-level attribute. This
+must be a string. Two different codec classes may set the same value for the
+`codec_id` attribute if and only if they are fully compatible, meaning that
+(1) configuration parameters are the same, and (2) given the same
+configuration, one class could correctly decode data encoded by the
+other and vice versa.
+
+"""
+
+from abc import ABC, abstractmethod
+
+
+class Codec(ABC):
+    """Codec abstract base class."""
+
+    # override in sub-class
+    codec_id = None
+    """Codec identifier."""
+
+    @abstractmethod
+    def encode(self, buf):  # pragma: no cover
+        """Encode data in `buf`.
+
+        Parameters
+        ----------
+        buf : buffer-like
+            Data to be encoded. May be any object supporting the new-style
+            buffer protocol.
+
+        Returns
+        -------
+        enc : buffer-like
+            Encoded data. May be any object supporting the new-style buffer
+            protocol.
+        """
+
+    @abstractmethod
+    def decode(self, buf, out=None):  # pragma: no cover
+        """Decode data in `buf`.
+
+        Parameters
+        ----------
+        buf : buffer-like
+            Encoded data. May be any object supporting the new-style buffer
+            protocol.
+        out : buffer-like, optional
+            Writeable buffer to store decoded data. N.B. if provided, this buffer must
+            be exactly the right size to store the decoded data.
+
+        Returns
+        -------
+        dec : buffer-like
+            Decoded data. May be any object supporting the new-style
+            buffer protocol.
+        """
+
+    def get_config(self):
+        """Return a dictionary holding configuration parameters for this
+        codec. Must include an 'id' field with the codec identifier. All
+        values must be compatible with JSON encoding."""
+
+        # override in sub-class if need special encoding of config values
+
+        # setup config object
+        config = dict(id=self.codec_id)
+
+        # by default, assume all non-private members are configuration
+        # parameters - override this in sub-class if not the case
+        for k in self.__dict__:
+            if not k.startswith('_'):
+                config[k] = getattr(self, k)
+
+        return config
+
+    @classmethod
+    def from_config(cls, config):
+        """Instantiate codec from a configuration object."""
+        # N.B., assume at this point the 'id' field has been removed from
+        # the config object
+
+        # override in sub-class if need special decoding of config values
+
+        # by default, assume constructor accepts configuration parameters as
+        # keyword arguments without any special decoding
+        return cls(**config)
+
+    def __eq__(self, other):
+        # override in sub-class if need special equality comparison
+        try:
+            return self.get_config() == other.get_config()
+        except AttributeError:
+            return False
+
+    def __repr__(self):
+        # override in sub-class if need special representation
+
+        # by default, assume all non-private members are configuration
+        # parameters and valid keyword arguments to constructor function
+
+        r = f'{type(self).__name__}('
+        params = [
+            f'{k}={getattr(self, k)!r}' for k in sorted(self.__dict__) if not k.startswith('_')
+        ]
+        r += ', '.join(params) + ')'
+        return r

numcodecs/astype.py

@@ -0,0 +1,76 @@
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_ndarray, ndarray_copy
+
+
+class AsType(Codec):
+    """Filter to convert data between different types.
+
+    Parameters
+    ----------
+    encode_dtype : dtype
+        Data type to use for encoded data.
+    decode_dtype : dtype, optional
+        Data type to use for decoded data.
+
+    Notes
+    -----
+    If `encode_dtype` is of lower precision than `decode_dtype`, please be
+    aware that data loss can occur by writing data to disk using this filter.
+    No checks are made to ensure the casting will work in that direction and
+    data corruption will occur.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> x = np.arange(100, 120, 2, dtype=np.int8)
+    >>> x
+    array([100, 102, 104, 106, 108, 110, 112, 114, 116, 118], dtype=int8)
+    >>> f = numcodecs.AsType(encode_dtype=x.dtype, decode_dtype=np.int16)
+    >>> y = f.decode(x)
+    >>> y
+    array([100, 102, 104, 106, 108, 110, 112, 114, 116, 118], dtype=int16)
+    >>> z = f.encode(y)
+    >>> z
+    array([100, 102, 104, 106, 108, 110, 112, 114, 116, 118], dtype=int8)
+
+    """
+
+    codec_id = 'astype'
+
+    def __init__(self, encode_dtype, decode_dtype):
+        self.encode_dtype = np.dtype(encode_dtype)
+        self.decode_dtype = np.dtype(decode_dtype)
+
+    def encode(self, buf):
+        # normalise input
+        arr = ensure_ndarray(buf).view(self.decode_dtype)
+
+        # convert and copy
+        enc = arr.astype(self.encode_dtype)
+
+        return enc
+
+    def decode(self, buf, out=None):
+        # normalise input
+        enc = ensure_ndarray(buf).view(self.encode_dtype)
+
+        # convert and copy
+        dec = enc.astype(self.decode_dtype)
+
+        # handle output
+        out = ndarray_copy(dec, out)
+
+        return out
+
+    def get_config(self):
+        return {
+            'id': self.codec_id,
+            'encode_dtype': self.encode_dtype.str,
+            'decode_dtype': self.decode_dtype.str,
+        }
+
+    def __repr__(self):
+        return f'{type(self).__name__}(encode_dtype={self.encode_dtype.str!r}, decode_dtype={self.decode_dtype.str!r})'

numcodecs/base64.py

@@ -0,0 +1,27 @@
+import base64 as _base64
+
+from .abc import Codec
+from .compat import ensure_contiguous_ndarray, ndarray_copy
+
+
+class Base64(Codec):
+    """Codec providing base64 compression via the Python standard library."""
+
+    codec_id = "base64"
+
+    def encode(self, buf):
+        # normalise inputs
+        buf = ensure_contiguous_ndarray(buf)
+        # do compression
+        compressed = _base64.standard_b64encode(buf)
+        return compressed
+
+    def decode(self, buf, out=None):
+        # normalise inputs
+        buf = ensure_contiguous_ndarray(buf)
+        if out is not None:
+            out = ensure_contiguous_ndarray(out)
+        # do decompression
+        decompressed = _base64.standard_b64decode(buf)
+        # handle destination
+        return ndarray_copy(decompressed, out)

numcodecs/bitround.py

@@ -0,0 +1,79 @@
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_ndarray_like, ndarray_copy
+
+# The size in bits of the mantissa/significand for the various floating types
+# You cannot keep more bits of data than you have available
+# https://en.wikipedia.org/wiki/IEEE_754
+max_bits = {
+    "float16": 10,
+    "float32": 23,
+    "float64": 52,
+}
+
+
+class BitRound(Codec):
+    """Floating-point bit rounding codec
+
+    Drops a specified number of bits from the floating point mantissa,
+    leaving an array more amenable to compression. The number of bits to keep should
+    be determined by an information analysis of the data to be compressed.
+    The approach is based on the paper by Klöwer et al. 2021
+    (https://www.nature.com/articles/s43588-021-00156-2). See
+    https://github.com/zarr-developers/numcodecs/issues/298 for discussion
+    and the original implementation in Julia referred to at
+    https://github.com/milankl/BitInformation.jl
+
+    Parameters
+    ----------
+
+    keepbits: int
+        The number of bits of the mantissa to keep. The range allowed
+        depends on the dtype input data. If keepbits is
+        equal to the maximum allowed for the data type, this is equivalent
+        to no transform.
+    """
+
+    codec_id = 'bitround'
+
+    def __init__(self, keepbits: int):
+        if keepbits < 0:
+            raise ValueError("keepbits must be zero or positive")
+        self.keepbits = keepbits
+
+    def encode(self, buf):
+        """Create int array by rounding floating-point data
+
+        The itemsize will be preserved, but the output should be much more
+        compressible.
+        """
+        a = ensure_ndarray_like(buf)
+        if not a.dtype.kind == "f" or a.dtype.itemsize > 8:
+            raise TypeError("Only float arrays (16-64bit) can be bit-rounded")
+        bits = max_bits[str(a.dtype)]
+        # cast float to int type of same width (preserve endianness)
+        a_int_dtype = np.dtype(a.dtype.str.replace("f", "i"))
+        all_set = np.array(-1, dtype=a_int_dtype)
+        if self.keepbits == bits:
+            return a
+        if self.keepbits > bits:
+            raise ValueError("Keepbits too large for given dtype")
+        b = a.view(a_int_dtype)
+        maskbits = bits - self.keepbits
+        mask = (all_set >> maskbits) << maskbits
+        half_quantum1 = (1 << (maskbits - 1)) - 1
+        b += ((b >> maskbits) & 1) + half_quantum1
+        b &= mask
+        return b
+
+    def decode(self, buf, out=None):
+        """Remake floats from ints
+
+        As with ``encode``, preserves itemsize.
+        """
+        buf = ensure_ndarray_like(buf)
+        # Cast back from `int` to `float` type (noop if a `float`ing type buffer is provided)
+        dt = np.dtype(buf.dtype.str.replace("i", "f"))
+        data = buf.view(dt)
+        return ndarray_copy(data, out)

numcodecs/bz2.py

@@ -0,0 +1,45 @@
+import bz2 as _bz2
+
+from numcodecs.abc import Codec
+from numcodecs.compat import ensure_contiguous_ndarray, ndarray_copy
+
+
+class BZ2(Codec):
+    """Codec providing compression using bzip2 via the Python standard library.
+
+    Parameters
+    ----------
+    level : int
+        Compression level.
+
+    """
+
+    codec_id = 'bz2'
+
+    def __init__(self, level=1):
+        self.level = level
+
+    def encode(self, buf):
+        # normalise input
+        buf = ensure_contiguous_ndarray(buf)
+
+        # do compression
+        return _bz2.compress(buf, self.level)
+
+    # noinspection PyMethodMayBeStatic
+    def decode(self, buf, out=None):
+        # normalise inputs
+        buf = ensure_contiguous_ndarray(buf)
+        if out is not None:
+            out = ensure_contiguous_ndarray(out)
+
+        # N.B., bz2 cannot handle ndarray directly because of truth testing issues
+        buf = memoryview(buf)
+
+        # do decompression
+        dec = _bz2.decompress(buf)
+
+        # handle destination - Python standard library bz2 module does not
+        # support direct decompression into buffer, so we have to copy into
+        # out if given
+        return ndarray_copy(dec, out)

numcodecs/categorize.py

@@ -0,0 +1,101 @@
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_ndarray, ensure_text, ndarray_copy
+
+
+class Categorize(Codec):
+    """Filter encoding categorical string data as integers.
+
+    Parameters
+    ----------
+    labels : sequence of strings
+        Category labels.
+    dtype : dtype
+        Data type to use for decoded data.
+    astype : dtype, optional
+        Data type to use for encoded data.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> x = np.array(['male', 'female', 'female', 'male', 'unexpected'], dtype=object)
+    >>> x
+    array(['male', 'female', 'female', 'male', 'unexpected'],
+          dtype=object)
+    >>> codec = numcodecs.Categorize(labels=['female', 'male'], dtype=object)
+    >>> y = codec.encode(x)
+    >>> y
+    array([2, 1, 1, 2, 0], dtype=uint8)
+    >>> z = codec.decode(y)
+    >>> z
+    array(['male', 'female', 'female', 'male', ''],
+          dtype=object)
+
+    """
+
+    codec_id = 'categorize'
+
+    def __init__(self, labels, dtype, astype='u1'):
+        self.dtype = np.dtype(dtype)
+        if self.dtype.kind not in 'UO':
+            raise TypeError("only unicode ('U') and object ('O') dtypes are supported")
+        self.labels = [ensure_text(label) for label in labels]
+        self.astype = np.dtype(astype)
+        if self.astype == np.dtype(object):
+            raise TypeError('encoding as object array not supported')
+
+    def encode(self, buf):
+        # normalise input
+        if self.dtype == np.dtype(object):
+            arr = np.asarray(buf, dtype=object)
+        else:
+            arr = ensure_ndarray(buf).view(self.dtype)
+
+        # flatten to simplify implementation
+        arr = arr.reshape(-1, order='A')
+
+        # setup output array
+        enc = np.zeros_like(arr, dtype=self.astype)
+
+        # apply encoding, reserving 0 for values not specified in labels
+        for i, label in enumerate(self.labels):
+            enc[arr == label] = i + 1
+
+        return enc
+
+    def decode(self, buf, out=None):
+        # normalise input
+        enc = ensure_ndarray(buf).view(self.astype)
+
+        # flatten to simplify implementation
+        enc = enc.reshape(-1, order='A')
+
+        # setup output
+        dec = np.full_like(enc, fill_value='', dtype=self.dtype)
+
+        # apply decoding
+        for i, label in enumerate(self.labels):
+            dec[enc == (i + 1)] = label
+
+        # handle output
+        dec = ndarray_copy(dec, out)
+
+        return dec
+
+    def get_config(self):
+        config = dict(
+            id=self.codec_id,
+            labels=self.labels,
+            dtype=self.dtype.str,
+            astype=self.astype.str,
+        )
+        return config
+
+    def __repr__(self):
+        # make sure labels part is not too long
+        labels = repr(self.labels[:3])
+        if len(self.labels) > 3:
+            labels = labels[:-1] + ', ...]'
+        return f'{type(self).__name__}(dtype={self.dtype.str!r}, astype={self.astype.str!r}, labels={labels})'

numcodecs/checksum32.py

@@ -0,0 +1,94 @@
+import struct
+import zlib
+
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_contiguous_ndarray, ndarray_copy
+from .jenkins import jenkins_lookup3
+
+
+class Checksum32(Codec):
+    # override in sub-class
+    checksum = None
+
+    def encode(self, buf):
+        arr = ensure_contiguous_ndarray(buf).view('u1')
+        checksum = self.checksum(arr) & 0xFFFFFFFF
+        enc = np.empty(arr.nbytes + 4, dtype='u1')
+        enc[:4].view('<u4')[0] = checksum
+        ndarray_copy(arr, enc[4:])
+        return enc
+
+    def decode(self, buf, out=None):
+        arr = ensure_contiguous_ndarray(buf).view('u1')
+        expect = arr[:4].view('<u4')[0]
+        checksum = self.checksum(arr[4:]) & 0xFFFFFFFF
+        if expect != checksum:
+            raise RuntimeError('checksum failed')
+        return ndarray_copy(arr[4:], out)
+
+
+class CRC32(Checksum32):
+    codec_id = 'crc32'
+    checksum = zlib.crc32
+
+
+class Adler32(Checksum32):
+    codec_id = 'adler32'
+    checksum = zlib.adler32
+
+
+class JenkinsLookup3(Checksum32):
+    """Bob Jenkin's lookup3 checksum with 32-bit output
+
+    This is the HDF5 implementation.
+    https://github.com/HDFGroup/hdf5/blob/577c192518598c7e2945683655feffcdbdf5a91b/src/H5checksum.c#L378-L472
+
+    With this codec, the checksum is concatenated on the end of the data
+    bytes when encoded. At decode time, the checksum is performed on
+    the data portion and compared with the four-byte checksum, raising
+    RuntimeError if inconsistent.
+
+    Attributes:
+       initval: initial seed passed to the hash algorithm, default: 0
+       prefix: bytes prepended to the buffer before evaluating the hash, default: None
+    """
+
+    checksum = jenkins_lookup3
+    codec_id = "jenkins_lookup3"
+
+    def __init__(self, initval: int = 0, prefix=None):
+        self.initval = initval
+        if prefix is None:
+            self.prefix = None
+        else:
+            self.prefix = np.frombuffer(prefix, dtype='uint8')
+
+    def encode(self, buf):
+        """Return buffer plus 4-byte Bob Jenkin's lookup3 checksum"""
+        buf = ensure_contiguous_ndarray(buf).ravel().view('uint8')
+        if self.prefix is None:
+            val = jenkins_lookup3(buf, self.initval)
+        else:
+            val = jenkins_lookup3(np.hstack((self.prefix, buf)), self.initval)
+        return buf.tobytes() + struct.pack("<I", val)
+
+    def decode(self, buf, out=None):
+        """Check Bob Jenkin's lookup3 checksum, and return buffer without it"""
+        b = ensure_contiguous_ndarray(buf).view('uint8')
+        if self.prefix is None:
+            val = jenkins_lookup3(b[:-4], self.initval)
+        else:
+            val = jenkins_lookup3(np.hstack((self.prefix, b[:-4])), self.initval)
+        found = b[-4:].view("<u4")[0]
+        if val != found:
+            raise RuntimeError(
+                f"The Bob Jenkin's lookup3 checksum of the data ({val}) did not"
+                f" match the expected checksum ({found}).\n"
+                "This could be a sign that the data has been corrupted."
+            )
+        if out is not None:
+            out.view("uint8")[:] = b[:-4]
+            return out
+        return memoryview(b[:-4])