numcodecs 0.13.1__cp313-cp313-macosx_10_13_x86_64.whl

numcodecs/msgpacks.py

@@ -0,0 +1,86 @@
+import msgpack
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_contiguous_ndarray
+
+
+class MsgPack(Codec):
+    """Codec to encode data as msgpacked bytes. Useful for encoding an array of Python
+    objects.
+
+    .. versionchanged:: 0.6
+        The encoding format has been changed to include the array shape in the encoded
+        data, which ensures that all object arrays can be correctly encoded and decoded.
+
+    Parameters
+    ----------
+    use_single_float : bool, optional
+        Use single precision float type for float.
+    use_bin_type : bool, optional
+        Use bin type introduced in msgpack spec 2.0 for bytes. It also enables str8 type
+        for unicode.
+    raw : bool, optional
+        If true, unpack msgpack raw to Python bytes. Otherwise, unpack to Python str
+        by decoding with UTF-8 encoding.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> x = np.array(['foo', 'bar', 'baz'], dtype='object')
+    >>> codec = numcodecs.MsgPack()
+    >>> codec.decode(codec.encode(x))
+    array(['foo', 'bar', 'baz'], dtype=object)
+
+    See Also
+    --------
+    numcodecs.pickles.Pickle, numcodecs.json.JSON, numcodecs.vlen.VLenUTF8
+
+    Notes
+    -----
+    Requires `msgpack <https://pypi.org/project/msgpack/>`_ to be installed.
+
+    """
+
+    codec_id = 'msgpack2'
+
+    def __init__(self, use_single_float=False, use_bin_type=True, raw=False):
+        self.use_single_float = use_single_float
+        self.use_bin_type = use_bin_type
+        self.raw = raw
+
+    def encode(self, buf):
+        try:
+            buf = np.asarray(buf)
+        except ValueError:
+            buf = np.asarray(buf, dtype=object)
+        items = buf.tolist()
+        items.extend((buf.dtype.str, buf.shape))
+        return msgpack.packb(
+            items,
+            use_bin_type=self.use_bin_type,
+            use_single_float=self.use_single_float,
+        )
+
+    def decode(self, buf, out=None):
+        buf = ensure_contiguous_ndarray(buf)
+        items = msgpack.unpackb(buf, raw=self.raw)
+        dec = np.empty(items[-1], dtype=items[-2])
+        dec[:] = items[:-2]
+        if out is not None:
+            np.copyto(out, dec)
+            return out
+        else:
+            return dec
+
+    def get_config(self):
+        return dict(
+            id=self.codec_id,
+            raw=self.raw,
+            use_single_float=self.use_single_float,
+            use_bin_type=self.use_bin_type,
+        )
+
+    def __repr__(self):
+        return f'MsgPack(raw={self.raw!r}, use_bin_type={self.use_bin_type!r}, use_single_float={self.use_single_float!r})'

numcodecs/ndarray_like.py

@@ -0,0 +1,65 @@
+from typing import Any, ClassVar, Optional, Protocol, runtime_checkable
+
+
+class _CachedProtocolMeta(Protocol.__class__):
+    """Custom implementation of @runtime_checkable
+
+    The native implementation of @runtime_checkable is slow,
+    see <https://github.com/zarr-developers/numcodecs/issues/379>.
+
+    This metaclass keeps an unbounded cache of the result of
+    isinstance checks using the object's class as the cache key.
+    """
+
+    _instancecheck_cache: ClassVar[dict[tuple[type, type], bool]] = {}
+
+    def __instancecheck__(self, instance):
+        key = (self, instance.__class__)
+        ret = self._instancecheck_cache.get(key)
+        if ret is None:
+            ret = super().__instancecheck__(instance)
+            self._instancecheck_cache[key] = ret
+        return ret
+
+
+@runtime_checkable
+class DType(Protocol, metaclass=_CachedProtocolMeta):
+    itemsize: int
+    name: str
+    kind: str
+
+
+@runtime_checkable
+class FlagsObj(Protocol, metaclass=_CachedProtocolMeta):
+    c_contiguous: bool
+    f_contiguous: bool
+    owndata: bool
+
+
+@runtime_checkable
+class NDArrayLike(Protocol, metaclass=_CachedProtocolMeta):
+    dtype: DType
+    shape: tuple[int, ...]
+    strides: tuple[int, ...]
+    ndim: int
+    size: int
+    itemsize: int
+    nbytes: int
+    flags: FlagsObj
+
+    def __len__(self) -> int: ...  # pragma: no cover
+
+    def __getitem__(self, key) -> Any: ...  # pragma: no cover
+
+    def __setitem__(self, key, value): ...  # pragma: no cover
+
+    def tobytes(self, order: Optional[str] = ...) -> bytes: ...  # pragma: no cover
+
+    def reshape(self, *shape: int, order: str = ...) -> "NDArrayLike": ...  # pragma: no cover
+
+    def view(self, dtype: DType = ...) -> "NDArrayLike": ...  # pragma: no cover
+
+
+def is_ndarray_like(obj: object) -> bool:
+    """Return True when `obj` is ndarray-like"""
+    return isinstance(obj, NDArrayLike)

numcodecs/packbits.py

@@ -0,0 +1,85 @@
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_ndarray, ndarray_copy
+
+
+class PackBits(Codec):
+    """Codec to pack elements of a boolean array into bits in a uint8 array.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> codec = numcodecs.PackBits()
+    >>> x = np.array([True, False, False, True], dtype=bool)
+    >>> y = codec.encode(x)
+    >>> y
+    array([  4, 144], dtype=uint8)
+    >>> z = codec.decode(y)
+    >>> z
+    array([ True, False, False,  True])
+
+    Notes
+    -----
+    The first element of the encoded array stores the number of bits that
+    were padded to complete the final byte.
+
+    """
+
+    codec_id = 'packbits'
+
+    def __init__(self):
+        pass
+
+    def encode(self, buf):
+        # normalise input
+        arr = ensure_ndarray(buf).view(bool)
+
+        # flatten to simplify implementation
+        arr = arr.reshape(-1, order='A')
+
+        # determine size of packed data
+        n = arr.size
+        n_bytes_packed = n // 8
+        n_bits_leftover = n % 8
+        if n_bits_leftover > 0:
+            n_bytes_packed += 1
+
+        # setup output
+        enc = np.empty(n_bytes_packed + 1, dtype='u1')
+
+        # store how many bits were padded
+        if n_bits_leftover:
+            n_bits_padded = 8 - n_bits_leftover
+        else:
+            n_bits_padded = 0
+        enc[0] = n_bits_padded
+
+        # apply encoding
+        enc[1:] = np.packbits(arr)
+
+        return enc
+
+    def decode(self, buf, out=None):
+        # normalise input
+        enc = ensure_ndarray(buf).view('u1')
+
+        # flatten to simplify implementation
+        enc = enc.reshape(-1, order='A')
+
+        # find out how many bits were padded
+        n_bits_padded = int(enc[0])
+
+        # apply decoding
+        dec = np.unpackbits(enc[1:])
+
+        # remove padded bits
+        if n_bits_padded:
+            dec = dec[:-n_bits_padded]
+
+        # view as boolean array
+        dec = dec.view(bool)
+
+        # handle destination
+        return ndarray_copy(dec, out)

numcodecs/pcodec.py

@@ -0,0 +1,89 @@
+from typing import Literal, Optional
+
+import numcodecs
+import numcodecs.abc
+from numcodecs.compat import ensure_contiguous_ndarray
+
+try:
+    from pcodec import ChunkConfig, ModeSpec, PagingSpec, standalone
+except ImportError:  # pragma: no cover
+    standalone = None
+
+
+DEFAULT_MAX_PAGE_N = 262144
+
+
+class PCodec(numcodecs.abc.Codec):
+    """
+    PCodec (or pco, pronounced "pico") losslessly compresses and decompresses
+    numerical sequences with high compression ratio and fast speed.
+
+    See `PCodec Repo <https://github.com/mwlon/pcodec>`_ for more information.
+
+    PCodec supports only the following numerical dtypes: uint16, uint32, uint64,
+    int16, int32, int64, float16, float32, and float64.
+
+    Parameters
+    ----------
+    level : int
+        A compression level from 0-12, where 12 take the longest and compresses
+        the most.
+    delta_encoding_order : init or None
+        Either a delta encoding level from 0-7 or None. If set to None, pcodec
+        will try to infer the optimal delta encoding order.
+    mode_spec : {'auto', 'classic'}
+        Configures whether Pcodec should try to infer the best "mode" or
+        structure of the data (e.g. approximate multiples of 0.1) to improve
+        compression ratio, or skip this step and just use the numbers as-is
+        (Classic mode).
+    equal_pages_up_to : int
+        Divide the chunk into equal pages of up to this many numbers.
+    """
+
+    codec_id = "pcodec"
+
+    def __init__(
+        self,
+        level: int = 8,
+        delta_encoding_order: Optional[int] = None,
+        equal_pages_up_to: int = 262144,
+        # TODO one day, add support for the Try* mode specs
+        mode_spec: Literal['auto', 'classic'] = 'auto',
+    ):
+        if standalone is None:  # pragma: no cover
+            raise ImportError("pcodec must be installed to use the PCodec codec.")
+
+        # note that we use `level` instead of `compression_level` to
+        # match other codecs
+        self.level = level
+        self.delta_encoding_order = delta_encoding_order
+        self.equal_pages_up_to = equal_pages_up_to
+        self.mode_spec = mode_spec
+
+    def encode(self, buf):
+        buf = ensure_contiguous_ndarray(buf)
+
+        match self.mode_spec:
+            case 'auto':
+                mode_spec = ModeSpec.auto()
+            case 'classic':
+                mode_spec = ModeSpec.classic()
+            case _:
+                raise ValueError(f"unknown value for mode_spec: {self.mode_spec}")
+        paging_spec = PagingSpec.equal_pages_up_to(self.equal_pages_up_to)
+
+        config = ChunkConfig(
+            compression_level=self.level,
+            delta_encoding_order=self.delta_encoding_order,
+            mode_spec=mode_spec,
+            paging_spec=paging_spec,
+        )
+        return standalone.simple_compress(buf, config)
+
+    def decode(self, buf, out=None):
+        if out is not None:
+            out = ensure_contiguous_ndarray(out)
+            standalone.simple_decompress_into(buf, out)
+            return out
+        else:
+            return standalone.simple_decompress(buf)

numcodecs/pickles.py

@@ -0,0 +1,55 @@
+import pickle
+
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_contiguous_ndarray
+
+
+class Pickle(Codec):
+    """Codec to encode data as as pickled bytes. Useful for encoding an array of Python string
+    objects.
+
+    Parameters
+    ----------
+    protocol : int, defaults to pickle.HIGHEST_PROTOCOL
+        The protocol used to pickle data.
+
+    Examples
+    --------
+    >>> import numcodecs as codecs
+    >>> import numpy as np
+    >>> x = np.array(['foo', 'bar', 'baz'], dtype='object')
+    >>> f = codecs.Pickle()
+    >>> f.decode(f.encode(x))
+    array(['foo', 'bar', 'baz'], dtype=object)
+
+    See Also
+    --------
+    numcodecs.msgpacks.MsgPack
+
+    """
+
+    codec_id = 'pickle'
+
+    def __init__(self, protocol=pickle.HIGHEST_PROTOCOL):
+        self.protocol = protocol
+
+    def encode(self, buf):
+        return pickle.dumps(buf, protocol=self.protocol)
+
+    def decode(self, buf, out=None):
+        buf = ensure_contiguous_ndarray(buf)
+        dec = pickle.loads(buf)
+
+        if out is not None:
+            np.copyto(out, dec)
+            return out
+        else:
+            return dec
+
+    def get_config(self):
+        return dict(id=self.codec_id, protocol=self.protocol)
+
+    def __repr__(self):
+        return f'Pickle(protocol={self.protocol})'

numcodecs/quantize.py

@@ -0,0 +1,100 @@
+import math
+
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_ndarray, ndarray_copy
+
+
+class Quantize(Codec):
+    """Lossy filter to reduce the precision of floating point data.
+
+    Parameters
+    ----------
+    digits : int
+        Desired precision (number of decimal digits).
+    dtype : dtype
+        Data type to use for decoded data.
+    astype : dtype, optional
+        Data type to use for encoded data.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> x = np.linspace(0, 1, 10, dtype='f8')
+    >>> x
+    array([0.        , 0.11111111, 0.22222222, 0.33333333, 0.44444444,
+           0.55555556, 0.66666667, 0.77777778, 0.88888889, 1.        ])
+    >>> codec = numcodecs.Quantize(digits=1, dtype='f8')
+    >>> codec.encode(x)
+    array([0.    , 0.125 , 0.25  , 0.3125, 0.4375, 0.5625, 0.6875,
+           0.75  , 0.875 , 1.    ])
+    >>> codec = numcodecs.Quantize(digits=2, dtype='f8')
+    >>> codec.encode(x)
+    array([0.       , 0.109375 , 0.21875  , 0.3359375, 0.4453125,
+           0.5546875, 0.6640625, 0.78125  , 0.890625 , 1.       ])
+    >>> codec = numcodecs.Quantize(digits=3, dtype='f8')
+    >>> codec.encode(x)
+    array([0.        , 0.11132812, 0.22265625, 0.33300781, 0.44433594,
+           0.55566406, 0.66699219, 0.77734375, 0.88867188, 1.        ])
+
+    See Also
+    --------
+    numcodecs.fixedscaleoffset.FixedScaleOffset
+
+    """
+
+    codec_id = 'quantize'
+
+    def __init__(self, digits, dtype, astype=None):
+        self.digits = digits
+        self.dtype = np.dtype(dtype)
+        if astype is None:
+            self.astype = self.dtype
+        else:
+            self.astype = np.dtype(astype)
+        if self.dtype.kind != 'f' or self.astype.kind != 'f':
+            raise ValueError('only floating point data types are supported')
+
+    def encode(self, buf):
+        # normalise input
+        arr = ensure_ndarray(buf).view(self.dtype)
+
+        # apply scaling
+        precision = 10.0**-self.digits
+        exp = math.log10(precision)
+        if exp < 0:
+            exp = int(math.floor(exp))
+        else:
+            exp = int(math.ceil(exp))
+        bits = math.ceil(math.log2(10.0**-exp))
+        scale = 2.0**bits
+        enc = np.around(scale * arr) / scale
+
+        # cast dtype
+        enc = enc.astype(self.astype, copy=False)
+
+        return enc
+
+    def decode(self, buf, out=None):
+        # filter is lossy, decoding is no-op
+        dec = ensure_ndarray(buf).view(self.astype)
+        dec = dec.astype(self.dtype, copy=False)
+        return ndarray_copy(dec, out)
+
+    def get_config(self):
+        # override to handle encoding dtypes
+        return dict(
+            id=self.codec_id,
+            digits=self.digits,
+            dtype=self.dtype.str,
+            astype=self.astype.str,
+        )
+
+    def __repr__(self):
+        r = f'{type(self).__name__}(digits={self.digits}, dtype={self.dtype.str!r}'
+        if self.astype != self.dtype:
+            r += f', astype={self.astype.str!r}'
+        r += ')'
+        return r

numcodecs/registry.py

@@ -0,0 +1,72 @@
+"""The registry module provides some simple convenience functions to enable
+applications to dynamically register and look-up codec classes."""
+
+import logging
+from importlib.metadata import entry_points
+
+logger = logging.getLogger("numcodecs")
+codec_registry = {}
+entries = {}
+
+
+def run_entrypoints():
+    entries.clear()
+    eps = entry_points()
+    entries.update({e.name: e for e in eps.select(group="numcodecs.codecs")})
+
+
+run_entrypoints()
+
+
+def get_codec(config):
+    """Obtain a codec for the given configuration.
+
+    Parameters
+    ----------
+    config : dict-like
+        Configuration object.
+
+    Returns
+    -------
+    codec : Codec
+
+    Examples
+    --------
+
+    >>> import numcodecs as codecs
+    >>> codec = codecs.get_codec(dict(id='zlib', level=1))
+    >>> codec
+    Zlib(level=1)
+
+    """
+    config = dict(config)
+    codec_id = config.pop('id', None)
+    cls = codec_registry.get(codec_id)
+    if cls is None:
+        if codec_id in entries:
+            logger.debug("Auto loading codec '%s' from entrypoint", codec_id)
+            cls = entries[codec_id].load()
+            register_codec(cls, codec_id=codec_id)
+    if cls:
+        return cls.from_config(config)
+    raise ValueError(f'codec not available: {codec_id!r}')
+
+
+def register_codec(cls, codec_id=None):
+    """Register a codec class.
+
+    Parameters
+    ----------
+    cls : Codec class
+
+    Notes
+    -----
+    This function maintains a mapping from codec identifiers to codec
+    classes. When a codec class is registered, it will replace any class
+    previously registered under the same codec identifier, if present.
+
+    """
+    if codec_id is None:
+        codec_id = cls.codec_id
+    logger.debug("Registering codec '%s'", codec_id)
+    codec_registry[codec_id] = cls

numcodecs/shuffle.py

@@ -0,0 +1,61 @@
+import numpy as np
+
+from ._shuffle import _doShuffle, _doUnshuffle
+from .abc import Codec
+from .compat import ensure_contiguous_ndarray
+
+
+class Shuffle(Codec):
+    """Codec providing shuffle
+
+    Parameters
+    ----------
+    elementsize : int
+        Size in bytes of the array elements.  Default = 4
+
+    """
+
+    codec_id = 'shuffle'
+
+    def __init__(self, elementsize=4):
+        self.elementsize = elementsize
+
+    def _prepare_arrays(self, buf, out):
+        buf = ensure_contiguous_ndarray(buf)
+
+        if out is None:
+            out = np.zeros(buf.nbytes, dtype='uint8')
+        else:
+            out = ensure_contiguous_ndarray(out)
+
+        if self.elementsize <= 1:
+            out.view(buf.dtype)[: len(buf)] = buf[:]  # no shuffling needed
+            return buf, out
+
+        if buf.nbytes % self.elementsize != 0:
+            raise ValueError("Shuffle buffer is not an integer multiple of elementsize")
+
+        return buf, out
+
+    def encode(self, buf, out=None):
+        buf, out = self._prepare_arrays(buf, out)
+
+        if self.elementsize <= 1:
+            return out  # no shuffling needed
+
+        _doShuffle(buf.view("uint8"), out.view("uint8"), self.elementsize)
+
+        return out
+
+    def decode(self, buf, out=None):
+        buf, out = self._prepare_arrays(buf, out)
+
+        if self.elementsize <= 1:
+            return out  # no shuffling needed
+
+        _doUnshuffle(buf.view("uint8"), out.view("uint8"), self.elementsize)
+
+        return out
+
+    def __repr__(self):
+        return f'{type(self).__name__}(elementsize={self.elementsize})'

numcodecs/tests/__init__.py

@@ -0,0 +1,3 @@
+import pytest
+
+pytest.register_assert_rewrite('numcodecs.tests.common')