numcodecs 0.16.0__cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

numcodecs/json.py

@@ -0,0 +1,107 @@
+import json as _json
+import textwrap
+
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_text
+
+
+class JSON(Codec):
+    """Codec to encode data as JSON. Useful for encoding an array of Python objects.
+
+    .. versionchanged:: 0.6
+        The encoding format has been changed to include the array shape in the encoded
+        data, which ensures that all object arrays can be correctly encoded and decoded.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> x = np.array(['foo', 'bar', 'baz'], dtype='object')
+    >>> codec = numcodecs.JSON()
+    >>> codec.decode(codec.encode(x))
+    array(['foo', 'bar', 'baz'], dtype=object)
+
+    See Also
+    --------
+    numcodecs.pickles.Pickle, numcodecs.msgpacks.MsgPack
+
+    """
+
+    codec_id = 'json2'
+
+    def __init__(
+        self,
+        encoding='utf-8',
+        skipkeys=False,
+        ensure_ascii=True,
+        check_circular=True,
+        allow_nan=True,
+        sort_keys=True,
+        indent=None,
+        separators=None,
+        strict=True,
+    ):
+        self._text_encoding = encoding
+        if separators is None:
+            # ensure separators are explicitly specified, and consistent behaviour across
+            # Python versions, and most compact representation if indent is None
+            if indent is None:
+                separators = ',', ':'
+            else:
+                separators = ', ', ': '
+        separators = tuple(separators)
+        self._encoder_config = {
+            'skipkeys': skipkeys,
+            'ensure_ascii': ensure_ascii,
+            'check_circular': check_circular,
+            'allow_nan': allow_nan,
+            'indent': indent,
+            'separators': separators,
+            'sort_keys': sort_keys,
+        }
+        self._encoder = _json.JSONEncoder(**self._encoder_config)
+        self._decoder_config = {'strict': strict}
+        self._decoder = _json.JSONDecoder(**self._decoder_config)
+
+    def encode(self, buf):
+        try:
+            buf = np.asarray(buf)
+        except ValueError:  # pragma: no cover
+            buf = np.asarray(buf, dtype=object)
+        items = np.atleast_1d(buf).tolist()
+        items.append(buf.dtype.str)
+        items.append(buf.shape)
+        return self._encoder.encode(items).encode(self._text_encoding)
+
+    def decode(self, buf, out=None):
+        items = self._decoder.decode(ensure_text(buf, self._text_encoding))
+        dec = np.empty(items[-1], dtype=items[-2])
+        if not items[-1]:
+            dec[...] = items[0]
+        else:
+            dec[:] = items[:-2]
+        if out is not None:
+            np.copyto(out, dec)
+            return out
+        else:
+            return dec
+
+    def get_config(self):
+        config = {'id': self.codec_id, 'encoding': self._text_encoding}
+        config.update(self._encoder_config)
+        config.update(self._decoder_config)
+        return config
+
+    def __repr__(self):
+        params = [f'encoding={self._text_encoding!r}']
+        for k, v in sorted(self._encoder_config.items()):
+            params.append(f'{k}={v!r}')
+        for k, v in sorted(self._decoder_config.items()):
+            params.append(f'{k}={v!r}')
+        classname = type(self).__name__
+        params = ', '.join(params)
+        return textwrap.fill(
+            f'{classname}({params})', width=80, break_long_words=False, subsequent_indent='     '
+        )

numcodecs/lzma.py

@@ -0,0 +1,72 @@
+from types import ModuleType
+from typing import Optional
+
+_lzma: Optional[ModuleType] = None
+try:
+    import lzma as _lzma
+except ImportError:  # pragma: no cover
+    try:  # noqa: SIM105
+        from backports import lzma as _lzma  # type: ignore[no-redef]
+    except ImportError:
+        pass
+
+
+if _lzma:
+    from .abc import Codec
+    from .compat import ensure_contiguous_ndarray, ndarray_copy
+
+    # noinspection PyShadowingBuiltins
+    class LZMA(Codec):
+        """Codec providing compression using lzma via the Python standard
+        library.
+
+        Parameters
+        ----------
+        format : integer, optional
+            One of the lzma format codes, e.g., ``lzma.FORMAT_XZ``.
+        check : integer, optional
+            One of the lzma check codes, e.g., ``lzma.CHECK_NONE``.
+        preset : integer, optional
+            An integer between 0 and 9 inclusive, specifying the compression
+            level.
+        filters : list, optional
+            A list of dictionaries specifying compression filters. If
+            filters are provided, 'preset' must be None.
+
+        """
+
+        codec_id = 'lzma'
+
+        def __init__(self, format=1, check=-1, preset=None, filters=None):
+            self.format = format
+            self.check = check
+            self.preset = preset
+            self.filters = filters
+
+        def encode(self, buf):
+            # normalise inputs
+            buf = ensure_contiguous_ndarray(buf)
+
+            # do compression
+            return _lzma.compress(
+                buf,
+                format=self.format,
+                check=self.check,
+                preset=self.preset,
+                filters=self.filters,
+            )
+
+        def decode(self, buf, out=None):
+            # normalise inputs
+            buf = ensure_contiguous_ndarray(buf)
+            if out is not None:
+                out = ensure_contiguous_ndarray(out)
+
+            # do decompression
+            dec = _lzma.decompress(buf, format=self.format, filters=self.filters)
+
+            # handle destination
+            return ndarray_copy(dec, out)
+
+        def __repr__(self):
+            return f'{type(self).__name__}(format={self.format!r}, check={self.check!r}, preset={self.preset!r}, filters={self.filters!r})'

numcodecs/msgpacks.py

@@ -0,0 +1,86 @@
+import msgpack
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_contiguous_ndarray
+
+
+class MsgPack(Codec):
+    """Codec to encode data as msgpacked bytes. Useful for encoding an array of Python
+    objects.
+
+    .. versionchanged:: 0.6
+        The encoding format has been changed to include the array shape in the encoded
+        data, which ensures that all object arrays can be correctly encoded and decoded.
+
+    Parameters
+    ----------
+    use_single_float : bool, optional
+        Use single precision float type for float.
+    use_bin_type : bool, optional
+        Use bin type introduced in msgpack spec 2.0 for bytes. It also enables str8 type
+        for unicode.
+    raw : bool, optional
+        If true, unpack msgpack raw to Python bytes. Otherwise, unpack to Python str
+        by decoding with UTF-8 encoding.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> x = np.array(['foo', 'bar', 'baz'], dtype='object')
+    >>> codec = numcodecs.MsgPack()
+    >>> codec.decode(codec.encode(x))
+    array(['foo', 'bar', 'baz'], dtype=object)
+
+    See Also
+    --------
+    numcodecs.pickles.Pickle, numcodecs.json.JSON, numcodecs.vlen.VLenUTF8
+
+    Notes
+    -----
+    Requires `msgpack <https://pypi.org/project/msgpack/>`_ to be installed.
+
+    """
+
+    codec_id = 'msgpack2'
+
+    def __init__(self, use_single_float=False, use_bin_type=True, raw=False):
+        self.use_single_float = use_single_float
+        self.use_bin_type = use_bin_type
+        self.raw = raw
+
+    def encode(self, buf):
+        try:
+            buf = np.asarray(buf)
+        except ValueError:
+            buf = np.asarray(buf, dtype=object)
+        items = buf.tolist()
+        items.extend((buf.dtype.str, buf.shape))
+        return msgpack.packb(
+            items,
+            use_bin_type=self.use_bin_type,
+            use_single_float=self.use_single_float,
+        )
+
+    def decode(self, buf, out=None):
+        buf = ensure_contiguous_ndarray(buf)
+        items = msgpack.unpackb(buf, raw=self.raw)
+        dec = np.empty(items[-1], dtype=items[-2])
+        dec[:] = items[:-2]
+        if out is not None:
+            np.copyto(out, dec)
+            return out
+        else:
+            return dec
+
+    def get_config(self):
+        return {
+            'id': self.codec_id,
+            'raw': self.raw,
+            'use_single_float': self.use_single_float,
+            'use_bin_type': self.use_bin_type,
+        }
+
+    def __repr__(self):
+        return f'MsgPack(raw={self.raw!r}, use_bin_type={self.use_bin_type!r}, use_single_float={self.use_single_float!r})'

numcodecs/ndarray_like.py

@@ -0,0 +1,65 @@
+from typing import Any, ClassVar, Protocol, runtime_checkable
+
+
+class _CachedProtocolMeta(Protocol.__class__):  # type: ignore[name-defined]
+    """Custom implementation of @runtime_checkable
+
+    The native implementation of @runtime_checkable is slow,
+    see <https://github.com/zarr-developers/numcodecs/issues/379>.
+
+    This metaclass keeps an unbounded cache of the result of
+    isinstance checks using the object's class as the cache key.
+    """
+
+    _instancecheck_cache: ClassVar[dict[tuple[type, type], bool]] = {}
+
+    def __instancecheck__(self, instance):
+        key = (self, instance.__class__)
+        ret = self._instancecheck_cache.get(key)
+        if ret is None:
+            ret = super().__instancecheck__(instance)
+            self._instancecheck_cache[key] = ret
+        return ret
+
+
+@runtime_checkable
+class DType(Protocol, metaclass=_CachedProtocolMeta):
+    itemsize: int
+    name: str
+    kind: str
+
+
+@runtime_checkable
+class FlagsObj(Protocol, metaclass=_CachedProtocolMeta):
+    c_contiguous: bool
+    f_contiguous: bool
+    owndata: bool
+
+
+@runtime_checkable
+class NDArrayLike(Protocol, metaclass=_CachedProtocolMeta):
+    dtype: DType
+    shape: tuple[int, ...]
+    strides: tuple[int, ...]
+    ndim: int
+    size: int
+    itemsize: int
+    nbytes: int
+    flags: FlagsObj
+
+    def __len__(self) -> int: ...  # pragma: no cover
+
+    def __getitem__(self, key) -> Any: ...  # pragma: no cover
+
+    def __setitem__(self, key, value): ...  # pragma: no cover
+
+    def tobytes(self, order: str | None = ...) -> bytes: ...  # pragma: no cover
+
+    def reshape(self, *shape: int, order: str = ...) -> "NDArrayLike": ...  # pragma: no cover
+
+    def view(self, dtype: DType = ...) -> "NDArrayLike": ...  # pragma: no cover
+
+
+def is_ndarray_like(obj: object) -> bool:
+    """Return True when `obj` is ndarray-like"""
+    return isinstance(obj, NDArrayLike)

numcodecs/packbits.py

@@ -0,0 +1,82 @@
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_ndarray, ndarray_copy
+
+
+class PackBits(Codec):
+    """Codec to pack elements of a boolean array into bits in a uint8 array.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> codec = numcodecs.PackBits()
+    >>> x = np.array([True, False, False, True], dtype=bool)
+    >>> y = codec.encode(x)
+    >>> y
+    array([  4, 144], dtype=uint8)
+    >>> z = codec.decode(y)
+    >>> z
+    array([ True, False, False,  True])
+
+    Notes
+    -----
+    The first element of the encoded array stores the number of bits that
+    were padded to complete the final byte.
+
+    """
+
+    codec_id = 'packbits'
+
+    def encode(self, buf):
+        # normalise input
+        arr = ensure_ndarray(buf).view(bool)
+
+        # flatten to simplify implementation
+        arr = arr.reshape(-1, order='A')
+
+        # determine size of packed data
+        n = arr.size
+        n_bytes_packed = n // 8
+        n_bits_leftover = n % 8
+        if n_bits_leftover > 0:
+            n_bytes_packed += 1
+
+        # setup output
+        enc = np.empty(n_bytes_packed + 1, dtype='u1')
+
+        # store how many bits were padded
+        if n_bits_leftover:
+            n_bits_padded = 8 - n_bits_leftover
+        else:
+            n_bits_padded = 0
+        enc[0] = n_bits_padded
+
+        # apply encoding
+        enc[1:] = np.packbits(arr)
+
+        return enc
+
+    def decode(self, buf, out=None):
+        # normalise input
+        enc = ensure_ndarray(buf).view('u1')
+
+        # flatten to simplify implementation
+        enc = enc.reshape(-1, order='A')
+
+        # find out how many bits were padded
+        n_bits_padded = int(enc[0])
+
+        # apply decoding
+        dec = np.unpackbits(enc[1:])
+
+        # remove padded bits
+        if n_bits_padded:
+            dec = dec[:-n_bits_padded]
+
+        # view as boolean array
+        dec = dec.view(bool)
+
+        # handle destination
+        return ndarray_copy(dec, out)

numcodecs/pcodec.py

@@ -0,0 +1,118 @@
+from typing import Literal
+
+from numcodecs.abc import Codec
+from numcodecs.compat import ensure_contiguous_ndarray
+from pcodec import ChunkConfig, DeltaSpec, ModeSpec, PagingSpec, standalone
+
+DEFAULT_MAX_PAGE_N = 262144
+
+
+class PCodec(Codec):
+    """
+    PCodec (or pco, pronounced "pico") losslessly compresses and decompresses
+    numerical sequences with high compression ratio and fast speed.
+
+    See `PCodec Repo <https://github.com/mwlon/pcodec>`_ for more information.
+
+    PCodec supports only the following numerical dtypes: uint16, uint32, uint64,
+    int16, int32, int64, float16, float32, and float64.
+
+    Parameters
+    ----------
+    level : int
+        A compression level from 0-12, where 12 take the longest and compresses
+        the most.
+    mode_spec : {"auto", "classic"}
+        Configures whether Pcodec should try to infer the best "mode" or
+        structure of the data (e.g. approximate multiples of 0.1) to improve
+        compression ratio, or skip this step and just use the numbers as-is
+        (Classic mode). Note that the "try*" specs are not currently supported.
+    delta_spec : {"auto", "none", "try_consecutive", "try_lookback"}
+        Configures the delta encoding strategy. By default, uses "auto" which
+        will try to infer the best encoding order.
+    paging_spec : {"equal_pages_up_to"}
+        Configures the paging strategy. Only "equal_pages_up_to" is currently
+        supported.
+    delta_encoding_order : int or None
+        Explicit delta encoding level from 0-7. Only valid if delta_spec is
+        "try_consecutive" or "auto" (to support backwards compatibility with
+        older versions of this codec).
+    equal_pages_up_to : int
+        Divide the chunk into equal pages of up to this many numbers.
+    """
+
+    codec_id = "pcodec"
+
+    def __init__(
+        self,
+        level: int = 8,
+        *,
+        mode_spec: Literal["auto", "classic"] = "auto",
+        delta_spec: Literal["auto", "none", "try_consecutive", "try_lookback"] = "auto",
+        paging_spec: Literal["equal_pages_up_to"] = "equal_pages_up_to",
+        delta_encoding_order: int | None = None,
+        equal_pages_up_to: int = DEFAULT_MAX_PAGE_N,
+    ):
+        # note that we use `level` instead of `compression_level` to
+        # match other codecs
+        self.level = level
+        self.mode_spec = mode_spec
+        self.delta_spec = delta_spec
+        self.paging_spec = paging_spec
+        self.delta_encoding_order = delta_encoding_order
+        self.equal_pages_up_to = equal_pages_up_to
+
+    def _get_chunk_config(self):
+        match self.mode_spec:
+            case "auto":
+                mode_spec = ModeSpec.auto()
+            case "classic":
+                mode_spec = ModeSpec.classic()
+            case _:
+                raise ValueError(f"mode_spec {self.mode_spec} is not supported")
+
+        if self.delta_encoding_order is not None and self.delta_spec == "auto":
+            # backwards compat for before delta_spec was introduced
+            delta_spec = DeltaSpec.try_consecutive(self.delta_encoding_order)
+        elif self.delta_encoding_order is not None and self.delta_spec != "try_consecutive":
+            raise ValueError(
+                "delta_encoding_order can only be set for delta_spec='try_consecutive'"
+            )
+        else:
+            match self.delta_spec:
+                case "auto":
+                    delta_spec = DeltaSpec.auto()
+                case "none":
+                    delta_spec = DeltaSpec.none()
+                case "try_consecutive":
+                    delta_spec = DeltaSpec.try_consecutive(self.delta_encoding_order)
+                case "try_lookback":
+                    delta_spec = DeltaSpec.try_lookback()
+                case _:
+                    raise ValueError(f"delta_spec {self.delta_spec} is not supported")
+
+        match self.paging_spec:
+            case "equal_pages_up_to":
+                paging_spec = PagingSpec.equal_pages_up_to(self.equal_pages_up_to)
+            case _:
+                raise ValueError(f"paging_spec {self.paging_spec} is not supported")
+
+        return ChunkConfig(
+            compression_level=self.level,
+            delta_spec=delta_spec,
+            mode_spec=mode_spec,
+            paging_spec=paging_spec,
+        )
+
+    def encode(self, buf):
+        buf = ensure_contiguous_ndarray(buf)
+        config = self._get_chunk_config()
+        return standalone.simple_compress(buf, config)
+
+    def decode(self, buf, out=None):
+        if out is not None:
+            out = ensure_contiguous_ndarray(out)
+            standalone.simple_decompress_into(buf, out)
+            return out
+        else:
+            return standalone.simple_decompress(buf)

numcodecs/pickles.py

@@ -0,0 +1,55 @@
+import pickle
+
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_contiguous_ndarray
+
+
+class Pickle(Codec):
+    """Codec to encode data as as pickled bytes. Useful for encoding an array of Python string
+    objects.
+
+    Parameters
+    ----------
+    protocol : int, defaults to pickle.HIGHEST_PROTOCOL
+        The protocol used to pickle data.
+
+    Examples
+    --------
+    >>> import numcodecs as codecs
+    >>> import numpy as np
+    >>> x = np.array(['foo', 'bar', 'baz'], dtype='object')
+    >>> f = codecs.Pickle()
+    >>> f.decode(f.encode(x))
+    array(['foo', 'bar', 'baz'], dtype=object)
+
+    See Also
+    --------
+    numcodecs.msgpacks.MsgPack
+
+    """
+
+    codec_id = 'pickle'
+
+    def __init__(self, protocol=pickle.HIGHEST_PROTOCOL):
+        self.protocol = protocol
+
+    def encode(self, buf):
+        return pickle.dumps(buf, protocol=self.protocol)
+
+    def decode(self, buf, out=None):
+        buf = ensure_contiguous_ndarray(buf)
+        dec = pickle.loads(buf)
+
+        if out is not None:
+            np.copyto(out, dec)
+            return out
+        else:
+            return dec
+
+    def get_config(self):
+        return {'id': self.codec_id, 'protocol': self.protocol}
+
+    def __repr__(self):
+        return f'Pickle(protocol={self.protocol})'

numcodecs/quantize.py

@@ -0,0 +1,98 @@
+import math
+
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_ndarray, ndarray_copy
+
+
+class Quantize(Codec):
+    """Lossy filter to reduce the precision of floating point data.
+
+    Parameters
+    ----------
+    digits : int
+        Desired precision (number of decimal digits).
+    dtype : dtype
+        Data type to use for decoded data.
+    astype : dtype, optional
+        Data type to use for encoded data.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> x = np.linspace(0, 1, 10, dtype='f8')
+    >>> x
+    array([0.        , 0.11111111, 0.22222222, 0.33333333, 0.44444444,
+           0.55555556, 0.66666667, 0.77777778, 0.88888889, 1.        ])
+    >>> codec = numcodecs.Quantize(digits=1, dtype='f8')
+    >>> codec.encode(x)
+    array([0.    , 0.125 , 0.25  , 0.3125, 0.4375, 0.5625, 0.6875,
+           0.75  , 0.875 , 1.    ])
+    >>> codec = numcodecs.Quantize(digits=2, dtype='f8')
+    >>> codec.encode(x)
+    array([0.       , 0.109375 , 0.21875  , 0.3359375, 0.4453125,
+           0.5546875, 0.6640625, 0.78125  , 0.890625 , 1.       ])
+    >>> codec = numcodecs.Quantize(digits=3, dtype='f8')
+    >>> codec.encode(x)
+    array([0.        , 0.11132812, 0.22265625, 0.33300781, 0.44433594,
+           0.55566406, 0.66699219, 0.77734375, 0.88867188, 1.        ])
+
+    See Also
+    --------
+    numcodecs.fixedscaleoffset.FixedScaleOffset
+
+    """
+
+    codec_id = 'quantize'
+
+    def __init__(self, digits, dtype, astype=None):
+        self.digits = digits
+        self.dtype = np.dtype(dtype)
+        if astype is None:
+            self.astype = self.dtype
+        else:
+            self.astype = np.dtype(astype)
+        if self.dtype.kind != 'f' or self.astype.kind != 'f':
+            raise ValueError('only floating point data types are supported')
+
+    def encode(self, buf):
+        # normalise input
+        arr = ensure_ndarray(buf).view(self.dtype)
+
+        # apply scaling
+        precision = 10.0**-self.digits
+        exp = math.log10(precision)
+        if exp < 0:
+            exp = math.floor(exp)
+        else:
+            exp = math.ceil(exp)
+        bits = math.ceil(math.log2(10.0**-exp))
+        scale = 2.0**bits
+        enc = np.around(scale * arr) / scale
+
+        # cast dtype
+        return enc.astype(self.astype, copy=False)
+
+    def decode(self, buf, out=None):
+        # filter is lossy, decoding is no-op
+        dec = ensure_ndarray(buf).view(self.astype)
+        dec = dec.astype(self.dtype, copy=False)
+        return ndarray_copy(dec, out)
+
+    def get_config(self):
+        # override to handle encoding dtypes
+        return {
+            'id': self.codec_id,
+            'digits': self.digits,
+            'dtype': self.dtype.str,
+            'astype': self.astype.str,
+        }
+
+    def __repr__(self):
+        r = f'{type(self).__name__}(digits={self.digits}, dtype={self.dtype.str!r}'
+        if self.astype != self.dtype:
+            r += f', astype={self.astype.str!r}'
+        r += ')'
+        return r