numcodecs 0.13.1__cp312-cp312-macosx_10_13_x86_64.whl

numcodecs/compat.py

@@ -0,0 +1,208 @@
+# ruff: noqa: F401
+import array
+import codecs
+import functools
+
+import numpy as np
+
+from .ndarray_like import NDArrayLike, is_ndarray_like
+
+
+def ensure_ndarray_like(buf) -> NDArrayLike:
+    """Convenience function to coerce `buf` to ndarray-like array.
+
+    Parameters
+    ----------
+    buf : ndarray-like, array-like, or bytes-like
+        A numpy array like object such as numpy.ndarray, cupy.ndarray, or
+        any object exporting a buffer interface.
+
+    Returns
+    -------
+    arr : NDArrayLike
+        A ndarray-like, sharing memory with `buf`.
+
+    Notes
+    -----
+    This function will not create a copy under any circumstances, it is guaranteed to
+    return a view on memory exported by `buf`.
+    """
+
+    if not is_ndarray_like(buf):
+        if isinstance(buf, array.array) and buf.typecode in "cu":
+            # Guard condition, do not support array.array with unicode type, this is
+            # problematic because numpy does not support it on all platforms. Also do not
+            # support char as it was removed in Python 3.
+            raise TypeError("array.array with char or unicode type is not supported")
+        else:
+            # N.B., first take a memoryview to make sure that we subsequently create a
+            # numpy array from a memory buffer with no copy
+            mem = memoryview(buf)
+            # instantiate array from memoryview, ensures no copy
+            buf = np.array(mem, copy=False)
+    return buf
+
+
+def ensure_ndarray(buf) -> np.ndarray:
+    """Convenience function to coerce `buf` to a numpy array, if it is not already a
+    numpy array.
+
+    Parameters
+    ----------
+    buf : array-like or bytes-like
+        A numpy array or any object exporting a buffer interface.
+
+    Returns
+    -------
+    arr : ndarray
+        A numpy array, sharing memory with `buf`.
+
+    Notes
+    -----
+    This function will not create a copy under any circumstances, it is guaranteed to
+    return a view on memory exported by `buf`.
+    """
+    return np.array(ensure_ndarray_like(buf), copy=False)
+
+
+def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> NDArrayLike:
+    """Convenience function to coerce `buf` to ndarray-like array.
+    Also ensures that the returned value exports fully contiguous memory,
+    and supports the new-style buffer interface. If the optional max_buffer_size is
+    provided, raise a ValueError if the number of bytes consumed by the returned
+    array exceeds this value.
+
+    Parameters
+    ----------
+    buf : ndarray-like, array-like, or bytes-like
+        A numpy array like object such as numpy.ndarray, cupy.ndarray, or
+        any object exporting a buffer interface.
+    max_buffer_size : int
+        If specified, the largest allowable value of arr.nbytes, where arr
+        is the returned array.
+    flatten : bool
+        If True, the array are flatten.
+
+    Returns
+    -------
+    arr : NDArrayLike
+        A ndarray-like, sharing memory with `buf`.
+
+    Notes
+    -----
+    This function will not create a copy under any circumstances, it is guaranteed to
+    return a view on memory exported by `buf`.
+    """
+    arr = ensure_ndarray_like(buf)
+
+    # check for object arrays, these are just memory pointers, actual memory holding
+    # item data is scattered elsewhere
+    if arr.dtype == object:
+        raise TypeError("object arrays are not supported")
+
+    # check for datetime or timedelta ndarray, the buffer interface doesn't support those
+    if arr.dtype.kind in "Mm":
+        arr = arr.view(np.int64)
+
+    # check memory is contiguous, if so flatten
+    if arr.flags.c_contiguous or arr.flags.f_contiguous:
+        if flatten:
+            # can flatten without copy
+            arr = arr.reshape(-1, order="A")
+    else:
+        raise ValueError("an array with contiguous memory is required")
+
+    if max_buffer_size is not None and arr.nbytes > max_buffer_size:
+        msg = f"Codec does not support buffers of > {max_buffer_size} bytes"
+        raise ValueError(msg)
+
+    return arr
+
+
+def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.array:
+    """Convenience function to coerce `buf` to a numpy array, if it is not already a
+    numpy array. Also ensures that the returned value exports fully contiguous memory,
+    and supports the new-style buffer interface. If the optional max_buffer_size is
+    provided, raise a ValueError if the number of bytes consumed by the returned
+    array exceeds this value.
+
+    Parameters
+    ----------
+    buf : array-like or bytes-like
+        A numpy array or any object exporting a buffer interface.
+    max_buffer_size : int
+        If specified, the largest allowable value of arr.nbytes, where arr
+        is the returned array.
+    flatten : bool
+        If True, the array are flatten.
+
+    Returns
+    -------
+    arr : ndarray
+        A numpy array, sharing memory with `buf`.
+
+    Notes
+    -----
+    This function will not create a copy under any circumstances, it is guaranteed to
+    return a view on memory exported by `buf`.
+    """
+
+    return ensure_ndarray(
+        ensure_contiguous_ndarray_like(buf, max_buffer_size=max_buffer_size, flatten=flatten)
+    )
+
+
+def ensure_bytes(buf) -> bytes:
+    """Obtain a bytes object from memory exposed by `buf`."""
+
+    if not isinstance(buf, bytes):
+        arr = ensure_ndarray_like(buf)
+
+        # check for object arrays, these are just memory pointers,
+        # actual memory holding item data is scattered elsewhere
+        if arr.dtype == object:
+            raise TypeError("object arrays are not supported")
+
+        # create bytes
+        buf = arr.tobytes(order="A")
+
+    return buf
+
+
+def ensure_text(s, encoding="utf-8"):
+    if not isinstance(s, str):
+        s = ensure_contiguous_ndarray(s)
+        s = codecs.decode(s, encoding)
+    return s
+
+
+def ndarray_copy(src, dst) -> NDArrayLike:
+    """Copy the contents of the array from `src` to `dst`."""
+
+    if dst is None:
+        # no-op
+        return src
+
+    # ensure ndarray like
+    src = ensure_ndarray_like(src)
+    dst = ensure_ndarray_like(dst)
+
+    # flatten source array
+    src = src.reshape(-1, order="A")
+
+    # ensure same data type
+    if dst.dtype != object:
+        src = src.view(dst.dtype)
+
+    # reshape source to match destination
+    if src.shape != dst.shape:
+        if dst.flags.f_contiguous:
+            order = "F"
+        else:
+            order = "C"
+        src = src.reshape(dst.shape, order=order)
+
+    # copy via numpy
+    np.copyto(dst, src)
+
+    return dst

numcodecs/delta.py

@@ -0,0 +1,97 @@
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_ndarray, ndarray_copy
+
+
+class Delta(Codec):
+    """Codec to encode data as the difference between adjacent values.
+
+    Parameters
+    ----------
+    dtype : dtype
+        Data type to use for decoded data.
+    astype : dtype, optional
+        Data type to use for encoded data.
+
+    Notes
+    -----
+    If `astype` is an integer data type, please ensure that it is
+    sufficiently large to store encoded values. No checks are made and data
+    may become corrupted due to integer overflow if `astype` is too small.
+    Note also that the encoded data for each chunk includes the absolute
+    value of the first element in the chunk, and so the encoded data type in
+    general needs to be large enough to store absolute values from the array.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> x = np.arange(100, 120, 2, dtype='i2')
+    >>> codec = numcodecs.Delta(dtype='i2', astype='i1')
+    >>> y = codec.encode(x)
+    >>> y
+    array([100,   2,   2,   2,   2,   2,   2,   2,   2,   2], dtype=int8)
+    >>> z = codec.decode(y)
+    >>> z
+    array([100, 102, 104, 106, 108, 110, 112, 114, 116, 118], dtype=int16)
+
+    """
+
+    codec_id = 'delta'
+
+    def __init__(self, dtype, astype=None):
+        self.dtype = np.dtype(dtype)
+        if astype is None:
+            self.astype = self.dtype
+        else:
+            self.astype = np.dtype(astype)
+        if self.dtype == np.dtype(object) or self.astype == np.dtype(object):
+            raise ValueError('object arrays are not supported')
+
+    def encode(self, buf):
+        # normalise input
+        arr = ensure_ndarray(buf).view(self.dtype)
+
+        # flatten to simplify implementation
+        arr = arr.reshape(-1, order='A')
+
+        # setup encoded output
+        enc = np.empty_like(arr, dtype=self.astype)
+
+        # set first element
+        enc[0] = arr[0]
+
+        # compute differences
+        enc[1:] = np.diff(arr)
+
+        return enc
+
+    def decode(self, buf, out=None):
+        # normalise input
+        enc = ensure_ndarray(buf).view(self.astype)
+
+        # flatten to simplify implementation
+        enc = enc.reshape(-1, order='A')
+
+        # setup decoded output
+        dec = np.empty_like(enc, dtype=self.dtype)
+
+        # decode differences
+        np.cumsum(enc, out=dec)
+
+        # handle output
+        out = ndarray_copy(dec, out)
+
+        return out
+
+    def get_config(self):
+        # override to handle encoding dtypes
+        return dict(id=self.codec_id, dtype=self.dtype.str, astype=self.astype.str)
+
+    def __repr__(self):
+        r = f'{type(self).__name__}(dtype={self.dtype.str!r}'
+        if self.astype != self.dtype:
+            r += f', astype={self.astype.str!r}'
+        r += ')'
+        return r

numcodecs/fixedscaleoffset.py

@@ -0,0 +1,132 @@
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_ndarray, ndarray_copy
+
+
+class FixedScaleOffset(Codec):
+    """Simplified version of the scale-offset filter available in HDF5.
+    Applies the transformation `(x - offset) * scale` to all chunks. Results
+    are rounded to the nearest integer but are not packed according to the
+    minimum number of bits.
+
+    Parameters
+    ----------
+    offset : float
+        Value to subtract from data.
+    scale : float
+        Value to multiply by data.
+    dtype : dtype
+        Data type to use for decoded data.
+    astype : dtype, optional
+        Data type to use for encoded data.
+
+    Notes
+    -----
+    If `astype` is an integer data type, please ensure that it is
+    sufficiently large to store encoded values. No checks are made and data
+    may become corrupted due to integer overflow if `astype` is too small.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> x = np.linspace(1000, 1001, 10, dtype='f8')
+    >>> x
+    array([1000.        , 1000.11111111, 1000.22222222, 1000.33333333,
+           1000.44444444, 1000.55555556, 1000.66666667, 1000.77777778,
+           1000.88888889, 1001.        ])
+    >>> codec = numcodecs.FixedScaleOffset(offset=1000, scale=10, dtype='f8', astype='u1')
+    >>> y1 = codec.encode(x)
+    >>> y1
+    array([ 0,  1,  2,  3,  4,  6,  7,  8,  9, 10], dtype=uint8)
+    >>> z1 = codec.decode(y1)
+    >>> z1
+    array([1000. , 1000.1, 1000.2, 1000.3, 1000.4, 1000.6, 1000.7,
+           1000.8, 1000.9, 1001. ])
+    >>> codec = numcodecs.FixedScaleOffset(offset=1000, scale=10**2, dtype='f8', astype='u1')
+    >>> y2 = codec.encode(x)
+    >>> y2
+    array([ 0,  11,  22,  33,  44,  56,  67,  78,  89, 100], dtype=uint8)
+    >>> z2 = codec.decode(y2)
+    >>> z2
+    array([1000.  , 1000.11, 1000.22, 1000.33, 1000.44, 1000.56,
+           1000.67, 1000.78, 1000.89, 1001.  ])
+    >>> codec = numcodecs.FixedScaleOffset(offset=1000, scale=10**3, dtype='f8', astype='u2')
+    >>> y3 = codec.encode(x)
+    >>> y3
+    array([ 0,  111,  222,  333,  444,  556,  667,  778,  889, 1000], dtype=uint16)
+    >>> z3 = codec.decode(y3)
+    >>> z3
+    array([1000.   , 1000.111, 1000.222, 1000.333, 1000.444, 1000.556,
+           1000.667, 1000.778, 1000.889, 1001.   ])
+
+    See Also
+    --------
+    numcodecs.quantize.Quantize
+
+    """
+
+    codec_id = 'fixedscaleoffset'
+
+    def __init__(self, offset, scale, dtype, astype=None):
+        self.offset = offset
+        self.scale = scale
+        self.dtype = np.dtype(dtype)
+        if astype is None:
+            self.astype = self.dtype
+        else:
+            self.astype = np.dtype(astype)
+        if self.dtype == np.dtype(object) or self.astype == np.dtype(object):
+            raise ValueError('object arrays are not supported')
+
+    def encode(self, buf):
+        # normalise input
+        arr = ensure_ndarray(buf).view(self.dtype)
+
+        # flatten to simplify implementation
+        arr = arr.reshape(-1, order='A')
+
+        # compute scale offset
+        enc = (arr - self.offset) * self.scale
+
+        # round to nearest integer
+        enc = np.around(enc)
+
+        # convert dtype
+        enc = enc.astype(self.astype, copy=False)
+
+        return enc
+
+    def decode(self, buf, out=None):
+        # interpret buffer as numpy array
+        enc = ensure_ndarray(buf).view(self.astype)
+
+        # flatten to simplify implementation
+        enc = enc.reshape(-1, order='A')
+
+        # decode scale offset
+        dec = (enc / self.scale) + self.offset
+
+        # convert dtype
+        dec = dec.astype(self.dtype, copy=False)
+
+        # handle output
+        return ndarray_copy(dec, out)
+
+    def get_config(self):
+        # override to handle encoding dtypes
+        return dict(
+            id=self.codec_id,
+            scale=self.scale,
+            offset=self.offset,
+            dtype=self.dtype.str,
+            astype=self.astype.str,
+        )
+
+    def __repr__(self):
+        r = f'{type(self).__name__}(scale={self.scale}, offset={self.offset}, dtype={self.dtype.str!r}'
+        if self.astype != self.dtype:
+            r += f', astype={self.astype.str!r}'
+        r += ')'
+        return r

numcodecs/gzip.py

@@ -0,0 +1,52 @@
+import gzip as _gzip
+import io
+
+from .abc import Codec
+from .compat import ensure_bytes, ensure_contiguous_ndarray
+
+
+class GZip(Codec):
+    """Codec providing gzip compression using zlib via the Python standard library.
+
+    Parameters
+    ----------
+    level : int
+        Compression level.
+
+    """
+
+    codec_id = 'gzip'
+
+    def __init__(self, level=1):
+        self.level = level
+
+    def encode(self, buf):
+        # normalise inputs
+        buf = ensure_contiguous_ndarray(buf)
+
+        # do compression
+        compressed = io.BytesIO()
+        with _gzip.GzipFile(fileobj=compressed, mode='wb', compresslevel=self.level) as compressor:
+            compressor.write(buf)
+        compressed = compressed.getvalue()
+
+        return compressed
+
+    # noinspection PyMethodMayBeStatic
+    def decode(self, buf, out=None):
+        # normalise inputs
+        # BytesIO only copies if the data is not of `bytes` type.
+        # This allows `bytes` objects to pass through without copying.
+        buf = io.BytesIO(ensure_bytes(buf))
+
+        # do decompression
+        with _gzip.GzipFile(fileobj=buf, mode='rb') as decompressor:
+            if out is not None:
+                out_view = ensure_contiguous_ndarray(out)
+                decompressor.readinto(out_view)
+                if decompressor.read(1) != b'':
+                    raise ValueError("Unable to fit data into `out`")
+            else:
+                out = decompressor.read()
+
+        return out

numcodecs/json.py

@@ -0,0 +1,107 @@
+import json as _json
+import textwrap
+
+import numpy as np
+
+from .abc import Codec
+from .compat import ensure_text
+
+
+class JSON(Codec):
+    """Codec to encode data as JSON. Useful for encoding an array of Python objects.
+
+    .. versionchanged:: 0.6
+        The encoding format has been changed to include the array shape in the encoded
+        data, which ensures that all object arrays can be correctly encoded and decoded.
+
+    Examples
+    --------
+    >>> import numcodecs
+    >>> import numpy as np
+    >>> x = np.array(['foo', 'bar', 'baz'], dtype='object')
+    >>> codec = numcodecs.JSON()
+    >>> codec.decode(codec.encode(x))
+    array(['foo', 'bar', 'baz'], dtype=object)
+
+    See Also
+    --------
+    numcodecs.pickles.Pickle, numcodecs.msgpacks.MsgPack
+
+    """
+
+    codec_id = 'json2'
+
+    def __init__(
+        self,
+        encoding='utf-8',
+        skipkeys=False,
+        ensure_ascii=True,
+        check_circular=True,
+        allow_nan=True,
+        sort_keys=True,
+        indent=None,
+        separators=None,
+        strict=True,
+    ):
+        self._text_encoding = encoding
+        if separators is None:
+            # ensure separators are explicitly specified, and consistent behaviour across
+            # Python versions, and most compact representation if indent is None
+            if indent is None:
+                separators = ',', ':'
+            else:
+                separators = ', ', ': '
+        separators = tuple(separators)
+        self._encoder_config = dict(
+            skipkeys=skipkeys,
+            ensure_ascii=ensure_ascii,
+            check_circular=check_circular,
+            allow_nan=allow_nan,
+            indent=indent,
+            separators=separators,
+            sort_keys=sort_keys,
+        )
+        self._encoder = _json.JSONEncoder(**self._encoder_config)
+        self._decoder_config = dict(strict=strict)
+        self._decoder = _json.JSONDecoder(**self._decoder_config)
+
+    def encode(self, buf):
+        try:
+            buf = np.asarray(buf)
+        except ValueError:
+            buf = np.asarray(buf, dtype=object)
+        items = np.atleast_1d(buf).tolist()
+        items.append(buf.dtype.str)
+        items.append(buf.shape)
+        return self._encoder.encode(items).encode(self._text_encoding)
+
+    def decode(self, buf, out=None):
+        items = self._decoder.decode(ensure_text(buf, self._text_encoding))
+        dec = np.empty(items[-1], dtype=items[-2])
+        if not items[-1]:
+            dec[...] = items[0]
+        else:
+            dec[:] = items[:-2]
+        if out is not None:
+            np.copyto(out, dec)
+            return out
+        else:
+            return dec
+
+    def get_config(self):
+        config = dict(id=self.codec_id, encoding=self._text_encoding)
+        config.update(self._encoder_config)
+        config.update(self._decoder_config)
+        return config
+
+    def __repr__(self):
+        params = [f'encoding={self._text_encoding!r}']
+        for k, v in sorted(self._encoder_config.items()):
+            params.append(f'{k}={v!r}')
+        for k, v in sorted(self._decoder_config.items()):
+            params.append(f'{k}={v!r}')
+        classname = type(self).__name__
+        params = ', '.join(params)
+        return textwrap.fill(
+            f'{classname}({params})', width=80, break_long_words=False, subsequent_indent='     '
+        )

numcodecs/lzma.py

@@ -0,0 +1,69 @@
+_lzma = None
+try:
+    import lzma as _lzma
+except ImportError:  # pragma: no cover
+    try:
+        from backports import lzma as _lzma
+    except ImportError:
+        pass
+
+
+if _lzma:
+    from .abc import Codec
+    from .compat import ensure_contiguous_ndarray, ndarray_copy
+
+    # noinspection PyShadowingBuiltins
+    class LZMA(Codec):
+        """Codec providing compression using lzma via the Python standard
+        library.
+
+        Parameters
+        ----------
+        format : integer, optional
+            One of the lzma format codes, e.g., ``lzma.FORMAT_XZ``.
+        check : integer, optional
+            One of the lzma check codes, e.g., ``lzma.CHECK_NONE``.
+        preset : integer, optional
+            An integer between 0 and 9 inclusive, specifying the compression
+            level.
+        filters : list, optional
+            A list of dictionaries specifying compression filters. If
+            filters are provided, 'preset' must be None.
+
+        """
+
+        codec_id = 'lzma'
+
+        def __init__(self, format=1, check=-1, preset=None, filters=None):
+            self.format = format
+            self.check = check
+            self.preset = preset
+            self.filters = filters
+
+        def encode(self, buf):
+            # normalise inputs
+            buf = ensure_contiguous_ndarray(buf)
+
+            # do compression
+            return _lzma.compress(
+                buf,
+                format=self.format,
+                check=self.check,
+                preset=self.preset,
+                filters=self.filters,
+            )
+
+        def decode(self, buf, out=None):
+            # normalise inputs
+            buf = ensure_contiguous_ndarray(buf)
+            if out is not None:
+                out = ensure_contiguous_ndarray(out)
+
+            # do decompression
+            dec = _lzma.decompress(buf, format=self.format, filters=self.filters)
+
+            # handle destination
+            return ndarray_copy(dec, out)
+
+        def __repr__(self):
+            return f'{type(self).__name__}(format={self.format!r}, check={self.check!r}, preset={self.preset!r}, filters={self.filters!r})'