numcodecs 0.16.0__tar.gz → 0.16.2__tar.gz

numcodecs-0.16.2/.gitattributes

@@ -0,0 +1,2 @@
+# GitHub syntax highlighting
+pixi.lock linguist-language=YAML linguist-generated=true

{numcodecs-0.16.0 → numcodecs-0.16.2}/.github/workflows/ci.yaml

@@ -70,3 +70,33 @@ jobs:
           fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
+
+  test-zarr:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0 # required for version resolution
+
+      - name: Set up Pixi
+        uses: prefix-dev/setup-pixi@v0.8.14
+        with:
+          pixi-version: v0.49.0
+          cache: false
+
+      - name: List deps
+        shell: "bash -l {0}"
+        run: pixi run -e default hatch run test:list-deps
+
+      - name: Run tests
+        shell: "bash -l {0}"
+        run: pixi run -e default hatch run test:test-zarr

{numcodecs-0.16.0 → numcodecs-0.16.2}/.github/workflows/wheel.yaml

@@ -26,7 +26,7 @@ jobs:
         with:
           submodules: true
 
-      - uses: pypa/cibuildwheel@v2.23.0
+      - uses: pypa/cibuildwheel@v2.23.2
 
       - uses: actions/upload-artifact@v4
         with:

{numcodecs-0.16.0 → numcodecs-0.16.2}/.gitignore

@@ -104,3 +104,7 @@ numcodecs/version.py
 
 # Cython generated
 numcodecs/*.c
+# pixi environments
+.pixi/*
+*.egg-info
+pixi.lock

{numcodecs-0.16.0 → numcodecs-0.16.2}/.pre-commit-config.yaml

@@ -14,19 +14,19 @@ repos:
     - id: debug-statements
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.9
+    rev: v0.11.12
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
     - id: ruff-format
 
   - repo: https://github.com/scientific-python/cookie
-    rev: 2025.01.22
+    rev: 2025.05.02
     hooks:
     - id: sp-repo-review
 
   -   repo: https://github.com/pre-commit/mirrors-mypy
-      rev: v1.15.0
+      rev: v1.16.0
       hooks:
       -   id: mypy
           args: [--config-file, pyproject.toml]

{numcodecs-0.16.0/numcodecs.egg-info → numcodecs-0.16.2}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: numcodecs
-Version: 0.16.0
+Version: 0.16.2
 Summary: A Python package providing buffer compression and transformation codecs for use in data storage and communication applications.
 Maintainer-email: Alistair Miles <alimanfoo@googlemail.com>
-License: MIT
+License-Expression: MIT
 Project-URL: Bug Tracker, https://github.com/zarr-developers/numcodecs/issues
 Project-URL: Changelog, https://numcodecs.readthedocs.io/en/stable/release.html
 Project-URL: Documentation, https://numcodecs.readthedocs.io/
@@ -12,7 +12,6 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Operating System :: Unix
@@ -21,6 +20,14 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
 License-File: LICENSE.txt
+License-File: c-blosc/LICENSE.txt
+License-File: c-blosc/LICENSES/BITSHUFFLE.txt
+License-File: c-blosc/LICENSES/FASTLZ.txt
+License-File: c-blosc/LICENSES/LZ4.txt
+License-File: c-blosc/LICENSES/SNAPPY.txt
+License-File: c-blosc/LICENSES/STDINT.txt
+License-File: c-blosc/LICENSES/ZLIB-NG.txt
+License-File: c-blosc/LICENSES/ZLIB.txt
 Requires-Dist: numpy>=1.24
 Requires-Dist: typing_extensions
 Provides-Extra: docs
@@ -32,6 +39,7 @@ Provides-Extra: test
 Requires-Dist: coverage; extra == "test"
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pyzstd; extra == "test"
 Provides-Extra: test-extras
 Requires-Dist: importlib_metadata; extra == "test-extras"
 Provides-Extra: msgpack

{numcodecs-0.16.0 → numcodecs-0.16.2}/docs/compression/zstd.rst

@@ -7,6 +7,9 @@ Zstd
     .. autoattribute:: codec_id
     .. automethod:: encode
     .. automethod:: decode
+        .. note::
+            If the compressed data does not contain the decompressed size, streaming
+            decompression will be used.
     .. automethod:: get_config
     .. automethod:: from_config
 

{numcodecs-0.16.0 → numcodecs-0.16.2}/docs/contributing.rst

@@ -203,12 +203,11 @@ contributors.
 Merging pull requests
 ~~~~~~~~~~~~~~~~~~~~~
 
-Pull requests submitted by an external contributor should be reviewed and approved by at least
-one core developers before being merged. Ideally, pull requests submitted by a core developer
-should be reviewed and approved by at least one other core developers before being merged.
+Pull requests should be reviewed and approved by at least one core developer
+(other than the pull request author) before being merged.
 
-Pull requests should not be merged until all CI checks have passed (Travis, AppVeyor,
-Codecov) against code that has had the latest main merged in.
+Pull requests should not be merged until all CI checks have passed against code
+that has had the latest main merged in.
 
 Compatibility and versioning policies
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -278,6 +277,10 @@ hard-and-fast rules, e.g., it is fine to make a minor release to make a single n
 feature available; equally, it is fine to make a minor release that includes a number of
 changes.
 
+When making a minor release, open an issue stating your intention so other developers
+know that a release is planned. At least a week's notice should be given for other
+developers to be aware of and possibly add to the contents of the release.
+
 Major releases obviously need to be given careful consideration, and should be done as
 infrequently as possible, as they will break existing code and/or affect data
 compatibility in some way.

{numcodecs-0.16.0 → numcodecs-0.16.2}/docs/release.rst

@@ -14,6 +14,39 @@ Release notes
 
 .. _unreleased:
 
+Unreleased
+----------
+
+Enhancements
+~~~~~~~~~~~~
+
+Improvements
+~~~~~~~~~~~~
+
+* In ``vlen``, define and use ``const`` ``HEADER_LENGTH``.
+  By :user:`John Kirkham <jakirkham>`, :issue:`723`
+* All codecs are now pickleable.
+  By :user:`Tom Nicholas <TomNicholas>`, :issue:`744`
+
+Fixes
+~~~~~
+
+* Fix ``const`` discard warnings in ``fletcher32``.
+  By :user:`John Kirkham <jakirkham>`, :issue:`728`
+
+* Update ``license*`` metadata and include third-party licenses.
+  By :user:`John Kirkham <jakirkham>`, :issue:`729`
+
+* Add ``#ifndef`` guard around ``PyBytes_RESIZE``.
+  By :user:`John Kirkham <jakirkham>`, :issue:`732`
+
+* Remove ``typesize`` from ``Blosc.get_config`` output
+  By :user:`Ilan Gold <ilan-gold>`
+
+Maintenance
+~~~~~~~~~~~
+
+
 0.16.0
 ------
 
@@ -62,6 +95,8 @@ Maintenance
 
 Improvements
 ~~~~~~~~~~~~
+* Add streaming decompression for ZSTD (:issue:`699`)
+  By :user:`Mark Kittisopikul <mkitti>`.
 * Raise a custom `UnknownCodecError` when trying to retrieve an unavailable codec.
   By :user:`Cas Wognum <cwognum>`.
 

{numcodecs-0.16.0 → numcodecs-0.16.2}/numcodecs/abc.py

@@ -29,14 +29,13 @@ other and vice versa.
 """
 
 from abc import ABC, abstractmethod
-from typing import Optional
 
 
 class Codec(ABC):
     """Codec abstract base class."""
 
     # override in sub-class
-    codec_id: Optional[str] = None
+    codec_id: str | None = None
     """Codec identifier."""
 
     @abstractmethod

{numcodecs-0.16.0 → numcodecs-0.16.2}/numcodecs/blosc.pyx

@@ -489,11 +489,11 @@ class Blosc(Codec):
         self.clevel = clevel
         self.shuffle = shuffle
         self.blocksize = blocksize
-        self.typesize = typesize
+        self._typesize = typesize
 
     def encode(self, buf):
         buf = ensure_contiguous_ndarray(buf, self.max_buffer_size)
-        return compress(buf, self._cname_bytes, self.clevel, self.shuffle, self.blocksize, self.typesize)
+        return compress(buf, self._cname_bytes, self.clevel, self.shuffle, self.blocksize, self._typesize)
 
     def decode(self, buf, out=None):
         buf = ensure_contiguous_ndarray(buf, self.max_buffer_size)

{numcodecs-0.16.0 → numcodecs-0.16.2}/numcodecs/checksum32.py

@@ -3,7 +3,7 @@ import struct
 import zlib
 from contextlib import suppress
 from types import ModuleType
-from typing import Literal, Optional
+from typing import Literal
 
 import numpy as np
 from typing_extensions import Buffer
@@ -12,7 +12,7 @@ from .abc import Codec
 from .compat import ensure_contiguous_ndarray, ndarray_copy
 from .jenkins import jenkins_lookup3
 
-_crc32c: Optional[ModuleType] = None
+_crc32c: ModuleType | None = None
 with suppress(ImportError):
     import crc32c as _crc32c  # type: ignore[no-redef, unused-ignore]
 

{numcodecs-0.16.0 → numcodecs-0.16.2}/numcodecs/compat_ext.pxd

@@ -3,7 +3,9 @@
 
 cdef extern from *:
     """
+    #ifndef PyBytes_RESIZE
     #define PyBytes_RESIZE(b, n) _PyBytes_Resize(&b, n)
+    #endif
     """
     int PyBytes_RESIZE(object b, Py_ssize_t n) except -1
 

{numcodecs-0.16.0 → numcodecs-0.16.2}/numcodecs/fletcher32.pyx

@@ -76,7 +76,7 @@ class Fletcher32(Codec):
         """Return buffer plus a footer with the fletcher checksum (4-bytes)"""
         buf = ensure_contiguous_ndarray(buf).ravel().view('uint8')
         cdef const uint8_t[::1] b_mv = buf
-        cdef uint8_t* b_ptr = &b_mv[0]
+        cdef const uint8_t* b_ptr = &b_mv[0]
         cdef Py_ssize_t b_len = len(b_mv)
 
         cdef Py_ssize_t out_len = b_len + FOOTER_LENGTH
@@ -92,7 +92,7 @@ class Fletcher32(Codec):
         """Check fletcher checksum, and return buffer without it"""
         b = ensure_contiguous_ndarray(buf).view('uint8')
         cdef const uint8_t[::1] b_mv = b
-        cdef uint8_t* b_ptr = &b_mv[0]
+        cdef const uint8_t* b_ptr = &b_mv[0]
         cdef Py_ssize_t b_len = len(b_mv)
 
         val = _fletcher32(b_mv[:-FOOTER_LENGTH])

{numcodecs-0.16.0 → numcodecs-0.16.2}/numcodecs/lzma.py

@@ -1,7 +1,6 @@
 from types import ModuleType
-from typing import Optional
 
-_lzma: Optional[ModuleType] = None
+_lzma: ModuleType | None = None
 try:
     import lzma as _lzma
 except ImportError:  # pragma: no cover

{numcodecs-0.16.0 → numcodecs-0.16.2}/numcodecs/tests/common.py

@@ -202,7 +202,7 @@ def check_backwards_compatibility(codec_id, arrays, codecs, precision=None, pref
 
         for j, codec in enumerate(codecs):
             if codec is None:
-                pytest.skip("codec has been removed")
+                continue
 
             # setup a directory to hold encoded data
             codec_dir = os.path.join(fixture_dir, f'codec.{j:02d}')

{numcodecs-0.16.0 → numcodecs-0.16.2}/numcodecs/tests/test_blosc.py

@@ -245,20 +245,20 @@ def test_err_encode_object_buffer():
     check_err_encode_object_buffer(Blosc())
 
 
-def test_decompression_error_handling():
-    for codec in codecs:
-        _skip_null(codec)
-        with pytest.raises(RuntimeError):
-            codec.decode(bytearray())
-        with pytest.raises(RuntimeError):
-            codec.decode(bytearray(0))
+@pytest.mark.parametrize('codec', codecs)
+def test_decompression_error_handling(codec):
+    _skip_null(codec)
+    with pytest.raises(RuntimeError):
+        codec.decode(bytearray())
+    with pytest.raises(RuntimeError):
+        codec.decode(bytearray(0))
 
 
-def test_max_buffer_size():
-    for codec in codecs:
-        _skip_null(codec)
-        assert codec.max_buffer_size == 2**31 - 1
-        check_max_buffer_size(codec)
+@pytest.mark.parametrize('codec', codecs)
+def test_max_buffer_size(codec):
+    _skip_null(codec)
+    assert codec.max_buffer_size == 2**31 - 1
+    check_max_buffer_size(codec)
 
 
 def test_typesize_explicit():
@@ -278,7 +278,13 @@ def test_typesize_less_than_1():
         Blosc(shuffle=Blosc.SHUFFLE, typesize=0)
     compressor = Blosc(shuffle=Blosc.SHUFFLE)
     # not really something that should be done in practice, but good for testing.
-    compressor.typesize = 0
+    compressor._typesize = 0
     arr = np.arange(100)
     with pytest.raises(ValueError, match=r"Cannot use typesize"):
         compressor.encode(arr.tobytes())
+
+
+def test_config_no_typesize():
+    codec = Blosc(shuffle=Blosc.SHUFFLE, typesize=5)
+    config = codec.get_config()
+    assert "typesize" not in config

numcodecs-0.16.2/numcodecs/tests/test_pyzstd.py

@@ -0,0 +1,77 @@
+# Check Zstd against pyzstd package
+
+import numpy as np
+import pytest
+import pyzstd
+
+from numcodecs.zstd import Zstd
+
+test_data = [
+    b"Hello World!",
+    np.arange(113).tobytes(),
+    np.arange(10, 15).tobytes(),
+    np.random.randint(3, 50, size=(53,), dtype=np.uint16).tobytes(),
+]
+
+
+@pytest.mark.parametrize("input", test_data)
+def test_pyzstd_simple(input):
+    """
+    Test if Zstd.[decode, encode] can perform the inverse operation to
+    pyzstd.[compress, decompress] in the simple case.
+    """
+    z = Zstd()
+    assert z.decode(pyzstd.compress(input)) == input
+    assert pyzstd.decompress(z.encode(input)) == input
+
+
+@pytest.mark.xfail
+@pytest.mark.parametrize("input", test_data)
+def test_pyzstd_simple_multiple_frames_decode(input):
+    """
+    Test decompression of two concatenated frames of known sizes
+
+    numcodecs.zstd.Zstd currently fails because it only assesses the size of the
+    first frame. Rather, it should keep iterating through all the frames until
+    the end of the input buffer.
+    """
+    z = Zstd()
+    assert pyzstd.decompress(pyzstd.compress(input) * 2) == input * 2
+    assert z.decode(pyzstd.compress(input) * 2) == input * 2
+
+
+@pytest.mark.parametrize("input", test_data)
+def test_pyzstd_simple_multiple_frames_encode(input):
+    """
+    Test if pyzstd can decompress two concatenated frames from Zstd.encode
+    """
+    z = Zstd()
+    assert pyzstd.decompress(z.encode(input) * 2) == input * 2
+
+
+@pytest.mark.parametrize("input", test_data)
+def test_pyzstd_streaming(input):
+    """
+    Test if Zstd can decode a single frame and concatenated frames in streaming
+    mode where the decompressed size is not recorded in the frame header.
+    """
+    pyzstd_c = pyzstd.ZstdCompressor()
+    pyzstd_d = pyzstd.ZstdDecompressor()
+    pyzstd_e = pyzstd.EndlessZstdDecompressor()
+    z = Zstd()
+
+    d_bytes = input
+    pyzstd_c.compress(d_bytes)
+    c_bytes = pyzstd_c.flush()
+    assert z.decode(c_bytes) == d_bytes
+    assert pyzstd_d.decompress(z.encode(d_bytes)) == d_bytes
+
+    # Test multiple streaming frames
+    assert z.decode(c_bytes * 2) == pyzstd_e.decompress(c_bytes * 2)
+    assert z.decode(c_bytes * 3) == pyzstd_e.decompress(c_bytes * 3)
+    assert z.decode(c_bytes * 4) == pyzstd_e.decompress(c_bytes * 4)
+    assert z.decode(c_bytes * 5) == pyzstd_e.decompress(c_bytes * 5)
+    assert z.decode(c_bytes * 7) == pyzstd_e.decompress(c_bytes * 7)
+    assert z.decode(c_bytes * 11) == pyzstd_e.decompress(c_bytes * 11)
+    assert z.decode(c_bytes * 13) == pyzstd_e.decompress(c_bytes * 13)
+    assert z.decode(c_bytes * 99) == pyzstd_e.decompress(c_bytes * 99)

{numcodecs-0.16.0 → numcodecs-0.16.2}/numcodecs/tests/test_vlen_bytes.py

@@ -1,4 +1,3 @@
-import sys
 import unittest
 
 import numpy as np
@@ -85,9 +84,6 @@ def test_decode_errors():
         codec.decode(enc, out=np.zeros(10, dtype='i4'))
 
 
-# TODO: fix this test on GitHub actions somehow...
-# See https://github.com/zarr-developers/numcodecs/issues/683
-@pytest.mark.skipif(sys.platform == "darwin", reason="Test is failing on macOS on GitHub actions.")
 def test_encode_none():
     a = np.array([b'foo', None, b'bar'], dtype=object)
     codec = VLenBytes()

{numcodecs-0.16.0 → numcodecs-0.16.2}/numcodecs/tests/test_zarr3.py

@@ -1,10 +1,13 @@
 from __future__ import annotations
 
+import pickle
 from typing import TYPE_CHECKING
 
 import numpy as np
 import pytest
 
+import numcodecs.bitround
+
 if TYPE_CHECKING:  # pragma: no cover
     import zarr
 else:
@@ -119,8 +122,8 @@ def test_generic_filter(
             ],
         )
 
-        a[:, :] = data.copy()
-        a = zarr.open_array(store / "generic", mode="r")
+    a[:, :] = data.copy()
+    a = zarr.open_array(store / "generic", mode="r")
     np.testing.assert_array_equal(data, a[:, :])
 
 
@@ -137,8 +140,8 @@ def test_generic_filter_bitround(store: StorePath):
             filters=[numcodecs.zarr3.BitRound(keepbits=3)],
         )
 
-        a[:, :] = data.copy()
-        a = zarr.open_array(store / "generic_bitround", mode="r")
+    a[:, :] = data.copy()
+    a = zarr.open_array(store / "generic_bitround", mode="r")
     assert np.allclose(data, a[:, :], atol=0.1)
 
 
@@ -155,8 +158,8 @@ def test_generic_filter_quantize(store: StorePath):
             filters=[numcodecs.zarr3.Quantize(digits=3)],
         )
 
-        a[:, :] = data.copy()
-        a = zarr.open_array(store / "generic_quantize", mode="r")
+    a[:, :] = data.copy()
+    a = zarr.open_array(store / "generic_quantize", mode="r")
     assert np.allclose(data, a[:, :], atol=0.001)
 
 
@@ -174,8 +177,8 @@ def test_generic_filter_packbits(store: StorePath):
             filters=[numcodecs.zarr3.PackBits()],
         )
 
-        a[:, :] = data.copy()
-        a = zarr.open_array(store / "generic_packbits", mode="r")
+    a[:, :] = data.copy()
+    a = zarr.open_array(store / "generic_packbits", mode="r")
     np.testing.assert_array_equal(data, a[:, :])
 
     with pytest.raises(ValueError, match=".*requires bool dtype.*"):
@@ -214,8 +217,8 @@ def test_generic_checksum(
             compressors=[codec_class()],
         )
 
-        a[:, :] = data.copy()
-        a = zarr.open_array(store / "generic_checksum", mode="r")
+    a[:, :] = data.copy()
+    a = zarr.open_array(store / "generic_checksum", mode="r")
     np.testing.assert_array_equal(data, a[:, :])
 
 
@@ -260,12 +263,12 @@ def test_delta_astype(store: StorePath):
             dtype=data.dtype,
             fill_value=0,
             filters=[
-                numcodecs.zarr3.Delta(dtype="i8", astype="i2"),  # type: ignore[arg-type]
+                numcodecs.zarr3.Delta(dtype="i8", astype="i2"),
             ],
         )
 
-        a[:, :] = data.copy()
-        a = zarr.open_array(store / "generic", mode="r")
+    a[:, :] = data.copy()
+    a = zarr.open_array(store / "generic", mode="r")
     np.testing.assert_array_equal(data, a[:, :])
 
 
@@ -277,3 +280,39 @@ def test_repr():
 def test_to_dict():
     codec = numcodecs.zarr3.LZ4(level=5)
     assert codec.to_dict() == {"name": "numcodecs.lz4", "configuration": {"level": 5}}
+
+
+@pytest.mark.parametrize(
+    "codec_cls",
+    [
+        numcodecs.zarr3.Blosc,
+        numcodecs.zarr3.LZ4,
+        numcodecs.zarr3.Zstd,
+        numcodecs.zarr3.Zlib,
+        numcodecs.zarr3.GZip,
+        numcodecs.zarr3.BZ2,
+        numcodecs.zarr3.LZMA,
+        numcodecs.zarr3.Shuffle,
+        numcodecs.zarr3.BitRound,
+        numcodecs.zarr3.Delta,
+        numcodecs.zarr3.FixedScaleOffset,
+        numcodecs.zarr3.Quantize,
+        numcodecs.zarr3.PackBits,
+        numcodecs.zarr3.AsType,
+        numcodecs.zarr3.CRC32,
+        numcodecs.zarr3.CRC32C,
+        numcodecs.zarr3.Adler32,
+        numcodecs.zarr3.Fletcher32,
+        numcodecs.zarr3.JenkinsLookup3,
+        numcodecs.zarr3.PCodec,
+        numcodecs.zarr3.ZFPY,
+    ],
+)
+def test_codecs_pickleable(codec_cls):
+    codec = codec_cls()
+
+    expected = codec
+
+    p = pickle.dumps(codec)
+    actual = pickle.loads(p)
+    assert actual == expected