numcodecs 0.14.0__cp312-cp312-win_amd64.whl → 0.15.0__cp312-cp312-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of numcodecs might be problematic. Click here for more details.
- numcodecs/__init__.py +44 -43
- numcodecs/_shuffle.cp312-win_amd64.pyd +0 -0
- numcodecs/abc.py +1 -1
- numcodecs/astype.py +2 -6
- numcodecs/base64.py +1 -2
- numcodecs/blosc.cp312-win_amd64.pyd +0 -0
- numcodecs/categorize.py +7 -10
- numcodecs/checksum32.py +24 -24
- numcodecs/compat_ext.cp312-win_amd64.pyd +0 -0
- numcodecs/delta.py +3 -10
- numcodecs/fixedscaleoffset.py +8 -10
- numcodecs/fletcher32.cp312-win_amd64.pyd +0 -0
- numcodecs/gzip.py +1 -3
- numcodecs/jenkins.cp312-win_amd64.pyd +0 -0
- numcodecs/json.py +11 -11
- numcodecs/lz4.cp312-win_amd64.pyd +0 -0
- numcodecs/lzma.py +1 -1
- numcodecs/msgpacks.py +6 -6
- numcodecs/ndarray_like.py +2 -2
- numcodecs/pcodec.py +61 -32
- numcodecs/pickles.py +1 -1
- numcodecs/quantize.py +7 -9
- numcodecs/registry.py +1 -1
- numcodecs/tests/common.py +3 -4
- numcodecs/tests/test_blosc.py +9 -11
- numcodecs/tests/test_checksum32.py +25 -9
- numcodecs/tests/test_lzma.py +1 -1
- numcodecs/tests/test_pcodec.py +18 -8
- numcodecs/tests/test_registry.py +2 -2
- numcodecs/tests/test_shuffle.py +2 -4
- numcodecs/tests/test_vlen_bytes.py +3 -0
- numcodecs/tests/test_zarr3.py +70 -44
- numcodecs/version.py +2 -2
- numcodecs/vlen.cp312-win_amd64.pyd +0 -0
- numcodecs/zarr3.py +44 -22
- numcodecs/zfpy.py +1 -1
- numcodecs/zstd.cp312-win_amd64.pyd +0 -0
- {numcodecs-0.14.0.dist-info → numcodecs-0.15.0.dist-info}/METADATA +20 -21
- numcodecs-0.15.0.dist-info/RECORD +78 -0
- {numcodecs-0.14.0.dist-info → numcodecs-0.15.0.dist-info}/WHEEL +1 -1
- numcodecs-0.14.0.dist-info/RECORD +0 -78
- {numcodecs-0.14.0.dist-info → numcodecs-0.15.0.dist-info}/LICENSE.txt +0 -0
- {numcodecs-0.14.0.dist-info → numcodecs-0.15.0.dist-info}/entry_points.txt +0 -0
- {numcodecs-0.14.0.dist-info → numcodecs-0.15.0.dist-info}/top_level.txt +0 -0
numcodecs/tests/test_blosc.py
CHANGED
|
@@ -124,12 +124,12 @@ def test_compress_blocksize_default(use_threads):
|
|
|
124
124
|
|
|
125
125
|
# default blocksize
|
|
126
126
|
enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE)
|
|
127
|
-
_, _, blocksize = blosc.
|
|
127
|
+
_, _, blocksize = blosc._cbuffer_sizes(enc)
|
|
128
128
|
assert blocksize > 0
|
|
129
129
|
|
|
130
130
|
# explicit default blocksize
|
|
131
131
|
enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE, 0)
|
|
132
|
-
_, _, blocksize = blosc.
|
|
132
|
+
_, _, blocksize = blosc._cbuffer_sizes(enc)
|
|
133
133
|
assert blocksize > 0
|
|
134
134
|
|
|
135
135
|
|
|
@@ -140,7 +140,7 @@ def test_compress_blocksize(use_threads, bs):
|
|
|
140
140
|
blosc.use_threads = use_threads
|
|
141
141
|
|
|
142
142
|
enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE, bs)
|
|
143
|
-
_, _, blocksize = blosc.
|
|
143
|
+
_, _, blocksize = blosc._cbuffer_sizes(enc)
|
|
144
144
|
assert blocksize == bs
|
|
145
145
|
|
|
146
146
|
|
|
@@ -174,7 +174,7 @@ def test_compress_metainfo(dtype, use_threads):
|
|
|
174
174
|
blosc.use_threads = use_threads
|
|
175
175
|
for cname in blosc.list_compressors():
|
|
176
176
|
enc = blosc.compress(arr, cname.encode(), 1, shuffle)
|
|
177
|
-
typesize, did_shuffle, _ = blosc.
|
|
177
|
+
typesize, did_shuffle, _ = blosc._cbuffer_metainfo(enc)
|
|
178
178
|
assert typesize == arr.dtype.itemsize
|
|
179
179
|
assert did_shuffle == shuffle
|
|
180
180
|
|
|
@@ -186,7 +186,7 @@ def test_compress_autoshuffle(use_threads):
|
|
|
186
186
|
blosc.use_threads = use_threads
|
|
187
187
|
for cname in blosc.list_compressors():
|
|
188
188
|
enc = blosc.compress(varr, cname.encode(), 1, Blosc.AUTOSHUFFLE)
|
|
189
|
-
typesize, did_shuffle, _ = blosc.
|
|
189
|
+
typesize, did_shuffle, _ = blosc._cbuffer_metainfo(enc)
|
|
190
190
|
assert typesize == varr.dtype.itemsize
|
|
191
191
|
if typesize == 1:
|
|
192
192
|
assert did_shuffle == Blosc.BITSHUFFLE
|
|
@@ -199,12 +199,12 @@ def test_config_blocksize():
|
|
|
199
199
|
# explicitly stated
|
|
200
200
|
|
|
201
201
|
# blocksize not stated
|
|
202
|
-
config =
|
|
202
|
+
config = {"cname": 'lz4', "clevel": 1, "shuffle": Blosc.SHUFFLE}
|
|
203
203
|
codec = Blosc.from_config(config)
|
|
204
204
|
assert codec.blocksize == 0
|
|
205
205
|
|
|
206
206
|
# blocksize stated
|
|
207
|
-
config =
|
|
207
|
+
config = {"cname": 'lz4', "clevel": 1, "shuffle": Blosc.SHUFFLE, "blocksize": 2**8}
|
|
208
208
|
codec = Blosc.from_config(config)
|
|
209
209
|
assert codec.blocksize == 2**8
|
|
210
210
|
|
|
@@ -215,14 +215,12 @@ def test_backwards_compatibility():
|
|
|
215
215
|
|
|
216
216
|
def _encode_worker(data):
|
|
217
217
|
compressor = Blosc(cname='zlib', clevel=9, shuffle=Blosc.SHUFFLE)
|
|
218
|
-
|
|
219
|
-
return enc
|
|
218
|
+
return compressor.encode(data)
|
|
220
219
|
|
|
221
220
|
|
|
222
221
|
def _decode_worker(enc):
|
|
223
222
|
compressor = Blosc()
|
|
224
|
-
|
|
225
|
-
return data
|
|
223
|
+
return compressor.decode(enc)
|
|
226
224
|
|
|
227
225
|
|
|
228
226
|
@pytest.mark.parametrize('pool', [Pool, ThreadPool])
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import itertools
|
|
2
|
+
from contextlib import suppress
|
|
2
3
|
|
|
3
4
|
import numpy as np
|
|
4
5
|
import pytest
|
|
5
6
|
|
|
6
|
-
from numcodecs.checksum32 import CRC32,
|
|
7
|
+
from numcodecs.checksum32 import CRC32, Adler32
|
|
7
8
|
from numcodecs.tests.common import (
|
|
8
9
|
check_backwards_compatibility,
|
|
9
10
|
check_config,
|
|
@@ -13,6 +14,12 @@ from numcodecs.tests.common import (
|
|
|
13
14
|
check_repr,
|
|
14
15
|
)
|
|
15
16
|
|
|
17
|
+
has_crc32c = False
|
|
18
|
+
with suppress(ImportError):
|
|
19
|
+
from numcodecs.checksum32 import CRC32C
|
|
20
|
+
|
|
21
|
+
has_crc32c = True
|
|
22
|
+
|
|
16
23
|
# mix of dtypes: integer, float, bool, string
|
|
17
24
|
# mix of shapes: 1D, 2D, 3D
|
|
18
25
|
# mix of orders: C, F
|
|
@@ -35,11 +42,16 @@ arrays = [
|
|
|
35
42
|
codecs = [
|
|
36
43
|
CRC32(),
|
|
37
44
|
CRC32(location="end"),
|
|
38
|
-
CRC32C(location="start"),
|
|
39
|
-
CRC32C(),
|
|
40
45
|
Adler32(),
|
|
41
46
|
Adler32(location="end"),
|
|
42
47
|
]
|
|
48
|
+
if has_crc32c:
|
|
49
|
+
codecs.extend(
|
|
50
|
+
[
|
|
51
|
+
CRC32C(location="start"),
|
|
52
|
+
CRC32C(),
|
|
53
|
+
]
|
|
54
|
+
)
|
|
43
55
|
|
|
44
56
|
|
|
45
57
|
@pytest.mark.parametrize(("codec", "arr"), itertools.product(codecs, arrays))
|
|
@@ -84,25 +96,28 @@ def test_err_encode_list(codec):
|
|
|
84
96
|
def test_err_location():
|
|
85
97
|
with pytest.raises(ValueError):
|
|
86
98
|
CRC32(location="foo")
|
|
87
|
-
with pytest.raises(ValueError):
|
|
88
|
-
CRC32C(location="foo")
|
|
89
99
|
with pytest.raises(ValueError):
|
|
90
100
|
Adler32(location="foo")
|
|
101
|
+
if has_crc32c:
|
|
102
|
+
with pytest.raises(ValueError):
|
|
103
|
+
CRC32C(location="foo")
|
|
91
104
|
|
|
92
105
|
|
|
93
106
|
def test_repr():
|
|
94
107
|
check_repr("CRC32(location='start')")
|
|
95
|
-
check_repr("CRC32C(location='start')")
|
|
96
|
-
check_repr("Adler32(location='start')")
|
|
97
108
|
check_repr("CRC32(location='end')")
|
|
98
|
-
check_repr("
|
|
109
|
+
check_repr("Adler32(location='start')")
|
|
99
110
|
check_repr("Adler32(location='end')")
|
|
111
|
+
if has_crc32c:
|
|
112
|
+
check_repr("CRC32C(location='start')")
|
|
113
|
+
check_repr("CRC32C(location='end')")
|
|
100
114
|
|
|
101
115
|
|
|
102
116
|
def test_backwards_compatibility():
|
|
103
117
|
check_backwards_compatibility(CRC32.codec_id, arrays, [CRC32()])
|
|
104
118
|
check_backwards_compatibility(Adler32.codec_id, arrays, [Adler32()])
|
|
105
|
-
|
|
119
|
+
if has_crc32c:
|
|
120
|
+
check_backwards_compatibility(CRC32C.codec_id, arrays, [CRC32C()])
|
|
106
121
|
|
|
107
122
|
|
|
108
123
|
@pytest.mark.parametrize("codec", codecs)
|
|
@@ -123,6 +138,7 @@ def test_err_out_too_small(codec):
|
|
|
123
138
|
codec.decode(codec.encode(arr), out)
|
|
124
139
|
|
|
125
140
|
|
|
141
|
+
@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed")
|
|
126
142
|
def test_crc32c_checksum():
|
|
127
143
|
arr = np.arange(0, 64, dtype="uint8")
|
|
128
144
|
buf = CRC32C(location="end").encode(arr)
|
numcodecs/tests/test_lzma.py
CHANGED
numcodecs/tests/test_pcodec.py
CHANGED
|
@@ -1,11 +1,8 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import pytest
|
|
3
3
|
|
|
4
|
-
from numcodecs.pcodec import PCodec
|
|
5
|
-
|
|
6
4
|
try:
|
|
7
|
-
|
|
8
|
-
PCodec()
|
|
5
|
+
from numcodecs.pcodec import PCodec
|
|
9
6
|
except ImportError: # pragma: no cover
|
|
10
7
|
pytest.skip("pcodec not available", allow_module_level=True)
|
|
11
8
|
|
|
@@ -23,8 +20,12 @@ codecs = [
|
|
|
23
20
|
PCodec(level=1),
|
|
24
21
|
PCodec(level=5),
|
|
25
22
|
PCodec(level=9),
|
|
26
|
-
PCodec(mode_spec=
|
|
23
|
+
PCodec(mode_spec="classic"),
|
|
27
24
|
PCodec(equal_pages_up_to=300),
|
|
25
|
+
PCodec(delta_encoding_order=2),
|
|
26
|
+
PCodec(delta_spec="try_lookback"),
|
|
27
|
+
PCodec(delta_spec="none"),
|
|
28
|
+
PCodec(delta_spec="try_consecutive", delta_encoding_order=1),
|
|
28
29
|
]
|
|
29
30
|
|
|
30
31
|
|
|
@@ -56,15 +57,24 @@ def test_config():
|
|
|
56
57
|
check_config(codec)
|
|
57
58
|
|
|
58
59
|
|
|
59
|
-
|
|
60
|
-
|
|
60
|
+
@pytest.mark.parametrize("param", ["mode_spec", "delta_spec", "paging_spec"])
|
|
61
|
+
def test_invalid_config_error(param):
|
|
62
|
+
codec = PCodec(**{param: "bogus"})
|
|
63
|
+
with pytest.raises(ValueError):
|
|
64
|
+
check_encode_decode_array_to_bytes(arrays[0], codec)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_invalid_delta_encoding_combo():
|
|
68
|
+
codec = PCodec(delta_encoding_order=2, delta_spec="none")
|
|
61
69
|
with pytest.raises(ValueError):
|
|
62
70
|
check_encode_decode_array_to_bytes(arrays[0], codec)
|
|
63
71
|
|
|
64
72
|
|
|
65
73
|
def test_repr():
|
|
66
74
|
check_repr(
|
|
67
|
-
"PCodec(delta_encoding_order=None,
|
|
75
|
+
"PCodec(delta_encoding_order=None, delta_spec='auto',"
|
|
76
|
+
" equal_pages_up_to=262144, level=3, mode_spec='auto',"
|
|
77
|
+
" paging_spec='equal_pages_up_to')"
|
|
68
78
|
)
|
|
69
79
|
|
|
70
80
|
|
numcodecs/tests/test_registry.py
CHANGED
|
@@ -26,7 +26,7 @@ def test_all_classes_registered():
|
|
|
26
26
|
|
|
27
27
|
see #346 for more info
|
|
28
28
|
"""
|
|
29
|
-
missing =
|
|
29
|
+
missing = {
|
|
30
30
|
obj.codec_id
|
|
31
31
|
for _, submod in inspect.getmembers(numcodecs, inspect.ismodule)
|
|
32
32
|
for _, obj in inspect.getmembers(submod)
|
|
@@ -36,7 +36,7 @@ def test_all_classes_registered():
|
|
|
36
36
|
and obj.codec_id not in numcodecs.registry.codec_registry
|
|
37
37
|
and obj.codec_id is not None # remove `None`
|
|
38
38
|
)
|
|
39
|
-
|
|
39
|
+
}
|
|
40
40
|
|
|
41
41
|
if missing:
|
|
42
42
|
raise Exception(f"these codecs are missing: {missing}") # pragma: no cover
|
numcodecs/tests/test_shuffle.py
CHANGED
|
@@ -79,14 +79,12 @@ def test_eq():
|
|
|
79
79
|
|
|
80
80
|
def _encode_worker(data):
|
|
81
81
|
compressor = Shuffle()
|
|
82
|
-
|
|
83
|
-
return enc
|
|
82
|
+
return compressor.encode(data)
|
|
84
83
|
|
|
85
84
|
|
|
86
85
|
def _decode_worker(enc):
|
|
87
86
|
compressor = Shuffle()
|
|
88
|
-
|
|
89
|
-
return data
|
|
87
|
+
return compressor.decode(enc)
|
|
90
88
|
|
|
91
89
|
|
|
92
90
|
@pytest.mark.parametrize('pool', [Pool, ThreadPool])
|
|
@@ -84,6 +84,9 @@ def test_decode_errors():
|
|
|
84
84
|
codec.decode(enc, out=np.zeros(10, dtype='i4'))
|
|
85
85
|
|
|
86
86
|
|
|
87
|
+
# TODO: fix this test on GitHub actions somehow...
|
|
88
|
+
# See https://github.com/zarr-developers/numcodecs/issues/683
|
|
89
|
+
@pytest.mark.skip("Test is failing on GitHub actions.")
|
|
87
90
|
def test_encode_none():
|
|
88
91
|
a = np.array([b'foo', None, b'bar'], dtype=object)
|
|
89
92
|
codec = VLenBytes()
|
numcodecs/tests/test_zarr3.py
CHANGED
|
@@ -5,10 +5,10 @@ from typing import TYPE_CHECKING
|
|
|
5
5
|
import numpy as np
|
|
6
6
|
import pytest
|
|
7
7
|
|
|
8
|
-
if
|
|
9
|
-
zarr = pytest.importorskip("zarr")
|
|
10
|
-
else:
|
|
8
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
11
9
|
import zarr
|
|
10
|
+
else:
|
|
11
|
+
zarr = pytest.importorskip("zarr")
|
|
12
12
|
|
|
13
13
|
import zarr.storage
|
|
14
14
|
from zarr.core.common import JSON
|
|
@@ -36,7 +36,7 @@ EXPECTED_WARNING_STR = "Numcodecs codecs are not in the Zarr version 3.*"
|
|
|
36
36
|
|
|
37
37
|
@pytest.fixture
|
|
38
38
|
def store() -> StorePath:
|
|
39
|
-
return StorePath(MemoryStore(
|
|
39
|
+
return StorePath(MemoryStore(read_only=False))
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
ALL_CODECS = [getattr(numcodecs.zarr3, cls_name) for cls_name in numcodecs.zarr3.__all__]
|
|
@@ -50,9 +50,7 @@ def test_entry_points(codec_class: type[numcodecs.zarr3._NumcodecsCodec]):
|
|
|
50
50
|
|
|
51
51
|
@pytest.mark.parametrize("codec_class", ALL_CODECS)
|
|
52
52
|
def test_docstring(codec_class: type[numcodecs.zarr3._NumcodecsCodec]):
|
|
53
|
-
|
|
54
|
-
pytest.skip()
|
|
55
|
-
assert "See :class:`numcodecs." in codec_class.__doc__
|
|
53
|
+
assert "See :class:`numcodecs." in codec_class.__doc__ # type: ignore[operator]
|
|
56
54
|
|
|
57
55
|
|
|
58
56
|
@pytest.mark.parametrize(
|
|
@@ -68,17 +66,19 @@ def test_docstring(codec_class: type[numcodecs.zarr3._NumcodecsCodec]):
|
|
|
68
66
|
numcodecs.zarr3.Shuffle,
|
|
69
67
|
],
|
|
70
68
|
)
|
|
71
|
-
def
|
|
69
|
+
def test_generic_compressor(
|
|
70
|
+
store: StorePath, codec_class: type[numcodecs.zarr3._NumcodecsBytesBytesCodec]
|
|
71
|
+
):
|
|
72
72
|
data = np.arange(0, 256, dtype="uint16").reshape((16, 16))
|
|
73
73
|
|
|
74
74
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
75
|
-
a =
|
|
75
|
+
a = zarr.create_array(
|
|
76
76
|
store / "generic",
|
|
77
77
|
shape=data.shape,
|
|
78
|
-
|
|
78
|
+
chunks=(16, 16),
|
|
79
79
|
dtype=data.dtype,
|
|
80
80
|
fill_value=0,
|
|
81
|
-
|
|
81
|
+
compressors=[codec_class()],
|
|
82
82
|
)
|
|
83
83
|
|
|
84
84
|
a[:, :] = data.copy()
|
|
@@ -102,26 +102,25 @@ def test_generic_codec_class(store: StorePath, codec_class: type[numcodecs.zarr3
|
|
|
102
102
|
)
|
|
103
103
|
def test_generic_filter(
|
|
104
104
|
store: StorePath,
|
|
105
|
-
codec_class: type[numcodecs.zarr3.
|
|
105
|
+
codec_class: type[numcodecs.zarr3._NumcodecsArrayArrayCodec],
|
|
106
106
|
codec_config: dict[str, JSON],
|
|
107
107
|
):
|
|
108
108
|
data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
|
|
109
109
|
|
|
110
110
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
111
|
-
a =
|
|
111
|
+
a = zarr.create_array(
|
|
112
112
|
store / "generic",
|
|
113
113
|
shape=data.shape,
|
|
114
|
-
|
|
114
|
+
chunks=(16, 16),
|
|
115
115
|
dtype=data.dtype,
|
|
116
116
|
fill_value=0,
|
|
117
|
-
|
|
117
|
+
filters=[
|
|
118
118
|
codec_class(**codec_config),
|
|
119
|
-
BytesCodec(),
|
|
120
119
|
],
|
|
121
120
|
)
|
|
122
121
|
|
|
123
122
|
a[:, :] = data.copy()
|
|
124
|
-
a =
|
|
123
|
+
a = zarr.open_array(store / "generic", mode="r")
|
|
125
124
|
np.testing.assert_array_equal(data, a[:, :])
|
|
126
125
|
|
|
127
126
|
|
|
@@ -129,17 +128,17 @@ def test_generic_filter_bitround(store: StorePath):
|
|
|
129
128
|
data = np.linspace(0, 1, 256, dtype="float32").reshape((16, 16))
|
|
130
129
|
|
|
131
130
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
132
|
-
a =
|
|
131
|
+
a = zarr.create_array(
|
|
133
132
|
store / "generic_bitround",
|
|
134
133
|
shape=data.shape,
|
|
135
|
-
|
|
134
|
+
chunks=(16, 16),
|
|
136
135
|
dtype=data.dtype,
|
|
137
136
|
fill_value=0,
|
|
138
|
-
|
|
137
|
+
filters=[numcodecs.zarr3.BitRound(keepbits=3)],
|
|
139
138
|
)
|
|
140
139
|
|
|
141
140
|
a[:, :] = data.copy()
|
|
142
|
-
a =
|
|
141
|
+
a = zarr.open_array(store / "generic_bitround", mode="r")
|
|
143
142
|
assert np.allclose(data, a[:, :], atol=0.1)
|
|
144
143
|
|
|
145
144
|
|
|
@@ -147,17 +146,17 @@ def test_generic_filter_quantize(store: StorePath):
|
|
|
147
146
|
data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
|
|
148
147
|
|
|
149
148
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
150
|
-
a =
|
|
149
|
+
a = zarr.create_array(
|
|
151
150
|
store / "generic_quantize",
|
|
152
151
|
shape=data.shape,
|
|
153
|
-
|
|
152
|
+
chunks=(16, 16),
|
|
154
153
|
dtype=data.dtype,
|
|
155
154
|
fill_value=0,
|
|
156
|
-
|
|
155
|
+
filters=[numcodecs.zarr3.Quantize(digits=3)],
|
|
157
156
|
)
|
|
158
157
|
|
|
159
158
|
a[:, :] = data.copy()
|
|
160
|
-
a =
|
|
159
|
+
a = zarr.open_array(store / "generic_quantize", mode="r")
|
|
161
160
|
assert np.allclose(data, a[:, :], atol=0.001)
|
|
162
161
|
|
|
163
162
|
|
|
@@ -166,27 +165,27 @@ def test_generic_filter_packbits(store: StorePath):
|
|
|
166
165
|
data[0:4, :] = True
|
|
167
166
|
|
|
168
167
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
169
|
-
a =
|
|
168
|
+
a = zarr.create_array(
|
|
170
169
|
store / "generic_packbits",
|
|
171
170
|
shape=data.shape,
|
|
172
|
-
|
|
171
|
+
chunks=(16, 16),
|
|
173
172
|
dtype=data.dtype,
|
|
174
173
|
fill_value=0,
|
|
175
|
-
|
|
174
|
+
filters=[numcodecs.zarr3.PackBits()],
|
|
176
175
|
)
|
|
177
176
|
|
|
178
177
|
a[:, :] = data.copy()
|
|
179
|
-
a =
|
|
178
|
+
a = zarr.open_array(store / "generic_packbits", mode="r")
|
|
180
179
|
np.testing.assert_array_equal(data, a[:, :])
|
|
181
180
|
|
|
182
181
|
with pytest.raises(ValueError, match=".*requires bool dtype.*"):
|
|
183
|
-
|
|
182
|
+
zarr.create_array(
|
|
184
183
|
store / "generic_packbits_err",
|
|
185
184
|
shape=data.shape,
|
|
186
|
-
|
|
185
|
+
chunks=(16, 16),
|
|
187
186
|
dtype="uint32",
|
|
188
187
|
fill_value=0,
|
|
189
|
-
|
|
188
|
+
filters=[numcodecs.zarr3.PackBits()],
|
|
190
189
|
)
|
|
191
190
|
|
|
192
191
|
|
|
@@ -200,26 +199,30 @@ def test_generic_filter_packbits(store: StorePath):
|
|
|
200
199
|
numcodecs.zarr3.JenkinsLookup3,
|
|
201
200
|
],
|
|
202
201
|
)
|
|
203
|
-
def test_generic_checksum(
|
|
202
|
+
def test_generic_checksum(
|
|
203
|
+
store: StorePath, codec_class: type[numcodecs.zarr3._NumcodecsBytesBytesCodec]
|
|
204
|
+
):
|
|
204
205
|
data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
|
|
205
206
|
|
|
206
207
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
207
|
-
a =
|
|
208
|
+
a = zarr.create_array(
|
|
208
209
|
store / "generic_checksum",
|
|
209
210
|
shape=data.shape,
|
|
210
|
-
|
|
211
|
+
chunks=(16, 16),
|
|
211
212
|
dtype=data.dtype,
|
|
212
213
|
fill_value=0,
|
|
213
|
-
|
|
214
|
+
compressors=[codec_class()],
|
|
214
215
|
)
|
|
215
216
|
|
|
216
217
|
a[:, :] = data.copy()
|
|
217
|
-
a =
|
|
218
|
+
a = zarr.open_array(store / "generic_checksum", mode="r")
|
|
218
219
|
np.testing.assert_array_equal(data, a[:, :])
|
|
219
220
|
|
|
220
221
|
|
|
221
222
|
@pytest.mark.parametrize("codec_class", [numcodecs.zarr3.PCodec, numcodecs.zarr3.ZFPY])
|
|
222
|
-
def test_generic_bytes_codec(
|
|
223
|
+
def test_generic_bytes_codec(
|
|
224
|
+
store: StorePath, codec_class: type[numcodecs.zarr3._NumcodecsArrayBytesCodec]
|
|
225
|
+
):
|
|
223
226
|
try:
|
|
224
227
|
codec_class()._codec # noqa: B018
|
|
225
228
|
except ValueError as e:
|
|
@@ -227,22 +230,45 @@ def test_generic_bytes_codec(store: StorePath, codec_class: type[numcodecs.zarr3
|
|
|
227
230
|
pytest.xfail(f"{codec_class.codec_name} is not available: {e}")
|
|
228
231
|
else:
|
|
229
232
|
raise # pragma: no cover
|
|
230
|
-
except ImportError as e:
|
|
233
|
+
except ImportError as e: # pragma: no cover
|
|
231
234
|
pytest.xfail(f"{codec_class.codec_name} is not available: {e}")
|
|
232
235
|
|
|
233
236
|
data = np.arange(0, 256, dtype="float32").reshape((16, 16))
|
|
234
237
|
|
|
235
238
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
236
|
-
a =
|
|
239
|
+
a = zarr.create_array(
|
|
237
240
|
store / "generic",
|
|
238
241
|
shape=data.shape,
|
|
239
|
-
|
|
242
|
+
chunks=(16, 16),
|
|
240
243
|
dtype=data.dtype,
|
|
241
244
|
fill_value=0,
|
|
242
|
-
|
|
243
|
-
codec_class(),
|
|
244
|
-
],
|
|
245
|
+
serializer=codec_class(),
|
|
245
246
|
)
|
|
246
247
|
|
|
247
248
|
a[:, :] = data.copy()
|
|
248
249
|
np.testing.assert_array_equal(data, a[:, :])
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def test_delta_astype(store: StorePath):
|
|
253
|
+
data = np.linspace(0, 10, 256, dtype="i8").reshape((16, 16))
|
|
254
|
+
|
|
255
|
+
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
256
|
+
a = zarr.create_array(
|
|
257
|
+
store / "generic",
|
|
258
|
+
shape=data.shape,
|
|
259
|
+
chunks=(16, 16),
|
|
260
|
+
dtype=data.dtype,
|
|
261
|
+
fill_value=0,
|
|
262
|
+
filters=[
|
|
263
|
+
numcodecs.zarr3.Delta(dtype="i8", astype="i2"), # type: ignore[arg-type]
|
|
264
|
+
],
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
a[:, :] = data.copy()
|
|
268
|
+
a = zarr.open_array(store / "generic", mode="r")
|
|
269
|
+
np.testing.assert_array_equal(data, a[:, :])
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def test_repr():
|
|
273
|
+
codec = numcodecs.zarr3.LZ4(level=5)
|
|
274
|
+
assert repr(codec) == "LZ4(codec_name='numcodecs.lz4', codec_config={'level': 5})"
|
numcodecs/version.py
CHANGED
|
Binary file
|
numcodecs/zarr3.py
CHANGED
|
@@ -8,12 +8,13 @@ You can use codecs from :py:mod:`numcodecs` by constructing codecs from :py:mod:
|
|
|
8
8
|
>>> import zarr
|
|
9
9
|
>>> import numcodecs.zarr3
|
|
10
10
|
>>>
|
|
11
|
-
>>>
|
|
12
|
-
|
|
13
|
-
...
|
|
14
|
-
...
|
|
11
|
+
>>> array = zarr.create_array(
|
|
12
|
+
... store="data.zarr",
|
|
13
|
+
... shape=(1024, 1024),
|
|
14
|
+
... chunks=(64, 64),
|
|
15
15
|
... dtype="uint32",
|
|
16
|
-
...
|
|
16
|
+
... filters=[numcodecs.zarr3.Delta()],
|
|
17
|
+
... compressors=[numcodecs.zarr3.BZ2(level=5)])
|
|
17
18
|
>>> array[:] = np.arange(*array.shape).astype(array.dtype)
|
|
18
19
|
|
|
19
20
|
.. note::
|
|
@@ -40,8 +41,10 @@ try:
|
|
|
40
41
|
|
|
41
42
|
if zarr.__version__ < "3.0.0": # pragma: no cover
|
|
42
43
|
raise ImportError("zarr 3.0.0 or later is required to use the numcodecs zarr integration.")
|
|
43
|
-
except ImportError: # pragma: no cover
|
|
44
|
-
raise ImportError(
|
|
44
|
+
except ImportError as e: # pragma: no cover
|
|
45
|
+
raise ImportError(
|
|
46
|
+
"zarr 3.0.0 or later is required to use the numcodecs zarr integration."
|
|
47
|
+
) from e
|
|
45
48
|
|
|
46
49
|
from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec
|
|
47
50
|
from zarr.abc.metadata import Metadata
|
|
@@ -95,6 +98,7 @@ class _NumcodecsCodec(Metadata):
|
|
|
95
98
|
"Numcodecs codecs are not in the Zarr version 3 specification and "
|
|
96
99
|
"may not be supported by other zarr implementations.",
|
|
97
100
|
category=UserWarning,
|
|
101
|
+
stacklevel=2,
|
|
98
102
|
)
|
|
99
103
|
|
|
100
104
|
@cached_property
|
|
@@ -116,6 +120,12 @@ class _NumcodecsCodec(Metadata):
|
|
|
116
120
|
def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int:
|
|
117
121
|
raise NotImplementedError # pragma: no cover
|
|
118
122
|
|
|
123
|
+
# Override __repr__ because dynamically constructed classes don't seem to work otherwise
|
|
124
|
+
def __repr__(self) -> str:
|
|
125
|
+
codec_config = self.codec_config.copy()
|
|
126
|
+
codec_config.pop("id", None)
|
|
127
|
+
return f"{self.__class__.__name__}(codec_name={self.codec_name!r}, codec_config={codec_config!r})"
|
|
128
|
+
|
|
119
129
|
|
|
120
130
|
class _NumcodecsBytesBytesCodec(_NumcodecsCodec, BytesBytesCodec):
|
|
121
131
|
def __init__(self, **codec_config: JSON) -> None:
|
|
@@ -266,7 +276,19 @@ class Shuffle(_NumcodecsBytesBytesCodec):
|
|
|
266
276
|
|
|
267
277
|
|
|
268
278
|
# array-to-array codecs ("filters")
|
|
269
|
-
|
|
279
|
+
@_add_docstring_wrapper("numcodecs.delta.Delta")
|
|
280
|
+
class Delta(_NumcodecsArrayArrayCodec):
|
|
281
|
+
codec_name = f"{CODEC_PREFIX}delta"
|
|
282
|
+
|
|
283
|
+
def __init__(self, **codec_config: dict[str, JSON]) -> None:
|
|
284
|
+
super().__init__(**codec_config)
|
|
285
|
+
|
|
286
|
+
def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
|
|
287
|
+
if astype := self.codec_config.get("astype"):
|
|
288
|
+
return replace(chunk_spec, dtype=np.dtype(astype)) # type: ignore[arg-type]
|
|
289
|
+
return chunk_spec
|
|
290
|
+
|
|
291
|
+
|
|
270
292
|
BitRound = _add_docstring(
|
|
271
293
|
_make_array_array_codec("bitround", "BitRound"), "numcodecs.bitround.BitRound"
|
|
272
294
|
)
|
|
@@ -355,25 +377,25 @@ PCodec = _add_docstring(_make_array_bytes_codec("pcodec", "PCodec"), "numcodecs.
|
|
|
355
377
|
ZFPY = _add_docstring(_make_array_bytes_codec("zfpy", "ZFPY"), "numcodecs.zfpy.ZFPY")
|
|
356
378
|
|
|
357
379
|
__all__ = [
|
|
358
|
-
"Blosc",
|
|
359
|
-
"LZ4",
|
|
360
|
-
"Zstd",
|
|
361
|
-
"Zlib",
|
|
362
|
-
"GZip",
|
|
363
380
|
"BZ2",
|
|
364
|
-
"LZMA",
|
|
365
|
-
"Shuffle",
|
|
366
|
-
"Delta",
|
|
367
|
-
"BitRound",
|
|
368
|
-
"FixedScaleOffset",
|
|
369
|
-
"Quantize",
|
|
370
|
-
"PackBits",
|
|
371
|
-
"AsType",
|
|
372
381
|
"CRC32",
|
|
373
382
|
"CRC32C",
|
|
383
|
+
"LZ4",
|
|
384
|
+
"LZMA",
|
|
385
|
+
"ZFPY",
|
|
374
386
|
"Adler32",
|
|
387
|
+
"AsType",
|
|
388
|
+
"BitRound",
|
|
389
|
+
"Blosc",
|
|
390
|
+
"Delta",
|
|
391
|
+
"FixedScaleOffset",
|
|
375
392
|
"Fletcher32",
|
|
393
|
+
"GZip",
|
|
376
394
|
"JenkinsLookup3",
|
|
377
395
|
"PCodec",
|
|
378
|
-
"
|
|
396
|
+
"PackBits",
|
|
397
|
+
"Quantize",
|
|
398
|
+
"Shuffle",
|
|
399
|
+
"Zlib",
|
|
400
|
+
"Zstd",
|
|
379
401
|
]
|
numcodecs/zfpy.py
CHANGED
|
@@ -13,7 +13,7 @@ with suppress(PackageNotFoundError):
|
|
|
13
13
|
if _zfpy_version:
|
|
14
14
|
# Check NumPy version
|
|
15
15
|
_numpy_version: tuple = tuple(map(int, version("numpy").split('.')))
|
|
16
|
-
if _numpy_version >= (2, 0, 0) and _zfpy_version
|
|
16
|
+
if _numpy_version >= (2, 0, 0) and _zfpy_version < (1, 0, 1): # pragma: no cover
|
|
17
17
|
_zfpy_version = ()
|
|
18
18
|
warnings.warn(
|
|
19
19
|
"NumPy version >= 2.0.0 detected. The zfpy library is incompatible with this version of NumPy. "
|
|
Binary file
|