numcodecs 0.14.1__cp312-cp312-macosx_11_0_arm64.whl → 0.15.1__cp312-cp312-macosx_11_0_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of numcodecs might be problematic. Click here for more details.
- numcodecs/__init__.py +43 -46
- numcodecs/_shuffle.cpython-312-darwin.so +0 -0
- numcodecs/abc.py +1 -1
- numcodecs/astype.py +2 -6
- numcodecs/base64.py +1 -2
- numcodecs/blosc.cpython-312-darwin.so +0 -0
- numcodecs/categorize.py +7 -10
- numcodecs/checksum32.py +1 -1
- numcodecs/compat_ext.cpython-312-darwin.so +0 -0
- numcodecs/delta.py +3 -10
- numcodecs/errors.py +26 -0
- numcodecs/fixedscaleoffset.py +8 -10
- numcodecs/fletcher32.cpython-312-darwin.so +0 -0
- numcodecs/gzip.py +1 -3
- numcodecs/jenkins.cpython-312-darwin.so +0 -0
- numcodecs/json.py +12 -12
- numcodecs/lz4.cpython-312-darwin.so +0 -0
- numcodecs/lzma.py +1 -1
- numcodecs/msgpacks.py +6 -6
- numcodecs/ndarray_like.py +2 -2
- numcodecs/pcodec.py +59 -29
- numcodecs/pickles.py +1 -1
- numcodecs/quantize.py +9 -11
- numcodecs/registry.py +3 -2
- numcodecs/tests/common.py +3 -4
- numcodecs/tests/test_blosc.py +9 -11
- numcodecs/tests/test_lzma.py +1 -1
- numcodecs/tests/test_pcodec.py +18 -8
- numcodecs/tests/test_registry.py +4 -3
- numcodecs/tests/test_shuffle.py +2 -4
- numcodecs/tests/test_vlen_bytes.py +3 -0
- numcodecs/tests/test_zarr3.py +73 -40
- numcodecs/version.py +2 -2
- numcodecs/vlen.cpython-312-darwin.so +0 -0
- numcodecs/zarr3.py +49 -27
- numcodecs/zfpy.py +1 -1
- numcodecs/zstd.cpython-312-darwin.so +0 -0
- {numcodecs-0.14.1.dist-info → numcodecs-0.15.1.dist-info}/METADATA +4 -4
- numcodecs-0.15.1.dist-info/RECORD +79 -0
- {numcodecs-0.14.1.dist-info → numcodecs-0.15.1.dist-info}/WHEEL +1 -1
- numcodecs-0.14.1.dist-info/RECORD +0 -78
- {numcodecs-0.14.1.dist-info → numcodecs-0.15.1.dist-info}/LICENSE.txt +0 -0
- {numcodecs-0.14.1.dist-info → numcodecs-0.15.1.dist-info}/entry_points.txt +0 -0
- {numcodecs-0.14.1.dist-info → numcodecs-0.15.1.dist-info}/top_level.txt +0 -0
numcodecs/tests/test_blosc.py
CHANGED
|
@@ -124,12 +124,12 @@ def test_compress_blocksize_default(use_threads):
|
|
|
124
124
|
|
|
125
125
|
# default blocksize
|
|
126
126
|
enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE)
|
|
127
|
-
_, _, blocksize = blosc.
|
|
127
|
+
_, _, blocksize = blosc._cbuffer_sizes(enc)
|
|
128
128
|
assert blocksize > 0
|
|
129
129
|
|
|
130
130
|
# explicit default blocksize
|
|
131
131
|
enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE, 0)
|
|
132
|
-
_, _, blocksize = blosc.
|
|
132
|
+
_, _, blocksize = blosc._cbuffer_sizes(enc)
|
|
133
133
|
assert blocksize > 0
|
|
134
134
|
|
|
135
135
|
|
|
@@ -140,7 +140,7 @@ def test_compress_blocksize(use_threads, bs):
|
|
|
140
140
|
blosc.use_threads = use_threads
|
|
141
141
|
|
|
142
142
|
enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE, bs)
|
|
143
|
-
_, _, blocksize = blosc.
|
|
143
|
+
_, _, blocksize = blosc._cbuffer_sizes(enc)
|
|
144
144
|
assert blocksize == bs
|
|
145
145
|
|
|
146
146
|
|
|
@@ -174,7 +174,7 @@ def test_compress_metainfo(dtype, use_threads):
|
|
|
174
174
|
blosc.use_threads = use_threads
|
|
175
175
|
for cname in blosc.list_compressors():
|
|
176
176
|
enc = blosc.compress(arr, cname.encode(), 1, shuffle)
|
|
177
|
-
typesize, did_shuffle, _ = blosc.
|
|
177
|
+
typesize, did_shuffle, _ = blosc._cbuffer_metainfo(enc)
|
|
178
178
|
assert typesize == arr.dtype.itemsize
|
|
179
179
|
assert did_shuffle == shuffle
|
|
180
180
|
|
|
@@ -186,7 +186,7 @@ def test_compress_autoshuffle(use_threads):
|
|
|
186
186
|
blosc.use_threads = use_threads
|
|
187
187
|
for cname in blosc.list_compressors():
|
|
188
188
|
enc = blosc.compress(varr, cname.encode(), 1, Blosc.AUTOSHUFFLE)
|
|
189
|
-
typesize, did_shuffle, _ = blosc.
|
|
189
|
+
typesize, did_shuffle, _ = blosc._cbuffer_metainfo(enc)
|
|
190
190
|
assert typesize == varr.dtype.itemsize
|
|
191
191
|
if typesize == 1:
|
|
192
192
|
assert did_shuffle == Blosc.BITSHUFFLE
|
|
@@ -199,12 +199,12 @@ def test_config_blocksize():
|
|
|
199
199
|
# explicitly stated
|
|
200
200
|
|
|
201
201
|
# blocksize not stated
|
|
202
|
-
config =
|
|
202
|
+
config = {"cname": 'lz4', "clevel": 1, "shuffle": Blosc.SHUFFLE}
|
|
203
203
|
codec = Blosc.from_config(config)
|
|
204
204
|
assert codec.blocksize == 0
|
|
205
205
|
|
|
206
206
|
# blocksize stated
|
|
207
|
-
config =
|
|
207
|
+
config = {"cname": 'lz4', "clevel": 1, "shuffle": Blosc.SHUFFLE, "blocksize": 2**8}
|
|
208
208
|
codec = Blosc.from_config(config)
|
|
209
209
|
assert codec.blocksize == 2**8
|
|
210
210
|
|
|
@@ -215,14 +215,12 @@ def test_backwards_compatibility():
|
|
|
215
215
|
|
|
216
216
|
def _encode_worker(data):
|
|
217
217
|
compressor = Blosc(cname='zlib', clevel=9, shuffle=Blosc.SHUFFLE)
|
|
218
|
-
|
|
219
|
-
return enc
|
|
218
|
+
return compressor.encode(data)
|
|
220
219
|
|
|
221
220
|
|
|
222
221
|
def _decode_worker(enc):
|
|
223
222
|
compressor = Blosc()
|
|
224
|
-
|
|
225
|
-
return data
|
|
223
|
+
return compressor.decode(enc)
|
|
226
224
|
|
|
227
225
|
|
|
228
226
|
@pytest.mark.parametrize('pool', [Pool, ThreadPool])
|
numcodecs/tests/test_lzma.py
CHANGED
numcodecs/tests/test_pcodec.py
CHANGED
|
@@ -1,11 +1,8 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import pytest
|
|
3
3
|
|
|
4
|
-
from numcodecs.pcodec import PCodec
|
|
5
|
-
|
|
6
4
|
try:
|
|
7
|
-
|
|
8
|
-
PCodec()
|
|
5
|
+
from numcodecs.pcodec import PCodec
|
|
9
6
|
except ImportError: # pragma: no cover
|
|
10
7
|
pytest.skip("pcodec not available", allow_module_level=True)
|
|
11
8
|
|
|
@@ -23,8 +20,12 @@ codecs = [
|
|
|
23
20
|
PCodec(level=1),
|
|
24
21
|
PCodec(level=5),
|
|
25
22
|
PCodec(level=9),
|
|
26
|
-
PCodec(mode_spec=
|
|
23
|
+
PCodec(mode_spec="classic"),
|
|
27
24
|
PCodec(equal_pages_up_to=300),
|
|
25
|
+
PCodec(delta_encoding_order=2),
|
|
26
|
+
PCodec(delta_spec="try_lookback"),
|
|
27
|
+
PCodec(delta_spec="none"),
|
|
28
|
+
PCodec(delta_spec="try_consecutive", delta_encoding_order=1),
|
|
28
29
|
]
|
|
29
30
|
|
|
30
31
|
|
|
@@ -56,15 +57,24 @@ def test_config():
|
|
|
56
57
|
check_config(codec)
|
|
57
58
|
|
|
58
59
|
|
|
59
|
-
|
|
60
|
-
|
|
60
|
+
@pytest.mark.parametrize("param", ["mode_spec", "delta_spec", "paging_spec"])
|
|
61
|
+
def test_invalid_config_error(param):
|
|
62
|
+
codec = PCodec(**{param: "bogus"})
|
|
63
|
+
with pytest.raises(ValueError):
|
|
64
|
+
check_encode_decode_array_to_bytes(arrays[0], codec)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_invalid_delta_encoding_combo():
|
|
68
|
+
codec = PCodec(delta_encoding_order=2, delta_spec="none")
|
|
61
69
|
with pytest.raises(ValueError):
|
|
62
70
|
check_encode_decode_array_to_bytes(arrays[0], codec)
|
|
63
71
|
|
|
64
72
|
|
|
65
73
|
def test_repr():
|
|
66
74
|
check_repr(
|
|
67
|
-
"PCodec(delta_encoding_order=None,
|
|
75
|
+
"PCodec(delta_encoding_order=None, delta_spec='auto',"
|
|
76
|
+
" equal_pages_up_to=262144, level=3, mode_spec='auto',"
|
|
77
|
+
" paging_spec='equal_pages_up_to')"
|
|
68
78
|
)
|
|
69
79
|
|
|
70
80
|
|
numcodecs/tests/test_registry.py
CHANGED
|
@@ -3,11 +3,12 @@ import inspect
|
|
|
3
3
|
import pytest
|
|
4
4
|
|
|
5
5
|
import numcodecs
|
|
6
|
+
from numcodecs.errors import UnknownCodecError
|
|
6
7
|
from numcodecs.registry import get_codec
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
def test_registry_errors():
|
|
10
|
-
with pytest.raises(
|
|
11
|
+
with pytest.raises(UnknownCodecError, match='foo'):
|
|
11
12
|
get_codec({'id': 'foo'})
|
|
12
13
|
|
|
13
14
|
|
|
@@ -26,7 +27,7 @@ def test_all_classes_registered():
|
|
|
26
27
|
|
|
27
28
|
see #346 for more info
|
|
28
29
|
"""
|
|
29
|
-
missing =
|
|
30
|
+
missing = {
|
|
30
31
|
obj.codec_id
|
|
31
32
|
for _, submod in inspect.getmembers(numcodecs, inspect.ismodule)
|
|
32
33
|
for _, obj in inspect.getmembers(submod)
|
|
@@ -36,7 +37,7 @@ def test_all_classes_registered():
|
|
|
36
37
|
and obj.codec_id not in numcodecs.registry.codec_registry
|
|
37
38
|
and obj.codec_id is not None # remove `None`
|
|
38
39
|
)
|
|
39
|
-
|
|
40
|
+
}
|
|
40
41
|
|
|
41
42
|
if missing:
|
|
42
43
|
raise Exception(f"these codecs are missing: {missing}") # pragma: no cover
|
numcodecs/tests/test_shuffle.py
CHANGED
|
@@ -79,14 +79,12 @@ def test_eq():
|
|
|
79
79
|
|
|
80
80
|
def _encode_worker(data):
|
|
81
81
|
compressor = Shuffle()
|
|
82
|
-
|
|
83
|
-
return enc
|
|
82
|
+
return compressor.encode(data)
|
|
84
83
|
|
|
85
84
|
|
|
86
85
|
def _decode_worker(enc):
|
|
87
86
|
compressor = Shuffle()
|
|
88
|
-
|
|
89
|
-
return data
|
|
87
|
+
return compressor.decode(enc)
|
|
90
88
|
|
|
91
89
|
|
|
92
90
|
@pytest.mark.parametrize('pool', [Pool, ThreadPool])
|
|
@@ -84,6 +84,9 @@ def test_decode_errors():
|
|
|
84
84
|
codec.decode(enc, out=np.zeros(10, dtype='i4'))
|
|
85
85
|
|
|
86
86
|
|
|
87
|
+
# TODO: fix this test on GitHub actions somehow...
|
|
88
|
+
# See https://github.com/zarr-developers/numcodecs/issues/683
|
|
89
|
+
@pytest.mark.skip("Test is failing on GitHub actions.")
|
|
87
90
|
def test_encode_none():
|
|
88
91
|
a = np.array([b'foo', None, b'bar'], dtype=object)
|
|
89
92
|
codec = VLenBytes()
|
numcodecs/tests/test_zarr3.py
CHANGED
|
@@ -66,17 +66,19 @@ def test_docstring(codec_class: type[numcodecs.zarr3._NumcodecsCodec]):
|
|
|
66
66
|
numcodecs.zarr3.Shuffle,
|
|
67
67
|
],
|
|
68
68
|
)
|
|
69
|
-
def
|
|
69
|
+
def test_generic_compressor(
|
|
70
|
+
store: StorePath, codec_class: type[numcodecs.zarr3._NumcodecsBytesBytesCodec]
|
|
71
|
+
):
|
|
70
72
|
data = np.arange(0, 256, dtype="uint16").reshape((16, 16))
|
|
71
73
|
|
|
72
74
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
73
|
-
a =
|
|
75
|
+
a = zarr.create_array(
|
|
74
76
|
store / "generic",
|
|
75
77
|
shape=data.shape,
|
|
76
|
-
|
|
78
|
+
chunks=(16, 16),
|
|
77
79
|
dtype=data.dtype,
|
|
78
80
|
fill_value=0,
|
|
79
|
-
|
|
81
|
+
compressors=[codec_class()],
|
|
80
82
|
)
|
|
81
83
|
|
|
82
84
|
a[:, :] = data.copy()
|
|
@@ -89,7 +91,7 @@ def test_generic_codec_class(store: StorePath, codec_class: type[numcodecs.zarr3
|
|
|
89
91
|
(numcodecs.zarr3.Delta, {"dtype": "float32"}),
|
|
90
92
|
(numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 25.5}),
|
|
91
93
|
(numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 51, "astype": "uint16"}),
|
|
92
|
-
(numcodecs.zarr3.AsType, {"encode_dtype": "float32", "decode_dtype": "
|
|
94
|
+
(numcodecs.zarr3.AsType, {"encode_dtype": "float32", "decode_dtype": "float32"}),
|
|
93
95
|
],
|
|
94
96
|
ids=[
|
|
95
97
|
"delta",
|
|
@@ -100,26 +102,25 @@ def test_generic_codec_class(store: StorePath, codec_class: type[numcodecs.zarr3
|
|
|
100
102
|
)
|
|
101
103
|
def test_generic_filter(
|
|
102
104
|
store: StorePath,
|
|
103
|
-
codec_class: type[numcodecs.zarr3.
|
|
105
|
+
codec_class: type[numcodecs.zarr3._NumcodecsArrayArrayCodec],
|
|
104
106
|
codec_config: dict[str, JSON],
|
|
105
107
|
):
|
|
106
108
|
data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
|
|
107
109
|
|
|
108
110
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
109
|
-
a =
|
|
111
|
+
a = zarr.create_array(
|
|
110
112
|
store / "generic",
|
|
111
113
|
shape=data.shape,
|
|
112
|
-
|
|
114
|
+
chunks=(16, 16),
|
|
113
115
|
dtype=data.dtype,
|
|
114
116
|
fill_value=0,
|
|
115
|
-
|
|
117
|
+
filters=[
|
|
116
118
|
codec_class(**codec_config),
|
|
117
|
-
BytesCodec(),
|
|
118
119
|
],
|
|
119
120
|
)
|
|
120
121
|
|
|
121
122
|
a[:, :] = data.copy()
|
|
122
|
-
a =
|
|
123
|
+
a = zarr.open_array(store / "generic", mode="r")
|
|
123
124
|
np.testing.assert_array_equal(data, a[:, :])
|
|
124
125
|
|
|
125
126
|
|
|
@@ -127,17 +128,17 @@ def test_generic_filter_bitround(store: StorePath):
|
|
|
127
128
|
data = np.linspace(0, 1, 256, dtype="float32").reshape((16, 16))
|
|
128
129
|
|
|
129
130
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
130
|
-
a =
|
|
131
|
+
a = zarr.create_array(
|
|
131
132
|
store / "generic_bitround",
|
|
132
133
|
shape=data.shape,
|
|
133
|
-
|
|
134
|
+
chunks=(16, 16),
|
|
134
135
|
dtype=data.dtype,
|
|
135
136
|
fill_value=0,
|
|
136
|
-
|
|
137
|
+
filters=[numcodecs.zarr3.BitRound(keepbits=3)],
|
|
137
138
|
)
|
|
138
139
|
|
|
139
140
|
a[:, :] = data.copy()
|
|
140
|
-
a =
|
|
141
|
+
a = zarr.open_array(store / "generic_bitround", mode="r")
|
|
141
142
|
assert np.allclose(data, a[:, :], atol=0.1)
|
|
142
143
|
|
|
143
144
|
|
|
@@ -145,17 +146,17 @@ def test_generic_filter_quantize(store: StorePath):
|
|
|
145
146
|
data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
|
|
146
147
|
|
|
147
148
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
148
|
-
a =
|
|
149
|
+
a = zarr.create_array(
|
|
149
150
|
store / "generic_quantize",
|
|
150
151
|
shape=data.shape,
|
|
151
|
-
|
|
152
|
+
chunks=(16, 16),
|
|
152
153
|
dtype=data.dtype,
|
|
153
154
|
fill_value=0,
|
|
154
|
-
|
|
155
|
+
filters=[numcodecs.zarr3.Quantize(digits=3)],
|
|
155
156
|
)
|
|
156
157
|
|
|
157
158
|
a[:, :] = data.copy()
|
|
158
|
-
a =
|
|
159
|
+
a = zarr.open_array(store / "generic_quantize", mode="r")
|
|
159
160
|
assert np.allclose(data, a[:, :], atol=0.001)
|
|
160
161
|
|
|
161
162
|
|
|
@@ -164,27 +165,27 @@ def test_generic_filter_packbits(store: StorePath):
|
|
|
164
165
|
data[0:4, :] = True
|
|
165
166
|
|
|
166
167
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
167
|
-
a =
|
|
168
|
+
a = zarr.create_array(
|
|
168
169
|
store / "generic_packbits",
|
|
169
170
|
shape=data.shape,
|
|
170
|
-
|
|
171
|
+
chunks=(16, 16),
|
|
171
172
|
dtype=data.dtype,
|
|
172
173
|
fill_value=0,
|
|
173
|
-
|
|
174
|
+
filters=[numcodecs.zarr3.PackBits()],
|
|
174
175
|
)
|
|
175
176
|
|
|
176
177
|
a[:, :] = data.copy()
|
|
177
|
-
a =
|
|
178
|
+
a = zarr.open_array(store / "generic_packbits", mode="r")
|
|
178
179
|
np.testing.assert_array_equal(data, a[:, :])
|
|
179
180
|
|
|
180
181
|
with pytest.raises(ValueError, match=".*requires bool dtype.*"):
|
|
181
|
-
|
|
182
|
+
zarr.create_array(
|
|
182
183
|
store / "generic_packbits_err",
|
|
183
184
|
shape=data.shape,
|
|
184
|
-
|
|
185
|
+
chunks=(16, 16),
|
|
185
186
|
dtype="uint32",
|
|
186
187
|
fill_value=0,
|
|
187
|
-
|
|
188
|
+
filters=[numcodecs.zarr3.PackBits()],
|
|
188
189
|
)
|
|
189
190
|
|
|
190
191
|
|
|
@@ -198,49 +199,81 @@ def test_generic_filter_packbits(store: StorePath):
|
|
|
198
199
|
numcodecs.zarr3.JenkinsLookup3,
|
|
199
200
|
],
|
|
200
201
|
)
|
|
201
|
-
def test_generic_checksum(
|
|
202
|
+
def test_generic_checksum(
|
|
203
|
+
store: StorePath, codec_class: type[numcodecs.zarr3._NumcodecsBytesBytesCodec]
|
|
204
|
+
):
|
|
202
205
|
data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
|
|
203
206
|
|
|
204
207
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
205
|
-
a =
|
|
208
|
+
a = zarr.create_array(
|
|
206
209
|
store / "generic_checksum",
|
|
207
210
|
shape=data.shape,
|
|
208
|
-
|
|
211
|
+
chunks=(16, 16),
|
|
209
212
|
dtype=data.dtype,
|
|
210
213
|
fill_value=0,
|
|
211
|
-
|
|
214
|
+
compressors=[codec_class()],
|
|
212
215
|
)
|
|
213
216
|
|
|
214
217
|
a[:, :] = data.copy()
|
|
215
|
-
a =
|
|
218
|
+
a = zarr.open_array(store / "generic_checksum", mode="r")
|
|
216
219
|
np.testing.assert_array_equal(data, a[:, :])
|
|
217
220
|
|
|
218
221
|
|
|
219
222
|
@pytest.mark.parametrize("codec_class", [numcodecs.zarr3.PCodec, numcodecs.zarr3.ZFPY])
|
|
220
|
-
def test_generic_bytes_codec(
|
|
223
|
+
def test_generic_bytes_codec(
|
|
224
|
+
store: StorePath, codec_class: type[numcodecs.zarr3._NumcodecsArrayBytesCodec]
|
|
225
|
+
):
|
|
221
226
|
try:
|
|
222
227
|
codec_class()._codec # noqa: B018
|
|
223
|
-
except ValueError as e:
|
|
228
|
+
except ValueError as e: # pragma: no cover
|
|
224
229
|
if "codec not available" in str(e):
|
|
225
230
|
pytest.xfail(f"{codec_class.codec_name} is not available: {e}")
|
|
226
231
|
else:
|
|
227
|
-
raise
|
|
228
|
-
except ImportError as e:
|
|
232
|
+
raise
|
|
233
|
+
except ImportError as e: # pragma: no cover
|
|
229
234
|
pytest.xfail(f"{codec_class.codec_name} is not available: {e}")
|
|
230
235
|
|
|
231
236
|
data = np.arange(0, 256, dtype="float32").reshape((16, 16))
|
|
232
237
|
|
|
233
238
|
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
234
|
-
a =
|
|
239
|
+
a = zarr.create_array(
|
|
235
240
|
store / "generic",
|
|
236
241
|
shape=data.shape,
|
|
237
|
-
|
|
242
|
+
chunks=(16, 16),
|
|
238
243
|
dtype=data.dtype,
|
|
239
244
|
fill_value=0,
|
|
240
|
-
|
|
241
|
-
codec_class(),
|
|
242
|
-
],
|
|
245
|
+
serializer=codec_class(),
|
|
243
246
|
)
|
|
244
247
|
|
|
245
248
|
a[:, :] = data.copy()
|
|
246
249
|
np.testing.assert_array_equal(data, a[:, :])
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def test_delta_astype(store: StorePath):
|
|
253
|
+
data = np.linspace(0, 10, 256, dtype="i8").reshape((16, 16))
|
|
254
|
+
|
|
255
|
+
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
|
|
256
|
+
a = zarr.create_array(
|
|
257
|
+
store / "generic",
|
|
258
|
+
shape=data.shape,
|
|
259
|
+
chunks=(16, 16),
|
|
260
|
+
dtype=data.dtype,
|
|
261
|
+
fill_value=0,
|
|
262
|
+
filters=[
|
|
263
|
+
numcodecs.zarr3.Delta(dtype="i8", astype="i2"), # type: ignore[arg-type]
|
|
264
|
+
],
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
a[:, :] = data.copy()
|
|
268
|
+
a = zarr.open_array(store / "generic", mode="r")
|
|
269
|
+
np.testing.assert_array_equal(data, a[:, :])
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def test_repr():
|
|
273
|
+
codec = numcodecs.zarr3.LZ4(level=5)
|
|
274
|
+
assert repr(codec) == "LZ4(codec_name='numcodecs.lz4', codec_config={'level': 5})"
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def test_to_dict():
|
|
278
|
+
codec = numcodecs.zarr3.LZ4(level=5)
|
|
279
|
+
assert codec.to_dict() == {"name": "numcodecs.lz4", "configuration": {"level": 5}}
|
numcodecs/version.py
CHANGED
|
Binary file
|
numcodecs/zarr3.py
CHANGED
|
@@ -8,12 +8,13 @@ You can use codecs from :py:mod:`numcodecs` by constructing codecs from :py:mod:
|
|
|
8
8
|
>>> import zarr
|
|
9
9
|
>>> import numcodecs.zarr3
|
|
10
10
|
>>>
|
|
11
|
-
>>>
|
|
12
|
-
|
|
13
|
-
...
|
|
14
|
-
...
|
|
11
|
+
>>> array = zarr.create_array(
|
|
12
|
+
... store="data.zarr",
|
|
13
|
+
... shape=(1024, 1024),
|
|
14
|
+
... chunks=(64, 64),
|
|
15
15
|
... dtype="uint32",
|
|
16
|
-
...
|
|
16
|
+
... filters=[numcodecs.zarr3.Delta()],
|
|
17
|
+
... compressors=[numcodecs.zarr3.BZ2(level=5)])
|
|
17
18
|
>>> array[:] = np.arange(*array.shape).astype(array.dtype)
|
|
18
19
|
|
|
19
20
|
.. note::
|
|
@@ -40,8 +41,10 @@ try:
|
|
|
40
41
|
|
|
41
42
|
if zarr.__version__ < "3.0.0": # pragma: no cover
|
|
42
43
|
raise ImportError("zarr 3.0.0 or later is required to use the numcodecs zarr integration.")
|
|
43
|
-
except ImportError: # pragma: no cover
|
|
44
|
-
raise ImportError(
|
|
44
|
+
except ImportError as e: # pragma: no cover
|
|
45
|
+
raise ImportError(
|
|
46
|
+
"zarr 3.0.0 or later is required to use the numcodecs zarr integration."
|
|
47
|
+
) from e
|
|
45
48
|
|
|
46
49
|
from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec
|
|
47
50
|
from zarr.abc.metadata import Metadata
|
|
@@ -95,6 +98,7 @@ class _NumcodecsCodec(Metadata):
|
|
|
95
98
|
"Numcodecs codecs are not in the Zarr version 3 specification and "
|
|
96
99
|
"may not be supported by other zarr implementations.",
|
|
97
100
|
category=UserWarning,
|
|
101
|
+
stacklevel=2,
|
|
98
102
|
)
|
|
99
103
|
|
|
100
104
|
@cached_property
|
|
@@ -108,6 +112,7 @@ class _NumcodecsCodec(Metadata):
|
|
|
108
112
|
|
|
109
113
|
def to_dict(self) -> dict[str, JSON]:
|
|
110
114
|
codec_config = self.codec_config.copy()
|
|
115
|
+
codec_config.pop("id", None)
|
|
111
116
|
return {
|
|
112
117
|
"name": self.codec_name,
|
|
113
118
|
"configuration": codec_config,
|
|
@@ -116,6 +121,12 @@ class _NumcodecsCodec(Metadata):
|
|
|
116
121
|
def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int:
|
|
117
122
|
raise NotImplementedError # pragma: no cover
|
|
118
123
|
|
|
124
|
+
# Override __repr__ because dynamically constructed classes don't seem to work otherwise
|
|
125
|
+
def __repr__(self) -> str:
|
|
126
|
+
codec_config = self.codec_config.copy()
|
|
127
|
+
codec_config.pop("id", None)
|
|
128
|
+
return f"{self.__class__.__name__}(codec_name={self.codec_name!r}, codec_config={codec_config!r})"
|
|
129
|
+
|
|
119
130
|
|
|
120
131
|
class _NumcodecsBytesBytesCodec(_NumcodecsCodec, BytesBytesCodec):
|
|
121
132
|
def __init__(self, **codec_config: JSON) -> None:
|
|
@@ -260,13 +271,25 @@ class Shuffle(_NumcodecsBytesBytesCodec):
|
|
|
260
271
|
super().__init__(**codec_config)
|
|
261
272
|
|
|
262
273
|
def evolve_from_array_spec(self, array_spec: ArraySpec) -> Shuffle:
|
|
263
|
-
if
|
|
274
|
+
if self.codec_config.get("elementsize", None) is None:
|
|
264
275
|
return Shuffle(**{**self.codec_config, "elementsize": array_spec.dtype.itemsize})
|
|
265
276
|
return self # pragma: no cover
|
|
266
277
|
|
|
267
278
|
|
|
268
279
|
# array-to-array codecs ("filters")
|
|
269
|
-
|
|
280
|
+
@_add_docstring_wrapper("numcodecs.delta.Delta")
|
|
281
|
+
class Delta(_NumcodecsArrayArrayCodec):
|
|
282
|
+
codec_name = f"{CODEC_PREFIX}delta"
|
|
283
|
+
|
|
284
|
+
def __init__(self, **codec_config: dict[str, JSON]) -> None:
|
|
285
|
+
super().__init__(**codec_config)
|
|
286
|
+
|
|
287
|
+
def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
|
|
288
|
+
if astype := self.codec_config.get("astype"):
|
|
289
|
+
return replace(chunk_spec, dtype=np.dtype(astype)) # type: ignore[arg-type]
|
|
290
|
+
return chunk_spec
|
|
291
|
+
|
|
292
|
+
|
|
270
293
|
BitRound = _add_docstring(
|
|
271
294
|
_make_array_array_codec("bitround", "BitRound"), "numcodecs.bitround.BitRound"
|
|
272
295
|
)
|
|
@@ -285,7 +308,7 @@ class FixedScaleOffset(_NumcodecsArrayArrayCodec):
|
|
|
285
308
|
return chunk_spec
|
|
286
309
|
|
|
287
310
|
def evolve_from_array_spec(self, array_spec: ArraySpec) -> FixedScaleOffset:
|
|
288
|
-
if
|
|
311
|
+
if self.codec_config.get("dtype", None) is None:
|
|
289
312
|
return FixedScaleOffset(**{**self.codec_config, "dtype": str(array_spec.dtype)})
|
|
290
313
|
return self
|
|
291
314
|
|
|
@@ -298,7 +321,7 @@ class Quantize(_NumcodecsArrayArrayCodec):
|
|
|
298
321
|
super().__init__(**codec_config)
|
|
299
322
|
|
|
300
323
|
def evolve_from_array_spec(self, array_spec: ArraySpec) -> Quantize:
|
|
301
|
-
if
|
|
324
|
+
if self.codec_config.get("dtype", None) is None:
|
|
302
325
|
return Quantize(**{**self.codec_config, "dtype": str(array_spec.dtype)})
|
|
303
326
|
return self
|
|
304
327
|
|
|
@@ -333,8 +356,7 @@ class AsType(_NumcodecsArrayArrayCodec):
|
|
|
333
356
|
return replace(chunk_spec, dtype=np.dtype(self.codec_config["encode_dtype"])) # type: ignore[arg-type]
|
|
334
357
|
|
|
335
358
|
def evolve_from_array_spec(self, array_spec: ArraySpec) -> AsType:
|
|
336
|
-
|
|
337
|
-
if str(array_spec.dtype) != decode_dtype:
|
|
359
|
+
if self.codec_config.get("decode_dtype", None) is None:
|
|
338
360
|
return AsType(**{**self.codec_config, "decode_dtype": str(array_spec.dtype)})
|
|
339
361
|
return self
|
|
340
362
|
|
|
@@ -355,25 +377,25 @@ PCodec = _add_docstring(_make_array_bytes_codec("pcodec", "PCodec"), "numcodecs.
|
|
|
355
377
|
ZFPY = _add_docstring(_make_array_bytes_codec("zfpy", "ZFPY"), "numcodecs.zfpy.ZFPY")
|
|
356
378
|
|
|
357
379
|
__all__ = [
|
|
358
|
-
"Blosc",
|
|
359
|
-
"LZ4",
|
|
360
|
-
"Zstd",
|
|
361
|
-
"Zlib",
|
|
362
|
-
"GZip",
|
|
363
380
|
"BZ2",
|
|
364
|
-
"LZMA",
|
|
365
|
-
"Shuffle",
|
|
366
|
-
"Delta",
|
|
367
|
-
"BitRound",
|
|
368
|
-
"FixedScaleOffset",
|
|
369
|
-
"Quantize",
|
|
370
|
-
"PackBits",
|
|
371
|
-
"AsType",
|
|
372
381
|
"CRC32",
|
|
373
382
|
"CRC32C",
|
|
383
|
+
"LZ4",
|
|
384
|
+
"LZMA",
|
|
385
|
+
"ZFPY",
|
|
374
386
|
"Adler32",
|
|
387
|
+
"AsType",
|
|
388
|
+
"BitRound",
|
|
389
|
+
"Blosc",
|
|
390
|
+
"Delta",
|
|
391
|
+
"FixedScaleOffset",
|
|
375
392
|
"Fletcher32",
|
|
393
|
+
"GZip",
|
|
376
394
|
"JenkinsLookup3",
|
|
377
395
|
"PCodec",
|
|
378
|
-
"
|
|
396
|
+
"PackBits",
|
|
397
|
+
"Quantize",
|
|
398
|
+
"Shuffle",
|
|
399
|
+
"Zlib",
|
|
400
|
+
"Zstd",
|
|
379
401
|
]
|
numcodecs/zfpy.py
CHANGED
|
@@ -13,7 +13,7 @@ with suppress(PackageNotFoundError):
|
|
|
13
13
|
if _zfpy_version:
|
|
14
14
|
# Check NumPy version
|
|
15
15
|
_numpy_version: tuple = tuple(map(int, version("numpy").split('.')))
|
|
16
|
-
if _numpy_version >= (2, 0, 0) and _zfpy_version
|
|
16
|
+
if _numpy_version >= (2, 0, 0) and _zfpy_version < (1, 0, 1): # pragma: no cover
|
|
17
17
|
_zfpy_version = ()
|
|
18
18
|
warnings.warn(
|
|
19
19
|
"NumPy version >= 2.0.0 detected. The zfpy library is incompatible with this version of NumPy. "
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: numcodecs
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.15.1
|
|
4
4
|
Summary: A Python package providing buffer compression and transformation codecs for use in data storage and communication applications.
|
|
5
5
|
Maintainer-email: Alistair Miles <alimanfoo@googlemail.com>
|
|
6
6
|
License: MIT
|
|
@@ -22,6 +22,7 @@ Requires-Python: >=3.11
|
|
|
22
22
|
Description-Content-Type: text/x-rst
|
|
23
23
|
License-File: LICENSE.txt
|
|
24
24
|
Requires-Dist: numpy>=1.24
|
|
25
|
+
Requires-Dist: deprecated
|
|
25
26
|
Provides-Extra: docs
|
|
26
27
|
Requires-Dist: sphinx; extra == "docs"
|
|
27
28
|
Requires-Dist: sphinx-issues; extra == "docs"
|
|
@@ -37,9 +38,8 @@ Provides-Extra: msgpack
|
|
|
37
38
|
Requires-Dist: msgpack; extra == "msgpack"
|
|
38
39
|
Provides-Extra: zfpy
|
|
39
40
|
Requires-Dist: zfpy>=1.0.0; extra == "zfpy"
|
|
40
|
-
Requires-Dist: numpy<2.0.0; extra == "zfpy"
|
|
41
41
|
Provides-Extra: pcodec
|
|
42
|
-
Requires-Dist: pcodec<0.
|
|
42
|
+
Requires-Dist: pcodec<0.4,>=0.3; extra == "pcodec"
|
|
43
43
|
Provides-Extra: crc32c
|
|
44
44
|
Requires-Dist: crc32c>=2.7; extra == "crc32c"
|
|
45
45
|
|