numcodecs 0.13.1__cp313-cp313-macosx_10_13_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of numcodecs might be problematic. Click here for more details.
- numcodecs/__init__.py +143 -0
- numcodecs/_shuffle.cpython-313-darwin.so +0 -0
- numcodecs/abc.py +126 -0
- numcodecs/astype.py +76 -0
- numcodecs/base64.py +27 -0
- numcodecs/bitround.py +79 -0
- numcodecs/blosc.cpython-313-darwin.so +0 -0
- numcodecs/bz2.py +45 -0
- numcodecs/categorize.py +101 -0
- numcodecs/checksum32.py +94 -0
- numcodecs/compat.py +208 -0
- numcodecs/compat_ext.cpython-313-darwin.so +0 -0
- numcodecs/delta.py +97 -0
- numcodecs/fixedscaleoffset.py +132 -0
- numcodecs/fletcher32.cpython-313-darwin.so +0 -0
- numcodecs/gzip.py +52 -0
- numcodecs/jenkins.cpython-313-darwin.so +0 -0
- numcodecs/json.py +107 -0
- numcodecs/lz4.cpython-313-darwin.so +0 -0
- numcodecs/lzma.py +69 -0
- numcodecs/msgpacks.py +86 -0
- numcodecs/ndarray_like.py +65 -0
- numcodecs/packbits.py +85 -0
- numcodecs/pcodec.py +89 -0
- numcodecs/pickles.py +55 -0
- numcodecs/quantize.py +100 -0
- numcodecs/registry.py +72 -0
- numcodecs/shuffle.py +61 -0
- numcodecs/tests/__init__.py +3 -0
- numcodecs/tests/common.py +354 -0
- numcodecs/tests/package_with_entrypoint/__init__.py +11 -0
- numcodecs/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt +2 -0
- numcodecs/tests/test_astype.py +74 -0
- numcodecs/tests/test_base64.py +81 -0
- numcodecs/tests/test_bitround.py +81 -0
- numcodecs/tests/test_blosc.py +277 -0
- numcodecs/tests/test_bz2.py +66 -0
- numcodecs/tests/test_categorize.py +87 -0
- numcodecs/tests/test_checksum32.py +58 -0
- numcodecs/tests/test_compat.py +108 -0
- numcodecs/tests/test_delta.py +60 -0
- numcodecs/tests/test_entrypoints.py +24 -0
- numcodecs/tests/test_entrypoints_backport.py +35 -0
- numcodecs/tests/test_fixedscaleoffset.py +69 -0
- numcodecs/tests/test_fletcher32.py +56 -0
- numcodecs/tests/test_gzip.py +110 -0
- numcodecs/tests/test_jenkins.py +150 -0
- numcodecs/tests/test_json.py +85 -0
- numcodecs/tests/test_lz4.py +83 -0
- numcodecs/tests/test_lzma.py +90 -0
- numcodecs/tests/test_msgpacks.py +123 -0
- numcodecs/tests/test_ndarray_like.py +48 -0
- numcodecs/tests/test_packbits.py +39 -0
- numcodecs/tests/test_pcodec.py +80 -0
- numcodecs/tests/test_pickles.py +61 -0
- numcodecs/tests/test_quantize.py +76 -0
- numcodecs/tests/test_registry.py +40 -0
- numcodecs/tests/test_shuffle.py +168 -0
- numcodecs/tests/test_vlen_array.py +97 -0
- numcodecs/tests/test_vlen_bytes.py +93 -0
- numcodecs/tests/test_vlen_utf8.py +91 -0
- numcodecs/tests/test_zfpy.py +98 -0
- numcodecs/tests/test_zlib.py +94 -0
- numcodecs/tests/test_zstd.py +92 -0
- numcodecs/version.py +16 -0
- numcodecs/vlen.cpython-313-darwin.so +0 -0
- numcodecs/zfpy.py +111 -0
- numcodecs/zlib.py +42 -0
- numcodecs/zstd.cpython-313-darwin.so +0 -0
- numcodecs-0.13.1.dist-info/LICENSE.txt +21 -0
- numcodecs-0.13.1.dist-info/METADATA +64 -0
- numcodecs-0.13.1.dist-info/RECORD +74 -0
- numcodecs-0.13.1.dist-info/WHEEL +5 -0
- numcodecs-0.13.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
from multiprocessing import Pool
|
|
2
|
+
from multiprocessing.pool import ThreadPool
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
from numcodecs import blosc
|
|
9
|
+
from numcodecs.blosc import Blosc
|
|
10
|
+
except ImportError: # pragma: no cover
|
|
11
|
+
pytest.skip("numcodecs.blosc not available", allow_module_level=True)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
from numcodecs.tests.common import (
|
|
15
|
+
check_backwards_compatibility,
|
|
16
|
+
check_config,
|
|
17
|
+
check_encode_decode,
|
|
18
|
+
check_encode_decode_partial,
|
|
19
|
+
check_err_decode_object_buffer,
|
|
20
|
+
check_err_encode_object_buffer,
|
|
21
|
+
check_max_buffer_size,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
codecs = [
|
|
25
|
+
Blosc(shuffle=Blosc.SHUFFLE),
|
|
26
|
+
Blosc(clevel=0, shuffle=Blosc.SHUFFLE),
|
|
27
|
+
Blosc(cname='lz4', shuffle=Blosc.SHUFFLE),
|
|
28
|
+
Blosc(cname='lz4', clevel=1, shuffle=Blosc.NOSHUFFLE),
|
|
29
|
+
Blosc(cname='lz4', clevel=5, shuffle=Blosc.SHUFFLE),
|
|
30
|
+
Blosc(cname='lz4', clevel=9, shuffle=Blosc.BITSHUFFLE),
|
|
31
|
+
Blosc(cname='zlib', clevel=1, shuffle=0),
|
|
32
|
+
Blosc(cname='zstd', clevel=1, shuffle=1),
|
|
33
|
+
Blosc(cname='blosclz', clevel=1, shuffle=2),
|
|
34
|
+
None, # was snappy
|
|
35
|
+
Blosc(shuffle=Blosc.SHUFFLE, blocksize=0),
|
|
36
|
+
Blosc(shuffle=Blosc.SHUFFLE, blocksize=2**8),
|
|
37
|
+
Blosc(cname='lz4', clevel=1, shuffle=Blosc.NOSHUFFLE, blocksize=2**8),
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# mix of dtypes: integer, float, bool, string
|
|
42
|
+
# mix of shapes: 1D, 2D, 3D
|
|
43
|
+
# mix of orders: C, F
|
|
44
|
+
arrays = [
|
|
45
|
+
np.arange(1000, dtype='i4'),
|
|
46
|
+
np.linspace(1000, 1001, 1000, dtype='f8'),
|
|
47
|
+
np.random.normal(loc=1000, scale=1, size=(100, 10)),
|
|
48
|
+
np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
|
|
49
|
+
np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
|
|
50
|
+
np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
|
|
51
|
+
np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
|
|
52
|
+
np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
|
|
53
|
+
np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
|
|
54
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
|
|
55
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
|
|
56
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
|
|
57
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _skip_null(codec):
|
|
62
|
+
if codec is None:
|
|
63
|
+
pytest.skip("codec has been removed")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@pytest.fixture(scope='module', params=[True, False, None])
|
|
67
|
+
def use_threads(request):
|
|
68
|
+
return request.param
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@pytest.mark.parametrize('array', arrays)
|
|
72
|
+
@pytest.mark.parametrize('codec', codecs)
|
|
73
|
+
def test_encode_decode(array, codec):
|
|
74
|
+
_skip_null(codec)
|
|
75
|
+
check_encode_decode(array, codec)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@pytest.mark.parametrize('codec', codecs)
|
|
79
|
+
@pytest.mark.parametrize(
|
|
80
|
+
'array',
|
|
81
|
+
[
|
|
82
|
+
pytest.param(x) if len(x.shape) == 1 else pytest.param(x, marks=[pytest.mark.xfail])
|
|
83
|
+
for x in arrays
|
|
84
|
+
],
|
|
85
|
+
)
|
|
86
|
+
def test_partial_decode(codec, array):
|
|
87
|
+
_skip_null(codec)
|
|
88
|
+
check_encode_decode_partial(array, codec)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_config():
|
|
92
|
+
codec = Blosc(cname='zstd', clevel=3, shuffle=1)
|
|
93
|
+
check_config(codec)
|
|
94
|
+
codec = Blosc(cname='lz4', clevel=1, shuffle=2, blocksize=2**8)
|
|
95
|
+
check_config(codec)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def test_repr():
|
|
99
|
+
expect = "Blosc(cname='zstd', clevel=3, shuffle=SHUFFLE, blocksize=0)"
|
|
100
|
+
actual = repr(Blosc(cname='zstd', clevel=3, shuffle=Blosc.SHUFFLE, blocksize=0))
|
|
101
|
+
assert expect == actual
|
|
102
|
+
expect = "Blosc(cname='lz4', clevel=1, shuffle=NOSHUFFLE, blocksize=256)"
|
|
103
|
+
actual = repr(Blosc(cname='lz4', clevel=1, shuffle=Blosc.NOSHUFFLE, blocksize=256))
|
|
104
|
+
assert expect == actual
|
|
105
|
+
expect = "Blosc(cname='zlib', clevel=9, shuffle=BITSHUFFLE, blocksize=512)"
|
|
106
|
+
actual = repr(Blosc(cname='zlib', clevel=9, shuffle=Blosc.BITSHUFFLE, blocksize=512))
|
|
107
|
+
assert expect == actual
|
|
108
|
+
expect = "Blosc(cname='blosclz', clevel=5, shuffle=AUTOSHUFFLE, blocksize=1024)"
|
|
109
|
+
actual = repr(Blosc(cname='blosclz', clevel=5, shuffle=Blosc.AUTOSHUFFLE, blocksize=1024))
|
|
110
|
+
assert expect == actual
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def test_eq():
|
|
114
|
+
assert Blosc() == Blosc()
|
|
115
|
+
assert Blosc(cname='lz4') != Blosc(cname='zstd')
|
|
116
|
+
assert Blosc(clevel=1) != Blosc(clevel=9)
|
|
117
|
+
assert Blosc(cname='lz4') != 'foo'
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_compress_blocksize_default(use_threads):
|
|
121
|
+
arr = np.arange(1000, dtype='i4')
|
|
122
|
+
|
|
123
|
+
blosc.use_threads = use_threads
|
|
124
|
+
|
|
125
|
+
# default blocksize
|
|
126
|
+
enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE)
|
|
127
|
+
_, _, blocksize = blosc.cbuffer_sizes(enc)
|
|
128
|
+
assert blocksize > 0
|
|
129
|
+
|
|
130
|
+
# explicit default blocksize
|
|
131
|
+
enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE, 0)
|
|
132
|
+
_, _, blocksize = blosc.cbuffer_sizes(enc)
|
|
133
|
+
assert blocksize > 0
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@pytest.mark.parametrize('bs', (2**7, 2**8))
|
|
137
|
+
def test_compress_blocksize(use_threads, bs):
|
|
138
|
+
arr = np.arange(1000, dtype='i4')
|
|
139
|
+
|
|
140
|
+
blosc.use_threads = use_threads
|
|
141
|
+
|
|
142
|
+
enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE, bs)
|
|
143
|
+
_, _, blocksize = blosc.cbuffer_sizes(enc)
|
|
144
|
+
assert blocksize == bs
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_compress_complib(use_threads):
|
|
148
|
+
arr = np.arange(1000, dtype='i4')
|
|
149
|
+
expected_complibs = {
|
|
150
|
+
'lz4': 'LZ4',
|
|
151
|
+
'lz4hc': 'LZ4',
|
|
152
|
+
'blosclz': 'BloscLZ',
|
|
153
|
+
'zlib': 'Zlib',
|
|
154
|
+
'zstd': 'Zstd',
|
|
155
|
+
}
|
|
156
|
+
blosc.use_threads = use_threads
|
|
157
|
+
for cname in blosc.list_compressors():
|
|
158
|
+
enc = blosc.compress(arr, cname.encode(), 1, Blosc.NOSHUFFLE)
|
|
159
|
+
complib = blosc.cbuffer_complib(enc)
|
|
160
|
+
expected_complib = expected_complibs[cname]
|
|
161
|
+
assert complib == expected_complib
|
|
162
|
+
with pytest.raises(ValueError):
|
|
163
|
+
# capitalized cname
|
|
164
|
+
blosc.compress(arr, b'LZ4', 1)
|
|
165
|
+
with pytest.raises(ValueError):
|
|
166
|
+
# bad cname
|
|
167
|
+
blosc.compress(arr, b'foo', 1)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@pytest.mark.parametrize('dtype', ['i1', 'i2', 'i4', 'i8'])
|
|
171
|
+
def test_compress_metainfo(dtype, use_threads):
|
|
172
|
+
arr = np.arange(1000, dtype=dtype)
|
|
173
|
+
for shuffle in Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE:
|
|
174
|
+
blosc.use_threads = use_threads
|
|
175
|
+
for cname in blosc.list_compressors():
|
|
176
|
+
enc = blosc.compress(arr, cname.encode(), 1, shuffle)
|
|
177
|
+
typesize, did_shuffle, _ = blosc.cbuffer_metainfo(enc)
|
|
178
|
+
assert typesize == arr.dtype.itemsize
|
|
179
|
+
assert did_shuffle == shuffle
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def test_compress_autoshuffle(use_threads):
|
|
183
|
+
arr = np.arange(8000)
|
|
184
|
+
for dtype in 'i1', 'i2', 'i4', 'i8', 'f2', 'f4', 'f8', 'bool', 'S10':
|
|
185
|
+
varr = arr.view(dtype)
|
|
186
|
+
blosc.use_threads = use_threads
|
|
187
|
+
for cname in blosc.list_compressors():
|
|
188
|
+
enc = blosc.compress(varr, cname.encode(), 1, Blosc.AUTOSHUFFLE)
|
|
189
|
+
typesize, did_shuffle, _ = blosc.cbuffer_metainfo(enc)
|
|
190
|
+
assert typesize == varr.dtype.itemsize
|
|
191
|
+
if typesize == 1:
|
|
192
|
+
assert did_shuffle == Blosc.BITSHUFFLE
|
|
193
|
+
else:
|
|
194
|
+
assert did_shuffle == Blosc.SHUFFLE
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def test_config_blocksize():
|
|
198
|
+
# N.B., we want to be backwards compatible with any config where blocksize is not
|
|
199
|
+
# explicitly stated
|
|
200
|
+
|
|
201
|
+
# blocksize not stated
|
|
202
|
+
config = dict(cname='lz4', clevel=1, shuffle=Blosc.SHUFFLE)
|
|
203
|
+
codec = Blosc.from_config(config)
|
|
204
|
+
assert codec.blocksize == 0
|
|
205
|
+
|
|
206
|
+
# blocksize stated
|
|
207
|
+
config = dict(cname='lz4', clevel=1, shuffle=Blosc.SHUFFLE, blocksize=2**8)
|
|
208
|
+
codec = Blosc.from_config(config)
|
|
209
|
+
assert codec.blocksize == 2**8
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def test_backwards_compatibility():
|
|
213
|
+
check_backwards_compatibility(Blosc.codec_id, arrays, codecs)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _encode_worker(data):
|
|
217
|
+
compressor = Blosc(cname='zlib', clevel=9, shuffle=Blosc.SHUFFLE)
|
|
218
|
+
enc = compressor.encode(data)
|
|
219
|
+
return enc
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _decode_worker(enc):
|
|
223
|
+
compressor = Blosc()
|
|
224
|
+
data = compressor.decode(enc)
|
|
225
|
+
return data
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@pytest.mark.parametrize('pool', (Pool, ThreadPool))
|
|
229
|
+
def test_multiprocessing(use_threads, pool):
|
|
230
|
+
data = np.arange(1000000)
|
|
231
|
+
enc = _encode_worker(data)
|
|
232
|
+
|
|
233
|
+
pool = pool(5)
|
|
234
|
+
|
|
235
|
+
try:
|
|
236
|
+
blosc.use_threads = use_threads
|
|
237
|
+
|
|
238
|
+
# test with process pool and thread pool
|
|
239
|
+
|
|
240
|
+
# test encoding
|
|
241
|
+
enc_results = pool.map(_encode_worker, [data] * 5)
|
|
242
|
+
assert all(len(enc) == len(e) for e in enc_results)
|
|
243
|
+
|
|
244
|
+
# test decoding
|
|
245
|
+
dec_results = pool.map(_decode_worker, [enc] * 5)
|
|
246
|
+
assert all(data.nbytes == len(d) for d in dec_results)
|
|
247
|
+
|
|
248
|
+
# tidy up
|
|
249
|
+
pool.close()
|
|
250
|
+
pool.join()
|
|
251
|
+
|
|
252
|
+
finally:
|
|
253
|
+
blosc.use_threads = None # restore default
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def test_err_decode_object_buffer():
|
|
257
|
+
check_err_decode_object_buffer(Blosc())
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def test_err_encode_object_buffer():
|
|
261
|
+
check_err_encode_object_buffer(Blosc())
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def test_decompression_error_handling():
|
|
265
|
+
for codec in codecs:
|
|
266
|
+
_skip_null(codec)
|
|
267
|
+
with pytest.raises(RuntimeError):
|
|
268
|
+
codec.decode(bytearray())
|
|
269
|
+
with pytest.raises(RuntimeError):
|
|
270
|
+
codec.decode(bytearray(0))
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def test_max_buffer_size():
|
|
274
|
+
for codec in codecs:
|
|
275
|
+
_skip_null(codec)
|
|
276
|
+
assert codec.max_buffer_size == 2**31 - 1
|
|
277
|
+
check_max_buffer_size(codec)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
from numcodecs.bz2 import BZ2
|
|
6
|
+
from numcodecs.tests.common import (
|
|
7
|
+
check_backwards_compatibility,
|
|
8
|
+
check_config,
|
|
9
|
+
check_encode_decode,
|
|
10
|
+
check_err_decode_object_buffer,
|
|
11
|
+
check_err_encode_object_buffer,
|
|
12
|
+
check_repr,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
codecs = [
|
|
16
|
+
BZ2(),
|
|
17
|
+
BZ2(level=1),
|
|
18
|
+
BZ2(level=5),
|
|
19
|
+
BZ2(level=9),
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# mix of dtypes: integer, float, bool, string
|
|
24
|
+
# mix of shapes: 1D, 2D, 3D
|
|
25
|
+
# mix of orders: C, F
|
|
26
|
+
arrays = [
|
|
27
|
+
np.arange(1000, dtype='i4'),
|
|
28
|
+
np.linspace(1000, 1001, 1000, dtype='f8'),
|
|
29
|
+
np.random.normal(loc=1000, scale=1, size=(100, 10)),
|
|
30
|
+
np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
|
|
31
|
+
np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
|
|
32
|
+
np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
|
|
33
|
+
np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
|
|
34
|
+
np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
|
|
35
|
+
np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
|
|
36
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
|
|
37
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
|
|
38
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
|
|
39
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_encode_decode():
|
|
44
|
+
for arr, codec in itertools.product(arrays, codecs):
|
|
45
|
+
check_encode_decode(arr, codec)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_config():
|
|
49
|
+
codec = BZ2(level=3)
|
|
50
|
+
check_config(codec)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_repr():
|
|
54
|
+
check_repr("BZ2(level=3)")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_backwards_compatibility():
|
|
58
|
+
check_backwards_compatibility(BZ2.codec_id, arrays, codecs)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_err_decode_object_buffer():
|
|
62
|
+
check_err_decode_object_buffer(BZ2())
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_err_encode_object_buffer():
|
|
66
|
+
check_err_encode_object_buffer(BZ2())
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pytest
|
|
3
|
+
from numpy.testing import assert_array_equal
|
|
4
|
+
|
|
5
|
+
from numcodecs.categorize import Categorize
|
|
6
|
+
from numcodecs.tests.common import (
|
|
7
|
+
check_backwards_compatibility,
|
|
8
|
+
check_config,
|
|
9
|
+
check_encode_decode,
|
|
10
|
+
check_encode_decode_array,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
labels = ['ƒöõ', 'ßàř', 'ßāẑ', 'ƪùüx']
|
|
14
|
+
arrays = [
|
|
15
|
+
np.random.choice(labels, size=1000),
|
|
16
|
+
np.random.choice(labels, size=(100, 10)),
|
|
17
|
+
np.random.choice(labels, size=(10, 10, 10)),
|
|
18
|
+
np.random.choice(labels, size=1000).reshape(100, 10, order='F'),
|
|
19
|
+
]
|
|
20
|
+
arrays_object = [a.astype(object) for a in arrays]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_encode_decode():
|
|
24
|
+
# unicode dtype
|
|
25
|
+
for arr in arrays:
|
|
26
|
+
codec = Categorize(labels, dtype=arr.dtype)
|
|
27
|
+
check_encode_decode(arr, codec)
|
|
28
|
+
|
|
29
|
+
# object dtype
|
|
30
|
+
for arr in arrays_object:
|
|
31
|
+
codec = Categorize(labels, dtype=arr.dtype)
|
|
32
|
+
check_encode_decode_array(arr, codec)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_encode():
|
|
36
|
+
for dtype in 'U', object:
|
|
37
|
+
arr = np.array(['ƒöõ', 'ßàř', 'ƒöõ', 'ßāẑ', 'ƪùüx'], dtype=dtype)
|
|
38
|
+
# miss off quux
|
|
39
|
+
codec = Categorize(labels=labels[:-1], dtype=arr.dtype, astype='u1')
|
|
40
|
+
|
|
41
|
+
# test encoding
|
|
42
|
+
expect = np.array([1, 2, 1, 3, 0], dtype='u1')
|
|
43
|
+
enc = codec.encode(arr)
|
|
44
|
+
assert_array_equal(expect, enc)
|
|
45
|
+
assert expect.dtype == enc.dtype
|
|
46
|
+
|
|
47
|
+
# test decoding with unexpected value
|
|
48
|
+
dec = codec.decode(enc)
|
|
49
|
+
expect = arr.copy()
|
|
50
|
+
expect[expect == 'ƪùüx'] = ''
|
|
51
|
+
assert_array_equal(expect, dec)
|
|
52
|
+
assert arr.dtype == dec.dtype
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_config():
|
|
56
|
+
codec = Categorize(labels=labels, dtype='U4')
|
|
57
|
+
check_config(codec)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_repr():
|
|
61
|
+
dtype = '<U3'
|
|
62
|
+
astype = '|u1'
|
|
63
|
+
codec = Categorize(labels=['foo', 'bar', 'baz', 'qux'], dtype=dtype, astype=astype)
|
|
64
|
+
expect = "Categorize(dtype='<U3', astype='|u1', labels=['foo', 'bar', 'baz', ...])"
|
|
65
|
+
actual = repr(codec)
|
|
66
|
+
assert expect == actual
|
|
67
|
+
|
|
68
|
+
dtype = '<U4'
|
|
69
|
+
astype = '|u1'
|
|
70
|
+
codec = Categorize(labels=labels, dtype=dtype, astype=astype)
|
|
71
|
+
expect = "Categorize(dtype='<U4', astype='|u1', labels=['ƒöõ', 'ßàř', 'ßāẑ', ...])"
|
|
72
|
+
actual = repr(codec)
|
|
73
|
+
assert expect == actual
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_backwards_compatibility():
|
|
77
|
+
codec = Categorize(labels=labels, dtype='<U4', astype='u1')
|
|
78
|
+
check_backwards_compatibility(Categorize.codec_id, arrays, [codec], prefix='U')
|
|
79
|
+
codec = Categorize(labels=labels, dtype=object, astype='u1')
|
|
80
|
+
check_backwards_compatibility(Categorize.codec_id, arrays_object, [codec], prefix='O')
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def test_errors():
|
|
84
|
+
with pytest.raises(TypeError):
|
|
85
|
+
Categorize(labels=['foo', 'bar'], dtype='S6')
|
|
86
|
+
with pytest.raises(TypeError):
|
|
87
|
+
Categorize(labels=['foo', 'bar'], dtype='U6', astype=object)
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from numcodecs.checksum32 import CRC32, Adler32
|
|
7
|
+
from numcodecs.tests.common import (
|
|
8
|
+
check_backwards_compatibility,
|
|
9
|
+
check_config,
|
|
10
|
+
check_encode_decode,
|
|
11
|
+
check_err_encode_object_buffer,
|
|
12
|
+
check_repr,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
# mix of dtypes: integer, float, bool, string
|
|
16
|
+
# mix of shapes: 1D, 2D, 3D
|
|
17
|
+
# mix of orders: C, F
|
|
18
|
+
arrays = [
|
|
19
|
+
np.arange(1000, dtype='i4'),
|
|
20
|
+
np.linspace(1000, 1001, 1000, dtype='f8'),
|
|
21
|
+
np.random.normal(loc=1000, scale=1, size=(100, 10)),
|
|
22
|
+
np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
|
|
23
|
+
np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
codecs = [CRC32(), Adler32()]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def test_encode_decode():
|
|
30
|
+
for codec, arr in itertools.product(codecs, arrays):
|
|
31
|
+
check_encode_decode(arr, codec)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_errors():
|
|
35
|
+
for codec, arr in itertools.product(codecs, arrays):
|
|
36
|
+
enc = codec.encode(arr)
|
|
37
|
+
with pytest.raises(RuntimeError):
|
|
38
|
+
codec.decode(enc[:-1])
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_config():
|
|
42
|
+
for codec in codecs:
|
|
43
|
+
check_config(codec)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_repr():
|
|
47
|
+
check_repr("CRC32()")
|
|
48
|
+
check_repr("Adler32()")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_backwards_compatibility():
|
|
52
|
+
check_backwards_compatibility(CRC32.codec_id, arrays, [CRC32()])
|
|
53
|
+
check_backwards_compatibility(Adler32.codec_id, arrays, [Adler32()])
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_err_encode_object_buffer():
|
|
57
|
+
check_err_encode_object_buffer(CRC32())
|
|
58
|
+
check_err_encode_object_buffer(Adler32())
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import array
|
|
2
|
+
import mmap
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray, ensure_text
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_ensure_text():
|
|
11
|
+
bufs = [
|
|
12
|
+
b'adsdasdas',
|
|
13
|
+
'adsdasdas',
|
|
14
|
+
np.asarray(memoryview(b'adsdasdas')),
|
|
15
|
+
array.array('B', b'qwertyuiqwertyui'),
|
|
16
|
+
]
|
|
17
|
+
for buf in bufs:
|
|
18
|
+
b = ensure_text(buf)
|
|
19
|
+
assert isinstance(b, str)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_ensure_bytes():
|
|
23
|
+
bufs = [
|
|
24
|
+
b'adsdasdas',
|
|
25
|
+
bytes(20),
|
|
26
|
+
np.arange(100),
|
|
27
|
+
array.array('l', b'qwertyuiqwertyui'),
|
|
28
|
+
]
|
|
29
|
+
for buf in bufs:
|
|
30
|
+
b = ensure_bytes(buf)
|
|
31
|
+
assert isinstance(b, bytes)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_ensure_contiguous_ndarray_shares_memory():
|
|
35
|
+
typed_bufs = [
|
|
36
|
+
('u', 1, b'adsdasdas'),
|
|
37
|
+
('u', 1, bytes(20)),
|
|
38
|
+
('i', 8, np.arange(100, dtype=np.int64)),
|
|
39
|
+
('f', 8, np.linspace(0, 1, 100, dtype=np.float64)),
|
|
40
|
+
('i', 4, array.array('i', b'qwertyuiqwertyui')),
|
|
41
|
+
('u', 4, array.array('I', b'qwertyuiqwertyui')),
|
|
42
|
+
('f', 4, array.array('f', b'qwertyuiqwertyui')),
|
|
43
|
+
('f', 8, array.array('d', b'qwertyuiqwertyui')),
|
|
44
|
+
('i', 1, array.array('b', b'qwertyuiqwertyui')),
|
|
45
|
+
('u', 1, array.array('B', b'qwertyuiqwertyui')),
|
|
46
|
+
('u', 1, mmap.mmap(-1, 10)),
|
|
47
|
+
]
|
|
48
|
+
for expected_kind, expected_itemsize, buf in typed_bufs:
|
|
49
|
+
a = ensure_contiguous_ndarray(buf)
|
|
50
|
+
assert isinstance(a, np.ndarray)
|
|
51
|
+
assert expected_kind == a.dtype.kind
|
|
52
|
+
if isinstance(buf, array.array):
|
|
53
|
+
assert buf.itemsize == a.dtype.itemsize
|
|
54
|
+
else:
|
|
55
|
+
assert expected_itemsize == a.dtype.itemsize
|
|
56
|
+
assert np.shares_memory(a, memoryview(buf))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_ensure_bytes_invalid_inputs():
|
|
60
|
+
# object array not allowed
|
|
61
|
+
a = np.array(['Xin chào thế giới'], dtype=object)
|
|
62
|
+
for e in (a, memoryview(a)):
|
|
63
|
+
with pytest.raises(TypeError):
|
|
64
|
+
ensure_bytes(e)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_ensure_contiguous_ndarray_invalid_inputs():
|
|
68
|
+
# object array not allowed
|
|
69
|
+
a = np.array(['Xin chào thế giới'], dtype=object)
|
|
70
|
+
for e in (a, memoryview(a)):
|
|
71
|
+
with pytest.raises(TypeError):
|
|
72
|
+
ensure_contiguous_ndarray(e)
|
|
73
|
+
|
|
74
|
+
# non-contiguous arrays not allowed
|
|
75
|
+
with pytest.raises(ValueError):
|
|
76
|
+
ensure_contiguous_ndarray(np.arange(100)[::2])
|
|
77
|
+
|
|
78
|
+
# unicode array.array not allowed
|
|
79
|
+
a = array.array('u', 'qwertyuiqwertyui')
|
|
80
|
+
with pytest.raises(TypeError):
|
|
81
|
+
ensure_contiguous_ndarray(a)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_ensure_contiguous_ndarray_writeable():
|
|
85
|
+
# check that the writeability of the underlying buffer is preserved
|
|
86
|
+
for writeable in (False, True):
|
|
87
|
+
a = np.arange(100)
|
|
88
|
+
a.setflags(write=writeable)
|
|
89
|
+
m = ensure_contiguous_ndarray(a)
|
|
90
|
+
assert m.flags.writeable == writeable
|
|
91
|
+
m = ensure_contiguous_ndarray(memoryview(a))
|
|
92
|
+
assert m.flags.writeable == writeable
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def test_ensure_contiguous_ndarray_max_buffer_size():
|
|
96
|
+
for max_buffer_size in (4, 64, 1024):
|
|
97
|
+
ensure_contiguous_ndarray(np.zeros(max_buffer_size - 1, dtype=np.int8), max_buffer_size)
|
|
98
|
+
ensure_contiguous_ndarray(np.zeros(max_buffer_size, dtype=np.int8), max_buffer_size)
|
|
99
|
+
buffers = [
|
|
100
|
+
bytes(b"x" * (max_buffer_size + 1)),
|
|
101
|
+
np.zeros(max_buffer_size + 1, dtype=np.int8),
|
|
102
|
+
np.zeros(max_buffer_size + 2, dtype=np.int8),
|
|
103
|
+
np.zeros(max_buffer_size, dtype=np.int16),
|
|
104
|
+
np.zeros(max_buffer_size, dtype=np.int32),
|
|
105
|
+
]
|
|
106
|
+
for buf in buffers:
|
|
107
|
+
with pytest.raises(ValueError):
|
|
108
|
+
ensure_contiguous_ndarray(buf, max_buffer_size=max_buffer_size)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pytest
|
|
3
|
+
from numpy.testing import assert_array_equal
|
|
4
|
+
|
|
5
|
+
from numcodecs.delta import Delta
|
|
6
|
+
from numcodecs.tests.common import (
|
|
7
|
+
check_backwards_compatibility,
|
|
8
|
+
check_config,
|
|
9
|
+
check_encode_decode,
|
|
10
|
+
check_repr,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
# mix of dtypes: integer, float
|
|
14
|
+
# mix of shapes: 1D, 2D, 3D
|
|
15
|
+
# mix of orders: C, F
|
|
16
|
+
arrays = [
|
|
17
|
+
np.arange(1000, dtype='<i4'),
|
|
18
|
+
np.linspace(1000, 1001, 1000, dtype='<f4').reshape(100, 10),
|
|
19
|
+
np.random.normal(loc=1000, scale=1, size=(10, 10, 10)).astype('<f8'),
|
|
20
|
+
np.random.randint(0, 200, size=1000, dtype='u2').astype('<u2').reshape(100, 10, order='F'),
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_encode_decode():
|
|
25
|
+
for arr in arrays:
|
|
26
|
+
codec = Delta(dtype=arr.dtype)
|
|
27
|
+
check_encode_decode(arr, codec)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_encode():
|
|
31
|
+
dtype = 'i8'
|
|
32
|
+
astype = 'i4'
|
|
33
|
+
codec = Delta(dtype=dtype, astype=astype)
|
|
34
|
+
arr = np.arange(10, 20, 1, dtype=dtype)
|
|
35
|
+
expect = np.array([10] + ([1] * 9), dtype=astype)
|
|
36
|
+
actual = codec.encode(arr)
|
|
37
|
+
assert_array_equal(expect, actual)
|
|
38
|
+
assert np.dtype(astype) == actual.dtype
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_config():
|
|
42
|
+
codec = Delta(dtype='<i4', astype='<i2')
|
|
43
|
+
check_config(codec)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_repr():
|
|
47
|
+
check_repr("Delta(dtype='<i4', astype='<i2')")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_backwards_compatibility():
|
|
51
|
+
for arr in arrays:
|
|
52
|
+
codec = Delta(dtype=arr.dtype)
|
|
53
|
+
check_backwards_compatibility(Delta.codec_id, [arr], [codec], prefix=str(arr.dtype))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_errors():
|
|
57
|
+
with pytest.raises(ValueError):
|
|
58
|
+
Delta(dtype=object)
|
|
59
|
+
with pytest.raises(ValueError):
|
|
60
|
+
Delta(dtype='i8', astype=object)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import os.path
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
import numcodecs.registry
|
|
7
|
+
|
|
8
|
+
here = os.path.abspath(os.path.dirname(__file__))
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@pytest.fixture()
|
|
12
|
+
def set_path():
|
|
13
|
+
sys.path.append(here)
|
|
14
|
+
numcodecs.registry.run_entrypoints()
|
|
15
|
+
yield
|
|
16
|
+
sys.path.remove(here)
|
|
17
|
+
numcodecs.registry.run_entrypoints()
|
|
18
|
+
numcodecs.registry.codec_registry.pop("test")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@pytest.mark.usefixtures("set_path")
|
|
22
|
+
def test_entrypoint_codec():
|
|
23
|
+
cls = numcodecs.registry.get_codec({"id": "test"})
|
|
24
|
+
assert cls.codec_id == "test"
|