numcodecs 0.13.1__cp313-cp313-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of numcodecs might be problematic. Click here for more details.

Files changed (74) hide show
  1. numcodecs/__init__.py +143 -0
  2. numcodecs/_shuffle.cp313-win_amd64.pyd +0 -0
  3. numcodecs/abc.py +126 -0
  4. numcodecs/astype.py +76 -0
  5. numcodecs/base64.py +27 -0
  6. numcodecs/bitround.py +79 -0
  7. numcodecs/blosc.cp313-win_amd64.pyd +0 -0
  8. numcodecs/bz2.py +45 -0
  9. numcodecs/categorize.py +101 -0
  10. numcodecs/checksum32.py +94 -0
  11. numcodecs/compat.py +208 -0
  12. numcodecs/compat_ext.cp313-win_amd64.pyd +0 -0
  13. numcodecs/delta.py +97 -0
  14. numcodecs/fixedscaleoffset.py +132 -0
  15. numcodecs/fletcher32.cp313-win_amd64.pyd +0 -0
  16. numcodecs/gzip.py +52 -0
  17. numcodecs/jenkins.cp313-win_amd64.pyd +0 -0
  18. numcodecs/json.py +107 -0
  19. numcodecs/lz4.cp313-win_amd64.pyd +0 -0
  20. numcodecs/lzma.py +69 -0
  21. numcodecs/msgpacks.py +86 -0
  22. numcodecs/ndarray_like.py +65 -0
  23. numcodecs/packbits.py +85 -0
  24. numcodecs/pcodec.py +89 -0
  25. numcodecs/pickles.py +55 -0
  26. numcodecs/quantize.py +100 -0
  27. numcodecs/registry.py +72 -0
  28. numcodecs/shuffle.py +61 -0
  29. numcodecs/tests/__init__.py +3 -0
  30. numcodecs/tests/common.py +354 -0
  31. numcodecs/tests/package_with_entrypoint/__init__.py +11 -0
  32. numcodecs/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt +2 -0
  33. numcodecs/tests/test_astype.py +74 -0
  34. numcodecs/tests/test_base64.py +81 -0
  35. numcodecs/tests/test_bitround.py +81 -0
  36. numcodecs/tests/test_blosc.py +277 -0
  37. numcodecs/tests/test_bz2.py +66 -0
  38. numcodecs/tests/test_categorize.py +87 -0
  39. numcodecs/tests/test_checksum32.py +58 -0
  40. numcodecs/tests/test_compat.py +108 -0
  41. numcodecs/tests/test_delta.py +60 -0
  42. numcodecs/tests/test_entrypoints.py +24 -0
  43. numcodecs/tests/test_entrypoints_backport.py +35 -0
  44. numcodecs/tests/test_fixedscaleoffset.py +69 -0
  45. numcodecs/tests/test_fletcher32.py +56 -0
  46. numcodecs/tests/test_gzip.py +110 -0
  47. numcodecs/tests/test_jenkins.py +150 -0
  48. numcodecs/tests/test_json.py +85 -0
  49. numcodecs/tests/test_lz4.py +83 -0
  50. numcodecs/tests/test_lzma.py +90 -0
  51. numcodecs/tests/test_msgpacks.py +123 -0
  52. numcodecs/tests/test_ndarray_like.py +48 -0
  53. numcodecs/tests/test_packbits.py +39 -0
  54. numcodecs/tests/test_pcodec.py +80 -0
  55. numcodecs/tests/test_pickles.py +61 -0
  56. numcodecs/tests/test_quantize.py +76 -0
  57. numcodecs/tests/test_registry.py +40 -0
  58. numcodecs/tests/test_shuffle.py +168 -0
  59. numcodecs/tests/test_vlen_array.py +97 -0
  60. numcodecs/tests/test_vlen_bytes.py +93 -0
  61. numcodecs/tests/test_vlen_utf8.py +91 -0
  62. numcodecs/tests/test_zfpy.py +98 -0
  63. numcodecs/tests/test_zlib.py +94 -0
  64. numcodecs/tests/test_zstd.py +92 -0
  65. numcodecs/version.py +16 -0
  66. numcodecs/vlen.cp313-win_amd64.pyd +0 -0
  67. numcodecs/zfpy.py +111 -0
  68. numcodecs/zlib.py +42 -0
  69. numcodecs/zstd.cp313-win_amd64.pyd +0 -0
  70. numcodecs-0.13.1.dist-info/LICENSE.txt +21 -0
  71. numcodecs-0.13.1.dist-info/METADATA +64 -0
  72. numcodecs-0.13.1.dist-info/RECORD +74 -0
  73. numcodecs-0.13.1.dist-info/WHEEL +5 -0
  74. numcodecs-0.13.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,277 @@
1
+ from multiprocessing import Pool
2
+ from multiprocessing.pool import ThreadPool
3
+
4
+ import numpy as np
5
+ import pytest
6
+
7
+ try:
8
+ from numcodecs import blosc
9
+ from numcodecs.blosc import Blosc
10
+ except ImportError: # pragma: no cover
11
+ pytest.skip("numcodecs.blosc not available", allow_module_level=True)
12
+
13
+
14
+ from numcodecs.tests.common import (
15
+ check_backwards_compatibility,
16
+ check_config,
17
+ check_encode_decode,
18
+ check_encode_decode_partial,
19
+ check_err_decode_object_buffer,
20
+ check_err_encode_object_buffer,
21
+ check_max_buffer_size,
22
+ )
23
+
24
+ codecs = [
25
+ Blosc(shuffle=Blosc.SHUFFLE),
26
+ Blosc(clevel=0, shuffle=Blosc.SHUFFLE),
27
+ Blosc(cname='lz4', shuffle=Blosc.SHUFFLE),
28
+ Blosc(cname='lz4', clevel=1, shuffle=Blosc.NOSHUFFLE),
29
+ Blosc(cname='lz4', clevel=5, shuffle=Blosc.SHUFFLE),
30
+ Blosc(cname='lz4', clevel=9, shuffle=Blosc.BITSHUFFLE),
31
+ Blosc(cname='zlib', clevel=1, shuffle=0),
32
+ Blosc(cname='zstd', clevel=1, shuffle=1),
33
+ Blosc(cname='blosclz', clevel=1, shuffle=2),
34
+ None, # was snappy
35
+ Blosc(shuffle=Blosc.SHUFFLE, blocksize=0),
36
+ Blosc(shuffle=Blosc.SHUFFLE, blocksize=2**8),
37
+ Blosc(cname='lz4', clevel=1, shuffle=Blosc.NOSHUFFLE, blocksize=2**8),
38
+ ]
39
+
40
+
41
+ # mix of dtypes: integer, float, bool, string
42
+ # mix of shapes: 1D, 2D, 3D
43
+ # mix of orders: C, F
44
+ arrays = [
45
+ np.arange(1000, dtype='i4'),
46
+ np.linspace(1000, 1001, 1000, dtype='f8'),
47
+ np.random.normal(loc=1000, scale=1, size=(100, 10)),
48
+ np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
49
+ np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
50
+ np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
51
+ np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
52
+ np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
53
+ np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
54
+ np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
55
+ np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
56
+ np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
57
+ np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
58
+ ]
59
+
60
+
61
+ def _skip_null(codec):
62
+ if codec is None:
63
+ pytest.skip("codec has been removed")
64
+
65
+
66
+ @pytest.fixture(scope='module', params=[True, False, None])
67
+ def use_threads(request):
68
+ return request.param
69
+
70
+
71
+ @pytest.mark.parametrize('array', arrays)
72
+ @pytest.mark.parametrize('codec', codecs)
73
+ def test_encode_decode(array, codec):
74
+ _skip_null(codec)
75
+ check_encode_decode(array, codec)
76
+
77
+
78
+ @pytest.mark.parametrize('codec', codecs)
79
+ @pytest.mark.parametrize(
80
+ 'array',
81
+ [
82
+ pytest.param(x) if len(x.shape) == 1 else pytest.param(x, marks=[pytest.mark.xfail])
83
+ for x in arrays
84
+ ],
85
+ )
86
+ def test_partial_decode(codec, array):
87
+ _skip_null(codec)
88
+ check_encode_decode_partial(array, codec)
89
+
90
+
91
+ def test_config():
92
+ codec = Blosc(cname='zstd', clevel=3, shuffle=1)
93
+ check_config(codec)
94
+ codec = Blosc(cname='lz4', clevel=1, shuffle=2, blocksize=2**8)
95
+ check_config(codec)
96
+
97
+
98
+ def test_repr():
99
+ expect = "Blosc(cname='zstd', clevel=3, shuffle=SHUFFLE, blocksize=0)"
100
+ actual = repr(Blosc(cname='zstd', clevel=3, shuffle=Blosc.SHUFFLE, blocksize=0))
101
+ assert expect == actual
102
+ expect = "Blosc(cname='lz4', clevel=1, shuffle=NOSHUFFLE, blocksize=256)"
103
+ actual = repr(Blosc(cname='lz4', clevel=1, shuffle=Blosc.NOSHUFFLE, blocksize=256))
104
+ assert expect == actual
105
+ expect = "Blosc(cname='zlib', clevel=9, shuffle=BITSHUFFLE, blocksize=512)"
106
+ actual = repr(Blosc(cname='zlib', clevel=9, shuffle=Blosc.BITSHUFFLE, blocksize=512))
107
+ assert expect == actual
108
+ expect = "Blosc(cname='blosclz', clevel=5, shuffle=AUTOSHUFFLE, blocksize=1024)"
109
+ actual = repr(Blosc(cname='blosclz', clevel=5, shuffle=Blosc.AUTOSHUFFLE, blocksize=1024))
110
+ assert expect == actual
111
+
112
+
113
+ def test_eq():
114
+ assert Blosc() == Blosc()
115
+ assert Blosc(cname='lz4') != Blosc(cname='zstd')
116
+ assert Blosc(clevel=1) != Blosc(clevel=9)
117
+ assert Blosc(cname='lz4') != 'foo'
118
+
119
+
120
+ def test_compress_blocksize_default(use_threads):
121
+ arr = np.arange(1000, dtype='i4')
122
+
123
+ blosc.use_threads = use_threads
124
+
125
+ # default blocksize
126
+ enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE)
127
+ _, _, blocksize = blosc.cbuffer_sizes(enc)
128
+ assert blocksize > 0
129
+
130
+ # explicit default blocksize
131
+ enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE, 0)
132
+ _, _, blocksize = blosc.cbuffer_sizes(enc)
133
+ assert blocksize > 0
134
+
135
+
136
+ @pytest.mark.parametrize('bs', (2**7, 2**8))
137
+ def test_compress_blocksize(use_threads, bs):
138
+ arr = np.arange(1000, dtype='i4')
139
+
140
+ blosc.use_threads = use_threads
141
+
142
+ enc = blosc.compress(arr, b'lz4', 1, Blosc.NOSHUFFLE, bs)
143
+ _, _, blocksize = blosc.cbuffer_sizes(enc)
144
+ assert blocksize == bs
145
+
146
+
147
+ def test_compress_complib(use_threads):
148
+ arr = np.arange(1000, dtype='i4')
149
+ expected_complibs = {
150
+ 'lz4': 'LZ4',
151
+ 'lz4hc': 'LZ4',
152
+ 'blosclz': 'BloscLZ',
153
+ 'zlib': 'Zlib',
154
+ 'zstd': 'Zstd',
155
+ }
156
+ blosc.use_threads = use_threads
157
+ for cname in blosc.list_compressors():
158
+ enc = blosc.compress(arr, cname.encode(), 1, Blosc.NOSHUFFLE)
159
+ complib = blosc.cbuffer_complib(enc)
160
+ expected_complib = expected_complibs[cname]
161
+ assert complib == expected_complib
162
+ with pytest.raises(ValueError):
163
+ # capitalized cname
164
+ blosc.compress(arr, b'LZ4', 1)
165
+ with pytest.raises(ValueError):
166
+ # bad cname
167
+ blosc.compress(arr, b'foo', 1)
168
+
169
+
170
+ @pytest.mark.parametrize('dtype', ['i1', 'i2', 'i4', 'i8'])
171
+ def test_compress_metainfo(dtype, use_threads):
172
+ arr = np.arange(1000, dtype=dtype)
173
+ for shuffle in Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE:
174
+ blosc.use_threads = use_threads
175
+ for cname in blosc.list_compressors():
176
+ enc = blosc.compress(arr, cname.encode(), 1, shuffle)
177
+ typesize, did_shuffle, _ = blosc.cbuffer_metainfo(enc)
178
+ assert typesize == arr.dtype.itemsize
179
+ assert did_shuffle == shuffle
180
+
181
+
182
+ def test_compress_autoshuffle(use_threads):
183
+ arr = np.arange(8000)
184
+ for dtype in 'i1', 'i2', 'i4', 'i8', 'f2', 'f4', 'f8', 'bool', 'S10':
185
+ varr = arr.view(dtype)
186
+ blosc.use_threads = use_threads
187
+ for cname in blosc.list_compressors():
188
+ enc = blosc.compress(varr, cname.encode(), 1, Blosc.AUTOSHUFFLE)
189
+ typesize, did_shuffle, _ = blosc.cbuffer_metainfo(enc)
190
+ assert typesize == varr.dtype.itemsize
191
+ if typesize == 1:
192
+ assert did_shuffle == Blosc.BITSHUFFLE
193
+ else:
194
+ assert did_shuffle == Blosc.SHUFFLE
195
+
196
+
197
+ def test_config_blocksize():
198
+ # N.B., we want to be backwards compatible with any config where blocksize is not
199
+ # explicitly stated
200
+
201
+ # blocksize not stated
202
+ config = dict(cname='lz4', clevel=1, shuffle=Blosc.SHUFFLE)
203
+ codec = Blosc.from_config(config)
204
+ assert codec.blocksize == 0
205
+
206
+ # blocksize stated
207
+ config = dict(cname='lz4', clevel=1, shuffle=Blosc.SHUFFLE, blocksize=2**8)
208
+ codec = Blosc.from_config(config)
209
+ assert codec.blocksize == 2**8
210
+
211
+
212
+ def test_backwards_compatibility():
213
+ check_backwards_compatibility(Blosc.codec_id, arrays, codecs)
214
+
215
+
216
+ def _encode_worker(data):
217
+ compressor = Blosc(cname='zlib', clevel=9, shuffle=Blosc.SHUFFLE)
218
+ enc = compressor.encode(data)
219
+ return enc
220
+
221
+
222
+ def _decode_worker(enc):
223
+ compressor = Blosc()
224
+ data = compressor.decode(enc)
225
+ return data
226
+
227
+
228
+ @pytest.mark.parametrize('pool', (Pool, ThreadPool))
229
+ def test_multiprocessing(use_threads, pool):
230
+ data = np.arange(1000000)
231
+ enc = _encode_worker(data)
232
+
233
+ pool = pool(5)
234
+
235
+ try:
236
+ blosc.use_threads = use_threads
237
+
238
+ # test with process pool and thread pool
239
+
240
+ # test encoding
241
+ enc_results = pool.map(_encode_worker, [data] * 5)
242
+ assert all(len(enc) == len(e) for e in enc_results)
243
+
244
+ # test decoding
245
+ dec_results = pool.map(_decode_worker, [enc] * 5)
246
+ assert all(data.nbytes == len(d) for d in dec_results)
247
+
248
+ # tidy up
249
+ pool.close()
250
+ pool.join()
251
+
252
+ finally:
253
+ blosc.use_threads = None # restore default
254
+
255
+
256
+ def test_err_decode_object_buffer():
257
+ check_err_decode_object_buffer(Blosc())
258
+
259
+
260
+ def test_err_encode_object_buffer():
261
+ check_err_encode_object_buffer(Blosc())
262
+
263
+
264
+ def test_decompression_error_handling():
265
+ for codec in codecs:
266
+ _skip_null(codec)
267
+ with pytest.raises(RuntimeError):
268
+ codec.decode(bytearray())
269
+ with pytest.raises(RuntimeError):
270
+ codec.decode(bytearray(0))
271
+
272
+
273
+ def test_max_buffer_size():
274
+ for codec in codecs:
275
+ _skip_null(codec)
276
+ assert codec.max_buffer_size == 2**31 - 1
277
+ check_max_buffer_size(codec)
@@ -0,0 +1,66 @@
1
+ import itertools
2
+
3
+ import numpy as np
4
+
5
+ from numcodecs.bz2 import BZ2
6
+ from numcodecs.tests.common import (
7
+ check_backwards_compatibility,
8
+ check_config,
9
+ check_encode_decode,
10
+ check_err_decode_object_buffer,
11
+ check_err_encode_object_buffer,
12
+ check_repr,
13
+ )
14
+
15
+ codecs = [
16
+ BZ2(),
17
+ BZ2(level=1),
18
+ BZ2(level=5),
19
+ BZ2(level=9),
20
+ ]
21
+
22
+
23
+ # mix of dtypes: integer, float, bool, string
24
+ # mix of shapes: 1D, 2D, 3D
25
+ # mix of orders: C, F
26
+ arrays = [
27
+ np.arange(1000, dtype='i4'),
28
+ np.linspace(1000, 1001, 1000, dtype='f8'),
29
+ np.random.normal(loc=1000, scale=1, size=(100, 10)),
30
+ np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
31
+ np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
32
+ np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
33
+ np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
34
+ np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
35
+ np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
36
+ np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
37
+ np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
38
+ np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
39
+ np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
40
+ ]
41
+
42
+
43
+ def test_encode_decode():
44
+ for arr, codec in itertools.product(arrays, codecs):
45
+ check_encode_decode(arr, codec)
46
+
47
+
48
+ def test_config():
49
+ codec = BZ2(level=3)
50
+ check_config(codec)
51
+
52
+
53
+ def test_repr():
54
+ check_repr("BZ2(level=3)")
55
+
56
+
57
+ def test_backwards_compatibility():
58
+ check_backwards_compatibility(BZ2.codec_id, arrays, codecs)
59
+
60
+
61
+ def test_err_decode_object_buffer():
62
+ check_err_decode_object_buffer(BZ2())
63
+
64
+
65
+ def test_err_encode_object_buffer():
66
+ check_err_encode_object_buffer(BZ2())
@@ -0,0 +1,87 @@
1
+ import numpy as np
2
+ import pytest
3
+ from numpy.testing import assert_array_equal
4
+
5
+ from numcodecs.categorize import Categorize
6
+ from numcodecs.tests.common import (
7
+ check_backwards_compatibility,
8
+ check_config,
9
+ check_encode_decode,
10
+ check_encode_decode_array,
11
+ )
12
+
13
+ labels = ['ƒöõ', 'ßàř', 'ßāẑ', 'ƪùüx']
14
+ arrays = [
15
+ np.random.choice(labels, size=1000),
16
+ np.random.choice(labels, size=(100, 10)),
17
+ np.random.choice(labels, size=(10, 10, 10)),
18
+ np.random.choice(labels, size=1000).reshape(100, 10, order='F'),
19
+ ]
20
+ arrays_object = [a.astype(object) for a in arrays]
21
+
22
+
23
+ def test_encode_decode():
24
+ # unicode dtype
25
+ for arr in arrays:
26
+ codec = Categorize(labels, dtype=arr.dtype)
27
+ check_encode_decode(arr, codec)
28
+
29
+ # object dtype
30
+ for arr in arrays_object:
31
+ codec = Categorize(labels, dtype=arr.dtype)
32
+ check_encode_decode_array(arr, codec)
33
+
34
+
35
+ def test_encode():
36
+ for dtype in 'U', object:
37
+ arr = np.array(['ƒöõ', 'ßàř', 'ƒöõ', 'ßāẑ', 'ƪùüx'], dtype=dtype)
38
+ # miss off quux
39
+ codec = Categorize(labels=labels[:-1], dtype=arr.dtype, astype='u1')
40
+
41
+ # test encoding
42
+ expect = np.array([1, 2, 1, 3, 0], dtype='u1')
43
+ enc = codec.encode(arr)
44
+ assert_array_equal(expect, enc)
45
+ assert expect.dtype == enc.dtype
46
+
47
+ # test decoding with unexpected value
48
+ dec = codec.decode(enc)
49
+ expect = arr.copy()
50
+ expect[expect == 'ƪùüx'] = ''
51
+ assert_array_equal(expect, dec)
52
+ assert arr.dtype == dec.dtype
53
+
54
+
55
+ def test_config():
56
+ codec = Categorize(labels=labels, dtype='U4')
57
+ check_config(codec)
58
+
59
+
60
+ def test_repr():
61
+ dtype = '<U3'
62
+ astype = '|u1'
63
+ codec = Categorize(labels=['foo', 'bar', 'baz', 'qux'], dtype=dtype, astype=astype)
64
+ expect = "Categorize(dtype='<U3', astype='|u1', labels=['foo', 'bar', 'baz', ...])"
65
+ actual = repr(codec)
66
+ assert expect == actual
67
+
68
+ dtype = '<U4'
69
+ astype = '|u1'
70
+ codec = Categorize(labels=labels, dtype=dtype, astype=astype)
71
+ expect = "Categorize(dtype='<U4', astype='|u1', labels=['ƒöõ', 'ßàř', 'ßāẑ', ...])"
72
+ actual = repr(codec)
73
+ assert expect == actual
74
+
75
+
76
+ def test_backwards_compatibility():
77
+ codec = Categorize(labels=labels, dtype='<U4', astype='u1')
78
+ check_backwards_compatibility(Categorize.codec_id, arrays, [codec], prefix='U')
79
+ codec = Categorize(labels=labels, dtype=object, astype='u1')
80
+ check_backwards_compatibility(Categorize.codec_id, arrays_object, [codec], prefix='O')
81
+
82
+
83
+ def test_errors():
84
+ with pytest.raises(TypeError):
85
+ Categorize(labels=['foo', 'bar'], dtype='S6')
86
+ with pytest.raises(TypeError):
87
+ Categorize(labels=['foo', 'bar'], dtype='U6', astype=object)
@@ -0,0 +1,58 @@
1
+ import itertools
2
+
3
+ import numpy as np
4
+ import pytest
5
+
6
+ from numcodecs.checksum32 import CRC32, Adler32
7
+ from numcodecs.tests.common import (
8
+ check_backwards_compatibility,
9
+ check_config,
10
+ check_encode_decode,
11
+ check_err_encode_object_buffer,
12
+ check_repr,
13
+ )
14
+
15
+ # mix of dtypes: integer, float, bool, string
16
+ # mix of shapes: 1D, 2D, 3D
17
+ # mix of orders: C, F
18
+ arrays = [
19
+ np.arange(1000, dtype='i4'),
20
+ np.linspace(1000, 1001, 1000, dtype='f8'),
21
+ np.random.normal(loc=1000, scale=1, size=(100, 10)),
22
+ np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
23
+ np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
24
+ ]
25
+
26
+ codecs = [CRC32(), Adler32()]
27
+
28
+
29
+ def test_encode_decode():
30
+ for codec, arr in itertools.product(codecs, arrays):
31
+ check_encode_decode(arr, codec)
32
+
33
+
34
+ def test_errors():
35
+ for codec, arr in itertools.product(codecs, arrays):
36
+ enc = codec.encode(arr)
37
+ with pytest.raises(RuntimeError):
38
+ codec.decode(enc[:-1])
39
+
40
+
41
+ def test_config():
42
+ for codec in codecs:
43
+ check_config(codec)
44
+
45
+
46
+ def test_repr():
47
+ check_repr("CRC32()")
48
+ check_repr("Adler32()")
49
+
50
+
51
+ def test_backwards_compatibility():
52
+ check_backwards_compatibility(CRC32.codec_id, arrays, [CRC32()])
53
+ check_backwards_compatibility(Adler32.codec_id, arrays, [Adler32()])
54
+
55
+
56
+ def test_err_encode_object_buffer():
57
+ check_err_encode_object_buffer(CRC32())
58
+ check_err_encode_object_buffer(Adler32())
@@ -0,0 +1,108 @@
1
+ import array
2
+ import mmap
3
+
4
+ import numpy as np
5
+ import pytest
6
+
7
+ from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray, ensure_text
8
+
9
+
10
+ def test_ensure_text():
11
+ bufs = [
12
+ b'adsdasdas',
13
+ 'adsdasdas',
14
+ np.asarray(memoryview(b'adsdasdas')),
15
+ array.array('B', b'qwertyuiqwertyui'),
16
+ ]
17
+ for buf in bufs:
18
+ b = ensure_text(buf)
19
+ assert isinstance(b, str)
20
+
21
+
22
+ def test_ensure_bytes():
23
+ bufs = [
24
+ b'adsdasdas',
25
+ bytes(20),
26
+ np.arange(100),
27
+ array.array('l', b'qwertyuiqwertyui'),
28
+ ]
29
+ for buf in bufs:
30
+ b = ensure_bytes(buf)
31
+ assert isinstance(b, bytes)
32
+
33
+
34
+ def test_ensure_contiguous_ndarray_shares_memory():
35
+ typed_bufs = [
36
+ ('u', 1, b'adsdasdas'),
37
+ ('u', 1, bytes(20)),
38
+ ('i', 8, np.arange(100, dtype=np.int64)),
39
+ ('f', 8, np.linspace(0, 1, 100, dtype=np.float64)),
40
+ ('i', 4, array.array('i', b'qwertyuiqwertyui')),
41
+ ('u', 4, array.array('I', b'qwertyuiqwertyui')),
42
+ ('f', 4, array.array('f', b'qwertyuiqwertyui')),
43
+ ('f', 8, array.array('d', b'qwertyuiqwertyui')),
44
+ ('i', 1, array.array('b', b'qwertyuiqwertyui')),
45
+ ('u', 1, array.array('B', b'qwertyuiqwertyui')),
46
+ ('u', 1, mmap.mmap(-1, 10)),
47
+ ]
48
+ for expected_kind, expected_itemsize, buf in typed_bufs:
49
+ a = ensure_contiguous_ndarray(buf)
50
+ assert isinstance(a, np.ndarray)
51
+ assert expected_kind == a.dtype.kind
52
+ if isinstance(buf, array.array):
53
+ assert buf.itemsize == a.dtype.itemsize
54
+ else:
55
+ assert expected_itemsize == a.dtype.itemsize
56
+ assert np.shares_memory(a, memoryview(buf))
57
+
58
+
59
+ def test_ensure_bytes_invalid_inputs():
60
+ # object array not allowed
61
+ a = np.array(['Xin chào thế giới'], dtype=object)
62
+ for e in (a, memoryview(a)):
63
+ with pytest.raises(TypeError):
64
+ ensure_bytes(e)
65
+
66
+
67
+ def test_ensure_contiguous_ndarray_invalid_inputs():
68
+ # object array not allowed
69
+ a = np.array(['Xin chào thế giới'], dtype=object)
70
+ for e in (a, memoryview(a)):
71
+ with pytest.raises(TypeError):
72
+ ensure_contiguous_ndarray(e)
73
+
74
+ # non-contiguous arrays not allowed
75
+ with pytest.raises(ValueError):
76
+ ensure_contiguous_ndarray(np.arange(100)[::2])
77
+
78
+ # unicode array.array not allowed
79
+ a = array.array('u', 'qwertyuiqwertyui')
80
+ with pytest.raises(TypeError):
81
+ ensure_contiguous_ndarray(a)
82
+
83
+
84
+ def test_ensure_contiguous_ndarray_writeable():
85
+ # check that the writeability of the underlying buffer is preserved
86
+ for writeable in (False, True):
87
+ a = np.arange(100)
88
+ a.setflags(write=writeable)
89
+ m = ensure_contiguous_ndarray(a)
90
+ assert m.flags.writeable == writeable
91
+ m = ensure_contiguous_ndarray(memoryview(a))
92
+ assert m.flags.writeable == writeable
93
+
94
+
95
+ def test_ensure_contiguous_ndarray_max_buffer_size():
96
+ for max_buffer_size in (4, 64, 1024):
97
+ ensure_contiguous_ndarray(np.zeros(max_buffer_size - 1, dtype=np.int8), max_buffer_size)
98
+ ensure_contiguous_ndarray(np.zeros(max_buffer_size, dtype=np.int8), max_buffer_size)
99
+ buffers = [
100
+ bytes(b"x" * (max_buffer_size + 1)),
101
+ np.zeros(max_buffer_size + 1, dtype=np.int8),
102
+ np.zeros(max_buffer_size + 2, dtype=np.int8),
103
+ np.zeros(max_buffer_size, dtype=np.int16),
104
+ np.zeros(max_buffer_size, dtype=np.int32),
105
+ ]
106
+ for buf in buffers:
107
+ with pytest.raises(ValueError):
108
+ ensure_contiguous_ndarray(buf, max_buffer_size=max_buffer_size)
@@ -0,0 +1,60 @@
1
+ import numpy as np
2
+ import pytest
3
+ from numpy.testing import assert_array_equal
4
+
5
+ from numcodecs.delta import Delta
6
+ from numcodecs.tests.common import (
7
+ check_backwards_compatibility,
8
+ check_config,
9
+ check_encode_decode,
10
+ check_repr,
11
+ )
12
+
13
+ # mix of dtypes: integer, float
14
+ # mix of shapes: 1D, 2D, 3D
15
+ # mix of orders: C, F
16
+ arrays = [
17
+ np.arange(1000, dtype='<i4'),
18
+ np.linspace(1000, 1001, 1000, dtype='<f4').reshape(100, 10),
19
+ np.random.normal(loc=1000, scale=1, size=(10, 10, 10)).astype('<f8'),
20
+ np.random.randint(0, 200, size=1000, dtype='u2').astype('<u2').reshape(100, 10, order='F'),
21
+ ]
22
+
23
+
24
+ def test_encode_decode():
25
+ for arr in arrays:
26
+ codec = Delta(dtype=arr.dtype)
27
+ check_encode_decode(arr, codec)
28
+
29
+
30
+ def test_encode():
31
+ dtype = 'i8'
32
+ astype = 'i4'
33
+ codec = Delta(dtype=dtype, astype=astype)
34
+ arr = np.arange(10, 20, 1, dtype=dtype)
35
+ expect = np.array([10] + ([1] * 9), dtype=astype)
36
+ actual = codec.encode(arr)
37
+ assert_array_equal(expect, actual)
38
+ assert np.dtype(astype) == actual.dtype
39
+
40
+
41
+ def test_config():
42
+ codec = Delta(dtype='<i4', astype='<i2')
43
+ check_config(codec)
44
+
45
+
46
+ def test_repr():
47
+ check_repr("Delta(dtype='<i4', astype='<i2')")
48
+
49
+
50
+ def test_backwards_compatibility():
51
+ for arr in arrays:
52
+ codec = Delta(dtype=arr.dtype)
53
+ check_backwards_compatibility(Delta.codec_id, [arr], [codec], prefix=str(arr.dtype))
54
+
55
+
56
+ def test_errors():
57
+ with pytest.raises(ValueError):
58
+ Delta(dtype=object)
59
+ with pytest.raises(ValueError):
60
+ Delta(dtype='i8', astype=object)
@@ -0,0 +1,24 @@
1
+ import os.path
2
+ import sys
3
+
4
+ import pytest
5
+
6
+ import numcodecs.registry
7
+
8
+ here = os.path.abspath(os.path.dirname(__file__))
9
+
10
+
11
+ @pytest.fixture()
12
+ def set_path():
13
+ sys.path.append(here)
14
+ numcodecs.registry.run_entrypoints()
15
+ yield
16
+ sys.path.remove(here)
17
+ numcodecs.registry.run_entrypoints()
18
+ numcodecs.registry.codec_registry.pop("test")
19
+
20
+
21
+ @pytest.mark.usefixtures("set_path")
22
+ def test_entrypoint_codec():
23
+ cls = numcodecs.registry.get_codec({"id": "test"})
24
+ assert cls.codec_id == "test"