numcodecs 0.16.0__cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of numcodecs might be problematic. Click here for more details.
- numcodecs/__init__.py +146 -0
- numcodecs/_shuffle.cpython-312-aarch64-linux-gnu.so +0 -0
- numcodecs/abc.py +127 -0
- numcodecs/astype.py +72 -0
- numcodecs/base64.py +26 -0
- numcodecs/bitround.py +80 -0
- numcodecs/blosc.cpython-312-aarch64-linux-gnu.so +0 -0
- numcodecs/bz2.py +45 -0
- numcodecs/categorize.py +98 -0
- numcodecs/checksum32.py +183 -0
- numcodecs/compat.py +206 -0
- numcodecs/compat_ext.cpython-312-aarch64-linux-gnu.so +0 -0
- numcodecs/delta.py +94 -0
- numcodecs/errors.py +26 -0
- numcodecs/fixedscaleoffset.py +130 -0
- numcodecs/fletcher32.cpython-312-aarch64-linux-gnu.so +0 -0
- numcodecs/gzip.py +50 -0
- numcodecs/jenkins.cpython-312-aarch64-linux-gnu.so +0 -0
- numcodecs/json.py +107 -0
- numcodecs/lz4.cpython-312-aarch64-linux-gnu.so +0 -0
- numcodecs/lzma.py +72 -0
- numcodecs/msgpacks.py +86 -0
- numcodecs/ndarray_like.py +65 -0
- numcodecs/packbits.py +82 -0
- numcodecs/pcodec.py +118 -0
- numcodecs/pickles.py +55 -0
- numcodecs/quantize.py +98 -0
- numcodecs/registry.py +74 -0
- numcodecs/shuffle.py +61 -0
- numcodecs/tests/__init__.py +3 -0
- numcodecs/tests/common.py +285 -0
- numcodecs/tests/package_with_entrypoint/__init__.py +11 -0
- numcodecs/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt +2 -0
- numcodecs/tests/test_astype.py +74 -0
- numcodecs/tests/test_base64.py +81 -0
- numcodecs/tests/test_bitround.py +81 -0
- numcodecs/tests/test_blosc.py +284 -0
- numcodecs/tests/test_bz2.py +66 -0
- numcodecs/tests/test_categorize.py +87 -0
- numcodecs/tests/test_checksum32.py +154 -0
- numcodecs/tests/test_compat.py +111 -0
- numcodecs/tests/test_delta.py +61 -0
- numcodecs/tests/test_entrypoints.py +24 -0
- numcodecs/tests/test_entrypoints_backport.py +36 -0
- numcodecs/tests/test_fixedscaleoffset.py +77 -0
- numcodecs/tests/test_fletcher32.py +56 -0
- numcodecs/tests/test_gzip.py +110 -0
- numcodecs/tests/test_jenkins.py +150 -0
- numcodecs/tests/test_json.py +85 -0
- numcodecs/tests/test_lz4.py +83 -0
- numcodecs/tests/test_lzma.py +94 -0
- numcodecs/tests/test_msgpacks.py +126 -0
- numcodecs/tests/test_ndarray_like.py +48 -0
- numcodecs/tests/test_packbits.py +39 -0
- numcodecs/tests/test_pcodec.py +90 -0
- numcodecs/tests/test_pickles.py +61 -0
- numcodecs/tests/test_quantize.py +76 -0
- numcodecs/tests/test_registry.py +43 -0
- numcodecs/tests/test_shuffle.py +166 -0
- numcodecs/tests/test_vlen_array.py +97 -0
- numcodecs/tests/test_vlen_bytes.py +97 -0
- numcodecs/tests/test_vlen_utf8.py +91 -0
- numcodecs/tests/test_zarr3.py +279 -0
- numcodecs/tests/test_zarr3_import.py +13 -0
- numcodecs/tests/test_zfpy.py +104 -0
- numcodecs/tests/test_zlib.py +94 -0
- numcodecs/tests/test_zstd.py +92 -0
- numcodecs/version.py +21 -0
- numcodecs/vlen.cpython-312-aarch64-linux-gnu.so +0 -0
- numcodecs/zarr3.py +401 -0
- numcodecs/zfpy.py +113 -0
- numcodecs/zlib.py +42 -0
- numcodecs/zstd.cpython-312-aarch64-linux-gnu.so +0 -0
- numcodecs-0.16.0.dist-info/METADATA +66 -0
- numcodecs-0.16.0.dist-info/RECORD +79 -0
- numcodecs-0.16.0.dist-info/WHEEL +6 -0
- numcodecs-0.16.0.dist-info/entry_points.txt +22 -0
- numcodecs-0.16.0.dist-info/licenses/LICENSE.txt +21 -0
- numcodecs-0.16.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pytest
|
|
3
|
+
from numpy.testing import assert_array_equal
|
|
4
|
+
|
|
5
|
+
from numcodecs.delta import Delta
|
|
6
|
+
from numcodecs.tests.common import (
|
|
7
|
+
check_backwards_compatibility,
|
|
8
|
+
check_config,
|
|
9
|
+
check_encode_decode,
|
|
10
|
+
check_repr,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
# mix of dtypes: integer, float
|
|
14
|
+
# mix of shapes: 1D, 2D, 3D
|
|
15
|
+
# mix of orders: C, F
|
|
16
|
+
arrays = [
|
|
17
|
+
np.random.randint(0, 1, size=110, dtype='?').reshape(10, 11),
|
|
18
|
+
np.arange(1000, dtype='<i4'),
|
|
19
|
+
np.linspace(1000, 1001, 1000, dtype='<f4').reshape(100, 10),
|
|
20
|
+
np.random.normal(loc=1000, scale=1, size=(10, 10, 10)).astype('<f8'),
|
|
21
|
+
np.random.randint(0, 200, size=1000, dtype='u2').astype('<u2').reshape(100, 10, order='F'),
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_encode_decode():
|
|
26
|
+
for arr in arrays:
|
|
27
|
+
codec = Delta(dtype=arr.dtype)
|
|
28
|
+
check_encode_decode(arr, codec)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_encode():
|
|
32
|
+
dtype = 'i8'
|
|
33
|
+
astype = 'i4'
|
|
34
|
+
codec = Delta(dtype=dtype, astype=astype)
|
|
35
|
+
arr = np.arange(10, 20, 1, dtype=dtype)
|
|
36
|
+
expect = np.array([10] + ([1] * 9), dtype=astype)
|
|
37
|
+
actual = codec.encode(arr)
|
|
38
|
+
assert_array_equal(expect, actual)
|
|
39
|
+
assert np.dtype(astype) == actual.dtype
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_config():
|
|
43
|
+
codec = Delta(dtype='<i4', astype='<i2')
|
|
44
|
+
check_config(codec)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_repr():
|
|
48
|
+
check_repr("Delta(dtype='<i4', astype='<i2')")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_backwards_compatibility():
|
|
52
|
+
for arr in arrays:
|
|
53
|
+
codec = Delta(dtype=arr.dtype)
|
|
54
|
+
check_backwards_compatibility(Delta.codec_id, [arr], [codec], prefix=str(arr.dtype))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_errors():
|
|
58
|
+
with pytest.raises(ValueError):
|
|
59
|
+
Delta(dtype=object)
|
|
60
|
+
with pytest.raises(ValueError):
|
|
61
|
+
Delta(dtype='i8', astype=object)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import os.path
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
import numcodecs.registry
|
|
7
|
+
|
|
8
|
+
here = os.path.abspath(os.path.dirname(__file__))
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@pytest.fixture
|
|
12
|
+
def set_path():
|
|
13
|
+
sys.path.append(here)
|
|
14
|
+
numcodecs.registry.run_entrypoints()
|
|
15
|
+
yield
|
|
16
|
+
sys.path.remove(here)
|
|
17
|
+
numcodecs.registry.run_entrypoints()
|
|
18
|
+
numcodecs.registry.codec_registry.pop("test")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@pytest.mark.usefixtures("set_path")
|
|
22
|
+
def test_entrypoint_codec():
|
|
23
|
+
cls = numcodecs.registry.get_codec({"id": "test"})
|
|
24
|
+
assert cls.codec_id == "test"
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import importlib.util
|
|
2
|
+
import os.path
|
|
3
|
+
import sys
|
|
4
|
+
from multiprocessing import Process
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
import numcodecs.registry
|
|
9
|
+
|
|
10
|
+
importlib_spec = importlib.util.find_spec("importlib_metadata")
|
|
11
|
+
if importlib_spec is None or importlib_spec.loader is None: # pragma: no cover
|
|
12
|
+
pytest.skip(
|
|
13
|
+
"This test module requires importlib_metadata to be installed",
|
|
14
|
+
allow_module_level=True,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
here = os.path.abspath(os.path.dirname(__file__))
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def get_entrypoints_with_importlib_metadata_loaded():
|
|
21
|
+
# importlib_metadata patches importlib.metadata, which can lead to breaking changes
|
|
22
|
+
# to the APIs of EntryPoint objects used when registering entrypoints. Attempt to
|
|
23
|
+
# isolate those changes to just this test.
|
|
24
|
+
import importlib_metadata # noqa: F401
|
|
25
|
+
|
|
26
|
+
sys.path.append(here)
|
|
27
|
+
numcodecs.registry.run_entrypoints()
|
|
28
|
+
cls = numcodecs.registry.get_codec({"id": "test"})
|
|
29
|
+
assert cls.codec_id == "test"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_entrypoint_codec_with_importlib_metadata():
|
|
33
|
+
p = Process(target=get_entrypoints_with_importlib_metadata_loaded)
|
|
34
|
+
p.start()
|
|
35
|
+
p.join()
|
|
36
|
+
assert p.exitcode == 0
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pytest
|
|
5
|
+
from numpy.testing import assert_array_equal
|
|
6
|
+
|
|
7
|
+
from numcodecs.fixedscaleoffset import FixedScaleOffset
|
|
8
|
+
from numcodecs.tests.common import (
|
|
9
|
+
check_backwards_compatibility,
|
|
10
|
+
check_config,
|
|
11
|
+
check_encode_decode,
|
|
12
|
+
check_repr,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
arrays = [
|
|
16
|
+
np.linspace(1000, 1001, 1000, dtype='<f8'),
|
|
17
|
+
np.random.normal(loc=1000, scale=1, size=1000).astype('<f8'),
|
|
18
|
+
np.linspace(1000, 1001, 1000, dtype='<f8').reshape(100, 10),
|
|
19
|
+
np.linspace(1000, 1001, 1000, dtype='<f8').reshape(100, 10, order='F'),
|
|
20
|
+
np.linspace(1000, 1001, 1000, dtype='<f8').reshape(10, 10, 10),
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
codecs = [
|
|
25
|
+
FixedScaleOffset(offset=1000, scale=10, dtype='<f8', astype='<i1'),
|
|
26
|
+
FixedScaleOffset(offset=1000, scale=10**2, dtype='<f8', astype='<i2'),
|
|
27
|
+
FixedScaleOffset(offset=1000, scale=10**6, dtype='<f8', astype='<i4'),
|
|
28
|
+
FixedScaleOffset(offset=1000, scale=10**12, dtype='<f8', astype='<i8'),
|
|
29
|
+
FixedScaleOffset(offset=1000, scale=10**12, dtype='<f8'),
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_encode_decode():
|
|
34
|
+
for arr, codec in itertools.product(arrays, codecs):
|
|
35
|
+
precision = int(np.log10(codec.scale))
|
|
36
|
+
check_encode_decode(arr, codec, precision=precision)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@pytest.mark.parametrize(
|
|
40
|
+
("offset", "scale", "expected"),
|
|
41
|
+
[
|
|
42
|
+
(1000, 10, [0, 6, 11, 17, 22, 28, 33, 39, 44, 50]),
|
|
43
|
+
(1002.5, 10, [-25, -19, -14, -8, -3, 3, 8, 14, 19, 25]),
|
|
44
|
+
(1000, 0.5, [0, 0, 1, 1, 1, 1, 2, 2, 2, 2]),
|
|
45
|
+
],
|
|
46
|
+
)
|
|
47
|
+
def test_encode(offset: float, scale: float, expected: list[int]):
|
|
48
|
+
dtype = '<f8'
|
|
49
|
+
astype = np.int16
|
|
50
|
+
codec = FixedScaleOffset(scale=scale, offset=offset, dtype=dtype, astype=astype)
|
|
51
|
+
arr = np.linspace(1000, 1005, 10, dtype=dtype)
|
|
52
|
+
expect = np.array(expected, dtype=astype)
|
|
53
|
+
actual = codec.encode(arr)
|
|
54
|
+
assert_array_equal(expect, actual)
|
|
55
|
+
assert np.dtype(astype) == actual.dtype
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def test_config():
|
|
59
|
+
codec = FixedScaleOffset(dtype='<f8', astype='<i4', scale=10, offset=100)
|
|
60
|
+
check_config(codec)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_repr():
|
|
64
|
+
stmt = "FixedScaleOffset(scale=10, offset=100, dtype='<f8', astype='<i4')"
|
|
65
|
+
check_repr(stmt)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_backwards_compatibility():
|
|
69
|
+
precision = [int(np.log10(codec.scale)) for codec in codecs]
|
|
70
|
+
check_backwards_compatibility(FixedScaleOffset.codec_id, arrays, codecs, precision=precision)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_errors():
|
|
74
|
+
with pytest.raises(ValueError):
|
|
75
|
+
FixedScaleOffset(dtype=object, astype='i4', scale=10, offset=100)
|
|
76
|
+
with pytest.raises(ValueError):
|
|
77
|
+
FixedScaleOffset(dtype='f8', astype=object, scale=10, offset=100)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
from numcodecs.fletcher32 import Fletcher32
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@pytest.mark.parametrize("dtype", ["uint8", "int32", "float32"])
|
|
8
|
+
def test_with_data(dtype):
|
|
9
|
+
data = np.arange(100, dtype=dtype)
|
|
10
|
+
f = Fletcher32()
|
|
11
|
+
arr = np.frombuffer(f.decode(f.encode(data)), dtype=dtype)
|
|
12
|
+
assert (arr == data).all()
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_error():
|
|
16
|
+
data = np.arange(100)
|
|
17
|
+
f = Fletcher32()
|
|
18
|
+
enc = f.encode(data)
|
|
19
|
+
enc2 = bytearray(enc)
|
|
20
|
+
enc2[0] += 1
|
|
21
|
+
with pytest.raises(RuntimeError) as e:
|
|
22
|
+
f.decode(enc2)
|
|
23
|
+
assert "fletcher32 checksum" in str(e.value)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_known():
|
|
27
|
+
data = (
|
|
28
|
+
b'w\x07\x00\x00\x00\x00\x00\x00\x85\xf6\xff\xff\xff\xff\xff\xff'
|
|
29
|
+
b'i\x07\x00\x00\x00\x00\x00\x00\x94\xf6\xff\xff\xff\xff\xff\xff'
|
|
30
|
+
b'\x88\t\x00\x00\x00\x00\x00\x00i\x03\x00\x00\x00\x00\x00\x00'
|
|
31
|
+
b'\x93\xfd\xff\xff\xff\xff\xff\xff\xc3\xfc\xff\xff\xff\xff\xff\xff'
|
|
32
|
+
b"'\x02\x00\x00\x00\x00\x00\x00\xba\xf7\xff\xff\xff\xff\xff\xff"
|
|
33
|
+
b'\xfd%\x86d'
|
|
34
|
+
)
|
|
35
|
+
data3 = Fletcher32().decode(data)
|
|
36
|
+
outarr = np.frombuffer(data3, dtype="<i8")
|
|
37
|
+
expected = [
|
|
38
|
+
1911,
|
|
39
|
+
-2427,
|
|
40
|
+
1897,
|
|
41
|
+
-2412,
|
|
42
|
+
2440,
|
|
43
|
+
873,
|
|
44
|
+
-621,
|
|
45
|
+
-829,
|
|
46
|
+
551,
|
|
47
|
+
-2118,
|
|
48
|
+
]
|
|
49
|
+
assert outarr.tolist() == expected
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_out():
|
|
53
|
+
data = np.frombuffer(bytearray(b"Hello World"), dtype="uint8")
|
|
54
|
+
f = Fletcher32()
|
|
55
|
+
result = f.encode(data)
|
|
56
|
+
f.decode(result, out=data)
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from numcodecs.gzip import GZip
|
|
7
|
+
from numcodecs.tests.common import (
|
|
8
|
+
check_backwards_compatibility,
|
|
9
|
+
check_config,
|
|
10
|
+
check_encode_decode,
|
|
11
|
+
check_err_decode_object_buffer,
|
|
12
|
+
check_err_encode_object_buffer,
|
|
13
|
+
check_repr,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
codecs = [
|
|
17
|
+
GZip(),
|
|
18
|
+
GZip(level=-1),
|
|
19
|
+
GZip(level=0),
|
|
20
|
+
GZip(level=1),
|
|
21
|
+
GZip(level=5),
|
|
22
|
+
GZip(level=9),
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# mix of dtypes: integer, float, bool, string
|
|
27
|
+
# mix of shapes: 1D, 2D, 3D
|
|
28
|
+
# mix of orders: C, F
|
|
29
|
+
arrays = [
|
|
30
|
+
np.arange(1000, dtype='i4'),
|
|
31
|
+
np.linspace(1000, 1001, 1000, dtype='f8'),
|
|
32
|
+
np.random.normal(loc=1000, scale=1, size=(100, 10)),
|
|
33
|
+
np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
|
|
34
|
+
np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
|
|
35
|
+
np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
|
|
36
|
+
np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
|
|
37
|
+
np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
|
|
38
|
+
np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
|
|
39
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
|
|
40
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
|
|
41
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
|
|
42
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_encode_decode():
|
|
47
|
+
for arr, codec in itertools.product(arrays, codecs):
|
|
48
|
+
check_encode_decode(arr, codec)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_config():
|
|
52
|
+
codec = GZip(level=3)
|
|
53
|
+
check_config(codec)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_repr():
|
|
57
|
+
check_repr("GZip(level=3)")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_eq():
|
|
61
|
+
assert GZip() == GZip()
|
|
62
|
+
assert not GZip() != GZip()
|
|
63
|
+
assert GZip(1) == GZip(1)
|
|
64
|
+
assert GZip(1) != GZip(9)
|
|
65
|
+
assert GZip() != 'foo'
|
|
66
|
+
assert 'foo' != GZip()
|
|
67
|
+
assert not GZip() == 'foo'
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_backwards_compatibility():
|
|
71
|
+
check_backwards_compatibility(GZip.codec_id, arrays, codecs)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_err_decode_object_buffer():
|
|
75
|
+
check_err_decode_object_buffer(GZip())
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_err_encode_object_buffer():
|
|
79
|
+
check_err_encode_object_buffer(GZip())
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_err_encode_list():
|
|
83
|
+
data = ['foo', 'bar', 'baz']
|
|
84
|
+
for codec in codecs:
|
|
85
|
+
with pytest.raises(TypeError):
|
|
86
|
+
codec.encode(data)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_err_encode_non_contiguous():
|
|
90
|
+
# non-contiguous memory
|
|
91
|
+
arr = np.arange(1000, dtype='i4')[::2]
|
|
92
|
+
for codec in codecs:
|
|
93
|
+
with pytest.raises(ValueError):
|
|
94
|
+
codec.encode(arr)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_err_out_too_small():
|
|
98
|
+
arr = np.arange(10, dtype='i4')
|
|
99
|
+
out = np.empty_like(arr)[:-1]
|
|
100
|
+
for codec in codecs:
|
|
101
|
+
with pytest.raises(ValueError):
|
|
102
|
+
codec.decode(codec.encode(arr), out)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_out_too_large():
|
|
106
|
+
out = np.empty((10,), dtype='i4')
|
|
107
|
+
arr = out[:-1]
|
|
108
|
+
arr[:] = 5
|
|
109
|
+
for codec in codecs:
|
|
110
|
+
codec.decode(codec.encode(arr), out)
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
from numcodecs.checksum32 import JenkinsLookup3
|
|
5
|
+
from numcodecs.jenkins import jenkins_lookup3
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_jenkins_lookup3():
|
|
9
|
+
h = jenkins_lookup3(b"", 0)
|
|
10
|
+
assert h == 0xDEADBEEF
|
|
11
|
+
h = jenkins_lookup3(b"", 0xDEADBEEF)
|
|
12
|
+
assert h == 0xBD5B7DDE
|
|
13
|
+
h = jenkins_lookup3(b"Four score and seven years ago", 0)
|
|
14
|
+
assert h == 0x17770551
|
|
15
|
+
h = jenkins_lookup3(b"Four score and seven years ago", 1)
|
|
16
|
+
assert h == 0xCD628161
|
|
17
|
+
|
|
18
|
+
# jenkins-cffi example
|
|
19
|
+
h = jenkins_lookup3(b"jenkins", 0)
|
|
20
|
+
assert h == 202276345
|
|
21
|
+
|
|
22
|
+
h_last = [0]
|
|
23
|
+
|
|
24
|
+
h = jenkins_lookup3(b"", h_last[-1])
|
|
25
|
+
assert h not in h_last
|
|
26
|
+
h_last.append(h)
|
|
27
|
+
|
|
28
|
+
h = jenkins_lookup3(b"", h_last[-1])
|
|
29
|
+
assert h not in h_last
|
|
30
|
+
h_last.append(h)
|
|
31
|
+
|
|
32
|
+
h = jenkins_lookup3(b"", h_last[-1])
|
|
33
|
+
assert h not in h_last
|
|
34
|
+
h_last.append(h)
|
|
35
|
+
|
|
36
|
+
h = jenkins_lookup3(b"", h_last[-1])
|
|
37
|
+
assert h not in h_last
|
|
38
|
+
h_last.append(h)
|
|
39
|
+
|
|
40
|
+
h = jenkins_lookup3(b"", h_last[-1])
|
|
41
|
+
assert h not in h_last
|
|
42
|
+
h_last.append(h)
|
|
43
|
+
|
|
44
|
+
h = jenkins_lookup3(b"", h_last[-1])
|
|
45
|
+
assert h not in h_last
|
|
46
|
+
h_last.append(h)
|
|
47
|
+
|
|
48
|
+
h = jenkins_lookup3(b"", h_last[-1])
|
|
49
|
+
assert h not in h_last
|
|
50
|
+
h_last.append(h)
|
|
51
|
+
|
|
52
|
+
h = jenkins_lookup3(b"", h_last[-1])
|
|
53
|
+
assert h not in h_last
|
|
54
|
+
h_last.append(h)
|
|
55
|
+
|
|
56
|
+
h = jenkins_lookup3(b"", h_last[-1])
|
|
57
|
+
assert h not in h_last
|
|
58
|
+
h_last.append(h)
|
|
59
|
+
|
|
60
|
+
a = np.frombuffer(b"Four score and seven years ago", dtype="uint8")
|
|
61
|
+
h = jenkins_lookup3(a, 0)
|
|
62
|
+
assert h == 0x17770551
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_jenkins_lookup3_codec():
|
|
66
|
+
s = b"Four score and seven years ago"
|
|
67
|
+
j = JenkinsLookup3()
|
|
68
|
+
result = j.encode(s)
|
|
69
|
+
assert result[-4:] == b'\x51\x05\x77\x17'
|
|
70
|
+
assert bytes(j.decode(result)) == s
|
|
71
|
+
|
|
72
|
+
j = JenkinsLookup3(initval=0xDEADBEEF)
|
|
73
|
+
result = j.encode(s)
|
|
74
|
+
assert bytes(j.decode(result)) == s
|
|
75
|
+
|
|
76
|
+
j = JenkinsLookup3(initval=1230)
|
|
77
|
+
result = j.encode(s)
|
|
78
|
+
assert result[-4:] == b'\xd7Z\xe2\x0e'
|
|
79
|
+
assert bytes(j.decode(result)) == s
|
|
80
|
+
|
|
81
|
+
j = JenkinsLookup3(initval=1230, prefix=b"Hello world")
|
|
82
|
+
result = j.encode(s)
|
|
83
|
+
assert bytes(j.decode(result)) == s
|
|
84
|
+
|
|
85
|
+
chunk_index = (
|
|
86
|
+
b"\x00\x08\x00\x00\x00\x00\x00\x00"
|
|
87
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
88
|
+
b"\xf7\x17\x00\x00\x00\x00\x00\x00"
|
|
89
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
90
|
+
b"\xee'\x00\x00\x00\x00\x00\x00"
|
|
91
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
92
|
+
b"\xe57\x00\x00\x00\x00\x00\x00"
|
|
93
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
94
|
+
b"\xdcG\x00\x00\x00\x00\x00\x00"
|
|
95
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
96
|
+
b"\xd3W\x00\x00\x00\x00\x00\x00"
|
|
97
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
98
|
+
b"\xcag\x00\x00\x00\x00\x00\x00"
|
|
99
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
100
|
+
b"\xc1w\x00\x00\x00\x00\x00\x00"
|
|
101
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
102
|
+
b"\xb8\x87\x00\x00\x00\x00\x00\x00"
|
|
103
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
104
|
+
b"\xaf\x97\x00\x00\x00\x00\x00\x00"
|
|
105
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
106
|
+
b"\xa6\xa7\x00\x00\x00\x00\x00\x00"
|
|
107
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
108
|
+
b"\x9d\xb7\x00\x00\x00\x00\x00\x00"
|
|
109
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
110
|
+
b"\x94\xc7\x00\x00\x00\x00\x00\x00"
|
|
111
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
112
|
+
b"\x8b\xd7\x00\x00\x00\x00\x00\x00"
|
|
113
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
114
|
+
b"\x82\xe7\x00\x00\x00\x00\x00\x00"
|
|
115
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
116
|
+
b"y\xf7\x00\x00\x00\x00\x00\x00"
|
|
117
|
+
b"\xf7\x0f\x00\x00\x00\x00\x00\x00"
|
|
118
|
+
b"n\x96\x07\x85"
|
|
119
|
+
)
|
|
120
|
+
hdf5_fadb_prefix = b'FADB\x00\x01\xcf\x01\x00\x00\x00\x00\x00\x00'
|
|
121
|
+
j = JenkinsLookup3(prefix=hdf5_fadb_prefix)
|
|
122
|
+
result = j.encode(chunk_index[:-4])
|
|
123
|
+
j.decode(result)
|
|
124
|
+
assert result == chunk_index
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@pytest.mark.parametrize("dtype", ["uint8", "int32", "float32"])
|
|
128
|
+
def test_with_data(dtype):
|
|
129
|
+
data = np.arange(100, dtype=dtype)
|
|
130
|
+
j = JenkinsLookup3()
|
|
131
|
+
arr = np.frombuffer(j.decode(j.encode(data)), dtype=dtype)
|
|
132
|
+
assert (arr == data).all()
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def test_error():
|
|
136
|
+
data = np.arange(100)
|
|
137
|
+
j = JenkinsLookup3()
|
|
138
|
+
enc = j.encode(data)
|
|
139
|
+
enc2 = bytearray(enc)
|
|
140
|
+
enc2[0] += 1
|
|
141
|
+
with pytest.raises(RuntimeError) as e:
|
|
142
|
+
j.decode(enc2)
|
|
143
|
+
assert "Bob Jenkin's lookup3 checksum" in str(e.value)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def test_out():
|
|
147
|
+
data = np.frombuffer(bytearray(b"Hello World"), dtype="uint8")
|
|
148
|
+
j = JenkinsLookup3()
|
|
149
|
+
result = j.encode(data)
|
|
150
|
+
j.decode(result, out=data)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from numcodecs.json import JSON
|
|
7
|
+
from numcodecs.tests.common import (
|
|
8
|
+
check_backwards_compatibility,
|
|
9
|
+
check_config,
|
|
10
|
+
check_encode_decode_array,
|
|
11
|
+
check_repr,
|
|
12
|
+
greetings,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
codecs = [
|
|
16
|
+
JSON(),
|
|
17
|
+
JSON(indent=True),
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# object array with strings
|
|
22
|
+
# object array with mix strings / nans
|
|
23
|
+
# object array with mix of string, int, float
|
|
24
|
+
# ...
|
|
25
|
+
arrays = [
|
|
26
|
+
np.array(['foo', 'bar', 'baz'] * 300, dtype=object),
|
|
27
|
+
np.array([['foo', 'bar', np.nan]] * 300, dtype=object),
|
|
28
|
+
np.array(['foo', 1.0, 2] * 300, dtype=object),
|
|
29
|
+
np.arange(1000, dtype='i4'),
|
|
30
|
+
np.array(['foo', 'bar', 'baz'] * 300),
|
|
31
|
+
np.array(['foo', ['bar', 1.0, 2], {'a': 'b', 'c': 42}] * 300, dtype=object),
|
|
32
|
+
np.array(greetings * 100),
|
|
33
|
+
np.array(greetings * 100, dtype=object),
|
|
34
|
+
np.array([[0, 1], [2, 3]], dtype=object),
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_encode_decode():
|
|
39
|
+
for arr, codec in itertools.product(arrays, codecs):
|
|
40
|
+
check_encode_decode_array(arr, codec)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_config():
|
|
44
|
+
for codec in codecs:
|
|
45
|
+
check_config(codec)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_repr():
|
|
49
|
+
r = (
|
|
50
|
+
"JSON(encoding='utf-8', allow_nan=True, check_circular=True, ensure_ascii=True,\n"
|
|
51
|
+
" indent=None, separators=(',', ':'), skipkeys=False, sort_keys=True,\n"
|
|
52
|
+
" strict=True)"
|
|
53
|
+
)
|
|
54
|
+
check_repr(r)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_backwards_compatibility():
|
|
58
|
+
check_backwards_compatibility(JSON.codec_id, arrays, codecs)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@pytest.mark.parametrize(
|
|
62
|
+
('input_data', 'dtype'),
|
|
63
|
+
[
|
|
64
|
+
([0, 1], None),
|
|
65
|
+
([[0, 1], [2, 3]], None),
|
|
66
|
+
([[0], [1], [2, 3]], object),
|
|
67
|
+
([[[0, 0]], [[1, 1]], [[2, 3]]], None),
|
|
68
|
+
(["1"], None),
|
|
69
|
+
(["11", "11"], None),
|
|
70
|
+
(["11", "1", "1"], None),
|
|
71
|
+
([{}], None),
|
|
72
|
+
([{"key": "value"}, ["list", "of", "strings"]], object),
|
|
73
|
+
([0], None),
|
|
74
|
+
([{'hi': 0}], "object"),
|
|
75
|
+
(["hi"], "object"),
|
|
76
|
+
(0, None),
|
|
77
|
+
],
|
|
78
|
+
)
|
|
79
|
+
def test_non_numpy_inputs(input_data, dtype):
|
|
80
|
+
# numpy will infer a range of different shapes and dtypes for these inputs.
|
|
81
|
+
# Make sure that round-tripping through encode preserves this.
|
|
82
|
+
data = np.array(input_data, dtype=dtype)
|
|
83
|
+
for codec in codecs:
|
|
84
|
+
output_data = codec.decode(codec.encode(data))
|
|
85
|
+
assert input_data == output_data.tolist()
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
from numcodecs.lz4 import LZ4
|
|
8
|
+
except ImportError: # pragma: no cover
|
|
9
|
+
pytest.skip("numcodecs.lz4 not available", allow_module_level=True)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
from numcodecs.tests.common import (
|
|
13
|
+
check_backwards_compatibility,
|
|
14
|
+
check_config,
|
|
15
|
+
check_encode_decode,
|
|
16
|
+
check_err_decode_object_buffer,
|
|
17
|
+
check_err_encode_object_buffer,
|
|
18
|
+
check_max_buffer_size,
|
|
19
|
+
check_repr,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
codecs = [
|
|
23
|
+
LZ4(),
|
|
24
|
+
LZ4(acceleration=-1),
|
|
25
|
+
LZ4(acceleration=0),
|
|
26
|
+
LZ4(acceleration=1),
|
|
27
|
+
LZ4(acceleration=10),
|
|
28
|
+
LZ4(acceleration=100),
|
|
29
|
+
LZ4(acceleration=1000000),
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# mix of dtypes: integer, float, bool, string
|
|
34
|
+
# mix of shapes: 1D, 2D, 3D
|
|
35
|
+
# mix of orders: C, F
|
|
36
|
+
arrays = [
|
|
37
|
+
np.arange(1000, dtype='i4'),
|
|
38
|
+
np.linspace(1000, 1001, 1000, dtype='f8'),
|
|
39
|
+
np.random.normal(loc=1000, scale=1, size=(100, 10)),
|
|
40
|
+
np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
|
|
41
|
+
np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
|
|
42
|
+
np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
|
|
43
|
+
np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
|
|
44
|
+
np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
|
|
45
|
+
np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
|
|
46
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
|
|
47
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
|
|
48
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
|
|
49
|
+
np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_encode_decode():
|
|
54
|
+
for arr, codec in itertools.product(arrays, codecs):
|
|
55
|
+
check_encode_decode(arr, codec)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def test_config():
|
|
59
|
+
for codec in codecs:
|
|
60
|
+
check_config(codec)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_repr():
|
|
64
|
+
check_repr("LZ4(acceleration=1)")
|
|
65
|
+
check_repr("LZ4(acceleration=100)")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_backwards_compatibility():
|
|
69
|
+
check_backwards_compatibility(LZ4.codec_id, arrays, codecs)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_err_decode_object_buffer():
|
|
73
|
+
check_err_decode_object_buffer(LZ4())
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_err_encode_object_buffer():
|
|
77
|
+
check_err_encode_object_buffer(LZ4())
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_max_buffer_size():
|
|
81
|
+
for codec in codecs:
|
|
82
|
+
assert codec.max_buffer_size == 0x7E000000
|
|
83
|
+
check_max_buffer_size(codec)
|