numcodecs 0.16.0__cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of numcodecs might be problematic. Click here for more details.
- numcodecs/__init__.py +146 -0
- numcodecs/_shuffle.cpython-313-aarch64-linux-gnu.so +0 -0
- numcodecs/abc.py +127 -0
- numcodecs/astype.py +72 -0
- numcodecs/base64.py +26 -0
- numcodecs/bitround.py +80 -0
- numcodecs/blosc.cpython-313-aarch64-linux-gnu.so +0 -0
- numcodecs/bz2.py +45 -0
- numcodecs/categorize.py +98 -0
- numcodecs/checksum32.py +183 -0
- numcodecs/compat.py +206 -0
- numcodecs/compat_ext.cpython-313-aarch64-linux-gnu.so +0 -0
- numcodecs/delta.py +94 -0
- numcodecs/errors.py +26 -0
- numcodecs/fixedscaleoffset.py +130 -0
- numcodecs/fletcher32.cpython-313-aarch64-linux-gnu.so +0 -0
- numcodecs/gzip.py +50 -0
- numcodecs/jenkins.cpython-313-aarch64-linux-gnu.so +0 -0
- numcodecs/json.py +107 -0
- numcodecs/lz4.cpython-313-aarch64-linux-gnu.so +0 -0
- numcodecs/lzma.py +72 -0
- numcodecs/msgpacks.py +86 -0
- numcodecs/ndarray_like.py +65 -0
- numcodecs/packbits.py +82 -0
- numcodecs/pcodec.py +118 -0
- numcodecs/pickles.py +55 -0
- numcodecs/quantize.py +98 -0
- numcodecs/registry.py +74 -0
- numcodecs/shuffle.py +61 -0
- numcodecs/tests/__init__.py +3 -0
- numcodecs/tests/common.py +285 -0
- numcodecs/tests/package_with_entrypoint/__init__.py +11 -0
- numcodecs/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt +2 -0
- numcodecs/tests/test_astype.py +74 -0
- numcodecs/tests/test_base64.py +81 -0
- numcodecs/tests/test_bitround.py +81 -0
- numcodecs/tests/test_blosc.py +284 -0
- numcodecs/tests/test_bz2.py +66 -0
- numcodecs/tests/test_categorize.py +87 -0
- numcodecs/tests/test_checksum32.py +154 -0
- numcodecs/tests/test_compat.py +111 -0
- numcodecs/tests/test_delta.py +61 -0
- numcodecs/tests/test_entrypoints.py +24 -0
- numcodecs/tests/test_entrypoints_backport.py +36 -0
- numcodecs/tests/test_fixedscaleoffset.py +77 -0
- numcodecs/tests/test_fletcher32.py +56 -0
- numcodecs/tests/test_gzip.py +110 -0
- numcodecs/tests/test_jenkins.py +150 -0
- numcodecs/tests/test_json.py +85 -0
- numcodecs/tests/test_lz4.py +83 -0
- numcodecs/tests/test_lzma.py +94 -0
- numcodecs/tests/test_msgpacks.py +126 -0
- numcodecs/tests/test_ndarray_like.py +48 -0
- numcodecs/tests/test_packbits.py +39 -0
- numcodecs/tests/test_pcodec.py +90 -0
- numcodecs/tests/test_pickles.py +61 -0
- numcodecs/tests/test_quantize.py +76 -0
- numcodecs/tests/test_registry.py +43 -0
- numcodecs/tests/test_shuffle.py +166 -0
- numcodecs/tests/test_vlen_array.py +97 -0
- numcodecs/tests/test_vlen_bytes.py +97 -0
- numcodecs/tests/test_vlen_utf8.py +91 -0
- numcodecs/tests/test_zarr3.py +279 -0
- numcodecs/tests/test_zarr3_import.py +13 -0
- numcodecs/tests/test_zfpy.py +104 -0
- numcodecs/tests/test_zlib.py +94 -0
- numcodecs/tests/test_zstd.py +92 -0
- numcodecs/version.py +21 -0
- numcodecs/vlen.cpython-313-aarch64-linux-gnu.so +0 -0
- numcodecs/zarr3.py +401 -0
- numcodecs/zfpy.py +113 -0
- numcodecs/zlib.py +42 -0
- numcodecs/zstd.cpython-313-aarch64-linux-gnu.so +0 -0
- numcodecs-0.16.0.dist-info/METADATA +66 -0
- numcodecs-0.16.0.dist-info/RECORD +79 -0
- numcodecs-0.16.0.dist-info/WHEEL +6 -0
- numcodecs-0.16.0.dist-info/entry_points.txt +22 -0
- numcodecs-0.16.0.dist-info/licenses/LICENSE.txt +21 -0
- numcodecs-0.16.0.dist-info/top_level.txt +1 -0
numcodecs/zarr3.py
ADDED
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module provides the compatibility for :py:mod:`numcodecs` in Zarr version 3.
|
|
3
|
+
|
|
4
|
+
A compatibility module is required because the codec handling in Zarr version 3 is different from Zarr version 2.
|
|
5
|
+
|
|
6
|
+
You can use codecs from :py:mod:`numcodecs` by constructing codecs from :py:mod:`numcodecs.zarr3` using the same parameters as the original codecs.
|
|
7
|
+
|
|
8
|
+
>>> import zarr
|
|
9
|
+
>>> import numcodecs.zarr3
|
|
10
|
+
>>>
|
|
11
|
+
>>> array = zarr.create_array(
|
|
12
|
+
... store="data.zarr",
|
|
13
|
+
... shape=(1024, 1024),
|
|
14
|
+
... chunks=(64, 64),
|
|
15
|
+
... dtype="uint32",
|
|
16
|
+
... filters=[numcodecs.zarr3.Delta()],
|
|
17
|
+
... compressors=[numcodecs.zarr3.BZ2(level=5)])
|
|
18
|
+
>>> array[:] = np.arange(*array.shape).astype(array.dtype)
|
|
19
|
+
|
|
20
|
+
.. note::
|
|
21
|
+
|
|
22
|
+
Please note that the codecs in :py:mod:`numcodecs.zarr3` are not part of the Zarr version 3 specification.
|
|
23
|
+
Using these codecs might cause interoperability issues with other Zarr implementations.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import asyncio
|
|
29
|
+
import math
|
|
30
|
+
from dataclasses import dataclass, replace
|
|
31
|
+
from functools import cached_property, partial
|
|
32
|
+
from typing import Any, Self, TypeVar
|
|
33
|
+
from warnings import warn
|
|
34
|
+
|
|
35
|
+
import numpy as np
|
|
36
|
+
|
|
37
|
+
import numcodecs
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
import zarr
|
|
41
|
+
|
|
42
|
+
if zarr.__version__ < "3.0.0": # pragma: no cover
|
|
43
|
+
raise ImportError("zarr 3.0.0 or later is required to use the numcodecs zarr integration.")
|
|
44
|
+
except ImportError as e: # pragma: no cover
|
|
45
|
+
raise ImportError(
|
|
46
|
+
"zarr 3.0.0 or later is required to use the numcodecs zarr integration."
|
|
47
|
+
) from e
|
|
48
|
+
|
|
49
|
+
from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec
|
|
50
|
+
from zarr.abc.metadata import Metadata
|
|
51
|
+
from zarr.core.array_spec import ArraySpec
|
|
52
|
+
from zarr.core.buffer import Buffer, BufferPrototype, NDBuffer
|
|
53
|
+
from zarr.core.buffer.cpu import as_numpy_array_wrapper
|
|
54
|
+
from zarr.core.common import JSON, parse_named_configuration, product
|
|
55
|
+
|
|
56
|
+
CODEC_PREFIX = "numcodecs."
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _expect_name_prefix(codec_name: str) -> str:
|
|
60
|
+
if not codec_name.startswith(CODEC_PREFIX):
|
|
61
|
+
raise ValueError(
|
|
62
|
+
f"Expected name to start with '{CODEC_PREFIX}'. Got {codec_name} instead."
|
|
63
|
+
) # pragma: no cover
|
|
64
|
+
return codec_name.removeprefix(CODEC_PREFIX)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _parse_codec_configuration(data: dict[str, JSON]) -> dict[str, JSON]:
|
|
68
|
+
parsed_name, parsed_configuration = parse_named_configuration(data)
|
|
69
|
+
if not parsed_name.startswith(CODEC_PREFIX):
|
|
70
|
+
raise ValueError(
|
|
71
|
+
f"Expected name to start with '{CODEC_PREFIX}'. Got {parsed_name} instead."
|
|
72
|
+
) # pragma: no cover
|
|
73
|
+
id = _expect_name_prefix(parsed_name)
|
|
74
|
+
return {"id": id, **parsed_configuration}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class _NumcodecsCodec(Metadata):
|
|
79
|
+
codec_name: str
|
|
80
|
+
codec_config: dict[str, JSON]
|
|
81
|
+
|
|
82
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
83
|
+
if not self.codec_name:
|
|
84
|
+
raise ValueError(
|
|
85
|
+
"The codec name needs to be supplied through the `codec_name` attribute."
|
|
86
|
+
) # pragma: no cover
|
|
87
|
+
unprefixed_codec_name = _expect_name_prefix(self.codec_name)
|
|
88
|
+
|
|
89
|
+
if "id" not in codec_config:
|
|
90
|
+
codec_config = {"id": unprefixed_codec_name, **codec_config}
|
|
91
|
+
elif codec_config["id"] != unprefixed_codec_name:
|
|
92
|
+
raise ValueError(
|
|
93
|
+
f"Codec id does not match {unprefixed_codec_name}. Got: {codec_config['id']}."
|
|
94
|
+
) # pragma: no cover
|
|
95
|
+
|
|
96
|
+
object.__setattr__(self, "codec_config", codec_config)
|
|
97
|
+
warn(
|
|
98
|
+
"Numcodecs codecs are not in the Zarr version 3 specification and "
|
|
99
|
+
"may not be supported by other zarr implementations.",
|
|
100
|
+
category=UserWarning,
|
|
101
|
+
stacklevel=2,
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
@cached_property
|
|
105
|
+
def _codec(self) -> numcodecs.abc.Codec:
|
|
106
|
+
return numcodecs.get_codec(self.codec_config)
|
|
107
|
+
|
|
108
|
+
@classmethod
|
|
109
|
+
def from_dict(cls, data: dict[str, JSON]) -> Self:
|
|
110
|
+
codec_config = _parse_codec_configuration(data)
|
|
111
|
+
return cls(**codec_config)
|
|
112
|
+
|
|
113
|
+
def to_dict(self) -> dict[str, JSON]:
|
|
114
|
+
codec_config = self.codec_config.copy()
|
|
115
|
+
codec_config.pop("id", None)
|
|
116
|
+
return {
|
|
117
|
+
"name": self.codec_name,
|
|
118
|
+
"configuration": codec_config,
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int:
|
|
122
|
+
raise NotImplementedError # pragma: no cover
|
|
123
|
+
|
|
124
|
+
# Override __repr__ because dynamically constructed classes don't seem to work otherwise
|
|
125
|
+
def __repr__(self) -> str:
|
|
126
|
+
codec_config = self.codec_config.copy()
|
|
127
|
+
codec_config.pop("id", None)
|
|
128
|
+
return f"{self.__class__.__name__}(codec_name={self.codec_name!r}, codec_config={codec_config!r})"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class _NumcodecsBytesBytesCodec(_NumcodecsCodec, BytesBytesCodec):
|
|
132
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
133
|
+
super().__init__(**codec_config)
|
|
134
|
+
|
|
135
|
+
async def _decode_single(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> Buffer:
|
|
136
|
+
return await asyncio.to_thread(
|
|
137
|
+
as_numpy_array_wrapper,
|
|
138
|
+
self._codec.decode,
|
|
139
|
+
chunk_bytes,
|
|
140
|
+
chunk_spec.prototype,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
def _encode(self, chunk_bytes: Buffer, prototype: BufferPrototype) -> Buffer:
|
|
144
|
+
encoded = self._codec.encode(chunk_bytes.as_array_like())
|
|
145
|
+
if isinstance(encoded, np.ndarray): # Required for checksum codecs
|
|
146
|
+
return prototype.buffer.from_bytes(encoded.tobytes())
|
|
147
|
+
return prototype.buffer.from_bytes(encoded)
|
|
148
|
+
|
|
149
|
+
async def _encode_single(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> Buffer:
|
|
150
|
+
return await asyncio.to_thread(self._encode, chunk_bytes, chunk_spec.prototype)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class _NumcodecsArrayArrayCodec(_NumcodecsCodec, ArrayArrayCodec):
|
|
154
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
155
|
+
super().__init__(**codec_config)
|
|
156
|
+
|
|
157
|
+
async def _decode_single(self, chunk_array: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer:
|
|
158
|
+
chunk_ndarray = chunk_array.as_ndarray_like()
|
|
159
|
+
out = await asyncio.to_thread(self._codec.decode, chunk_ndarray)
|
|
160
|
+
return chunk_spec.prototype.nd_buffer.from_ndarray_like(out.reshape(chunk_spec.shape))
|
|
161
|
+
|
|
162
|
+
async def _encode_single(self, chunk_array: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer:
|
|
163
|
+
chunk_ndarray = chunk_array.as_ndarray_like()
|
|
164
|
+
out = await asyncio.to_thread(self._codec.encode, chunk_ndarray)
|
|
165
|
+
return chunk_spec.prototype.nd_buffer.from_ndarray_like(out)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class _NumcodecsArrayBytesCodec(_NumcodecsCodec, ArrayBytesCodec):
|
|
169
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
170
|
+
super().__init__(**codec_config)
|
|
171
|
+
|
|
172
|
+
async def _decode_single(self, chunk_buffer: Buffer, chunk_spec: ArraySpec) -> NDBuffer:
|
|
173
|
+
chunk_bytes = chunk_buffer.to_bytes()
|
|
174
|
+
out = await asyncio.to_thread(self._codec.decode, chunk_bytes)
|
|
175
|
+
return chunk_spec.prototype.nd_buffer.from_ndarray_like(out.reshape(chunk_spec.shape))
|
|
176
|
+
|
|
177
|
+
async def _encode_single(self, chunk_ndbuffer: NDBuffer, chunk_spec: ArraySpec) -> Buffer:
|
|
178
|
+
chunk_ndarray = chunk_ndbuffer.as_ndarray_like()
|
|
179
|
+
out = await asyncio.to_thread(self._codec.encode, chunk_ndarray)
|
|
180
|
+
return chunk_spec.prototype.buffer.from_bytes(out)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
T = TypeVar("T", bound=_NumcodecsCodec)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _add_docstring(cls: type[T], ref_class_name: str) -> type[T]:
|
|
187
|
+
cls.__doc__ = f"""
|
|
188
|
+
See :class:`{ref_class_name}` for more details and parameters.
|
|
189
|
+
"""
|
|
190
|
+
return cls
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _add_docstring_wrapper(ref_class_name: str) -> partial:
|
|
194
|
+
return partial(_add_docstring, ref_class_name=ref_class_name)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _make_bytes_bytes_codec(codec_name: str, cls_name: str) -> type[_NumcodecsBytesBytesCodec]:
|
|
198
|
+
# rename for class scope
|
|
199
|
+
_codec_name = CODEC_PREFIX + codec_name
|
|
200
|
+
|
|
201
|
+
class _Codec(_NumcodecsBytesBytesCodec):
|
|
202
|
+
codec_name = _codec_name
|
|
203
|
+
|
|
204
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
205
|
+
super().__init__(**codec_config)
|
|
206
|
+
|
|
207
|
+
_Codec.__name__ = cls_name
|
|
208
|
+
return _Codec
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _make_array_array_codec(codec_name: str, cls_name: str) -> type[_NumcodecsArrayArrayCodec]:
|
|
212
|
+
# rename for class scope
|
|
213
|
+
_codec_name = CODEC_PREFIX + codec_name
|
|
214
|
+
|
|
215
|
+
class _Codec(_NumcodecsArrayArrayCodec):
|
|
216
|
+
codec_name = _codec_name
|
|
217
|
+
|
|
218
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
219
|
+
super().__init__(**codec_config)
|
|
220
|
+
|
|
221
|
+
_Codec.__name__ = cls_name
|
|
222
|
+
return _Codec
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _make_array_bytes_codec(codec_name: str, cls_name: str) -> type[_NumcodecsArrayBytesCodec]:
|
|
226
|
+
# rename for class scope
|
|
227
|
+
_codec_name = CODEC_PREFIX + codec_name
|
|
228
|
+
|
|
229
|
+
class _Codec(_NumcodecsArrayBytesCodec):
|
|
230
|
+
codec_name = _codec_name
|
|
231
|
+
|
|
232
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
233
|
+
super().__init__(**codec_config)
|
|
234
|
+
|
|
235
|
+
_Codec.__name__ = cls_name
|
|
236
|
+
return _Codec
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _make_checksum_codec(codec_name: str, cls_name: str) -> type[_NumcodecsBytesBytesCodec]:
|
|
240
|
+
# rename for class scope
|
|
241
|
+
_codec_name = CODEC_PREFIX + codec_name
|
|
242
|
+
|
|
243
|
+
class _ChecksumCodec(_NumcodecsBytesBytesCodec):
|
|
244
|
+
codec_name = _codec_name
|
|
245
|
+
|
|
246
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
247
|
+
super().__init__(**codec_config)
|
|
248
|
+
|
|
249
|
+
def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int:
|
|
250
|
+
return input_byte_length + 4 # pragma: no cover
|
|
251
|
+
|
|
252
|
+
_ChecksumCodec.__name__ = cls_name
|
|
253
|
+
return _ChecksumCodec
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
# bytes-to-bytes codecs
|
|
257
|
+
Blosc = _add_docstring(_make_bytes_bytes_codec("blosc", "Blosc"), "numcodecs.blosc.Blosc")
|
|
258
|
+
LZ4 = _add_docstring(_make_bytes_bytes_codec("lz4", "LZ4"), "numcodecs.lz4.LZ4")
|
|
259
|
+
Zstd = _add_docstring(_make_bytes_bytes_codec("zstd", "Zstd"), "numcodecs.zstd.Zstd")
|
|
260
|
+
Zlib = _add_docstring(_make_bytes_bytes_codec("zlib", "Zlib"), "numcodecs.zlib.Zlib")
|
|
261
|
+
GZip = _add_docstring(_make_bytes_bytes_codec("gzip", "GZip"), "numcodecs.gzip.GZip")
|
|
262
|
+
BZ2 = _add_docstring(_make_bytes_bytes_codec("bz2", "BZ2"), "numcodecs.bz2.BZ2")
|
|
263
|
+
LZMA = _add_docstring(_make_bytes_bytes_codec("lzma", "LZMA"), "numcodecs.lzma.LZMA")
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
@_add_docstring_wrapper("numcodecs.shuffle.Shuffle")
|
|
267
|
+
class Shuffle(_NumcodecsBytesBytesCodec):
|
|
268
|
+
codec_name = f"{CODEC_PREFIX}shuffle"
|
|
269
|
+
|
|
270
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
271
|
+
super().__init__(**codec_config)
|
|
272
|
+
|
|
273
|
+
def evolve_from_array_spec(self, array_spec: ArraySpec) -> Shuffle:
|
|
274
|
+
if self.codec_config.get("elementsize", None) is None:
|
|
275
|
+
return Shuffle(**{**self.codec_config, "elementsize": array_spec.dtype.itemsize})
|
|
276
|
+
return self # pragma: no cover
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
# array-to-array codecs ("filters")
|
|
280
|
+
@_add_docstring_wrapper("numcodecs.delta.Delta")
|
|
281
|
+
class Delta(_NumcodecsArrayArrayCodec):
|
|
282
|
+
codec_name = f"{CODEC_PREFIX}delta"
|
|
283
|
+
|
|
284
|
+
def __init__(self, **codec_config: dict[str, JSON]) -> None:
|
|
285
|
+
super().__init__(**codec_config)
|
|
286
|
+
|
|
287
|
+
def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
|
|
288
|
+
if astype := self.codec_config.get("astype"):
|
|
289
|
+
return replace(chunk_spec, dtype=np.dtype(astype)) # type: ignore[call-overload]
|
|
290
|
+
return chunk_spec
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
BitRound = _add_docstring(
|
|
294
|
+
_make_array_array_codec("bitround", "BitRound"), "numcodecs.bitround.BitRound"
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
@_add_docstring_wrapper("numcodecs.fixedscaleoffset.FixedScaleOffset")
|
|
299
|
+
class FixedScaleOffset(_NumcodecsArrayArrayCodec):
|
|
300
|
+
codec_name = f"{CODEC_PREFIX}fixedscaleoffset"
|
|
301
|
+
|
|
302
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
303
|
+
super().__init__(**codec_config)
|
|
304
|
+
|
|
305
|
+
def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
|
|
306
|
+
if astype := self.codec_config.get("astype"):
|
|
307
|
+
return replace(chunk_spec, dtype=np.dtype(astype)) # type: ignore[call-overload]
|
|
308
|
+
return chunk_spec
|
|
309
|
+
|
|
310
|
+
def evolve_from_array_spec(self, array_spec: ArraySpec) -> FixedScaleOffset:
|
|
311
|
+
if self.codec_config.get("dtype", None) is None:
|
|
312
|
+
return FixedScaleOffset(**{**self.codec_config, "dtype": str(array_spec.dtype)})
|
|
313
|
+
return self
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
@_add_docstring_wrapper("numcodecs.quantize.Quantize")
|
|
317
|
+
class Quantize(_NumcodecsArrayArrayCodec):
|
|
318
|
+
codec_name = f"{CODEC_PREFIX}quantize"
|
|
319
|
+
|
|
320
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
321
|
+
super().__init__(**codec_config)
|
|
322
|
+
|
|
323
|
+
def evolve_from_array_spec(self, array_spec: ArraySpec) -> Quantize:
|
|
324
|
+
if self.codec_config.get("dtype", None) is None:
|
|
325
|
+
return Quantize(**{**self.codec_config, "dtype": str(array_spec.dtype)})
|
|
326
|
+
return self
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
@_add_docstring_wrapper("numcodecs.packbits.PackBits")
|
|
330
|
+
class PackBits(_NumcodecsArrayArrayCodec):
|
|
331
|
+
codec_name = f"{CODEC_PREFIX}packbits"
|
|
332
|
+
|
|
333
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
334
|
+
super().__init__(**codec_config)
|
|
335
|
+
|
|
336
|
+
def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
|
|
337
|
+
return replace(
|
|
338
|
+
chunk_spec,
|
|
339
|
+
shape=(1 + math.ceil(product(chunk_spec.shape) / 8),),
|
|
340
|
+
dtype=np.dtype("uint8"),
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
def validate(self, *, dtype: np.dtype[Any], **_kwargs) -> None:
|
|
344
|
+
if dtype != np.dtype("bool"):
|
|
345
|
+
raise ValueError(f"Packbits filter requires bool dtype. Got {dtype}.")
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
@_add_docstring_wrapper("numcodecs.astype.AsType")
|
|
349
|
+
class AsType(_NumcodecsArrayArrayCodec):
|
|
350
|
+
codec_name = f"{CODEC_PREFIX}astype"
|
|
351
|
+
|
|
352
|
+
def __init__(self, **codec_config: JSON) -> None:
|
|
353
|
+
super().__init__(**codec_config)
|
|
354
|
+
|
|
355
|
+
def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
|
|
356
|
+
return replace(chunk_spec, dtype=np.dtype(self.codec_config["encode_dtype"])) # type: ignore[arg-type]
|
|
357
|
+
|
|
358
|
+
def evolve_from_array_spec(self, array_spec: ArraySpec) -> AsType:
|
|
359
|
+
if self.codec_config.get("decode_dtype", None) is None:
|
|
360
|
+
return AsType(**{**self.codec_config, "decode_dtype": str(array_spec.dtype)})
|
|
361
|
+
return self
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
# bytes-to-bytes checksum codecs
|
|
365
|
+
CRC32 = _add_docstring(_make_checksum_codec("crc32", "CRC32"), "numcodecs.checksum32.CRC32")
|
|
366
|
+
CRC32C = _add_docstring(_make_checksum_codec("crc32c", "CRC32C"), "numcodecs.checksum32.CRC32C")
|
|
367
|
+
Adler32 = _add_docstring(_make_checksum_codec("adler32", "Adler32"), "numcodecs.checksum32.Adler32")
|
|
368
|
+
Fletcher32 = _add_docstring(
|
|
369
|
+
_make_checksum_codec("fletcher32", "Fletcher32"), "numcodecs.fletcher32.Fletcher32"
|
|
370
|
+
)
|
|
371
|
+
JenkinsLookup3 = _add_docstring(
|
|
372
|
+
_make_checksum_codec("jenkins_lookup3", "JenkinsLookup3"), "numcodecs.checksum32.JenkinsLookup3"
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
# array-to-bytes codecs
|
|
376
|
+
PCodec = _add_docstring(_make_array_bytes_codec("pcodec", "PCodec"), "numcodecs.pcodec.PCodec")
|
|
377
|
+
ZFPY = _add_docstring(_make_array_bytes_codec("zfpy", "ZFPY"), "numcodecs.zfpy.ZFPY")
|
|
378
|
+
|
|
379
|
+
__all__ = [
|
|
380
|
+
"BZ2",
|
|
381
|
+
"CRC32",
|
|
382
|
+
"CRC32C",
|
|
383
|
+
"LZ4",
|
|
384
|
+
"LZMA",
|
|
385
|
+
"ZFPY",
|
|
386
|
+
"Adler32",
|
|
387
|
+
"AsType",
|
|
388
|
+
"BitRound",
|
|
389
|
+
"Blosc",
|
|
390
|
+
"Delta",
|
|
391
|
+
"FixedScaleOffset",
|
|
392
|
+
"Fletcher32",
|
|
393
|
+
"GZip",
|
|
394
|
+
"JenkinsLookup3",
|
|
395
|
+
"PCodec",
|
|
396
|
+
"PackBits",
|
|
397
|
+
"Quantize",
|
|
398
|
+
"Shuffle",
|
|
399
|
+
"Zlib",
|
|
400
|
+
"Zstd",
|
|
401
|
+
]
|
numcodecs/zfpy.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
from contextlib import suppress
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
from types import ModuleType
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
_zfpy: Optional[ModuleType] = None
|
|
8
|
+
|
|
9
|
+
_zfpy_version: tuple = ()
|
|
10
|
+
with suppress(PackageNotFoundError):
|
|
11
|
+
_zfpy_version = tuple(map(int, version("zfpy").split(".")))
|
|
12
|
+
|
|
13
|
+
if _zfpy_version:
|
|
14
|
+
# Check NumPy version
|
|
15
|
+
_numpy_version: tuple = tuple(map(int, version("numpy").split('.')))
|
|
16
|
+
if _numpy_version >= (2, 0, 0) and _zfpy_version < (1, 0, 1): # pragma: no cover
|
|
17
|
+
_zfpy_version = ()
|
|
18
|
+
warnings.warn(
|
|
19
|
+
"NumPy version >= 2.0.0 detected. The zfpy library is incompatible with this version of NumPy. "
|
|
20
|
+
"Please downgrade to NumPy < 2.0.0 or wait for an update from zfpy.",
|
|
21
|
+
UserWarning,
|
|
22
|
+
stacklevel=2,
|
|
23
|
+
)
|
|
24
|
+
else:
|
|
25
|
+
with suppress(ImportError):
|
|
26
|
+
import zfpy as _zfpy # type: ignore[no-redef]
|
|
27
|
+
|
|
28
|
+
if _zfpy:
|
|
29
|
+
import numpy as np
|
|
30
|
+
|
|
31
|
+
from .abc import Codec
|
|
32
|
+
from .compat import ensure_bytes, ensure_contiguous_ndarray, ndarray_copy
|
|
33
|
+
|
|
34
|
+
# noinspection PyShadowingBuiltins
|
|
35
|
+
class ZFPY(Codec):
|
|
36
|
+
"""Codec providing compression using zfpy via the Python standard
|
|
37
|
+
library.
|
|
38
|
+
|
|
39
|
+
Parameters
|
|
40
|
+
----------
|
|
41
|
+
mode : integer
|
|
42
|
+
One of the zfpy mode choice, e.g., ``zfpy.mode_fixed_accuracy``.
|
|
43
|
+
tolerance : double, optional
|
|
44
|
+
A double-precision number, specifying the compression accuracy needed.
|
|
45
|
+
rate : double, optional
|
|
46
|
+
A double-precision number, specifying the compression rate needed.
|
|
47
|
+
precision : int, optional
|
|
48
|
+
A integer number, specifying the compression precision needed.
|
|
49
|
+
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
codec_id = "zfpy"
|
|
53
|
+
|
|
54
|
+
def __init__(
|
|
55
|
+
self,
|
|
56
|
+
mode=_zfpy.mode_fixed_accuracy,
|
|
57
|
+
tolerance=-1,
|
|
58
|
+
rate=-1,
|
|
59
|
+
precision=-1,
|
|
60
|
+
compression_kwargs=None,
|
|
61
|
+
):
|
|
62
|
+
self.mode = mode
|
|
63
|
+
if mode == _zfpy.mode_fixed_accuracy:
|
|
64
|
+
self.compression_kwargs = {"tolerance": tolerance}
|
|
65
|
+
elif mode == _zfpy.mode_fixed_rate:
|
|
66
|
+
self.compression_kwargs = {"rate": rate}
|
|
67
|
+
elif mode == _zfpy.mode_fixed_precision:
|
|
68
|
+
self.compression_kwargs = {"precision": precision}
|
|
69
|
+
|
|
70
|
+
self.tolerance = tolerance
|
|
71
|
+
self.rate = rate
|
|
72
|
+
self.precision = precision
|
|
73
|
+
|
|
74
|
+
def encode(self, buf):
|
|
75
|
+
# not flatten c-order array and raise exception for f-order array
|
|
76
|
+
if not isinstance(buf, np.ndarray):
|
|
77
|
+
raise TypeError(
|
|
78
|
+
"The zfp codec does not support none numpy arrays."
|
|
79
|
+
f" Your buffers were {type(buf)}."
|
|
80
|
+
)
|
|
81
|
+
if buf.flags.c_contiguous:
|
|
82
|
+
flatten = False
|
|
83
|
+
else:
|
|
84
|
+
raise ValueError(
|
|
85
|
+
"The zfp codec does not support F order arrays. "
|
|
86
|
+
f"Your arrays flags were {buf.flags}."
|
|
87
|
+
)
|
|
88
|
+
buf = ensure_contiguous_ndarray(buf, flatten=flatten)
|
|
89
|
+
|
|
90
|
+
# do compression
|
|
91
|
+
return _zfpy.compress_numpy(buf, write_header=True, **self.compression_kwargs)
|
|
92
|
+
|
|
93
|
+
def decode(self, buf, out=None):
|
|
94
|
+
# normalise inputs
|
|
95
|
+
buf = ensure_bytes(buf)
|
|
96
|
+
if out is not None:
|
|
97
|
+
out = ensure_contiguous_ndarray(out)
|
|
98
|
+
|
|
99
|
+
# do decompression
|
|
100
|
+
dec = _zfpy.decompress_numpy(buf)
|
|
101
|
+
|
|
102
|
+
# handle destination
|
|
103
|
+
if out is not None:
|
|
104
|
+
return ndarray_copy(dec, out)
|
|
105
|
+
else:
|
|
106
|
+
return dec
|
|
107
|
+
|
|
108
|
+
def __repr__(self):
|
|
109
|
+
return (
|
|
110
|
+
f"{type(self).__name__}(mode={self.mode!r}, "
|
|
111
|
+
f"tolerance={self.tolerance}, rate={self.rate}, "
|
|
112
|
+
f"precision={self.precision})"
|
|
113
|
+
)
|
numcodecs/zlib.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import zlib as _zlib
|
|
2
|
+
|
|
3
|
+
from .abc import Codec
|
|
4
|
+
from .compat import ensure_contiguous_ndarray, ndarray_copy
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Zlib(Codec):
|
|
8
|
+
"""Codec providing compression using zlib via the Python standard library.
|
|
9
|
+
|
|
10
|
+
Parameters
|
|
11
|
+
----------
|
|
12
|
+
level : int
|
|
13
|
+
Compression level.
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
codec_id = 'zlib'
|
|
18
|
+
|
|
19
|
+
def __init__(self, level=1):
|
|
20
|
+
self.level = level
|
|
21
|
+
|
|
22
|
+
def encode(self, buf):
|
|
23
|
+
# normalise inputs
|
|
24
|
+
buf = ensure_contiguous_ndarray(buf)
|
|
25
|
+
|
|
26
|
+
# do compression
|
|
27
|
+
return _zlib.compress(buf, self.level)
|
|
28
|
+
|
|
29
|
+
# noinspection PyMethodMayBeStatic
|
|
30
|
+
def decode(self, buf, out=None):
|
|
31
|
+
# normalise inputs
|
|
32
|
+
buf = ensure_contiguous_ndarray(buf)
|
|
33
|
+
if out is not None:
|
|
34
|
+
out = ensure_contiguous_ndarray(out)
|
|
35
|
+
|
|
36
|
+
# do decompression
|
|
37
|
+
dec = _zlib.decompress(buf)
|
|
38
|
+
|
|
39
|
+
# handle destination - Python standard library zlib module does not
|
|
40
|
+
# support direct decompression into buffer, so we have to copy into
|
|
41
|
+
# out if given
|
|
42
|
+
return ndarray_copy(dec, out)
|
|
Binary file
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: numcodecs
|
|
3
|
+
Version: 0.16.0
|
|
4
|
+
Summary: A Python package providing buffer compression and transformation codecs for use in data storage and communication applications.
|
|
5
|
+
Maintainer-email: Alistair Miles <alimanfoo@googlemail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Bug Tracker, https://github.com/zarr-developers/numcodecs/issues
|
|
8
|
+
Project-URL: Changelog, https://numcodecs.readthedocs.io/en/stable/release.html
|
|
9
|
+
Project-URL: Documentation, https://numcodecs.readthedocs.io/
|
|
10
|
+
Project-URL: Homepage, https://github.com/zarr-developers/numcodecs
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Information Technology
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
+
Classifier: Operating System :: Unix
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Description-Content-Type: text/x-rst
|
|
23
|
+
License-File: LICENSE.txt
|
|
24
|
+
Requires-Dist: numpy>=1.24
|
|
25
|
+
Requires-Dist: typing_extensions
|
|
26
|
+
Provides-Extra: docs
|
|
27
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
28
|
+
Requires-Dist: sphinx-issues; extra == "docs"
|
|
29
|
+
Requires-Dist: pydata-sphinx-theme; extra == "docs"
|
|
30
|
+
Requires-Dist: numpydoc; extra == "docs"
|
|
31
|
+
Provides-Extra: test
|
|
32
|
+
Requires-Dist: coverage; extra == "test"
|
|
33
|
+
Requires-Dist: pytest; extra == "test"
|
|
34
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
35
|
+
Provides-Extra: test-extras
|
|
36
|
+
Requires-Dist: importlib_metadata; extra == "test-extras"
|
|
37
|
+
Provides-Extra: msgpack
|
|
38
|
+
Requires-Dist: msgpack; extra == "msgpack"
|
|
39
|
+
Provides-Extra: zfpy
|
|
40
|
+
Requires-Dist: zfpy>=1.0.0; extra == "zfpy"
|
|
41
|
+
Provides-Extra: pcodec
|
|
42
|
+
Requires-Dist: pcodec<0.4,>=0.3; extra == "pcodec"
|
|
43
|
+
Provides-Extra: crc32c
|
|
44
|
+
Requires-Dist: crc32c>=2.7; extra == "crc32c"
|
|
45
|
+
Dynamic: license-file
|
|
46
|
+
|
|
47
|
+
Numcodecs
|
|
48
|
+
=========
|
|
49
|
+
|
|
50
|
+
Numcodecs is a Python package providing buffer compression and transformation
|
|
51
|
+
codecs for use in data storage and communication applications.
|
|
52
|
+
|
|
53
|
+
.. image:: https://readthedocs.org/projects/numcodecs/badge/?version=latest
|
|
54
|
+
:target: https://numcodecs.readthedocs.io/en/latest/?badge=latest
|
|
55
|
+
|
|
56
|
+
.. image:: https://github.com/zarr-developers/numcodecs/workflows/Linux%20CI/badge.svg?branch=main
|
|
57
|
+
:target: https://github.com/zarr-developers/numcodecs/actions?query=workflow%3A%22Linux+CI%22
|
|
58
|
+
|
|
59
|
+
.. image:: https://github.com/zarr-developers/numcodecs/workflows/OSX%20CI/badge.svg?branch=main
|
|
60
|
+
:target: https://github.com/zarr-developers/numcodecs/actions?query=workflow%3A%22OSX+CI%22
|
|
61
|
+
|
|
62
|
+
.. image:: https://github.com/zarr-developers/numcodecs/workflows/Wheels/badge.svg?branch=main
|
|
63
|
+
:target: https://github.com/zarr-developers/numcodecs/actions?query=workflow%3AWheels
|
|
64
|
+
|
|
65
|
+
.. image:: https://codecov.io/gh/zarr-developers/numcodecs/branch/main/graph/badge.svg
|
|
66
|
+
:target: https://codecov.io/gh/zarr-developers/numcodecs
|