zarrs 0.1.5__cp311-abi3-manylinux_2_28_aarch64.whl → 0.2.2__cp311-abi3-manylinux_2_28_aarch64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zarrs/__init__.py +0 -4
- zarrs/_internal.abi3.so +0 -0
- zarrs/_internal.pyi +19 -27
- zarrs/pipeline.py +78 -60
- zarrs/utils.py +19 -10
- {zarrs-0.1.5.dist-info → zarrs-0.2.2.dist-info}/METADATA +17 -13
- zarrs-0.2.2.dist-info/RECORD +11 -0
- {zarrs-0.1.5.dist-info → zarrs-0.2.2.dist-info}/WHEEL +1 -1
- zarrs-0.2.2.dist-info/entry_points.txt +2 -0
- zarrs-0.1.5.dist-info/RECORD +0 -10
- {zarrs-0.1.5.dist-info → zarrs-0.2.2.dist-info}/licenses/LICENSE +0 -0
zarrs/__init__.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
from zarr.registry import register_pipeline
|
|
2
|
-
|
|
3
1
|
from ._internal import __version__
|
|
4
2
|
from .pipeline import ZarrsCodecPipeline as _ZarrsCodecPipeline
|
|
5
3
|
from .utils import CollapsedDimensionError, DiscontiguousArrayError
|
|
@@ -10,8 +8,6 @@ class ZarrsCodecPipeline(_ZarrsCodecPipeline):
|
|
|
10
8
|
pass
|
|
11
9
|
|
|
12
10
|
|
|
13
|
-
register_pipeline(ZarrsCodecPipeline)
|
|
14
|
-
|
|
15
11
|
__all__ = [
|
|
16
12
|
"ZarrsCodecPipeline",
|
|
17
13
|
"DiscontiguousArrayError",
|
zarrs/_internal.abi3.so
CHANGED
|
Binary file
|
zarrs/_internal.pyi
CHANGED
|
@@ -3,50 +3,42 @@
|
|
|
3
3
|
|
|
4
4
|
import builtins
|
|
5
5
|
import typing
|
|
6
|
-
from enum import Enum
|
|
7
6
|
|
|
8
7
|
import numpy.typing
|
|
8
|
+
import zarr.abc.store
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
@typing.final
|
|
11
|
+
class ChunkItem:
|
|
12
|
+
def __new__(
|
|
13
|
+
cls,
|
|
14
|
+
key: builtins.str,
|
|
15
|
+
chunk_subset: typing.Sequence[slice],
|
|
16
|
+
chunk_shape: typing.Sequence[builtins.int],
|
|
17
|
+
subset: typing.Sequence[slice],
|
|
18
|
+
shape: typing.Sequence[builtins.int],
|
|
19
|
+
) -> ChunkItem: ...
|
|
12
20
|
|
|
21
|
+
@typing.final
|
|
13
22
|
class CodecPipelineImpl:
|
|
14
23
|
def __new__(
|
|
15
24
|
cls,
|
|
16
|
-
|
|
25
|
+
array_metadata: builtins.str,
|
|
26
|
+
store_config: zarr.abc.store.Store,
|
|
17
27
|
*,
|
|
18
|
-
validate_checksums: builtins.bool
|
|
19
|
-
store_empty_chunks: builtins.bool | None = None,
|
|
28
|
+
validate_checksums: builtins.bool = False,
|
|
20
29
|
chunk_concurrent_minimum: builtins.int | None = None,
|
|
21
30
|
chunk_concurrent_maximum: builtins.int | None = None,
|
|
22
31
|
num_threads: builtins.int | None = None,
|
|
32
|
+
direct_io: builtins.bool = False,
|
|
23
33
|
) -> CodecPipelineImpl: ...
|
|
24
34
|
def retrieve_chunks_and_apply_index(
|
|
25
35
|
self,
|
|
26
|
-
chunk_descriptions: typing.Sequence[
|
|
36
|
+
chunk_descriptions: typing.Sequence[ChunkItem],
|
|
27
37
|
value: numpy.typing.NDArray[typing.Any],
|
|
28
38
|
) -> None: ...
|
|
29
39
|
def store_chunks_with_indices(
|
|
30
40
|
self,
|
|
31
|
-
chunk_descriptions: typing.Sequence[
|
|
41
|
+
chunk_descriptions: typing.Sequence[ChunkItem],
|
|
32
42
|
value: numpy.typing.NDArray[typing.Any],
|
|
43
|
+
write_empty_chunks: builtins.bool,
|
|
33
44
|
) -> None: ...
|
|
34
|
-
|
|
35
|
-
class FilesystemStoreConfig:
|
|
36
|
-
root: builtins.str
|
|
37
|
-
|
|
38
|
-
class HttpStoreConfig:
|
|
39
|
-
endpoint: builtins.str
|
|
40
|
-
|
|
41
|
-
class WithSubset:
|
|
42
|
-
def __new__(
|
|
43
|
-
cls,
|
|
44
|
-
item: Basic,
|
|
45
|
-
chunk_subset: typing.Sequence[slice],
|
|
46
|
-
subset: typing.Sequence[slice],
|
|
47
|
-
shape: typing.Sequence[builtins.int],
|
|
48
|
-
) -> WithSubset: ...
|
|
49
|
-
|
|
50
|
-
class StoreConfig(Enum):
|
|
51
|
-
Filesystem = ...
|
|
52
|
-
Http = ...
|
zarrs/pipeline.py
CHANGED
|
@@ -2,27 +2,29 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import json
|
|
5
|
-
import re
|
|
6
5
|
from dataclasses import dataclass
|
|
7
6
|
from typing import TYPE_CHECKING, TypedDict
|
|
7
|
+
from warnings import warn
|
|
8
8
|
|
|
9
9
|
import numpy as np
|
|
10
10
|
from zarr.abc.codec import Codec, CodecPipeline
|
|
11
|
+
from zarr.codecs._v2 import V2Codec
|
|
11
12
|
from zarr.core import BatchedCodecPipeline
|
|
12
13
|
from zarr.core.config import config
|
|
14
|
+
from zarr.core.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata
|
|
13
15
|
|
|
14
16
|
if TYPE_CHECKING:
|
|
15
|
-
from collections.abc import
|
|
16
|
-
from typing import
|
|
17
|
+
from collections.abc import Iterable, Iterator
|
|
18
|
+
from typing import Self
|
|
17
19
|
|
|
18
|
-
from zarr.abc.store import ByteGetter, ByteSetter
|
|
20
|
+
from zarr.abc.store import ByteGetter, ByteSetter, Store
|
|
19
21
|
from zarr.core.array_spec import ArraySpec
|
|
20
22
|
from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer
|
|
21
23
|
from zarr.core.chunk_grids import ChunkGrid
|
|
22
|
-
from zarr.core.common import ChunkCoords
|
|
23
24
|
from zarr.core.indexing import SelectorTuple
|
|
25
|
+
from zarr.dtype import ZDType
|
|
24
26
|
|
|
25
|
-
from ._internal import CodecPipelineImpl
|
|
27
|
+
from ._internal import CodecPipelineImpl
|
|
26
28
|
from .utils import (
|
|
27
29
|
CollapsedDimensionError,
|
|
28
30
|
DiscontiguousArrayError,
|
|
@@ -39,12 +41,19 @@ class UnsupportedMetadataError(Exception):
|
|
|
39
41
|
pass
|
|
40
42
|
|
|
41
43
|
|
|
42
|
-
def get_codec_pipeline_impl(
|
|
44
|
+
def get_codec_pipeline_impl(
|
|
45
|
+
metadata: ArrayMetadata, store: Store, *, strict: bool
|
|
46
|
+
) -> CodecPipelineImpl | None:
|
|
43
47
|
try:
|
|
48
|
+
array_metadata_json = json.dumps(metadata.to_dict())
|
|
49
|
+
# Maintain old behavior: https://github.com/zarrs/zarrs-python/tree/b36ba797cafec77f5f41a25316be02c718a2b4f8?tab=readme-ov-file#configuration
|
|
50
|
+
validate_checksums = config.get("codec_pipeline.validate_checksums", True)
|
|
51
|
+
if validate_checksums is None:
|
|
52
|
+
validate_checksums = True
|
|
44
53
|
return CodecPipelineImpl(
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
54
|
+
array_metadata_json,
|
|
55
|
+
store_config=store,
|
|
56
|
+
validate_checksums=validate_checksums,
|
|
48
57
|
chunk_concurrent_minimum=config.get(
|
|
49
58
|
"codec_pipeline.chunk_concurrent_minimum", None
|
|
50
59
|
),
|
|
@@ -52,35 +61,27 @@ def get_codec_pipeline_impl(codec_metadata_json: str) -> CodecPipelineImpl | Non
|
|
|
52
61
|
"codec_pipeline.chunk_concurrent_maximum", None
|
|
53
62
|
),
|
|
54
63
|
num_threads=config.get("threading.max_workers", None),
|
|
64
|
+
direct_io=config.get("codec_pipeline.direct_io", False),
|
|
55
65
|
)
|
|
56
66
|
except TypeError as e:
|
|
57
|
-
if
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
compressor_json = codec_dict.get("compressor").get_config()
|
|
76
|
-
compressor = json.dumps(compressor_json)
|
|
77
|
-
else:
|
|
78
|
-
compressor = None
|
|
79
|
-
codecs_v3 = codec_metadata_v2_to_v3(filters, compressor)
|
|
80
|
-
for codec in codecs_v3:
|
|
81
|
-
yield json.loads(codec)
|
|
82
|
-
else:
|
|
83
|
-
yield codec.to_dict()
|
|
67
|
+
if strict:
|
|
68
|
+
raise UnsupportedMetadataError() from e
|
|
69
|
+
|
|
70
|
+
warn(
|
|
71
|
+
f"Array is unsupported by ZarrsCodecPipeline: {e}",
|
|
72
|
+
category=UserWarning,
|
|
73
|
+
)
|
|
74
|
+
return None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def get_codec_pipeline_fallback(
|
|
78
|
+
metadata: ArrayMetadata, *, strict: bool
|
|
79
|
+
) -> BatchedCodecPipeline | None:
|
|
80
|
+
if strict:
|
|
81
|
+
return None
|
|
82
|
+
else:
|
|
83
|
+
codecs = array_metadata_to_codecs(metadata)
|
|
84
|
+
return BatchedCodecPipeline.from_codecs(codecs)
|
|
84
85
|
|
|
85
86
|
|
|
86
87
|
class ZarrsCodecPipelineState(TypedDict):
|
|
@@ -88,38 +89,48 @@ class ZarrsCodecPipelineState(TypedDict):
|
|
|
88
89
|
codecs: tuple[Codec, ...]
|
|
89
90
|
|
|
90
91
|
|
|
92
|
+
def array_metadata_to_codecs(metadata: ArrayMetadata) -> list[Codec]:
|
|
93
|
+
if isinstance(metadata, ArrayV3Metadata):
|
|
94
|
+
return metadata.codecs
|
|
95
|
+
elif isinstance(metadata, ArrayV2Metadata):
|
|
96
|
+
v2_codec = V2Codec(filters=metadata.filters, compressor=metadata.compressor)
|
|
97
|
+
return [v2_codec]
|
|
98
|
+
|
|
99
|
+
|
|
91
100
|
@dataclass
|
|
92
101
|
class ZarrsCodecPipeline(CodecPipeline):
|
|
93
|
-
|
|
102
|
+
metadata: ArrayMetadata
|
|
103
|
+
store: Store
|
|
94
104
|
impl: CodecPipelineImpl | None
|
|
95
|
-
|
|
96
|
-
python_impl: BatchedCodecPipeline
|
|
105
|
+
python_impl: BatchedCodecPipeline | None
|
|
97
106
|
|
|
98
107
|
def __getstate__(self) -> ZarrsCodecPipelineState:
|
|
99
|
-
return {"
|
|
108
|
+
return {"metadata": self.metadata, "store": self.store}
|
|
100
109
|
|
|
101
110
|
def __setstate__(self, state: ZarrsCodecPipelineState):
|
|
102
|
-
self.
|
|
103
|
-
self.
|
|
104
|
-
|
|
105
|
-
self.
|
|
111
|
+
self.metadata = state["metadata"]
|
|
112
|
+
self.store = state["store"]
|
|
113
|
+
strict = config.get("codec_pipeline.strict", False)
|
|
114
|
+
self.impl = get_codec_pipeline_impl(self.metadata, self.store, strict=strict)
|
|
115
|
+
self.python_impl = get_codec_pipeline_fallback(self.metadata, strict=strict)
|
|
106
116
|
|
|
107
117
|
def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
|
|
108
|
-
|
|
118
|
+
return self
|
|
109
119
|
|
|
110
120
|
@classmethod
|
|
111
121
|
def from_codecs(cls, codecs: Iterable[Codec]) -> Self:
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
122
|
+
return BatchedCodecPipeline.from_codecs(codecs)
|
|
123
|
+
|
|
124
|
+
@classmethod
|
|
125
|
+
def from_array_metadata_and_store(
|
|
126
|
+
cls, array_metadata: ArrayMetadata, store: Store
|
|
127
|
+
) -> Self:
|
|
128
|
+
strict = config.get("codec_pipeline.strict", False)
|
|
118
129
|
return cls(
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
impl=get_codec_pipeline_impl(
|
|
122
|
-
python_impl=
|
|
130
|
+
metadata=array_metadata,
|
|
131
|
+
store=store,
|
|
132
|
+
impl=get_codec_pipeline_impl(array_metadata, store, strict=strict),
|
|
133
|
+
python_impl=get_codec_pipeline_fallback(array_metadata, strict=strict),
|
|
123
134
|
)
|
|
124
135
|
|
|
125
136
|
@property
|
|
@@ -134,7 +145,7 @@ class ZarrsCodecPipeline(CodecPipeline):
|
|
|
134
145
|
yield from self.codecs
|
|
135
146
|
|
|
136
147
|
def validate(
|
|
137
|
-
self, *, shape:
|
|
148
|
+
self, *, shape: tuple[int, ...], dtype: ZDType, chunk_grid: ChunkGrid
|
|
138
149
|
) -> None:
|
|
139
150
|
raise NotImplementedError("validate")
|
|
140
151
|
|
|
@@ -178,13 +189,15 @@ class ZarrsCodecPipeline(CodecPipeline):
|
|
|
178
189
|
UnsupportedDataTypeError,
|
|
179
190
|
FillValueNoneError,
|
|
180
191
|
):
|
|
192
|
+
if self.python_impl is None:
|
|
193
|
+
raise
|
|
181
194
|
await self.python_impl.read(batch_info, out, drop_axes)
|
|
182
195
|
return None
|
|
183
196
|
else:
|
|
184
197
|
out: NDArrayLike = out.as_ndarray_like()
|
|
185
198
|
await asyncio.to_thread(
|
|
186
199
|
self.impl.retrieve_chunks_and_apply_index,
|
|
187
|
-
chunks_desc,
|
|
200
|
+
chunks_desc.chunk_info_with_indices,
|
|
188
201
|
out,
|
|
189
202
|
)
|
|
190
203
|
return None
|
|
@@ -211,6 +224,8 @@ class ZarrsCodecPipeline(CodecPipeline):
|
|
|
211
224
|
UnsupportedDataTypeError,
|
|
212
225
|
FillValueNoneError,
|
|
213
226
|
):
|
|
227
|
+
if self.python_impl is None:
|
|
228
|
+
raise
|
|
214
229
|
await self.python_impl.write(batch_info, value, drop_axes)
|
|
215
230
|
return None
|
|
216
231
|
else:
|
|
@@ -223,7 +238,10 @@ class ZarrsCodecPipeline(CodecPipeline):
|
|
|
223
238
|
elif not value_np.flags.c_contiguous:
|
|
224
239
|
value_np = np.ascontiguousarray(value_np)
|
|
225
240
|
await asyncio.to_thread(
|
|
226
|
-
self.impl.store_chunks_with_indices,
|
|
241
|
+
self.impl.store_chunks_with_indices,
|
|
242
|
+
chunks_desc.chunk_info_with_indices,
|
|
243
|
+
value_np,
|
|
244
|
+
chunks_desc.write_empty_chunks,
|
|
227
245
|
)
|
|
228
246
|
return None
|
|
229
247
|
|
|
@@ -236,7 +254,7 @@ class ZarrsCodecPipeline(CodecPipeline):
|
|
|
236
254
|
# https://github.com/LDeakin/zarrs/blob/0532fe983b7b42b59dbf84e50a2fe5e6f7bad4ce/zarrs_metadata/src/v2_to_v3.rs#L289-L293 for VSUMm
|
|
237
255
|
# Further, our pipeline does not support variable-length objects due to limitations on decode_into, so object/np.dtypes.StringDType is also out
|
|
238
256
|
if any(
|
|
239
|
-
info.dtype.kind in {"V", "S", "U", "M", "m", "O", "T"}
|
|
257
|
+
info.dtype.to_native_dtype().kind in {"V", "S", "U", "M", "m", "O", "T"}
|
|
240
258
|
for (_, info, _, _, _) in batch_info
|
|
241
259
|
):
|
|
242
260
|
raise UnsupportedDataTypeError()
|
zarrs/utils.py
CHANGED
|
@@ -2,21 +2,22 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import operator
|
|
4
4
|
import os
|
|
5
|
+
from dataclasses import dataclass
|
|
5
6
|
from functools import reduce
|
|
6
7
|
from typing import TYPE_CHECKING, Any
|
|
7
8
|
|
|
8
9
|
import numpy as np
|
|
9
10
|
from zarr.core.array_spec import ArraySpec
|
|
10
11
|
from zarr.core.indexing import SelectorTuple, is_integer
|
|
11
|
-
from zarr.core.metadata.v2 import _default_fill_value
|
|
12
12
|
|
|
13
|
-
from zarrs._internal import
|
|
13
|
+
from zarrs._internal import ChunkItem
|
|
14
14
|
|
|
15
15
|
if TYPE_CHECKING:
|
|
16
16
|
from collections.abc import Iterable
|
|
17
17
|
from types import EllipsisType
|
|
18
18
|
|
|
19
19
|
from zarr.abc.store import ByteGetter, ByteSetter
|
|
20
|
+
from zarr.dtype import ZDType
|
|
20
21
|
|
|
21
22
|
|
|
22
23
|
# adapted from https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
|
|
@@ -139,21 +140,28 @@ def get_shape_for_selector(
|
|
|
139
140
|
return resulting_shape_from_index(shape, selector_tuple, drop_axes, pad=pad)
|
|
140
141
|
|
|
141
142
|
|
|
142
|
-
def get_implicit_fill_value(dtype:
|
|
143
|
+
def get_implicit_fill_value(dtype: ZDType, fill_value: Any) -> Any:
|
|
143
144
|
if fill_value is None:
|
|
144
|
-
fill_value =
|
|
145
|
+
fill_value = dtype.default_scalar()
|
|
145
146
|
return fill_value
|
|
146
147
|
|
|
147
148
|
|
|
149
|
+
@dataclass(frozen=True)
|
|
150
|
+
class RustChunkInfo:
|
|
151
|
+
chunk_info_with_indices: list[ChunkItem]
|
|
152
|
+
write_empty_chunks: bool
|
|
153
|
+
|
|
154
|
+
|
|
148
155
|
def make_chunk_info_for_rust_with_indices(
|
|
149
156
|
batch_info: Iterable[
|
|
150
157
|
tuple[ByteGetter | ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]
|
|
151
158
|
],
|
|
152
159
|
drop_axes: tuple[int, ...],
|
|
153
160
|
shape: tuple[int, ...],
|
|
154
|
-
) ->
|
|
161
|
+
) -> RustChunkInfo:
|
|
155
162
|
shape = shape if shape else (1,) # constant array
|
|
156
|
-
chunk_info_with_indices: list[
|
|
163
|
+
chunk_info_with_indices: list[ChunkItem] = []
|
|
164
|
+
write_empty_chunks: bool = True
|
|
157
165
|
for (
|
|
158
166
|
byte_getter,
|
|
159
167
|
chunk_spec,
|
|
@@ -161,6 +169,7 @@ def make_chunk_info_for_rust_with_indices(
|
|
|
161
169
|
out_selection,
|
|
162
170
|
_,
|
|
163
171
|
) in batch_info:
|
|
172
|
+
write_empty_chunks = chunk_spec.config.write_empty_chunks
|
|
164
173
|
if chunk_spec.fill_value is None:
|
|
165
174
|
chunk_spec = ArraySpec(
|
|
166
175
|
chunk_spec.shape,
|
|
@@ -169,7 +178,6 @@ def make_chunk_info_for_rust_with_indices(
|
|
|
169
178
|
chunk_spec.config,
|
|
170
179
|
chunk_spec.prototype,
|
|
171
180
|
)
|
|
172
|
-
chunk_info = Basic(byte_getter, chunk_spec)
|
|
173
181
|
out_selection_as_slices = selector_tuple_to_slice_selection(out_selection)
|
|
174
182
|
chunk_selection_as_slices = selector_tuple_to_slice_selection(chunk_selection)
|
|
175
183
|
shape_chunk_selection_slices = get_shape_for_selector(
|
|
@@ -186,11 +194,12 @@ def make_chunk_info_for_rust_with_indices(
|
|
|
186
194
|
f"{shape_chunk_selection} != {shape_chunk_selection_slices}"
|
|
187
195
|
)
|
|
188
196
|
chunk_info_with_indices.append(
|
|
189
|
-
|
|
190
|
-
|
|
197
|
+
ChunkItem(
|
|
198
|
+
key=byte_getter.path,
|
|
191
199
|
chunk_subset=chunk_selection_as_slices,
|
|
200
|
+
chunk_shape=chunk_spec.shape,
|
|
192
201
|
subset=out_selection_as_slices,
|
|
193
202
|
shape=shape,
|
|
194
203
|
)
|
|
195
204
|
)
|
|
196
|
-
return chunk_info_with_indices
|
|
205
|
+
return RustChunkInfo(chunk_info_with_indices, write_empty_chunks)
|
|
@@ -1,15 +1,16 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: zarrs
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Classifier: Programming Language :: Rust
|
|
5
5
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
6
6
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
7
7
|
Classifier: Typing :: Typed
|
|
8
|
-
Requires-Dist: numpy
|
|
9
|
-
Requires-Dist: zarr
|
|
8
|
+
Requires-Dist: numpy>=1.24
|
|
9
|
+
Requires-Dist: zarr>=3.1
|
|
10
10
|
License-File: LICENSE
|
|
11
|
+
Summary: A CodecPipeline for zarr-python backed by the zarrs Rust crate
|
|
11
12
|
Author: Ilan Gold, Lachlan Deakin, Philipp Angerer
|
|
12
|
-
License: MIT
|
|
13
|
+
License-Expression: MIT
|
|
13
14
|
Requires-Python: >=3.11
|
|
14
15
|
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
15
16
|
|
|
@@ -28,7 +29,6 @@ To use the project, simply install our package (which depends on `zarr-python>=3
|
|
|
28
29
|
|
|
29
30
|
```python
|
|
30
31
|
import zarr
|
|
31
|
-
import zarrs
|
|
32
32
|
zarr.config.set({"codec_pipeline.path": "zarrs.ZarrsCodecPipeline"})
|
|
33
33
|
```
|
|
34
34
|
|
|
@@ -40,12 +40,11 @@ We export a `ZarrsCodecPipeline` class so that `zarr-python` can use the class b
|
|
|
40
40
|
|
|
41
41
|
At the moment, we only support a subset of the `zarr-python` stores:
|
|
42
42
|
|
|
43
|
-
- [
|
|
44
|
-
- [
|
|
45
|
-
|
|
43
|
+
- [`LocalStore`](https://zarr.readthedocs.io/en/latest/api/zarr/storage/#zarr.storage.LocalStore) (local filesystem)
|
|
44
|
+
- [`ObjectStore`](https://zarr.readthedocs.io/en/latest/user-guide/storage/#object-store) (cloud storage)
|
|
45
|
+
- [`HTTPFileSystem`](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.implementations.http.HTTPFileSystem) via [`FsspecStore`](https://zarr.readthedocs.io/en/latest/api/zarr/storage/#zarr.storage.FsspecStore)
|
|
46
46
|
|
|
47
47
|
A `NotImplementedError` will be raised if a store is not supported.
|
|
48
|
-
We intend to support more stores in the future: https://github.com/zarrs/zarrs-python/issues/44.
|
|
49
48
|
|
|
50
49
|
### Configuration
|
|
51
50
|
|
|
@@ -63,7 +62,11 @@ The `ZarrsCodecPipeline` specific options are:
|
|
|
63
62
|
- `codec_pipeline.chunk_concurrent_minimum`: the minimum number of chunks retrieved/stored concurrently when balancing chunk/codec concurrency.
|
|
64
63
|
- Defaults to 4 if `None`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#chunk-concurrent-minimum) for more info.
|
|
65
64
|
- `codec_pipeline.validate_checksums`: enable checksum validation (e.g. with the CRC32C codec).
|
|
66
|
-
- Defaults to
|
|
65
|
+
- Defaults to `True`. See [here](https://docs.rs/zarrs/latest/zarrs/config/struct.Config.html#validate-checksums) for more info.
|
|
66
|
+
- `codec_pipeline.direct_io`: enable `O_DIRECT` read/write, needs support from the operating system (currently only Linux) and file system.
|
|
67
|
+
- Defaults to `False`.
|
|
68
|
+
- `codec_pipeline.strict`: raise exceptions for unsupported operations instead of falling back to the default codec pipeline of `zarr-python`.
|
|
69
|
+
- Defaults to `False`.
|
|
67
70
|
|
|
68
71
|
For example:
|
|
69
72
|
```python
|
|
@@ -73,14 +76,15 @@ zarr.config.set({
|
|
|
73
76
|
"codec_pipeline": {
|
|
74
77
|
"path": "zarrs.ZarrsCodecPipeline",
|
|
75
78
|
"validate_checksums": True,
|
|
76
|
-
"store_empty_chunks": False,
|
|
77
79
|
"chunk_concurrent_maximum": None,
|
|
78
80
|
"chunk_concurrent_minimum": 4,
|
|
81
|
+
"direct_io": False,
|
|
82
|
+
"strict": False
|
|
79
83
|
}
|
|
80
84
|
})
|
|
81
85
|
```
|
|
82
86
|
|
|
83
|
-
If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `
|
|
87
|
+
If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value. However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed. This may change in the future as guidance from the `zarr` community becomes clear.
|
|
84
88
|
|
|
85
89
|
## Concurrency
|
|
86
90
|
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
zarrs-0.2.2.dist-info/METADATA,sha256=8K1AOS_SVQgRLzQ2rWtNbJxCAb12XmTLRrT6QklQgOI,8054
|
|
2
|
+
zarrs-0.2.2.dist-info/WHEEL,sha256=pQhpX1zEYym7lHVMSudaqVXS44siTf8XBrjmsFdZ39M,108
|
|
3
|
+
zarrs-0.2.2.dist-info/entry_points.txt,sha256=EzI6yCIUPDHBHzjDdexuGGYbOLXf8x2ICokOJXnuX3k,68
|
|
4
|
+
zarrs-0.2.2.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
|
|
5
|
+
zarrs/__init__.py,sha256=lRVtAPzCzJkGs4vQrW4UgANq-pC-khS0ZF7HTj4__Hg,489
|
|
6
|
+
zarrs/_internal.abi3.so,sha256=2oa6VVFlcQ-C3qa6UK_LKKzRFc36GhmZfzKXqiJnvLA,15343720
|
|
7
|
+
zarrs/_internal.pyi,sha256=a_D4yx99r4xeQX1ntY_A_Q4wVmLeLwJZHWAQV_mVu9A,1308
|
|
8
|
+
zarrs/pipeline.py,sha256=YfB13GWNfxELerXVtJ_ipFwSL7bN-YuPys6jCB9lnms,9008
|
|
9
|
+
zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
+
zarrs/utils.py,sha256=W2XCjJDVrdHYJgtVaRKN533Ljw1MF7o0YwXuz5ZAk2g,7020
|
|
11
|
+
zarrs-0.2.2.dist-info/RECORD,,
|
zarrs-0.1.5.dist-info/RECORD
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
zarrs-0.1.5.dist-info/METADATA,sha256=MraaZvJnr2BaC4xLsvHjNIWkaE_KMYsHfDRJ5CJ4nsY,7693
|
|
2
|
-
zarrs-0.1.5.dist-info/WHEEL,sha256=vlVK9XRfjbZsffx4VpMMyFV477nGomrs3TkCmK8HoZM,108
|
|
3
|
-
zarrs-0.1.5.dist-info/licenses/LICENSE,sha256=vwIsJjEfVFehyyqcb7B3dAXAniaFMmk8u7IoiJAfBJ4,1099
|
|
4
|
-
zarrs/__init__.py,sha256=4oWtWDZO8r7z4Uh7Fy_brmkxXDpULQdgjlA0iFw98eA,573
|
|
5
|
-
zarrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
-
zarrs/pipeline.py,sha256=GV6Z1xoMdLQpnp4iX4nCuIYZTK1bPDG9nO0-t23F0_I,8708
|
|
7
|
-
zarrs/_internal.pyi,sha256=H2afxhX5LDKOSR4bOBoH3mXoVqdM24Rc-xdjqMiOE5Y,1421
|
|
8
|
-
zarrs/utils.py,sha256=uKIDi2EKRJRFdlyCElEvZrooVXkbmO4ZoE5fTHFNHGo,6763
|
|
9
|
-
zarrs/_internal.abi3.so,sha256=cYjWNpNHDGGY71MVDv6Z7oE-3k2PrhLUHR1dtwByln4,9704088
|
|
10
|
-
zarrs-0.1.5.dist-info/RECORD,,
|
|
File without changes
|