ngio 0.5.0b6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ngio/__init__.py +69 -0
- ngio/common/__init__.py +28 -0
- ngio/common/_dimensions.py +335 -0
- ngio/common/_masking_roi.py +153 -0
- ngio/common/_pyramid.py +408 -0
- ngio/common/_roi.py +315 -0
- ngio/common/_synt_images_utils.py +101 -0
- ngio/common/_zoom.py +188 -0
- ngio/experimental/__init__.py +5 -0
- ngio/experimental/iterators/__init__.py +15 -0
- ngio/experimental/iterators/_abstract_iterator.py +390 -0
- ngio/experimental/iterators/_feature.py +189 -0
- ngio/experimental/iterators/_image_processing.py +130 -0
- ngio/experimental/iterators/_mappers.py +48 -0
- ngio/experimental/iterators/_rois_utils.py +126 -0
- ngio/experimental/iterators/_segmentation.py +235 -0
- ngio/hcs/__init__.py +19 -0
- ngio/hcs/_plate.py +1354 -0
- ngio/images/__init__.py +44 -0
- ngio/images/_abstract_image.py +967 -0
- ngio/images/_create_synt_container.py +132 -0
- ngio/images/_create_utils.py +423 -0
- ngio/images/_image.py +926 -0
- ngio/images/_label.py +411 -0
- ngio/images/_masked_image.py +531 -0
- ngio/images/_ome_zarr_container.py +1237 -0
- ngio/images/_table_ops.py +471 -0
- ngio/io_pipes/__init__.py +75 -0
- ngio/io_pipes/_io_pipes.py +361 -0
- ngio/io_pipes/_io_pipes_masked.py +488 -0
- ngio/io_pipes/_io_pipes_roi.py +146 -0
- ngio/io_pipes/_io_pipes_types.py +56 -0
- ngio/io_pipes/_match_shape.py +377 -0
- ngio/io_pipes/_ops_axes.py +344 -0
- ngio/io_pipes/_ops_slices.py +411 -0
- ngio/io_pipes/_ops_slices_utils.py +199 -0
- ngio/io_pipes/_ops_transforms.py +104 -0
- ngio/io_pipes/_zoom_transform.py +180 -0
- ngio/ome_zarr_meta/__init__.py +65 -0
- ngio/ome_zarr_meta/_meta_handlers.py +536 -0
- ngio/ome_zarr_meta/ngio_specs/__init__.py +77 -0
- ngio/ome_zarr_meta/ngio_specs/_axes.py +515 -0
- ngio/ome_zarr_meta/ngio_specs/_channels.py +462 -0
- ngio/ome_zarr_meta/ngio_specs/_dataset.py +89 -0
- ngio/ome_zarr_meta/ngio_specs/_ngio_hcs.py +539 -0
- ngio/ome_zarr_meta/ngio_specs/_ngio_image.py +438 -0
- ngio/ome_zarr_meta/ngio_specs/_pixel_size.py +122 -0
- ngio/ome_zarr_meta/v04/__init__.py +27 -0
- ngio/ome_zarr_meta/v04/_custom_models.py +18 -0
- ngio/ome_zarr_meta/v04/_v04_spec.py +473 -0
- ngio/ome_zarr_meta/v05/__init__.py +27 -0
- ngio/ome_zarr_meta/v05/_custom_models.py +18 -0
- ngio/ome_zarr_meta/v05/_v05_spec.py +511 -0
- ngio/resources/20200812-CardiomyocyteDifferentiation14-Cycle1_B03/mask.png +0 -0
- ngio/resources/20200812-CardiomyocyteDifferentiation14-Cycle1_B03/nuclei.png +0 -0
- ngio/resources/20200812-CardiomyocyteDifferentiation14-Cycle1_B03/raw.jpg +0 -0
- ngio/resources/__init__.py +55 -0
- ngio/resources/resource_model.py +36 -0
- ngio/tables/__init__.py +43 -0
- ngio/tables/_abstract_table.py +270 -0
- ngio/tables/_tables_container.py +449 -0
- ngio/tables/backends/__init__.py +57 -0
- ngio/tables/backends/_abstract_backend.py +240 -0
- ngio/tables/backends/_anndata.py +139 -0
- ngio/tables/backends/_anndata_utils.py +90 -0
- ngio/tables/backends/_csv.py +19 -0
- ngio/tables/backends/_json.py +92 -0
- ngio/tables/backends/_parquet.py +19 -0
- ngio/tables/backends/_py_arrow_backends.py +222 -0
- ngio/tables/backends/_table_backends.py +226 -0
- ngio/tables/backends/_utils.py +608 -0
- ngio/tables/v1/__init__.py +23 -0
- ngio/tables/v1/_condition_table.py +71 -0
- ngio/tables/v1/_feature_table.py +125 -0
- ngio/tables/v1/_generic_table.py +49 -0
- ngio/tables/v1/_roi_table.py +575 -0
- ngio/transforms/__init__.py +5 -0
- ngio/transforms/_zoom.py +19 -0
- ngio/utils/__init__.py +45 -0
- ngio/utils/_cache.py +48 -0
- ngio/utils/_datasets.py +165 -0
- ngio/utils/_errors.py +37 -0
- ngio/utils/_fractal_fsspec_store.py +42 -0
- ngio/utils/_zarr_utils.py +534 -0
- ngio-0.5.0b6.dist-info/METADATA +148 -0
- ngio-0.5.0b6.dist-info/RECORD +88 -0
- ngio-0.5.0b6.dist-info/WHEEL +4 -0
- ngio-0.5.0b6.dist-info/licenses/LICENSE +28 -0
|
@@ -0,0 +1,534 @@
|
|
|
1
|
+
"""Common utilities for working with Zarr groups in consistent ways."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import warnings
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Literal, TypeAlias
|
|
7
|
+
|
|
8
|
+
import dask.array as da
|
|
9
|
+
import fsspec
|
|
10
|
+
import zarr
|
|
11
|
+
from filelock import BaseFileLock, FileLock
|
|
12
|
+
from pydantic_zarr.v2 import ArraySpec as AnyArraySpecV2
|
|
13
|
+
from pydantic_zarr.v3 import ArraySpec as AnyArraySpecV3
|
|
14
|
+
from zarr.abc.store import Store
|
|
15
|
+
from zarr.errors import ContainsGroupError
|
|
16
|
+
from zarr.storage import FsspecStore, LocalStore, MemoryStore, ZipStore
|
|
17
|
+
|
|
18
|
+
from ngio.utils._cache import NgioCache
|
|
19
|
+
from ngio.utils._errors import (
|
|
20
|
+
NgioFileExistsError,
|
|
21
|
+
NgioFileNotFoundError,
|
|
22
|
+
NgioValueError,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
AccessModeLiteral = Literal["r", "r+", "w", "w-", "a"]
|
|
26
|
+
# StoreLike is more restrictive than it could be
|
|
27
|
+
# but to make sure we can handle the store correctly
|
|
28
|
+
# we need to be more restrictive
|
|
29
|
+
NgioSupportedStore: TypeAlias = (
|
|
30
|
+
str | Path | fsspec.mapping.FSMap | FsspecStore | MemoryStore | dict | LocalStore
|
|
31
|
+
)
|
|
32
|
+
GenericStore: TypeAlias = NgioSupportedStore | Store
|
|
33
|
+
StoreOrGroup: TypeAlias = NgioSupportedStore | zarr.Group
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _check_store(store) -> NgioSupportedStore:
|
|
37
|
+
"""Check the store and return a valid store."""
|
|
38
|
+
if not isinstance(store, NgioSupportedStore):
|
|
39
|
+
warnings.warn(
|
|
40
|
+
f"Store type {type(store)} is not explicitly supported. "
|
|
41
|
+
f"Supported types are: {NgioSupportedStore}. "
|
|
42
|
+
"Proceeding, but this may lead to unexpected behavior.",
|
|
43
|
+
UserWarning,
|
|
44
|
+
stacklevel=2,
|
|
45
|
+
)
|
|
46
|
+
return store
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _check_group(
|
|
50
|
+
group: zarr.Group, mode: AccessModeLiteral | None = None
|
|
51
|
+
) -> zarr.Group:
|
|
52
|
+
"""Check the group and return a valid group."""
|
|
53
|
+
if group.read_only and mode not in [None, "r"]:
|
|
54
|
+
raise NgioValueError(f"The group is read only. Cannot open in mode {mode}.")
|
|
55
|
+
|
|
56
|
+
if mode == "r" and not group.read_only:
|
|
57
|
+
# let's make sure we don't accidentally write to the group
|
|
58
|
+
group = zarr.open_group(store=group.store, path=group.path, mode="r")
|
|
59
|
+
return group
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def open_group_wrapper(
|
|
63
|
+
store: StoreOrGroup,
|
|
64
|
+
mode: AccessModeLiteral | None = None,
|
|
65
|
+
zarr_format: Literal[2, 3] | None = None,
|
|
66
|
+
) -> zarr.Group:
|
|
67
|
+
"""Wrapper around zarr.open_group with some additional checks.
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
store (StoreOrGroup): The store or group to open.
|
|
71
|
+
mode (AccessModeLiteral): The mode to open the group in.
|
|
72
|
+
zarr_format (int): The Zarr format version to use.
|
|
73
|
+
|
|
74
|
+
Returns:
|
|
75
|
+
zarr.Group: The opened Zarr group.
|
|
76
|
+
"""
|
|
77
|
+
if isinstance(store, zarr.Group):
|
|
78
|
+
group = _check_group(store, mode)
|
|
79
|
+
_check_store(group.store)
|
|
80
|
+
return group
|
|
81
|
+
|
|
82
|
+
try:
|
|
83
|
+
_check_store(store)
|
|
84
|
+
mode = mode if mode is not None else "a"
|
|
85
|
+
group = zarr.open_group(store=store, mode=mode, zarr_format=zarr_format)
|
|
86
|
+
|
|
87
|
+
except FileExistsError as e:
|
|
88
|
+
raise NgioFileExistsError(
|
|
89
|
+
f"A Zarr group already exists at {store}, consider setting overwrite=True."
|
|
90
|
+
) from e
|
|
91
|
+
|
|
92
|
+
except FileNotFoundError as e:
|
|
93
|
+
raise NgioFileNotFoundError(f"No Zarr group found at {store}") from e
|
|
94
|
+
|
|
95
|
+
except ContainsGroupError as e:
|
|
96
|
+
raise NgioFileExistsError(
|
|
97
|
+
f"A Zarr group already exists at {store}, consider setting overwrite=True."
|
|
98
|
+
) from e
|
|
99
|
+
|
|
100
|
+
return group
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class ZarrGroupHandler:
|
|
104
|
+
"""A simple wrapper around a Zarr group to handle metadata."""
|
|
105
|
+
|
|
106
|
+
def __init__(
|
|
107
|
+
self,
|
|
108
|
+
store: StoreOrGroup,
|
|
109
|
+
zarr_format: Literal[2, 3] | None = None,
|
|
110
|
+
cache: bool = False,
|
|
111
|
+
mode: AccessModeLiteral | None = None,
|
|
112
|
+
):
|
|
113
|
+
"""Initialize the handler.
|
|
114
|
+
|
|
115
|
+
Args:
|
|
116
|
+
store (StoreOrGroup): The Zarr store or group containing the image data.
|
|
117
|
+
meta_mode (str): The mode of the metadata handler.
|
|
118
|
+
zarr_format (int | None): The Zarr format version to use.
|
|
119
|
+
cache (bool): Whether to cache the metadata.
|
|
120
|
+
mode (str | None): The mode of the store.
|
|
121
|
+
"""
|
|
122
|
+
if mode not in ["r", "r+", "w", "w-", "a", None]:
|
|
123
|
+
raise NgioValueError(f"Mode {mode} is not supported.")
|
|
124
|
+
|
|
125
|
+
group = open_group_wrapper(store=store, mode=mode, zarr_format=zarr_format)
|
|
126
|
+
self._group = group
|
|
127
|
+
self.use_cache = cache
|
|
128
|
+
|
|
129
|
+
self._group_cache: NgioCache[zarr.Group] = NgioCache(use_cache=cache)
|
|
130
|
+
self._array_cache: NgioCache[zarr.Array] = NgioCache(use_cache=cache)
|
|
131
|
+
self._handlers_cache: NgioCache[ZarrGroupHandler] = NgioCache(use_cache=cache)
|
|
132
|
+
self._lock: tuple[Path, BaseFileLock] | None = None
|
|
133
|
+
|
|
134
|
+
def __repr__(self) -> str:
|
|
135
|
+
"""Return a string representation of the handler."""
|
|
136
|
+
return (
|
|
137
|
+
f"ZarrGroupHandler(full_url={self.full_url}, read_only={self.read_only}, "
|
|
138
|
+
f"cache={self.use_cache}"
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def store(self) -> Store:
|
|
143
|
+
"""Return the store of the group."""
|
|
144
|
+
return self._group.store
|
|
145
|
+
|
|
146
|
+
@property
|
|
147
|
+
def full_url(self) -> str | None:
|
|
148
|
+
"""Return the store path."""
|
|
149
|
+
if isinstance(self.store, LocalStore):
|
|
150
|
+
return (self.store.root / self.group.path).as_posix()
|
|
151
|
+
elif isinstance(self.store, FsspecStore):
|
|
152
|
+
return f"{self.store.path}/{self.group.path}"
|
|
153
|
+
elif isinstance(self.store, ZipStore):
|
|
154
|
+
return (self.store.path / self.group.path).as_posix()
|
|
155
|
+
elif isinstance(self.store, MemoryStore):
|
|
156
|
+
return None
|
|
157
|
+
warnings.warn(
|
|
158
|
+
f"Cannot determine full URL for store type {type(self.store)}. ",
|
|
159
|
+
UserWarning,
|
|
160
|
+
stacklevel=2,
|
|
161
|
+
)
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
@property
|
|
165
|
+
def zarr_format(self) -> Literal[2, 3]:
|
|
166
|
+
"""Return the Zarr format version."""
|
|
167
|
+
return self._group.metadata.zarr_format
|
|
168
|
+
|
|
169
|
+
@property
|
|
170
|
+
def read_only(self) -> bool:
|
|
171
|
+
"""Return whether the group is read only."""
|
|
172
|
+
return self._group.read_only
|
|
173
|
+
|
|
174
|
+
def _create_lock(self) -> tuple[Path, BaseFileLock]:
|
|
175
|
+
"""Create the lock."""
|
|
176
|
+
if self._lock is not None:
|
|
177
|
+
return self._lock
|
|
178
|
+
|
|
179
|
+
if self.use_cache is True:
|
|
180
|
+
raise NgioValueError(
|
|
181
|
+
"Lock mechanism is not compatible with caching. "
|
|
182
|
+
"Please set cache=False to use the lock mechanism."
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
if not isinstance(self.store, LocalStore):
|
|
186
|
+
raise NgioValueError(
|
|
187
|
+
"The store needs to be a LocalStore to use the lock mechanism. "
|
|
188
|
+
f"Instead, got {self.store.__class__.__name__}."
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
store_path = Path(self.store.root) / self.group.path
|
|
192
|
+
_lock_path = store_path.with_suffix(".lock")
|
|
193
|
+
_lock = FileLock(_lock_path, timeout=10)
|
|
194
|
+
return _lock_path, _lock
|
|
195
|
+
|
|
196
|
+
@property
|
|
197
|
+
def lock(self) -> BaseFileLock:
|
|
198
|
+
"""Return the lock."""
|
|
199
|
+
if self._lock is None:
|
|
200
|
+
self._lock = self._create_lock()
|
|
201
|
+
return self._lock[1]
|
|
202
|
+
|
|
203
|
+
@property
|
|
204
|
+
def lock_path(self) -> Path:
|
|
205
|
+
"""Return the lock path."""
|
|
206
|
+
if self._lock is None:
|
|
207
|
+
self._lock = self._create_lock()
|
|
208
|
+
return self._lock[0]
|
|
209
|
+
|
|
210
|
+
def remove_lock(self) -> None:
|
|
211
|
+
"""Return the lock."""
|
|
212
|
+
if self._lock is None:
|
|
213
|
+
return None
|
|
214
|
+
|
|
215
|
+
lock_path, lock = self._lock
|
|
216
|
+
if lock_path.exists() and lock.lock_counter == 0:
|
|
217
|
+
lock_path.unlink()
|
|
218
|
+
self._lock = None
|
|
219
|
+
return None
|
|
220
|
+
|
|
221
|
+
raise NgioValueError("The lock is still in use. Cannot remove it.")
|
|
222
|
+
|
|
223
|
+
def reopen_group(self) -> zarr.Group:
|
|
224
|
+
"""Reopen the group.
|
|
225
|
+
|
|
226
|
+
This is useful when the group has been modified
|
|
227
|
+
outside of the handler.
|
|
228
|
+
"""
|
|
229
|
+
mode = "r" if self.read_only else "r+"
|
|
230
|
+
return zarr.open_group(
|
|
231
|
+
store=self._group.store,
|
|
232
|
+
path=self._group.path,
|
|
233
|
+
mode=mode,
|
|
234
|
+
zarr_format=self._group.metadata.zarr_format,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
def reopen_handler(self) -> "ZarrGroupHandler":
|
|
238
|
+
"""Reopen the handler.
|
|
239
|
+
|
|
240
|
+
This is useful when the group has been modified
|
|
241
|
+
outside of the handler.
|
|
242
|
+
"""
|
|
243
|
+
mode = "r" if self.read_only else "r+"
|
|
244
|
+
group = self.reopen_group()
|
|
245
|
+
return ZarrGroupHandler(
|
|
246
|
+
store=group,
|
|
247
|
+
zarr_format=group.metadata.zarr_format,
|
|
248
|
+
cache=self.use_cache,
|
|
249
|
+
mode=mode,
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
def clean_cache(self) -> None:
|
|
253
|
+
"""Clear the cached metadata."""
|
|
254
|
+
group = self.reopen_group()
|
|
255
|
+
self.__init__(
|
|
256
|
+
store=group,
|
|
257
|
+
zarr_format=group.metadata.zarr_format,
|
|
258
|
+
cache=self.use_cache,
|
|
259
|
+
mode="r" if self.read_only else "r+",
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
@property
|
|
263
|
+
def group(self) -> zarr.Group:
|
|
264
|
+
"""Return the group."""
|
|
265
|
+
if self.use_cache is False:
|
|
266
|
+
# If we are not using cache, we need to reopen the group
|
|
267
|
+
# to make sure that the attributes are up to date
|
|
268
|
+
return self.reopen_group()
|
|
269
|
+
return self._group
|
|
270
|
+
|
|
271
|
+
def load_attrs(self) -> dict:
|
|
272
|
+
"""Load the attributes of the group."""
|
|
273
|
+
return self.reopen_group().attrs.asdict()
|
|
274
|
+
|
|
275
|
+
def write_attrs(self, attrs: dict, overwrite: bool = False) -> None:
|
|
276
|
+
"""Write the metadata to the store."""
|
|
277
|
+
# Maybe we should use the lock here
|
|
278
|
+
if self.read_only:
|
|
279
|
+
raise NgioValueError("The group is read only. Cannot write metadata.")
|
|
280
|
+
group = self.reopen_group()
|
|
281
|
+
if overwrite:
|
|
282
|
+
group.attrs.clear()
|
|
283
|
+
group.attrs.update(attrs)
|
|
284
|
+
|
|
285
|
+
def create_group(self, path: str, overwrite: bool = False) -> zarr.Group:
|
|
286
|
+
"""Create a group in the group."""
|
|
287
|
+
if self.group.read_only:
|
|
288
|
+
raise NgioValueError("Cannot create a group in read only mode.")
|
|
289
|
+
|
|
290
|
+
try:
|
|
291
|
+
group = self.group.create_group(path, overwrite=overwrite)
|
|
292
|
+
except ContainsGroupError as e:
|
|
293
|
+
raise NgioFileExistsError(
|
|
294
|
+
f"A Zarr group already exists at {path}, "
|
|
295
|
+
"consider setting overwrite=True."
|
|
296
|
+
) from e
|
|
297
|
+
self._group_cache.set(path, group, overwrite=overwrite)
|
|
298
|
+
return group
|
|
299
|
+
|
|
300
|
+
def get_group(
|
|
301
|
+
self,
|
|
302
|
+
path: str,
|
|
303
|
+
create_mode: bool = False,
|
|
304
|
+
overwrite: bool = False,
|
|
305
|
+
) -> zarr.Group:
|
|
306
|
+
"""Get a group from the group.
|
|
307
|
+
|
|
308
|
+
Args:
|
|
309
|
+
path (str): The path to the group.
|
|
310
|
+
create_mode (bool): If True, create the group if it does not exist.
|
|
311
|
+
overwrite (bool): If True, overwrite the group if it exists.
|
|
312
|
+
|
|
313
|
+
Returns:
|
|
314
|
+
zarr.Group: The Zarr group.
|
|
315
|
+
|
|
316
|
+
"""
|
|
317
|
+
if overwrite and not create_mode:
|
|
318
|
+
raise NgioValueError("Cannot overwrite a group without create_mode=True.")
|
|
319
|
+
|
|
320
|
+
if overwrite:
|
|
321
|
+
return self.create_group(path, overwrite=overwrite)
|
|
322
|
+
|
|
323
|
+
group = self._group_cache.get(path)
|
|
324
|
+
if isinstance(group, zarr.Group):
|
|
325
|
+
return group
|
|
326
|
+
|
|
327
|
+
group = self.group.get(path, default=None)
|
|
328
|
+
if isinstance(group, zarr.Group):
|
|
329
|
+
self._group_cache.set(path, group, overwrite=overwrite)
|
|
330
|
+
return group
|
|
331
|
+
|
|
332
|
+
if isinstance(group, zarr.Array):
|
|
333
|
+
raise NgioValueError(f"The object at {path} is not a group, but an array.")
|
|
334
|
+
|
|
335
|
+
if not create_mode:
|
|
336
|
+
raise NgioFileNotFoundError(f"No group found at {path}")
|
|
337
|
+
group = self.create_group(path)
|
|
338
|
+
self._group_cache.set(path, group, overwrite=overwrite)
|
|
339
|
+
return group
|
|
340
|
+
|
|
341
|
+
def get_array(self, path: str) -> zarr.Array:
|
|
342
|
+
"""Get an array from the group."""
|
|
343
|
+
array = self._array_cache.get(path)
|
|
344
|
+
if isinstance(array, zarr.Array):
|
|
345
|
+
return array
|
|
346
|
+
array = self.group.get(path, default=None)
|
|
347
|
+
if isinstance(array, zarr.Array):
|
|
348
|
+
self._array_cache.set(path, array)
|
|
349
|
+
return array
|
|
350
|
+
|
|
351
|
+
if isinstance(array, zarr.Group):
|
|
352
|
+
raise NgioValueError(f"The object at {path} is not an array, but a group.")
|
|
353
|
+
raise NgioFileNotFoundError(f"No array found at {path}")
|
|
354
|
+
|
|
355
|
+
def get_handler(
|
|
356
|
+
self,
|
|
357
|
+
path: str,
|
|
358
|
+
create_mode: bool = True,
|
|
359
|
+
overwrite: bool = False,
|
|
360
|
+
) -> "ZarrGroupHandler":
|
|
361
|
+
"""Get a new handler for a group in the current handler group.
|
|
362
|
+
|
|
363
|
+
Args:
|
|
364
|
+
path (str): The path to the group.
|
|
365
|
+
create_mode (bool): If True, create the group if it does not exist.
|
|
366
|
+
overwrite (bool): If True, overwrite the group if it exists.
|
|
367
|
+
"""
|
|
368
|
+
handler = self._handlers_cache.get(path)
|
|
369
|
+
if handler is not None:
|
|
370
|
+
return handler
|
|
371
|
+
group = self.get_group(path, create_mode=create_mode, overwrite=overwrite)
|
|
372
|
+
mode = "r" if group.read_only else "r+"
|
|
373
|
+
handler = ZarrGroupHandler(
|
|
374
|
+
store=group, zarr_format=self.zarr_format, cache=self.use_cache, mode=mode
|
|
375
|
+
)
|
|
376
|
+
self._handlers_cache.set(path, handler)
|
|
377
|
+
return handler
|
|
378
|
+
|
|
379
|
+
@property
|
|
380
|
+
def is_listable(self) -> bool:
|
|
381
|
+
return is_group_listable(self.group)
|
|
382
|
+
|
|
383
|
+
def delete_group(self, path: str) -> None:
|
|
384
|
+
"""Delete a group from the current group.
|
|
385
|
+
|
|
386
|
+
Args:
|
|
387
|
+
path (str): The path to the group to delete.
|
|
388
|
+
"""
|
|
389
|
+
if self.group.read_only:
|
|
390
|
+
raise NgioValueError("Cannot delete a group in read only mode.")
|
|
391
|
+
self.group.__delitem__(path)
|
|
392
|
+
self._group_cache._cache.pop(path, None)
|
|
393
|
+
self._handlers_cache._cache.pop(path, None)
|
|
394
|
+
|
|
395
|
+
def delete_self(self) -> None:
|
|
396
|
+
"""Delete the current group."""
|
|
397
|
+
if self.group.read_only:
|
|
398
|
+
raise NgioValueError("Cannot delete a group in read only mode.")
|
|
399
|
+
self.group.__delitem__("/")
|
|
400
|
+
|
|
401
|
+
def copy_group(self, dest_group: zarr.Group):
|
|
402
|
+
"""Copy the group to a new store."""
|
|
403
|
+
copy_group(self.group, dest_group)
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def find_dimension_separator(array: zarr.Array) -> Literal[".", "/"]:
|
|
407
|
+
"""Find the dimension separator used in the Zarr store.
|
|
408
|
+
|
|
409
|
+
Args:
|
|
410
|
+
array (zarr.Array): The Zarr array to check.
|
|
411
|
+
|
|
412
|
+
Returns:
|
|
413
|
+
Literal[".", "/"]: The dimension separator used in the store.
|
|
414
|
+
"""
|
|
415
|
+
from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding
|
|
416
|
+
|
|
417
|
+
if array.metadata.zarr_format == 2:
|
|
418
|
+
separator = array.metadata.dimension_separator
|
|
419
|
+
else:
|
|
420
|
+
separator = array.metadata.chunk_key_encoding
|
|
421
|
+
if not isinstance(separator, DefaultChunkKeyEncoding):
|
|
422
|
+
raise NgioValueError(
|
|
423
|
+
"Only DefaultChunkKeyEncoding is supported in this example."
|
|
424
|
+
)
|
|
425
|
+
separator = separator.separator
|
|
426
|
+
return separator
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def is_group_listable(group: zarr.Group) -> bool:
|
|
430
|
+
"""Check if a Zarr group is listable.
|
|
431
|
+
|
|
432
|
+
A group is considered listable if it contains at least one array or subgroup.
|
|
433
|
+
|
|
434
|
+
Args:
|
|
435
|
+
group (zarr.Group): The Zarr group to check.
|
|
436
|
+
|
|
437
|
+
Returns:
|
|
438
|
+
bool: True if the group is listable, False otherwise.
|
|
439
|
+
"""
|
|
440
|
+
if not group.store.supports_listing:
|
|
441
|
+
# If the store does not support listing
|
|
442
|
+
# then for sure it is not listable
|
|
443
|
+
return False
|
|
444
|
+
try:
|
|
445
|
+
next(group.keys())
|
|
446
|
+
return True
|
|
447
|
+
except StopIteration:
|
|
448
|
+
# Group is listable but empty
|
|
449
|
+
return True
|
|
450
|
+
except Exception as _:
|
|
451
|
+
# Some stores may raise errors when listing
|
|
452
|
+
# consider those not listable
|
|
453
|
+
return False
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _make_sync_fs(fs: fsspec.AbstractFileSystem) -> fsspec.AbstractFileSystem:
|
|
457
|
+
fs_dict = json.loads(fs.to_json())
|
|
458
|
+
fs_dict["asynchronous"] = False
|
|
459
|
+
return fsspec.AbstractFileSystem.from_json(json.dumps(fs_dict))
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _get_mapper(store: LocalStore | FsspecStore, path: str):
|
|
463
|
+
if isinstance(store, LocalStore):
|
|
464
|
+
fs = fsspec.filesystem("file")
|
|
465
|
+
full_path = (store.root / path).as_posix()
|
|
466
|
+
else:
|
|
467
|
+
fs = _make_sync_fs(store.fs)
|
|
468
|
+
full_path = f"{store.path}/{path}"
|
|
469
|
+
return fs.get_mapper(full_path)
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _fsspec_copy(
|
|
473
|
+
src_fs: LocalStore | FsspecStore,
|
|
474
|
+
src_path: str,
|
|
475
|
+
dest_fs: LocalStore | FsspecStore,
|
|
476
|
+
dest_path: str,
|
|
477
|
+
):
|
|
478
|
+
src_mapper = _get_mapper(src_fs, src_path)
|
|
479
|
+
dest_mapper = _get_mapper(dest_fs, dest_path)
|
|
480
|
+
for key in src_mapper.keys():
|
|
481
|
+
dest_mapper[key] = src_mapper[key]
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def _zarr_python_copy(src_group: zarr.Group, dest_group: zarr.Group):
|
|
485
|
+
# Copy attributes
|
|
486
|
+
dest_group.attrs.put(src_group.attrs.asdict())
|
|
487
|
+
# Copy arrays
|
|
488
|
+
for name, array in src_group.arrays():
|
|
489
|
+
if array.metadata.zarr_format == 2:
|
|
490
|
+
spec = AnyArraySpecV2.from_zarr(array)
|
|
491
|
+
else:
|
|
492
|
+
spec = AnyArraySpecV3.from_zarr(array)
|
|
493
|
+
dst = spec.to_zarr(
|
|
494
|
+
store=dest_group.store,
|
|
495
|
+
path=f"{dest_group.path}/{name}",
|
|
496
|
+
overwrite=True,
|
|
497
|
+
)
|
|
498
|
+
if array.ndim > 0:
|
|
499
|
+
dask_array = da.from_zarr(array)
|
|
500
|
+
da.to_zarr(dask_array, dst, overwrite=False)
|
|
501
|
+
# Copy subgroups
|
|
502
|
+
for name, subgroup in src_group.groups():
|
|
503
|
+
dest_subgroup = dest_group.create_group(name, overwrite=True)
|
|
504
|
+
_zarr_python_copy(subgroup, dest_subgroup)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def copy_group(
|
|
508
|
+
src_group: zarr.Group, dest_group: zarr.Group, suppress_warnings: bool = False
|
|
509
|
+
):
|
|
510
|
+
if src_group.metadata.zarr_format != dest_group.metadata.zarr_format:
|
|
511
|
+
raise NgioValueError(
|
|
512
|
+
"Different Zarr format versions between source and destination, "
|
|
513
|
+
"cannot copy."
|
|
514
|
+
)
|
|
515
|
+
|
|
516
|
+
if not is_group_listable(src_group):
|
|
517
|
+
raise NgioValueError("Source group is not listable, cannot copy.")
|
|
518
|
+
|
|
519
|
+
if dest_group.read_only:
|
|
520
|
+
raise NgioValueError("Destination group is read only, cannot copy.")
|
|
521
|
+
if isinstance(src_group.store, LocalStore | FsspecStore) and isinstance(
|
|
522
|
+
dest_group.store, LocalStore | FsspecStore
|
|
523
|
+
):
|
|
524
|
+
_fsspec_copy(src_group.store, src_group.path, dest_group.store, dest_group.path)
|
|
525
|
+
return
|
|
526
|
+
if not suppress_warnings:
|
|
527
|
+
warnings.warn(
|
|
528
|
+
"Fsspec copy not possible, falling back to Zarr Python API for the copy. "
|
|
529
|
+
"This will preserve some tabular data non-zarr native (parquet, and csv), "
|
|
530
|
+
"and it will be slower for large datasets.",
|
|
531
|
+
UserWarning,
|
|
532
|
+
stacklevel=2,
|
|
533
|
+
)
|
|
534
|
+
_zarr_python_copy(src_group, dest_group)
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ngio
|
|
3
|
+
Version: 0.5.0b6
|
|
4
|
+
Summary: Next Generation file format IO
|
|
5
|
+
Project-URL: homepage, https://github.com/BioVisionCenter/ngio
|
|
6
|
+
Project-URL: repository, https://github.com/BioVisionCenter/ngio
|
|
7
|
+
Author-email: Lorenzo Cerrone <lorenzo.cerrone@uzh.ch>
|
|
8
|
+
License: BSD-3-Clause
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Requires-Python: <3.15,>=3.11
|
|
19
|
+
Requires-Dist: aiohttp
|
|
20
|
+
Requires-Dist: anndata
|
|
21
|
+
Requires-Dist: dask[array]<2025.11.0
|
|
22
|
+
Requires-Dist: dask[distributed]<2025.11.0
|
|
23
|
+
Requires-Dist: filelock
|
|
24
|
+
Requires-Dist: numpy
|
|
25
|
+
Requires-Dist: ome-zarr-models
|
|
26
|
+
Requires-Dist: pandas>=1.2.0
|
|
27
|
+
Requires-Dist: pillow
|
|
28
|
+
Requires-Dist: polars
|
|
29
|
+
Requires-Dist: pooch
|
|
30
|
+
Requires-Dist: pyarrow
|
|
31
|
+
Requires-Dist: pydantic
|
|
32
|
+
Requires-Dist: requests
|
|
33
|
+
Requires-Dist: zarr>3
|
|
34
|
+
Provides-Extra: dev
|
|
35
|
+
Requires-Dist: matplotlib; extra == 'dev'
|
|
36
|
+
Requires-Dist: mypy; extra == 'dev'
|
|
37
|
+
Requires-Dist: napari; extra == 'dev'
|
|
38
|
+
Requires-Dist: notebook; extra == 'dev'
|
|
39
|
+
Requires-Dist: pdbpp; extra == 'dev'
|
|
40
|
+
Requires-Dist: pre-commit; extra == 'dev'
|
|
41
|
+
Requires-Dist: pympler; extra == 'dev'
|
|
42
|
+
Requires-Dist: pyqt5; extra == 'dev'
|
|
43
|
+
Requires-Dist: rich; extra == 'dev'
|
|
44
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
45
|
+
Requires-Dist: scikit-image; extra == 'dev'
|
|
46
|
+
Requires-Dist: zarrs; extra == 'dev'
|
|
47
|
+
Provides-Extra: docs
|
|
48
|
+
Requires-Dist: griffe-typingdoc; extra == 'docs'
|
|
49
|
+
Requires-Dist: markdown-exec[ansi]; extra == 'docs'
|
|
50
|
+
Requires-Dist: matplotlib; extra == 'docs'
|
|
51
|
+
Requires-Dist: mike; extra == 'docs'
|
|
52
|
+
Requires-Dist: mkdocs; extra == 'docs'
|
|
53
|
+
Requires-Dist: mkdocs-autorefs; extra == 'docs'
|
|
54
|
+
Requires-Dist: mkdocs-git-committers-plugin-2; extra == 'docs'
|
|
55
|
+
Requires-Dist: mkdocs-git-revision-date-localized-plugin; extra == 'docs'
|
|
56
|
+
Requires-Dist: mkdocs-include-markdown-plugin; extra == 'docs'
|
|
57
|
+
Requires-Dist: mkdocs-jupyter; extra == 'docs'
|
|
58
|
+
Requires-Dist: mkdocs-material; extra == 'docs'
|
|
59
|
+
Requires-Dist: mkdocstrings[python]; extra == 'docs'
|
|
60
|
+
Requires-Dist: rich; extra == 'docs'
|
|
61
|
+
Requires-Dist: scikit-image; extra == 'docs'
|
|
62
|
+
Requires-Dist: tabulate; extra == 'docs'
|
|
63
|
+
Provides-Extra: test
|
|
64
|
+
Requires-Dist: boto; extra == 'test'
|
|
65
|
+
Requires-Dist: devtools; extra == 'test'
|
|
66
|
+
Requires-Dist: moto[server]; extra == 'test'
|
|
67
|
+
Requires-Dist: pytest; extra == 'test'
|
|
68
|
+
Requires-Dist: pytest-cov; extra == 'test'
|
|
69
|
+
Requires-Dist: pytest-httpserver; extra == 'test'
|
|
70
|
+
Requires-Dist: s3fs; extra == 'test'
|
|
71
|
+
Requires-Dist: scikit-image; extra == 'test'
|
|
72
|
+
Description-Content-Type: text/markdown
|
|
73
|
+
|
|
74
|
+
# Ngio - Next Generation file format IO
|
|
75
|
+
|
|
76
|
+
[](https://github.com/BioVisionCenter/ngio/raw/main/LICENSE)
|
|
77
|
+
[](https://pypi.org/project/ngio)
|
|
78
|
+
[](https://python.org)
|
|
79
|
+
[](https://github.com/BioVisionCenter/ngio/actions/workflows/ci.yml)
|
|
80
|
+
[](https://codecov.io/gh/BioVisionCenter/ngio)
|
|
81
|
+
|
|
82
|
+
ngio is a Python library designed to simplify bioimage analysis workflows, offering an intuitive interface for working with OME-Zarr files.
|
|
83
|
+
|
|
84
|
+
## What is Ngio?
|
|
85
|
+
|
|
86
|
+
Ngio is built for the [OME-Zarr](https://ngff.openmicroscopy.org/) file format, a modern, cloud-optimized format for biological imaging data. OME-Zarr stores large, multi-dimensional microscopy images and metadata in an efficient and scalable way.
|
|
87
|
+
|
|
88
|
+
Ngio's mission is to streamline working with OME-Zarr files by providing a simple, object-based API for opening, exploring, and manipulating OME-Zarr images and high-content screening (HCS) plates. It also offers comprehensive support for labels, tables and regions of interest (ROIs), making it easy to extract and analyze specific regions in your data.
|
|
89
|
+
|
|
90
|
+
## Key Features
|
|
91
|
+
|
|
92
|
+
### 🔍 Simple Object-Based API
|
|
93
|
+
|
|
94
|
+
- Easily open, explore, and manipulate OME-Zarr images and HCS plates
|
|
95
|
+
- Create and derive new images and labels with minimal boilerplate code
|
|
96
|
+
|
|
97
|
+
### 📊 Rich Tables and Regions of Interest (ROI) Support
|
|
98
|
+
|
|
99
|
+
- Tight integration with [tabular data](https://biovisioncenter.github.io/ngio/stable/table_specs/overview/)
|
|
100
|
+
- Extract and analyze specific regions of interest
|
|
101
|
+
- Store measurements and other metadata in the OME-Zarr container
|
|
102
|
+
- Extensible & modular allowing users to define custom table schemas and on disk serialization
|
|
103
|
+
|
|
104
|
+
### 🔄 Scalable Data Processing
|
|
105
|
+
|
|
106
|
+
- Powerful iterators for building scalable and generalizable image processing pipelines
|
|
107
|
+
- Extensible mapping mechanism for custom parallelization strategies
|
|
108
|
+
|
|
109
|
+
## Installation
|
|
110
|
+
|
|
111
|
+
You can install ngio via pip:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
pip install ngio
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
To get started check out the [Quickstart Guide](https://BioVisionCenter.github.io/ngio/stable/getting_started/0_quickstart/).
|
|
118
|
+
|
|
119
|
+
## Supported OME-Zarr versions
|
|
120
|
+
|
|
121
|
+
Currently, ngio only supports OME-Zarr v0.4. Support for version 0.5 and higher is planned for future releases.
|
|
122
|
+
|
|
123
|
+
## Development Status
|
|
124
|
+
|
|
125
|
+
Ngio is under active development and is not yet stable. The API is subject to change, and bugs and breaking changes are expected.
|
|
126
|
+
We follow [Semantic Versioning](https://semver.org/). Which means for 0.x releases potentially breaking changes can be introduced in minor releases.
|
|
127
|
+
|
|
128
|
+
### Available Features
|
|
129
|
+
|
|
130
|
+
- ✅ OME-Zarr metadata handling and validation
|
|
131
|
+
- ✅ Image and label access across pyramid levels
|
|
132
|
+
- ✅ ROI and table support
|
|
133
|
+
- ✅ Image processing iterators
|
|
134
|
+
- ✅ Streaming from remote sources
|
|
135
|
+
- ✅ Documentation and examples
|
|
136
|
+
|
|
137
|
+
### Upcoming Features
|
|
138
|
+
|
|
139
|
+
- Support for OME-Zarr v0.5 and Zarr v3 (via `zarr-python` v3)
|
|
140
|
+
- Enhanced performance optimizations (parallel iterators, optimized io strategies)
|
|
141
|
+
|
|
142
|
+
## Contributors
|
|
143
|
+
|
|
144
|
+
Ngio is developed at the [BioVisionCenter](https://www.biovisioncenter.uzh.ch/en.html), University of Zurich, by [@lorenzocerrone](https://github.com/lorenzocerrone) and [@jluethi](https://github.com/jluethi).
|
|
145
|
+
|
|
146
|
+
## License
|
|
147
|
+
|
|
148
|
+
Ngio is released under the BSD-3-Clause License. See [LICENSE](https://github.com/BioVisionCenter/ngio/blob/main/LICENSE) for details.
|