worldarray 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- worldarray/__init__.py +43 -0
- worldarray/array.py +935 -0
- worldarray/context.py +53 -0
- worldarray/geometry.py +173 -0
- worldarray/levels.py +147 -0
- worldarray/placement.py +739 -0
- worldarray/resample.py +229 -0
- worldarray/retry.py +76 -0
- worldarray/store.py +427 -0
- worldarray/units.py +77 -0
- worldarray-0.1.0.dist-info/METADATA +214 -0
- worldarray-0.1.0.dist-info/RECORD +14 -0
- worldarray-0.1.0.dist-info/WHEEL +4 -0
- worldarray-0.1.0.dist-info/licenses/LICENSE +28 -0
worldarray/placement.py
ADDED
|
@@ -0,0 +1,739 @@
|
|
|
1
|
+
"""What an array is and where it sits: ``ArrayMeta``, read per format by one precedence.
|
|
2
|
+
|
|
3
|
+
The rule everything else rests on: a **translation is the centre of voxel 0** (OME-NGFF,
|
|
4
|
+
COSEM ``transform``, CF coordinates, Neuroglancer's own zarr reader). Formats that store
|
|
5
|
+
voxel 0's corner are moved half a voxel along the spatial axes on reading: precomputed's
|
|
6
|
+
``voxel_offset`` and funlib's ``offset`` (0 when only a ``resolution`` is given). N5 levels
|
|
7
|
+
with ``pixelResolution`` and ``downsamplingFactors`` and no offset follow BigDataViewer: a
|
|
8
|
+
downsampled voxel's centre is the centre of the block it covers (spec, Conventions 1 and 4).
|
|
9
|
+
|
|
10
|
+
Per-axis lists shorter than the array (a 3-entry ``voxel_size`` on a ``(c, z, y, x)``
|
|
11
|
+
prediction) cover the trailing axes; the leading axes get 1.0, 0.0 or ``""``. OME lists
|
|
12
|
+
must match the array exactly.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import logging
|
|
19
|
+
import re
|
|
20
|
+
from dataclasses import dataclass, field, replace
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
import tensorstore as ts
|
|
24
|
+
|
|
25
|
+
from . import units as U
|
|
26
|
+
from .store import Location, Node, basename, is_root, multiscales_of, parent
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
DEFAULT_AXES = ("t", "c", "z", "y", "x")
|
|
31
|
+
SPATIAL_NAMES = frozenset({"z", "y", "x"})
|
|
32
|
+
RTOL = 1e-6
|
|
33
|
+
PLACED_BY = ("ome", "cosem", "funlib", "n5", "precomputed", "dimension_units")
|
|
34
|
+
|
|
35
|
+
_warned_unitless: set[str] = set()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class ArrayMeta:
|
|
40
|
+
"""One entry per axis, in the array's C order. The same field names as chunkmirage's
|
|
41
|
+
``ArrayInfo`` so its adapter is a one-liner."""
|
|
42
|
+
|
|
43
|
+
shape: tuple[int, ...]
|
|
44
|
+
dtype: np.dtype
|
|
45
|
+
chunk_shape: tuple[int, ...]
|
|
46
|
+
axes: tuple[str, ...]
|
|
47
|
+
axes_named: bool
|
|
48
|
+
units: tuple[str, ...] # "" when the store gives none; any string is accepted
|
|
49
|
+
voxel_size: tuple[float, ...] # one index step, in `units`; 1.0 on unitless axes
|
|
50
|
+
translation: tuple[float, ...] # the CENTRE of voxel 0, in `units`
|
|
51
|
+
format: str # zarr2 | zarr3 | n5 | precomputed
|
|
52
|
+
path: str
|
|
53
|
+
parent: str | None = None
|
|
54
|
+
attrs: dict = field(default_factory=dict)
|
|
55
|
+
group_attrs: dict = field(default_factory=dict)
|
|
56
|
+
placed_by: str | None = None # which rule gave voxel_size and translation; None: nothing did
|
|
57
|
+
fill_value: object = None # the store's fill value, as tensorstore reports it
|
|
58
|
+
|
|
59
|
+
def __post_init__(self) -> None:
|
|
60
|
+
object.__setattr__(self, "dtype", np.dtype(self.dtype))
|
|
61
|
+
n = len(self.shape)
|
|
62
|
+
for name in ("chunk_shape", "axes", "units", "voxel_size", "translation"):
|
|
63
|
+
if len(getattr(self, name)) != n:
|
|
64
|
+
raise ValueError(f"{name} must have length {n}, got {getattr(self, name)}")
|
|
65
|
+
object.__setattr__(self, "voxel_size", tuple(float(v) for v in self.voxel_size))
|
|
66
|
+
object.__setattr__(self, "translation", tuple(float(v) for v in self.translation))
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def ndim(self) -> int:
|
|
70
|
+
return len(self.shape)
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def spatial(self) -> tuple[int, ...]:
|
|
74
|
+
"""Indices of the axes named z, y or x when any are; else the last three."""
|
|
75
|
+
return spatial_axes(self.axes)
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def corner(self) -> tuple[float, ...]:
|
|
79
|
+
"""``translation - voxel_size / 2`` along the spatial axes, unchanged elsewhere."""
|
|
80
|
+
sp = set(self.spatial)
|
|
81
|
+
return tuple(
|
|
82
|
+
t - v / 2 if i in sp else t
|
|
83
|
+
for i, (t, v) in enumerate(zip(self.translation, self.voxel_size))
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
def in_units(
|
|
87
|
+
self, space: str | None = None, time: str | None = None, strict: bool = True
|
|
88
|
+
) -> ArrayMeta:
|
|
89
|
+
"""The same metadata with spatial axes in ``space`` and time axes in ``time``.
|
|
90
|
+
Axes not asked about keep their unit string, whatever it is. A spatial axis with no
|
|
91
|
+
unit is taken as nanometres, with a warning once per array (a BigDataViewer N5 with
|
|
92
|
+
a unitless micrometre ``resolution`` would otherwise be read a thousandfold too
|
|
93
|
+
small, silently); an unknown unit on an axis being converted raises."""
|
|
94
|
+
sp = set(self.spatial)
|
|
95
|
+
units, vs, tr = list(self.units), list(self.voxel_size), list(self.translation)
|
|
96
|
+
for i, unit in enumerate(self.units):
|
|
97
|
+
k = U.kind(unit)
|
|
98
|
+
target = None
|
|
99
|
+
if i in sp and space is not None and k in ("space", "unknown"):
|
|
100
|
+
target = space
|
|
101
|
+
if unit == "" and self.path not in _warned_unitless:
|
|
102
|
+
_warned_unitless.add(self.path)
|
|
103
|
+
logger.warning(
|
|
104
|
+
"%s: no unit stored on spatial axis %r; taking nanometres (pass units=None to keep the store's numbers)",
|
|
105
|
+
self.path,
|
|
106
|
+
self.axes[i],
|
|
107
|
+
)
|
|
108
|
+
if not strict and unit != "" and U.kind(unit) == "unknown":
|
|
109
|
+
logger.warning(
|
|
110
|
+
"%s: unknown unit %r on axis %r taken as %s",
|
|
111
|
+
self.path,
|
|
112
|
+
unit,
|
|
113
|
+
self.axes[i],
|
|
114
|
+
space,
|
|
115
|
+
)
|
|
116
|
+
f = U.factor(unit, space, missing_as=U.NANOMETER, lenient=not strict)
|
|
117
|
+
elif k == "time" and time is not None:
|
|
118
|
+
target = time
|
|
119
|
+
f = U.factor(unit, time)
|
|
120
|
+
if target is not None:
|
|
121
|
+
units[i], vs[i], tr[i] = target, vs[i] * f, tr[i] * f
|
|
122
|
+
return replace(self, units=tuple(units), voxel_size=tuple(vs), translation=tuple(tr))
|
|
123
|
+
|
|
124
|
+
def in_nm(self) -> ArrayMeta:
|
|
125
|
+
return self.in_units(space=U.NANOMETER)
|
|
126
|
+
|
|
127
|
+
def drop_axis(self, axis: int) -> ArrayMeta:
|
|
128
|
+
keep = [i for i in range(self.ndim) if i != axis]
|
|
129
|
+
pick = lambda t: tuple(t[i] for i in keep) # noqa: E731
|
|
130
|
+
return replace(
|
|
131
|
+
self,
|
|
132
|
+
shape=pick(self.shape),
|
|
133
|
+
chunk_shape=pick(self.chunk_shape),
|
|
134
|
+
axes=pick(self.axes),
|
|
135
|
+
units=pick(self.units),
|
|
136
|
+
voxel_size=pick(self.voxel_size),
|
|
137
|
+
translation=pick(self.translation),
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def spatial_axes(axes: tuple[str, ...]) -> tuple[int, ...]:
|
|
142
|
+
named = tuple(i for i, a in enumerate(axes) if a in SPATIAL_NAMES)
|
|
143
|
+
if named:
|
|
144
|
+
return named
|
|
145
|
+
return tuple(range(max(0, len(axes) - 3), len(axes)))
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def default_axes(ndim: int) -> tuple[str, ...]:
|
|
149
|
+
if ndim <= len(DEFAULT_AXES):
|
|
150
|
+
return DEFAULT_AXES[len(DEFAULT_AXES) - ndim :]
|
|
151
|
+
return tuple(f"dim_{i}" for i in range(ndim - len(DEFAULT_AXES))) + DEFAULT_AXES
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def corner_to_centre(corner, voxel_size, axes=None) -> tuple[float, ...]:
|
|
155
|
+
"""Voxel 0's centre from its corner: half a voxel on along the spatial axes (those named
|
|
156
|
+
z, y or x; the last three when ``axes`` is None or names none of them)."""
|
|
157
|
+
corner, voxel_size = tuple(corner), tuple(voxel_size)
|
|
158
|
+
sp = set(spatial_axes(tuple(axes) if axes is not None else default_axes(len(corner))))
|
|
159
|
+
return tuple(
|
|
160
|
+
float(c) + (float(v) / 2 if i in sp else 0.0)
|
|
161
|
+
for i, (c, v) in enumerate(zip(corner, voxel_size))
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
# ---------------------------------------------------------------------------
|
|
166
|
+
# Reading
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@dataclass
|
|
170
|
+
class _Resolved:
|
|
171
|
+
"""What a parser found: per-axis tuples, each None when the store said nothing."""
|
|
172
|
+
|
|
173
|
+
axes: tuple[str, ...] | None = None
|
|
174
|
+
units: tuple[str, ...] | None = None
|
|
175
|
+
voxel_size: tuple[float, ...] | None = None
|
|
176
|
+
translation: tuple[float, ...] | None = None
|
|
177
|
+
placed_by: str | None = None
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def read_meta(
|
|
181
|
+
node: Node,
|
|
182
|
+
store: ts.TensorStore,
|
|
183
|
+
*,
|
|
184
|
+
scale_index: int | None = None,
|
|
185
|
+
conflicts: str = "error",
|
|
186
|
+
) -> ArrayMeta:
|
|
187
|
+
"""The metadata of ``store``, an array ``node`` (or one scale of a precomputed node)
|
|
188
|
+
opened by ``store.open_array``, so already in C order with its origin at 0.
|
|
189
|
+
``conflicts``: what a per-array ``voxel_size`` that disagrees with the OME scale does,
|
|
190
|
+
``"error"`` (cellmap-analyze's choice, catches stale attributes) or ``"warn"`` (a server
|
|
191
|
+
that would rather show the image)."""
|
|
192
|
+
return _assemble(
|
|
193
|
+
node,
|
|
194
|
+
tuple(int(s) for s in store.shape),
|
|
195
|
+
store.dtype.numpy_dtype,
|
|
196
|
+
tuple(int(c) for c in store.chunk_layout.read_chunk.shape),
|
|
197
|
+
_fill_value(store),
|
|
198
|
+
scale_index=scale_index,
|
|
199
|
+
conflicts=conflicts,
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def read_meta_light(
|
|
204
|
+
node: Node, *, scale_index: int | None = None, conflicts: str = "error"
|
|
205
|
+
) -> ArrayMeta:
|
|
206
|
+
"""``read_meta`` from the array's documents alone, with no tensorstore opened: shape,
|
|
207
|
+
dtype, chunks and fill value from ``.zarray``, ``zarr.json`` (the inner chunk of a
|
|
208
|
+
sharded array), ``attributes.json`` or ``info``. For listing a pyramid's levels cheaply;
|
|
209
|
+
a precomputed single channel is dropped as ``open_array`` drops it."""
|
|
210
|
+
shape, dtype, chunk_shape, fill = _dims_from_docs(node, scale_index)
|
|
211
|
+
return _assemble(
|
|
212
|
+
node, shape, dtype, chunk_shape, fill, scale_index=scale_index, conflicts=conflicts
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _dims_from_docs(node: Node, scale_index: int | None):
|
|
217
|
+
if node.format == "zarr2":
|
|
218
|
+
d = node.docs[".zarray"]
|
|
219
|
+
return tuple(d["shape"]), np.dtype(d["dtype"]), tuple(d["chunks"]), d.get("fill_value")
|
|
220
|
+
if node.format == "zarr3":
|
|
221
|
+
d = node.docs["zarr.json"]
|
|
222
|
+
chunks = tuple(d["chunk_grid"]["configuration"]["chunk_shape"])
|
|
223
|
+
for codec in d.get("codecs", []) or []:
|
|
224
|
+
if codec.get("name") == "sharding_indexed":
|
|
225
|
+
chunks = tuple(codec["configuration"]["chunk_shape"])
|
|
226
|
+
return tuple(d["shape"]), np.dtype(d["data_type"]), chunks, d.get("fill_value")
|
|
227
|
+
if node.format == "n5":
|
|
228
|
+
d = node.docs["attributes.json"]
|
|
229
|
+
return (
|
|
230
|
+
tuple(d["dimensions"])[::-1],
|
|
231
|
+
np.dtype(d["dataType"]),
|
|
232
|
+
tuple(d["blockSize"])[::-1],
|
|
233
|
+
None,
|
|
234
|
+
)
|
|
235
|
+
info = node.docs["info"]
|
|
236
|
+
scale = info["scales"][scale_index or 0]
|
|
237
|
+
size, chunk = tuple(scale["size"])[::-1], tuple(scale["chunk_sizes"][0])[::-1]
|
|
238
|
+
channels = int(info.get("num_channels", 1))
|
|
239
|
+
if channels > 1:
|
|
240
|
+
return (channels, *size), np.dtype(info["data_type"]), (channels, *chunk), None
|
|
241
|
+
return size, np.dtype(info["data_type"]), chunk, None
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _assemble(
|
|
245
|
+
node: Node,
|
|
246
|
+
shape: tuple[int, ...],
|
|
247
|
+
dtype,
|
|
248
|
+
chunk_shape: tuple[int, ...],
|
|
249
|
+
fill_value,
|
|
250
|
+
*,
|
|
251
|
+
scale_index: int | None,
|
|
252
|
+
conflicts: str,
|
|
253
|
+
) -> ArrayMeta:
|
|
254
|
+
if conflicts not in ("error", "warn"):
|
|
255
|
+
raise ValueError("conflicts is 'error' or 'warn'")
|
|
256
|
+
ndim = len(shape)
|
|
257
|
+
up = parent(node.path)
|
|
258
|
+
|
|
259
|
+
if node.format == "precomputed":
|
|
260
|
+
attrs, group_attrs = {}, node.docs["info"]
|
|
261
|
+
found = _precomputed(node.docs["info"], scale_index or 0, ndim)
|
|
262
|
+
elif node.format == "n5":
|
|
263
|
+
attrs = node.docs.get("attributes.json", {})
|
|
264
|
+
group_attrs = (Location(up).read_json("attributes.json") or {}) if up else {}
|
|
265
|
+
found = _n5(attrs, group_attrs, node, ndim)
|
|
266
|
+
else:
|
|
267
|
+
attrs, group_attrs = _zarr_attrs(node, up)
|
|
268
|
+
found = _zarr(attrs, group_attrs, basename(node.path), ndim, node, conflicts)
|
|
269
|
+
|
|
270
|
+
axes_named = found.axes is not None
|
|
271
|
+
axes = tuple(str(a) for a in found.axes) if found.axes is not None else default_axes(ndim)
|
|
272
|
+
return ArrayMeta(
|
|
273
|
+
shape=shape,
|
|
274
|
+
dtype=dtype,
|
|
275
|
+
chunk_shape=chunk_shape,
|
|
276
|
+
axes=axes,
|
|
277
|
+
axes_named=axes_named,
|
|
278
|
+
units=found.units if found.units is not None else ("",) * ndim,
|
|
279
|
+
voxel_size=found.voxel_size if found.voxel_size is not None else (1.0,) * ndim,
|
|
280
|
+
translation=found.translation
|
|
281
|
+
if found.translation is not None
|
|
282
|
+
else corner_to_centre((0.0,) * ndim, (1.0,) * ndim, axes), # unplaced: a corner at 0
|
|
283
|
+
format=node.format,
|
|
284
|
+
path=node.path
|
|
285
|
+
if scale_index is None
|
|
286
|
+
else f"{node.path}/{node.docs['info']['scales'][scale_index]['key']}",
|
|
287
|
+
parent=up,
|
|
288
|
+
attrs=attrs,
|
|
289
|
+
group_attrs=group_attrs,
|
|
290
|
+
placed_by=found.placed_by if found.voxel_size is not None else None,
|
|
291
|
+
fill_value=fill_value,
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _fill_value(store: ts.TensorStore):
|
|
296
|
+
try:
|
|
297
|
+
fv = store.fill_value
|
|
298
|
+
except Exception:
|
|
299
|
+
return None
|
|
300
|
+
if fv is None:
|
|
301
|
+
return None
|
|
302
|
+
arr = np.asarray(fv)
|
|
303
|
+
return arr.item() if arr.size == 1 else arr
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _floats(values, ndim: int, what: str, pad: float | None = None) -> tuple[float, ...]:
|
|
307
|
+
"""``values`` as floats, one per axis. Shorter lists cover the trailing axes and the
|
|
308
|
+
leading ones get ``pad``; with ``pad`` None (OME) the length must match."""
|
|
309
|
+
values = tuple(float(v) for v in values)
|
|
310
|
+
if len(values) < ndim and pad is not None:
|
|
311
|
+
values = (float(pad),) * (ndim - len(values)) + values
|
|
312
|
+
if len(values) != ndim:
|
|
313
|
+
raise ValueError(
|
|
314
|
+
f"{what} has {len(values)} entries for a {ndim}-dimensional array: {values}"
|
|
315
|
+
)
|
|
316
|
+
return values
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _units(values, ndim: int, pad: str | None = None) -> tuple[str, ...]:
|
|
320
|
+
out = tuple("" if v is None else str(v) for v in values)
|
|
321
|
+
if len(out) < ndim and pad is not None:
|
|
322
|
+
out = (pad,) * (ndim - len(out)) + out
|
|
323
|
+
if len(out) != ndim:
|
|
324
|
+
raise ValueError(f"units has {len(out)} entries for a {ndim}-dimensional array: {out}")
|
|
325
|
+
return out
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _axes(names, ndim: int) -> tuple[str, ...] | None:
|
|
329
|
+
"""Axis names, the leading axes defaulted when the list is short; None when it is empty
|
|
330
|
+
or too long."""
|
|
331
|
+
names = tuple(str(a) for a in names or ())
|
|
332
|
+
if not names or len(names) > ndim:
|
|
333
|
+
return None
|
|
334
|
+
return default_axes(ndim)[: ndim - len(names)] + names
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
# --- zarr ------------------------------------------------------------------
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _zarr_attrs(node: Node, up: str | None) -> tuple[dict, dict]:
|
|
341
|
+
if node.format == "zarr3":
|
|
342
|
+
attrs = node.docs["zarr.json"].get("attributes", {}) or {}
|
|
343
|
+
else:
|
|
344
|
+
attrs = node.location.read_json(".zattrs") or {}
|
|
345
|
+
return attrs, _group_attrs_at(up, node.format)
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _group_attrs_at(path: str | None, fmt: str) -> dict:
|
|
349
|
+
"""A group's user attributes. zarr groups are read in both formats, because a zarr v3
|
|
350
|
+
array inside a v2 group whose ``.zattrs`` holds the OME multiscales (cellmap-analyze
|
|
351
|
+
writes these) is a layout in the wild; the array's own format wins where both exist."""
|
|
352
|
+
if path is None:
|
|
353
|
+
return {}
|
|
354
|
+
loc = Location(path)
|
|
355
|
+
if fmt == "n5":
|
|
356
|
+
return loc.read_json("attributes.json") or {}
|
|
357
|
+
docs = loc.read_many(("zarr.json", ".zattrs"))
|
|
358
|
+
v3 = (json.loads(docs["zarr.json"]) if docs["zarr.json"] else {}).get("attributes", {}) or {}
|
|
359
|
+
v2 = json.loads(docs[".zattrs"]) if docs[".zattrs"] else {}
|
|
360
|
+
return {**v2, **v3} if fmt == "zarr3" else {**v3, **v2}
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _ome_hit(node: Node, group_attrs: dict, max_depth: int = 4) -> tuple[dict, dict] | None:
|
|
364
|
+
"""The ``(multiscale, dataset)`` listing this array: in the parent, or in a further
|
|
365
|
+
ancestor when the dataset path has more than one component (``"seg/s0"``). A parent that
|
|
366
|
+
lists levels but not this array gets a warning, and the array is placed by its own
|
|
367
|
+
attributes or not at all."""
|
|
368
|
+
name = basename(node.path)
|
|
369
|
+
hit = _dataset_for(multiscales_of(group_attrs), name)
|
|
370
|
+
if hit is not None:
|
|
371
|
+
return hit
|
|
372
|
+
if multiscales_of(group_attrs): # the parent is a pyramid, just not ours: no walk
|
|
373
|
+
logger.warning(
|
|
374
|
+
"%s: the parent group's multiscales list no dataset named %r; the array is placed by its own attributes, or not at all",
|
|
375
|
+
node.path,
|
|
376
|
+
name,
|
|
377
|
+
)
|
|
378
|
+
return None
|
|
379
|
+
ancestor, rel = parent(node.path), name
|
|
380
|
+
for _ in range(max_depth - 1):
|
|
381
|
+
up = parent(ancestor) if ancestor is not None else None
|
|
382
|
+
if is_root(up): # never probe the filesystem root, a URL's host root or a bucket root
|
|
383
|
+
break
|
|
384
|
+
rel = f"{basename(ancestor)}/{rel}"
|
|
385
|
+
ancestor = up
|
|
386
|
+
try: # an ancestor that answers anything but a document (a 301 from a server root) is no group
|
|
387
|
+
multiscales = multiscales_of(_group_attrs_at(ancestor, node.format))
|
|
388
|
+
except Exception as e: # noqa: BLE001
|
|
389
|
+
logger.debug("%s: stopped looking for multiscales at %s (%s)", node.path, ancestor, e)
|
|
390
|
+
break
|
|
391
|
+
hit = _dataset_for(multiscales, rel)
|
|
392
|
+
if hit is not None:
|
|
393
|
+
return hit
|
|
394
|
+
if multiscales: # an ancestor pyramid that does not list us: stop here
|
|
395
|
+
break
|
|
396
|
+
return None
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _dataset_for(multiscales: list[dict], name: str) -> tuple[dict, dict] | None:
|
|
400
|
+
"""``(multiscale, dataset)`` whose dataset ``path`` is ``name``: matched by name, never by
|
|
401
|
+
position, so an array the group does not list borrows no other level's scale."""
|
|
402
|
+
for ms in multiscales:
|
|
403
|
+
for ds in ms.get("datasets", []) or []:
|
|
404
|
+
if str(ds.get("path", "")).strip("/") == name:
|
|
405
|
+
return ms, ds
|
|
406
|
+
return None
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _zarr(
|
|
410
|
+
attrs: dict, group_attrs: dict, name: str, ndim: int, node: Node, conflicts: str
|
|
411
|
+
) -> _Resolved:
|
|
412
|
+
found = _Resolved()
|
|
413
|
+
hit = _ome_hit(node, group_attrs)
|
|
414
|
+
if hit is not None:
|
|
415
|
+
found = _ome(hit[0], hit[1], ndim, node.path)
|
|
416
|
+
if found.voxel_size is None:
|
|
417
|
+
found = _array_attrs(attrs, ndim, node.path)
|
|
418
|
+
else:
|
|
419
|
+
_check_against_attrs(attrs, found, ndim, node.path, conflicts)
|
|
420
|
+
if found.axes is None:
|
|
421
|
+
dims = attrs.get("_ARRAY_DIMENSIONS") or node.docs.get("zarr.json", {}).get(
|
|
422
|
+
"dimension_names"
|
|
423
|
+
)
|
|
424
|
+
if dims and len(dims) == ndim and all(d for d in dims):
|
|
425
|
+
found.axes = tuple(str(d) for d in dims)
|
|
426
|
+
if found.voxel_size is None and node.format == "zarr3" and attrs.get("dimension_units"):
|
|
427
|
+
units, vs = _dimension_units(attrs["dimension_units"], ndim)
|
|
428
|
+
found.units, found.voxel_size, found.placed_by = units, vs, "dimension_units"
|
|
429
|
+
return found
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _dimension_units(spec: list, ndim: int) -> tuple[tuple[str, ...], tuple[float, ...]]:
|
|
433
|
+
"""tensorstore's ``dimension_units`` attribute: ``"4nm"``, ``null``, ``""``."""
|
|
434
|
+
units, vs = [], []
|
|
435
|
+
for entry in spec:
|
|
436
|
+
if not entry:
|
|
437
|
+
units.append("")
|
|
438
|
+
vs.append(1.0)
|
|
439
|
+
continue
|
|
440
|
+
m = re.match(r"^\s*([-+0-9.eE]*)\s*(.*)$", str(entry))
|
|
441
|
+
vs.append(float(m.group(1)) if m and m.group(1) else 1.0)
|
|
442
|
+
units.append(m.group(2).strip() if m else "")
|
|
443
|
+
return _units(units, ndim), _floats(vs, ndim, "dimension_units")
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _ome(multiscale: dict, dataset: dict, ndim: int, path: str) -> _Resolved:
|
|
447
|
+
"""OME-NGFF 0.4 and 0.5: the dataset's own transforms, then the multiscale-level ones.
|
|
448
|
+
0.6 (``coordinateSystems``): the intrinsic system only, chunkmirage's rule."""
|
|
449
|
+
transforms = list(dataset.get("coordinateTransformations", []) or [])
|
|
450
|
+
systems = multiscale.get("coordinateSystems")
|
|
451
|
+
if systems:
|
|
452
|
+
axes_spec = _ome06_system(multiscale, transforms, systems).get("axes", [])
|
|
453
|
+
level_transforms: list[dict] = []
|
|
454
|
+
transforms = _unpack_sequence(transforms)
|
|
455
|
+
else:
|
|
456
|
+
axes_spec = multiscale.get("axes", []) or []
|
|
457
|
+
level_transforms = list(multiscale.get("coordinateTransformations", []) or [])
|
|
458
|
+
|
|
459
|
+
scale = np.ones(ndim)
|
|
460
|
+
translation = np.zeros(ndim)
|
|
461
|
+
for t in transforms + level_transforms:
|
|
462
|
+
kind = t.get("type")
|
|
463
|
+
if kind == "scale":
|
|
464
|
+
s = np.asarray(_floats(t["scale"], ndim, f"{path}: OME scale"))
|
|
465
|
+
scale, translation = scale * s, translation * s
|
|
466
|
+
elif kind == "translation":
|
|
467
|
+
translation = translation + np.asarray(
|
|
468
|
+
_floats(t["translation"], ndim, f"{path}: OME translation")
|
|
469
|
+
)
|
|
470
|
+
# identity and anything else: no effect (0.6's other types lead to other systems)
|
|
471
|
+
|
|
472
|
+
axes: tuple[str, ...] | None = None
|
|
473
|
+
units: tuple[str, ...] | None = None
|
|
474
|
+
if axes_spec and len(axes_spec) == ndim:
|
|
475
|
+
axes = tuple(str(a["name"] if isinstance(a, dict) else a) for a in axes_spec)
|
|
476
|
+
units = tuple(
|
|
477
|
+
str((a.get("unit") if isinstance(a, dict) else None) or "") for a in axes_spec
|
|
478
|
+
)
|
|
479
|
+
return _Resolved(
|
|
480
|
+
axes, units, tuple(float(v) for v in scale), tuple(float(v) for v in translation), "ome"
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def _ome06_system(multiscale: dict, transforms: list[dict], systems: list[dict]) -> dict:
|
|
485
|
+
wanted = next(
|
|
486
|
+
(t.get("output") for t in transforms if isinstance(t, dict) and t.get("output")), None
|
|
487
|
+
)
|
|
488
|
+
for system in systems:
|
|
489
|
+
if wanted is None or system.get("name") == wanted:
|
|
490
|
+
return system
|
|
491
|
+
return systems[0]
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def _unpack_sequence(transforms: list[dict]) -> list[dict]:
|
|
495
|
+
out: list[dict] = []
|
|
496
|
+
for t in transforms:
|
|
497
|
+
if t.get("type") == "sequence":
|
|
498
|
+
out.extend(_unpack_sequence(list(t.get("transformations", []) or [])))
|
|
499
|
+
elif t.get("type") in ("scale", "translation"):
|
|
500
|
+
out.append(t)
|
|
501
|
+
return out
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _array_attrs(attrs: dict, ndim: int, path: str) -> _Resolved:
|
|
505
|
+
"""funlib (``resolution``/``voxel_size``, ``offset``, ``units``, ``axis_names``) or COSEM
|
|
506
|
+
(``transform``) attributes on the array itself, C order. An ``offset`` with no voxel
|
|
507
|
+
size places the array with voxels of 1."""
|
|
508
|
+
found = _Resolved()
|
|
509
|
+
transform = attrs.get("transform")
|
|
510
|
+
if isinstance(transform, dict) and "scale" in transform:
|
|
511
|
+
return _cosem_transform(transform, ndim, path)
|
|
512
|
+
found.axes = _axes(attrs.get("axis_names"), ndim)
|
|
513
|
+
vs = attrs.get("voxel_size", attrs.get("resolution", attrs.get("scale")))
|
|
514
|
+
if not isinstance(vs, (list, tuple)):
|
|
515
|
+
vs = None
|
|
516
|
+
offset = attrs.get("offset")
|
|
517
|
+
if vs is None and offset is None:
|
|
518
|
+
return found
|
|
519
|
+
found.voxel_size = _floats(
|
|
520
|
+
vs if vs is not None else [1.0] * ndim, ndim, f"{path}: voxel_size", pad=1.0
|
|
521
|
+
)
|
|
522
|
+
if attrs.get("units") is not None:
|
|
523
|
+
raw = attrs["units"] if isinstance(attrs["units"], list) else [attrs["units"]] * ndim
|
|
524
|
+
found.units = _units(raw, ndim, pad="")
|
|
525
|
+
corner = _floats(
|
|
526
|
+
offset if offset is not None else [0.0] * ndim, ndim, f"{path}: offset", pad=0.0
|
|
527
|
+
)
|
|
528
|
+
found.translation = corner_to_centre(corner, found.voxel_size, found.axes)
|
|
529
|
+
found.placed_by = "funlib"
|
|
530
|
+
return found
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def _cosem_transform(transform: dict, ndim: int, path: str) -> _Resolved:
|
|
534
|
+
order = transform.get("ordering", "C")
|
|
535
|
+
rev = (lambda v: tuple(v)[::-1]) if order == "F" else tuple
|
|
536
|
+
scale = _floats(rev(transform["scale"]), ndim, f"{path}: transform.scale", pad=1.0)
|
|
537
|
+
translate = _floats(
|
|
538
|
+
rev(transform.get("translate", [0.0] * ndim)), ndim, f"{path}: transform.translate", pad=0.0
|
|
539
|
+
)
|
|
540
|
+
axes = _axes(rev(transform["axes"]), ndim) if transform.get("axes") else None
|
|
541
|
+
units = _units(rev(transform["units"]), ndim, pad="") if transform.get("units") else None
|
|
542
|
+
return _Resolved(axes, units, scale, translate, "cosem") # translate is already a centre
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def _check_against_attrs(
|
|
546
|
+
attrs: dict, found: _Resolved, ndim: int, path: str, conflicts: str
|
|
547
|
+
) -> None:
|
|
548
|
+
"""An OME translation beside a funlib ``offset``: equal (cellmap-analyze's writer) or a
|
|
549
|
+
corner half a voxel below (funlib's writer) pass silently; anything else warns. A
|
|
550
|
+
disagreeing voxel size raises (or warns, per ``conflicts``): that is how stale
|
|
551
|
+
attributes are caught."""
|
|
552
|
+
assert found.voxel_size is not None and found.translation is not None
|
|
553
|
+
vs = np.asarray(found.voxel_size)
|
|
554
|
+
attr_vs = attrs.get("voxel_size", attrs.get("resolution"))
|
|
555
|
+
if attr_vs is not None and len(attr_vs) <= ndim:
|
|
556
|
+
padded = np.asarray(_floats(attr_vs, ndim, f"{path}: voxel_size", pad=1.0))
|
|
557
|
+
if not np.allclose(padded, vs, rtol=RTOL, atol=0):
|
|
558
|
+
message = f"{path}: OME scale {found.voxel_size} disagrees with the array's voxel_size {list(attr_vs)}"
|
|
559
|
+
if conflicts == "error":
|
|
560
|
+
raise ValueError(message)
|
|
561
|
+
logger.warning("%s; using the OME scale", message)
|
|
562
|
+
offset = attrs.get("offset")
|
|
563
|
+
if offset is None or len(offset) > ndim:
|
|
564
|
+
return
|
|
565
|
+
offset = np.asarray(_floats(offset, ndim, f"{path}: offset", pad=0.0))
|
|
566
|
+
tr = np.asarray(found.translation)
|
|
567
|
+
corner = np.asarray(corner_to_centre(offset, vs, found.axes))
|
|
568
|
+
tol = RTOL * np.maximum(np.abs(vs), 1.0)
|
|
569
|
+
if np.all(np.abs(offset - tr) <= tol) or np.all(np.abs(corner - tr) <= tol):
|
|
570
|
+
return
|
|
571
|
+
logger.warning(
|
|
572
|
+
"%s: offset %s is neither the OME translation %s nor the corner below it; using the translation",
|
|
573
|
+
path,
|
|
574
|
+
offset.tolist(),
|
|
575
|
+
tr.tolist(),
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
# --- N5 --------------------------------------------------------------------
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _n5(attrs: dict, group_attrs: dict, node: Node, ndim: int) -> _Resolved:
|
|
583
|
+
"""COSEM ``transform`` on the array; else the group's ``multiscales[].datasets[].transform``
|
|
584
|
+
for this path; else ``pixelResolution``/``resolution`` (array, else group) times the
|
|
585
|
+
level's ``downsamplingFactors``, plus ``offset``. Lists other than ``transform`` are
|
|
586
|
+
x-first and reversed to C order here. Without an ``offset``: a level with ``downsamplingFactors`` follows
|
|
587
|
+
BigDataViewer, ``(f - 1) / 2`` full-resolution voxels on, whichever key gave the base
|
|
588
|
+
(BigStitcher writes ``resolution``); a funlib ``resolution`` with no
|
|
589
|
+
``downsamplingFactors`` means a corner at 0 (translation half a voxel)."""
|
|
590
|
+
name = basename(node.path)
|
|
591
|
+
transform = attrs.get("transform")
|
|
592
|
+
if isinstance(transform, dict) and "scale" in transform:
|
|
593
|
+
return _cosem_transform(transform, ndim, name)
|
|
594
|
+
hit = _ome_hit(node, group_attrs)
|
|
595
|
+
if hit is not None and isinstance(hit[1].get("transform"), dict):
|
|
596
|
+
return _cosem_transform(hit[1]["transform"], ndim, name)
|
|
597
|
+
|
|
598
|
+
axes = _axes(list(attrs.get("axes", group_attrs.get("axes", [])) or [])[::-1], ndim)
|
|
599
|
+
unit: str | None = None
|
|
600
|
+
base, key = None, None
|
|
601
|
+
for source in (attrs, group_attrs):
|
|
602
|
+
if "pixelResolution" in source:
|
|
603
|
+
pr = source["pixelResolution"]
|
|
604
|
+
base, unit = (
|
|
605
|
+
(pr.get("dimensions"), pr.get("unit")) if isinstance(pr, dict) else (pr, None)
|
|
606
|
+
)
|
|
607
|
+
key = "pixelResolution"
|
|
608
|
+
break
|
|
609
|
+
if "resolution" in source:
|
|
610
|
+
base, key = source["resolution"], "resolution"
|
|
611
|
+
break
|
|
612
|
+
if base is None:
|
|
613
|
+
return _Resolved(axes)
|
|
614
|
+
base = np.asarray(_floats(list(base)[::-1], ndim, f"{name}: {key}", pad=1.0)) # x-first -> C
|
|
615
|
+
factors = np.asarray(
|
|
616
|
+
_floats(
|
|
617
|
+
list(attrs.get("downsamplingFactors", [1]))[::-1],
|
|
618
|
+
ndim,
|
|
619
|
+
f"{name}: downsamplingFactors",
|
|
620
|
+
pad=1.0,
|
|
621
|
+
)
|
|
622
|
+
)
|
|
623
|
+
voxel_size = base * factors
|
|
624
|
+
|
|
625
|
+
units_list = attrs.get("units", group_attrs.get("units"))
|
|
626
|
+
if isinstance(units_list, str): # "um" written once for every axis
|
|
627
|
+
units_list = [units_list] * ndim
|
|
628
|
+
if unit is None and units_list is not None:
|
|
629
|
+
units = _units(list(units_list)[::-1], ndim, pad="")
|
|
630
|
+
else:
|
|
631
|
+
units = tuple(
|
|
632
|
+
(str(unit or "") if i in set(spatial_axes(axes or default_axes(ndim))) else "")
|
|
633
|
+
for i in range(ndim)
|
|
634
|
+
)
|
|
635
|
+
|
|
636
|
+
if "offset" in attrs:
|
|
637
|
+
corner = _floats(list(attrs["offset"])[::-1], ndim, f"{name}: offset", pad=0.0)
|
|
638
|
+
translation = corner_to_centre(corner, voxel_size, axes)
|
|
639
|
+
elif key == "resolution" and "downsamplingFactors" not in attrs:
|
|
640
|
+
# funlib/daisy N5: a resolution with no offset is a corner at 0 (funlib never writes
|
|
641
|
+
# downsamplingFactors; BigStitcher writes "resolution" AND downsamplingFactors, and means BDV)
|
|
642
|
+
translation = corner_to_centre((0.0,) * ndim, voxel_size, axes)
|
|
643
|
+
else:
|
|
644
|
+
# BigDataViewer: the downsampled voxel's centre is the centre of the block it covers
|
|
645
|
+
translation = tuple(float(v) for v in (factors - 1) / 2 * base)
|
|
646
|
+
return _Resolved(axes, units, tuple(float(v) for v in voxel_size), tuple(translation), "n5")
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
# --- precomputed -------------------------------------------------------------
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def _precomputed(info: dict, scale_index: int, ndim: int) -> _Resolved:
|
|
653
|
+
scale = info["scales"][scale_index]
|
|
654
|
+
resolution = [float(v) for v in scale["resolution"]][::-1] # xyz nm -> zyx
|
|
655
|
+
voxel_offset = [float(v) for v in scale.get("voxel_offset", [0, 0, 0])][::-1]
|
|
656
|
+
spatial_tr = [(o + 0.5) * r for o, r in zip(voxel_offset, resolution)]
|
|
657
|
+
if ndim == 4: # channel kept (more than one)
|
|
658
|
+
return _Resolved(
|
|
659
|
+
("c", "z", "y", "x"),
|
|
660
|
+
("", "nm", "nm", "nm"),
|
|
661
|
+
(1.0, *resolution),
|
|
662
|
+
(0.0, *spatial_tr),
|
|
663
|
+
"precomputed",
|
|
664
|
+
)
|
|
665
|
+
return _Resolved(
|
|
666
|
+
("z", "y", "x"), ("nm", "nm", "nm"), tuple(resolution), tuple(spatial_tr), "precomputed"
|
|
667
|
+
)
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
# --- pyramids from the group document alone ---------------------------------
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def light_voxel_sizes(
|
|
674
|
+
group: Node, names: list[str], space: str | None
|
|
675
|
+
) -> list[tuple[str, tuple[float, ...]]] | None:
|
|
676
|
+
levels = light_levels(group, names, space)
|
|
677
|
+
return None if levels is None else [(name, vs) for name, vs, _ in levels]
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def light_levels(
|
|
681
|
+
group: Node, names: list[str], space: str | None
|
|
682
|
+
) -> list[tuple[str, tuple[float, ...], tuple[float, ...]]] | None:
|
|
683
|
+
"""``(name, spatial voxel size)`` for every level of a group, from the group's own
|
|
684
|
+
document, in ``space`` units (native when None): OME-NGFF datasets, COSEM N5
|
|
685
|
+
``multiscales[].datasets[].transform`` and precomputed ``scales``. None when the
|
|
686
|
+
document does not say (then the levels have to be opened), or when a unit cannot be
|
|
687
|
+
converted. ``light_levels`` adds each level's spatial translation (the centre of its
|
|
688
|
+
voxel 0, in the same units), so a pyramid can be placed without opening any level."""
|
|
689
|
+
try:
|
|
690
|
+
if group.format == "precomputed":
|
|
691
|
+
f = U.factor("nm", space) if space else 1.0
|
|
692
|
+
out = []
|
|
693
|
+
for i, scale in enumerate(group.docs["info"]["scales"]):
|
|
694
|
+
res = [float(v) * f for v in scale["resolution"]][::-1]
|
|
695
|
+
vo = [float(v) for v in scale.get("voxel_offset", [0, 0, 0])][::-1]
|
|
696
|
+
tr = [(o + 0.5) * r for o, r in zip(vo, res)]
|
|
697
|
+
out.append((str(scale.get("key", i)), tuple(res), tuple(tr)))
|
|
698
|
+
return out
|
|
699
|
+
multiscales = group.multiscales
|
|
700
|
+
if not multiscales:
|
|
701
|
+
return None
|
|
702
|
+
out = []
|
|
703
|
+
for name in names:
|
|
704
|
+
hit = _dataset_for(multiscales, name)
|
|
705
|
+
if hit is None:
|
|
706
|
+
return None
|
|
707
|
+
ms, ds = hit
|
|
708
|
+
if group.format == "n5":
|
|
709
|
+
transform = ds.get("transform")
|
|
710
|
+
if not isinstance(transform, dict) or "scale" not in transform:
|
|
711
|
+
return None
|
|
712
|
+
ndim = len(transform["scale"])
|
|
713
|
+
found = _cosem_transform(transform, ndim, name)
|
|
714
|
+
else:
|
|
715
|
+
axes_spec = ms.get("axes") or (
|
|
716
|
+
_ome06_system(
|
|
717
|
+
ms,
|
|
718
|
+
list(ds.get("coordinateTransformations", []) or []),
|
|
719
|
+
ms["coordinateSystems"],
|
|
720
|
+
).get("axes", [])
|
|
721
|
+
if ms.get("coordinateSystems")
|
|
722
|
+
else []
|
|
723
|
+
)
|
|
724
|
+
ndim = len(axes_spec)
|
|
725
|
+
if ndim == 0:
|
|
726
|
+
return None
|
|
727
|
+
found = _ome(ms, ds, ndim, name)
|
|
728
|
+
assert found.voxel_size is not None
|
|
729
|
+
axes = found.axes or default_axes(ndim)
|
|
730
|
+
units = found.units or ("",) * ndim
|
|
731
|
+
sizes, trs = [], []
|
|
732
|
+
for i in spatial_axes(axes):
|
|
733
|
+
f = U.factor(units[i], space, missing_as=U.NANOMETER) if space else 1.0
|
|
734
|
+
sizes.append(found.voxel_size[i] * f)
|
|
735
|
+
trs.append((found.translation[i] if found.translation is not None else 0.0) * f)
|
|
736
|
+
out.append((name, tuple(sizes), tuple(trs)))
|
|
737
|
+
return out
|
|
738
|
+
except (ValueError, KeyError, TypeError):
|
|
739
|
+
return None
|