worldarray 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {worldarray-0.3.0 → worldarray-0.4.0}/CHANGELOG.md +40 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/PKG-INFO +19 -9
- {worldarray-0.3.0 → worldarray-0.4.0}/README.md +18 -8
- {worldarray-0.3.0 → worldarray-0.4.0}/docs/spec.md +77 -20
- {worldarray-0.3.0 → worldarray-0.4.0}/pyproject.toml +1 -1
- {worldarray-0.3.0 → worldarray-0.4.0}/src/worldarray/__init__.py +1 -1
- {worldarray-0.3.0 → worldarray-0.4.0}/src/worldarray/array.py +122 -13
- {worldarray-0.3.0 → worldarray-0.4.0}/src/worldarray/resample.py +12 -4
- {worldarray-0.3.0 → worldarray-0.4.0}/src/worldarray/units.py +3 -1
- {worldarray-0.3.0 → worldarray-0.4.0}/src/worldarray/write.py +10 -1
- {worldarray-0.3.0 → worldarray-0.4.0}/tests/test_grid.py +1 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/tests/test_read.py +28 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/tests/test_resample.py +42 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/tests/test_write.py +52 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/uv.lock +1 -1
- {worldarray-0.3.0 → worldarray-0.4.0}/.github/workflows/ci.yml +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/.github/workflows/docs.yml +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/.github/workflows/publish.yml +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/.gitignore +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/LICENSE +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/docs/changelog.md +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/docs/index.md +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/mkdocs.yml +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/src/worldarray/context.py +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/src/worldarray/geometry.py +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/src/worldarray/grid.py +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/src/worldarray/levels.py +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/src/worldarray/placement.py +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/src/worldarray/retry.py +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/src/worldarray/store.py +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/tests/conftest.py +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/tests/fixtures/metadata.json +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/tests/fixtures/write.json +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/tests/test_access.py +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/tests/test_geometry.py +0 -0
- {worldarray-0.3.0 → worldarray-0.4.0}/tests/test_metadata.py +0 -0
|
@@ -1,5 +1,45 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.4.0 (2026-10-22)
|
|
4
|
+
|
|
5
|
+
cellmap-analyze's three asks, so it can call worldarray directly with no wrapper, and two
|
|
6
|
+
of cellmap-flow's. No placement change and no unit table change.
|
|
7
|
+
|
|
8
|
+
- `vol.read_onto(grid, begin=None, end=None, ...)` reads voxels `[begin, end)` of a
|
|
9
|
+
`Grid`'s spatial axes from the array, exactly that shape (Behaviour 20). It is the
|
|
10
|
+
`read` of the grid's index box at the grid's voxel size anchored at its corner, so at
|
|
11
|
+
the array's own voxel size nothing is resampled, whatever the sub-voxel offset, and
|
|
12
|
+
otherwise the array is resampled onto the grid. The grid's units are converted.
|
|
13
|
+
- A pickled handle, and `reopen`, use the path resolved against the working directory at
|
|
14
|
+
open, so a worker that changes directory before unpickling opens the same store; it
|
|
15
|
+
used to raise `FileNotFoundError` for a relative path.
|
|
16
|
+
- `write_index`, `write` and `write_chunk` take a scalar, which fills the box. It is
|
|
17
|
+
converted to the dtype only when its value survives (`0` into `uint8`; `300`, `-1`, `0.5`
|
|
18
|
+
and NaN raise), and the fill value over whole units leaves them absent.
|
|
19
|
+
- `vol.source_translation`: the stored level's translation in `units`, beside
|
|
20
|
+
`source_voxel_size`, under resample and relabel too (cellmap-flow's `stored_placement`).
|
|
21
|
+
- `create(exists="adopt")` also compares the keys of `attrs` with the existing array's
|
|
22
|
+
attributes and raises naming each that differs or is missing; keys not given are not
|
|
23
|
+
compared. An adopt given different `attrs` used to keep the old ones silently, as funlib's
|
|
24
|
+
`prepare_ds` kept an old offset (cellmap-flow's `_refuse_another_region`).
|
|
25
|
+
|
|
26
|
+
## 0.3.1 (2026-10-22)
|
|
27
|
+
|
|
28
|
+
- `source_voxel_size` reports the stored level's voxel size under `on_mismatch="relabel"`,
|
|
29
|
+
as the spec says; it reported the relabelled size (cellmap-flow's report). Reads,
|
|
30
|
+
`voxel_size`, `translation` and `corner` are unchanged.
|
|
31
|
+
- `box` of an array opened with a resample target is the extent of the target voxels
|
|
32
|
+
`shape` counts, as the spec's `[corner, corner + shape * voxel_size)` says and as
|
|
33
|
+
`Grid.from_meta(meta, voxel_size=t).box` already was; it was the stored data's extent, so
|
|
34
|
+
`read(box)` could return fewer voxels than `shape` (cellmap-flow's 7 voxels of 8 nm at
|
|
35
|
+
24 nm: box ended at 56 nm, now 72). No voxel moves.
|
|
36
|
+
- **Unit table change.** The unit table knows kilometres (`kilometer`, `kilometre`, `km`,
|
|
37
|
+
1e3 m), so they convert, abbreviate to `km` and are written as OME's `kilometer`
|
|
38
|
+
(chunkmirage's ask). This should have been a minor release (spec section 8): cellmap-flow
|
|
39
|
+
exports the table and its CI failed until it moved its pin to `>=0.3.1`.
|
|
40
|
+
- README: the placement rule gains the corner formula, a worked example, why the centre
|
|
41
|
+
is the model, and how the read rule follows from it.
|
|
42
|
+
|
|
3
43
|
## 0.3.0 (2026-10-22)
|
|
4
44
|
|
|
5
45
|
Grids and blocks, built to spec draft 3 (Behaviours 17 to 19). Nothing on the read or
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: worldarray
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Read zarr, N5 and Neuroglancer precomputed volumes in world coordinates through tensorstore: correct voxel placement, pyramid levels, padded region reads, resampling. numpy + tensorstore only.
|
|
5
5
|
Project-URL: Homepage, https://janeliascicomp.github.io/worldarray/
|
|
6
6
|
Project-URL: Documentation, https://janeliascicomp.github.io/worldarray/
|
|
@@ -109,6 +109,7 @@ six = wa.open(path, voxel_size=(6, 6, 6), on_mismatch="resample")
|
|
|
109
109
|
|
|
110
110
|
# one volume read onto another volume's grid, for masks, contact sites, statistics
|
|
111
111
|
onto = raw.read(box, voxel_size=labels.voxel_size, anchor=labels.corner)
|
|
112
|
+
onto = raw.read_onto(grid, begin, end) # voxels [begin, end) of a Grid, exactly that shape
|
|
112
113
|
|
|
113
114
|
# index space, native units: what an engine or a viewer wants
|
|
114
115
|
vol = wa.open(path, units=None)
|
|
@@ -159,15 +160,24 @@ s1 = wa.open_pyramid("out.zarr", mode="r+").add_level("s1", factors=2) # place
|
|
|
159
160
|
- **Plans outputs and blocks** (0.3): a store-less `Grid` with the same geometry as an
|
|
160
161
|
array, so an output is planned, cut into blocks and only then created; `blocks` tiling an
|
|
161
162
|
array or a box into write boxes with context-grown read boxes, in C order, as plain
|
|
162
|
-
picklable records a cluster job takes by id; the anisotropy-aware pyramid schedule
|
|
163
|
-
Nothing is scheduled or run.
|
|
163
|
+
picklable records a cluster job takes by id; the anisotropy-aware pyramid schedule; every
|
|
164
|
+
input read onto that grid by its voxel indices (0.4). Nothing is scheduled or run.
|
|
164
165
|
|
|
165
166
|
## The placement rule
|
|
166
167
|
|
|
167
168
|
A translation is where the centre of voxel 0 is. Index `i` sits at
|
|
168
|
-
`translation + i * voxel_size
|
|
169
|
-
|
|
170
|
-
|
|
169
|
+
`translation + i * voxel_size` and covers half a voxel either side, so the array's corner
|
|
170
|
+
is `corner = translation - voxel_size / 2`. With 8 nm voxels and the corner at 0, the
|
|
171
|
+
translation is 4: voxel 0 covers 0 to 8 nm around its centre at 4, voxel 1 covers 8 to
|
|
172
|
+
16 nm around 12. That is how OME-NGFF defines it and how Neuroglancer draws it.
|
|
173
|
+
|
|
174
|
+
The centre is the model because a voxel is a sample taken at a point. Everything that
|
|
175
|
+
compares or resamples grids (picking a level, reading one volume onto another's grid,
|
|
176
|
+
interpolating) works on those points, and the box a voxel covers follows from its point.
|
|
177
|
+
The read rule follows too: a voxel is read iff its centre lies in the box, so boxes that
|
|
178
|
+
adjoin never share a voxel or drop one.
|
|
179
|
+
|
|
180
|
+
Dialects that store the corner are moved half a voxel on reading:
|
|
171
181
|
|
|
172
182
|
| Stored | Means | Read as `translation` |
|
|
173
183
|
|---|---|---|
|
|
@@ -218,12 +228,12 @@ wa.units.abbreviation("nanometer") ; wa.store.join(path, "s0") ; wa.store.parent
|
|
|
218
228
|
```
|
|
219
229
|
|
|
220
230
|
`WorldArray`: `ts`, `meta`, `grid`, `shape`, `dtype`, `axes`, `voxel_size`, `translation`,
|
|
221
|
-
`corner`, `box`, `levels`, `level_name`, `level_path`, `source_voxel_size`, `resampled`,
|
|
231
|
+
`corner`, `box`, `levels`, `level_name`, `level_path`, `source_voxel_size`, `source_translation`, `resampled`,
|
|
222
232
|
`writable`, `write_shape`;
|
|
223
233
|
`read(box, voxel_size=, padding=, method=, mask=, anchor=, transform=, post_transform=, select=, strict=)`,
|
|
224
|
-
`read_index(begin, end, padding=, mask=, select=, transform=)`, `index_box(box)`,
|
|
234
|
+
`read_index(begin, end, padding=, mask=, select=, transform=)`, `read_onto(grid, begin=, end=, ...)`, `index_box(box)`,
|
|
225
235
|
`chunks_covering(begin, end, write=)`, `list_chunks()`, `chunk_key(index)`, `parse_chunk_key(key)`,
|
|
226
|
-
`write_index(begin, end, data, partial=)
|
|
236
|
+
`write_index(begin, end, data, partial=)` and `write(box, data, partial=)` (`data` an array or a scalar), `reopen(context=)`.
|
|
227
237
|
|
|
228
238
|
A box is a `(begin, end)` pair of floats in your units, half-open per axis, over the
|
|
229
239
|
spatial axes or every axis; any object with `.begin`/`.end` or `.offset`/`.shape` also
|
|
@@ -50,6 +50,7 @@ six = wa.open(path, voxel_size=(6, 6, 6), on_mismatch="resample")
|
|
|
50
50
|
|
|
51
51
|
# one volume read onto another volume's grid, for masks, contact sites, statistics
|
|
52
52
|
onto = raw.read(box, voxel_size=labels.voxel_size, anchor=labels.corner)
|
|
53
|
+
onto = raw.read_onto(grid, begin, end) # voxels [begin, end) of a Grid, exactly that shape
|
|
53
54
|
|
|
54
55
|
# index space, native units: what an engine or a viewer wants
|
|
55
56
|
vol = wa.open(path, units=None)
|
|
@@ -100,15 +101,24 @@ s1 = wa.open_pyramid("out.zarr", mode="r+").add_level("s1", factors=2) # place
|
|
|
100
101
|
- **Plans outputs and blocks** (0.3): a store-less `Grid` with the same geometry as an
|
|
101
102
|
array, so an output is planned, cut into blocks and only then created; `blocks` tiling an
|
|
102
103
|
array or a box into write boxes with context-grown read boxes, in C order, as plain
|
|
103
|
-
picklable records a cluster job takes by id; the anisotropy-aware pyramid schedule
|
|
104
|
-
Nothing is scheduled or run.
|
|
104
|
+
picklable records a cluster job takes by id; the anisotropy-aware pyramid schedule; every
|
|
105
|
+
input read onto that grid by its voxel indices (0.4). Nothing is scheduled or run.
|
|
105
106
|
|
|
106
107
|
## The placement rule
|
|
107
108
|
|
|
108
109
|
A translation is where the centre of voxel 0 is. Index `i` sits at
|
|
109
|
-
`translation + i * voxel_size
|
|
110
|
-
|
|
111
|
-
|
|
110
|
+
`translation + i * voxel_size` and covers half a voxel either side, so the array's corner
|
|
111
|
+
is `corner = translation - voxel_size / 2`. With 8 nm voxels and the corner at 0, the
|
|
112
|
+
translation is 4: voxel 0 covers 0 to 8 nm around its centre at 4, voxel 1 covers 8 to
|
|
113
|
+
16 nm around 12. That is how OME-NGFF defines it and how Neuroglancer draws it.
|
|
114
|
+
|
|
115
|
+
The centre is the model because a voxel is a sample taken at a point. Everything that
|
|
116
|
+
compares or resamples grids (picking a level, reading one volume onto another's grid,
|
|
117
|
+
interpolating) works on those points, and the box a voxel covers follows from its point.
|
|
118
|
+
The read rule follows too: a voxel is read iff its centre lies in the box, so boxes that
|
|
119
|
+
adjoin never share a voxel or drop one.
|
|
120
|
+
|
|
121
|
+
Dialects that store the corner are moved half a voxel on reading:
|
|
112
122
|
|
|
113
123
|
| Stored | Means | Read as `translation` |
|
|
114
124
|
|---|---|---|
|
|
@@ -159,12 +169,12 @@ wa.units.abbreviation("nanometer") ; wa.store.join(path, "s0") ; wa.store.parent
|
|
|
159
169
|
```
|
|
160
170
|
|
|
161
171
|
`WorldArray`: `ts`, `meta`, `grid`, `shape`, `dtype`, `axes`, `voxel_size`, `translation`,
|
|
162
|
-
`corner`, `box`, `levels`, `level_name`, `level_path`, `source_voxel_size`, `resampled`,
|
|
172
|
+
`corner`, `box`, `levels`, `level_name`, `level_path`, `source_voxel_size`, `source_translation`, `resampled`,
|
|
163
173
|
`writable`, `write_shape`;
|
|
164
174
|
`read(box, voxel_size=, padding=, method=, mask=, anchor=, transform=, post_transform=, select=, strict=)`,
|
|
165
|
-
`read_index(begin, end, padding=, mask=, select=, transform=)`, `index_box(box)`,
|
|
175
|
+
`read_index(begin, end, padding=, mask=, select=, transform=)`, `read_onto(grid, begin=, end=, ...)`, `index_box(box)`,
|
|
166
176
|
`chunks_covering(begin, end, write=)`, `list_chunks()`, `chunk_key(index)`, `parse_chunk_key(key)`,
|
|
167
|
-
`write_index(begin, end, data, partial=)
|
|
177
|
+
`write_index(begin, end, data, partial=)` and `write(box, data, partial=)` (`data` an array or a scalar), `reopen(context=)`.
|
|
168
178
|
|
|
169
179
|
A box is a `(begin, end)` pair of floats in your units, half-open per axis, over the
|
|
170
180
|
spatial axes or every axis; any object with `.begin`/`.end` or `.offset`/`.shape` also
|
|
@@ -149,7 +149,9 @@ vol.meta # ArrayMeta in the store's own units
|
|
|
149
149
|
vol.grid # ArrayMeta in the requested units, every axis
|
|
150
150
|
vol.shape, vol.dtype, vol.axes, vol.spatial, vol.level_name, vol.levels, vol.remote
|
|
151
151
|
# shape is the stored shape, or with a resample target the target voxels covering the data
|
|
152
|
-
vol.source_voxel_size, vol.resampled
|
|
152
|
+
vol.source_voxel_size, vol.source_translation, vol.resampled
|
|
153
|
+
# the stored level's voxel size and translation (0.4), also under resample and relabel; whether
|
|
154
|
+
# reads resample to a target
|
|
153
155
|
vol.list_chunks() # the chunk indices that exist in storage, by listing keys (zarr v2/v3, N5, unsharded precomputed)
|
|
154
156
|
vol.path # what the caller passed (a group, say); vol.level_path is the array actually opened
|
|
155
157
|
vol.voxel_size # tuple[float] in `units`, spatial axes, the array's order (zyx for a volume)
|
|
@@ -212,7 +214,8 @@ window and a spatial box at once); anything with `.begin` and `.end`, or `.offse
|
|
|
212
214
|
`shift`, `snap(voxel_size, translation, mode="grow" | "shrink" | "closest")` to a grid's
|
|
213
215
|
voxel corners, `to_index` (the centre rule) and `Box.from_index`; `vol.box` is the array's
|
|
214
216
|
own extent on the spatial axes, `[corner, corner + shape * voxel_size)`, what funlib calls
|
|
215
|
-
the array's roi
|
|
217
|
+
the array's roi; with a resample target, the target voxels `shape` counts, so `read(vol.box)`
|
|
218
|
+
returns `shape` and equals `Grid.from_meta(meta, voxel_size=t).box`. So `vol.read(vol.box.intersect(roi).snap(vol.voxel_size, vol.translation))`
|
|
216
219
|
is the migration of a clipped, snapped Roi read, without funlib.
|
|
217
220
|
|
|
218
221
|
There is no `offset` attribute anywhere in this API, because that word means the corner in
|
|
@@ -283,7 +286,8 @@ vol = wa.create(
|
|
|
283
286
|
attrs=None, # extra array attributes, merged into the array's own
|
|
284
287
|
group_attrs=None, # extra group attributes, merged; no one else's key is dropped or rewritten (Behaviour 12)
|
|
285
288
|
exists="error", # "error": anything already at `path` raises | "adopt": an identical array (shape, dtype, chunks,
|
|
286
|
-
# shards, placement within rtol 1e-6
|
|
289
|
+
# shards, placement within rtol 1e-6, and from 0.4 every key of `attrs` holding the value given)
|
|
290
|
+
# is opened for writing; a different one raises naming each
|
|
287
291
|
# difference. There is no third value. An adopt writes no group document when the entry
|
|
288
292
|
# already says the same (Behaviour 12)
|
|
289
293
|
store_fill=False, # True stores write units equal to fill_value instead of leaving them absent
|
|
@@ -335,6 +339,20 @@ wa.store.join(path, *parts) ; wa.store.parent(path) ; wa.store.basename(path)
|
|
|
335
339
|
wa.units.abbreviation(unit) -> str # the OME short form, "nm" for any spelling of nanometre the table knows
|
|
336
340
|
```
|
|
337
341
|
|
|
342
|
+
### 0.4: reading onto a grid
|
|
343
|
+
|
|
344
|
+
```python
|
|
345
|
+
vol.read_onto(grid, begin=None, end=None, padding=None, method=None, mask=False, retries=None, transform=None,
|
|
346
|
+
post_transform=None, select=None)
|
|
347
|
+
# voxels [begin, end) of a Grid's spatial axes (all of it when None), read from this array:
|
|
348
|
+
# exactly that shape, by the centre rule at the array's voxel size, else resampled onto the
|
|
349
|
+
# grid anchored at its corner (Behaviour 20)
|
|
350
|
+
vol.write_index(begin, end, 0) ; vol.write(box, 0) ; vol.write_chunk(index, 0)
|
|
351
|
+
# a scalar fills the box, converted to the dtype only when its value survives (Behaviour 13)
|
|
352
|
+
vol.source_translation # the stored level's translation in `units`, beside source_voxel_size, under resample and relabel
|
|
353
|
+
wa.create(..., attrs={...}, exists="adopt") # an adopt also compares the keys of `attrs` (Behaviour 11)
|
|
354
|
+
```
|
|
355
|
+
|
|
338
356
|
## 3. Conventions
|
|
339
357
|
|
|
340
358
|
These are the rules the package exists to hold in one place. They match chunkmirage's
|
|
@@ -393,7 +411,7 @@ this document and that one differ, fix one of them.
|
|
|
393
411
|
3. **Units are kept as stored and converted only on the axes a caller asks to convert**
|
|
394
412
|
(`ArrayMeta.in_units(space=, time=)`, `in_nm()`, the `units` argument), through one table:
|
|
395
413
|
for length, American and British spellings, `nm`, `um`, `µm` (U+00B5), `μm` (U+03BC),
|
|
396
|
-
`mm`, `cm`, `m`, `angstrom`/`Å`, `pm`; for time, `s`/`second`, `ms`, `us`/`µs`, `ns`,
|
|
414
|
+
`mm`, `cm`, `m`, `km`, `angstrom`/`Å`, `pm`; for time, `s`/`second`, `ms`, `us`/`µs`, `ns`,
|
|
397
415
|
`minute`, `hour`, `day`. Any unit string on any axis is accepted and never rejected until
|
|
398
416
|
a conversion is asked for (`"s"`, `"m"`, `"degrees_north"`, `""`). A spatial axis with no
|
|
399
417
|
unit is taken as nanometres, with a **warning once per array** (every repo assumed this
|
|
@@ -549,8 +567,8 @@ this document and that one differ, fix one of them.
|
|
|
549
567
|
reads that level voxel for voxel and reports `voxel_size` as the requested size with
|
|
550
568
|
**voxel `i` staying voxel `i`**: every world quantity, the translation and the corner
|
|
551
569
|
included, scales by the ratio of the sizes (cellmap-flow's rule, which chunkmirage's
|
|
552
|
-
server relies on), so a box of `m` requested voxels returns `m` stored voxels of the
|
|
553
|
-
level; `"keep"` reads the chosen level at its own voxel size (a level pick, nothing
|
|
570
|
+
server relies on), so a box of `m` requested voxels returns `m` stored voxels of the level,
|
|
571
|
+
and `source_voxel_size` still reports the level's own size; `"keep"` reads the chosen level at its own voxel size (a level pick, nothing
|
|
554
572
|
more). `relabel` and `nearest` are different things and both exist: one is a grid
|
|
555
573
|
policy, the other a resampling kernel. `level` also takes a position, `0` or `"first"`
|
|
556
574
|
for the first listed level. The pyramid an array belongs to is found through
|
|
@@ -680,9 +698,11 @@ Behaviours 10 to 16 are 0.2; 17 to 19 are 0.3.
|
|
|
680
698
|
compression; cellmap-flow's blosc stays blosc by passing it. There is no overwrite: with
|
|
681
699
|
`exists="error"` any document already at `path` raises `FileExistsError`; with
|
|
682
700
|
`exists="adopt"` an array whose shape, dtype, chunks, shards and placement match
|
|
683
|
-
(placement within `rtol 1e-6`)
|
|
684
|
-
|
|
685
|
-
and
|
|
701
|
+
(placement within `rtol 1e-6`), and whose attributes hold every key of `attrs` with the
|
|
702
|
+
value given as JSON stores it (keys not given are someone else's and not compared), is
|
|
703
|
+
opened for writing and its dataset entry ensured, and one that differs raises
|
|
704
|
+
`ValueError` listing every differing field and key with the value found and the value
|
|
705
|
+
wanted (cellmap-flow's `_refuse_another_region`, registration-station's
|
|
686
706
|
missing check, which let a shorter last band's chunk grid win a creation race). A
|
|
687
707
|
creation that loses a race, the store reporting the array present after the probe said
|
|
688
708
|
absent, is re-read with backoff, 5 attempts at `0.5 s * attempt`, and then treated as an
|
|
@@ -695,7 +715,9 @@ Behaviours 10 to 16 are 0.2; 17 to 19 are 0.3.
|
|
|
695
715
|
being the one it replaces. `create` returns a writable `WorldArray`;
|
|
696
716
|
`wa.open(path, mode="r+")` opens an existing one with every rule of Conventions 3 to 5
|
|
697
717
|
unchanged, and a handle pickles by its arguments and its context's spec, so each worker
|
|
698
|
-
process opens its own cheaply
|
|
718
|
+
process opens its own cheaply (a local path goes resolved against the working directory
|
|
719
|
+
at open, so a worker that changes directory before unpickling, as cellmap-analyze's dask
|
|
720
|
+
workers do, opens the same store; `reopen` uses it too), on the shared context when the handle had one, else on
|
|
699
721
|
a context rebuilt once per process from the same spec, so a capped context stays
|
|
700
722
|
capped (mesh-n-bone's per-process cache and thread caps).
|
|
701
723
|
|
|
@@ -720,7 +742,12 @@ Behaviours 10 to 16 are 0.2; 17 to 19 are 0.3.
|
|
|
720
742
|
|
|
721
743
|
13. **Aligned writes need no coordination.** `write_index(begin, end, data)` requires
|
|
722
744
|
`data.shape == end - begin`, `data.dtype == vol.dtype` (no silent cast; `ValueError`),
|
|
723
|
-
|
|
745
|
+
or `data` a scalar (a number or a 0-d array) that fills the box: it is converted to the
|
|
746
|
+
dtype only when its value survives, so an integer or bool array takes whole numbers in
|
|
747
|
+
its range and a float array any number it does not overflow, rounded to its precision
|
|
748
|
+
(`0` into `uint8` is written; `300`, `-1`, `0.5` and NaN raise), and a scalar equal to
|
|
749
|
+
`fill_value` over whole units leaves them absent as an all-fill array does (cellmap-analyze
|
|
750
|
+
writes a constant for every empty or masked-out block). The box must be a union of whole write units: every bound a multiple of
|
|
724
751
|
`write_shape` on its axis or the array's end on that axis, so the clipped last unit is
|
|
725
752
|
allowed; otherwise `ValueError` names the axis, the bound and the unit size. A unit
|
|
726
753
|
written whole is one put, so writers in different processes writing different units
|
|
@@ -807,6 +834,22 @@ Behaviours 10 to 16 are 0.2; 17 to 19 are 0.3.
|
|
|
807
834
|
returns the OME short form for any spelling the table knows and raises for one it does
|
|
808
835
|
not.
|
|
809
836
|
|
|
837
|
+
20. **Reading onto a grid.** `vol.read_onto(grid, begin, end)` returns voxels `[begin, end)`
|
|
838
|
+
of a `Grid`'s spatial axes, all of it when `begin` and `end` are None, read from this
|
|
839
|
+
array, with exactly `end - begin` voxels on those axes; the array's other axes are read
|
|
840
|
+
as `read` reads them (whole, or pinned by `select`), the open-time `padding`, `method`,
|
|
841
|
+
`retries` and `transform` apply, and `mask` marks the grid voxels that held data. It is
|
|
842
|
+
`read(Box.from_index(begin, end, grid voxel size, grid translation), voxel_size=grid
|
|
843
|
+
voxel size, anchor=grid corner)`, so Behaviours 2 and 4 decide every voxel: at the
|
|
844
|
+
array's own voxel size (within level selection's rtol 1e-6) nothing is resampled,
|
|
845
|
+
whatever the offset between the grids, and each grid voxel takes the stored voxel whose
|
|
846
|
+
centre lies in it, never an interpolation, with `method="linear"` too; at any other
|
|
847
|
+
voxel size the array is resampled onto the grid, anchored at its corner. Adjoining
|
|
848
|
+
`[begin, end)` reads therefore agree with one read across both. The grid's spatial axes
|
|
849
|
+
must be named as the array's are (`ValueError` otherwise), and its units are converted
|
|
850
|
+
to the array's `units`. This is cellmap-analyze's per-operation grid: blocks come from
|
|
851
|
+
one `Grid`, every input is read onto it, and outputs are created on it.
|
|
852
|
+
|
|
810
853
|
## 5. Tests (behavioural; synthetic stores in `tmp_path`; one optional network test)
|
|
811
854
|
|
|
812
855
|
Conformance, one tiny store each, checking `voxel_size`, `translation`, `axes`,
|
|
@@ -896,8 +939,10 @@ factor; missing ancestor documents down from the `.zarr` root and none above it,
|
|
|
896
939
|
ancestor's format followed and a contradiction refused, an array in the way refused; an
|
|
897
940
|
adopt writing no document and a changed placement writing one; a lost entry written again
|
|
898
941
|
and the error after the attempts; `ensure_group_entry` on a foreign array, idempotent; a
|
|
899
|
-
pickled handle keeping a capped context's spec and sharing the shared one
|
|
900
|
-
path; an all-fill write removing its own key
|
|
942
|
+
pickled handle keeping a capped context's spec and sharing the shared one, and a relative
|
|
943
|
+
path reopening after a `chdir`; `codecs_of` by path; an all-fill write removing its own key;
|
|
944
|
+
a scalar filling a box through `write_index`, `write` and `write_chunk`, refused when it
|
|
945
|
+
does not fit the dtype.
|
|
901
946
|
|
|
902
947
|
Grids (0.3): `blocks` partitioning with and without a box, `context` as an int, per axis
|
|
903
948
|
and as before/after pairs, clipping on and off, `len`, `blocks[id]` and `index_of`;
|
|
@@ -906,6 +951,14 @@ and as before/after pairs, clipping on and off, `len`, `blocks[id]` and `index_o
|
|
|
906
951
|
reproducing mesh-n-bone's anisotropic table (8, 8, 30 nm: z stays at 1 until y and x
|
|
907
952
|
reach 32) and all 2s for isotropic data; `abbreviation` for every spelling in the table.
|
|
908
953
|
|
|
954
|
+
Reading onto a grid (0.4): `read_onto` at the array's voxel size returning stored values
|
|
955
|
+
under a sub-voxel offset with `method="linear"`, and the next voxel past a half-voxel one;
|
|
956
|
+
onto a finer grid equal to the `read` it is defined as, in micrometres too, adjoining
|
|
957
|
+
reads agreeing, the mask marking grid voxels before the data; its own grid at voxel sizes
|
|
958
|
+
3.54, 4 and 5.3 returning the stored array exactly; a grid with other axis names refused.
|
|
959
|
+
`source_translation` under relabel and resample and in micrometres; an adopt keeping keys
|
|
960
|
+
it was not given and refusing a different or missing one, naming both values.
|
|
961
|
+
|
|
909
962
|
Target: about 600 lines of test cases for 0.1 and about 300 more for 0.2 and 0.3, the
|
|
910
963
|
store builders in `conftest.py` aside; no wording pins; no `/nrs`; no cluster.
|
|
911
964
|
|
|
@@ -1082,10 +1135,14 @@ tests, minimal reusable code. The README is sections 1 to 5 of this file; a beha
|
|
|
1082
1135
|
is not stated here is not implemented without adding the statement first.
|
|
1083
1136
|
|
|
1084
1137
|
Versioning. Every consumer pins a floating range and its CI installs the newest version in
|
|
1085
|
-
it
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1138
|
+
it within the hour of a release, so the version number says what a release can break. A
|
|
1139
|
+
**patch** release brings the code back to what this spec says, or reads more stores or
|
|
1140
|
+
reads them faster; it places nothing differently and adds nothing public. A **minor**
|
|
1141
|
+
release is anything else: it may move voxels or add API, and an entry added to, renamed in
|
|
1142
|
+
or removed from the unit table is API (`wa.units.TABLE` is public, and cellmap-flow exports
|
|
1143
|
+
it to its browser app behind a freshness test). Both were once shipped as patches and both
|
|
1144
|
+
turned cellmap-flow's CI red on every open PR within the hour: 0.1.1 moved offset-less N5
|
|
1145
|
+
pyramids by half a base voxel, and 0.3.1 added kilometres. A changelog entry starts with
|
|
1146
|
+
**Placement change** when voxels move and **Unit table change** when the table does, and the
|
|
1147
|
+
release message to each consumer repeats those lines; nothing before 1.0 promises more.
|
|
1148
|
+
Consumers that cannot absorb a move between CI runs pin `>=0.x,<0.(x+1)`.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "worldarray"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.0"
|
|
4
4
|
description = "Read zarr, N5 and Neuroglancer precomputed volumes in world coordinates through tensorstore: correct voxel placement, pyramid levels, padded region reads, resampling. numpy + tensorstore only."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = { file = "LICENSE" }
|
|
@@ -36,7 +36,15 @@ from .placement import (
|
|
|
36
36
|
read_meta_light,
|
|
37
37
|
)
|
|
38
38
|
from .retry import Retry, read_with_retries
|
|
39
|
-
from .store import
|
|
39
|
+
from .store import (
|
|
40
|
+
Node,
|
|
41
|
+
detect,
|
|
42
|
+
is_remote,
|
|
43
|
+
join,
|
|
44
|
+
normalize_path,
|
|
45
|
+
open_array,
|
|
46
|
+
precomputed_scale_index,
|
|
47
|
+
)
|
|
40
48
|
|
|
41
49
|
logger = logging.getLogger(__name__)
|
|
42
50
|
|
|
@@ -90,6 +98,9 @@ class WorldArray:
|
|
|
90
98
|
if mode not in ("r", "r+"):
|
|
91
99
|
raise ValueError("mode is 'r' (read) or 'r+' (read and write)")
|
|
92
100
|
self.path = path
|
|
101
|
+
# where later opens find it (reopen, a pickle, the pyramid): a local path resolved
|
|
102
|
+
# against the working directory now, so a worker that changes directory opens the same store
|
|
103
|
+
self._where = normalize_path(path)
|
|
93
104
|
self._mode = mode
|
|
94
105
|
self._store_fill = bool(store_fill)
|
|
95
106
|
self.padding = padding
|
|
@@ -139,6 +150,9 @@ class WorldArray:
|
|
|
139
150
|
self._meta = self._rescaled_to_base(base_voxel_size, base_units)
|
|
140
151
|
space, time = _unit_targets(units)
|
|
141
152
|
self._grid = self._meta.in_units(space=space, time=time, strict=strict_units)
|
|
153
|
+
# the stored level's spatial placement, before any relabel renames the grid
|
|
154
|
+
self._stored_voxel_size = tuple(self._grid.voxel_size[i] for i in self._grid.spatial)
|
|
155
|
+
self._stored_translation = tuple(self._grid.translation[i] for i in self._grid.spatial)
|
|
142
156
|
|
|
143
157
|
self._target: tuple[float, ...] | None = None
|
|
144
158
|
if voxel_size is not None:
|
|
@@ -299,8 +313,16 @@ class WorldArray:
|
|
|
299
313
|
|
|
300
314
|
@property
|
|
301
315
|
def source_voxel_size(self) -> tuple[float, ...]:
|
|
302
|
-
"""The stored level's voxel size on the spatial axes, in ``units
|
|
303
|
-
|
|
316
|
+
"""The stored level's voxel size on the spatial axes, in ``units``: what the level
|
|
317
|
+
is, whether reads resample it or ``on_mismatch="relabel"`` renames its grid."""
|
|
318
|
+
return self._stored_voxel_size
|
|
319
|
+
|
|
320
|
+
@property
|
|
321
|
+
def source_translation(self) -> tuple[float, ...]:
|
|
322
|
+
"""The stored level's translation (the centre of its voxel 0) on the spatial axes, in
|
|
323
|
+
``units``: where the level is, whether reads resample it or ``on_mismatch="relabel"``
|
|
324
|
+
renames its grid."""
|
|
325
|
+
return self._stored_translation
|
|
304
326
|
|
|
305
327
|
@property
|
|
306
328
|
def resampled(self) -> bool:
|
|
@@ -337,8 +359,22 @@ class WorldArray:
|
|
|
337
359
|
def box(self) -> Box:
|
|
338
360
|
"""The array's extent on the spatial axes as a world box, ``[corner, corner + shape *
|
|
339
361
|
voxel_size)``: what funlib calls the array's roi. ``Box.from_index`` gives the same
|
|
340
|
-
over every axis.
|
|
362
|
+
over every axis. With a resample target it is the extent of the target voxels that
|
|
363
|
+
``shape`` counts, which may reach past the stored data by less than one target voxel,
|
|
364
|
+
so ``read(box)`` returns ``shape``."""
|
|
341
365
|
sp = self._grid.spatial
|
|
366
|
+
if self._target is not None:
|
|
367
|
+
begin, end = [], []
|
|
368
|
+
for k, axis in enumerate(sp):
|
|
369
|
+
first, last = _resample.target_range(
|
|
370
|
+
self._grid.shape[axis],
|
|
371
|
+
self._grid.voxel_size[axis],
|
|
372
|
+
self._target[k],
|
|
373
|
+
self._origins()[k],
|
|
374
|
+
)
|
|
375
|
+
begin.append(self.anchor[k] + first * self._target[k])
|
|
376
|
+
end.append(self.anchor[k] + last * self._target[k])
|
|
377
|
+
return Box(tuple(begin), tuple(end))
|
|
342
378
|
return Box.from_index(
|
|
343
379
|
(0,) * len(sp),
|
|
344
380
|
tuple(self._grid.shape[i] for i in sp),
|
|
@@ -356,7 +392,7 @@ class WorldArray:
|
|
|
356
392
|
def levels_pyramid(self) -> Pyramid:
|
|
357
393
|
if self._pyramid is None:
|
|
358
394
|
self._pyramid = Pyramid(
|
|
359
|
-
self.
|
|
395
|
+
self._where, context=self.context, units=self.units, select=self._select
|
|
360
396
|
)
|
|
361
397
|
return self._pyramid
|
|
362
398
|
|
|
@@ -405,14 +441,15 @@ class WorldArray:
|
|
|
405
441
|
**overrides,
|
|
406
442
|
}
|
|
407
443
|
return WorldArray(
|
|
408
|
-
self.
|
|
444
|
+
self._where, context=context if context is not None else self.context, **opts
|
|
409
445
|
)
|
|
410
446
|
|
|
411
447
|
def __reduce__(self):
|
|
412
448
|
"""A handle pickles by its arguments and its context's recipe, and reopens in the
|
|
413
449
|
receiving process (Behaviour 11): a worker opens its own store, cheaply, on a shared
|
|
414
450
|
context when the handle had one, else on a context rebuilt from the same spec, so a
|
|
415
|
-
capped context stays capped.
|
|
451
|
+
capped context stays capped. A local path goes as resolved at open, so a worker that
|
|
452
|
+
changed directory still finds the store."""
|
|
416
453
|
opts = {
|
|
417
454
|
**self._opts,
|
|
418
455
|
"padding": self.padding,
|
|
@@ -422,7 +459,7 @@ class WorldArray:
|
|
|
422
459
|
"retries": self.retries,
|
|
423
460
|
"transform": self.transform,
|
|
424
461
|
}
|
|
425
|
-
return (_reopen, (self.
|
|
462
|
+
return (_reopen, (self._where, opts, recipe_of(self.context)))
|
|
426
463
|
|
|
427
464
|
# --- index space ------------------------------------------------------------
|
|
428
465
|
|
|
@@ -503,7 +540,9 @@ class WorldArray:
|
|
|
503
540
|
or the array's end), so writers in different processes need no coordination; a
|
|
504
541
|
unit written whole is one put. ``partial=True`` lifts the check and does tensorstore's
|
|
505
542
|
read-modify-write of the touched units, unsafe while anything else writes the same
|
|
506
|
-
unit.
|
|
543
|
+
unit. ``data`` may be a scalar, which fills the box: it is converted to the array's
|
|
544
|
+
dtype only when the value survives (``0`` into ``uint8``, not ``300`` or ``0.5``).
|
|
545
|
+
No other cast, no padding, no resampling; the write commits before returning."""
|
|
507
546
|
if not self.writable:
|
|
508
547
|
raise ValueError(
|
|
509
548
|
f"{self.path} was opened read-only; open it with mode='r+' or create it"
|
|
@@ -518,9 +557,11 @@ class WorldArray:
|
|
|
518
557
|
if len(begin) != len(shape) or len(end) != len(shape):
|
|
519
558
|
raise ValueError(f"{self.path}: begin and end need one entry per axis of {shape}")
|
|
520
559
|
wanted = tuple(e - b for b, e in zip(begin, end))
|
|
521
|
-
if data.
|
|
560
|
+
if data.ndim == 0:
|
|
561
|
+
data = self._scalar(data)
|
|
562
|
+
elif data.shape != wanted:
|
|
522
563
|
raise ValueError(f"{self.path}: data has shape {data.shape}; the box is {wanted}")
|
|
523
|
-
|
|
564
|
+
elif data.dtype != self.dtype:
|
|
524
565
|
raise ValueError(
|
|
525
566
|
f"{self.path}: data is {data.dtype}, the array {self.dtype}; nothing is cast on write"
|
|
526
567
|
)
|
|
@@ -538,6 +579,22 @@ class WorldArray:
|
|
|
538
579
|
)
|
|
539
580
|
self._store[tuple(slice(b, e) for b, e in zip(begin, end))].write(data).result()
|
|
540
581
|
|
|
582
|
+
def _scalar(self, value: np.ndarray) -> np.ndarray:
|
|
583
|
+
"""A 0-d ``value`` in the array's dtype, or ``ValueError`` when converting changes it:
|
|
584
|
+
an integer or bool array takes only whole numbers in its range, a float array any
|
|
585
|
+
number that does not overflow it (rounded to its precision)."""
|
|
586
|
+
dtype = self.dtype
|
|
587
|
+
with np.errstate(invalid="ignore", over="ignore"), warnings.catch_warnings():
|
|
588
|
+
warnings.simplefilter("ignore")
|
|
589
|
+
out = value.astype(dtype)
|
|
590
|
+
if dtype.kind in "biu":
|
|
591
|
+
kept = bool(out == value)
|
|
592
|
+
else:
|
|
593
|
+
kept = bool(np.isfinite(out) or not np.isfinite(value))
|
|
594
|
+
if not kept:
|
|
595
|
+
raise ValueError(f"{self.path}: {value.item()!r} does not fit {dtype}")
|
|
596
|
+
return out
|
|
597
|
+
|
|
541
598
|
def write(self, box, data: np.ndarray, partial: bool = False) -> None:
|
|
542
599
|
"""Write ``data`` over a world box: the voxels Behaviour 2's centre rule selects, then
|
|
543
600
|
``write_index``. A box reaching outside the array raises; nothing is padded or
|
|
@@ -863,6 +920,57 @@ class WorldArray:
|
|
|
863
920
|
data[~inside] = padding
|
|
864
921
|
return _squeeze((data, inside) if mask else data, drop)
|
|
865
922
|
|
|
923
|
+
def read_onto(
|
|
924
|
+
self,
|
|
925
|
+
grid,
|
|
926
|
+
begin: Sequence[int] | None = None,
|
|
927
|
+
end: Sequence[int] | None = None,
|
|
928
|
+
padding: Any = None,
|
|
929
|
+
method: str | None = None,
|
|
930
|
+
mask: bool = False,
|
|
931
|
+
retries: Retry | None = None,
|
|
932
|
+
transform: Transform | None = None,
|
|
933
|
+
post_transform: Transform | None = None,
|
|
934
|
+
select: dict[str, int] | None = None,
|
|
935
|
+
) -> np.ndarray | tuple[np.ndarray, np.ndarray]:
|
|
936
|
+
"""Voxels ``[begin, end)`` of another grid (a ``Grid``), read from this array
|
|
937
|
+
(Behaviour 20): ``begin`` and ``end`` index the grid's spatial axes, all of it when
|
|
938
|
+
None, and the result has exactly that shape there; other axes are read as ``read``
|
|
939
|
+
reads them. At the array's own voxel size nothing is resampled, however the grids are
|
|
940
|
+
offset: each grid voxel takes the stored voxel whose centre lies in it. Otherwise the
|
|
941
|
+
array is resampled onto the grid by ``method``, anchored at the grid's corner. The
|
|
942
|
+
grid's units are converted to ``units``; its spatial axes must be named as this
|
|
943
|
+
array's are."""
|
|
944
|
+
sp, mine = grid.spatial, self._grid.spatial
|
|
945
|
+
names = tuple(grid.axes[i] for i in sp)
|
|
946
|
+
if names != tuple(self._grid.axes[i] for i in mine):
|
|
947
|
+
raise ValueError(
|
|
948
|
+
f"{self.path}: the grid's spatial axes are {names}; the array's "
|
|
949
|
+
f"{tuple(self._grid.axes[i] for i in mine)}"
|
|
950
|
+
)
|
|
951
|
+
vs, tr = [], []
|
|
952
|
+
for i, j in zip(sp, mine):
|
|
953
|
+
have, want = grid.units[i], self._grid.units[j]
|
|
954
|
+
f = 1.0 if have == want else U.factor(have, want)
|
|
955
|
+
vs.append(grid.voxel_size[i] * f)
|
|
956
|
+
tr.append(grid.translation[i] * f)
|
|
957
|
+
begin = (0,) * len(sp) if begin is None else tuple(int(b) for b in begin)
|
|
958
|
+
end = tuple(grid.shape[i] for i in sp) if end is None else tuple(int(e) for e in end)
|
|
959
|
+
if len(begin) != len(sp) or len(end) != len(sp):
|
|
960
|
+
raise ValueError(f"begin and end need one entry per spatial axis of the grid {names}")
|
|
961
|
+
return self.read(
|
|
962
|
+
Box.from_index(begin, end, vs, tr),
|
|
963
|
+
voxel_size=vs,
|
|
964
|
+
anchor=tuple(t - v / 2 for t, v in zip(tr, vs)),
|
|
965
|
+
padding=padding,
|
|
966
|
+
method=method,
|
|
967
|
+
mask=mask,
|
|
968
|
+
retries=retries,
|
|
969
|
+
transform=transform,
|
|
970
|
+
post_transform=post_transform,
|
|
971
|
+
select=select,
|
|
972
|
+
)
|
|
973
|
+
|
|
866
974
|
|
|
867
975
|
def _squeeze(out, drop: tuple[int, ...]):
|
|
868
976
|
if not drop:
|
|
@@ -1106,8 +1214,8 @@ def create(
|
|
|
1106
1214
|
0.5) following the group's format (Behaviours 11 and 12). ``chunks`` is required.
|
|
1107
1215
|
Nothing is ever deleted or overwritten: with ``exists="error"`` anything at ``path``
|
|
1108
1216
|
raises ``FileExistsError``; with ``"adopt"`` an identical array (shape, dtype, chunks,
|
|
1109
|
-
shards, placement
|
|
1110
|
-
naming every difference. Returns a writable ``WorldArray``."""
|
|
1217
|
+
shards, placement, and every key of ``attrs`` already holding the value given) is opened
|
|
1218
|
+
for writing and a different one raises ``ValueError`` naming every difference. Returns a writable ``WorldArray``."""
|
|
1111
1219
|
if grid is not None:
|
|
1112
1220
|
# a Grid planned the output (Behaviour 18): its facts are the array's, chunks and
|
|
1113
1221
|
# shards overridable; the placement arguments must not also be given
|
|
@@ -1148,6 +1256,7 @@ def create(
|
|
|
1148
1256
|
shards=shards,
|
|
1149
1257
|
voxel_size=vs,
|
|
1150
1258
|
translation=tr,
|
|
1259
|
+
attrs=attrs,
|
|
1151
1260
|
)
|
|
1152
1261
|
if diffs:
|
|
1153
1262
|
raise ValueError(
|
|
@@ -153,14 +153,22 @@ def apply_axis(block: np.ndarray, axis: int, plan: AxisPlan) -> np.ndarray:
|
|
|
153
153
|
raise ValueError(plan.kind)
|
|
154
154
|
|
|
155
155
|
|
|
156
|
+
def target_range(n: int, s: float, t: float, origin: float = 0.0) -> tuple[int, int]:
|
|
157
|
+
"""``[first, last)``: the target voxels of size ``t`` that overlap ``n`` source voxels of
|
|
158
|
+
size ``s``, indexed on a target grid whose voxel 0 corner sits ``origin`` source voxels
|
|
159
|
+
from the source corner."""
|
|
160
|
+
ratio = t / s
|
|
161
|
+
first = int(np.floor(snap_integral(-origin / ratio)))
|
|
162
|
+
last = int(np.ceil(snap_integral((n - origin) / ratio)))
|
|
163
|
+
return first, max(last, first)
|
|
164
|
+
|
|
165
|
+
|
|
156
166
|
def target_shape(n: int, s: float, t: float, origin: float = 0.0) -> int:
|
|
157
167
|
"""How many target voxels of size ``t`` cover ``n`` source voxels of size ``s``, counting
|
|
158
168
|
from a target grid whose voxel 0 corner sits ``origin`` source voxels from the source
|
|
159
169
|
corner: the first and last target voxels that overlap the data, inclusive."""
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
last = np.ceil(snap_integral((n - origin) / ratio))
|
|
163
|
-
return int(max(last - first, 0))
|
|
170
|
+
first, last = target_range(n, s, t, origin)
|
|
171
|
+
return last - first
|
|
164
172
|
|
|
165
173
|
|
|
166
174
|
def resample(
|
|
@@ -19,6 +19,7 @@ LENGTH_M = {
|
|
|
19
19
|
"millimeter": 1e-3, "millimetre": 1e-3, "mm": 1e-3,
|
|
20
20
|
"centimeter": 1e-2, "centimetre": 1e-2, "cm": 1e-2,
|
|
21
21
|
"meter": 1.0, "metre": 1.0, "m": 1.0,
|
|
22
|
+
"kilometer": 1e3, "kilometre": 1e3, "km": 1e3,
|
|
22
23
|
} # fmt: skip
|
|
23
24
|
TIME_S = {
|
|
24
25
|
"nanosecond": 1e-9, "ns": 1e-9,
|
|
@@ -81,7 +82,7 @@ def factor(
|
|
|
81
82
|
# entry through the table above, so "nanometre", "nm" and "Nanometers" agree.
|
|
82
83
|
_SHORT = {
|
|
83
84
|
"picometer": "pm", "angstrom": "Å", "nanometer": "nm", "micrometer": "µm",
|
|
84
|
-
"millimeter": "mm", "centimeter": "cm", "meter": "m",
|
|
85
|
+
"millimeter": "mm", "centimeter": "cm", "meter": "m", "kilometer": "km",
|
|
85
86
|
"nanosecond": "ns", "microsecond": "µs", "millisecond": "ms", "second": "s",
|
|
86
87
|
"minute": "min", "hour": "h", "day": "d",
|
|
87
88
|
} # fmt: skip
|
|
@@ -103,6 +104,7 @@ def abbreviation(unit: str) -> str:
|
|
|
103
104
|
# The American spelling is OME's.
|
|
104
105
|
_OME_NAMES = (
|
|
105
106
|
"picometer", "angstrom", "nanometer", "micrometer", "millimeter", "centimeter", "meter",
|
|
107
|
+
"kilometer",
|
|
106
108
|
"nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day",
|
|
107
109
|
) # fmt: skip
|
|
108
110
|
_OME_BY_SIZE = {_lookup(name): name for name in _OME_NAMES}
|
|
@@ -803,9 +803,12 @@ def differences(
|
|
|
803
803
|
shards: Sequence[int] | None,
|
|
804
804
|
voxel_size: Sequence[float],
|
|
805
805
|
translation: Sequence[float],
|
|
806
|
+
attrs: dict | None = None,
|
|
806
807
|
) -> list[str]:
|
|
807
808
|
"""How an existing array's ``ArrayMeta`` differs from what a caller wanted to create:
|
|
808
|
-
one line per field, empty when identical (placement within ``RTOL``).
|
|
809
|
+
one line per field, empty when identical (placement within ``RTOL``). Of the array's
|
|
810
|
+
attributes only the keys of ``attrs`` are compared, as JSON stores them; every other key
|
|
811
|
+
is someone else's."""
|
|
809
812
|
out = []
|
|
810
813
|
if tuple(meta.shape) != tuple(int(n) for n in shape):
|
|
811
814
|
out.append(f"shape: found {tuple(meta.shape)}, wanted {tuple(shape)}")
|
|
@@ -826,4 +829,10 @@ def differences(
|
|
|
826
829
|
out.append(
|
|
827
830
|
f"{what}: found {tuple(float(x) for x in f)}, wanted {tuple(float(x) for x in w)}"
|
|
828
831
|
)
|
|
832
|
+
found_attrs = meta.attrs or {}
|
|
833
|
+
for key, wanted in json.loads(json.dumps(attrs or {})).items():
|
|
834
|
+
if key not in found_attrs:
|
|
835
|
+
out.append(f"attrs[{key!r}]: missing, wanted {wanted!r}")
|
|
836
|
+
elif found_attrs[key] != wanted:
|
|
837
|
+
out.append(f"attrs[{key!r}]: found {found_attrs[key]!r}, wanted {wanted!r}")
|
|
829
838
|
return out
|
|
@@ -299,6 +299,7 @@ SHORT_FORMS = {
|
|
|
299
299
|
"mm": ("millimeter", "millimetre", "mm"),
|
|
300
300
|
"cm": ("centimeter", "centimetre", "cm"),
|
|
301
301
|
"m": ("meter", "metre", "m"),
|
|
302
|
+
"km": ("kilometer", "kilometre", "km"),
|
|
302
303
|
"ns": ("nanosecond", "ns"),
|
|
303
304
|
"µs": ("microsecond", "us", "µs", "μs"),
|
|
304
305
|
"ms": ("millisecond", "ms"),
|
|
@@ -550,6 +550,34 @@ def test_relabel_rescales_everything_by_the_ratio(tmp_path):
|
|
|
550
550
|
and vol.translation == (20.0, 20.0, 20.0)
|
|
551
551
|
and vol.corner == (16.0, 16.0, 16.0)
|
|
552
552
|
)
|
|
553
|
+
assert vol.source_voxel_size == (4.0, 4.0, 4.0) # what the level is (cellmap-flow's report)
|
|
554
|
+
resampled = wa.open(str(group / "s0"), voxel_size=(8, 8, 8), on_mismatch="resample")
|
|
555
|
+
assert resampled.source_voxel_size == (4.0, 4.0, 4.0)
|
|
556
|
+
assert resampled.voxel_size == (8.0, 8.0, 8.0)
|
|
557
|
+
for v in (
|
|
558
|
+
vol,
|
|
559
|
+
resampled,
|
|
560
|
+
): # where the level is, beside what it is (cellmap-flow's stored_placement)
|
|
561
|
+
assert v.source_translation == (10.0, 10.0, 10.0)
|
|
562
|
+
assert wa.open(str(group / "s0"), units="micrometer").source_translation == (0.01, 0.01, 0.01)
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def test_a_resampled_box_is_the_target_voxels_shape_counts(tmp_path):
|
|
566
|
+
"""cellmap-flow's 7 voxels of 8 nm read at 24 nm: 3 target voxels, box 0 to 72, not 56."""
|
|
567
|
+
group = tmp_path / "rs.zarr"
|
|
568
|
+
write_zarr2_group(group, ome_multiscales("zyx", [("s0", (8, 8, 8), (4, 4, 4))]))
|
|
569
|
+
write_zarr2(group / "s0", pattern((7, 7, 7)))
|
|
570
|
+
vol = wa.open(str(group / "s0"), voxel_size=(24, 24, 24), on_mismatch="resample")
|
|
571
|
+
assert vol.shape == (3, 3, 3) and vol.box == wa.Box((0.0,) * 3, (72.0,) * 3)
|
|
572
|
+
assert vol.read(vol.box).shape == vol.shape
|
|
573
|
+
g = wa.Grid.from_meta(wa.open(str(group / "s0")).grid, voxel_size=(24, 24, 24))
|
|
574
|
+
assert g.box == vol.box # the planned grid and the opened array agree
|
|
575
|
+
# target corners at 4 + 24 k: the voxels from -20 cover the data from 0
|
|
576
|
+
anchored = wa.open(
|
|
577
|
+
str(group / "s0"), voxel_size=(24, 24, 24), on_mismatch="resample", anchor=(4, 4, 4)
|
|
578
|
+
)
|
|
579
|
+
assert anchored.shape == (4, 4, 4) and anchored.box == wa.Box((-20.0,) * 3, (76.0,) * 3)
|
|
580
|
+
assert anchored.read(anchored.box).shape == anchored.shape
|
|
553
581
|
|
|
554
582
|
|
|
555
583
|
def test_post_transform_runs_after_resampling_and_before_padding(tmp_path):
|
|
@@ -139,3 +139,45 @@ def test_partial_block_mean_and_edge_fill(tmp_path):
|
|
|
139
139
|
((0, 0, 0), (48, 16, 16)), voxel_size=(8, 8, 8), method="nearest", padding="edge"
|
|
140
140
|
)
|
|
141
141
|
assert list(near[:, 0, 0]) == pytest.approx([1, 3, 5, 6, 6, 6])
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def test_read_onto_reads_another_grids_voxels(tmp_path):
|
|
145
|
+
"""cellmap-analyze's per-operation grid (Behaviour 20): voxels [begin, end) of a grid,
|
|
146
|
+
exactly that many, from any array; at the array's own voxel size nothing is resampled,
|
|
147
|
+
however far under a voxel the grids are offset."""
|
|
148
|
+
path, data = _ramp(tmp_path, "float32")
|
|
149
|
+
vol = wa.open(path, method="linear")
|
|
150
|
+
near = wa.Grid((8, 4, 4), (4, 4, 4), (3, 3, 3)) # corner 1 nm past the array's
|
|
151
|
+
np.testing.assert_array_equal(vol.read_onto(near), data) # stored values, not interpolated
|
|
152
|
+
far = wa.Grid(
|
|
153
|
+
(8, 4, 4), (4, 4, 4), (5, 2, 2)
|
|
154
|
+
) # 3 nm past on z: each grid voxel holds the next centre
|
|
155
|
+
out = vol.read_onto(far)
|
|
156
|
+
assert out.shape == (8, 4, 4)
|
|
157
|
+
np.testing.assert_array_equal(out[:7], data[1:])
|
|
158
|
+
assert np.all(out[7] == 0) # the last grid voxel holds no stored centre: the fill
|
|
159
|
+
finer = wa.Grid((16, 8, 8), (2, 2, 2), (-1, -1, -1), chunks=(4, 4, 4)) # corner -2
|
|
160
|
+
expect = vol.read(finer.box, voxel_size=(2, 2, 2), anchor=finer.corner)
|
|
161
|
+
np.testing.assert_array_equal(vol.read_onto(finer), expect)
|
|
162
|
+
halves = np.concatenate(
|
|
163
|
+
[vol.read_onto(finer, (0, 0, 0), (6, 8, 8)), vol.read_onto(finer, (6, 0, 0), (16, 8, 8))]
|
|
164
|
+
)
|
|
165
|
+
np.testing.assert_array_equal(halves, expect) # adjoining reads agree with one read
|
|
166
|
+
in_um = wa.Grid((16, 8, 8), (0.002,) * 3, (-0.001,) * 3, units="micrometer")
|
|
167
|
+
np.testing.assert_array_equal(vol.read_onto(in_um), expect)
|
|
168
|
+
# the grid's voxels -1 and 0 are centred at -3 and -1 nm, before the data: fill, masked
|
|
169
|
+
out, inside = vol.read_onto(finer, (-1, 1, 1), (3, 3, 3), mask=True)
|
|
170
|
+
assert out.shape == (4, 2, 2) and not inside[:2].any() and inside[2:].all()
|
|
171
|
+
with pytest.raises(ValueError, match="spatial axes"):
|
|
172
|
+
vol.read_onto(wa.Grid((4, 4), (4, 4), (2, 2), axes=("y", "x")))
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def test_read_onto_its_own_grid_is_the_stored_array(tmp_path):
|
|
176
|
+
data = np.arange(7 * 6 * 5, dtype="uint16").reshape(7, 6, 5)
|
|
177
|
+
group = tmp_path / "odd.zarr"
|
|
178
|
+
write_zarr2_group(group, ome_multiscales("zyx", [("s0", (3.54, 4, 5.3), (1.0, -2.3, 7.7))]))
|
|
179
|
+
write_zarr2(group / "s0", data, chunks=(3, 3, 3))
|
|
180
|
+
vol = wa.open(str(group / "s0"))
|
|
181
|
+
np.testing.assert_array_equal(vol.read_onto(wa.Grid.from_meta(vol.grid)), data)
|
|
182
|
+
twice = wa.Grid.from_meta(vol.grid, voxel_size=(1.77, 2, 2.65))
|
|
183
|
+
assert vol.read_onto(twice).shape == twice.shape == (14, 12, 10)
|
|
@@ -57,6 +57,7 @@ def test_units_are_written_in_ome_names_and_unknown_ones_as_given():
|
|
|
57
57
|
assert wa.units.ome_name(spelling) == "nanometer"
|
|
58
58
|
assert wa.units.ome_name("um") == wa.units.ome_name("\u00b5m") == "micrometer"
|
|
59
59
|
assert wa.units.ome_name("s") == "second"
|
|
60
|
+
assert wa.units.ome_name("km") == wa.units.ome_name("Kilometres") == "kilometer"
|
|
60
61
|
with pytest.raises(ValueError, match="unknown unit"):
|
|
61
62
|
wa.units.ome_name("pixel")
|
|
62
63
|
doc = wa.ome_multiscales(("z", "y", "x"), "pixel", [("s0", (1, 1, 1), (0.5,) * 3)])
|
|
@@ -271,6 +272,19 @@ def test_create_never_overwrites_and_adopts_only_an_identical_array(tmp_path):
|
|
|
271
272
|
wa.create(path, *args[:3], (3, 2, 2), chunks=(4, 4, 4), exists="adopt")
|
|
272
273
|
|
|
273
274
|
|
|
275
|
+
def test_adopt_checks_the_attributes_it_was_given(tmp_path):
|
|
276
|
+
"""cellmap-flow's _refuse_another_region: funlib's prepare_ds kept an old offset silently."""
|
|
277
|
+
path = str(tmp_path / "vol.zarr/s0")
|
|
278
|
+
args = ((8, 8, 8), "uint8", (4, 4, 4), (2, 2, 2))
|
|
279
|
+
wa.create(path, *args, chunks=(4, 4, 4), attrs={"offset": (0, 0, 0), "model": "a"})
|
|
280
|
+
again = wa.create(path, *args, chunks=(4, 4, 4), attrs={"offset": [0.0, 0, 0]}, exists="adopt")
|
|
281
|
+
assert again.meta.attrs["model"] == "a" # keys not given are not compared
|
|
282
|
+
with pytest.raises(ValueError) as e:
|
|
283
|
+
wa.create(path, *args, chunks=(4, 4, 4), attrs={"model": "b", "run": 2}, exists="adopt")
|
|
284
|
+
assert "attrs['model']: found 'a', wanted 'b'" in str(e.value)
|
|
285
|
+
assert "attrs['run']: missing, wanted 2" in str(e.value)
|
|
286
|
+
|
|
287
|
+
|
|
274
288
|
def test_create_keeps_everyone_elses_attributes(tmp_path):
|
|
275
289
|
group = tmp_path / "shared.zarr"
|
|
276
290
|
write_zarr3_group(
|
|
@@ -719,6 +733,44 @@ def test_ensure_group_entry_lists_an_array_the_package_did_not_create(tmp_path,
|
|
|
719
733
|
)
|
|
720
734
|
|
|
721
735
|
|
|
736
|
+
def test_a_pickled_relative_path_reopens_after_a_chdir(tmp_path, monkeypatch):
|
|
737
|
+
"""cellmap-analyze's dask workers change directory before they unpickle."""
|
|
738
|
+
monkeypatch.chdir(tmp_path)
|
|
739
|
+
wa.create("b.zarr/seg/s0", (8, 8, 8), "uint8", (4, 4, 4), (2, 2, 2), chunks=(4, 4, 4))
|
|
740
|
+
vol = wa.open("b.zarr/seg/s0", mode="r+")
|
|
741
|
+
vol.write_index((0, 0, 0), (4, 4, 4), np.full((4, 4, 4), 9, np.uint8))
|
|
742
|
+
blob = pickle.dumps(vol)
|
|
743
|
+
(tmp_path / "elsewhere").mkdir()
|
|
744
|
+
monkeypatch.chdir(tmp_path / "elsewhere")
|
|
745
|
+
again = pickle.loads(blob)
|
|
746
|
+
assert again.path == str(tmp_path / "b.zarr/seg/s0")
|
|
747
|
+
assert int(again.read_index((0, 0, 0), (1, 1, 1))[0, 0, 0]) == 9
|
|
748
|
+
assert vol.reopen().level_path == again.level_path
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def test_a_scalar_fills_the_box(tmp_path):
|
|
752
|
+
vol = wa.create(
|
|
753
|
+
str(tmp_path / "vol.zarr/s0"), (8, 8, 8), "uint8", (1, 1, 1), (0.5,) * 3, chunks=(4, 4, 4)
|
|
754
|
+
)
|
|
755
|
+
vol.write_index((0, 0, 0), (4, 8, 8), 7)
|
|
756
|
+
vol.write(((4, 0, 0), (8, 4, 4)), np.uint8(3))
|
|
757
|
+
vol.write_chunk((1, 1, 1), 5)
|
|
758
|
+
got = vol.read_index((0, 0, 0), (8, 8, 8))
|
|
759
|
+
assert np.all(got[:4] == 7) and np.all(got[4:, :4, :4] == 3) and np.all(got[4:, 4:, 4:] == 5)
|
|
760
|
+
for bad in (300, -1, 0.5, float("nan"), True + 255):
|
|
761
|
+
with pytest.raises(ValueError, match="does not fit uint8"):
|
|
762
|
+
vol.write_index((0, 0, 0), (4, 4, 4), bad)
|
|
763
|
+
vol.write_index((0, 0, 0), (4, 4, 4), 0) # the fill over a whole unit leaves it absent
|
|
764
|
+
assert (0, 0, 0) not in vol.list_chunks() and (0, 1, 0) in vol.list_chunks()
|
|
765
|
+
f = wa.create(
|
|
766
|
+
str(tmp_path / "f.zarr/s0"), (4,), "float32", (1,), (0.5,), axes=("x",), chunks=(4,)
|
|
767
|
+
)
|
|
768
|
+
f.write_index((0,), (4,), 0.1) # rounded to the array's precision
|
|
769
|
+
assert np.all(f.read_index((0,), (4,)) == np.float32(0.1))
|
|
770
|
+
with pytest.raises(ValueError, match="does not fit float32"):
|
|
771
|
+
f.write_index((0,), (4,), 1e300)
|
|
772
|
+
|
|
773
|
+
|
|
722
774
|
def test_a_pickled_handle_keeps_its_context(tmp_path):
|
|
723
775
|
path = str(tmp_path / "vol.zarr/s0")
|
|
724
776
|
wa.create(path, (8, 8, 8), "uint16", (4, 4, 4), (2, 2, 2), chunks=(4, 4, 4))
|
|
@@ -1247,7 +1247,7 @@ wheels = [
|
|
|
1247
1247
|
|
|
1248
1248
|
[[package]]
|
|
1249
1249
|
name = "worldarray"
|
|
1250
|
-
version = "0.
|
|
1250
|
+
version = "0.4.0"
|
|
1251
1251
|
source = { editable = "." }
|
|
1252
1252
|
dependencies = [
|
|
1253
1253
|
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|