worldarray 0.2.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {worldarray-0.2.0 → worldarray-0.3.1}/CHANGELOG.md +38 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/PKG-INFO +26 -4
- {worldarray-0.2.0 → worldarray-0.3.1}/README.md +25 -3
- {worldarray-0.2.0 → worldarray-0.3.1}/docs/spec.md +39 -22
- {worldarray-0.2.0 → worldarray-0.3.1}/pyproject.toml +1 -1
- {worldarray-0.2.0 → worldarray-0.3.1}/src/worldarray/__init__.py +12 -4
- {worldarray-0.2.0 → worldarray-0.3.1}/src/worldarray/array.py +66 -7
- worldarray-0.3.1/src/worldarray/grid.py +492 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/src/worldarray/resample.py +12 -4
- {worldarray-0.2.0 → worldarray-0.3.1}/src/worldarray/store.py +5 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/src/worldarray/units.py +23 -0
- worldarray-0.3.1/tests/test_grid.py +386 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/tests/test_read.py +22 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/tests/test_write.py +1 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/uv.lock +1 -1
- {worldarray-0.2.0 → worldarray-0.3.1}/.github/workflows/ci.yml +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/.github/workflows/docs.yml +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/.github/workflows/publish.yml +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/.gitignore +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/LICENSE +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/docs/changelog.md +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/docs/index.md +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/mkdocs.yml +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/src/worldarray/context.py +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/src/worldarray/geometry.py +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/src/worldarray/levels.py +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/src/worldarray/placement.py +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/src/worldarray/retry.py +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/src/worldarray/write.py +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/tests/conftest.py +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/tests/fixtures/metadata.json +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/tests/fixtures/write.json +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/tests/test_access.py +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/tests/test_geometry.py +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/tests/test_metadata.py +0 -0
- {worldarray-0.2.0 → worldarray-0.3.1}/tests/test_resample.py +0 -0
|
@@ -1,5 +1,43 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.1 (2026-10-22)
|
|
4
|
+
|
|
5
|
+
- `source_voxel_size` reports the stored level's voxel size under `on_mismatch="relabel"`,
|
|
6
|
+
as the spec says; it reported the relabelled size (cellmap-flow's report). Reads,
|
|
7
|
+
`voxel_size`, `translation` and `corner` are unchanged.
|
|
8
|
+
- `box` of an array opened with a resample target is the extent of the target voxels
|
|
9
|
+
`shape` counts, as the spec's `[corner, corner + shape * voxel_size)` says and as
|
|
10
|
+
`Grid.from_meta(meta, voxel_size=t).box` already was; it was the stored data's extent, so
|
|
11
|
+
`read(box)` could return fewer voxels than `shape` (cellmap-flow's 7 voxels of 8 nm at
|
|
12
|
+
24 nm: box ended at 56 nm, now 72). No voxel moves.
|
|
13
|
+
- The unit table knows kilometres (`kilometer`, `kilometre`, `km`, 1e3 m), so they
|
|
14
|
+
convert, abbreviate to `km` and are written as OME's `kilometer` (chunkmirage's ask).
|
|
15
|
+
- README: the placement rule gains the corner formula, a worked example, why the centre
|
|
16
|
+
is the model, and how the read rule follows from it.
|
|
17
|
+
|
|
18
|
+
## 0.3.0 (2026-10-22)
|
|
19
|
+
|
|
20
|
+
Grids and blocks, built to spec draft 3 (Behaviours 17 to 19). Nothing on the read or
|
|
21
|
+
write side changes.
|
|
22
|
+
|
|
23
|
+
- `blocks(shape, block_shape, context=0, box=None, clip=True)`: the array (or a voxel
|
|
24
|
+
box, which may reach past the array) cut into write boxes that tile it with no overlap
|
|
25
|
+
and no gap, each with a read box grown by `context` (one int, one per axis, or
|
|
26
|
+
`(before, after)` pairs) and clipped to the array when `clip`; C order, `len`,
|
|
27
|
+
`blocks[id]` in O(1), `index_of(begin)`; a `Block` carries `index`, `id`, `begin`,
|
|
28
|
+
`end`, `read_begin`, `read_end`, `at_start` and `at_end` per axis and `inner`, the write
|
|
29
|
+
box's slices inside the read box. Plain picklable records; nothing is scheduled.
|
|
30
|
+
`vol.blocks()` defaults the block to `write_shape`, so default blocks are aligned writes.
|
|
31
|
+
`vol.write_chunk(index, data)` writes one whole unit.
|
|
32
|
+
- `Grid(shape, voxel_size, translation, axes, units, chunks, shards)`: a `WorldArray`
|
|
33
|
+
without a store, with `box`, `corner`, `index_box`, `chunks_covering`, `write_shape`,
|
|
34
|
+
`blocks` and `meta(dtype)`; `Grid.from_meta(meta, voxel_size=)` is an existing array's
|
|
35
|
+
grid or the resample target grid over it. `create(path, grid=g, dtype=)` creates from it.
|
|
36
|
+
- `level_factors(voxel_size, ratio=1.5, ...)`: the anisotropy-aware pyramid schedule
|
|
37
|
+
mesh-n-bone and cellmap-analyze share, per level.
|
|
38
|
+
- `units.abbreviation`, and `store.join`, `store.parent`, `store.basename` as public
|
|
39
|
+
URL-aware path helpers.
|
|
40
|
+
|
|
3
41
|
## 0.2.0 (2026-10-22)
|
|
4
42
|
|
|
5
43
|
Metadata and placed creation, built to spec draft 3 (Behaviours 10 to 16). Nothing on the
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: worldarray
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Read zarr, N5 and Neuroglancer precomputed volumes in world coordinates through tensorstore: correct voxel placement, pyramid levels, padded region reads, resampling. numpy + tensorstore only.
|
|
5
5
|
Project-URL: Homepage, https://janeliascicomp.github.io/worldarray/
|
|
6
6
|
Project-URL: Documentation, https://janeliascicomp.github.io/worldarray/
|
|
@@ -156,13 +156,27 @@ s1 = wa.open_pyramid("out.zarr", mode="r+").add_level("s1", factors=2) # place
|
|
|
156
156
|
and only two answers to "it exists": raise, or adopt an identical array after checking;
|
|
157
157
|
whole-unit writes from many processes with no coordination; `add_level` as metadata plus
|
|
158
158
|
an empty array; chunk keys for every format. Nothing is ever deleted or overwritten.
|
|
159
|
+
- **Plans outputs and blocks** (0.3): a store-less `Grid` with the same geometry as an
|
|
160
|
+
array, so an output is planned, cut into blocks and only then created; `blocks` tiling an
|
|
161
|
+
array or a box into write boxes with context-grown read boxes, in C order, as plain
|
|
162
|
+
picklable records a cluster job takes by id; the anisotropy-aware pyramid schedule.
|
|
163
|
+
Nothing is scheduled or run.
|
|
159
164
|
|
|
160
165
|
## The placement rule
|
|
161
166
|
|
|
162
167
|
A translation is where the centre of voxel 0 is. Index `i` sits at
|
|
163
|
-
`translation + i * voxel_size
|
|
164
|
-
|
|
165
|
-
|
|
168
|
+
`translation + i * voxel_size` and covers half a voxel either side, so the array's corner
|
|
169
|
+
is `corner = translation - voxel_size / 2`. With 8 nm voxels and the corner at 0, the
|
|
170
|
+
translation is 4: voxel 0 covers 0 to 8 nm around its centre at 4, voxel 1 covers 8 to
|
|
171
|
+
16 nm around 12. That is how OME-NGFF defines it and how Neuroglancer draws it.
|
|
172
|
+
|
|
173
|
+
The centre is the model because a voxel is a sample taken at a point. Everything that
|
|
174
|
+
compares or resamples grids (picking a level, reading one volume onto another's grid,
|
|
175
|
+
interpolating) works on those points, and the box a voxel covers follows from its point.
|
|
176
|
+
The read rule follows too: a voxel is read iff its centre lies in the box, so boxes that
|
|
177
|
+
adjoin never share a voxel or drop one.
|
|
178
|
+
|
|
179
|
+
Dialects that store the corner are moved half a voxel on reading:
|
|
166
180
|
|
|
167
181
|
| Stored | Means | Read as `translation` |
|
|
168
182
|
|---|---|---|
|
|
@@ -202,6 +216,14 @@ wa.ome_multiscales(axes, units, levels, version="0.5", type=None) ; wa.level_pla
|
|
|
202
216
|
wa.level_shape(shape, factors, origin=0) ; wa.chunk_key(index, format, ...) ; wa.parse_chunk_key(key, format, ...)
|
|
203
217
|
wa.ensure_group_entry(group_path, name, voxel_size, translation, axes, units="nanometer") # list an array you made yourself
|
|
204
218
|
wa.units.ome_name("nm") -> "nanometer" ; wa.write.codecs_of(path) -> (codecs, shards)
|
|
219
|
+
|
|
220
|
+
# grids and blocks (0.3)
|
|
221
|
+
g = wa.Grid(shape, voxel_size, translation, axes=None, units="nanometer", chunks=None, shards=None) # or Grid.from_meta(meta, voxel_size=)
|
|
222
|
+
wa.create(path, grid=g, dtype="uint64")
|
|
223
|
+
for b in wa.blocks(shape, block_shape, context=0, box=None, clip=True): # also vol.blocks() and g.blocks()
|
|
224
|
+
b.begin, b.end, b.read_begin, b.read_end, b.inner, b.at_end, b.id # plain records; blocks[id] is O(1)
|
|
225
|
+
wa.level_factors(voxel_size, ratio=1.5, max_levels=None, stop_shape=None, shape=None)
|
|
226
|
+
wa.units.abbreviation("nanometer") ; wa.store.join(path, "s0") ; wa.store.parent(path) ; wa.store.basename(path)
|
|
205
227
|
```
|
|
206
228
|
|
|
207
229
|
`WorldArray`: `ts`, `meta`, `grid`, `shape`, `dtype`, `axes`, `voxel_size`, `translation`,
|
|
@@ -97,13 +97,27 @@ s1 = wa.open_pyramid("out.zarr", mode="r+").add_level("s1", factors=2) # place
|
|
|
97
97
|
and only two answers to "it exists": raise, or adopt an identical array after checking;
|
|
98
98
|
whole-unit writes from many processes with no coordination; `add_level` as metadata plus
|
|
99
99
|
an empty array; chunk keys for every format. Nothing is ever deleted or overwritten.
|
|
100
|
+
- **Plans outputs and blocks** (0.3): a store-less `Grid` with the same geometry as an
|
|
101
|
+
array, so an output is planned, cut into blocks and only then created; `blocks` tiling an
|
|
102
|
+
array or a box into write boxes with context-grown read boxes, in C order, as plain
|
|
103
|
+
picklable records a cluster job takes by id; the anisotropy-aware pyramid schedule.
|
|
104
|
+
Nothing is scheduled or run.
|
|
100
105
|
|
|
101
106
|
## The placement rule
|
|
102
107
|
|
|
103
108
|
A translation is where the centre of voxel 0 is. Index `i` sits at
|
|
104
|
-
`translation + i * voxel_size
|
|
105
|
-
|
|
106
|
-
|
|
109
|
+
`translation + i * voxel_size` and covers half a voxel either side, so the array's corner
|
|
110
|
+
is `corner = translation - voxel_size / 2`. With 8 nm voxels and the corner at 0, the
|
|
111
|
+
translation is 4: voxel 0 covers 0 to 8 nm around its centre at 4, voxel 1 covers 8 to
|
|
112
|
+
16 nm around 12. That is how OME-NGFF defines it and how Neuroglancer draws it.
|
|
113
|
+
|
|
114
|
+
The centre is the model because a voxel is a sample taken at a point. Everything that
|
|
115
|
+
compares or resamples grids (picking a level, reading one volume onto another's grid,
|
|
116
|
+
interpolating) works on those points, and the box a voxel covers follows from its point.
|
|
117
|
+
The read rule follows too: a voxel is read iff its centre lies in the box, so boxes that
|
|
118
|
+
adjoin never share a voxel or drop one.
|
|
119
|
+
|
|
120
|
+
Dialects that store the corner are moved half a voxel on reading:
|
|
107
121
|
|
|
108
122
|
| Stored | Means | Read as `translation` |
|
|
109
123
|
|---|---|---|
|
|
@@ -143,6 +157,14 @@ wa.ome_multiscales(axes, units, levels, version="0.5", type=None) ; wa.level_pla
|
|
|
143
157
|
wa.level_shape(shape, factors, origin=0) ; wa.chunk_key(index, format, ...) ; wa.parse_chunk_key(key, format, ...)
|
|
144
158
|
wa.ensure_group_entry(group_path, name, voxel_size, translation, axes, units="nanometer") # list an array you made yourself
|
|
145
159
|
wa.units.ome_name("nm") -> "nanometer" ; wa.write.codecs_of(path) -> (codecs, shards)
|
|
160
|
+
|
|
161
|
+
# grids and blocks (0.3)
|
|
162
|
+
g = wa.Grid(shape, voxel_size, translation, axes=None, units="nanometer", chunks=None, shards=None) # or Grid.from_meta(meta, voxel_size=)
|
|
163
|
+
wa.create(path, grid=g, dtype="uint64")
|
|
164
|
+
for b in wa.blocks(shape, block_shape, context=0, box=None, clip=True): # also vol.blocks() and g.blocks()
|
|
165
|
+
b.begin, b.end, b.read_begin, b.read_end, b.inner, b.at_end, b.id # plain records; blocks[id] is O(1)
|
|
166
|
+
wa.level_factors(voxel_size, ratio=1.5, max_levels=None, stop_shape=None, shape=None)
|
|
167
|
+
wa.units.abbreviation("nanometer") ; wa.store.join(path, "s0") ; wa.store.parent(path) ; wa.store.basename(path)
|
|
146
168
|
```
|
|
147
169
|
|
|
148
170
|
`WorldArray`: `ts`, `meta`, `grid`, `shape`, `dtype`, `axes`, `voxel_size`, `translation`,
|
|
@@ -149,7 +149,8 @@ vol.meta # ArrayMeta in the store's own units
|
|
|
149
149
|
vol.grid # ArrayMeta in the requested units, every axis
|
|
150
150
|
vol.shape, vol.dtype, vol.axes, vol.spatial, vol.level_name, vol.levels, vol.remote
|
|
151
151
|
# shape is the stored shape, or with a resample target the target voxels covering the data
|
|
152
|
-
vol.source_voxel_size, vol.resampled # the stored level's voxel size
|
|
152
|
+
vol.source_voxel_size, vol.resampled # the stored level's voxel size, also under resample and relabel; whether reads
|
|
153
|
+
# resample to a target
|
|
153
154
|
vol.list_chunks() # the chunk indices that exist in storage, by listing keys (zarr v2/v3, N5, unsharded precomputed)
|
|
154
155
|
vol.path # what the caller passed (a group, say); vol.level_path is the array actually opened
|
|
155
156
|
vol.voxel_size # tuple[float] in `units`, spatial axes, the array's order (zyx for a volume)
|
|
@@ -212,7 +213,8 @@ window and a spatial box at once); anything with `.begin` and `.end`, or `.offse
|
|
|
212
213
|
`shift`, `snap(voxel_size, translation, mode="grow" | "shrink" | "closest")` to a grid's
|
|
213
214
|
voxel corners, `to_index` (the centre rule) and `Box.from_index`; `vol.box` is the array's
|
|
214
215
|
own extent on the spatial axes, `[corner, corner + shape * voxel_size)`, what funlib calls
|
|
215
|
-
the array's roi
|
|
216
|
+
the array's roi; with a resample target, the target voxels `shape` counts, so `read(vol.box)`
|
|
217
|
+
returns `shape` and equals `Grid.from_meta(meta, voxel_size=t).box`. So `vol.read(vol.box.intersect(roi).snap(vol.voxel_size, vol.translation))`
|
|
216
218
|
is the migration of a clipped, snapped Roi read, without funlib.
|
|
217
219
|
|
|
218
220
|
There is no `offset` attribute anywhere in this API, because that word means the corner in
|
|
@@ -393,7 +395,7 @@ this document and that one differ, fix one of them.
|
|
|
393
395
|
3. **Units are kept as stored and converted only on the axes a caller asks to convert**
|
|
394
396
|
(`ArrayMeta.in_units(space=, time=)`, `in_nm()`, the `units` argument), through one table:
|
|
395
397
|
for length, American and British spellings, `nm`, `um`, `µm` (U+00B5), `μm` (U+03BC),
|
|
396
|
-
`mm`, `cm`, `m`, `angstrom`/`Å`, `pm`; for time, `s`/`second`, `ms`, `us`/`µs`, `ns`,
|
|
398
|
+
`mm`, `cm`, `m`, `km`, `angstrom`/`Å`, `pm`; for time, `s`/`second`, `ms`, `us`/`µs`, `ns`,
|
|
397
399
|
`minute`, `hour`, `day`. Any unit string on any axis is accepted and never rejected until
|
|
398
400
|
a conversion is asked for (`"s"`, `"m"`, `"degrees_north"`, `""`). A spatial axis with no
|
|
399
401
|
unit is taken as nanometres, with a **warning once per array** (every repo assumed this
|
|
@@ -549,8 +551,8 @@ this document and that one differ, fix one of them.
|
|
|
549
551
|
reads that level voxel for voxel and reports `voxel_size` as the requested size with
|
|
550
552
|
**voxel `i` staying voxel `i`**: every world quantity, the translation and the corner
|
|
551
553
|
included, scales by the ratio of the sizes (cellmap-flow's rule, which chunkmirage's
|
|
552
|
-
server relies on), so a box of `m` requested voxels returns `m` stored voxels of the
|
|
553
|
-
level; `"keep"` reads the chosen level at its own voxel size (a level pick, nothing
|
|
554
|
+
server relies on), so a box of `m` requested voxels returns `m` stored voxels of the level,
|
|
555
|
+
and `source_voxel_size` still reports the level's own size; `"keep"` reads the chosen level at its own voxel size (a level pick, nothing
|
|
554
556
|
more). `relabel` and `nearest` are different things and both exist: one is a grid
|
|
555
557
|
policy, the other a resampling kernel. `level` also takes a position, `0` or `"first"`
|
|
556
558
|
for the first listed level. The pyramid an array belongs to is found through
|
|
@@ -765,24 +767,38 @@ Behaviours 10 to 16 are 0.2; 17 to 19 are 0.3.
|
|
|
765
767
|
17. **Blocks tile, context grows, nothing runs.** `blocks(shape, block_shape, context, box,
|
|
766
768
|
clip)` yields, in C order, blocks whose write boxes `[begin, end)` partition `[0,
|
|
767
769
|
shape)`, or `box` (a voxel box, the grid anchored at its begin) when given, with no
|
|
768
|
-
overlap and no gap, the last block on an axis clipped
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
770
|
+
overlap and no gap, the last block on an axis clipped to the box. `box` may reach past
|
|
771
|
+
the array on any side (a ROI rounded outward to a factor multiple, mesh-n-bone's halo
|
|
772
|
+
mode): the write boxes tile it exactly, outside the array too. A block's read box is
|
|
773
|
+
its write box grown by `context` on each side, where `context` is one int, one per
|
|
774
|
+
axis, or a `(before, after)` pair per axis, clipped to the **array** (never to `box`,
|
|
775
|
+
never a write box) when `clip=True` (the default; cellmap-analyze's `create_block` edge
|
|
776
|
+
rule) and left unclipped when `clip=False` for a padded `read_index` (its
|
|
777
|
+
`neg_with_edge_pos` mode, which reads past the array for fill). A block also carries
|
|
778
|
+
`at_start` and `at_end` per axis, whether it is the first or last block on that axis,
|
|
779
|
+
so a caller adds context at the array's far faces itself, and `inner`, the slices of
|
|
780
|
+
the write box inside the read box, correct also when `clip=False` lets `read_begin`
|
|
781
|
+
go below 0. `len(blocks)`, `blocks[b.id]` (O(1)), `blocks.index_of(b.begin)` and
|
|
773
782
|
`b.index` agree. `vol.blocks()` defaults `block_shape` to `write_shape`, so default
|
|
774
783
|
blocks are aligned writes. Nothing is scheduled: the iterable and its blocks are plain,
|
|
775
|
-
picklable records
|
|
776
|
-
|
|
784
|
+
picklable records, and the iterable pickles without its blocks, so a worker takes an
|
|
785
|
+
id. One grid per call: a consumer that writes several levels from one block derives
|
|
786
|
+
the per-level boxes itself (mesh-n-bone's direct mode, two lines). `write_chunk(index,
|
|
787
|
+
data)` writes the unit at `index` whole; `data.shape` must equal that unit's clipped
|
|
788
|
+
shape.
|
|
777
789
|
|
|
778
790
|
18. **A `Grid` is a `WorldArray` without a store.** `Grid(shape, voxel_size, translation,
|
|
779
791
|
axes, units, chunks, shards)` answers `box`, `corner`, `index_box`, `chunks_covering`,
|
|
780
792
|
`write_shape` and `blocks` exactly as a `WorldArray` of those facts does, and
|
|
781
|
-
`g.meta(dtype)`
|
|
793
|
+
`g.meta(dtype)` agrees with `wa.create(path, grid=g, dtype=dtype).meta` in every
|
|
794
|
+
field a grid has: shape, dtype, chunk and shard shape, axes, units, voxel size and
|
|
795
|
+
translation (a grid has no path, format, documents or fill value), so a pipeline plans
|
|
782
796
|
its output before anything exists. `Grid.from_meta(meta, voxel_size=t)` is Behaviour
|
|
783
797
|
4's target grid over that array: anchored at its corner with `ceil(n * s / t)` voxels,
|
|
784
|
-
so a `read(voxel_size=t)` of the source fills it exactly
|
|
785
|
-
|
|
798
|
+
so a `read(voxel_size=t)` of the source fills it exactly, and its `box` is the target
|
|
799
|
+
grid's extent, which may reach past the source's own `box` by less than one target
|
|
800
|
+
voxel; this is the output grid cellmap-analyze's commands compute today in the integer
|
|
801
|
+
frame.
|
|
786
802
|
|
|
787
803
|
19. **Schedules and helpers.** `level_factors(voxel_size, ratio, max_levels, stop_shape,
|
|
788
804
|
shape)` returns per level the per-axis factors, 2 on every axis whose current voxel size
|
|
@@ -1017,11 +1033,12 @@ over funlib geometry, by this spec's own assignment.
|
|
|
1017
1033
|
attributes? If so, should the package offer `funlib_attrs(voxel_size, translation, axes,
|
|
1018
1034
|
units)`, writing `offset` as the corner, funlib's meaning, for `create(attrs=)`, or do
|
|
1019
1035
|
the repos keep writing their own?
|
|
1020
|
-
6. **The block iterator's shape** (Behaviour 17)
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1036
|
+
6. **The block iterator's shape** (Behaviour 17). Resolved 2026-10-22: both sessions
|
|
1037
|
+
confirmed it. mesh-n-bone asked that `box` may reach past the array and that `clip`
|
|
1038
|
+
clips read boxes only, and derives its per-level boxes from one grid itself;
|
|
1039
|
+
cellmap-analyze asked for `at_start`/`at_end` per axis and `inner`, and confirmed that
|
|
1040
|
+
`(before, after)` pairs cover its context modes and that `clip=False` with a padded
|
|
1041
|
+
`read_index` is its edge-reading mode. All are in Behaviour 17.
|
|
1025
1042
|
7. **Sidecar attributes.** Resolved 2026-10-22: registration-station keeps `fs` and
|
|
1026
1043
|
`fix_origin_um` on the group through `group_attrs` and the part attributes (`y0`, `y1`,
|
|
1027
1044
|
`y0_core`, `y1_core`, `feather_vox`) on the array through `attrs`, all known at creation;
|
|
@@ -1056,8 +1073,8 @@ over funlib geometry, by this spec's own assignment.
|
|
|
1056
1073
|
13. **World boxes on blocks.** cellmap-flow reads its input at one voxel size by a world
|
|
1057
1074
|
box grown by a context in nanometres and writes at another; `blocks` (0.3) gives index
|
|
1058
1075
|
boxes on one grid. `Grid.blocks` could carry world boxes too (`b.box`, `b.read_box` as
|
|
1059
|
-
`Box`, a context in world units). Two lines on cellmap-flow's side today
|
|
1060
|
-
|
|
1076
|
+
`Box`, a context in world units). Two lines on cellmap-flow's side today. Not planned
|
|
1077
|
+
(2026-10-22): daisy does cellmap-flow's tiling and stays; reopen if cellmap-flow drops it.
|
|
1061
1078
|
|
|
1062
1079
|
## 8. Constraints for whoever builds on this
|
|
1063
1080
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "worldarray"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.1"
|
|
4
4
|
description = "Read zarr, N5 and Neuroglancer precomputed volumes in world coordinates through tensorstore: correct voxel placement, pyramid levels, padded region reads, resampling. numpy + tensorstore only."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = { file = "LICENSE" }
|
|
@@ -2,16 +2,19 @@
|
|
|
2
2
|
|
|
3
3
|
Open zarr v2, zarr v3, N5 and Neuroglancer precomputed arrays and pyramids by path, know
|
|
4
4
|
where each sits (voxel size, the centre of voxel 0, units, axes), and read voxel boxes or
|
|
5
|
-
world boxes as numpy arrays, padded where the box leaves the data
|
|
6
|
-
|
|
5
|
+
world boxes as numpy arrays, padded where the box leaves the data; create placed OME-NGFF
|
|
6
|
+
levels and write them from any number of jobs; plan an output grid and cut it into blocks
|
|
7
|
+
before anything exists. ``docs/spec.md`` is the contract; the README is its first five
|
|
8
|
+
sections.
|
|
7
9
|
"""
|
|
8
10
|
|
|
9
11
|
from __future__ import annotations
|
|
10
12
|
|
|
11
|
-
from . import store, units, write
|
|
13
|
+
from . import grid, store, units, write
|
|
12
14
|
from .array import Pyramid, WorldArray, create, metadata, open, open_pyramid, open_tensorstore
|
|
13
15
|
from .context import capped_context, capped_context_spec, shared_context
|
|
14
16
|
from .geometry import Box, index_range
|
|
17
|
+
from .grid import Block, Grid, blocks, level_factors
|
|
15
18
|
from .levels import select_level
|
|
16
19
|
from .placement import ArrayMeta, corner_to_centre, default_axes
|
|
17
20
|
from .retry import Retry
|
|
@@ -25,19 +28,24 @@ from .write import (
|
|
|
25
28
|
parse_chunk_key,
|
|
26
29
|
)
|
|
27
30
|
|
|
28
|
-
__version__ = "0.
|
|
31
|
+
__version__ = "0.3.1"
|
|
29
32
|
|
|
30
33
|
__all__ = [
|
|
31
34
|
"ArrayMeta",
|
|
35
|
+
"Block",
|
|
32
36
|
"Box",
|
|
37
|
+
"Grid",
|
|
33
38
|
"Node",
|
|
39
|
+
"blocks",
|
|
34
40
|
"chunk_key",
|
|
35
41
|
"corner_to_centre",
|
|
36
42
|
"create",
|
|
37
43
|
"default_axes",
|
|
38
44
|
"detect",
|
|
39
45
|
"ensure_group_entry",
|
|
46
|
+
"grid",
|
|
40
47
|
"index_range",
|
|
48
|
+
"level_factors",
|
|
41
49
|
"level_placement",
|
|
42
50
|
"level_shape",
|
|
43
51
|
"list_chunk_keys",
|
|
@@ -139,6 +139,8 @@ class WorldArray:
|
|
|
139
139
|
self._meta = self._rescaled_to_base(base_voxel_size, base_units)
|
|
140
140
|
space, time = _unit_targets(units)
|
|
141
141
|
self._grid = self._meta.in_units(space=space, time=time, strict=strict_units)
|
|
142
|
+
# the stored level's spatial voxel size, before any relabel renames the grid
|
|
143
|
+
self._stored_voxel_size = tuple(self._grid.voxel_size[i] for i in self._grid.spatial)
|
|
142
144
|
|
|
143
145
|
self._target: tuple[float, ...] | None = None
|
|
144
146
|
if voxel_size is not None:
|
|
@@ -299,8 +301,9 @@ class WorldArray:
|
|
|
299
301
|
|
|
300
302
|
@property
|
|
301
303
|
def source_voxel_size(self) -> tuple[float, ...]:
|
|
302
|
-
"""The stored level's voxel size on the spatial axes, in ``units
|
|
303
|
-
|
|
304
|
+
"""The stored level's voxel size on the spatial axes, in ``units``: what the level
|
|
305
|
+
is, whether reads resample it or ``on_mismatch="relabel"`` renames its grid."""
|
|
306
|
+
return self._stored_voxel_size
|
|
304
307
|
|
|
305
308
|
@property
|
|
306
309
|
def resampled(self) -> bool:
|
|
@@ -337,8 +340,22 @@ class WorldArray:
|
|
|
337
340
|
def box(self) -> Box:
|
|
338
341
|
"""The array's extent on the spatial axes as a world box, ``[corner, corner + shape *
|
|
339
342
|
voxel_size)``: what funlib calls the array's roi. ``Box.from_index`` gives the same
|
|
340
|
-
over every axis.
|
|
343
|
+
over every axis. With a resample target it is the extent of the target voxels that
|
|
344
|
+
``shape`` counts, which may reach past the stored data by less than one target voxel,
|
|
345
|
+
so ``read(box)`` returns ``shape``."""
|
|
341
346
|
sp = self._grid.spatial
|
|
347
|
+
if self._target is not None:
|
|
348
|
+
begin, end = [], []
|
|
349
|
+
for k, axis in enumerate(sp):
|
|
350
|
+
first, last = _resample.target_range(
|
|
351
|
+
self._grid.shape[axis],
|
|
352
|
+
self._grid.voxel_size[axis],
|
|
353
|
+
self._target[k],
|
|
354
|
+
self._origins()[k],
|
|
355
|
+
)
|
|
356
|
+
begin.append(self.anchor[k] + first * self._target[k])
|
|
357
|
+
end.append(self.anchor[k] + last * self._target[k])
|
|
358
|
+
return Box(tuple(begin), tuple(end))
|
|
342
359
|
return Box.from_index(
|
|
343
360
|
(0,) * len(sp),
|
|
344
361
|
tuple(self._grid.shape[i] for i in sp),
|
|
@@ -551,6 +568,34 @@ class WorldArray:
|
|
|
551
568
|
)
|
|
552
569
|
self.write_index(begin, end, data, partial=partial)
|
|
553
570
|
|
|
571
|
+
def blocks(self, block_shape=None, context=0, box=None, clip: bool = True):
|
|
572
|
+
"""The array cut into blocks (Behaviour 17): write boxes tiling the stored grid (or a
|
|
573
|
+
voxel ``box``) with ``block_shape`` (default ``write_shape``, so the default blocks are
|
|
574
|
+
aligned writes), read boxes grown by ``context`` and clipped to the array when
|
|
575
|
+
``clip``. Plain records, nothing scheduled."""
|
|
576
|
+
from .grid import blocks as _blocks
|
|
577
|
+
|
|
578
|
+
return _blocks(
|
|
579
|
+
self._grid.shape,
|
|
580
|
+
self.write_shape if block_shape is None else block_shape,
|
|
581
|
+
context=context,
|
|
582
|
+
box=box,
|
|
583
|
+
clip=clip,
|
|
584
|
+
)
|
|
585
|
+
|
|
586
|
+
def write_chunk(self, index: Sequence[int], data: np.ndarray) -> None:
|
|
587
|
+
"""Write the whole write unit at ``index`` on the storage grid (the shard of a sharded
|
|
588
|
+
array); ``data`` has that unit's shape, clipped at the array's end."""
|
|
589
|
+
unit, shape = self.write_shape, self._grid.shape
|
|
590
|
+
index = tuple(int(i) for i in index)
|
|
591
|
+
if len(index) != len(shape):
|
|
592
|
+
raise ValueError(f"{self.path}: index needs one entry per axis of {shape}")
|
|
593
|
+
begin = tuple(i * u for i, u in zip(index, unit))
|
|
594
|
+
if any(i < 0 or b >= n for i, b, n in zip(index, begin, shape)):
|
|
595
|
+
raise ValueError(f"{self.path}: no write unit at {index}")
|
|
596
|
+
end = tuple(min(b + u, n) for b, u, n in zip(begin, unit, shape))
|
|
597
|
+
self.write_index(begin, end, data)
|
|
598
|
+
|
|
554
599
|
def read_index(
|
|
555
600
|
self,
|
|
556
601
|
begin: Sequence[int],
|
|
@@ -1051,11 +1096,12 @@ class Pyramid:
|
|
|
1051
1096
|
|
|
1052
1097
|
def create(
|
|
1053
1098
|
path: str,
|
|
1054
|
-
shape: Sequence[int],
|
|
1055
|
-
dtype,
|
|
1056
|
-
voxel_size,
|
|
1057
|
-
translation,
|
|
1099
|
+
shape: Sequence[int] | None = None,
|
|
1100
|
+
dtype=None,
|
|
1101
|
+
voxel_size=None,
|
|
1102
|
+
translation=None,
|
|
1058
1103
|
*,
|
|
1104
|
+
grid=None,
|
|
1059
1105
|
axes: Sequence[str] | None = None,
|
|
1060
1106
|
units: Units = "nanometer",
|
|
1061
1107
|
chunks: Sequence[int] | None = None,
|
|
@@ -1079,6 +1125,19 @@ def create(
|
|
|
1079
1125
|
raises ``FileExistsError``; with ``"adopt"`` an identical array (shape, dtype, chunks,
|
|
1080
1126
|
shards, placement) is opened for writing and a different one raises ``ValueError``
|
|
1081
1127
|
naming every difference. Returns a writable ``WorldArray``."""
|
|
1128
|
+
if grid is not None:
|
|
1129
|
+
# a Grid planned the output (Behaviour 18): its facts are the array's, chunks and
|
|
1130
|
+
# shards overridable; the placement arguments must not also be given
|
|
1131
|
+
if any(v is not None for v in (shape, voxel_size, translation, axes)):
|
|
1132
|
+
raise ValueError("pass grid= or shape, voxel_size, translation and axes, not both")
|
|
1133
|
+
shape, voxel_size, translation = grid.shape, grid.voxel_size, grid.translation
|
|
1134
|
+
axes, units = grid.axes, grid.units
|
|
1135
|
+
chunks = grid.chunks if chunks is None else chunks
|
|
1136
|
+
shards = grid.shards if shards is None else shards
|
|
1137
|
+
if shape is None or dtype is None or voxel_size is None or translation is None:
|
|
1138
|
+
raise ValueError(
|
|
1139
|
+
"create needs shape, dtype, voxel_size and translation, or grid= and dtype"
|
|
1140
|
+
)
|
|
1082
1141
|
if chunks is None:
|
|
1083
1142
|
raise ValueError("chunks is required: the package never picks a chunk shape")
|
|
1084
1143
|
ndim = len(shape)
|