patchworks 2.5.2__tar.gz → 2.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {patchworks-2.5.2 → patchworks-2.6.0}/.gitignore +3 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/PKG-INFO +8 -1
- {patchworks-2.5.2 → patchworks-2.6.0}/README.md +7 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/guide/snakemake.md +26 -1
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_distributed.py +26 -6
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/plugins/cellpose.py +41 -22
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/plugins/napari.py +49 -13
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_distributed.py +93 -0
- patchworks-2.6.0/workflow/config/config_cyto.yaml +37 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/scripts/_pw.py +50 -3
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/scripts/segment_tile.py +6 -1
- patchworks-2.5.2/workflow/config/config_cyto.yaml +0 -22
- {patchworks-2.5.2 → patchworks-2.6.0}/.github/workflows/docs.yml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/.github/workflows/lint.yml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/.github/workflows/release.yml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/.markdownlint-cli2.yaml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/LICENSE +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/cliff.toml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/api/chunks.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/api/cluster.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/api/io.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/api/merge_tile_labels.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/api/plugins/cellpose.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/api/plugins/dog.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/api/plugins/napari.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/api/plugins/ome_zarr.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/api/postprocess.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/api/relabel.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/api/tile_process.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/assets/logo.png +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/examples/cellpose_2d.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/examples/cellpose_2d.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/examples/cellpose_3d.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/examples/cellpose_3d.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/examples/custom.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/examples/custom_method.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/examples/dog.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/examples/dog.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/examples/standalone_merge.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/examples/stardist.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/examples/stardist_2d.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/getting_started.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/guide/custom_segmentation.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/guide/gpu_distributed.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/guide/label_relations.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/guide/measurements.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/guide/merging.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/guide/ome_zarr_napari.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/guide/performance.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/guide/pitfalls.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/guide/skip_empty.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/guide/tiling.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/docs/index.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/mkdocs.yml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/pyproject.toml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/__init__.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_chunks.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_cluster.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_core.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_gpu.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_io.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_merge.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_notify.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_occupancy.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_postprocess.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_progress.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_relabel.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/_relations.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/plugins/__init__.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/plugins/dog.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/src/patchworks/plugins/ome_zarr.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_allocation.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_core.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_dog.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_gpu.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_napari.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_notify.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_occupancy.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_ome_zarr.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_postprocess.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_progress.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_relations.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/tests/test_run_multi.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/README.md +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/Snakefile +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/config/common.yaml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/config/config.yaml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/config/config_cilia.yaml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/config/config_nuclei.yaml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/config/multi.yaml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/pixi.toml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/profile/slurm/config.yaml +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/rules/common.smk +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/rules/convert.smk +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/rules/merge.smk +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/rules/segment.smk +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/scripts/build_occupancy.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/scripts/convert.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/scripts/fetch_model.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/scripts/merge.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/scripts/prepare_tiles.py +0 -0
- {patchworks-2.5.2 → patchworks-2.6.0}/workflow/scripts/run_multi.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: patchworks
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.6.0
|
|
4
4
|
Summary: Tiled processing of arbitrarily large images with globally consistent labels
|
|
5
5
|
Project-URL: Homepage, https://github.com/imcf/patchworks
|
|
6
6
|
Project-URL: Issues, https://github.com/imcf/patchworks/issues
|
|
@@ -115,6 +115,13 @@ patchworks splits a large image into tiles, runs **any callable** on each
|
|
|
115
115
|
tile in parallel, and merges the results into a globally consistent label array.
|
|
116
116
|
It handles terabyte-scale images without loading them into memory.
|
|
117
117
|
|
|
118
|
+
> [!NOTE]
|
|
119
|
+
> **On how this was written.** Large parts of patchworks were vibe coded —
|
|
120
|
+
> written with heavy LLM assistance rather than line by line. It is covered by
|
|
121
|
+
> a test suite and has been run on real data, so it is not untested, but the
|
|
122
|
+
> usual caveats apply: read the code before you trust it with anything
|
|
123
|
+
> irreplaceable, and please open an issue if something looks off.
|
|
124
|
+
|
|
118
125
|
---
|
|
119
126
|
|
|
120
127
|
## Installation
|
|
@@ -25,6 +25,13 @@ patchworks splits a large image into tiles, runs **any callable** on each
|
|
|
25
25
|
tile in parallel, and merges the results into a globally consistent label array.
|
|
26
26
|
It handles terabyte-scale images without loading them into memory.
|
|
27
27
|
|
|
28
|
+
> [!NOTE]
|
|
29
|
+
> **On how this was written.** Large parts of patchworks were vibe coded —
|
|
30
|
+
> written with heavy LLM assistance rather than line by line. It is covered by
|
|
31
|
+
> a test suite and has been run on real data, so it is not untested, but the
|
|
32
|
+
> usual caveats apply: read the code before you trust it with anything
|
|
33
|
+
> irreplaceable, and please open an issue if something looks off.
|
|
34
|
+
|
|
28
35
|
---
|
|
29
36
|
|
|
30
37
|
## Installation
|
|
@@ -53,7 +53,8 @@ convert_chunks: null # null → bounded auto chunks; or [c,z,y,x]
|
|
|
53
53
|
shard: false # true → pack chunks into shards (fewer files)
|
|
54
54
|
|
|
55
55
|
# tiling
|
|
56
|
-
channel: 0 # channel to segment (null = keep all)
|
|
56
|
+
channel: 0 # channel to segment, 0-based (null = keep all)
|
|
57
|
+
nuclei_channel: null # optional 2nd channel for Cellpose (see below)
|
|
57
58
|
level: 0 # pyramid level (0 = full resolution)
|
|
58
59
|
tile_shape: "auto" # "auto", or e.g. [16, 1024, 1024] (zyx)
|
|
59
60
|
gpu_memory_gb: null # for "auto" on SLURM: your segment GPU's VRAM
|
|
@@ -299,6 +300,7 @@ cellpose:
|
|
|
299
300
|
# config/config_cyto.yaml — only the differences
|
|
300
301
|
label_name: "cyto_labels"
|
|
301
302
|
channel: 0 # cytoplasm/membrane channel
|
|
303
|
+
nuclei_channel: 1 # optional: nuclear stain, as Cellpose's 2nd input
|
|
302
304
|
overlap: [4, 30, 30]
|
|
303
305
|
method: "cellpose"
|
|
304
306
|
cellpose:
|
|
@@ -307,6 +309,29 @@ cellpose:
|
|
|
307
309
|
do_3D: true
|
|
308
310
|
```
|
|
309
311
|
|
|
312
|
+
### Giving Cellpose a nuclei channel
|
|
313
|
+
|
|
314
|
+
`nuclei_channel` hands Cellpose a second channel — the nuclear stain — which
|
|
315
|
+
usually improves cytoplasm segmentation. Both indices are 0-based, like
|
|
316
|
+
`channel`.
|
|
317
|
+
|
|
318
|
+
Only the `segment` step reads it. The pair is stacked on a leading axis that
|
|
319
|
+
is *carried* into each tile rather than tiled, so the tile geometry, the
|
|
320
|
+
occupancy map and the staged labels are byte-for-byte what a single-channel
|
|
321
|
+
run produces, and `merge` and `label_relations` need no changes. Two things
|
|
322
|
+
follow from that:
|
|
323
|
+
|
|
324
|
+
- A tile holds twice the bytes, so a hand-set `tile_shape` sized to fill a GPU
|
|
325
|
+
may need halving. `tile_shape: "auto"` sizes from the single-channel array
|
|
326
|
+
and does not yet know about the pair.
|
|
327
|
+
- The translation is version-specific. Cellpose 3 gets `channels: [1, 2]`
|
|
328
|
+
(1-based into the channel axis, `0` = grayscale); Cellpose 4 (cpsam) dropped
|
|
329
|
+
`channels` entirely and simply reads both. Either is overridable by setting
|
|
330
|
+
`channels:` or `channel_axis:` in the `cellpose:` block.
|
|
331
|
+
|
|
332
|
+
`nuclei_channel` applies to the SLURM/Snakemake path. The single-process
|
|
333
|
+
`tile_process` API still takes one `channel`.
|
|
334
|
+
|
|
310
335
|
Run them as two independent SLURM submissions — they touch disjoint files, so
|
|
311
336
|
they can run concurrently. Give each its own `--directory`, because
|
|
312
337
|
Snakemake's lock lives in the working directory, not in the config:
|
|
@@ -166,6 +166,7 @@ def stage_tile(
|
|
|
166
166
|
tile_shape: tuple[int, ...],
|
|
167
167
|
overlap: Overlap = 0,
|
|
168
168
|
component: str = "staged",
|
|
169
|
+
channel_axis: int | None = None,
|
|
169
170
|
) -> int:
|
|
170
171
|
"""Run *fn* on a single tile and write it into the shared stage store.
|
|
171
172
|
|
|
@@ -192,6 +193,13 @@ def stage_tile(
|
|
|
192
193
|
:func:`normalize_overlap`).
|
|
193
194
|
component : str, optional
|
|
194
195
|
Array name inside the stage store.
|
|
196
|
+
channel_axis : int or None, optional
|
|
197
|
+
Axis of *image* holding channels, which is **not** tiled: it is read
|
|
198
|
+
whole and handed to *fn* alongside the tile's voxels. ``tile_shape``,
|
|
199
|
+
``overlap`` and the stage store stay purely spatial, so *fn* still
|
|
200
|
+
returns one label per voxel with no channel axis (e.g. Cellpose fed a
|
|
201
|
+
cytoplasm + nuclei pair returns a single label volume). ``None`` (the
|
|
202
|
+
default) means *image* is already single-channel.
|
|
195
203
|
|
|
196
204
|
Returns
|
|
197
205
|
-------
|
|
@@ -202,18 +210,30 @@ def stage_tile(
|
|
|
202
210
|
by a cumulative sum, instead of rewriting the whole store to make the
|
|
203
211
|
ids unique.
|
|
204
212
|
"""
|
|
205
|
-
shape = image.shape
|
|
206
|
-
|
|
213
|
+
shape = tuple(image.shape)
|
|
214
|
+
# The channel axis is carried, not tiled: geometry (tiles, halo, the stage
|
|
215
|
+
# store) stays spatial, so nothing downstream of fn learns about channels.
|
|
216
|
+
if channel_axis is None:
|
|
217
|
+
spatial_shape = shape
|
|
218
|
+
else:
|
|
219
|
+
channel_axis %= len(shape)
|
|
220
|
+
spatial_shape = shape[:channel_axis] + shape[channel_axis + 1 :]
|
|
221
|
+
sl = spatial_tiles(spatial_shape, tile_shape)[index]
|
|
207
222
|
halo = normalize_overlap(overlap, len(sl), tile_shape=tile_shape)
|
|
208
223
|
expanded, trims = [], []
|
|
209
|
-
for s, dim, ov in zip(sl,
|
|
224
|
+
for s, dim, ov in zip(sl, spatial_shape, halo):
|
|
210
225
|
lo = max(0, s.start - ov)
|
|
211
226
|
hi = min(dim, s.stop + ov)
|
|
212
227
|
expanded.append(slice(lo, hi))
|
|
213
228
|
trims.append((s.start - lo, hi - s.stop))
|
|
214
|
-
|
|
229
|
+
read = list(expanded)
|
|
230
|
+
if channel_axis is not None:
|
|
231
|
+
read.insert(channel_axis, slice(None))
|
|
232
|
+
block = np.asarray(image[tuple(read)])
|
|
233
|
+
# What fn owes us back: one label per voxel, channel axis consumed.
|
|
234
|
+
block_spatial = tuple(e.stop - e.start for e in expanded)
|
|
215
235
|
out = np.asarray(fn(block))
|
|
216
|
-
if out.shape !=
|
|
236
|
+
if out.shape != block_spatial:
|
|
217
237
|
# Caught here rather than 6 frames deep in zarr's codec pipeline as
|
|
218
238
|
# "could not broadcast input array from shape (13,1020,1020) into
|
|
219
239
|
# shape (14,1024,1024)", which says nothing about which function is
|
|
@@ -224,7 +244,7 @@ def stage_tile(
|
|
|
224
244
|
name = getattr(fn, "__name__", type(fn).__name__)
|
|
225
245
|
raise ValueError(
|
|
226
246
|
f"segmentation function {name!r} returned shape {out.shape} for "
|
|
227
|
-
f"a tile of shape {
|
|
247
|
+
f"a tile of shape {block_spatial} (tile {index}). It must return "
|
|
228
248
|
"one label per input voxel. Some deconvolution backends crop "
|
|
229
249
|
"their output -- pad or centre it back to the input shape before "
|
|
230
250
|
"returning."
|
|
@@ -161,9 +161,13 @@ def _make_config(
|
|
|
161
161
|
gpu : bool
|
|
162
162
|
Run on the GPU.
|
|
163
163
|
channels : list of int or None
|
|
164
|
-
Cellpose
|
|
164
|
+
*Cellpose 3 only.* ``[cyto, nucleus]``, 1-based into the channel axis
|
|
165
|
+
(0 = grayscale). ``None`` resolves per tile: ``[1, 2]`` when the tile
|
|
166
|
+
carries two channels, else ``[0, 0]``. Cellpose 4 dropped this
|
|
167
|
+
argument, so it is ignored there.
|
|
165
168
|
channel_axis : int or None
|
|
166
|
-
|
|
169
|
+
Axis of the tile holding channels, forwarded to ``eval`` for both
|
|
170
|
+
Cellpose 3 and 4. ``None`` means single-channel tiles.
|
|
167
171
|
diameter : float or None
|
|
168
172
|
Expected cell diameter in pixels.
|
|
169
173
|
do_3D : bool
|
|
@@ -179,7 +183,9 @@ def _make_config(
|
|
|
179
183
|
return {
|
|
180
184
|
"model": model,
|
|
181
185
|
"gpu": gpu,
|
|
182
|
-
|
|
186
|
+
# Left as None ("auto") rather than [0, 0]: _run only knows how many
|
|
187
|
+
# channels a tile actually carries once it has one in hand.
|
|
188
|
+
"channels": channels,
|
|
183
189
|
"channel_axis": channel_axis,
|
|
184
190
|
"diameter": diameter,
|
|
185
191
|
"do_3D": do_3D,
|
|
@@ -284,37 +290,50 @@ def _run(block: np.ndarray, cellpose_dict: dict[str, Any]) -> np.ndarray:
|
|
|
284
290
|
Integer (``int32``) label array of the same spatial shape.
|
|
285
291
|
"""
|
|
286
292
|
do_3D = cellpose_dict["do_3D"]
|
|
293
|
+
channel_axis = cellpose_dict.get("channel_axis")
|
|
294
|
+
n_channels = block.shape[channel_axis] if channel_axis is not None else 1
|
|
295
|
+
|
|
296
|
+
kwargs: dict[str, Any] = dict(
|
|
297
|
+
channel_axis=channel_axis,
|
|
298
|
+
diameter=cellpose_dict["diameter"],
|
|
299
|
+
do_3D=do_3D,
|
|
300
|
+
**cellpose_dict.get("cellpose_kwargs", {}),
|
|
301
|
+
)
|
|
302
|
+
if not _CELLPOSE_V4:
|
|
303
|
+
# Cellpose 4 (cpsam) dropped `channels` and reads whatever channels
|
|
304
|
+
# the array carries; Cellpose 3 needs the cyto/nucleus pairing named.
|
|
305
|
+
channels = cellpose_dict.get("channels")
|
|
306
|
+
if channels is None:
|
|
307
|
+
channels = [1, 2] if n_channels >= 2 else [0, 0]
|
|
308
|
+
kwargs["channels"] = channels
|
|
287
309
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
channel_axis=cellpose_dict.get("channel_axis"),
|
|
291
|
-
diameter=cellpose_dict["diameter"],
|
|
292
|
-
do_3D=do_3D,
|
|
293
|
-
**cellpose_dict.get("cellpose_kwargs", {}),
|
|
294
|
-
)
|
|
295
|
-
else:
|
|
296
|
-
kwargs = dict(
|
|
297
|
-
channels=cellpose_dict["channels"],
|
|
298
|
-
diameter=cellpose_dict["diameter"],
|
|
299
|
-
do_3D=do_3D,
|
|
300
|
-
**cellpose_dict.get("cellpose_kwargs", {}),
|
|
301
|
-
)
|
|
310
|
+
# Where z sits once the channel axis is accounted for.
|
|
311
|
+
z_axis = 1 if channel_axis == 0 else 0
|
|
302
312
|
|
|
303
313
|
if do_3D:
|
|
304
|
-
kwargs["z_axis"] =
|
|
314
|
+
kwargs["z_axis"] = z_axis
|
|
305
315
|
masks = _eval_with_oom_fallback(block, kwargs, cellpose_dict)
|
|
306
316
|
return masks.astype("int32")
|
|
307
317
|
else:
|
|
308
318
|
# Squeeze singleton z so Cellpose gets a clean 2-D image
|
|
309
|
-
|
|
310
|
-
if
|
|
319
|
+
spatial = list(block.shape)
|
|
320
|
+
if channel_axis is not None:
|
|
321
|
+
spatial.pop(channel_axis)
|
|
322
|
+
squeeze = len(spatial) == 3 and spatial[0] == 1
|
|
323
|
+
if len(spatial) == 3 and not squeeze:
|
|
311
324
|
raise ValueError(
|
|
312
|
-
f"do_3D is False but this tile has {
|
|
325
|
+
f"do_3D is False but this tile has {spatial[0]} z-planes. "
|
|
313
326
|
"Cellpose would receive the stack with no z_axis and treat "
|
|
314
327
|
"the leading axis as channels. Set do_3D: true, or tile with "
|
|
315
328
|
"z=1 to segment plane by plane."
|
|
316
329
|
)
|
|
317
|
-
|
|
330
|
+
if squeeze:
|
|
331
|
+
img = block[(slice(None),) * z_axis + (0,)]
|
|
332
|
+
# Dropping z shifts any channel axis that sat behind it.
|
|
333
|
+
if channel_axis is not None and channel_axis > z_axis:
|
|
334
|
+
kwargs["channel_axis"] = channel_axis - 1
|
|
335
|
+
else:
|
|
336
|
+
img = block
|
|
318
337
|
masks = _eval_with_oom_fallback(img, kwargs, cellpose_dict)
|
|
319
338
|
masks = masks.astype("int32")
|
|
320
339
|
return masks[np.newaxis] if squeeze else masks
|
|
@@ -406,21 +406,57 @@ def view_in_napari(
|
|
|
406
406
|
# unwrapped to a single array) even for one level, so napari always
|
|
407
407
|
# treats it as multiscale — required for 3D resolution switching, see
|
|
408
408
|
# https://napari.org/stable/gallery/add_multiscale_volume.html
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
409
|
+
names = _inner_label_names(image)
|
|
410
|
+
if not names:
|
|
411
|
+
logger.warning(
|
|
412
|
+
"%s has no label images under labels/, so nothing was "
|
|
413
|
+
"overlaid. Pass labels=<path> explicitly if they live "
|
|
414
|
+
"somewhere else.",
|
|
415
|
+
image,
|
|
416
|
+
)
|
|
417
|
+
else:
|
|
418
|
+
logger.info(
|
|
419
|
+
"auto-loading %d label image(s) from %s/labels: %s",
|
|
420
|
+
len(names),
|
|
421
|
+
image,
|
|
422
|
+
", ".join(names),
|
|
422
423
|
)
|
|
424
|
+
loaded = 0
|
|
425
|
+
for name in names:
|
|
426
|
+
store = f"{image}/labels/{name}"
|
|
427
|
+
# Guarded per label: without this, one bad label group raised
|
|
428
|
+
# *after* the image had been added, so the viewer opened showing
|
|
429
|
+
# the image alone and every remaining label was skipped -- which
|
|
430
|
+
# looks exactly like "there were no labels".
|
|
431
|
+
try:
|
|
432
|
+
levels = _multiscale_levels(store, None)
|
|
433
|
+
lab = [lvl.astype("int32") for lvl in levels]
|
|
434
|
+
lab_scale, lab_units = _pyramid_calibration(store, lab[0].ndim)
|
|
435
|
+
viewer.add_labels(
|
|
436
|
+
lab,
|
|
437
|
+
name=name,
|
|
438
|
+
multiscale=True,
|
|
439
|
+
scale=lab_scale,
|
|
440
|
+
units=lab_units,
|
|
441
|
+
metadata=_label_hint(store),
|
|
442
|
+
**label_kwargs,
|
|
443
|
+
)
|
|
444
|
+
except Exception:
|
|
445
|
+
logger.exception(
|
|
446
|
+
"could not add labels/%s as a layer; skipping it and "
|
|
447
|
+
"continuing with the rest.",
|
|
448
|
+
name,
|
|
449
|
+
)
|
|
450
|
+
continue
|
|
451
|
+
loaded += 1
|
|
423
452
|
logger.info("auto-loaded labels/%s from %s", name, image)
|
|
453
|
+
if names and not loaded:
|
|
454
|
+
logger.error(
|
|
455
|
+
"found %d label image(s) in %s/labels but none could be "
|
|
456
|
+
"added -- see the errors above.",
|
|
457
|
+
len(names),
|
|
458
|
+
image,
|
|
459
|
+
)
|
|
424
460
|
|
|
425
461
|
if show:
|
|
426
462
|
napari.run()
|
|
@@ -399,3 +399,96 @@ def test_separate_objects_keep_distinct_labels(tmp_path):
|
|
|
399
399
|
ids = np.unique(merged[merged > 0])
|
|
400
400
|
assert ids.size == 4, f"expected 4 distinct objects, got {ids.size}"
|
|
401
401
|
assert set(ids.tolist()) == {1, 2, 3, 4} # contiguous after relabel
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def test_stage_tile_carries_a_channel_axis(tmp_path):
|
|
405
|
+
"""A channel axis reaches fn but is never tiled.
|
|
406
|
+
|
|
407
|
+
Cellpose's cyto+nucleus pair: tile geometry, the halo and the stage store
|
|
408
|
+
all stay purely spatial, so fn sees ``(2, z, y, x)`` and owes back
|
|
409
|
+
``(z, y, x)``. Without this, a second channel would either be tiled like a
|
|
410
|
+
spatial axis or trip stage_tile's shape check.
|
|
411
|
+
"""
|
|
412
|
+
import numpy as np
|
|
413
|
+
import zarr
|
|
414
|
+
|
|
415
|
+
from patchworks import create_stage, stage_tile
|
|
416
|
+
|
|
417
|
+
# Channel 0 carries the signal, channel 1 is blank -- so a swapped or
|
|
418
|
+
# collapsed channel axis fails rather than quietly segmenting the wrong one.
|
|
419
|
+
img = np.stack(
|
|
420
|
+
[
|
|
421
|
+
np.arange(4 * 8 * 8).reshape(4, 8, 8) % 5,
|
|
422
|
+
np.zeros((4, 8, 8), dtype=int),
|
|
423
|
+
]
|
|
424
|
+
)
|
|
425
|
+
assert img.shape == (2, 4, 8, 8)
|
|
426
|
+
|
|
427
|
+
tile_shape = (4, 4, 4)
|
|
428
|
+
stage = create_stage(tmp_path / "s.zarr", (4, 8, 8), tile_shape)
|
|
429
|
+
|
|
430
|
+
seen = {}
|
|
431
|
+
|
|
432
|
+
def fn(block):
|
|
433
|
+
seen["shape"] = block.shape
|
|
434
|
+
assert not block[1].any(), "channel 1 should be the blank one"
|
|
435
|
+
return (block[0] > 2).astype("int32")
|
|
436
|
+
|
|
437
|
+
n = stage_tile(
|
|
438
|
+
img,
|
|
439
|
+
fn,
|
|
440
|
+
stage,
|
|
441
|
+
0,
|
|
442
|
+
tile_shape=tile_shape,
|
|
443
|
+
overlap=1,
|
|
444
|
+
channel_axis=0,
|
|
445
|
+
)
|
|
446
|
+
|
|
447
|
+
# z is already the full extent, y/x grow by the 1-voxel halo.
|
|
448
|
+
assert seen["shape"] == (2, 4, 5, 5)
|
|
449
|
+
assert n == 1
|
|
450
|
+
staged = zarr.open_group(str(stage))["staged"][:]
|
|
451
|
+
assert staged.shape == (4, 8, 8) # store never learned about channels
|
|
452
|
+
assert staged[:, :4, :4].any() # the tile actually landed
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def test_cellpose_run_pairs_channels_and_shifts_z(monkeypatch):
|
|
456
|
+
"""A 2-channel tile must set z_axis past the channel axis.
|
|
457
|
+
|
|
458
|
+
With channels stacked on axis 0, z moves to axis 1; leaving z_axis=0 would
|
|
459
|
+
tell Cellpose the channel axis is z. Cellpose 3 also needs the pair named
|
|
460
|
+
as ``channels=[1, 2]`` (1-based, 0 = grayscale), which Cellpose 4 dropped.
|
|
461
|
+
"""
|
|
462
|
+
import numpy as np
|
|
463
|
+
|
|
464
|
+
from patchworks.plugins import cellpose as cp
|
|
465
|
+
|
|
466
|
+
calls = {}
|
|
467
|
+
|
|
468
|
+
class _FakeModel:
|
|
469
|
+
def eval(self, img, **kwargs):
|
|
470
|
+
calls["img_shape"] = img.shape
|
|
471
|
+
calls["kwargs"] = kwargs
|
|
472
|
+
return np.zeros(img.shape[1:], dtype="int32"), None
|
|
473
|
+
|
|
474
|
+
monkeypatch.setattr(cp, "_get_model", lambda _cfg: _FakeModel())
|
|
475
|
+
|
|
476
|
+
cfg = cp._make_config("cyto3", diameter=30, do_3D=True, channel_axis=0)
|
|
477
|
+
block = np.zeros((2, 4, 8, 8), dtype="uint16")
|
|
478
|
+
out = cp._run(block, cfg)
|
|
479
|
+
|
|
480
|
+
assert out.shape == (4, 8, 8) # labels come back spatial
|
|
481
|
+
assert calls["kwargs"]["z_axis"] == 1
|
|
482
|
+
assert calls["kwargs"]["channel_axis"] == 0
|
|
483
|
+
if cp._CELLPOSE_V4:
|
|
484
|
+
assert "channels" not in calls["kwargs"]
|
|
485
|
+
else:
|
|
486
|
+
assert calls["kwargs"]["channels"] == [1, 2]
|
|
487
|
+
|
|
488
|
+
# Single-channel tiles keep the old geometry and grayscale pairing.
|
|
489
|
+
cfg1 = cp._make_config("cyto3", diameter=30, do_3D=True)
|
|
490
|
+
cp._run(np.zeros((4, 8, 8), dtype="uint16"), cfg1)
|
|
491
|
+
assert calls["kwargs"]["z_axis"] == 0
|
|
492
|
+
assert calls["kwargs"]["channel_axis"] is None
|
|
493
|
+
if not cp._CELLPOSE_V4:
|
|
494
|
+
assert calls["kwargs"]["channels"] == [0, 0]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Example segmentation config: cytoplasm/membrane channel.
|
|
2
|
+
#
|
|
3
|
+
# Only what differs from config/common.yaml — the input, work_dir, tiling and
|
|
4
|
+
# conversion settings all live there and are merged in ahead of this file.
|
|
5
|
+
# Run it via config/multi.yaml, or on its own with both files:
|
|
6
|
+
#
|
|
7
|
+
# snakemake -s Snakefile --configfile config/common.yaml config/config_cyto.yaml
|
|
8
|
+
|
|
9
|
+
channel: 0 # cytoplasm/membrane channel
|
|
10
|
+
|
|
11
|
+
# Optional second channel handed to Cellpose as the nucleus input, which
|
|
12
|
+
# usually improves cytoplasm segmentation. Channel indices are 0-based, same
|
|
13
|
+
# as `channel`. Leave it out (or null) for a single-channel run.
|
|
14
|
+
#
|
|
15
|
+
# Only the segment step reads it: the tile geometry, the occupancy map and the
|
|
16
|
+
# staged labels all stay exactly as they are for a single channel -- the pair
|
|
17
|
+
# is stacked on a leading axis that is carried into each tile, not tiled. It
|
|
18
|
+
# does double the bytes a tile holds, so a hand-set `tile_shape` sized to the
|
|
19
|
+
# GPU may need halving.
|
|
20
|
+
#
|
|
21
|
+
# Cellpose 3 turns this into channels: [1, 2]; Cellpose 4 (cpsam) dropped
|
|
22
|
+
# `channels` and just reads both. Override either by setting `channels:` or
|
|
23
|
+
# `channel_axis:` in the cellpose: block below.
|
|
24
|
+
nuclei_channel: 1
|
|
25
|
+
|
|
26
|
+
# Per-axis halo [z, y, x]. A scalar 30 would expand a [16, 1024, 1024] tile to
|
|
27
|
+
# 76 x 1084 x 1084 = 5.3x the voxels it keeps, nearly all of it wasted z. The
|
|
28
|
+
# z-halo only needs to cover one cell in z, not one cell in x/y.
|
|
29
|
+
overlap: [4, 30, 30]
|
|
30
|
+
|
|
31
|
+
method: "cellpose"
|
|
32
|
+
label_name: "cyto_labels"
|
|
33
|
+
cellpose:
|
|
34
|
+
model: "cyto3"
|
|
35
|
+
diameter: 30
|
|
36
|
+
do_3D: true
|
|
37
|
+
gpu: true
|
|
@@ -71,7 +71,7 @@ def start_log(path, *, append=True):
|
|
|
71
71
|
return handle
|
|
72
72
|
|
|
73
73
|
|
|
74
|
-
def open_image(work_dir, channel, level):
|
|
74
|
+
def open_image(work_dir, channel, level, nuclei_channel=None):
|
|
75
75
|
"""Open the converted image for segmentation.
|
|
76
76
|
|
|
77
77
|
Parameters
|
|
@@ -82,14 +82,38 @@ def open_image(work_dir, channel, level):
|
|
|
82
82
|
Channel to select.
|
|
83
83
|
level : int
|
|
84
84
|
Pyramid level to read.
|
|
85
|
+
nuclei_channel : int or None, optional
|
|
86
|
+
Second channel to stack in front of *channel*, giving a
|
|
87
|
+
``(2, z, y, x)`` array whose leading axis is ``[channel,
|
|
88
|
+
nuclei_channel]``. This is Cellpose's cyto+nucleus pair (see
|
|
89
|
+
``config/config_cyto.yaml``). ``None`` (default) returns the plain
|
|
90
|
+
``(z, y, x)`` single-channel array.
|
|
85
91
|
|
|
86
92
|
Returns
|
|
87
93
|
-------
|
|
88
94
|
da.Array
|
|
89
95
|
The (lazy) image array.
|
|
90
96
|
"""
|
|
91
|
-
|
|
92
|
-
|
|
97
|
+
path = str(Path(work_dir) / "image.zarr")
|
|
98
|
+
if nuclei_channel is None:
|
|
99
|
+
return load_ome_zarr(path, channel=channel, level=level)
|
|
100
|
+
|
|
101
|
+
import dask.array as da
|
|
102
|
+
|
|
103
|
+
if nuclei_channel == channel:
|
|
104
|
+
raise ValueError(
|
|
105
|
+
f"nuclei_channel ({nuclei_channel}) is the same as channel "
|
|
106
|
+
f"({channel}); Cellpose would get the same image twice. Channel "
|
|
107
|
+
"indices are 0-based."
|
|
108
|
+
)
|
|
109
|
+
# Two lazy reads of the same spatial box, stacked. The tiling stays
|
|
110
|
+
# spatial -- stage_tile carries this axis through rather than tiling it.
|
|
111
|
+
return da.stack(
|
|
112
|
+
[
|
|
113
|
+
load_ome_zarr(path, channel=c, level=level)
|
|
114
|
+
for c in (channel, nuclei_channel)
|
|
115
|
+
],
|
|
116
|
+
axis=0,
|
|
93
117
|
)
|
|
94
118
|
|
|
95
119
|
|
|
@@ -226,6 +250,24 @@ def validate_config(cfg) -> None:
|
|
|
226
250
|
elif ts is not None and not all(isinstance(v, int) and v > 0 for v in ts):
|
|
227
251
|
problems.append(f"tile_shape must be positive integers; got {ts!r}")
|
|
228
252
|
|
|
253
|
+
nuc = cfg.get("nuclei_channel")
|
|
254
|
+
if nuc is not None:
|
|
255
|
+
if isinstance(nuc, bool) or not isinstance(nuc, int):
|
|
256
|
+
problems.append(
|
|
257
|
+
"nuclei_channel must be null or a 0-based channel index "
|
|
258
|
+
f"(e.g. 1); got {nuc!r}"
|
|
259
|
+
)
|
|
260
|
+
elif nuc == cfg.get("channel"):
|
|
261
|
+
problems.append(
|
|
262
|
+
f"nuclei_channel ({nuc}) is the same as channel; Cellpose "
|
|
263
|
+
"would get the same image twice. Indices are 0-based."
|
|
264
|
+
)
|
|
265
|
+
if cfg.get("method", "cellpose") == "threshold":
|
|
266
|
+
problems.append(
|
|
267
|
+
'nuclei_channel has no meaning for method: "threshold", '
|
|
268
|
+
"which segments a single channel"
|
|
269
|
+
)
|
|
270
|
+
|
|
229
271
|
method = cfg.get("method", "cellpose")
|
|
230
272
|
if method not in KNOWN_METHODS:
|
|
231
273
|
listed = ", ".join(f'"{m}"' for m in KNOWN_METHODS)
|
|
@@ -378,6 +420,11 @@ def _build_method_fn(cfg):
|
|
|
378
420
|
for k, v in cp.items()
|
|
379
421
|
if k not in ("model", "diameter", "do_3D", "gpu")
|
|
380
422
|
}
|
|
423
|
+
if cfg.get("nuclei_channel") is not None:
|
|
424
|
+
# open_image stacks [channel, nuclei_channel] on axis 0; tell
|
|
425
|
+
# Cellpose where they are. setdefault so an explicit
|
|
426
|
+
# cellpose.channel_axis in the config still wins.
|
|
427
|
+
extra.setdefault("channel_axis", 0)
|
|
381
428
|
return cellpose_fn(
|
|
382
429
|
cp.get("model", "cyto3"),
|
|
383
430
|
gpu=cp.get("gpu", True),
|
|
@@ -21,7 +21,11 @@ work_dir = cfg["work_dir"]
|
|
|
21
21
|
label_name = cfg.get("label_name", "labels")
|
|
22
22
|
|
|
23
23
|
manifest = load_tiles_json(snakemake.input.tiles) # noqa: F821
|
|
24
|
-
|
|
24
|
+
# Optional second channel (Cellpose's nucleus input): open_image stacks it on
|
|
25
|
+
# a leading axis that stage_tile carries through without tiling it, so the
|
|
26
|
+
# tile geometry and the staged labels stay exactly as single-channel runs.
|
|
27
|
+
nuclei_channel = cfg.get("nuclei_channel")
|
|
28
|
+
image = open_image(work_dir, cfg["channel"], cfg["level"], nuclei_channel)
|
|
25
29
|
indices = manifest["batches"][batch]
|
|
26
30
|
|
|
27
31
|
# Built once for the whole batch: this is what makes the model load amortize.
|
|
@@ -46,6 +50,7 @@ for n, index in enumerate(indices, 1):
|
|
|
46
50
|
# Scalar (older manifests) or per-axis list; stage_tile normalizes both.
|
|
47
51
|
overlap=manifest["overlap"],
|
|
48
52
|
component=component,
|
|
53
|
+
channel_axis=0 if nuclei_channel is not None else None,
|
|
49
54
|
)
|
|
50
55
|
# The per-tile time is what `tiles_per_job` has to be sized from: a job's
|
|
51
56
|
# wall time is roughly N x this, and it must stay inside the QOS ceiling.
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
# Example segmentation config: cytoplasm/membrane channel.
|
|
2
|
-
#
|
|
3
|
-
# Only what differs from config/common.yaml — the input, work_dir, tiling and
|
|
4
|
-
# conversion settings all live there and are merged in ahead of this file.
|
|
5
|
-
# Run it via config/multi.yaml, or on its own with both files:
|
|
6
|
-
#
|
|
7
|
-
# snakemake -s Snakefile --configfile config/common.yaml config/config_cyto.yaml
|
|
8
|
-
|
|
9
|
-
channel: 0 # cytoplasm/membrane channel
|
|
10
|
-
|
|
11
|
-
# Per-axis halo [z, y, x]. A scalar 30 would expand a [16, 1024, 1024] tile to
|
|
12
|
-
# 76 x 1084 x 1084 = 5.3x the voxels it keeps, nearly all of it wasted z. The
|
|
13
|
-
# z-halo only needs to cover one cell in z, not one cell in x/y.
|
|
14
|
-
overlap: [4, 30, 30]
|
|
15
|
-
|
|
16
|
-
method: "cellpose"
|
|
17
|
-
label_name: "cyto_labels"
|
|
18
|
-
cellpose:
|
|
19
|
-
model: "cyto3"
|
|
20
|
-
diameter: 30
|
|
21
|
-
do_3D: true
|
|
22
|
-
gpu: true
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|