patchworks 2.5.1__tar.gz → 2.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {patchworks-2.5.1 → patchworks-2.6.0}/.gitignore +3 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/PKG-INFO +8 -1
- {patchworks-2.5.1 → patchworks-2.6.0}/README.md +7 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/guide/snakemake.md +44 -1
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_distributed.py +26 -6
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_io.py +55 -1
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_notify.py +42 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/plugins/cellpose.py +41 -22
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/plugins/napari.py +49 -13
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_core.py +32 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_distributed.py +93 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_notify.py +35 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/Snakefile +21 -11
- patchworks-2.6.0/workflow/config/config_cyto.yaml +37 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/scripts/_pw.py +50 -3
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/scripts/run_multi.py +70 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/scripts/segment_tile.py +6 -1
- patchworks-2.5.1/workflow/config/config_cyto.yaml +0 -22
- {patchworks-2.5.1 → patchworks-2.6.0}/.github/workflows/docs.yml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/.github/workflows/lint.yml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/.github/workflows/release.yml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/.markdownlint-cli2.yaml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/LICENSE +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/cliff.toml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/api/chunks.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/api/cluster.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/api/io.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/api/merge_tile_labels.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/api/plugins/cellpose.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/api/plugins/dog.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/api/plugins/napari.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/api/plugins/ome_zarr.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/api/postprocess.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/api/relabel.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/api/tile_process.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/assets/logo.png +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/examples/cellpose_2d.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/examples/cellpose_2d.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/examples/cellpose_3d.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/examples/cellpose_3d.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/examples/custom.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/examples/custom_method.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/examples/dog.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/examples/dog.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/examples/standalone_merge.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/examples/stardist.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/examples/stardist_2d.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/getting_started.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/guide/custom_segmentation.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/guide/gpu_distributed.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/guide/label_relations.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/guide/measurements.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/guide/merging.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/guide/ome_zarr_napari.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/guide/performance.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/guide/pitfalls.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/guide/skip_empty.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/guide/tiling.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/docs/index.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/mkdocs.yml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/pyproject.toml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/__init__.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_chunks.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_cluster.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_core.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_gpu.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_merge.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_occupancy.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_postprocess.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_progress.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_relabel.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/_relations.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/plugins/__init__.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/plugins/dog.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/src/patchworks/plugins/ome_zarr.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_allocation.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_dog.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_gpu.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_napari.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_occupancy.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_ome_zarr.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_postprocess.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_progress.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_relations.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/tests/test_run_multi.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/README.md +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/config/common.yaml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/config/config.yaml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/config/config_cilia.yaml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/config/config_nuclei.yaml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/config/multi.yaml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/pixi.toml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/profile/slurm/config.yaml +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/rules/common.smk +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/rules/convert.smk +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/rules/merge.smk +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/rules/segment.smk +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/scripts/build_occupancy.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/scripts/convert.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/scripts/fetch_model.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/scripts/merge.py +0 -0
- {patchworks-2.5.1 → patchworks-2.6.0}/workflow/scripts/prepare_tiles.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: patchworks
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.6.0
|
|
4
4
|
Summary: Tiled processing of arbitrarily large images with globally consistent labels
|
|
5
5
|
Project-URL: Homepage, https://github.com/imcf/patchworks
|
|
6
6
|
Project-URL: Issues, https://github.com/imcf/patchworks/issues
|
|
@@ -115,6 +115,13 @@ patchworks splits a large image into tiles, runs **any callable** on each
|
|
|
115
115
|
tile in parallel, and merges the results into a globally consistent label array.
|
|
116
116
|
It handles terabyte-scale images without loading them into memory.
|
|
117
117
|
|
|
118
|
+
> [!NOTE]
|
|
119
|
+
> **On how this was written.** Large parts of patchworks were vibe coded —
|
|
120
|
+
> written with heavy LLM assistance rather than line by line. It is covered by
|
|
121
|
+
> a test suite and has been run on real data, so it is not untested, but the
|
|
122
|
+
> usual caveats apply: read the code before you trust it with anything
|
|
123
|
+
> irreplaceable, and please open an issue if something looks off.
|
|
124
|
+
|
|
118
125
|
---
|
|
119
126
|
|
|
120
127
|
## Installation
|
|
@@ -25,6 +25,13 @@ patchworks splits a large image into tiles, runs **any callable** on each
|
|
|
25
25
|
tile in parallel, and merges the results into a globally consistent label array.
|
|
26
26
|
It handles terabyte-scale images without loading them into memory.
|
|
27
27
|
|
|
28
|
+
> [!NOTE]
|
|
29
|
+
> **On how this was written.** Large parts of patchworks were vibe coded —
|
|
30
|
+
> written with heavy LLM assistance rather than line by line. It is covered by
|
|
31
|
+
> a test suite and has been run on real data, so it is not untested, but the
|
|
32
|
+
> usual caveats apply: read the code before you trust it with anything
|
|
33
|
+
> irreplaceable, and please open an issue if something looks off.
|
|
34
|
+
|
|
28
35
|
---
|
|
29
36
|
|
|
30
37
|
## Installation
|
|
@@ -53,7 +53,8 @@ convert_chunks: null # null → bounded auto chunks; or [c,z,y,x]
|
|
|
53
53
|
shard: false # true → pack chunks into shards (fewer files)
|
|
54
54
|
|
|
55
55
|
# tiling
|
|
56
|
-
channel: 0 # channel to segment (null = keep all)
|
|
56
|
+
channel: 0 # channel to segment, 0-based (null = keep all)
|
|
57
|
+
nuclei_channel: null # optional 2nd channel for Cellpose (see below)
|
|
57
58
|
level: 0 # pyramid level (0 = full resolution)
|
|
58
59
|
tile_shape: "auto" # "auto", or e.g. [16, 1024, 1024] (zyx)
|
|
59
60
|
gpu_memory_gb: null # for "auto" on SLURM: your segment GPU's VRAM
|
|
@@ -299,6 +300,7 @@ cellpose:
|
|
|
299
300
|
# config/config_cyto.yaml — only the differences
|
|
300
301
|
label_name: "cyto_labels"
|
|
301
302
|
channel: 0 # cytoplasm/membrane channel
|
|
303
|
+
nuclei_channel: 1 # optional: nuclear stain, as Cellpose's 2nd input
|
|
302
304
|
overlap: [4, 30, 30]
|
|
303
305
|
method: "cellpose"
|
|
304
306
|
cellpose:
|
|
@@ -307,6 +309,29 @@ cellpose:
|
|
|
307
309
|
do_3D: true
|
|
308
310
|
```
|
|
309
311
|
|
|
312
|
+
### Giving Cellpose a nuclei channel
|
|
313
|
+
|
|
314
|
+
`nuclei_channel` hands Cellpose a second channel — the nuclear stain — which
|
|
315
|
+
usually improves cytoplasm segmentation. Both indices are 0-based, like
|
|
316
|
+
`channel`.
|
|
317
|
+
|
|
318
|
+
Only the `segment` step reads it. The pair is stacked on a leading axis that
|
|
319
|
+
is *carried* into each tile rather than tiled, so the tile geometry, the
|
|
320
|
+
occupancy map and the staged labels are byte-for-byte what a single-channel
|
|
321
|
+
run produces, and `merge` and `label_relations` need no changes. Two things
|
|
322
|
+
follow from that:
|
|
323
|
+
|
|
324
|
+
- A tile holds twice the bytes, so a hand-set `tile_shape` sized to fill a GPU
|
|
325
|
+
may need halving. `tile_shape: "auto"` sizes from the single-channel array
|
|
326
|
+
and does not yet know about the pair.
|
|
327
|
+
- The translation is version-specific. Cellpose 3 gets `channels: [1, 2]`
|
|
328
|
+
(1-based into the channel axis, `0` = grayscale); Cellpose 4 (cpsam) dropped
|
|
329
|
+
`channels` entirely and simply reads both. Either is overridable by setting
|
|
330
|
+
`channels:` or `channel_axis:` in the `cellpose:` block.
|
|
331
|
+
|
|
332
|
+
`nuclei_channel` applies to the SLURM/Snakemake path. The single-process
|
|
333
|
+
`tile_process` API still takes one `channel`.
|
|
334
|
+
|
|
310
335
|
Run them as two independent SLURM submissions — they touch disjoint files, so
|
|
311
336
|
they can run concurrently. Give each its own `--directory`, because
|
|
312
337
|
Snakemake's lock lives in the working directory, not in the config:
|
|
@@ -475,6 +500,24 @@ These cover what SLURM cannot: a local run with no scheduler at all, and
|
|
|
475
500
|
failures where the useful content is the Python traceback rather than an exit
|
|
476
501
|
code.
|
|
477
502
|
|
|
503
|
+
Check it works without waiting for a multi-hour step:
|
|
504
|
+
|
|
505
|
+
```bash
|
|
506
|
+
python scripts/run_multi.py --config config/multi.yaml --test-email
|
|
507
|
+
```
|
|
508
|
+
|
|
509
|
+
It prints the address it actually resolved from the merged config, the exact
|
|
510
|
+
`sbatch` flags the jobs will carry, and whether a test message was accepted —
|
|
511
|
+
which separates "the address never reached the config" from "it did and the
|
|
512
|
+
mail was dropped downstream". Those look identical otherwise: no email either
|
|
513
|
+
way.
|
|
514
|
+
|
|
515
|
+
!!! warning "No mail from `segment`"
|
|
516
|
+
`segment` is excluded on purpose, so a run that is only segmenting tiles
|
|
517
|
+
sends nothing. Mail comes from `convert`, `occupancy` and `merge`, plus
|
|
518
|
+
the workflow-level success/failure message — if those already completed
|
|
519
|
+
before you set the address, there is nothing left in the run to mail you.
|
|
520
|
+
|
|
478
521
|
!!! note "Delivery is best-effort, by design"
|
|
479
522
|
A notification can never fail a run. If no local `sendmail` exists and no
|
|
480
523
|
SMTP server answers on localhost, the failure is logged as a warning and
|
|
@@ -166,6 +166,7 @@ def stage_tile(
|
|
|
166
166
|
tile_shape: tuple[int, ...],
|
|
167
167
|
overlap: Overlap = 0,
|
|
168
168
|
component: str = "staged",
|
|
169
|
+
channel_axis: int | None = None,
|
|
169
170
|
) -> int:
|
|
170
171
|
"""Run *fn* on a single tile and write it into the shared stage store.
|
|
171
172
|
|
|
@@ -192,6 +193,13 @@ def stage_tile(
|
|
|
192
193
|
:func:`normalize_overlap`).
|
|
193
194
|
component : str, optional
|
|
194
195
|
Array name inside the stage store.
|
|
196
|
+
channel_axis : int or None, optional
|
|
197
|
+
Axis of *image* holding channels, which is **not** tiled: it is read
|
|
198
|
+
whole and handed to *fn* alongside the tile's voxels. ``tile_shape``,
|
|
199
|
+
``overlap`` and the stage store stay purely spatial, so *fn* still
|
|
200
|
+
returns one label per voxel with no channel axis (e.g. Cellpose fed a
|
|
201
|
+
cytoplasm + nuclei pair returns a single label volume). ``None`` (the
|
|
202
|
+
default) means *image* is already single-channel.
|
|
195
203
|
|
|
196
204
|
Returns
|
|
197
205
|
-------
|
|
@@ -202,18 +210,30 @@ def stage_tile(
|
|
|
202
210
|
by a cumulative sum, instead of rewriting the whole store to make the
|
|
203
211
|
ids unique.
|
|
204
212
|
"""
|
|
205
|
-
shape = image.shape
|
|
206
|
-
|
|
213
|
+
shape = tuple(image.shape)
|
|
214
|
+
# The channel axis is carried, not tiled: geometry (tiles, halo, the stage
|
|
215
|
+
# store) stays spatial, so nothing downstream of fn learns about channels.
|
|
216
|
+
if channel_axis is None:
|
|
217
|
+
spatial_shape = shape
|
|
218
|
+
else:
|
|
219
|
+
channel_axis %= len(shape)
|
|
220
|
+
spatial_shape = shape[:channel_axis] + shape[channel_axis + 1 :]
|
|
221
|
+
sl = spatial_tiles(spatial_shape, tile_shape)[index]
|
|
207
222
|
halo = normalize_overlap(overlap, len(sl), tile_shape=tile_shape)
|
|
208
223
|
expanded, trims = [], []
|
|
209
|
-
for s, dim, ov in zip(sl,
|
|
224
|
+
for s, dim, ov in zip(sl, spatial_shape, halo):
|
|
210
225
|
lo = max(0, s.start - ov)
|
|
211
226
|
hi = min(dim, s.stop + ov)
|
|
212
227
|
expanded.append(slice(lo, hi))
|
|
213
228
|
trims.append((s.start - lo, hi - s.stop))
|
|
214
|
-
|
|
229
|
+
read = list(expanded)
|
|
230
|
+
if channel_axis is not None:
|
|
231
|
+
read.insert(channel_axis, slice(None))
|
|
232
|
+
block = np.asarray(image[tuple(read)])
|
|
233
|
+
# What fn owes us back: one label per voxel, channel axis consumed.
|
|
234
|
+
block_spatial = tuple(e.stop - e.start for e in expanded)
|
|
215
235
|
out = np.asarray(fn(block))
|
|
216
|
-
if out.shape !=
|
|
236
|
+
if out.shape != block_spatial:
|
|
217
237
|
# Caught here rather than 6 frames deep in zarr's codec pipeline as
|
|
218
238
|
# "could not broadcast input array from shape (13,1020,1020) into
|
|
219
239
|
# shape (14,1024,1024)", which says nothing about which function is
|
|
@@ -224,7 +244,7 @@ def stage_tile(
|
|
|
224
244
|
name = getattr(fn, "__name__", type(fn).__name__)
|
|
225
245
|
raise ValueError(
|
|
226
246
|
f"segmentation function {name!r} returned shape {out.shape} for "
|
|
227
|
-
f"a tile of shape {
|
|
247
|
+
f"a tile of shape {block_spatial} (tile {index}). It must return "
|
|
228
248
|
"one label per input voxel. Some deconvolution backends crop "
|
|
229
249
|
"their output -- pad or centre it back to the input shape before "
|
|
230
250
|
"returning."
|
|
@@ -95,7 +95,61 @@ def load_ome_zarr(
|
|
|
95
95
|
|
|
96
96
|
arr = da.from_zarr(str(store_path), component=path, chunks=zarr_chunks)
|
|
97
97
|
if channel is not None:
|
|
98
|
-
arr = arr[
|
|
98
|
+
arr = _select_channel(arr, channel, _ms[0], store_path)
|
|
99
|
+
return arr
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _select_channel(arr, channel: int, multiscale: dict, store_path):
|
|
103
|
+
"""Index *arr*'s channel axis, or leave it alone when there isn't one.
|
|
104
|
+
|
|
105
|
+
``arr[channel]`` used to be applied unconditionally, so on a
|
|
106
|
+
single-channel store written as plain ``zyx`` the default ``channel: 0``
|
|
107
|
+
silently sliced away **z** instead. The result stayed a valid array, just
|
|
108
|
+
one dimension short, and surfaced much later as a tile-count mismatch
|
|
109
|
+
against the occupancy grid rather than as anything about channels.
|
|
110
|
+
|
|
111
|
+
Parameters
|
|
112
|
+
----------
|
|
113
|
+
arr : da.Array
|
|
114
|
+
The full array as stored.
|
|
115
|
+
channel : int
|
|
116
|
+
Requested channel index.
|
|
117
|
+
multiscale : dict
|
|
118
|
+
The store's multiscales entry, read for its ``axes``.
|
|
119
|
+
store_path : str or Path
|
|
120
|
+
Only used in messages.
|
|
121
|
+
|
|
122
|
+
Returns
|
|
123
|
+
-------
|
|
124
|
+
da.Array
|
|
125
|
+
*arr* with the channel axis indexed away, or unchanged when the store
|
|
126
|
+
has no channel axis and channel 0 was requested.
|
|
127
|
+
"""
|
|
128
|
+
axes = [
|
|
129
|
+
(a.get("name") if isinstance(a, dict) else a) or ""
|
|
130
|
+
for a in (multiscale.get("axes") or [])
|
|
131
|
+
]
|
|
132
|
+
if "c" in axes:
|
|
133
|
+
idx = axes.index("c")
|
|
134
|
+
return arr[(slice(None),) * idx + (channel,)]
|
|
135
|
+
|
|
136
|
+
# No axes metadata: fall back to shape. A 4-D array is c,z,y,x by the
|
|
137
|
+
# convention this package writes; a 3-D one is z,y,x.
|
|
138
|
+
if not axes and arr.ndim >= 4:
|
|
139
|
+
return arr[channel]
|
|
140
|
+
|
|
141
|
+
if channel:
|
|
142
|
+
raise ValueError(
|
|
143
|
+
f"channel={channel} was requested but {store_path!r} has no "
|
|
144
|
+
f"channel axis (axes={axes or 'unknown'}, shape={arr.shape}). "
|
|
145
|
+
"Set channel: null in the config for a single-channel image."
|
|
146
|
+
)
|
|
147
|
+
logger.info(
|
|
148
|
+
"%s has no channel axis; ignoring channel=0 and using the whole "
|
|
149
|
+
"array (shape %s).",
|
|
150
|
+
store_path,
|
|
151
|
+
arr.shape,
|
|
152
|
+
)
|
|
99
153
|
return arr
|
|
100
154
|
|
|
101
155
|
|
|
@@ -14,6 +14,7 @@ turn a real error into a confusing one about email.
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
16
|
import logging
|
|
17
|
+
import re
|
|
17
18
|
import shutil
|
|
18
19
|
import socket
|
|
19
20
|
import subprocess
|
|
@@ -155,3 +156,44 @@ def slurm_mail_extra(
|
|
|
155
156
|
# Keep SLURM's own order, not the config's, so the string is stable.
|
|
156
157
|
types = [mapping[k] for k in ("start", "finish", "error") if k in chosen]
|
|
157
158
|
return f"--mail-type={','.join(types)} --mail-user={email}"
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def failing_step(
|
|
162
|
+
snakemake_log: Union[str, Path, None],
|
|
163
|
+
) -> "tuple[Union[str, None], Union[str, None]]":
|
|
164
|
+
"""Find which rule failed, and its log, from Snakemake's own log file.
|
|
165
|
+
|
|
166
|
+
Guessing from step-log timestamps does not work: a `segment` failure
|
|
167
|
+
leaves `prepare.log` as the most recently written of the sequential
|
|
168
|
+
steps, so a mail built that way quotes a log that *succeeded* and names
|
|
169
|
+
the wrong step. Snakemake records the failing rule and the exact log path
|
|
170
|
+
it used, so read that instead.
|
|
171
|
+
|
|
172
|
+
Parameters
|
|
173
|
+
----------
|
|
174
|
+
snakemake_log : str or Path or None
|
|
175
|
+
Path to Snakemake's own log (the ``log`` variable inside an
|
|
176
|
+
``onerror`` handler).
|
|
177
|
+
|
|
178
|
+
Returns
|
|
179
|
+
-------
|
|
180
|
+
tuple
|
|
181
|
+
``(rule_name, log_path)``, either of which may be None when the log
|
|
182
|
+
is unreadable or records no rule error.
|
|
183
|
+
"""
|
|
184
|
+
try:
|
|
185
|
+
text = Path(snakemake_log).read_text(errors="replace")
|
|
186
|
+
except (OSError, TypeError, ValueError):
|
|
187
|
+
return None, None
|
|
188
|
+
|
|
189
|
+
blocks = text.split("Error in rule ")
|
|
190
|
+
if len(blocks) < 2:
|
|
191
|
+
return None, None
|
|
192
|
+
last = blocks[-1]
|
|
193
|
+
rule = re.match(r"(\S+?):", last)
|
|
194
|
+
# The log: line inside that error block points at the step's own log.
|
|
195
|
+
path = re.search(r"^\s*log:\s*(\S+?)(?:,|\s|$)", last, re.M)
|
|
196
|
+
return (
|
|
197
|
+
rule.group(1) if rule else None,
|
|
198
|
+
path.group(1) if path else None,
|
|
199
|
+
)
|
|
@@ -161,9 +161,13 @@ def _make_config(
|
|
|
161
161
|
gpu : bool
|
|
162
162
|
Run on the GPU.
|
|
163
163
|
channels : list of int or None
|
|
164
|
-
Cellpose
|
|
164
|
+
*Cellpose 3 only.* ``[cyto, nucleus]``, 1-based into the channel axis
|
|
165
|
+
(0 = grayscale). ``None`` resolves per tile: ``[1, 2]`` when the tile
|
|
166
|
+
carries two channels, else ``[0, 0]``. Cellpose 4 dropped this
|
|
167
|
+
argument, so it is ignored there.
|
|
165
168
|
channel_axis : int or None
|
|
166
|
-
|
|
169
|
+
Axis of the tile holding channels, forwarded to ``eval`` for both
|
|
170
|
+
Cellpose 3 and 4. ``None`` means single-channel tiles.
|
|
167
171
|
diameter : float or None
|
|
168
172
|
Expected cell diameter in pixels.
|
|
169
173
|
do_3D : bool
|
|
@@ -179,7 +183,9 @@ def _make_config(
|
|
|
179
183
|
return {
|
|
180
184
|
"model": model,
|
|
181
185
|
"gpu": gpu,
|
|
182
|
-
|
|
186
|
+
# Left as None ("auto") rather than [0, 0]: _run only knows how many
|
|
187
|
+
# channels a tile actually carries once it has one in hand.
|
|
188
|
+
"channels": channels,
|
|
183
189
|
"channel_axis": channel_axis,
|
|
184
190
|
"diameter": diameter,
|
|
185
191
|
"do_3D": do_3D,
|
|
@@ -284,37 +290,50 @@ def _run(block: np.ndarray, cellpose_dict: dict[str, Any]) -> np.ndarray:
|
|
|
284
290
|
Integer (``int32``) label array of the same spatial shape.
|
|
285
291
|
"""
|
|
286
292
|
do_3D = cellpose_dict["do_3D"]
|
|
293
|
+
channel_axis = cellpose_dict.get("channel_axis")
|
|
294
|
+
n_channels = block.shape[channel_axis] if channel_axis is not None else 1
|
|
295
|
+
|
|
296
|
+
kwargs: dict[str, Any] = dict(
|
|
297
|
+
channel_axis=channel_axis,
|
|
298
|
+
diameter=cellpose_dict["diameter"],
|
|
299
|
+
do_3D=do_3D,
|
|
300
|
+
**cellpose_dict.get("cellpose_kwargs", {}),
|
|
301
|
+
)
|
|
302
|
+
if not _CELLPOSE_V4:
|
|
303
|
+
# Cellpose 4 (cpsam) dropped `channels` and reads whatever channels
|
|
304
|
+
# the array carries; Cellpose 3 needs the cyto/nucleus pairing named.
|
|
305
|
+
channels = cellpose_dict.get("channels")
|
|
306
|
+
if channels is None:
|
|
307
|
+
channels = [1, 2] if n_channels >= 2 else [0, 0]
|
|
308
|
+
kwargs["channels"] = channels
|
|
287
309
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
channel_axis=cellpose_dict.get("channel_axis"),
|
|
291
|
-
diameter=cellpose_dict["diameter"],
|
|
292
|
-
do_3D=do_3D,
|
|
293
|
-
**cellpose_dict.get("cellpose_kwargs", {}),
|
|
294
|
-
)
|
|
295
|
-
else:
|
|
296
|
-
kwargs = dict(
|
|
297
|
-
channels=cellpose_dict["channels"],
|
|
298
|
-
diameter=cellpose_dict["diameter"],
|
|
299
|
-
do_3D=do_3D,
|
|
300
|
-
**cellpose_dict.get("cellpose_kwargs", {}),
|
|
301
|
-
)
|
|
310
|
+
# Where z sits once the channel axis is accounted for.
|
|
311
|
+
z_axis = 1 if channel_axis == 0 else 0
|
|
302
312
|
|
|
303
313
|
if do_3D:
|
|
304
|
-
kwargs["z_axis"] =
|
|
314
|
+
kwargs["z_axis"] = z_axis
|
|
305
315
|
masks = _eval_with_oom_fallback(block, kwargs, cellpose_dict)
|
|
306
316
|
return masks.astype("int32")
|
|
307
317
|
else:
|
|
308
318
|
# Squeeze singleton z so Cellpose gets a clean 2-D image
|
|
309
|
-
|
|
310
|
-
if
|
|
319
|
+
spatial = list(block.shape)
|
|
320
|
+
if channel_axis is not None:
|
|
321
|
+
spatial.pop(channel_axis)
|
|
322
|
+
squeeze = len(spatial) == 3 and spatial[0] == 1
|
|
323
|
+
if len(spatial) == 3 and not squeeze:
|
|
311
324
|
raise ValueError(
|
|
312
|
-
f"do_3D is False but this tile has {
|
|
325
|
+
f"do_3D is False but this tile has {spatial[0]} z-planes. "
|
|
313
326
|
"Cellpose would receive the stack with no z_axis and treat "
|
|
314
327
|
"the leading axis as channels. Set do_3D: true, or tile with "
|
|
315
328
|
"z=1 to segment plane by plane."
|
|
316
329
|
)
|
|
317
|
-
|
|
330
|
+
if squeeze:
|
|
331
|
+
img = block[(slice(None),) * z_axis + (0,)]
|
|
332
|
+
# Dropping z shifts any channel axis that sat behind it.
|
|
333
|
+
if channel_axis is not None and channel_axis > z_axis:
|
|
334
|
+
kwargs["channel_axis"] = channel_axis - 1
|
|
335
|
+
else:
|
|
336
|
+
img = block
|
|
318
337
|
masks = _eval_with_oom_fallback(img, kwargs, cellpose_dict)
|
|
319
338
|
masks = masks.astype("int32")
|
|
320
339
|
return masks[np.newaxis] if squeeze else masks
|
|
@@ -406,21 +406,57 @@ def view_in_napari(
|
|
|
406
406
|
# unwrapped to a single array) even for one level, so napari always
|
|
407
407
|
# treats it as multiscale — required for 3D resolution switching, see
|
|
408
408
|
# https://napari.org/stable/gallery/add_multiscale_volume.html
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
409
|
+
names = _inner_label_names(image)
|
|
410
|
+
if not names:
|
|
411
|
+
logger.warning(
|
|
412
|
+
"%s has no label images under labels/, so nothing was "
|
|
413
|
+
"overlaid. Pass labels=<path> explicitly if they live "
|
|
414
|
+
"somewhere else.",
|
|
415
|
+
image,
|
|
416
|
+
)
|
|
417
|
+
else:
|
|
418
|
+
logger.info(
|
|
419
|
+
"auto-loading %d label image(s) from %s/labels: %s",
|
|
420
|
+
len(names),
|
|
421
|
+
image,
|
|
422
|
+
", ".join(names),
|
|
422
423
|
)
|
|
424
|
+
loaded = 0
|
|
425
|
+
for name in names:
|
|
426
|
+
store = f"{image}/labels/{name}"
|
|
427
|
+
# Guarded per label: without this, one bad label group raised
|
|
428
|
+
# *after* the image had been added, so the viewer opened showing
|
|
429
|
+
# the image alone and every remaining label was skipped -- which
|
|
430
|
+
# looks exactly like "there were no labels".
|
|
431
|
+
try:
|
|
432
|
+
levels = _multiscale_levels(store, None)
|
|
433
|
+
lab = [lvl.astype("int32") for lvl in levels]
|
|
434
|
+
lab_scale, lab_units = _pyramid_calibration(store, lab[0].ndim)
|
|
435
|
+
viewer.add_labels(
|
|
436
|
+
lab,
|
|
437
|
+
name=name,
|
|
438
|
+
multiscale=True,
|
|
439
|
+
scale=lab_scale,
|
|
440
|
+
units=lab_units,
|
|
441
|
+
metadata=_label_hint(store),
|
|
442
|
+
**label_kwargs,
|
|
443
|
+
)
|
|
444
|
+
except Exception:
|
|
445
|
+
logger.exception(
|
|
446
|
+
"could not add labels/%s as a layer; skipping it and "
|
|
447
|
+
"continuing with the rest.",
|
|
448
|
+
name,
|
|
449
|
+
)
|
|
450
|
+
continue
|
|
451
|
+
loaded += 1
|
|
423
452
|
logger.info("auto-loaded labels/%s from %s", name, image)
|
|
453
|
+
if names and not loaded:
|
|
454
|
+
logger.error(
|
|
455
|
+
"found %d label image(s) in %s/labels but none could be "
|
|
456
|
+
"added -- see the errors above.",
|
|
457
|
+
len(names),
|
|
458
|
+
image,
|
|
459
|
+
)
|
|
424
460
|
|
|
425
461
|
if show:
|
|
426
462
|
napari.run()
|
|
@@ -357,3 +357,35 @@ def test_tile_process_max_workers():
|
|
|
357
357
|
arr = da.from_array(_make_image((2, 32, 32)), chunks=(1, 32, 32))
|
|
358
358
|
result = tile_process(arr, _label_fn, max_workers=1).compute()
|
|
359
359
|
assert result.shape == (2, 32, 32)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def test_channel_selection_respects_the_stores_axes(tmp_path):
|
|
363
|
+
"""`channel: 0` on a store with no channel axis must not slice away z.
|
|
364
|
+
|
|
365
|
+
arr[channel] was applied unconditionally, so a single-channel image
|
|
366
|
+
written as plain zyx silently lost its z axis. The array stayed valid,
|
|
367
|
+
just one dimension short, and only surfaced later as a tile count that
|
|
368
|
+
disagreed with the occupancy grid.
|
|
369
|
+
"""
|
|
370
|
+
import dask.array as da
|
|
371
|
+
import pytest
|
|
372
|
+
|
|
373
|
+
from patchworks import load_ome_zarr
|
|
374
|
+
from patchworks.plugins.ome_zarr import to_ome_zarr
|
|
375
|
+
|
|
376
|
+
vol = da.zeros((8, 64, 64), chunks=(4, 32, 32), dtype="uint16")
|
|
377
|
+
store = str(tmp_path / "zyx.zarr")
|
|
378
|
+
to_ome_zarr(vol, store, axes="zyx", n_levels=1, progress=False)
|
|
379
|
+
|
|
380
|
+
# channel 0 on a zyx store: keep the whole volume, do not index axis 0.
|
|
381
|
+
assert load_ome_zarr(store, channel=0, level=0).shape == (8, 64, 64)
|
|
382
|
+
assert load_ome_zarr(store, channel=None, level=0).shape == (8, 64, 64)
|
|
383
|
+
# A non-zero channel really is a mistake here, and is named as one.
|
|
384
|
+
with pytest.raises(ValueError, match="no channel axis"):
|
|
385
|
+
load_ome_zarr(store, channel=2, level=0)
|
|
386
|
+
|
|
387
|
+
# A czyx store still selects the channel as before.
|
|
388
|
+
vol4 = da.zeros((3, 8, 64, 64), chunks=(1, 4, 32, 32), dtype="uint16")
|
|
389
|
+
store4 = str(tmp_path / "czyx.zarr")
|
|
390
|
+
to_ome_zarr(vol4, store4, axes="czyx", n_levels=1, progress=False)
|
|
391
|
+
assert load_ome_zarr(store4, channel=1, level=0).shape == (8, 64, 64)
|
|
@@ -399,3 +399,96 @@ def test_separate_objects_keep_distinct_labels(tmp_path):
|
|
|
399
399
|
ids = np.unique(merged[merged > 0])
|
|
400
400
|
assert ids.size == 4, f"expected 4 distinct objects, got {ids.size}"
|
|
401
401
|
assert set(ids.tolist()) == {1, 2, 3, 4} # contiguous after relabel
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def test_stage_tile_carries_a_channel_axis(tmp_path):
|
|
405
|
+
"""A channel axis reaches fn but is never tiled.
|
|
406
|
+
|
|
407
|
+
Cellpose's cyto+nucleus pair: tile geometry, the halo and the stage store
|
|
408
|
+
all stay purely spatial, so fn sees ``(2, z, y, x)`` and owes back
|
|
409
|
+
``(z, y, x)``. Without this, a second channel would either be tiled like a
|
|
410
|
+
spatial axis or trip stage_tile's shape check.
|
|
411
|
+
"""
|
|
412
|
+
import numpy as np
|
|
413
|
+
import zarr
|
|
414
|
+
|
|
415
|
+
from patchworks import create_stage, stage_tile
|
|
416
|
+
|
|
417
|
+
# Channel 0 carries the signal, channel 1 is blank -- so a swapped or
|
|
418
|
+
# collapsed channel axis fails rather than quietly segmenting the wrong one.
|
|
419
|
+
img = np.stack(
|
|
420
|
+
[
|
|
421
|
+
np.arange(4 * 8 * 8).reshape(4, 8, 8) % 5,
|
|
422
|
+
np.zeros((4, 8, 8), dtype=int),
|
|
423
|
+
]
|
|
424
|
+
)
|
|
425
|
+
assert img.shape == (2, 4, 8, 8)
|
|
426
|
+
|
|
427
|
+
tile_shape = (4, 4, 4)
|
|
428
|
+
stage = create_stage(tmp_path / "s.zarr", (4, 8, 8), tile_shape)
|
|
429
|
+
|
|
430
|
+
seen = {}
|
|
431
|
+
|
|
432
|
+
def fn(block):
|
|
433
|
+
seen["shape"] = block.shape
|
|
434
|
+
assert not block[1].any(), "channel 1 should be the blank one"
|
|
435
|
+
return (block[0] > 2).astype("int32")
|
|
436
|
+
|
|
437
|
+
n = stage_tile(
|
|
438
|
+
img,
|
|
439
|
+
fn,
|
|
440
|
+
stage,
|
|
441
|
+
0,
|
|
442
|
+
tile_shape=tile_shape,
|
|
443
|
+
overlap=1,
|
|
444
|
+
channel_axis=0,
|
|
445
|
+
)
|
|
446
|
+
|
|
447
|
+
# z is already the full extent, y/x grow by the 1-voxel halo.
|
|
448
|
+
assert seen["shape"] == (2, 4, 5, 5)
|
|
449
|
+
assert n == 1
|
|
450
|
+
staged = zarr.open_group(str(stage))["staged"][:]
|
|
451
|
+
assert staged.shape == (4, 8, 8) # store never learned about channels
|
|
452
|
+
assert staged[:, :4, :4].any() # the tile actually landed
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def test_cellpose_run_pairs_channels_and_shifts_z(monkeypatch):
|
|
456
|
+
"""A 2-channel tile must set z_axis past the channel axis.
|
|
457
|
+
|
|
458
|
+
With channels stacked on axis 0, z moves to axis 1; leaving z_axis=0 would
|
|
459
|
+
tell Cellpose the channel axis is z. Cellpose 3 also needs the pair named
|
|
460
|
+
as ``channels=[1, 2]`` (1-based, 0 = grayscale), which Cellpose 4 dropped.
|
|
461
|
+
"""
|
|
462
|
+
import numpy as np
|
|
463
|
+
|
|
464
|
+
from patchworks.plugins import cellpose as cp
|
|
465
|
+
|
|
466
|
+
calls = {}
|
|
467
|
+
|
|
468
|
+
class _FakeModel:
|
|
469
|
+
def eval(self, img, **kwargs):
|
|
470
|
+
calls["img_shape"] = img.shape
|
|
471
|
+
calls["kwargs"] = kwargs
|
|
472
|
+
return np.zeros(img.shape[1:], dtype="int32"), None
|
|
473
|
+
|
|
474
|
+
monkeypatch.setattr(cp, "_get_model", lambda _cfg: _FakeModel())
|
|
475
|
+
|
|
476
|
+
cfg = cp._make_config("cyto3", diameter=30, do_3D=True, channel_axis=0)
|
|
477
|
+
block = np.zeros((2, 4, 8, 8), dtype="uint16")
|
|
478
|
+
out = cp._run(block, cfg)
|
|
479
|
+
|
|
480
|
+
assert out.shape == (4, 8, 8) # labels come back spatial
|
|
481
|
+
assert calls["kwargs"]["z_axis"] == 1
|
|
482
|
+
assert calls["kwargs"]["channel_axis"] == 0
|
|
483
|
+
if cp._CELLPOSE_V4:
|
|
484
|
+
assert "channels" not in calls["kwargs"]
|
|
485
|
+
else:
|
|
486
|
+
assert calls["kwargs"]["channels"] == [1, 2]
|
|
487
|
+
|
|
488
|
+
# Single-channel tiles keep the old geometry and grayscale pairing.
|
|
489
|
+
cfg1 = cp._make_config("cyto3", diameter=30, do_3D=True)
|
|
490
|
+
cp._run(np.zeros((4, 8, 8), dtype="uint16"), cfg1)
|
|
491
|
+
assert calls["kwargs"]["z_axis"] == 0
|
|
492
|
+
assert calls["kwargs"]["channel_axis"] is None
|
|
493
|
+
if not cp._CELLPOSE_V4:
|
|
494
|
+
assert calls["kwargs"]["channels"] == [0, 0]
|
|
@@ -73,3 +73,38 @@ def test_log_tail_quotes_the_end_and_survives_a_missing_file(tmp_path):
|
|
|
73
73
|
assert "could not read" in log_tail(tmp_path / "nope.log")
|
|
74
74
|
(tmp_path / "empty.log").write_text("")
|
|
75
75
|
assert "empty" in log_tail(tmp_path / "empty.log")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_failing_step_reads_the_rule_from_snakemakes_log(tmp_path):
|
|
79
|
+
"""The failing rule must come from Snakemake, not from log timestamps.
|
|
80
|
+
|
|
81
|
+
Guessing by mtime named the wrong step in the most common case: a
|
|
82
|
+
`segment` failure leaves prepare.log the newest of the sequential steps,
|
|
83
|
+
so the failure mail quoted a log that had *succeeded*.
|
|
84
|
+
"""
|
|
85
|
+
from patchworks._notify import failing_step
|
|
86
|
+
|
|
87
|
+
log = tmp_path / "sm.log"
|
|
88
|
+
log.write_text(
|
|
89
|
+
"Building DAG of jobs...\n"
|
|
90
|
+
"[Wed Jul 30 11:17:20 2026]\n"
|
|
91
|
+
"Error in rule segment:\n"
|
|
92
|
+
" jobid: 812\n"
|
|
93
|
+
" output: seg/203.done\n"
|
|
94
|
+
" log: /w/nuclei_labels/logs/segment/203.log (check log file(s))\n"
|
|
95
|
+
)
|
|
96
|
+
assert failing_step(log) == (
|
|
97
|
+
"segment",
|
|
98
|
+
"/w/nuclei_labels/logs/segment/203.log",
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_failing_step_degrades_quietly(tmp_path):
|
|
103
|
+
"""It runs inside an error handler, so it must never raise itself."""
|
|
104
|
+
from patchworks._notify import failing_step
|
|
105
|
+
|
|
106
|
+
assert failing_step(None) == (None, None)
|
|
107
|
+
assert failing_step(tmp_path / "missing.log") == (None, None)
|
|
108
|
+
clean = tmp_path / "ok.log"
|
|
109
|
+
clean.write_text("Building DAG of jobs...\n3 of 3 steps (100%) done\n")
|
|
110
|
+
assert failing_step(clean) == (None, None)
|