patchworks 1.4.4__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {patchworks-1.4.4 → patchworks-2.1.0}/PKG-INFO +5 -2
- {patchworks-1.4.4 → patchworks-2.1.0}/README.md +4 -1
- patchworks-2.1.0/docs/api/chunks.md +19 -0
- patchworks-2.1.0/docs/api/io.md +19 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/api/plugins/ome_zarr.md +16 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/examples/dog.md +39 -9
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/guide/custom_segmentation.md +31 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/guide/gpu_distributed.md +25 -2
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/guide/merging.md +64 -12
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/guide/ome_zarr_napari.md +32 -3
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/guide/performance.md +12 -4
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/guide/skip_empty.md +47 -6
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/guide/snakemake.md +95 -27
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/guide/tiling.md +29 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/__init__.py +30 -4
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/_chunks.py +138 -18
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/_cluster.py +3 -2
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/_core.py +9 -8
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/_distributed.py +99 -13
- patchworks-2.1.0/src/patchworks/_gpu.py +158 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/_io.py +37 -3
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/_merge.py +415 -67
- patchworks-2.1.0/src/patchworks/_occupancy.py +400 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/_relations.py +3 -2
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/plugins/cellpose.py +57 -56
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/plugins/dog.py +129 -9
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/plugins/napari.py +4 -3
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/plugins/ome_zarr.py +261 -25
- patchworks-2.1.0/tests/test_allocation.py +78 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/tests/test_core.py +30 -0
- patchworks-2.1.0/tests/test_distributed.py +401 -0
- patchworks-2.1.0/tests/test_dog.py +109 -0
- patchworks-2.1.0/tests/test_gpu.py +93 -0
- patchworks-2.1.0/tests/test_occupancy.py +234 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/tests/test_ome_zarr.py +125 -4
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/README.md +22 -7
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/config/config.yaml +17 -4
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/config/config_cilia.yaml +18 -5
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/config/config_cyto.yaml +7 -1
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/config/config_nuclei.yaml +7 -1
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/profile/slurm/config.yaml +29 -11
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/rules/common.smk +9 -4
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/rules/merge.smk +2 -1
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/rules/segment.smk +9 -3
- patchworks-2.1.0/workflow/scripts/_pw.py +389 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/scripts/convert.py +10 -0
- patchworks-2.1.0/workflow/scripts/merge.py +120 -0
- patchworks-2.1.0/workflow/scripts/prepare_tiles.py +193 -0
- patchworks-2.1.0/workflow/scripts/run_multi.py +350 -0
- patchworks-2.1.0/workflow/scripts/segment_tile.py +55 -0
- patchworks-1.4.4/docs/api/chunks.md +0 -5
- patchworks-1.4.4/docs/api/io.md +0 -5
- patchworks-1.4.4/tests/test_distributed.py +0 -75
- patchworks-1.4.4/tests/test_dog.py +0 -49
- patchworks-1.4.4/workflow/scripts/_pw.py +0 -219
- patchworks-1.4.4/workflow/scripts/merge.py +0 -58
- patchworks-1.4.4/workflow/scripts/prepare_tiles.py +0 -64
- patchworks-1.4.4/workflow/scripts/run_multi.py +0 -194
- patchworks-1.4.4/workflow/scripts/segment_tile.py +0 -30
- {patchworks-1.4.4 → patchworks-2.1.0}/.github/workflows/docs.yml +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/.github/workflows/lint.yml +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/.github/workflows/release.yml +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/.gitignore +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/.markdownlint-cli2.yaml +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/LICENSE +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/cliff.toml +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/api/cluster.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/api/merge_tile_labels.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/api/plugins/cellpose.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/api/plugins/dog.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/api/plugins/napari.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/api/postprocess.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/api/relabel.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/api/tile_process.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/assets/logo.png +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/examples/cellpose_2d.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/examples/cellpose_2d.py +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/examples/cellpose_3d.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/examples/cellpose_3d.py +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/examples/custom.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/examples/custom_method.py +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/examples/dog.py +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/examples/standalone_merge.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/examples/stardist.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/examples/stardist_2d.py +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/getting_started.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/guide/label_relations.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/guide/measurements.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/guide/pitfalls.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/docs/index.md +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/mkdocs.yml +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/pyproject.toml +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/_postprocess.py +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/_relabel.py +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/src/patchworks/plugins/__init__.py +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/tests/test_napari.py +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/tests/test_postprocess.py +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/tests/test_relations.py +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/Snakefile +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/config/multi.yaml +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/pixi.toml +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/rules/convert.smk +0 -0
- {patchworks-1.4.4 → patchworks-2.1.0}/workflow/scripts/fetch_model.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: patchworks
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Tiled processing of arbitrarily large images with globally consistent labels
|
|
5
5
|
Project-URL: Homepage, https://github.com/imcf/patchworks
|
|
6
6
|
Project-URL: Issues, https://github.com/imcf/patchworks/issues
|
|
@@ -376,9 +376,12 @@ tiles where the dask-image approach stalls.
|
|
|
376
376
|
| ------------------------------ | ----------------------------------------- | ------------------------------------------------------------- |
|
|
377
377
|
| In-process Dask client | `FutureCancelledError: lost dependencies` | Detected at startup, raises immediately with fix instructions |
|
|
378
378
|
| 3-4× fn recompute during merge | Cellpose runs 3× per tile | Staging writes labels once, merge reads from disk |
|
|
379
|
-
| O(n²) sequential relabelling | Graph construction hangs at 1000+ tiles |
|
|
379
|
+
| O(n²) sequential relabelling | Graph construction hangs at 1000+ tiles | Folded into the merge's own LUT — no extra pass over the volume |
|
|
380
380
|
| Wrong overlap boundary | Output shape mismatch | Always uses `boundary="none"` |
|
|
381
381
|
| Persisting large arrays | Worker OOM | Never persists; keeps dask graph lazy and streams |
|
|
382
|
+
| Sizing work to the whole node | Job OOM-killed on a shared cluster node | Reads the SLURM/cgroup allocation, not `os.cpu_count()` |
|
|
383
|
+
| Rechunking a pyramid level | Threaded scheduler stockpiles intermediates | Levels stream one source chunk per task, bounded by construction |
|
|
384
|
+
| Isotropic halo on flat tiles | 5× the voxels read and segmented, then trimmed | `overlap` takes one width per axis |
|
|
382
385
|
|
|
383
386
|
---
|
|
384
387
|
|
|
@@ -286,9 +286,12 @@ tiles where the dask-image approach stalls.
|
|
|
286
286
|
| ------------------------------ | ----------------------------------------- | ------------------------------------------------------------- |
|
|
287
287
|
| In-process Dask client | `FutureCancelledError: lost dependencies` | Detected at startup, raises immediately with fix instructions |
|
|
288
288
|
| 3-4× fn recompute during merge | Cellpose runs 3× per tile | Staging writes labels once, merge reads from disk |
|
|
289
|
-
| O(n²) sequential relabelling | Graph construction hangs at 1000+ tiles |
|
|
289
|
+
| O(n²) sequential relabelling | Graph construction hangs at 1000+ tiles | Folded into the merge's own LUT — no extra pass over the volume |
|
|
290
290
|
| Wrong overlap boundary | Output shape mismatch | Always uses `boundary="none"` |
|
|
291
291
|
| Persisting large arrays | Worker OOM | Never persists; keeps dask graph lazy and streams |
|
|
292
|
+
| Sizing work to the whole node | Job OOM-killed on a shared cluster node | Reads the SLURM/cgroup allocation, not `os.cpu_count()` |
|
|
293
|
+
| Rechunking a pyramid level | Threaded scheduler stockpiles intermediates | Levels stream one source chunk per task, bounded by construction |
|
|
294
|
+
| Isotropic halo on flat tiles | 5× the voxels read and segmented, then trimmed | `overlap` takes one width per axis |
|
|
292
295
|
|
|
293
296
|
---
|
|
294
297
|
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Tile sizing
|
|
2
|
+
|
|
3
|
+
::: patchworks.auto_tile_shape
|
|
4
|
+
|
|
5
|
+
::: patchworks.auto_tile_shape_cellpose
|
|
6
|
+
|
|
7
|
+
::: patchworks.auto_overlap
|
|
8
|
+
|
|
9
|
+
::: patchworks.normalize_overlap
|
|
10
|
+
|
|
11
|
+
## Cluster resource detection
|
|
12
|
+
|
|
13
|
+
On a shared node the machine's core count and free RAM say nothing about what
|
|
14
|
+
this job was granted. These read the allocation instead, and everything that
|
|
15
|
+
sizes a worker pool goes through them.
|
|
16
|
+
|
|
17
|
+
::: patchworks.cpu_allocation
|
|
18
|
+
|
|
19
|
+
::: patchworks.safe_worker_count
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# I/O helpers
|
|
2
|
+
|
|
3
|
+
::: patchworks.load_ome_zarr
|
|
4
|
+
|
|
5
|
+
## Deciding which tiles hold signal
|
|
6
|
+
|
|
7
|
+
`estimate_empty_tiles` is a fast **preview** — it samples a centred window per
|
|
8
|
+
tile, so it can miss signal at a tile's edge. `build_occupancy_map` +
|
|
9
|
+
`tile_occupancy` are **exact**: a brick maximum exceeds the threshold exactly
|
|
10
|
+
when some voxel in that brick does. Use the latter pair when the result is
|
|
11
|
+
used as a skip list. See [Skipping empty tiles](../guide/skip_empty.md).
|
|
12
|
+
|
|
13
|
+
::: patchworks.estimate_empty_tiles
|
|
14
|
+
|
|
15
|
+
::: patchworks.build_occupancy_map
|
|
16
|
+
|
|
17
|
+
::: patchworks.tile_occupancy
|
|
18
|
+
|
|
19
|
+
::: patchworks.auto_empty_threshold
|
|
@@ -24,3 +24,19 @@ resolution, matching anisotropic microscopy stacks.
|
|
|
24
24
|
## register_labels
|
|
25
25
|
|
|
26
26
|
::: patchworks.plugins.ome_zarr.register_labels
|
|
27
|
+
|
|
28
|
+
## read_pixel_size
|
|
29
|
+
|
|
30
|
+
::: patchworks.plugins.ome_zarr.read_pixel_size
|
|
31
|
+
|
|
32
|
+
## NGFF metadata layout
|
|
33
|
+
|
|
34
|
+
NGFF 0.4 is defined over zarr v2 and puts its keys at the top level; 0.5 is
|
|
35
|
+
the zarr-v3 revision and nests them under `ome`. patchworks writes whichever
|
|
36
|
+
matches the store, and reads both.
|
|
37
|
+
|
|
38
|
+
::: patchworks.plugins.ome_zarr.ngff_version
|
|
39
|
+
|
|
40
|
+
::: patchworks.plugins.ome_zarr.read_ngff_attr
|
|
41
|
+
|
|
42
|
+
::: patchworks.plugins.ome_zarr.write_ngff_attrs
|
|
@@ -65,18 +65,36 @@ Requires `cupy` (matching your CUDA version, e.g. `pip install cupy-cuda12x`)
|
|
|
65
65
|
|
|
66
66
|
## With deconvolution first
|
|
67
67
|
|
|
68
|
+
Let the voxel sizes come from the image rather than retyping them — the
|
|
69
|
+
lateral ones from X/Y, the axial ones from Z:
|
|
70
|
+
|
|
68
71
|
```python
|
|
72
|
+
from patchworks.plugins.ome_zarr import read_pixel_size
|
|
73
|
+
|
|
69
74
|
fn = dog_label_fn(
|
|
70
75
|
low_sigma=1.0, high_sigma=3.0, threshold=0.02,
|
|
71
|
-
decon_kwargs=dict(
|
|
72
|
-
|
|
73
|
-
dzpsf=z_scale, dzdata=z_scale,
|
|
74
|
-
wavelength=525, na=1.4, nimm=1.515,
|
|
75
|
-
),
|
|
76
|
+
decon_kwargs=dict(psf=psf, wavelength=525, na=1.4, nimm=1.515),
|
|
77
|
+
voxel_size=read_pixel_size(IMAGE), # -> dxdata/dzdata/dxpsf/dzpsf
|
|
76
78
|
)
|
|
77
79
|
result = tile_process(IMAGE, fn, tile_shape=(1, 1024, 1024), overlap=32)
|
|
78
80
|
```
|
|
79
81
|
|
|
82
|
+
Anything you set in `decon_kwargs` yourself wins, so a PSF sampled
|
|
83
|
+
differently from the data keeps its own sizes:
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
decon_kwargs=dict(psf=psf, dxpsf=0.05, dzpsf=0.1, wavelength=525, ...)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
!!! warning "A wrong voxel size does not fail loudly"
|
|
90
|
+
Deconvolution given the wrong sampling still runs and still returns an
|
|
91
|
+
image — just a subtly wrong one. That is the reason to derive these from
|
|
92
|
+
the store's own calibration instead of keeping a second copy in a config
|
|
93
|
+
that can drift.
|
|
94
|
+
|
|
95
|
+
`read_pixel_size` returns `{}` for an uncalibrated store; then nothing is
|
|
96
|
+
filled in and you must supply the sizes yourself.
|
|
97
|
+
|
|
80
98
|
!!! note "Deconvolution always needs a GPU"
|
|
81
99
|
`pycudadecon` is CUDA-only, independent of `dog_label_fn`'s own `use_gpu`
|
|
82
100
|
flag (which only picks the backend for the blur/label steps). A SLURM job
|
|
@@ -146,15 +164,27 @@ custom:
|
|
|
146
164
|
threshold: 0.02
|
|
147
165
|
decon_kwargs:
|
|
148
166
|
psf: "/path/to/psf.tif"
|
|
149
|
-
dxpsf: 0.1
|
|
150
|
-
dxdata: 0.1
|
|
151
|
-
dzpsf: 0.2
|
|
152
|
-
dzdata: 0.2
|
|
153
167
|
wavelength: 525
|
|
154
168
|
na: 1.4
|
|
155
169
|
nimm: 1.515
|
|
156
170
|
```
|
|
157
171
|
|
|
172
|
+
No voxel sizes: the workflow reads `image.zarr`'s own calibration and fills
|
|
173
|
+
in `dxdata`/`dxpsf` from X/Y and `dzdata`/`dzpsf` from Z. Set any of them in
|
|
174
|
+
`decon_kwargs` to override — for instance a PSF sampled finer than the data:
|
|
175
|
+
|
|
176
|
+
```yaml
|
|
177
|
+
dxpsf: 0.05
|
|
178
|
+
dzpsf: 0.1
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
!!! tip "This works for your own methods too"
|
|
182
|
+
The injection is not DoG-specific. Any `custom` function that declares a
|
|
183
|
+
`voxel_size` parameter receives `{"z": .., "y": .., "x": ..}` from the
|
|
184
|
+
store, so a method needing physical units never has to keep a second copy
|
|
185
|
+
of the calibration in its config. If the store is uncalibrated the
|
|
186
|
+
workflow says so and passes nothing.
|
|
187
|
+
|
|
158
188
|
Run it exactly like a Cellpose config:
|
|
159
189
|
|
|
160
190
|
```bash
|
|
@@ -118,6 +118,34 @@ def segment(tile: np.ndarray, prob_thresh: float = 0.5) -> np.ndarray:
|
|
|
118
118
|
|
|
119
119
|
Using a GPU? Just let your framework see it — nothing extra needed.
|
|
120
120
|
|
|
121
|
+
!!! tip "Ask for the voxel size instead of configuring it"
|
|
122
|
+
Declare a `voxel_size` parameter and the workflow passes the image's own
|
|
123
|
+
NGFF calibration, `{"z": .., "y": .., "x": ..}` in micrometers:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
def segment(tile, *, voxel_size=None, min_diameter_um=5.0):
|
|
127
|
+
min_px = min_diameter_um / voxel_size["x"] # µm -> pixels
|
|
128
|
+
...
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
That keeps physical parameters tied to the image rather than duplicated
|
|
132
|
+
in a config that can drift away from it. Setting `voxel_size` in
|
|
133
|
+
`custom.kwargs` overrides it; an uncalibrated store passes nothing and
|
|
134
|
+
logs a warning. This is how the DoG plugin gets its deconvolution voxel
|
|
135
|
+
sizes.
|
|
136
|
+
|
|
137
|
+
!!! tip "You do not need to modify patchworks"
|
|
138
|
+
`method: "custom"` imports any `(tile) -> labels` callable, so a new
|
|
139
|
+
method is a module of your own and a config block — nothing in the
|
|
140
|
+
package changes. It then inherits everything the pipeline does: empty
|
|
141
|
+
tiles are skipped, tiles are batched per job, labels are stitched across
|
|
142
|
+
boundaries and renumbered, and the result is written as a calibrated
|
|
143
|
+
pyramid.
|
|
144
|
+
|
|
145
|
+
(The `KNOWN_METHODS` list in `workflow/scripts/_pw.py` is only for
|
|
146
|
+
*built-in* shortcuts like `"cellpose"`. Adding to it is for methods that
|
|
147
|
+
ship with patchworks, not for your own.)
|
|
148
|
+
|
|
121
149
|
## Test it before you submit
|
|
122
150
|
|
|
123
151
|
Run your function on one real tile first — it catches shape/dtype bugs in
|
|
@@ -148,6 +176,9 @@ custom:
|
|
|
148
176
|
module: "my_seg" # import name
|
|
149
177
|
function: "segment" # default is "segment"
|
|
150
178
|
kwargs: # optional — forwarded as segment(tile, **kwargs)
|
|
179
|
+
# checked against your function's signature during
|
|
180
|
+
# `prepare`, so a typo fails on a cheap CPU job
|
|
181
|
+
# rather than in the first GPU job hours later
|
|
151
182
|
sigma: 1.5
|
|
152
183
|
```
|
|
153
184
|
|
|
@@ -73,8 +73,16 @@ FutureCancelledError: lost dependencies
|
|
|
73
73
|
## GPU memory sizing
|
|
74
74
|
|
|
75
75
|
When `use_gpu=True`, patchworks queries free GPU VRAM via `nvidia-ml-py`
|
|
76
|
-
(install: `pip install "patchworks[gpu]"`)
|
|
77
|
-
|
|
76
|
+
(install: `pip install "patchworks[gpu]"`) and keeps 20% of it as headroom,
|
|
77
|
+
because `info.free` is a point-in-time reading of a device you usually do not
|
|
78
|
+
own outright. `auto_tile_shape` then sizes each tile to at most half of that
|
|
79
|
+
budget; `auto_tile_shape_cellpose` uses Cellpose's own memory model instead
|
|
80
|
+
(roughly 20× the raw tile bytes, plus ~2 GiB for the model).
|
|
81
|
+
|
|
82
|
+
The device is resolved from `CUDA_VISIBLE_DEVICES`. This matters on
|
|
83
|
+
multi-GPU nodes: NVML enumerates **every** GPU regardless of `--gres=gpu:1`,
|
|
84
|
+
so querying index 0 unconditionally would read a different card's free memory
|
|
85
|
+
than the one your job was granted.
|
|
78
86
|
|
|
79
87
|
Without `nvidia-ml-py`, a conservative 8 GiB default is used with a warning.
|
|
80
88
|
Install it for accurate sizing on large-VRAM cards (A100, H100):
|
|
@@ -82,3 +90,18 @@ Install it for accurate sizing on large-VRAM cards (A100, H100):
|
|
|
82
90
|
```bash
|
|
83
91
|
pip install "patchworks[gpu]"
|
|
84
92
|
```
|
|
93
|
+
|
|
94
|
+
!!! tip "Sizing for a GPU you can't see"
|
|
95
|
+
The Snakemake workflow plans tiles in `prepare`, which runs on a CPU
|
|
96
|
+
node, so no GPU can be queried there. Set `gpu_memory_gb` in the config
|
|
97
|
+
to the segment GPU's VRAM (e.g. `24` for an RTX 4090) instead of relying
|
|
98
|
+
on the fallback.
|
|
99
|
+
|
|
100
|
+
## Surviving a shared GPU
|
|
101
|
+
|
|
102
|
+
An out-of-memory error on a shared device is often transient — a co-tenant
|
|
103
|
+
job grew, not a tile that doesn't fit. patchworks retries on the GPU with a
|
|
104
|
+
backoff rather than falling back to the CPU, where a single tile can take
|
|
105
|
+
over an hour. Before each backoff it releases its own device memory
|
|
106
|
+
(including any cached Cellpose model), since holding that is exactly what
|
|
107
|
+
starves the other job. Both torch and cupy OOM errors are recognised.
|
|
@@ -28,9 +28,9 @@ This is the same approach used by
|
|
|
28
28
|
|
|
29
29
|
### Step 1: stage
|
|
30
30
|
|
|
31
|
-
Each tile's labels are written to
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
Each tile's labels are written to zarr once. This is critical: without
|
|
32
|
+
staging, any downstream operation that reads the label array re-runs your
|
|
33
|
+
segmentation function. The merge internally reads labels multiple times.
|
|
34
34
|
|
|
35
35
|
```text
|
|
36
36
|
tile_process calls fn once per tile → staged zarr
|
|
@@ -38,14 +38,37 @@ tile_process calls fn once per tile → staged zarr
|
|
|
38
38
|
merge reads from staged zarr (no fn calls)
|
|
39
39
|
```
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
The Snakemake workflow goes further and stages **directly into**
|
|
42
|
+
`image.zarr/labels/<name>/0`, then has the merge rewrite that array in place
|
|
43
|
+
— saving a whole extra write of the volume plus the scratch store's disk. It
|
|
44
|
+
falls back to a separate store when the tile is larger than the label chunk
|
|
45
|
+
cap, since in place the chunking cannot be changed and level 0 has to stay
|
|
46
|
+
pageable for a viewer.
|
|
47
|
+
|
|
48
|
+
### Step 2: make the ids globally unique
|
|
49
|
+
|
|
50
|
+
Tiles write local `1..n`, which collide, so the boundary scan could not
|
|
51
|
+
otherwise tell two different objects apart. If each tile's label **count** is
|
|
52
|
+
known, this is just an exclusive cumulative sum — global id is
|
|
53
|
+
`offset[tile] + local`, computed in `O(n_tiles)` with no read of the volume
|
|
54
|
+
at all. `stage_tile` returns that count for exactly this purpose; pass the
|
|
55
|
+
counts as `label_counts=`.
|
|
56
|
+
|
|
57
|
+
Without counts, the merge falls back to streaming every chunk and renumbering
|
|
58
|
+
it in place — correct, but a full read **and write** of the volume.
|
|
59
|
+
|
|
60
|
+
### Step 3: boundary scan
|
|
42
61
|
|
|
43
62
|
Only the two voxels on either side of each tile boundary are read. For any
|
|
44
|
-
pair of touching non-zero labels `(a, b)`, they must be the same object.
|
|
63
|
+
pair of touching non-zero labels `(a, b)`, they must be the same object. The
|
|
64
|
+
per-tile offsets are applied here, on the fly.
|
|
45
65
|
|
|
46
|
-
I/O cost: `O(n_boundaries × face_area)`, not `O(full_volume)`.
|
|
66
|
+
I/O cost: `O(n_boundaries × face_area)`, not `O(full_volume)`. The columns
|
|
67
|
+
are read in parallel, and a boundary next to a chunk that holds no labels is
|
|
68
|
+
skipped outright — a pair needs a non-zero label on *both* sides, so it could
|
|
69
|
+
never produce one.
|
|
47
70
|
|
|
48
|
-
### Step
|
|
71
|
+
### Step 4: connected components
|
|
49
72
|
|
|
50
73
|
scipy sparse connected components on the touching pairs produces a relabeling
|
|
51
74
|
lookup table. All labels that transitively touch each other are mapped to the
|
|
@@ -53,12 +76,38 @@ same canonical label.
|
|
|
53
76
|
|
|
54
77
|
Cost: `O(n_touching_pairs)`.
|
|
55
78
|
|
|
56
|
-
|
|
79
|
+
With `sequential_labels=True` the contiguous renumbering is folded into this
|
|
80
|
+
same LUT. Because the id domain is dense by construction, the surviving ids
|
|
81
|
+
are exactly the distinct LUT values — a `np.unique` over an array the length
|
|
82
|
+
of the object count, with no scan of the volume.
|
|
83
|
+
|
|
84
|
+
### Step 5: parallel relabel
|
|
57
85
|
|
|
58
86
|
The LUT is applied to every tile in parallel via `multiprocessing.Pool`. The
|
|
59
87
|
LUT is shared via process initializer to avoid re-pickling it for every chunk
|
|
60
88
|
(LUTs can be hundreds of MB for dense label volumes).
|
|
61
89
|
|
|
90
|
+
Chunks whose tile wrote no labels are skipped entirely — not read, and not
|
|
91
|
+
written. Zarr never materialises an unwritten chunk and reads it back as the
|
|
92
|
+
fill value, so background regions cost neither I/O nor disk. On a sparse
|
|
93
|
+
image that is most of the volume.
|
|
94
|
+
|
|
95
|
+
When the merge's output *is* its input, this pass rewrites the array in
|
|
96
|
+
place. That is safe because the boundary scan (step 3) has already finished,
|
|
97
|
+
so nothing still needs the original ids.
|
|
98
|
+
|
|
99
|
+
Because an in-place merge destroys its own input, it records how far it got
|
|
100
|
+
on the array itself, and refuses to guess on a re-run:
|
|
101
|
+
|
|
102
|
+
| State found | What happens |
|
|
103
|
+
| --- | --- |
|
|
104
|
+
| nothing recorded | fresh tile-local ids — merge normally |
|
|
105
|
+
| `running` | a previous attempt died mid-relabel, so the array is part local and part global. **Refuses**: re-segment to rebuild it. |
|
|
106
|
+
| `done` | already merged — a no-op, so a failure *after* the relabel (the pyramid, say) can simply be retried |
|
|
107
|
+
|
|
108
|
+
Without that, a second pass would add each tile's offset to ids that are
|
|
109
|
+
already global, which can land two unrelated objects on the same id.
|
|
110
|
+
|
|
62
111
|
## Using the merge step standalone
|
|
63
112
|
|
|
64
113
|
You can call the merge step directly on any existing label array or zarr:
|
|
@@ -86,9 +135,10 @@ merged = merge_tile_labels(
|
|
|
86
135
|
|
|
87
136
|
## Sequential label numbering
|
|
88
137
|
|
|
89
|
-
By default, merged labels are globally unique but may be **gappy**
|
|
90
|
-
|
|
91
|
-
counting, `regionprops`, and measurement — the IDs just aren't
|
|
138
|
+
By default, merged labels are globally unique but may be **gappy** — boundary
|
|
139
|
+
merging fuses ids, leaving holes where the absorbed ones were. This is fine
|
|
140
|
+
for counting, `regionprops`, and measurement — the IDs just aren't
|
|
141
|
+
consecutive.
|
|
92
142
|
|
|
93
143
|
For contiguous 1..N numbering, use `sequential_labels=True`:
|
|
94
144
|
|
|
@@ -96,7 +146,9 @@ For contiguous 1..N numbering, use `sequential_labels=True`:
|
|
|
96
146
|
tile_process("image.zarr", fn, write_to="labels.zarr", sequential_labels=True)
|
|
97
147
|
```
|
|
98
148
|
|
|
99
|
-
This
|
|
149
|
+
This is free: it composes into the relabel LUT the merge already applies, so
|
|
150
|
+
it costs a `np.unique` over the object count rather than another pass over
|
|
151
|
+
the volume.
|
|
100
152
|
|
|
101
153
|
!!! warning "Do not use dask's built-in sequential relabel"
|
|
102
154
|
`dask_image.ndmeasure.merge_labels_across_chunk_boundaries` has a
|
|
@@ -22,9 +22,23 @@ natively. Pass `write_to="other.zarr"` to instead write a separate
|
|
|
22
22
|
single-resolution label store, or `output_component="cells"` to name the label
|
|
23
23
|
image.
|
|
24
24
|
|
|
25
|
-
The label pyramid is
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
The label pyramid is streamed chunk by chunk, so it stays OOM-safe even for
|
|
26
|
+
terabyte volumes. Control it with `pyramid_levels` and `pyramid_downscale`.
|
|
27
|
+
|
|
28
|
+
!!! note "NGFF version"
|
|
29
|
+
The metadata layout follows the zarr format being written: **NGFF 0.5**
|
|
30
|
+
(keys nested under an `ome` attribute) on zarr v3, **0.4** (keys at the
|
|
31
|
+
top level) on v2. Writing v3 data with 0.4's top-level layout — which
|
|
32
|
+
earlier versions did — matches neither revision, and a strict 0.5 reader
|
|
33
|
+
finds nothing there.
|
|
34
|
+
|
|
35
|
+
Reading accepts **both** layouts, so stores written by any patchworks
|
|
36
|
+
version still load. Use `read_ngff_attr()` if you need to inspect the
|
|
37
|
+
metadata yourself rather than indexing `attrs["multiscales"]` directly.
|
|
38
|
+
|
|
39
|
+
patchworks' own hints (`n_objects`, `sequential_labels`) are not NGFF
|
|
40
|
+
keys, so they stay at the top level where a consumer finds them without
|
|
41
|
+
knowing the layout.
|
|
28
42
|
|
|
29
43
|
## Why a pyramid?
|
|
30
44
|
|
|
@@ -107,6 +121,21 @@ input — see [Cluster usage](snakemake.md).
|
|
|
107
121
|
to_ome_zarr("scan.ims", "scan.zarr", reuse_pyramid=True)
|
|
108
122
|
```
|
|
109
123
|
|
|
124
|
+
### Axis names
|
|
125
|
+
|
|
126
|
+
For a file, axes come from the reader's own dimension metadata (bioio's
|
|
127
|
+
`dims.order`, Imaris, or the `sequence_pattern` groups) — not guessed. Only
|
|
128
|
+
non-spatial singleton axes are dropped, so a genuine `z=1` survives.
|
|
129
|
+
|
|
130
|
+
For a **bare array** there is nothing to read, so axes are inferred from the
|
|
131
|
+
number of dimensions (`yx`, `zyx`, `czyx`, `tczyx`). That is unambiguous up to
|
|
132
|
+
3-D; from 4-D the leading axis could be channel or time, and patchworks logs a
|
|
133
|
+
warning saying what it guessed. Pass `axes=` to settle it:
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
to_ome_zarr(arr, "out.zarr", axes="tzyx") # not the czyx it would assume
|
|
137
|
+
```
|
|
138
|
+
|
|
110
139
|
### Pixel calibration
|
|
111
140
|
|
|
112
141
|
The physical voxel size is read from the input — bioio's `physical_pixel_sizes`,
|
|
@@ -3,10 +3,10 @@
|
|
|
3
3
|
`tile_process` is built so a run **adapts to whatever machine it lands on** and
|
|
4
4
|
can't run out of RAM/VRAM or freeze the box — without you tuning anything.
|
|
5
5
|
|
|
6
|
-
## Automatic,
|
|
6
|
+
## Automatic, allocation-aware concurrency
|
|
7
7
|
|
|
8
8
|
The staging step (running your `fn` once per tile to a temp store) and the
|
|
9
|
-
merge step are sized
|
|
9
|
+
merge step are sized automatically:
|
|
10
10
|
|
|
11
11
|
- **GPU** (`use_gpu=True`) → **one tile at a time**, so concurrent evaluations
|
|
12
12
|
can never exhaust VRAM.
|
|
@@ -14,8 +14,16 @@ merge step are sized to the host automatically:
|
|
|
14
14
|
from the tile size), and always **leaving one core free** so the machine
|
|
15
15
|
stays responsive — it never pins every core.
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
"Available" means available **to this process**, not to the machine. On a
|
|
18
|
+
shared cluster node those differ wildly — a 32-core, 128 GB job on a 128-core,
|
|
19
|
+
512 GB node would otherwise size itself for the whole box and get OOM-killed
|
|
20
|
+
while its own accounting said it had room. patchworks takes the smallest of
|
|
21
|
+
`SLURM_MEM_PER_NODE`, `SLURM_MEM_PER_CPU × cpus`, the cgroup limit (the one
|
|
22
|
+
that actually triggers the kill) and `psutil`'s free RAM, and reads the core
|
|
23
|
+
count from `SLURM_CPUS_PER_TASK` or the process' CPU affinity mask.
|
|
24
|
+
|
|
25
|
+
Without any of those signals, a conservative default is used instead of
|
|
26
|
+
guessing high.
|
|
19
27
|
|
|
20
28
|
## Live progress dashboard (GPU runs)
|
|
21
29
|
|
|
@@ -37,7 +37,9 @@ tile_process(
|
|
|
37
37
|
)
|
|
38
38
|
```
|
|
39
39
|
|
|
40
|
-
##
|
|
40
|
+
## Two ways to decide
|
|
41
|
+
|
|
42
|
+
### `estimate_empty_tiles` — a fast preview
|
|
41
43
|
|
|
42
44
|
For each tile in the grid, only a small centred **sample window** is read
|
|
43
45
|
(default: 24×256×256 voxels). If the maximum value in that window exceeds
|
|
@@ -47,11 +49,50 @@ This is **bounded I/O**: the total data read is `n_tiles × sample_window`,
|
|
|
47
49
|
not the full image. For a 2200-tile image with the default window, this reads
|
|
48
50
|
≈ 30 MB instead of 250 GB — and it runs in seconds.
|
|
49
51
|
|
|
50
|
-
!!! warning "Approximate"
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
the **full tile** max
|
|
54
|
-
|
|
52
|
+
!!! warning "Approximate — a preview, not a skip list"
|
|
53
|
+
Only the tile centre is inspected. On a `(16, 1024, 1024)` tile the
|
|
54
|
+
default window covers **6.25% of the tile's area**, so an object in the
|
|
55
|
+
outer ring is invisible. `tile_process` re-tests the **full tile** max
|
|
56
|
+
inline, so nothing is dropped in a real run — but anything that uses this
|
|
57
|
+
result *as* the skip list would drop those tiles for good.
|
|
58
|
+
|
|
59
|
+
### `build_occupancy_map` + `tile_occupancy` — exact
|
|
60
|
+
|
|
61
|
+
Reduces every `block`-sized brick of the image to its **maximum** and stores
|
|
62
|
+
that beside `image.zarr` (about 1/16384 of the image at the default 128 px
|
|
63
|
+
block). `tile_occupancy` then reduces the map over each tile's full footprint.
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from patchworks import (
|
|
67
|
+
auto_empty_threshold, block_for_tile, build_occupancy_map, tile_occupancy
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
# Size the block from the tile, or every tile over-covers the same block and
|
|
71
|
+
# tests occupied — correct, but useless as a skip list.
|
|
72
|
+
build_occupancy_map("image.zarr", block=block_for_tile(TILE))
|
|
73
|
+
info = tile_occupancy(
|
|
74
|
+
"image.zarr", TILE, channel=0, threshold=auto_empty_threshold(img, 0, 0)
|
|
75
|
+
)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
The map is stored **beside** the image (`image.occupancy.zarr`), not inside
|
|
79
|
+
it: it is not an NGFF array, and a zarr hierarchy containing one cannot be
|
|
80
|
+
walked. It is rebuilt automatically if an existing map was built at a
|
|
81
|
+
different block.
|
|
82
|
+
|
|
83
|
+
This is **exact, not approximate**: `block_max > threshold` is true exactly
|
|
84
|
+
when some voxel in that block exceeds the threshold, so comparing pooled
|
|
85
|
+
maxima against a threshold derived from raw voxels answers the same question
|
|
86
|
+
as scanning every voxel. Max-pooling cannot lose a bright voxel.
|
|
87
|
+
|
|
88
|
+
The map is built once and shared by every segmentation reading that image, so
|
|
89
|
+
a three-config run pays for one pooling pass instead of three sampling passes.
|
|
90
|
+
This is what the Snakemake workflow uses, and it builds the map on first use
|
|
91
|
+
for stores converted before the map existed.
|
|
92
|
+
|
|
93
|
+
Skipping a tile pays off twice over: it is never segmented, **and** the merge
|
|
94
|
+
skips its chunk too — neither reading it nor writing zeros over it, so the
|
|
95
|
+
background never reaches disk at all.
|
|
55
96
|
|
|
56
97
|
## Threshold selection
|
|
57
98
|
|