patchworks 0.4.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {patchworks-0.4.0 → patchworks-0.6.0}/.github/workflows/docs.yml +4 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/.github/workflows/lint.yml +2 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/.github/workflows/release.yml +14 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/PKG-INFO +51 -12
- {patchworks-0.4.0 → patchworks-0.6.0}/README.md +47 -11
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/custom.md +3 -3
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/custom_method.py +1 -2
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/getting_started.md +5 -3
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/guide/ome_zarr_napari.md +21 -7
- patchworks-0.6.0/docs/guide/performance.md +60 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/index.md +1 -1
- {patchworks-0.4.0 → patchworks-0.6.0}/mkdocs.yml +4 -1
- {patchworks-0.4.0 → patchworks-0.6.0}/pyproject.toml +3 -1
- {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/_chunks.py +45 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/_core.py +48 -42
- {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/_io.py +9 -2
- patchworks-0.6.0/src/patchworks/plugins/ome_zarr.py +570 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/tests/test_core.py +24 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/tests/test_ome_zarr.py +35 -0
- patchworks-0.4.0/src/patchworks/plugins/ome_zarr.py +0 -459
- {patchworks-0.4.0 → patchworks-0.6.0}/.gitignore +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/cliff.toml +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/chunks.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/cluster.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/io.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/merge_tile_labels.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/plugins/cellpose.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/plugins/napari.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/plugins/ome_zarr.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/relabel.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/tile_process.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/cellpose_2d.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/cellpose_2d.py +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/cellpose_3d.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/cellpose_3d.py +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/standalone_merge.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/stardist.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/stardist_2d.py +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/guide/gpu_distributed.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/guide/merging.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/guide/pitfalls.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/guide/skip_empty.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/docs/guide/tiling.md +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/__init__.py +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/_cluster.py +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/_merge.py +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/_relabel.py +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/plugins/__init__.py +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/plugins/cellpose.py +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/plugins/napari.py +0 -0
- {patchworks-0.4.0 → patchworks-0.6.0}/tests/test_napari.py +0 -0
|
@@ -48,3 +48,17 @@ jobs:
|
|
|
48
48
|
|
|
49
49
|
- name: Publish to PyPI
|
|
50
50
|
uses: pypa/gh-action-pypi-publish@release/v1
|
|
51
|
+
|
|
52
|
+
# Rebuild the org-wide pdoc apidocs site so it picks up the new version.
|
|
53
|
+
apidocs:
|
|
54
|
+
needs: release
|
|
55
|
+
runs-on: ubuntu-latest
|
|
56
|
+
steps:
|
|
57
|
+
- name: Trigger imcf.github.io apidocs rebuild
|
|
58
|
+
uses: peter-evans/repository-dispatch@v3
|
|
59
|
+
with:
|
|
60
|
+
# Fine-grained PAT with "Contents: write" on imcf/imcf.github.io,
|
|
61
|
+
# stored as the APIDOCS_DISPATCH_TOKEN secret in this repo.
|
|
62
|
+
token: ${{ secrets.APIDOCS_DISPATCH_TOKEN }}
|
|
63
|
+
repository: imcf/imcf.github.io
|
|
64
|
+
event-type: dispatch-event
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: patchworks
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Tiled processing of arbitrarily large images with globally consistent labels
|
|
5
5
|
Project-URL: Homepage, https://github.com/imcf/patchworks
|
|
6
6
|
Project-URL: Issues, https://github.com/imcf/patchworks/issues
|
|
@@ -29,6 +29,7 @@ Requires-Dist: bioio-lif; extra == 'all'
|
|
|
29
29
|
Requires-Dist: bioio-nd2; extra == 'all'
|
|
30
30
|
Requires-Dist: bioio-ome-tiff; extra == 'all'
|
|
31
31
|
Requires-Dist: bioio-tifffile; extra == 'all'
|
|
32
|
+
Requires-Dist: imaris-ims-file-reader; extra == 'all'
|
|
32
33
|
Requires-Dist: nvidia-ml-py; extra == 'all'
|
|
33
34
|
Requires-Dist: psutil; extra == 'all'
|
|
34
35
|
Requires-Dist: scikit-image; extra == 'all'
|
|
@@ -54,6 +55,8 @@ Requires-Dist: mkdocs-material>=9.0; extra == 'docs'
|
|
|
54
55
|
Requires-Dist: mkdocstrings[python]>=0.24; extra == 'docs'
|
|
55
56
|
Provides-Extra: gpu
|
|
56
57
|
Requires-Dist: nvidia-ml-py; extra == 'gpu'
|
|
58
|
+
Provides-Extra: imaris
|
|
59
|
+
Requires-Dist: imaris-ims-file-reader; extra == 'imaris'
|
|
57
60
|
Provides-Extra: io
|
|
58
61
|
Requires-Dist: psutil; extra == 'io'
|
|
59
62
|
Requires-Dist: tqdm; extra == 'io'
|
|
@@ -71,7 +74,7 @@ Description-Content-Type: text/markdown
|
|
|
71
74
|
> Tiled processing of arbitrarily large images — any image, any function.
|
|
72
75
|
|
|
73
76
|
```
|
|
74
|
-
┌──────┬──────┬──────┐ fn(tile) → labels
|
|
77
|
+
┌──────┬──────┬──────┐ fn(tile) → labels ┌──────┬──────┬──────┐
|
|
75
78
|
│ tile │ tile │ tile │ ─────────────────────► │ 1 │ 2 │ 3 │
|
|
76
79
|
├──────┼──────┼──────┤ ├──────┼──────┼──────┤
|
|
77
80
|
│ tile │ tile │ tile │ │ 4 │ 5 │ 6 │ globally
|
|
@@ -98,6 +101,7 @@ Optional extras:
|
|
|
98
101
|
pip install "patchworks[gpu]" # GPU VRAM querying (nvidia-ml-py)
|
|
99
102
|
pip install "patchworks[cellpose]" # Cellpose plugin
|
|
100
103
|
pip install "patchworks[bioio]" # convert any image format to OME-ZARR
|
|
104
|
+
pip install "patchworks[imaris]" # convert Imaris .ims files to OME-ZARR
|
|
101
105
|
pip install "patchworks[napari]" # interactive napari viewer plugin
|
|
102
106
|
pip install "patchworks[all]" # Everything (except napari GUI)
|
|
103
107
|
```
|
|
@@ -105,6 +109,8 @@ pip install "patchworks[all]" # Everything (except napari GUI)
|
|
|
105
109
|
> `bioio` reads CZI/LIF/ND2/OME-TIFF/… The `[bioio]` extra bundles the common
|
|
106
110
|
> native readers (`bioio-nd2`, `bioio-ome-tiff`, `bioio-czi`, `bioio-tifffile`,
|
|
107
111
|
> `bioio-lif`) plus `bioio-bioformats`, the Bio-Formats catch-all reader (JVM).
|
|
112
|
+
> `[imaris]` adds native `.ims` support (HDF5, no JVM). Physical pixel
|
|
113
|
+
> calibration is read from the input and written into the OME-ZARR.
|
|
108
114
|
|
|
109
115
|
---
|
|
110
116
|
|
|
@@ -121,11 +127,15 @@ def my_fn(tile):
|
|
|
121
127
|
return label(tile > threshold_otsu(tile)).astype("int32")
|
|
122
128
|
|
|
123
129
|
|
|
124
|
-
result = tile_process("image.zarr", my_fn
|
|
130
|
+
result = tile_process("image.zarr", my_fn)
|
|
125
131
|
```
|
|
126
132
|
|
|
127
|
-
Done. `result` is a
|
|
128
|
-
input, with globally unique IDs
|
|
133
|
+
Done. `result` is a **lazy dask array** of integer labels (call `.compute()`
|
|
134
|
+
for a NumPy array), same spatial shape as the input, with globally unique IDs
|
|
135
|
+
across all tiles. By default the labels are also written **into the input
|
|
136
|
+
store** at `image.zarr/labels/labels/` as a multi-scale pyramid, so the image
|
|
137
|
+
and its segmentation live in one OME-ZARR. Pass `write_to="labels.zarr"` to
|
|
138
|
+
write a separate store instead.
|
|
129
139
|
|
|
130
140
|
---
|
|
131
141
|
|
|
@@ -197,6 +207,26 @@ tile_process("image.zarr", my_custom_fn, tile_shape=(1, 512, 512))
|
|
|
197
207
|
|
|
198
208
|
---
|
|
199
209
|
|
|
210
|
+
## Convert to OME-ZARR & view in napari
|
|
211
|
+
|
|
212
|
+
Optional plugins close the loop: convert any image (Imaris `.ims`, CZI, LIF,
|
|
213
|
+
ND2, OME-TIFF, … via bioio) to a pyramidal, **calibrated** OME-ZARR, then view
|
|
214
|
+
the image and its labels in napari.
|
|
215
|
+
|
|
216
|
+
```python
|
|
217
|
+
from patchworks.plugins.ome_zarr import to_ome_zarr
|
|
218
|
+
from patchworks.plugins.napari import view_in_napari
|
|
219
|
+
|
|
220
|
+
to_ome_zarr("scan.ims", "scan.zarr") # lazy, OOM-safe, keeps µm calibration
|
|
221
|
+
view_in_napari("scan.zarr", labels="scan.zarr/labels/labels")
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
Pyramids downsample **X/Y only** (Z kept full-res) and are built level-by-level
|
|
225
|
+
from disk, so terabyte volumes convert in bounded RAM. See the
|
|
226
|
+
[OME-ZARR & napari guide](https://imcf.one/patchworks/guide/ome_zarr_napari/).
|
|
227
|
+
|
|
228
|
+
---
|
|
229
|
+
|
|
200
230
|
## Common patterns
|
|
201
231
|
|
|
202
232
|
### Auto-size tiles from available memory
|
|
@@ -282,8 +312,8 @@ merged = merge_tile_labels(
|
|
|
282
312
|
|
|
283
313
|
## How tiling and merging work
|
|
284
314
|
|
|
285
|
-
See [
|
|
286
|
-
Short version:
|
|
315
|
+
See the [Merging labels guide](https://imcf.one/patchworks/guide/merging/) for
|
|
316
|
+
a full explanation. Short version:
|
|
287
317
|
|
|
288
318
|
1. Image is split into tiles (with optional overlap for boundary context).
|
|
289
319
|
2. Your function is called independently on each tile. Dask handles parallelism
|
|
@@ -313,10 +343,15 @@ tiles where the dask-image approach stalls.
|
|
|
313
343
|
|
|
314
344
|
## Documentation
|
|
315
345
|
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
- [
|
|
319
|
-
- [
|
|
346
|
+
Full docs, guides and tutorials: **<https://imcf.one/patchworks/>**
|
|
347
|
+
|
|
348
|
+
- [Getting Started](https://imcf.one/patchworks/getting_started/)
|
|
349
|
+
- [User Guide](https://imcf.one/patchworks/guide/tiling/) — tiling, merging,
|
|
350
|
+
empty-tile skipping, GPU/distributed, OME-ZARR & napari, pitfalls
|
|
351
|
+
- [Examples](https://imcf.one/patchworks/examples/cellpose_2d/) — Cellpose,
|
|
352
|
+
StarDist, custom functions, standalone merge
|
|
353
|
+
- [API Reference](https://imcf.one/patchworks/api/tile_process/) ·
|
|
354
|
+
[pdoc API](https://imcf.one/apidocs/patchworks/)
|
|
320
355
|
|
|
321
356
|
---
|
|
322
357
|
|
|
@@ -329,7 +364,11 @@ Optional:
|
|
|
329
364
|
- `psutil` — accurate RAM sizing for `tile_shape="auto"`
|
|
330
365
|
- `nvidia-ml-py` — accurate GPU VRAM sizing
|
|
331
366
|
- `tqdm` — progress bars
|
|
332
|
-
- `cellpose` — Cellpose plugin
|
|
367
|
+
- `cellpose` — Cellpose plugin (`patchworks[cellpose]`)
|
|
368
|
+
- `bioio` + readers — convert CZI/LIF/ND2/OME-TIFF/… to OME-ZARR
|
|
369
|
+
(`patchworks[bioio]`)
|
|
370
|
+
- `imaris-ims-file-reader` — convert Imaris `.ims` (`patchworks[imaris]`)
|
|
371
|
+
- `napari` — interactive viewer plugin (`patchworks[napari]`)
|
|
333
372
|
|
|
334
373
|
---
|
|
335
374
|
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
> Tiled processing of arbitrarily large images — any image, any function.
|
|
9
9
|
|
|
10
10
|
```
|
|
11
|
-
┌──────┬──────┬──────┐ fn(tile) → labels
|
|
11
|
+
┌──────┬──────┬──────┐ fn(tile) → labels ┌──────┬──────┬──────┐
|
|
12
12
|
│ tile │ tile │ tile │ ─────────────────────► │ 1 │ 2 │ 3 │
|
|
13
13
|
├──────┼──────┼──────┤ ├──────┼──────┼──────┤
|
|
14
14
|
│ tile │ tile │ tile │ │ 4 │ 5 │ 6 │ globally
|
|
@@ -35,6 +35,7 @@ Optional extras:
|
|
|
35
35
|
pip install "patchworks[gpu]" # GPU VRAM querying (nvidia-ml-py)
|
|
36
36
|
pip install "patchworks[cellpose]" # Cellpose plugin
|
|
37
37
|
pip install "patchworks[bioio]" # convert any image format to OME-ZARR
|
|
38
|
+
pip install "patchworks[imaris]" # convert Imaris .ims files to OME-ZARR
|
|
38
39
|
pip install "patchworks[napari]" # interactive napari viewer plugin
|
|
39
40
|
pip install "patchworks[all]" # Everything (except napari GUI)
|
|
40
41
|
```
|
|
@@ -42,6 +43,8 @@ pip install "patchworks[all]" # Everything (except napari GUI)
|
|
|
42
43
|
> `bioio` reads CZI/LIF/ND2/OME-TIFF/… The `[bioio]` extra bundles the common
|
|
43
44
|
> native readers (`bioio-nd2`, `bioio-ome-tiff`, `bioio-czi`, `bioio-tifffile`,
|
|
44
45
|
> `bioio-lif`) plus `bioio-bioformats`, the Bio-Formats catch-all reader (JVM).
|
|
46
|
+
> `[imaris]` adds native `.ims` support (HDF5, no JVM). Physical pixel
|
|
47
|
+
> calibration is read from the input and written into the OME-ZARR.
|
|
45
48
|
|
|
46
49
|
---
|
|
47
50
|
|
|
@@ -58,11 +61,15 @@ def my_fn(tile):
|
|
|
58
61
|
return label(tile > threshold_otsu(tile)).astype("int32")
|
|
59
62
|
|
|
60
63
|
|
|
61
|
-
result = tile_process("image.zarr", my_fn
|
|
64
|
+
result = tile_process("image.zarr", my_fn)
|
|
62
65
|
```
|
|
63
66
|
|
|
64
|
-
Done. `result` is a
|
|
65
|
-
input, with globally unique IDs
|
|
67
|
+
Done. `result` is a **lazy dask array** of integer labels (call `.compute()`
|
|
68
|
+
for a NumPy array), same spatial shape as the input, with globally unique IDs
|
|
69
|
+
across all tiles. By default the labels are also written **into the input
|
|
70
|
+
store** at `image.zarr/labels/labels/` as a multi-scale pyramid, so the image
|
|
71
|
+
and its segmentation live in one OME-ZARR. Pass `write_to="labels.zarr"` to
|
|
72
|
+
write a separate store instead.
|
|
66
73
|
|
|
67
74
|
---
|
|
68
75
|
|
|
@@ -134,6 +141,26 @@ tile_process("image.zarr", my_custom_fn, tile_shape=(1, 512, 512))
|
|
|
134
141
|
|
|
135
142
|
---
|
|
136
143
|
|
|
144
|
+
## Convert to OME-ZARR & view in napari
|
|
145
|
+
|
|
146
|
+
Optional plugins close the loop: convert any image (Imaris `.ims`, CZI, LIF,
|
|
147
|
+
ND2, OME-TIFF, … via bioio) to a pyramidal, **calibrated** OME-ZARR, then view
|
|
148
|
+
the image and its labels in napari.
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
from patchworks.plugins.ome_zarr import to_ome_zarr
|
|
152
|
+
from patchworks.plugins.napari import view_in_napari
|
|
153
|
+
|
|
154
|
+
to_ome_zarr("scan.ims", "scan.zarr") # lazy, OOM-safe, keeps µm calibration
|
|
155
|
+
view_in_napari("scan.zarr", labels="scan.zarr/labels/labels")
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Pyramids downsample **X/Y only** (Z kept full-res) and are built level-by-level
|
|
159
|
+
from disk, so terabyte volumes convert in bounded RAM. See the
|
|
160
|
+
[OME-ZARR & napari guide](https://imcf.one/patchworks/guide/ome_zarr_napari/).
|
|
161
|
+
|
|
162
|
+
---
|
|
163
|
+
|
|
137
164
|
## Common patterns
|
|
138
165
|
|
|
139
166
|
### Auto-size tiles from available memory
|
|
@@ -219,8 +246,8 @@ merged = merge_tile_labels(
|
|
|
219
246
|
|
|
220
247
|
## How tiling and merging work
|
|
221
248
|
|
|
222
|
-
See [
|
|
223
|
-
Short version:
|
|
249
|
+
See the [Merging labels guide](https://imcf.one/patchworks/guide/merging/) for
|
|
250
|
+
a full explanation. Short version:
|
|
224
251
|
|
|
225
252
|
1. Image is split into tiles (with optional overlap for boundary context).
|
|
226
253
|
2. Your function is called independently on each tile. Dask handles parallelism
|
|
@@ -250,10 +277,15 @@ tiles where the dask-image approach stalls.
|
|
|
250
277
|
|
|
251
278
|
## Documentation
|
|
252
279
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
- [
|
|
256
|
-
- [
|
|
280
|
+
Full docs, guides and tutorials: **<https://imcf.one/patchworks/>**
|
|
281
|
+
|
|
282
|
+
- [Getting Started](https://imcf.one/patchworks/getting_started/)
|
|
283
|
+
- [User Guide](https://imcf.one/patchworks/guide/tiling/) — tiling, merging,
|
|
284
|
+
empty-tile skipping, GPU/distributed, OME-ZARR & napari, pitfalls
|
|
285
|
+
- [Examples](https://imcf.one/patchworks/examples/cellpose_2d/) — Cellpose,
|
|
286
|
+
StarDist, custom functions, standalone merge
|
|
287
|
+
- [API Reference](https://imcf.one/patchworks/api/tile_process/) ·
|
|
288
|
+
[pdoc API](https://imcf.one/apidocs/patchworks/)
|
|
257
289
|
|
|
258
290
|
---
|
|
259
291
|
|
|
@@ -266,7 +298,11 @@ Optional:
|
|
|
266
298
|
- `psutil` — accurate RAM sizing for `tile_shape="auto"`
|
|
267
299
|
- `nvidia-ml-py` — accurate GPU VRAM sizing
|
|
268
300
|
- `tqdm` — progress bars
|
|
269
|
-
- `cellpose` — Cellpose plugin
|
|
301
|
+
- `cellpose` — Cellpose plugin (`patchworks[cellpose]`)
|
|
302
|
+
- `bioio` + readers — convert CZI/LIF/ND2/OME-TIFF/… to OME-ZARR
|
|
303
|
+
(`patchworks[bioio]`)
|
|
304
|
+
- `imaris-ims-file-reader` — convert Imaris `.ims` (`patchworks[imaris]`)
|
|
305
|
+
- `napari` — interactive viewer plugin (`patchworks[napari]`)
|
|
270
306
|
|
|
271
307
|
---
|
|
272
308
|
|
|
@@ -17,7 +17,7 @@ def threshold_fn(tile: np.ndarray) -> np.ndarray:
|
|
|
17
17
|
return label(tile > thr).astype("int32")
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
result = tile_process("image.zarr", threshold_fn
|
|
20
|
+
result = tile_process("image.zarr", threshold_fn)
|
|
21
21
|
```
|
|
22
22
|
|
|
23
23
|
## Gaussian + morphological operations
|
|
@@ -86,12 +86,12 @@ from patchworks import tile_process
|
|
|
86
86
|
|
|
87
87
|
# From any array-like source
|
|
88
88
|
arr = da.from_array(my_numpy_array, chunks=(1, 1024, 1024))
|
|
89
|
-
result = tile_process(arr, my_fn
|
|
89
|
+
result = tile_process(arr, my_fn)
|
|
90
90
|
|
|
91
91
|
# From tifffile
|
|
92
92
|
import tifffile
|
|
93
93
|
import dask.array as da
|
|
94
94
|
|
|
95
95
|
arr = da.from_array(tifffile.imread("image.tif", aszarr=True))
|
|
96
|
-
result = tile_process(arr, my_fn
|
|
96
|
+
result = tile_process(arr, my_fn)
|
|
97
97
|
```
|
|
@@ -89,9 +89,11 @@ objects spanning tile boundaries are merged into a single label.
|
|
|
89
89
|
```python
|
|
90
90
|
from patchworks import tile_process
|
|
91
91
|
|
|
92
|
-
|
|
92
|
+
# returns a lazy dask array; labels are also written into image.zarr by
|
|
93
|
+
# default (image.zarr/labels/labels/, as a pyramid)
|
|
94
|
+
result = tile_process("image.zarr", my_fn)
|
|
93
95
|
print(result.shape) # (z, y, x)
|
|
94
|
-
print(result.max()) # number of objects found
|
|
96
|
+
print(int(result.max().compute())) # number of objects found
|
|
95
97
|
```
|
|
96
98
|
|
|
97
99
|
=== "From a dask array"
|
|
@@ -101,7 +103,7 @@ objects spanning tile boundaries are merged into a single label.
|
|
|
101
103
|
from patchworks import tile_process
|
|
102
104
|
|
|
103
105
|
arr = da.from_zarr("image.zarr")
|
|
104
|
-
result = tile_process(arr, my_fn
|
|
106
|
+
result = tile_process(arr, my_fn)
|
|
105
107
|
```
|
|
106
108
|
|
|
107
109
|
=== "Stream to zarr (recommended for large images)"
|
|
@@ -38,19 +38,33 @@ existed.
|
|
|
38
38
|
|
|
39
39
|
## Convert any image to OME-ZARR
|
|
40
40
|
|
|
41
|
-
`to_ome_zarr` accepts a dask/NumPy array, an existing `.zarr` store,
|
|
42
|
-
file format** readable by
|
|
43
|
-
LIF, ND2, OME-TIFF, …). File
|
|
44
|
-
|
|
45
|
-
bounded RAM.
|
|
41
|
+
`to_ome_zarr` accepts a dask/NumPy array, an existing `.zarr` store, an
|
|
42
|
+
**Imaris `.ims`** file, or **any format** readable by
|
|
43
|
+
[bioio](https://github.com/bioio-devs/bioio) (CZI, LIF, ND2, OME-TIFF, …). File
|
|
44
|
+
inputs are read **lazily**.
|
|
46
45
|
|
|
47
46
|
```python
|
|
48
47
|
from patchworks.plugins.ome_zarr import to_ome_zarr
|
|
49
48
|
|
|
50
|
-
|
|
51
|
-
to_ome_zarr("scan.
|
|
49
|
+
to_ome_zarr("scan.czi", "scan.zarr", n_levels=5) # via bioio
|
|
50
|
+
to_ome_zarr("scan.ims", "scan.zarr") # Imaris, native HDF5
|
|
52
51
|
```
|
|
53
52
|
|
|
53
|
+
### Pixel calibration
|
|
54
|
+
|
|
55
|
+
The physical voxel size is read from the input — bioio's `physical_pixel_sizes`,
|
|
56
|
+
the Imaris resolution metadata, or an existing OME-ZARR's scale — and written
|
|
57
|
+
into the NGFF `coordinateTransformations` (in micrometers), so calibration is
|
|
58
|
+
preserved regardless of input. Override or supply it for bare arrays with
|
|
59
|
+
`pixel_size={"z": 2.0, "y": 0.32, "x": 0.32}`.
|
|
60
|
+
|
|
61
|
+
### Won't OOM
|
|
62
|
+
|
|
63
|
+
Each pyramid level is built by reading the **previous level back from disk**
|
|
64
|
+
and streaming the downsampled result out through dask with bounded chunks. The
|
|
65
|
+
graph never chains level-on-level and no whole plane/volume is held in RAM, so
|
|
66
|
+
terabyte images convert in bounded memory.
|
|
67
|
+
|
|
54
68
|
!!! note "Install the readers you need"
|
|
55
69
|
`pip install "patchworks[bioio]"` pulls `bioio` plus the `bioio-bioformats`
|
|
56
70
|
catch-all reader (needs a JVM). For speed, add native readers for your
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Performance & memory safety
|
|
2
|
+
|
|
3
|
+
`tile_process` is built so a run **adapts to whatever machine it lands on** and
|
|
4
|
+
can't run out of RAM/VRAM or freeze the box — without you tuning anything.
|
|
5
|
+
|
|
6
|
+
## Automatic, machine-aware concurrency
|
|
7
|
+
|
|
8
|
+
The staging step (running your `fn` once per tile to a temp store) and the
|
|
9
|
+
merge step are sized to the host automatically:
|
|
10
|
+
|
|
11
|
+
- **GPU** (`use_gpu=True`) → **one tile at a time**, so concurrent evaluations
|
|
12
|
+
can never exhaust VRAM.
|
|
13
|
+
- **CPU** → as many tiles in flight as fit **80 % of available RAM** (estimated
|
|
14
|
+
from the tile size), and always **leaving one core free** so the machine
|
|
15
|
+
stays responsive — it never pins every core.
|
|
16
|
+
|
|
17
|
+
The RAM figure is read live via `psutil`; without it, a conservative default is
|
|
18
|
+
used instead of guessing high.
|
|
19
|
+
|
|
20
|
+
## Overriding the worker count
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from patchworks import tile_process
|
|
24
|
+
|
|
25
|
+
# let patchworks pick (recommended)
|
|
26
|
+
tile_process("scan.zarr", fn)
|
|
27
|
+
|
|
28
|
+
# or cap it yourself (staging threads + merge processes)
|
|
29
|
+
tile_process("scan.zarr", fn, max_workers=8)
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
`max_workers` bounds both staging and merging. A running **distributed client**
|
|
33
|
+
manages its own concurrency, so the override is skipped there — configure the
|
|
34
|
+
cluster's memory limits instead.
|
|
35
|
+
|
|
36
|
+
## Why it won't OOM or freeze
|
|
37
|
+
|
|
38
|
+
| Resource | Guard |
|
|
39
|
+
|----------|-------|
|
|
40
|
+
| RAM | concurrent tiles × tile size × overhead ≤ 80 % of available RAM |
|
|
41
|
+
| VRAM | GPU path runs one tile at a time |
|
|
42
|
+
| CPU | always leaves at least one core free |
|
|
43
|
+
| Disk I/O | each pyramid/stage level is streamed chunk-by-chunk; no whole volume in memory |
|
|
44
|
+
|
|
45
|
+
The staging graph itself is kept small — a single fused `map_overlap`
|
|
46
|
+
(halo → `fn` → trim) rather than three separate passes — and there is **no**
|
|
47
|
+
extra read-back of the staged data.
|
|
48
|
+
|
|
49
|
+
## Getting more speed
|
|
50
|
+
|
|
51
|
+
- `tile_shape="auto"` sizes tiles to free RAM (or VRAM with `use_gpu=True`).
|
|
52
|
+
- `skip_empty=True` with `estimate_empty_tiles()` skips background tiles.
|
|
53
|
+
- A Dask **distributed** cluster (`make_local_cluster`) parallelises across
|
|
54
|
+
workers/GPUs; patchworks then defers concurrency to the cluster.
|
|
55
|
+
|
|
56
|
+
!!! note "What doesn't help here"
|
|
57
|
+
The merge and relabel steps are already vectorised NumPy + SciPy (C-level)
|
|
58
|
+
with no per-voxel Python loop, and the pipeline is I/O-bound — so `numba`,
|
|
59
|
+
`cupy`, `arrow` and `xarray` bring essentially nothing. The real levers are
|
|
60
|
+
tile size, concurrency (above) and zarr chunking.
|
|
@@ -22,9 +22,11 @@ theme:
|
|
|
22
22
|
- content.code.annotate
|
|
23
23
|
- content.code.copy
|
|
24
24
|
- content.tabs.link
|
|
25
|
-
- navigation.
|
|
25
|
+
- navigation.expand
|
|
26
|
+
- navigation.sections
|
|
26
27
|
- navigation.top
|
|
27
28
|
- navigation.footer
|
|
29
|
+
- toc.integrate
|
|
28
30
|
- search.highlight
|
|
29
31
|
- search.share
|
|
30
32
|
|
|
@@ -36,6 +38,7 @@ nav:
|
|
|
36
38
|
- Merging labels: guide/merging.md
|
|
37
39
|
- Empty tile skipping: guide/skip_empty.md
|
|
38
40
|
- GPU & distributed: guide/gpu_distributed.md
|
|
41
|
+
- Performance & memory: guide/performance.md
|
|
39
42
|
- OME-ZARR & napari: guide/ome_zarr_napari.md
|
|
40
43
|
- Pitfalls: guide/pitfalls.md
|
|
41
44
|
- Examples:
|
|
@@ -54,11 +54,13 @@ bioio = [
|
|
|
54
54
|
"bioio-tifffile",
|
|
55
55
|
"bioio-lif",
|
|
56
56
|
]
|
|
57
|
+
# imaris reads .ims files natively (HDF5, no JVM) for OME-ZARR conversion.
|
|
58
|
+
imaris = ["imaris-ims-file-reader"]
|
|
57
59
|
# napari enables the interactive viewer plugin (GUI-heavy, kept out of [all]).
|
|
58
60
|
napari = ["napari[all]"]
|
|
59
61
|
dev = ["pytest", "pytest-cov", "scikit-image", "psutil", "tqdm"]
|
|
60
62
|
docs = ["mkdocs-material>=9.0", "mkdocstrings[python]>=0.24"]
|
|
61
|
-
all = ["patchworks[io,gpu,bioio]", "psutil", "tqdm", "scikit-image"]
|
|
63
|
+
all = ["patchworks[io,gpu,bioio,imaris]", "psutil", "tqdm", "scikit-image"]
|
|
62
64
|
|
|
63
65
|
[project.urls]
|
|
64
66
|
Homepage = "https://github.com/imcf/patchworks"
|
|
@@ -57,6 +57,51 @@ def _get_available_memory() -> int:
|
|
|
57
57
|
return 8 * 1024**3
|
|
58
58
|
|
|
59
59
|
|
|
60
|
+
def safe_worker_count(
|
|
61
|
+
tile_nbytes: int,
|
|
62
|
+
*,
|
|
63
|
+
use_gpu: bool = False,
|
|
64
|
+
fn_overhead: int = 4,
|
|
65
|
+
ram_fraction: float = 0.8,
|
|
66
|
+
) -> int:
|
|
67
|
+
"""Concurrent tiles that fit the machine without OOM or a CPU freeze.
|
|
68
|
+
|
|
69
|
+
Bounds the threaded scheduler by two limits and takes the smaller:
|
|
70
|
+
|
|
71
|
+
* **CPU** — leaves at least one core free so the box stays responsive
|
|
72
|
+
(never pins every core).
|
|
73
|
+
* **RAM** — at most ``ram_fraction`` of available memory, assuming each
|
|
74
|
+
in-flight tile needs ``fn_overhead`` copies (halo + output + temporaries).
|
|
75
|
+
|
|
76
|
+
On GPU the answer is always 1: one evaluation at a time so concurrent
|
|
77
|
+
tiles can never exhaust VRAM. Without ``psutil`` it returns a conservative
|
|
78
|
+
default rather than guessing high.
|
|
79
|
+
|
|
80
|
+
Parameters
|
|
81
|
+
----------
|
|
82
|
+
tile_nbytes : int
|
|
83
|
+
Size of one tile in bytes (``prod(tile_shape) * dtype.itemsize``).
|
|
84
|
+
use_gpu : bool, optional
|
|
85
|
+
Whether tiles are processed on the GPU.
|
|
86
|
+
fn_overhead : int, optional
|
|
87
|
+
Assumed peak number of tile-sized buffers alive per worker.
|
|
88
|
+
ram_fraction : float, optional
|
|
89
|
+
Fraction of available RAM the staging step may use.
|
|
90
|
+
|
|
91
|
+
Returns
|
|
92
|
+
-------
|
|
93
|
+
int
|
|
94
|
+
Worker-thread count (always >= 1).
|
|
95
|
+
"""
|
|
96
|
+
cpu_cap = max(1, (os.cpu_count() or 1) - 1)
|
|
97
|
+
if use_gpu:
|
|
98
|
+
return 1
|
|
99
|
+
avail = _get_available_memory()
|
|
100
|
+
per_tile = max(1, int(tile_nbytes) * max(1, fn_overhead))
|
|
101
|
+
mem_cap = max(1, int(avail * ram_fraction) // per_tile)
|
|
102
|
+
return max(1, min(cpu_cap, mem_cap))
|
|
103
|
+
|
|
104
|
+
|
|
60
105
|
def _get_gpu_memory() -> int:
|
|
61
106
|
"""Return free GPU VRAM in bytes. Falls back to 8 GiB default."""
|
|
62
107
|
try:
|