patchworks 0.4.0__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. {patchworks-0.4.0 → patchworks-0.6.0}/.github/workflows/docs.yml +4 -0
  2. {patchworks-0.4.0 → patchworks-0.6.0}/.github/workflows/lint.yml +2 -0
  3. {patchworks-0.4.0 → patchworks-0.6.0}/.github/workflows/release.yml +14 -0
  4. {patchworks-0.4.0 → patchworks-0.6.0}/PKG-INFO +51 -12
  5. {patchworks-0.4.0 → patchworks-0.6.0}/README.md +47 -11
  6. {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/custom.md +3 -3
  7. {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/custom_method.py +1 -2
  8. {patchworks-0.4.0 → patchworks-0.6.0}/docs/getting_started.md +5 -3
  9. {patchworks-0.4.0 → patchworks-0.6.0}/docs/guide/ome_zarr_napari.md +21 -7
  10. patchworks-0.6.0/docs/guide/performance.md +60 -0
  11. {patchworks-0.4.0 → patchworks-0.6.0}/docs/index.md +1 -1
  12. {patchworks-0.4.0 → patchworks-0.6.0}/mkdocs.yml +4 -1
  13. {patchworks-0.4.0 → patchworks-0.6.0}/pyproject.toml +3 -1
  14. {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/_chunks.py +45 -0
  15. {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/_core.py +48 -42
  16. {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/_io.py +9 -2
  17. patchworks-0.6.0/src/patchworks/plugins/ome_zarr.py +570 -0
  18. {patchworks-0.4.0 → patchworks-0.6.0}/tests/test_core.py +24 -0
  19. {patchworks-0.4.0 → patchworks-0.6.0}/tests/test_ome_zarr.py +35 -0
  20. patchworks-0.4.0/src/patchworks/plugins/ome_zarr.py +0 -459
  21. {patchworks-0.4.0 → patchworks-0.6.0}/.gitignore +0 -0
  22. {patchworks-0.4.0 → patchworks-0.6.0}/cliff.toml +0 -0
  23. {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/chunks.md +0 -0
  24. {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/cluster.md +0 -0
  25. {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/io.md +0 -0
  26. {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/merge_tile_labels.md +0 -0
  27. {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/plugins/cellpose.md +0 -0
  28. {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/plugins/napari.md +0 -0
  29. {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/plugins/ome_zarr.md +0 -0
  30. {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/relabel.md +0 -0
  31. {patchworks-0.4.0 → patchworks-0.6.0}/docs/api/tile_process.md +0 -0
  32. {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/cellpose_2d.md +0 -0
  33. {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/cellpose_2d.py +0 -0
  34. {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/cellpose_3d.md +0 -0
  35. {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/cellpose_3d.py +0 -0
  36. {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/standalone_merge.md +0 -0
  37. {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/stardist.md +0 -0
  38. {patchworks-0.4.0 → patchworks-0.6.0}/docs/examples/stardist_2d.py +0 -0
  39. {patchworks-0.4.0 → patchworks-0.6.0}/docs/guide/gpu_distributed.md +0 -0
  40. {patchworks-0.4.0 → patchworks-0.6.0}/docs/guide/merging.md +0 -0
  41. {patchworks-0.4.0 → patchworks-0.6.0}/docs/guide/pitfalls.md +0 -0
  42. {patchworks-0.4.0 → patchworks-0.6.0}/docs/guide/skip_empty.md +0 -0
  43. {patchworks-0.4.0 → patchworks-0.6.0}/docs/guide/tiling.md +0 -0
  44. {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/__init__.py +0 -0
  45. {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/_cluster.py +0 -0
  46. {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/_merge.py +0 -0
  47. {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/_relabel.py +0 -0
  48. {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/plugins/__init__.py +0 -0
  49. {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/plugins/cellpose.py +0 -0
  50. {patchworks-0.4.0 → patchworks-0.6.0}/src/patchworks/plugins/napari.py +0 -0
  51. {patchworks-0.4.0 → patchworks-0.6.0}/tests/test_napari.py +0 -0
@@ -4,6 +4,10 @@ on:
4
4
  push:
5
5
  branches:
6
6
  - main
7
+ # Rebuild after each release so the version shown in the header refreshes.
8
+ release:
9
+ types: [published]
10
+ workflow_dispatch:
7
11
 
8
12
  permissions:
9
13
  contents: write
@@ -15,9 +15,11 @@ jobs:
15
15
  - name: ruff check
16
16
  uses: astral-sh/ruff-action@v3
17
17
  with:
18
+ version: "0.15.18"
18
19
  args: "check"
19
20
 
20
21
  - name: ruff format --check
21
22
  uses: astral-sh/ruff-action@v3
22
23
  with:
24
+ version: "0.15.18"
23
25
  args: "format --check"
@@ -48,3 +48,17 @@ jobs:
48
48
 
49
49
  - name: Publish to PyPI
50
50
  uses: pypa/gh-action-pypi-publish@release/v1
51
+
52
+ # Rebuild the org-wide pdoc apidocs site so it picks up the new version.
53
+ apidocs:
54
+ needs: release
55
+ runs-on: ubuntu-latest
56
+ steps:
57
+ - name: Trigger imcf.github.io apidocs rebuild
58
+ uses: peter-evans/repository-dispatch@v3
59
+ with:
60
+ # Fine-grained PAT with "Contents: write" on imcf/imcf.github.io,
61
+ # stored as the APIDOCS_DISPATCH_TOKEN secret in this repo.
62
+ token: ${{ secrets.APIDOCS_DISPATCH_TOKEN }}
63
+ repository: imcf/imcf.github.io
64
+ event-type: dispatch-event
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: patchworks
3
- Version: 0.4.0
3
+ Version: 0.6.0
4
4
  Summary: Tiled processing of arbitrarily large images with globally consistent labels
5
5
  Project-URL: Homepage, https://github.com/imcf/patchworks
6
6
  Project-URL: Issues, https://github.com/imcf/patchworks/issues
@@ -29,6 +29,7 @@ Requires-Dist: bioio-lif; extra == 'all'
29
29
  Requires-Dist: bioio-nd2; extra == 'all'
30
30
  Requires-Dist: bioio-ome-tiff; extra == 'all'
31
31
  Requires-Dist: bioio-tifffile; extra == 'all'
32
+ Requires-Dist: imaris-ims-file-reader; extra == 'all'
32
33
  Requires-Dist: nvidia-ml-py; extra == 'all'
33
34
  Requires-Dist: psutil; extra == 'all'
34
35
  Requires-Dist: scikit-image; extra == 'all'
@@ -54,6 +55,8 @@ Requires-Dist: mkdocs-material>=9.0; extra == 'docs'
54
55
  Requires-Dist: mkdocstrings[python]>=0.24; extra == 'docs'
55
56
  Provides-Extra: gpu
56
57
  Requires-Dist: nvidia-ml-py; extra == 'gpu'
58
+ Provides-Extra: imaris
59
+ Requires-Dist: imaris-ims-file-reader; extra == 'imaris'
57
60
  Provides-Extra: io
58
61
  Requires-Dist: psutil; extra == 'io'
59
62
  Requires-Dist: tqdm; extra == 'io'
@@ -71,7 +74,7 @@ Description-Content-Type: text/markdown
71
74
  > Tiled processing of arbitrarily large images — any image, any function.
72
75
 
73
76
  ```
74
- ┌──────┬──────┬──────┐ fn(tile) → labels ┌──────┬──────┬──────┐
77
+ ┌──────┬──────┬──────┐ fn(tile) → labels ┌──────┬──────┬──────┐
75
78
  │ tile │ tile │ tile │ ─────────────────────► │ 1 │ 2 │ 3 │
76
79
  ├──────┼──────┼──────┤ ├──────┼──────┼──────┤
77
80
  │ tile │ tile │ tile │ │ 4 │ 5 │ 6 │ globally
@@ -98,6 +101,7 @@ Optional extras:
98
101
  pip install "patchworks[gpu]" # GPU VRAM querying (nvidia-ml-py)
99
102
  pip install "patchworks[cellpose]" # Cellpose plugin
100
103
  pip install "patchworks[bioio]" # convert any image format to OME-ZARR
104
+ pip install "patchworks[imaris]" # convert Imaris .ims files to OME-ZARR
101
105
  pip install "patchworks[napari]" # interactive napari viewer plugin
102
106
  pip install "patchworks[all]" # Everything (except napari GUI)
103
107
  ```
@@ -105,6 +109,8 @@ pip install "patchworks[all]" # Everything (except napari GUI)
105
109
  > `bioio` reads CZI/LIF/ND2/OME-TIFF/… The `[bioio]` extra bundles the common
106
110
  > native readers (`bioio-nd2`, `bioio-ome-tiff`, `bioio-czi`, `bioio-tifffile`,
107
111
  > `bioio-lif`) plus `bioio-bioformats`, the Bio-Formats catch-all reader (JVM).
112
+ > `[imaris]` adds native `.ims` support (HDF5, no JVM). Physical pixel
113
+ > calibration is read from the input and written into the OME-ZARR.
108
114
 
109
115
  ---
110
116
 
@@ -121,11 +127,15 @@ def my_fn(tile):
121
127
  return label(tile > threshold_otsu(tile)).astype("int32")
122
128
 
123
129
 
124
- result = tile_process("image.zarr", my_fn, compute=True)
130
+ result = tile_process("image.zarr", my_fn)
125
131
  ```
126
132
 
127
- Done. `result` is a NumPy array of integer labels, same spatial shape as the
128
- input, with globally unique IDs across all tiles.
133
+ Done. `result` is a **lazy dask array** of integer labels (call `.compute()`
134
+ for a NumPy array), same spatial shape as the input, with globally unique IDs
135
+ across all tiles. By default the labels are also written **into the input
136
+ store** at `image.zarr/labels/labels/` as a multi-scale pyramid, so the image
137
+ and its segmentation live in one OME-ZARR. Pass `write_to="labels.zarr"` to
138
+ write a separate store instead.
129
139
 
130
140
  ---
131
141
 
@@ -197,6 +207,26 @@ tile_process("image.zarr", my_custom_fn, tile_shape=(1, 512, 512))
197
207
 
198
208
  ---
199
209
 
210
+ ## Convert to OME-ZARR & view in napari
211
+
212
+ Optional plugins close the loop: convert any image (Imaris `.ims`, CZI, LIF,
213
+ ND2, OME-TIFF, … via bioio) to a pyramidal, **calibrated** OME-ZARR, then view
214
+ the image and its labels in napari.
215
+
216
+ ```python
217
+ from patchworks.plugins.ome_zarr import to_ome_zarr
218
+ from patchworks.plugins.napari import view_in_napari
219
+
220
+ to_ome_zarr("scan.ims", "scan.zarr") # lazy, OOM-safe, keeps µm calibration
221
+ view_in_napari("scan.zarr", labels="scan.zarr/labels/labels")
222
+ ```
223
+
224
+ Pyramids downsample **X/Y only** (Z kept full-res) and are built level-by-level
225
+ from disk, so terabyte volumes convert in bounded RAM. See the
226
+ [OME-ZARR & napari guide](https://imcf.one/patchworks/guide/ome_zarr_napari/).
227
+
228
+ ---
229
+
200
230
  ## Common patterns
201
231
 
202
232
  ### Auto-size tiles from available memory
@@ -282,8 +312,8 @@ merged = merge_tile_labels(
282
312
 
283
313
  ## How tiling and merging work
284
314
 
285
- See [docs/how-it-works.md](docs/how-it-works.md) for a full explanation.
286
- Short version:
315
+ See the [Merging labels guide](https://imcf.one/patchworks/guide/merging/) for
316
+ a full explanation. Short version:
287
317
 
288
318
  1. Image is split into tiles (with optional overlap for boundary context).
289
319
  2. Your function is called independently on each tile. Dask handles parallelism
@@ -313,10 +343,15 @@ tiles where the dask-image approach stalls.
313
343
 
314
344
  ## Documentation
315
345
 
316
- - [Quick Start](docs/quickstart.md)
317
- - [API Reference](docs/api-reference.md)
318
- - [How It Works](docs/how-it-works.md)
319
- - [Examples](docs/examples/)
346
+ Full docs, guides and tutorials: **<https://imcf.one/patchworks/>**
347
+
348
+ - [Getting Started](https://imcf.one/patchworks/getting_started/)
349
+ - [User Guide](https://imcf.one/patchworks/guide/tiling/) — tiling, merging,
350
+ empty-tile skipping, GPU/distributed, OME-ZARR & napari, pitfalls
351
+ - [Examples](https://imcf.one/patchworks/examples/cellpose_2d/) — Cellpose,
352
+ StarDist, custom functions, standalone merge
353
+ - [API Reference](https://imcf.one/patchworks/api/tile_process/) ·
354
+ [pdoc API](https://imcf.one/apidocs/patchworks/)
320
355
 
321
356
  ---
322
357
 
@@ -329,7 +364,11 @@ Optional:
329
364
  - `psutil` — accurate RAM sizing for `tile_shape="auto"`
330
365
  - `nvidia-ml-py` — accurate GPU VRAM sizing
331
366
  - `tqdm` — progress bars
332
- - `cellpose` — Cellpose plugin
367
+ - `cellpose` — Cellpose plugin (`patchworks[cellpose]`)
368
+ - `bioio` + readers — convert CZI/LIF/ND2/OME-TIFF/… to OME-ZARR
369
+ (`patchworks[bioio]`)
370
+ - `imaris-ims-file-reader` — convert Imaris `.ims` (`patchworks[imaris]`)
371
+ - `napari` — interactive viewer plugin (`patchworks[napari]`)
333
372
 
334
373
  ---
335
374
 
@@ -8,7 +8,7 @@
8
8
  > Tiled processing of arbitrarily large images — any image, any function.
9
9
 
10
10
  ```
11
- ┌──────┬──────┬──────┐ fn(tile) → labels ┌──────┬──────┬──────┐
11
+ ┌──────┬──────┬──────┐ fn(tile) → labels ┌──────┬──────┬──────┐
12
12
  │ tile │ tile │ tile │ ─────────────────────► │ 1 │ 2 │ 3 │
13
13
  ├──────┼──────┼──────┤ ├──────┼──────┼──────┤
14
14
  │ tile │ tile │ tile │ │ 4 │ 5 │ 6 │ globally
@@ -35,6 +35,7 @@ Optional extras:
35
35
  pip install "patchworks[gpu]" # GPU VRAM querying (nvidia-ml-py)
36
36
  pip install "patchworks[cellpose]" # Cellpose plugin
37
37
  pip install "patchworks[bioio]" # convert any image format to OME-ZARR
38
+ pip install "patchworks[imaris]" # convert Imaris .ims files to OME-ZARR
38
39
  pip install "patchworks[napari]" # interactive napari viewer plugin
39
40
  pip install "patchworks[all]" # Everything (except napari GUI)
40
41
  ```
@@ -42,6 +43,8 @@ pip install "patchworks[all]" # Everything (except napari GUI)
42
43
  > `bioio` reads CZI/LIF/ND2/OME-TIFF/… The `[bioio]` extra bundles the common
43
44
  > native readers (`bioio-nd2`, `bioio-ome-tiff`, `bioio-czi`, `bioio-tifffile`,
44
45
  > `bioio-lif`) plus `bioio-bioformats`, the Bio-Formats catch-all reader (JVM).
46
+ > `[imaris]` adds native `.ims` support (HDF5, no JVM). Physical pixel
47
+ > calibration is read from the input and written into the OME-ZARR.
45
48
 
46
49
  ---
47
50
 
@@ -58,11 +61,15 @@ def my_fn(tile):
58
61
  return label(tile > threshold_otsu(tile)).astype("int32")
59
62
 
60
63
 
61
- result = tile_process("image.zarr", my_fn, compute=True)
64
+ result = tile_process("image.zarr", my_fn)
62
65
  ```
63
66
 
64
- Done. `result` is a NumPy array of integer labels, same spatial shape as the
65
- input, with globally unique IDs across all tiles.
67
+ Done. `result` is a **lazy dask array** of integer labels (call `.compute()`
68
+ for a NumPy array), same spatial shape as the input, with globally unique IDs
69
+ across all tiles. By default the labels are also written **into the input
70
+ store** at `image.zarr/labels/labels/` as a multi-scale pyramid, so the image
71
+ and its segmentation live in one OME-ZARR. Pass `write_to="labels.zarr"` to
72
+ write a separate store instead.
66
73
 
67
74
  ---
68
75
 
@@ -134,6 +141,26 @@ tile_process("image.zarr", my_custom_fn, tile_shape=(1, 512, 512))
134
141
 
135
142
  ---
136
143
 
144
+ ## Convert to OME-ZARR & view in napari
145
+
146
+ Optional plugins close the loop: convert any image (Imaris `.ims`, CZI, LIF,
147
+ ND2, OME-TIFF, … via bioio) to a pyramidal, **calibrated** OME-ZARR, then view
148
+ the image and its labels in napari.
149
+
150
+ ```python
151
+ from patchworks.plugins.ome_zarr import to_ome_zarr
152
+ from patchworks.plugins.napari import view_in_napari
153
+
154
+ to_ome_zarr("scan.ims", "scan.zarr") # lazy, OOM-safe, keeps µm calibration
155
+ view_in_napari("scan.zarr", labels="scan.zarr/labels/labels")
156
+ ```
157
+
158
+ Pyramids downsample **X/Y only** (Z kept full-res) and are built level-by-level
159
+ from disk, so terabyte volumes convert in bounded RAM. See the
160
+ [OME-ZARR & napari guide](https://imcf.one/patchworks/guide/ome_zarr_napari/).
161
+
162
+ ---
163
+
137
164
  ## Common patterns
138
165
 
139
166
  ### Auto-size tiles from available memory
@@ -219,8 +246,8 @@ merged = merge_tile_labels(
219
246
 
220
247
  ## How tiling and merging work
221
248
 
222
- See [docs/how-it-works.md](docs/how-it-works.md) for a full explanation.
223
- Short version:
249
+ See the [Merging labels guide](https://imcf.one/patchworks/guide/merging/) for
250
+ a full explanation. Short version:
224
251
 
225
252
  1. Image is split into tiles (with optional overlap for boundary context).
226
253
  2. Your function is called independently on each tile. Dask handles parallelism
@@ -250,10 +277,15 @@ tiles where the dask-image approach stalls.
250
277
 
251
278
  ## Documentation
252
279
 
253
- - [Quick Start](docs/quickstart.md)
254
- - [API Reference](docs/api-reference.md)
255
- - [How It Works](docs/how-it-works.md)
256
- - [Examples](docs/examples/)
280
+ Full docs, guides and tutorials: **<https://imcf.one/patchworks/>**
281
+
282
+ - [Getting Started](https://imcf.one/patchworks/getting_started/)
283
+ - [User Guide](https://imcf.one/patchworks/guide/tiling/) — tiling, merging,
284
+ empty-tile skipping, GPU/distributed, OME-ZARR & napari, pitfalls
285
+ - [Examples](https://imcf.one/patchworks/examples/cellpose_2d/) — Cellpose,
286
+ StarDist, custom functions, standalone merge
287
+ - [API Reference](https://imcf.one/patchworks/api/tile_process/) ·
288
+ [pdoc API](https://imcf.one/apidocs/patchworks/)
257
289
 
258
290
  ---
259
291
 
@@ -266,7 +298,11 @@ Optional:
266
298
  - `psutil` — accurate RAM sizing for `tile_shape="auto"`
267
299
  - `nvidia-ml-py` — accurate GPU VRAM sizing
268
300
  - `tqdm` — progress bars
269
- - `cellpose` — Cellpose plugin
301
+ - `cellpose` — Cellpose plugin (`patchworks[cellpose]`)
302
+ - `bioio` + readers — convert CZI/LIF/ND2/OME-TIFF/… to OME-ZARR
303
+ (`patchworks[bioio]`)
304
+ - `imaris-ims-file-reader` — convert Imaris `.ims` (`patchworks[imaris]`)
305
+ - `napari` — interactive viewer plugin (`patchworks[napari]`)
270
306
 
271
307
  ---
272
308
 
@@ -17,7 +17,7 @@ def threshold_fn(tile: np.ndarray) -> np.ndarray:
17
17
  return label(tile > thr).astype("int32")
18
18
 
19
19
 
20
- result = tile_process("image.zarr", threshold_fn, compute=True)
20
+ result = tile_process("image.zarr", threshold_fn)
21
21
  ```
22
22
 
23
23
  ## Gaussian + morphological operations
@@ -86,12 +86,12 @@ from patchworks import tile_process
86
86
 
87
87
  # From any array-like source
88
88
  arr = da.from_array(my_numpy_array, chunks=(1, 1024, 1024))
89
- result = tile_process(arr, my_fn, compute=True)
89
+ result = tile_process(arr, my_fn)
90
90
 
91
91
  # From tifffile
92
92
  import tifffile
93
93
  import dask.array as da
94
94
 
95
95
  arr = da.from_array(tifffile.imread("image.tif", aszarr=True))
96
- result = tile_process(arr, my_fn, compute=True)
96
+ result = tile_process(arr, my_fn)
97
97
  ```
@@ -41,8 +41,7 @@ result = tile_process(
41
41
  my_fn,
42
42
  tile_shape=(1, 512, 512),
43
43
  overlap=16,
44
- compute=True,
45
44
  progress=True,
46
45
  )
47
46
 
48
- print(f"Found {result.max()} objects")
47
+ print(f"Found {int(result.max().compute())} objects")
@@ -89,9 +89,11 @@ objects spanning tile boundaries are merged into a single label.
89
89
  ```python
90
90
  from patchworks import tile_process
91
91
 
92
- result = tile_process("image.zarr", my_fn, compute=True)
92
+ # returns a lazy dask array; labels are also written into image.zarr by
93
+ # default (image.zarr/labels/labels/, as a pyramid)
94
+ result = tile_process("image.zarr", my_fn)
93
95
  print(result.shape) # (z, y, x)
94
- print(result.max()) # number of objects found
96
+ print(int(result.max().compute())) # number of objects found
95
97
  ```
96
98
 
97
99
  === "From a dask array"
@@ -101,7 +103,7 @@ objects spanning tile boundaries are merged into a single label.
101
103
  from patchworks import tile_process
102
104
 
103
105
  arr = da.from_zarr("image.zarr")
104
- result = tile_process(arr, my_fn, compute=True)
106
+ result = tile_process(arr, my_fn)
105
107
  ```
106
108
 
107
109
  === "Stream to zarr (recommended for large images)"
@@ -38,19 +38,33 @@ existed.
38
38
 
39
39
  ## Convert any image to OME-ZARR
40
40
 
41
- `to_ome_zarr` accepts a dask/NumPy array, an existing `.zarr` store, or **any
42
- file format** readable by [bioio](https://github.com/bioio-devs/bioio) (CZI,
43
- LIF, ND2, OME-TIFF, …). File inputs are read **lazily** — pixels stream from
44
- disk and are written level by level through dask, so terabyte images convert in
45
- bounded RAM.
41
+ `to_ome_zarr` accepts a dask/NumPy array, an existing `.zarr` store, an
42
+ **Imaris `.ims`** file, or **any format** readable by
43
+ [bioio](https://github.com/bioio-devs/bioio) (CZI, LIF, ND2, OME-TIFF, …). File
44
+ inputs are read **lazily**.
46
45
 
47
46
  ```python
48
47
  from patchworks.plugins.ome_zarr import to_ome_zarr
49
48
 
50
- # From a proprietary microscope file (lazy, via bioio):
51
- to_ome_zarr("scan.czi", "scan.zarr", n_levels=5)
49
+ to_ome_zarr("scan.czi", "scan.zarr", n_levels=5) # via bioio
50
+ to_ome_zarr("scan.ims", "scan.zarr") # Imaris, native HDF5
52
51
  ```
53
52
 
53
+ ### Pixel calibration
54
+
55
+ The physical voxel size is read from the input — bioio's `physical_pixel_sizes`,
56
+ the Imaris resolution metadata, or an existing OME-ZARR's scale — and written
57
+ into the NGFF `coordinateTransformations` (in micrometers), so calibration is
58
+ preserved regardless of input. Override or supply it for bare arrays with
59
+ `pixel_size={"z": 2.0, "y": 0.32, "x": 0.32}`.
60
+
61
+ ### Won't OOM
62
+
63
+ Each pyramid level is built by reading the **previous level back from disk**
64
+ and streaming the downsampled result out through dask with bounded chunks. The
65
+ graph never chains level-on-level and no whole plane/volume is held in RAM, so
66
+ terabyte images convert in bounded memory.
67
+
54
68
  !!! note "Install the readers you need"
55
69
  `pip install "patchworks[bioio]"` pulls `bioio` plus the `bioio-bioformats`
56
70
  catch-all reader (needs a JVM). For speed, add native readers for your
@@ -0,0 +1,60 @@
1
+ # Performance & memory safety
2
+
3
+ `tile_process` is built so a run **adapts to whatever machine it lands on** and
4
+ can't run out of RAM/VRAM or freeze the box — without you tuning anything.
5
+
6
+ ## Automatic, machine-aware concurrency
7
+
8
+ The staging step (running your `fn` once per tile to a temp store) and the
9
+ merge step are sized to the host automatically:
10
+
11
+ - **GPU** (`use_gpu=True`) → **one tile at a time**, so concurrent evaluations
12
+ can never exhaust VRAM.
13
+ - **CPU** → as many tiles in flight as fit **80 % of available RAM** (estimated
14
+ from the tile size), and always **leaving one core free** so the machine
15
+ stays responsive — it never pins every core.
16
+
17
+ The RAM figure is read live via `psutil`; without it, a conservative default is
18
+ used instead of guessing high.
19
+
20
+ ## Overriding the worker count
21
+
22
+ ```python
23
+ from patchworks import tile_process
24
+
25
+ # let patchworks pick (recommended)
26
+ tile_process("scan.zarr", fn)
27
+
28
+ # or cap it yourself (staging threads + merge processes)
29
+ tile_process("scan.zarr", fn, max_workers=8)
30
+ ```
31
+
32
+ `max_workers` bounds both staging and merging. A running **distributed client**
33
+ manages its own concurrency, so the override is skipped there — configure the
34
+ cluster's memory limits instead.
35
+
36
+ ## Why it won't OOM or freeze
37
+
38
+ | Resource | Guard |
39
+ |----------|-------|
40
+ | RAM | concurrent tiles × tile size × overhead ≤ 80 % of available RAM |
41
+ | VRAM | GPU path runs one tile at a time |
42
+ | CPU | always leaves at least one core free |
43
+ | Disk I/O | each pyramid/stage level is streamed chunk-by-chunk; no whole volume in memory |
44
+
45
+ The staging graph itself is kept small — a single fused `map_overlap`
46
+ (halo → `fn` → trim) rather than three separate passes — and there is **no**
47
+ extra read-back of the staged data.
48
+
49
+ ## Getting more speed
50
+
51
+ - `tile_shape="auto"` sizes tiles to free RAM (or VRAM with `use_gpu=True`).
52
+ - `skip_empty=True` with `estimate_empty_tiles()` skips background tiles.
53
+ - A Dask **distributed** cluster (`make_local_cluster`) parallelises across
54
+ workers/GPUs; patchworks then defers concurrency to the cluster.
55
+
56
+ !!! note "What doesn't help here"
57
+ The merge and relabel steps are already vectorised NumPy + SciPy (C-level)
58
+ with no per-voxel Python loop, and the pipeline is I/O-bound — so `numba`,
59
+ `cupy`, `arrow` and `xarray` bring essentially nothing. The real levers are
60
+ tile size, concurrency (above) and zarr chunking.
@@ -53,7 +53,7 @@ def my_fn(tile):
53
53
  return label(tile > threshold_otsu(tile)).astype("int32")
54
54
 
55
55
 
56
- result = tile_process("image.zarr", my_fn, compute=True)
56
+ result = tile_process("image.zarr", my_fn)
57
57
  ```
58
58
 
59
59
  Any function. Any image.
@@ -22,9 +22,11 @@ theme:
22
22
  - content.code.annotate
23
23
  - content.code.copy
24
24
  - content.tabs.link
25
- - navigation.tabs
25
+ - navigation.expand
26
+ - navigation.sections
26
27
  - navigation.top
27
28
  - navigation.footer
29
+ - toc.integrate
28
30
  - search.highlight
29
31
  - search.share
30
32
 
@@ -36,6 +38,7 @@ nav:
36
38
  - Merging labels: guide/merging.md
37
39
  - Empty tile skipping: guide/skip_empty.md
38
40
  - GPU & distributed: guide/gpu_distributed.md
41
+ - Performance & memory: guide/performance.md
39
42
  - OME-ZARR & napari: guide/ome_zarr_napari.md
40
43
  - Pitfalls: guide/pitfalls.md
41
44
  - Examples:
@@ -54,11 +54,13 @@ bioio = [
54
54
  "bioio-tifffile",
55
55
  "bioio-lif",
56
56
  ]
57
+ # imaris reads .ims files natively (HDF5, no JVM) for OME-ZARR conversion.
58
+ imaris = ["imaris-ims-file-reader"]
57
59
  # napari enables the interactive viewer plugin (GUI-heavy, kept out of [all]).
58
60
  napari = ["napari[all]"]
59
61
  dev = ["pytest", "pytest-cov", "scikit-image", "psutil", "tqdm"]
60
62
  docs = ["mkdocs-material>=9.0", "mkdocstrings[python]>=0.24"]
61
- all = ["patchworks[io,gpu,bioio]", "psutil", "tqdm", "scikit-image"]
63
+ all = ["patchworks[io,gpu,bioio,imaris]", "psutil", "tqdm", "scikit-image"]
62
64
 
63
65
  [project.urls]
64
66
  Homepage = "https://github.com/imcf/patchworks"
@@ -57,6 +57,51 @@ def _get_available_memory() -> int:
57
57
  return 8 * 1024**3
58
58
 
59
59
 
60
+ def safe_worker_count(
61
+ tile_nbytes: int,
62
+ *,
63
+ use_gpu: bool = False,
64
+ fn_overhead: int = 4,
65
+ ram_fraction: float = 0.8,
66
+ ) -> int:
67
+ """Concurrent tiles that fit the machine without OOM or a CPU freeze.
68
+
69
+ Bounds the threaded scheduler by two limits and takes the smaller:
70
+
71
+ * **CPU** — leaves at least one core free so the box stays responsive
72
+ (never pins every core).
73
+ * **RAM** — at most ``ram_fraction`` of available memory, assuming each
74
+ in-flight tile needs ``fn_overhead`` copies (halo + output + temporaries).
75
+
76
+ On GPU the answer is always 1: one evaluation at a time so concurrent
77
+ tiles can never exhaust VRAM. Without ``psutil`` it returns a conservative
78
+ default rather than guessing high.
79
+
80
+ Parameters
81
+ ----------
82
+ tile_nbytes : int
83
+ Size of one tile in bytes (``prod(tile_shape) * dtype.itemsize``).
84
+ use_gpu : bool, optional
85
+ Whether tiles are processed on the GPU.
86
+ fn_overhead : int, optional
87
+ Assumed peak number of tile-sized buffers alive per worker.
88
+ ram_fraction : float, optional
89
+ Fraction of available RAM the staging step may use.
90
+
91
+ Returns
92
+ -------
93
+ int
94
+ Worker-thread count (always >= 1).
95
+ """
96
+ cpu_cap = max(1, (os.cpu_count() or 1) - 1)
97
+ if use_gpu:
98
+ return 1
99
+ avail = _get_available_memory()
100
+ per_tile = max(1, int(tile_nbytes) * max(1, fn_overhead))
101
+ mem_cap = max(1, int(avail * ram_fraction) // per_tile)
102
+ return max(1, min(cpu_cap, mem_cap))
103
+
104
+
60
105
  def _get_gpu_memory() -> int:
61
106
  """Return free GPU VRAM in bytes. Falls back to 8 GiB default."""
62
107
  try: