patchworks 2.6.12__tar.gz → 2.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. {patchworks-2.6.12 → patchworks-2.8.0}/PKG-INFO +2 -1
  2. {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/plugins/ome_zarr.md +12 -0
  3. {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/custom_segmentation.md +5 -2
  4. {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/snakemake.md +299 -10
  5. {patchworks-2.6.12 → patchworks-2.8.0}/pyproject.toml +10 -1
  6. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_chunks.py +18 -4
  7. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_io.py +105 -6
  8. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_relations.py +26 -2
  9. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/plugins/napari.py +7 -8
  10. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/plugins/ome_zarr.py +466 -166
  11. patchworks-2.8.0/tests/ngff_schemas/0.4/image.schema +233 -0
  12. patchworks-2.8.0/tests/ngff_schemas/0.4/label.schema +77 -0
  13. patchworks-2.8.0/tests/ngff_schemas/0.4/ome.schema +17 -0
  14. patchworks-2.8.0/tests/ngff_schemas/0.5/_version.schema +10 -0
  15. patchworks-2.8.0/tests/ngff_schemas/0.5/image.schema +268 -0
  16. patchworks-2.8.0/tests/ngff_schemas/0.5/label.schema +91 -0
  17. patchworks-2.8.0/tests/ngff_schemas/0.5/ome.schema +33 -0
  18. patchworks-2.8.0/tests/ngff_schemas/README.md +16 -0
  19. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_allocation.py +28 -0
  20. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_napari.py +75 -0
  21. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_ome_zarr.py +381 -0
  22. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_pw.py +60 -0
  23. patchworks-2.8.0/tests/test_relations.py +104 -0
  24. patchworks-2.8.0/tests/test_run_multi.py +1120 -0
  25. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/config/common.yaml +6 -0
  26. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/config/config.yaml +14 -0
  27. patchworks-2.8.0/workflow/config/multi.yaml +75 -0
  28. patchworks-2.8.0/workflow/pixi.toml +126 -0
  29. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/profile/slurm/config.yaml +28 -0
  30. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/rules/common.smk +9 -2
  31. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/_pw.py +38 -0
  32. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/build_occupancy.py +5 -1
  33. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/convert.py +8 -2
  34. patchworks-2.8.0/workflow/scripts/export_iso.py +314 -0
  35. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/merge.py +73 -2
  36. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/relate.py +35 -0
  37. patchworks-2.8.0/workflow/scripts/reshard_store.py +196 -0
  38. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/run_multi.py +268 -18
  39. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/view.py +10 -1
  40. patchworks-2.6.12/tests/test_relations.py +0 -47
  41. patchworks-2.6.12/tests/test_run_multi.py +0 -537
  42. patchworks-2.6.12/workflow/config/multi.yaml +0 -36
  43. patchworks-2.6.12/workflow/pixi.toml +0 -66
  44. {patchworks-2.6.12 → patchworks-2.8.0}/.github/workflows/docs.yml +0 -0
  45. {patchworks-2.6.12 → patchworks-2.8.0}/.github/workflows/lint.yml +0 -0
  46. {patchworks-2.6.12 → patchworks-2.8.0}/.github/workflows/release.yml +0 -0
  47. {patchworks-2.6.12 → patchworks-2.8.0}/.gitignore +0 -0
  48. {patchworks-2.6.12 → patchworks-2.8.0}/.markdownlint-cli2.yaml +0 -0
  49. {patchworks-2.6.12 → patchworks-2.8.0}/LICENSE +0 -0
  50. {patchworks-2.6.12 → patchworks-2.8.0}/README.md +0 -0
  51. {patchworks-2.6.12 → patchworks-2.8.0}/cliff.toml +0 -0
  52. {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/chunks.md +0 -0
  53. {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/cluster.md +0 -0
  54. {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/io.md +0 -0
  55. {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/merge_tile_labels.md +0 -0
  56. {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/plugins/cellpose.md +0 -0
  57. {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/plugins/dog.md +0 -0
  58. {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/plugins/napari.md +0 -0
  59. {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/postprocess.md +0 -0
  60. {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/relabel.md +0 -0
  61. {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/tile_process.md +0 -0
  62. {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/volume_filter.md +0 -0
  63. {patchworks-2.6.12 → patchworks-2.8.0}/docs/assets/logo.png +0 -0
  64. {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/cellpose_2d.md +0 -0
  65. {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/cellpose_2d.py +0 -0
  66. {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/cellpose_3d.md +0 -0
  67. {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/cellpose_3d.py +0 -0
  68. {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/custom.md +0 -0
  69. {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/custom_method.py +0 -0
  70. {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/dog.md +0 -0
  71. {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/dog.py +0 -0
  72. {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/standalone_merge.md +0 -0
  73. {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/stardist.md +0 -0
  74. {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/stardist_2d.py +0 -0
  75. {patchworks-2.6.12 → patchworks-2.8.0}/docs/getting_started.md +0 -0
  76. {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/gpu_distributed.md +0 -0
  77. {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/label_relations.md +0 -0
  78. {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/measurements.md +0 -0
  79. {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/merging.md +0 -0
  80. {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/ome_zarr_napari.md +0 -0
  81. {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/performance.md +0 -0
  82. {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/pitfalls.md +0 -0
  83. {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/skip_empty.md +0 -0
  84. {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/tiling.md +0 -0
  85. {patchworks-2.6.12 → patchworks-2.8.0}/docs/index.md +0 -0
  86. {patchworks-2.6.12 → patchworks-2.8.0}/mkdocs.yml +0 -0
  87. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/__init__.py +0 -0
  88. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_cluster.py +0 -0
  89. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_core.py +0 -0
  90. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_distributed.py +0 -0
  91. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_gpu.py +0 -0
  92. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_merge.py +0 -0
  93. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_notify.py +0 -0
  94. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_occupancy.py +0 -0
  95. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_postprocess.py +0 -0
  96. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_progress.py +0 -0
  97. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_relabel.py +0 -0
  98. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_volume_filter.py +0 -0
  99. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/plugins/__init__.py +0 -0
  100. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/plugins/cellpose.py +0 -0
  101. {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/plugins/dog.py +0 -0
  102. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_cellpose.py +0 -0
  103. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_core.py +0 -0
  104. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_distributed.py +0 -0
  105. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_dog.py +0 -0
  106. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_gpu.py +0 -0
  107. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_notify.py +0 -0
  108. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_occupancy.py +0 -0
  109. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_postprocess.py +0 -0
  110. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_progress.py +0 -0
  111. {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_volume_filter.py +0 -0
  112. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/README.md +0 -0
  113. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/Snakefile +0 -0
  114. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/config/config_cilia.yaml +0 -0
  115. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/config/config_cyto.yaml +0 -0
  116. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/config/config_nuclei.yaml +0 -0
  117. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/rules/convert.smk +0 -0
  118. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/rules/merge.smk +0 -0
  119. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/rules/segment.smk +0 -0
  120. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/fetch_model.py +0 -0
  121. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/prepare_tiles.py +0 -0
  122. {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/segment_tile.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: patchworks
3
- Version: 2.6.12
3
+ Version: 2.8.0
4
4
  Summary: Tiled processing of arbitrarily large images with globally consistent labels
5
5
  Project-URL: Homepage, https://github.com/imcf/patchworks
6
6
  Project-URL: Issues, https://github.com/imcf/patchworks/issues
@@ -57,6 +57,7 @@ Requires-Dist: cellpose<4,>=3.0; extra == 'cellpose3'
57
57
  Provides-Extra: cellpose4
58
58
  Requires-Dist: cellpose>=4; extra == 'cellpose4'
59
59
  Provides-Extra: dev
60
+ Requires-Dist: jsonschema>=4.18; extra == 'dev'
60
61
  Requires-Dist: psutil; extra == 'dev'
61
62
  Requires-Dist: pytest; extra == 'dev'
62
63
  Requires-Dist: pytest-cov; extra == 'dev'
@@ -25,6 +25,10 @@ resolution, matching anisotropic microscopy stacks.
25
25
 
26
26
  ::: patchworks.plugins.ome_zarr.register_labels
27
27
 
28
+ ## reshard_level
29
+
30
+ ::: patchworks.plugins.ome_zarr.reshard_level
31
+
28
32
  ## read_pixel_size
29
33
 
30
34
  ::: patchworks.plugins.ome_zarr.read_pixel_size
@@ -37,6 +41,14 @@ matches the store, and reads both.
37
41
 
38
42
  ::: patchworks.plugins.ome_zarr.ngff_version
39
43
 
44
+ Every writer takes an `ngff_version=` keyword: `"auto"` (default) follows the
45
+ installed zarr — 0.5 on v3, 0.4 on v2 — and `"0.4"` pins the older, zarr-v2
46
+ layout. Writing into an existing store always follows *that store's* format,
47
+ so a label pyramid added later can never disagree with the image it sits in.
48
+ NGFF 0.6 is released but not written: RFC-5 replaces `axes` with
49
+ `coordinateSystems` and requires `input`/`output` on every coordinate
50
+ transformation, and no reader supports it yet.
51
+
40
52
  ::: patchworks.plugins.ome_zarr.read_ngff_attr
41
53
 
42
54
  ::: patchworks.plugins.ome_zarr.write_ngff_attrs
@@ -81,8 +81,11 @@ backend `fn` used — pass `use_gpu=True` to dilate via cupy instead:
81
81
  fn = dilate_labels(fn, iterations=2, use_gpu=True)
82
82
  ```
83
83
 
84
- Needs `cupy` installed **manually**, matching your CUDA version (e.g.
85
- `pip install cupy-cuda12x`) — it's never installed automatically by
84
+ Needs `cupy`, matching your CUDA version. Under the Snakemake workflow, use
85
+ the environment that already carries it — `pixi install -e cuda12` (or
86
+ `-e cuda13`), then `pixi run -e cuda12 <task>`; outside it, install the
87
+ matching wheel yourself (e.g. `pip install cupy-cuda12x`). It's never
88
+ installed automatically by
86
89
  patchworks, unlike Cellpose's GPU support (which comes for free via
87
90
  PyTorch's self-contained CUDA wheels); cupy ships one wheel per CUDA major
88
91
  version, so there's no single generic pin that works everywhere. On the
@@ -50,7 +50,9 @@ work_dir: "/scratch/results" # everything is written here
50
50
  # conversion (input → pyramidal OME-ZARR)
51
51
  reuse_pyramid: true # .ims: copy its own pyramid (fast)
52
52
  convert_chunks: null # null → bounded auto chunks; or [c,z,y,x]
53
- shard: false # true → pack chunks into shards (fewer files)
53
+ shard: false # true → pack chunks into shards (fewer files);
54
+ # covers the image and the label pyramids
55
+ ngff_version: "auto" # OME-ZARR version: "auto" (0.5), or "0.4"
54
56
 
55
57
  # tiling
56
58
  channel: 0 # channel to segment, 0-based (null = keep all)
@@ -83,21 +85,241 @@ cellpose:
83
85
  pyramid_levels: 5
84
86
  pyramid_downscale: 2
85
87
  sequential_labels: true # renumber labels to a contiguous 1..N
88
+ shard_labels: false # true → also reshard label level 0 after the
89
+ # merge (one extra pass; see tip below)
86
90
  ```
87
91
 
88
92
  !!! tip "Growing labels after segmentation"
89
93
  `dilate: N` grows every label by `N` pixels once segmentation finishes,
90
94
  regardless of `method`. `0` (default) disables it. Runs on CPU (scipy)
91
95
  by default; set `dilate_gpu: true` to dilate via cupy instead — that
92
- needs `cupy` installed in the segment job's environment (matching your
93
- CUDA version, e.g. `pip install cupy-cuda12x`) and a GPU allocated for
96
+ needs `cupy` in the segment job's environment and a GPU allocated for
94
97
  that job (`set-resources: segment:` in `profile/slurm/config.yaml`,
95
- same as for a GPU `method`). It's independent of whatever `method`
98
+ same as for a GPU `method`). cupy ships one wheel per CUDA major
99
+ version, so pick the environment that matches yours rather than editing
100
+ `pixi.toml`:
101
+
102
+ ```bash
103
+ nvidia-smi # read the CUDA version off a GPU node
104
+ pixi install -e cuda12 # or -e cuda13
105
+ pixi run -e cuda12 multi-slurm # every task works in these too
106
+ ```
107
+
108
+ A pixi environment includes the default feature as well, so `-e cuda12`
109
+ is "everything the default has, plus cupy". `cellpose4-cuda12` and
110
+ `cellpose4-cuda13` combine it with the pinned Cellpose. It's independent of whatever `method`
96
111
  itself runs on — you can dilate on GPU even with `method: "threshold"`
97
112
  (CPU), or on CPU even with `method: "cellpose"` (GPU). See [Growing
98
113
  labels afterwards](custom_segmentation.md#growing-labels-afterwards-dilation)
99
114
  for how it works and the equivalent direct-API call.
100
115
 
116
+ !!! tip "What `shard: true` covers, and `shard_labels` for label level 0"
117
+ Without sharding, one chunk is one file, and a fine-chunked level 0 can
118
+ run to ~950k of them — painful on a shared filesystem at write time and
119
+ on every read after. `shard: true` packs chunks into far fewer files
120
+ without changing the chunking or the memory profile.
121
+
122
+ `shard: true` applies to **the converted image (all levels)** and to
123
+ **the label pyramid levels (1..N)**. It cannot apply to **label level
124
+ 0** *while that level is being written*: a shard has to be written whole
125
+ by a single writer, while level 0 is filled one chunk at a time by
126
+ concurrent `segment` jobs (or by the merge's own worker pool). Two
127
+ writers doing a read-modify-write on the same shard file silently lose
128
+ each other's chunks.
129
+
130
+ Once the merge has finished, though, nothing is writing it anymore, so a
131
+ single-threaded pass *can* rewrite it sharded. That is `shard_labels`:
132
+
133
+ ```yaml
134
+ shard: true
135
+ shard_labels: true # or an explicit shape, e.g. [16, 512, 512]
136
+ ```
137
+
138
+ `true` reuses whatever spec `shard` carries; a list overrides it.
139
+ `shard: false` does not veto it — an unsharded image with sharded labels
140
+ is a valid combination. It is opt-in because it costs one extra full
141
+ read+write of level 0. The array's attributes (including the merge's own
142
+ completion marker) are carried across, so a later rerun still sees the
143
+ merge as done.
144
+
145
+ **Set both keys**, but expect level 0 to dominate. `shard: true` sends
146
+ the pyramid down the dask path, where each level is rechunked to the
147
+ `(16, 1024, 1024)` cap — so levels 1..N shrink fourfold each, the way
148
+ you would expect. Level 0 does not: its chunks come from `tile_shape`,
149
+ it is the full-resolution level, and it is the one `shard` cannot reach.
150
+
151
+ Measured on a real `(126, 45961, 42072)` label group whose tiles are 32%
152
+ occupied (empty chunks are never written):
153
+
154
+ | level | files, `shard` only | with `shard_labels` too |
155
+ |---|---|---|
156
+ | 0 | ~10,656 | ~666 |
157
+ | 1–4 | ~462 | ~462 |
158
+ | **total** | **~11,100** | **~1,130** |
159
+
160
+ So level 0 is ~96% of a label group here, and `shard` alone barely moves
161
+ the file count. The two keys are not interchangeable and neither is
162
+ redundant — `shard` handles the pyramid, `shard_labels` handles the level
163
+ that actually holds the files.
164
+
165
+ Check whether the file count is actually a problem for your data first —
166
+ a `(126, 34000, 28500)` image at the `(16, 1024, 1024)` label chunk cap
167
+ is 7,616 files per label group, well under the 200,000 at which the
168
+ conversion starts warning. On scicore that is fine; on a filesystem with
169
+ a tighter inode or per-directory budget it may not be.
170
+
171
+ !!! tip "Which OME-ZARR version gets written (`ngff_version`)"
172
+ OME-ZARR is versioned, and the version decides the **zarr format** as
173
+ well as the metadata layout — the two are not separate choices. NGFF 0.4
174
+ is specified over zarr v2 and puts `multiscales`, `labels` and
175
+ `image-label` at the top level of the store's attributes; 0.5 is the
176
+ zarr-v3 revision and nests them under an `ome` key.
177
+
178
+ `ngff_version: "auto"` (the default) follows the installed zarr, which on
179
+ any current environment means **0.5**. Pin `"0.4"` only when a downstream
180
+ tool still cannot read 0.5:
181
+
182
+ ```yaml
183
+ ngff_version: "0.4"
184
+ shard: false # required: zarr v2 has no sharding codec
185
+ ```
186
+
187
+ That is a real trade-off, not a formality — 0.4 means a zarr-v2 store, and
188
+ sharding is a zarr-v3 feature, so `shard`/`shard_labels` stop doing
189
+ anything. The workflow refuses the combination rather than silently
190
+ ignoring it. The store's own root file changes too (`.zgroup` instead of
191
+ `zarr.json`), which the rules account for.
192
+
193
+ Writing into an **existing** store always follows that store's format,
194
+ whatever this key says: the label pyramid is written by `merge`, long
195
+ after `convert` made the image, and a store with v2 arrays and v3
196
+ metadata is one no reader can open.
197
+
198
+ **What about 0.6?** It was released on 14 September 2026 and patchworks
199
+ does not write it. It is not a version bump but a different metadata
200
+ document: RFC-5 replaces a multiscale's `axes` with `coordinateSystems`
201
+ and requires an `input`/`output` pair on every coordinate transformation.
202
+ Nothing reads it yet either — ome-zarr-py, and so napari, still default
203
+ to 0.5. Setting `ngff_version: "0.6"` is rejected with that explanation
204
+ rather than writing a store you could not open.
205
+
206
+ !!! tip "Repacking a store that was written unsharded (`pixi run reshard`)"
207
+ Sharding normally has to be chosen before a store is written, because
208
+ concurrent writers cannot share a shard file. Once the store is finished
209
+ nothing is writing it, so a single pass can repack it in place — same
210
+ data, same chunking, same metadata, far fewer files, no re-conversion and
211
+ no re-segmentation:
212
+
213
+ ```bash
214
+ pixi run reshard --store /path/to/image.zarr --dry-run # report only
215
+ pixi run reshard --store /path/to/image.zarr --labels-only
216
+ ```
217
+
218
+ It skips anything already sharded, so re-running it is a no-op, and it
219
+ carries each array's attributes across — including the merge's own
220
+ completion marker, without which a later re-run would merge already-merged
221
+ ids together.
222
+
223
+ Submit it rather than running it on a login node: it reads and writes
224
+ every level. Mind the QOS ceiling (see the warning in section 5b) —
225
+ `sbatch --qos=1day --time=12:00:00 --cpus-per-task=8 --mem=64G`.
226
+
227
+ !!! tip "Exporting a store as a single file (`.zip` or `.iso`)"
228
+ A zarr store is tens of thousands of small files, which copies slowly
229
+ everywhere and badly to Windows. Two ways to make it one file:
230
+
231
+ ```bash
232
+ pixi run zip --store /path/to/image.zarr # needs nothing extra
233
+ pixi run iso --store /path/to/image.zarr # needs an ISO builder
234
+ ```
235
+
236
+ **`.zip` is the one that always works.** It needs nothing beyond Python,
237
+ and zarr reads a store straight out of it *without unpacking*:
238
+
239
+ ```python
240
+ import zarr
241
+ store = zarr.storage.ZipStore("image.zarr.zip", mode="r")
242
+ group = zarr.open_group(store, path="image.zarr", mode="r")
243
+ ```
244
+
245
+ Windows Explorer opens it natively, and unzipping gives the store back
246
+ byte for byte. patchworks reads a bundle wherever it takes a store path,
247
+ so the viewer works on one directly — same layers, same calibration,
248
+ same auto-loaded label groups:
249
+
250
+ ```bash
251
+ pixi run -e viewer napari /path/to/image.zarr.zip
252
+ ``` It is written `ZIP_STORED` — the chunks are already
253
+ zstd-compressed, so deflating them again would cost a full pass to save
254
+ almost nothing — and entry by entry, so memory stays flat.
255
+
256
+ **Automatically, at the end of a run.** Add a `bundle:` block to the
257
+ multi config and `pixi run multi-slurm` packs the store itself, once
258
+ every segmentation *and* relation has succeeded:
259
+
260
+ ```yaml
261
+ # config/multi.yaml
262
+ bundle:
263
+ format: "zip" # or "iso"; omit the block for no bundle
264
+ qos: "1day" # `time` must stay under this QOS's MaxWall
265
+ ```
266
+
267
+ Or `--bundle zip` for a one-off. Under `--profile` it is submitted as
268
+ its own job, for the same reason the occupancy and relate steps are. A
269
+ failed relation skips it deliberately: a bundle of a half-finished run
270
+ would look complete while missing workbooks. If the packing itself
271
+ fails, nothing is lost — the store is complete on disk and
272
+ `pixi run zip` retries just that step.
273
+
274
+ **`.iso` mounts as a read-only drive**, which `.zip` does not, so the
275
+ store can be opened in place by anything that takes a path. The cost is
276
+ that it needs `xorriso`, `genisoimage` or `mkisofs` on the system, and
277
+ none of them is on conda-forge, so on a cluster without one this option
278
+ is simply unavailable.
279
+
280
+ !!! tip "Details of the `.iso` format"
281
+ A zarr store is tens of thousands of small files, which copies slowly
282
+ everywhere and badly to Windows. `pixi run iso` packs a finished store
283
+ into one image that mounts read-only with a double-click:
284
+
285
+ ```bash
286
+ pixi run iso --store /path/to/image.zarr --dry-run # report, write nothing
287
+ pixi run iso --store /path/to/image.zarr
288
+ ```
289
+
290
+ Mount it: **Windows** right-click → Mount; **macOS** double-click or
291
+ `hdiutil attach`; **Linux** `sudo mount -o loop <iso> /mnt/point`. The
292
+ store inside opens with napari/patchworks unchanged — same bytes, just
293
+ packaged.
294
+
295
+ **Submit it rather than running it on a login node**: packing a store
296
+ reads every file and writes the whole image, and a shared login node
297
+ kills a process that large with no message — the run just returns to the
298
+ prompt partway through, leaving no usable `.iso`. Same QOS ceiling as
299
+ everything else:
300
+
301
+ ```bash
302
+ sbatch --qos=1day --time=12:00:00 --cpus-per-task=4 --mem=8G \
303
+ --wrap "cd $PWD && pixi run iso --store /path/to/image.zarr"
304
+ ```
305
+
306
+ It needs **`xorriso`, `genisoimage` or `mkisofs`** from the system —
307
+ whichever is present. None of them is on conda-forge (it carries no
308
+ ISO-building C tool), so this is deliberately *not* a pixi dependency:
309
+ declaring one makes `pixi install` unsolvable and takes every
310
+ environment down with it. Check with
311
+ `which xorriso genisoimage mkisofs`, and try `module avail` before
312
+ asking an admin.
313
+
314
+ Do not substitute a pure-Python ISO builder such as `pycdlib`: those
315
+ assemble the whole image in RAM and die partway through a store with
316
+ tens of thousands of files. The C tools stream straight to the output
317
+ file, so the file count costs no memory at all. The
318
+ image is ISO-9660 level 3 with Rock Ridge *and* Joliet and deep-directory
319
+ relocation disabled, because a zarr v3 chunk path nests deeper than
320
+ ISO-9660's 8 levels — without that, Windows sees a tree flattened into
321
+ `RR_MOVED` that still looks like it copied correctly.
322
+
101
323
  !!! tip "Dropping objects by size with `min_volume`/`max_volume`"
102
324
  `min_volume: N` drops any object smaller than `N` µm³; `max_volume: N`
103
325
  drops any object larger than `N` µm³ (e.g. several objects merged into
@@ -258,6 +480,41 @@ set-resources:
258
480
  runtime: 240
259
481
  ```
260
482
 
483
+ !!! warning "`runtime` is capped by the QOS, and sbatch rejects — it does not truncate"
484
+ Every `runtime:` above is bounded by the QOS the job lands in. Ask for
485
+ more and **`sbatch` refuses the job**, so it never starts:
486
+
487
+ ```
488
+ sbatch: error: QOSMaxWallDurationPerJobLimit
489
+ sbatch: error: Batch job submission failed: Job violates accounting/QOS policy
490
+ ```
491
+
492
+ The giveaway is an **empty log file**: the rule's `logs/<rule>.log` is
493
+ created by Snakemake but nothing ever writes to it, because the script
494
+ never ran. The reason appears only in the submission error, not in the
495
+ log.
496
+
497
+ Raising `runtime` past the cap therefore does not work on its own — you
498
+ have to request a QOS that allows it, per rule:
499
+
500
+ ```yaml
501
+ set-resources:
502
+ merge:
503
+ qos: "1day" # a QOS your account may use on that partition
504
+ runtime: 720
505
+ ```
506
+
507
+ List what you may ask for, and each one's ceiling:
508
+
509
+ ```bash
510
+ sacctmgr show assoc user=$USER format=partition,qos%40
511
+ sacctmgr show qos format=name,maxwall
512
+ ```
513
+
514
+ On scicore the default QOS allows 6h, which is why the shipped profile
515
+ keeps every CPU rule at or below `runtime: 360` and gives the long
516
+ `segment` rule an explicit `qos:`.
517
+
261
518
  Then launch (from a login node — Snakemake submits and watches the jobs):
262
519
 
263
520
  ```bash
@@ -362,6 +619,7 @@ input: "/data/scan.ims"
362
619
  work_dir: "/scratch/results"
363
620
  tile_shape: [16, 1024, 1024]
364
621
  shard: false # true → far fewer files, same chunks
622
+ ngff_version: "auto" # convert reads it, so it belongs here
365
623
  tiles_per_job: 4
366
624
  ```
367
625
 
@@ -447,11 +705,21 @@ snakemake --workflow-profile profile/slurm --configfile config/common.yaml confi
447
705
  ```
448
706
 
449
707
  !!! warning "Conversion settings belong in the shared file"
450
- `convert` runs **once**, from the first config only. A `shard`, `input` or
451
- `pyramid_levels` set on the second config is therefore never read, and
452
- nothing logs that it was dropped. `run_multi` refuses to start when those
453
- keys disagree across configs and tells you which one — but if you drive
454
- the configs by hand, keep them in `common.yaml`.
708
+ `convert` runs **once**, from the first config only. A `shard`, `input`,
709
+ `convert_chunks`, `sequence_pattern` or `reuse_pyramid` set on the second
710
+ config is therefore never read, and nothing logs that it was dropped.
711
+ `run_multi` refuses to start when those keys disagree across configs and
712
+ tells you which one — but if you drive the configs by hand, keep them in
713
+ `common.yaml`.
714
+
715
+ `ngff_version` is in that list: it decides the store's zarr format, so a
716
+ second config disagreeing about it would describe a store that is not the
717
+ one on disk.
718
+
719
+ `merge` runs once **per config**, so the keys it reads —
720
+ `pyramid_levels`, `pyramid_downscale`, `sequential_labels`,
721
+ `min_volume`/`max_volume` and `shard_labels` — may legitimately differ
722
+ between them, and are only in `common.yaml` for convenience.
455
723
 
456
724
  Splitting the configs is optional: a self-contained config still works,
457
725
  and `common:` can simply be left out of `multi.yaml`.
@@ -560,9 +828,30 @@ abort the others; you get a per-config status and a non-zero exit.
560
828
  caps the wall time below `--relate-time` — `srun` fails immediately with
561
829
  `QOSMaxWallDurationPerJobLimit` when that happens; `sacctmgr -p show
562
830
  assoc user=$USER` and `sacctmgr -p show qos` list what's available and
563
- each one's `MaxWall`. Under plain `multi` (no `--profile`), relations
831
+ each one's `MaxWall`.
832
+
833
+ The same settings can live in the multi config as a `relate:` block, so
834
+ the shipped `pixi run multi-slurm` task keeps working without extra
835
+ flags:
836
+
837
+ ```yaml
838
+ # config/multi.yaml
839
+ relate:
840
+ qos: "1day"
841
+ time: 720 # minutes, per pair; must stay under that QOS's MaxWall
842
+ ```
843
+
844
+ A `--relate-*` flag overrides the block for that one key; anything the
845
+ block does not set keeps its default. An unknown key there is an error
846
+ rather than silently ignored, since a typo would otherwise run with the
847
+ default you meant to replace. Under plain `multi` (no `--profile`), relations
564
848
  still run locally, in-process, one after another, as before.
565
849
 
850
+ Each pair logs its shape, chunk count and object count before it starts,
851
+ then a progress line roughly once a minute (`label_relations: 412/3,600
852
+ (11%) after 7m, ~55m left`), so a long relation is distinguishable from a
853
+ hung one in `logs/relate/<a>_to_<b>.log`.
854
+
566
855
  Because every pair gets its own job, one running long no longer starves
567
856
  the others out of a shared time budget, and a pair that gets killed no
568
857
  longer takes an already-finished sibling's workbook down with it.
@@ -96,7 +96,16 @@ napari = [
96
96
  # openpyxl -> scripts/run_multi.py writes label_relations() output as an
97
97
  # Excel workbook (per-object + per-container sheets), not a plain CSV.
98
98
  workflow = ["snakemake>=8", "snakemake-executor-plugin-slurm", "openpyxl"]
99
- dev = ["pytest", "pytest-cov", "scikit-image", "psutil", "tqdm"]
99
+ # jsonschema validates what we write against the vendored official
100
+ # OME-NGFF schemas (tests/ngff_schemas/); without it that one test skips.
101
+ dev = [
102
+ "pytest",
103
+ "pytest-cov",
104
+ "scikit-image",
105
+ "psutil",
106
+ "tqdm",
107
+ "jsonschema>=4.18",
108
+ ]
100
109
  docs = ["mkdocs-material>=9.0", "mkdocstrings[python]>=0.24"]
101
110
  all = [
102
111
  "patchworks[io,gpu,bioio,imaris,napari]",
@@ -449,6 +449,18 @@ def auto_tile_shape_cellpose(
449
449
  (1, 2048, 2048)
450
450
  """
451
451
  n_workers = n_workers or cpu_allocation()
452
+ # Cellpose resizes the tile before the net runs -- by `rescale`
453
+ # (= 30 / diameter) on every axis, and by `anisotropy` on z as well -- so
454
+ # the array it actually holds is bigger than the one it was handed, and a
455
+ # budget computed from the unresized tile under-counts by that factor. A
456
+ # diameter half the model's 30 px means a 2x upsample per axis: 8x the
457
+ # voxels, enough to turn a comfortable tile into an OOM.
458
+ #
459
+ # Both only ever *shrink* the tile. A predicted downsample (diameter > 30)
460
+ # would license a bigger one, but these are a safety margin against a
461
+ # rough memory model, not a measurement to spend headroom on.
462
+ rescale = max(1.0, 30.0 / diameter) if diameter else 1.0
463
+ z_resize = max(1.0, anisotropy or 1.0)
452
464
  # A tile holds n_channels planes per voxel (e.g. Cellpose's
453
465
  # cyto+nuclei pair), so the per-voxel cost -- and every budget
454
466
  # derived from it below -- scales with them.
@@ -475,7 +487,8 @@ def auto_tile_shape_cellpose(
475
487
  min_tile = int(4 * diameter) if diameter is not None else 1
476
488
 
477
489
  if n_spatial == 2 or not do_3D:
478
- max_pixels_2d = max(1, max_raw_bytes // itemsize)
490
+ # Two axes resized, so the cost per configured pixel is rescale**2.
491
+ max_pixels_2d = max(1, int(max_raw_bytes // (itemsize * rescale**2)))
479
492
  tile_side = max(min_tile, int(max_pixels_2d**0.5))
480
493
  if n_spatial == 2:
481
494
  y, x = shape[-2], shape[-1]
@@ -485,9 +498,10 @@ def auto_tile_shape_cellpose(
485
498
  chunk_spatial = [1, min(y, tile_side), min(x, tile_side)]
486
499
  else:
487
500
  z, y, x = shape[-3], shape[-2], shape[-1]
488
- # Cellpose resizes z by `anisotropy` before the net runs, so the tile
489
- # it actually holds is that much deeper than the one handed to it.
490
- effective_z = z * max(1.0, anisotropy or 1.0)
501
+ # All three axes are resized: z by anisotropy * rescale, y/x by
502
+ # rescale each -- so one configured voxel costs
503
+ # anisotropy * rescale**3 of them.
504
+ effective_z = z * z_resize * rescale**3
491
505
  max_pixels_per_slice = max(
492
506
  1, int((max_raw_bytes // 3) // (effective_z * itemsize))
493
507
  )
@@ -18,32 +18,128 @@ _ZARR_V3 = int(zarr.__version__.split(".")[0]) >= 3
18
18
  _ZARR_V3 = int(zarr.__version__.split(".")[0]) >= 3
19
19
 
20
20
 
21
- def zarr_compressor_kwargs() -> dict:
21
+ def zarr_compressor_kwargs(zarr_format: int = 3) -> dict:
22
22
  """Keyword arguments pinning the compression codec for a new array.
23
23
 
24
24
  zstd is already zarr v3's default, but relying on a library default means
25
25
  the stores patchworks writes change silently if that default ever moves.
26
26
  Labels in particular are highly compressible, so this is worth stating.
27
27
 
28
+ The codec *object* depends on the format of the array being written, not
29
+ on the installed zarr: zarr-python 3 can write a zarr-v2 array (which is
30
+ what NGFF 0.4 needs), and a v2 array rejects ``zarr.codecs.ZstdCodec`` --
31
+ it wants the numcodecs one.
32
+
33
+ Parameters
34
+ ----------
35
+ zarr_format : int, optional
36
+ Format of the array about to be created, 2 or 3 (default 3).
37
+
28
38
  Returns
29
39
  -------
30
40
  dict
31
- ``compressors=``/``compressor=`` as the installed zarr expects, or
41
+ ``compressors=``/``compressor=`` as that combination expects, or
32
42
  empty if the codec cannot be built (then the default applies).
33
43
  """
34
44
  try:
35
- if _ZARR_V3:
45
+ if _ZARR_V3 and zarr_format != 2:
36
46
  from zarr.codecs import ZstdCodec
37
47
 
38
48
  return {"compressors": (ZstdCodec(level=1),)}
39
49
  import numcodecs
40
50
 
41
- return {"compressor": numcodecs.Zstd(level=1)}
51
+ codec = numcodecs.Zstd(level=1)
52
+ return {"compressors": (codec,)} if _ZARR_V3 else {"compressor": codec}
42
53
  except Exception: # pragma: no cover - depends on the installed zarr
43
54
  logger.debug("could not pin a compressor; using zarr's default")
44
55
  return {}
45
56
 
46
57
 
58
+ def open_zarr_source(
59
+ store_path: Union[str, Path],
60
+ ) -> tuple[Union[str, "zarr.storage.StoreLike"], str]:
61
+ """Resolve a store path, transparently opening a ``.zip`` bundle.
62
+
63
+ A store packed by ``pixi run zip`` is one file holding
64
+ ``<name>.zarr/...``. zarr reads it in place through a ``ZipStore``, so
65
+ nothing has to be unpacked first -- but a plain path string does not,
66
+ and every reader here takes a path. This returns what zarr and dask
67
+ should actually be handed, plus the prefix to prepend to a component.
68
+
69
+ Parameters
70
+ ----------
71
+ store_path : str or Path
72
+ A ``.zarr`` directory, or a ``.zip`` bundle containing one.
73
+
74
+ Returns
75
+ -------
76
+ tuple
77
+ ``(source, prefix)``. For a directory, the path and ``""``. For a
78
+ bundle, an open read-only ``ZipStore`` and the store's name inside
79
+ it, so a component is addressed as ``f"{prefix}/{component}"``.
80
+
81
+ Raises
82
+ ------
83
+ ValueError
84
+ If a ``.zip`` does not hold exactly one top-level store.
85
+ """
86
+ text = str(store_path)
87
+ if ".zip" not in text:
88
+ return text, ""
89
+
90
+ import zipfile
91
+
92
+ # The bundle may be addressed with a group path after it, e.g.
93
+ # "scan.zarr.zip/labels/cells" -- callers build those by string-joining.
94
+ head, _, tail = text.partition(".zip")
95
+ archive_path = head + ".zip"
96
+ with zipfile.ZipFile(archive_path) as archive:
97
+ tops = {
98
+ name.split("/", 1)[0] for name in archive.namelist() if "/" in name
99
+ }
100
+ if len(tops) != 1:
101
+ raise ValueError(
102
+ f"{archive_path} must contain exactly one top-level store; found "
103
+ f"{sorted(tops) or 'nothing'}. Bundles written by "
104
+ "`pixi run zip` always do."
105
+ )
106
+ prefix = tops.pop()
107
+ inner = tail.strip("/")
108
+ if inner:
109
+ prefix = f"{prefix}/{inner}"
110
+ return zarr.storage.ZipStore(archive_path, mode="r"), prefix
111
+
112
+
113
+ def open_group_any(path: Union[str, Path], mode: str = "r"):
114
+ """``zarr.open_group`` that also accepts a path *inside* a .zip bundle.
115
+
116
+ Callers build group paths by string-joining (``f"{store}/labels"``),
117
+ which a bundle breaks: the archive is a file, not a directory. Split on
118
+ the ``.zip`` instead, so ``bundle.zip/labels/cells`` resolves to the
119
+ right group inside it.
120
+ """
121
+ source, prefix = open_zarr_source(path)
122
+ return zarr.open_group(source, path=prefix, mode=mode)
123
+
124
+
125
+ def from_zarr_any(path: Union[str, Path], component: str | None = None):
126
+ """``dask.array.from_zarr`` that also accepts a .zip bundle."""
127
+ import dask.array as _da
128
+
129
+ source, prefix = open_zarr_source(path)
130
+ inner = _component(prefix, component) if component else prefix
131
+ return (
132
+ _da.from_zarr(source, component=inner)
133
+ if inner
134
+ else _da.from_zarr(source)
135
+ )
136
+
137
+
138
+ def _component(prefix: str, name: str) -> str:
139
+ """Join a bundle prefix and a component, tolerating an empty prefix."""
140
+ return f"{prefix}/{name}" if prefix else name
141
+
142
+
47
143
  def load_ome_zarr(
48
144
  store_path: Union[str, Path],
49
145
  channel: int | None = 0,
@@ -75,7 +171,8 @@ def load_ome_zarr(
75
171
  >>> arr.shape
76
172
  (128, 2048, 2048)
77
173
  """
78
- root = zarr.open_group(str(store_path), mode="r")
174
+ source, prefix = open_zarr_source(store_path)
175
+ root = zarr.open_group(source, path=prefix, mode="r")
79
176
  # OME-ZARR 0.5 nests under "ome" key; older stores use "multiscales" directly
80
177
  _attrs = dict(root.attrs)
81
178
  _ms = _attrs.get("multiscales") or _attrs.get("ome", {}).get("multiscales")
@@ -93,7 +190,9 @@ def load_ome_zarr(
93
190
  if zarr_ndim > len(chunks):
94
191
  zarr_chunks = (1,) * (zarr_ndim - len(chunks)) + tuple(chunks)
95
192
 
96
- arr = da.from_zarr(str(store_path), component=path, chunks=zarr_chunks)
193
+ arr = da.from_zarr(
194
+ source, component=_component(prefix, path), chunks=zarr_chunks
195
+ )
97
196
  if channel is not None:
98
197
  arr = _select_channel(arr, channel, _ms[0], store_path)
99
198
  return arr