patchworks 2.6.12__tar.gz → 2.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {patchworks-2.6.12 → patchworks-2.8.0}/PKG-INFO +2 -1
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/plugins/ome_zarr.md +12 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/custom_segmentation.md +5 -2
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/snakemake.md +299 -10
- {patchworks-2.6.12 → patchworks-2.8.0}/pyproject.toml +10 -1
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_chunks.py +18 -4
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_io.py +105 -6
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_relations.py +26 -2
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/plugins/napari.py +7 -8
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/plugins/ome_zarr.py +466 -166
- patchworks-2.8.0/tests/ngff_schemas/0.4/image.schema +233 -0
- patchworks-2.8.0/tests/ngff_schemas/0.4/label.schema +77 -0
- patchworks-2.8.0/tests/ngff_schemas/0.4/ome.schema +17 -0
- patchworks-2.8.0/tests/ngff_schemas/0.5/_version.schema +10 -0
- patchworks-2.8.0/tests/ngff_schemas/0.5/image.schema +268 -0
- patchworks-2.8.0/tests/ngff_schemas/0.5/label.schema +91 -0
- patchworks-2.8.0/tests/ngff_schemas/0.5/ome.schema +33 -0
- patchworks-2.8.0/tests/ngff_schemas/README.md +16 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_allocation.py +28 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_napari.py +75 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_ome_zarr.py +381 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_pw.py +60 -0
- patchworks-2.8.0/tests/test_relations.py +104 -0
- patchworks-2.8.0/tests/test_run_multi.py +1120 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/config/common.yaml +6 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/config/config.yaml +14 -0
- patchworks-2.8.0/workflow/config/multi.yaml +75 -0
- patchworks-2.8.0/workflow/pixi.toml +126 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/profile/slurm/config.yaml +28 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/rules/common.smk +9 -2
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/_pw.py +38 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/build_occupancy.py +5 -1
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/convert.py +8 -2
- patchworks-2.8.0/workflow/scripts/export_iso.py +314 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/merge.py +73 -2
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/relate.py +35 -0
- patchworks-2.8.0/workflow/scripts/reshard_store.py +196 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/run_multi.py +268 -18
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/view.py +10 -1
- patchworks-2.6.12/tests/test_relations.py +0 -47
- patchworks-2.6.12/tests/test_run_multi.py +0 -537
- patchworks-2.6.12/workflow/config/multi.yaml +0 -36
- patchworks-2.6.12/workflow/pixi.toml +0 -66
- {patchworks-2.6.12 → patchworks-2.8.0}/.github/workflows/docs.yml +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/.github/workflows/lint.yml +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/.github/workflows/release.yml +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/.gitignore +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/.markdownlint-cli2.yaml +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/LICENSE +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/README.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/cliff.toml +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/chunks.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/cluster.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/io.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/merge_tile_labels.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/plugins/cellpose.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/plugins/dog.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/plugins/napari.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/postprocess.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/relabel.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/tile_process.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/api/volume_filter.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/assets/logo.png +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/cellpose_2d.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/cellpose_2d.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/cellpose_3d.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/cellpose_3d.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/custom.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/custom_method.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/dog.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/dog.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/standalone_merge.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/stardist.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/examples/stardist_2d.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/getting_started.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/gpu_distributed.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/label_relations.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/measurements.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/merging.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/ome_zarr_napari.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/performance.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/pitfalls.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/skip_empty.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/guide/tiling.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/docs/index.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/mkdocs.yml +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/__init__.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_cluster.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_core.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_distributed.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_gpu.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_merge.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_notify.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_occupancy.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_postprocess.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_progress.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_relabel.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/_volume_filter.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/plugins/__init__.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/plugins/cellpose.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/src/patchworks/plugins/dog.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_cellpose.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_core.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_distributed.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_dog.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_gpu.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_notify.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_occupancy.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_postprocess.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_progress.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/tests/test_volume_filter.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/README.md +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/Snakefile +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/config/config_cilia.yaml +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/config/config_cyto.yaml +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/config/config_nuclei.yaml +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/rules/convert.smk +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/rules/merge.smk +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/rules/segment.smk +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/fetch_model.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/prepare_tiles.py +0 -0
- {patchworks-2.6.12 → patchworks-2.8.0}/workflow/scripts/segment_tile.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: patchworks
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.8.0
|
|
4
4
|
Summary: Tiled processing of arbitrarily large images with globally consistent labels
|
|
5
5
|
Project-URL: Homepage, https://github.com/imcf/patchworks
|
|
6
6
|
Project-URL: Issues, https://github.com/imcf/patchworks/issues
|
|
@@ -57,6 +57,7 @@ Requires-Dist: cellpose<4,>=3.0; extra == 'cellpose3'
|
|
|
57
57
|
Provides-Extra: cellpose4
|
|
58
58
|
Requires-Dist: cellpose>=4; extra == 'cellpose4'
|
|
59
59
|
Provides-Extra: dev
|
|
60
|
+
Requires-Dist: jsonschema>=4.18; extra == 'dev'
|
|
60
61
|
Requires-Dist: psutil; extra == 'dev'
|
|
61
62
|
Requires-Dist: pytest; extra == 'dev'
|
|
62
63
|
Requires-Dist: pytest-cov; extra == 'dev'
|
|
@@ -25,6 +25,10 @@ resolution, matching anisotropic microscopy stacks.
|
|
|
25
25
|
|
|
26
26
|
::: patchworks.plugins.ome_zarr.register_labels
|
|
27
27
|
|
|
28
|
+
## reshard_level
|
|
29
|
+
|
|
30
|
+
::: patchworks.plugins.ome_zarr.reshard_level
|
|
31
|
+
|
|
28
32
|
## read_pixel_size
|
|
29
33
|
|
|
30
34
|
::: patchworks.plugins.ome_zarr.read_pixel_size
|
|
@@ -37,6 +41,14 @@ matches the store, and reads both.
|
|
|
37
41
|
|
|
38
42
|
::: patchworks.plugins.ome_zarr.ngff_version
|
|
39
43
|
|
|
44
|
+
Every writer takes an `ngff_version=` keyword: `"auto"` (default) follows the
|
|
45
|
+
installed zarr — 0.5 on v3, 0.4 on v2 — and `"0.4"` pins the older, zarr-v2
|
|
46
|
+
layout. Writing into an existing store always follows *that store's* format,
|
|
47
|
+
so a label pyramid added later can never disagree with the image it sits in.
|
|
48
|
+
NGFF 0.6 is released but not written: RFC-5 replaces `axes` with
|
|
49
|
+
`coordinateSystems` and requires `input`/`output` on every coordinate
|
|
50
|
+
transformation, and no reader supports it yet.
|
|
51
|
+
|
|
40
52
|
::: patchworks.plugins.ome_zarr.read_ngff_attr
|
|
41
53
|
|
|
42
54
|
::: patchworks.plugins.ome_zarr.write_ngff_attrs
|
|
@@ -81,8 +81,11 @@ backend `fn` used — pass `use_gpu=True` to dilate via cupy instead:
|
|
|
81
81
|
fn = dilate_labels(fn, iterations=2, use_gpu=True)
|
|
82
82
|
```
|
|
83
83
|
|
|
84
|
-
Needs `cupy
|
|
85
|
-
|
|
84
|
+
Needs `cupy`, matching your CUDA version. Under the Snakemake workflow, use
|
|
85
|
+
the environment that already carries it — `pixi install -e cuda12` (or
|
|
86
|
+
`-e cuda13`), then `pixi run -e cuda12 <task>`; outside it, install the
|
|
87
|
+
matching wheel yourself (e.g. `pip install cupy-cuda12x`). It's never
|
|
88
|
+
installed automatically by
|
|
86
89
|
patchworks, unlike Cellpose's GPU support (which comes for free via
|
|
87
90
|
PyTorch's self-contained CUDA wheels); cupy ships one wheel per CUDA major
|
|
88
91
|
version, so there's no single generic pin that works everywhere. On the
|
|
@@ -50,7 +50,9 @@ work_dir: "/scratch/results" # everything is written here
|
|
|
50
50
|
# conversion (input → pyramidal OME-ZARR)
|
|
51
51
|
reuse_pyramid: true # .ims: copy its own pyramid (fast)
|
|
52
52
|
convert_chunks: null # null → bounded auto chunks; or [c,z,y,x]
|
|
53
|
-
shard: false # true → pack chunks into shards (fewer files)
|
|
53
|
+
shard: false # true → pack chunks into shards (fewer files);
|
|
54
|
+
# covers the image and the label pyramids
|
|
55
|
+
ngff_version: "auto" # OME-ZARR version: "auto" (0.5), or "0.4"
|
|
54
56
|
|
|
55
57
|
# tiling
|
|
56
58
|
channel: 0 # channel to segment, 0-based (null = keep all)
|
|
@@ -83,21 +85,241 @@ cellpose:
|
|
|
83
85
|
pyramid_levels: 5
|
|
84
86
|
pyramid_downscale: 2
|
|
85
87
|
sequential_labels: true # renumber labels to a contiguous 1..N
|
|
88
|
+
shard_labels: false # true → also reshard label level 0 after the
|
|
89
|
+
# merge (one extra pass; see tip below)
|
|
86
90
|
```
|
|
87
91
|
|
|
88
92
|
!!! tip "Growing labels after segmentation"
|
|
89
93
|
`dilate: N` grows every label by `N` pixels once segmentation finishes,
|
|
90
94
|
regardless of `method`. `0` (default) disables it. Runs on CPU (scipy)
|
|
91
95
|
by default; set `dilate_gpu: true` to dilate via cupy instead — that
|
|
92
|
-
needs `cupy`
|
|
93
|
-
CUDA version, e.g. `pip install cupy-cuda12x`) and a GPU allocated for
|
|
96
|
+
needs `cupy` in the segment job's environment and a GPU allocated for
|
|
94
97
|
that job (`set-resources: segment:` in `profile/slurm/config.yaml`,
|
|
95
|
-
same as for a GPU `method`).
|
|
98
|
+
same as for a GPU `method`). cupy ships one wheel per CUDA major
|
|
99
|
+
version, so pick the environment that matches yours rather than editing
|
|
100
|
+
`pixi.toml`:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
nvidia-smi # read the CUDA version off a GPU node
|
|
104
|
+
pixi install -e cuda12 # or -e cuda13
|
|
105
|
+
pixi run -e cuda12 multi-slurm # every task works in these too
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
A pixi environment includes the default feature as well, so `-e cuda12`
|
|
109
|
+
is "everything the default has, plus cupy". `cellpose4-cuda12` and
|
|
110
|
+
`cellpose4-cuda13` combine it with the pinned Cellpose. It's independent of whatever `method`
|
|
96
111
|
itself runs on — you can dilate on GPU even with `method: "threshold"`
|
|
97
112
|
(CPU), or on CPU even with `method: "cellpose"` (GPU). See [Growing
|
|
98
113
|
labels afterwards](custom_segmentation.md#growing-labels-afterwards-dilation)
|
|
99
114
|
for how it works and the equivalent direct-API call.
|
|
100
115
|
|
|
116
|
+
!!! tip "What `shard: true` covers, and `shard_labels` for label level 0"
|
|
117
|
+
Without sharding, one chunk is one file, and a fine-chunked level 0 can
|
|
118
|
+
run to ~950k of them — painful on a shared filesystem at write time and
|
|
119
|
+
on every read after. `shard: true` packs chunks into far fewer files
|
|
120
|
+
without changing the chunking or the memory profile.
|
|
121
|
+
|
|
122
|
+
`shard: true` applies to **the converted image (all levels)** and to
|
|
123
|
+
**the label pyramid levels (1..N)**. It cannot apply to **label level
|
|
124
|
+
0** *while that level is being written*: a shard has to be written whole
|
|
125
|
+
by a single writer, while level 0 is filled one chunk at a time by
|
|
126
|
+
concurrent `segment` jobs (or by the merge's own worker pool). Two
|
|
127
|
+
writers doing a read-modify-write on the same shard file silently lose
|
|
128
|
+
each other's chunks.
|
|
129
|
+
|
|
130
|
+
Once the merge has finished, though, nothing is writing it anymore, so a
|
|
131
|
+
single-threaded pass *can* rewrite it sharded. That is `shard_labels`:
|
|
132
|
+
|
|
133
|
+
```yaml
|
|
134
|
+
shard: true
|
|
135
|
+
shard_labels: true # or an explicit shape, e.g. [16, 512, 512]
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
`true` reuses whatever spec `shard` carries; a list overrides it.
|
|
139
|
+
`shard: false` does not veto it — an unsharded image with sharded labels
|
|
140
|
+
is a valid combination. It is opt-in because it costs one extra full
|
|
141
|
+
read+write of level 0. The array's attributes (including the merge's own
|
|
142
|
+
completion marker) are carried across, so a later rerun still sees the
|
|
143
|
+
merge as done.
|
|
144
|
+
|
|
145
|
+
**Set both keys**, but expect level 0 to dominate. `shard: true` sends
|
|
146
|
+
the pyramid down the dask path, where each level is rechunked to the
|
|
147
|
+
`(16, 1024, 1024)` cap — so levels 1..N shrink fourfold each, the way
|
|
148
|
+
you would expect. Level 0 does not: its chunks come from `tile_shape`,
|
|
149
|
+
it is the full-resolution level, and it is the one `shard` cannot reach.
|
|
150
|
+
|
|
151
|
+
Measured on a real `(126, 45961, 42072)` label group whose tiles are 32%
|
|
152
|
+
occupied (empty chunks are never written):
|
|
153
|
+
|
|
154
|
+
| level | files, `shard` only | with `shard_labels` too |
|
|
155
|
+
|---|---|---|
|
|
156
|
+
| 0 | ~10,656 | ~666 |
|
|
157
|
+
| 1–4 | ~462 | ~462 |
|
|
158
|
+
| **total** | **~11,100** | **~1,130** |
|
|
159
|
+
|
|
160
|
+
So level 0 is ~96% of a label group here, and `shard` alone barely moves
|
|
161
|
+
the file count. The two keys are not interchangeable and neither is
|
|
162
|
+
redundant — `shard` handles the pyramid, `shard_labels` handles the level
|
|
163
|
+
that actually holds the files.
|
|
164
|
+
|
|
165
|
+
Check whether the file count is actually a problem for your data first —
|
|
166
|
+
a `(126, 34000, 28500)` image at the `(16, 1024, 1024)` label chunk cap
|
|
167
|
+
is 7,616 files per label group, well under the 200,000 at which the
|
|
168
|
+
conversion starts warning. On scicore that is fine; on a filesystem with
|
|
169
|
+
a tighter inode or per-directory budget it may not be.
|
|
170
|
+
|
|
171
|
+
!!! tip "Which OME-ZARR version gets written (`ngff_version`)"
|
|
172
|
+
OME-ZARR is versioned, and the version decides the **zarr format** as
|
|
173
|
+
well as the metadata layout — the two are not separate choices. NGFF 0.4
|
|
174
|
+
is specified over zarr v2 and puts `multiscales`, `labels` and
|
|
175
|
+
`image-label` at the top level of the store's attributes; 0.5 is the
|
|
176
|
+
zarr-v3 revision and nests them under an `ome` key.
|
|
177
|
+
|
|
178
|
+
`ngff_version: "auto"` (the default) follows the installed zarr, which on
|
|
179
|
+
any current environment means **0.5**. Pin `"0.4"` only when a downstream
|
|
180
|
+
tool still cannot read 0.5:
|
|
181
|
+
|
|
182
|
+
```yaml
|
|
183
|
+
ngff_version: "0.4"
|
|
184
|
+
shard: false # required: zarr v2 has no sharding codec
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
That is a real trade-off, not a formality — 0.4 means a zarr-v2 store, and
|
|
188
|
+
sharding is a zarr-v3 feature, so `shard`/`shard_labels` stop doing
|
|
189
|
+
anything. The workflow refuses the combination rather than silently
|
|
190
|
+
ignoring it. The store's own root file changes too (`.zgroup` instead of
|
|
191
|
+
`zarr.json`), which the rules account for.
|
|
192
|
+
|
|
193
|
+
Writing into an **existing** store always follows that store's format,
|
|
194
|
+
whatever this key says: the label pyramid is written by `merge`, long
|
|
195
|
+
after `convert` made the image, and a store with v2 arrays and v3
|
|
196
|
+
metadata is one no reader can open.
|
|
197
|
+
|
|
198
|
+
**What about 0.6?** It was released on 14 September 2026 and patchworks
|
|
199
|
+
does not write it. It is not a version bump but a different metadata
|
|
200
|
+
document: RFC-5 replaces a multiscale's `axes` with `coordinateSystems`
|
|
201
|
+
and requires an `input`/`output` pair on every coordinate transformation.
|
|
202
|
+
Nothing reads it yet either — ome-zarr-py, and so napari, still default
|
|
203
|
+
to 0.5. Setting `ngff_version: "0.6"` is rejected with that explanation
|
|
204
|
+
rather than writing a store you could not open.
|
|
205
|
+
|
|
206
|
+
!!! tip "Repacking a store that was written unsharded (`pixi run reshard`)"
|
|
207
|
+
Sharding normally has to be chosen before a store is written, because
|
|
208
|
+
concurrent writers cannot share a shard file. Once the store is finished
|
|
209
|
+
nothing is writing it, so a single pass can repack it in place — same
|
|
210
|
+
data, same chunking, same metadata, far fewer files, no re-conversion and
|
|
211
|
+
no re-segmentation:
|
|
212
|
+
|
|
213
|
+
```bash
|
|
214
|
+
pixi run reshard --store /path/to/image.zarr --dry-run # report only
|
|
215
|
+
pixi run reshard --store /path/to/image.zarr --labels-only
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
It skips anything already sharded, so re-running it is a no-op, and it
|
|
219
|
+
carries each array's attributes across — including the merge's own
|
|
220
|
+
completion marker, without which a later re-run would merge already-merged
|
|
221
|
+
ids together.
|
|
222
|
+
|
|
223
|
+
Submit it rather than running it on a login node: it reads and writes
|
|
224
|
+
every level. Mind the QOS ceiling (see the warning in section 5b) —
|
|
225
|
+
`sbatch --qos=1day --time=12:00:00 --cpus-per-task=8 --mem=64G`.
|
|
226
|
+
|
|
227
|
+
!!! tip "Exporting a store as a single file (`.zip` or `.iso`)"
|
|
228
|
+
A zarr store is tens of thousands of small files, which copies slowly
|
|
229
|
+
everywhere and badly to Windows. Two ways to make it one file:
|
|
230
|
+
|
|
231
|
+
```bash
|
|
232
|
+
pixi run zip --store /path/to/image.zarr # needs nothing extra
|
|
233
|
+
pixi run iso --store /path/to/image.zarr # needs an ISO builder
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
**`.zip` is the one that always works.** It needs nothing beyond Python,
|
|
237
|
+
and zarr reads a store straight out of it *without unpacking*:
|
|
238
|
+
|
|
239
|
+
```python
|
|
240
|
+
import zarr
|
|
241
|
+
store = zarr.storage.ZipStore("image.zarr.zip", mode="r")
|
|
242
|
+
group = zarr.open_group(store, path="image.zarr", mode="r")
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
Windows Explorer opens it natively, and unzipping gives the store back
|
|
246
|
+
byte for byte. patchworks reads a bundle wherever it takes a store path,
|
|
247
|
+
so the viewer works on one directly — same layers, same calibration,
|
|
248
|
+
same auto-loaded label groups:
|
|
249
|
+
|
|
250
|
+
```bash
|
|
251
|
+
pixi run -e viewer napari /path/to/image.zarr.zip
|
|
252
|
+
``` It is written `ZIP_STORED` — the chunks are already
|
|
253
|
+
zstd-compressed, so deflating them again would cost a full pass to save
|
|
254
|
+
almost nothing — and entry by entry, so memory stays flat.
|
|
255
|
+
|
|
256
|
+
**Automatically, at the end of a run.** Add a `bundle:` block to the
|
|
257
|
+
multi config and `pixi run multi-slurm` packs the store itself, once
|
|
258
|
+
every segmentation *and* relation has succeeded:
|
|
259
|
+
|
|
260
|
+
```yaml
|
|
261
|
+
# config/multi.yaml
|
|
262
|
+
bundle:
|
|
263
|
+
format: "zip" # or "iso"; omit the block for no bundle
|
|
264
|
+
qos: "1day" # `time` must stay under this QOS's MaxWall
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
Or `--bundle zip` for a one-off. Under `--profile` it is submitted as
|
|
268
|
+
its own job, for the same reason the occupancy and relate steps are. A
|
|
269
|
+
failed relation skips it deliberately: a bundle of a half-finished run
|
|
270
|
+
would look complete while missing workbooks. If the packing itself
|
|
271
|
+
fails, nothing is lost — the store is complete on disk and
|
|
272
|
+
`pixi run zip` retries just that step.
|
|
273
|
+
|
|
274
|
+
**`.iso` mounts as a read-only drive**, which `.zip` does not, so the
|
|
275
|
+
store can be opened in place by anything that takes a path. The cost is
|
|
276
|
+
that it needs `xorriso`, `genisoimage` or `mkisofs` on the system, and
|
|
277
|
+
none of them is on conda-forge, so on a cluster without one this option
|
|
278
|
+
is simply unavailable.
|
|
279
|
+
|
|
280
|
+
!!! tip "Details of the `.iso` format"
|
|
281
|
+
A zarr store is tens of thousands of small files, which copies slowly
|
|
282
|
+
everywhere and badly to Windows. `pixi run iso` packs a finished store
|
|
283
|
+
into one image that mounts read-only with a double-click:
|
|
284
|
+
|
|
285
|
+
```bash
|
|
286
|
+
pixi run iso --store /path/to/image.zarr --dry-run # report, write nothing
|
|
287
|
+
pixi run iso --store /path/to/image.zarr
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
Mount it: **Windows** right-click → Mount; **macOS** double-click or
|
|
291
|
+
`hdiutil attach`; **Linux** `sudo mount -o loop <iso> /mnt/point`. The
|
|
292
|
+
store inside opens with napari/patchworks unchanged — same bytes, just
|
|
293
|
+
packaged.
|
|
294
|
+
|
|
295
|
+
**Submit it rather than running it on a login node**: packing a store
|
|
296
|
+
reads every file and writes the whole image, and a shared login node
|
|
297
|
+
kills a process that large with no message — the run just returns to the
|
|
298
|
+
prompt partway through, leaving no usable `.iso`. Same QOS ceiling as
|
|
299
|
+
everything else:
|
|
300
|
+
|
|
301
|
+
```bash
|
|
302
|
+
sbatch --qos=1day --time=12:00:00 --cpus-per-task=4 --mem=8G \
|
|
303
|
+
--wrap "cd $PWD && pixi run iso --store /path/to/image.zarr"
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
It needs **`xorriso`, `genisoimage` or `mkisofs`** from the system —
|
|
307
|
+
whichever is present. None of them is on conda-forge (it carries no
|
|
308
|
+
ISO-building C tool), so this is deliberately *not* a pixi dependency:
|
|
309
|
+
declaring one makes `pixi install` unsolvable and takes every
|
|
310
|
+
environment down with it. Check with
|
|
311
|
+
`which xorriso genisoimage mkisofs`, and try `module avail` before
|
|
312
|
+
asking an admin.
|
|
313
|
+
|
|
314
|
+
Do not substitute a pure-Python ISO builder such as `pycdlib`: those
|
|
315
|
+
assemble the whole image in RAM and die partway through a store with
|
|
316
|
+
tens of thousands of files. The C tools stream straight to the output
|
|
317
|
+
file, so the file count costs no memory at all. The
|
|
318
|
+
image is ISO-9660 level 3 with Rock Ridge *and* Joliet and deep-directory
|
|
319
|
+
relocation disabled, because a zarr v3 chunk path nests deeper than
|
|
320
|
+
ISO-9660's 8 levels — without that, Windows sees a tree flattened into
|
|
321
|
+
`RR_MOVED` that still looks like it copied correctly.
|
|
322
|
+
|
|
101
323
|
!!! tip "Dropping objects by size with `min_volume`/`max_volume`"
|
|
102
324
|
`min_volume: N` drops any object smaller than `N` µm³; `max_volume: N`
|
|
103
325
|
drops any object larger than `N` µm³ (e.g. several objects merged into
|
|
@@ -258,6 +480,41 @@ set-resources:
|
|
|
258
480
|
runtime: 240
|
|
259
481
|
```
|
|
260
482
|
|
|
483
|
+
!!! warning "`runtime` is capped by the QOS, and sbatch rejects — it does not truncate"
|
|
484
|
+
Every `runtime:` above is bounded by the QOS the job lands in. Ask for
|
|
485
|
+
more and **`sbatch` refuses the job**, so it never starts:
|
|
486
|
+
|
|
487
|
+
```
|
|
488
|
+
sbatch: error: QOSMaxWallDurationPerJobLimit
|
|
489
|
+
sbatch: error: Batch job submission failed: Job violates accounting/QOS policy
|
|
490
|
+
```
|
|
491
|
+
|
|
492
|
+
The giveaway is an **empty log file**: the rule's `logs/<rule>.log` is
|
|
493
|
+
created by Snakemake but nothing ever writes to it, because the script
|
|
494
|
+
never ran. The reason appears only in the submission error, not in the
|
|
495
|
+
log.
|
|
496
|
+
|
|
497
|
+
Raising `runtime` past the cap therefore does not work on its own — you
|
|
498
|
+
have to request a QOS that allows it, per rule:
|
|
499
|
+
|
|
500
|
+
```yaml
|
|
501
|
+
set-resources:
|
|
502
|
+
merge:
|
|
503
|
+
qos: "1day" # a QOS your account may use on that partition
|
|
504
|
+
runtime: 720
|
|
505
|
+
```
|
|
506
|
+
|
|
507
|
+
List what you may ask for, and each one's ceiling:
|
|
508
|
+
|
|
509
|
+
```bash
|
|
510
|
+
sacctmgr show assoc user=$USER format=partition,qos%40
|
|
511
|
+
sacctmgr show qos format=name,maxwall
|
|
512
|
+
```
|
|
513
|
+
|
|
514
|
+
On scicore the default QOS allows 6h, which is why the shipped profile
|
|
515
|
+
keeps every CPU rule at or below `runtime: 360` and gives the long
|
|
516
|
+
`segment` rule an explicit `qos:`.
|
|
517
|
+
|
|
261
518
|
Then launch (from a login node — Snakemake submits and watches the jobs):
|
|
262
519
|
|
|
263
520
|
```bash
|
|
@@ -362,6 +619,7 @@ input: "/data/scan.ims"
|
|
|
362
619
|
work_dir: "/scratch/results"
|
|
363
620
|
tile_shape: [16, 1024, 1024]
|
|
364
621
|
shard: false # true → far fewer files, same chunks
|
|
622
|
+
ngff_version: "auto" # convert reads it, so it belongs here
|
|
365
623
|
tiles_per_job: 4
|
|
366
624
|
```
|
|
367
625
|
|
|
@@ -447,11 +705,21 @@ snakemake --workflow-profile profile/slurm --configfile config/common.yaml confi
|
|
|
447
705
|
```
|
|
448
706
|
|
|
449
707
|
!!! warning "Conversion settings belong in the shared file"
|
|
450
|
-
`convert` runs **once**, from the first config only. A `shard`, `input
|
|
451
|
-
`
|
|
452
|
-
nothing logs that it was dropped.
|
|
453
|
-
|
|
454
|
-
the configs by hand, keep them in
|
|
708
|
+
`convert` runs **once**, from the first config only. A `shard`, `input`,
|
|
709
|
+
`convert_chunks`, `sequence_pattern` or `reuse_pyramid` set on the second
|
|
710
|
+
config is therefore never read, and nothing logs that it was dropped.
|
|
711
|
+
`run_multi` refuses to start when those keys disagree across configs and
|
|
712
|
+
tells you which one — but if you drive the configs by hand, keep them in
|
|
713
|
+
`common.yaml`.
|
|
714
|
+
|
|
715
|
+
`ngff_version` is in that list: it decides the store's zarr format, so a
|
|
716
|
+
second config disagreeing about it would describe a store that is not the
|
|
717
|
+
one on disk.
|
|
718
|
+
|
|
719
|
+
`merge` runs once **per config**, so the keys it reads —
|
|
720
|
+
`pyramid_levels`, `pyramid_downscale`, `sequential_labels`,
|
|
721
|
+
`min_volume`/`max_volume` and `shard_labels` — may legitimately differ
|
|
722
|
+
between them, and are only in `common.yaml` for convenience.
|
|
455
723
|
|
|
456
724
|
Splitting the configs is optional: a self-contained config still works,
|
|
457
725
|
and `common:` can simply be left out of `multi.yaml`.
|
|
@@ -560,9 +828,30 @@ abort the others; you get a per-config status and a non-zero exit.
|
|
|
560
828
|
caps the wall time below `--relate-time` — `srun` fails immediately with
|
|
561
829
|
`QOSMaxWallDurationPerJobLimit` when that happens; `sacctmgr -p show
|
|
562
830
|
assoc user=$USER` and `sacctmgr -p show qos` list what's available and
|
|
563
|
-
each one's `MaxWall`.
|
|
831
|
+
each one's `MaxWall`.
|
|
832
|
+
|
|
833
|
+
The same settings can live in the multi config as a `relate:` block, so
|
|
834
|
+
the shipped `pixi run multi-slurm` task keeps working without extra
|
|
835
|
+
flags:
|
|
836
|
+
|
|
837
|
+
```yaml
|
|
838
|
+
# config/multi.yaml
|
|
839
|
+
relate:
|
|
840
|
+
qos: "1day"
|
|
841
|
+
time: 720 # minutes, per pair; must stay under that QOS's MaxWall
|
|
842
|
+
```
|
|
843
|
+
|
|
844
|
+
A `--relate-*` flag overrides the block for that one key; anything the
|
|
845
|
+
block does not set keeps its default. An unknown key there is an error
|
|
846
|
+
rather than silently ignored, since a typo would otherwise run with the
|
|
847
|
+
default you meant to replace. Under plain `multi` (no `--profile`), relations
|
|
564
848
|
still run locally, in-process, one after another, as before.
|
|
565
849
|
|
|
850
|
+
Each pair logs its shape, chunk count and object count before it starts,
|
|
851
|
+
then a progress line roughly once a minute (`label_relations: 412/3,600
|
|
852
|
+
(11%) after 7m, ~55m left`), so a long relation is distinguishable from a
|
|
853
|
+
hung one in `logs/relate/<a>_to_<b>.log`.
|
|
854
|
+
|
|
566
855
|
Because every pair gets its own job, one running long no longer starves
|
|
567
856
|
the others out of a shared time budget, and a pair that gets killed no
|
|
568
857
|
longer takes an already-finished sibling's workbook down with it.
|
|
@@ -96,7 +96,16 @@ napari = [
|
|
|
96
96
|
# openpyxl -> scripts/run_multi.py writes label_relations() output as an
|
|
97
97
|
# Excel workbook (per-object + per-container sheets), not a plain CSV.
|
|
98
98
|
workflow = ["snakemake>=8", "snakemake-executor-plugin-slurm", "openpyxl"]
|
|
99
|
-
|
|
99
|
+
# jsonschema validates what we write against the vendored official
|
|
100
|
+
# OME-NGFF schemas (tests/ngff_schemas/); without it that one test skips.
|
|
101
|
+
dev = [
|
|
102
|
+
"pytest",
|
|
103
|
+
"pytest-cov",
|
|
104
|
+
"scikit-image",
|
|
105
|
+
"psutil",
|
|
106
|
+
"tqdm",
|
|
107
|
+
"jsonschema>=4.18",
|
|
108
|
+
]
|
|
100
109
|
docs = ["mkdocs-material>=9.0", "mkdocstrings[python]>=0.24"]
|
|
101
110
|
all = [
|
|
102
111
|
"patchworks[io,gpu,bioio,imaris,napari]",
|
|
@@ -449,6 +449,18 @@ def auto_tile_shape_cellpose(
|
|
|
449
449
|
(1, 2048, 2048)
|
|
450
450
|
"""
|
|
451
451
|
n_workers = n_workers or cpu_allocation()
|
|
452
|
+
# Cellpose resizes the tile before the net runs -- by `rescale`
|
|
453
|
+
# (= 30 / diameter) on every axis, and by `anisotropy` on z as well -- so
|
|
454
|
+
# the array it actually holds is bigger than the one it was handed, and a
|
|
455
|
+
# budget computed from the unresized tile under-counts by that factor. A
|
|
456
|
+
# diameter half the model's 30 px means a 2x upsample per axis: 8x the
|
|
457
|
+
# voxels, enough to turn a comfortable tile into an OOM.
|
|
458
|
+
#
|
|
459
|
+
# Both only ever *shrink* the tile. A predicted downsample (diameter > 30)
|
|
460
|
+
# would license a bigger one, but these are a safety margin against a
|
|
461
|
+
# rough memory model, not a measurement to spend headroom on.
|
|
462
|
+
rescale = max(1.0, 30.0 / diameter) if diameter else 1.0
|
|
463
|
+
z_resize = max(1.0, anisotropy or 1.0)
|
|
452
464
|
# A tile holds n_channels planes per voxel (e.g. Cellpose's
|
|
453
465
|
# cyto+nuclei pair), so the per-voxel cost -- and every budget
|
|
454
466
|
# derived from it below -- scales with them.
|
|
@@ -475,7 +487,8 @@ def auto_tile_shape_cellpose(
|
|
|
475
487
|
min_tile = int(4 * diameter) if diameter is not None else 1
|
|
476
488
|
|
|
477
489
|
if n_spatial == 2 or not do_3D:
|
|
478
|
-
|
|
490
|
+
# Two axes resized, so the cost per configured pixel is rescale**2.
|
|
491
|
+
max_pixels_2d = max(1, int(max_raw_bytes // (itemsize * rescale**2)))
|
|
479
492
|
tile_side = max(min_tile, int(max_pixels_2d**0.5))
|
|
480
493
|
if n_spatial == 2:
|
|
481
494
|
y, x = shape[-2], shape[-1]
|
|
@@ -485,9 +498,10 @@ def auto_tile_shape_cellpose(
|
|
|
485
498
|
chunk_spatial = [1, min(y, tile_side), min(x, tile_side)]
|
|
486
499
|
else:
|
|
487
500
|
z, y, x = shape[-3], shape[-2], shape[-1]
|
|
488
|
-
#
|
|
489
|
-
#
|
|
490
|
-
|
|
501
|
+
# All three axes are resized: z by anisotropy * rescale, y/x by
|
|
502
|
+
# rescale each -- so one configured voxel costs
|
|
503
|
+
# anisotropy * rescale**3 of them.
|
|
504
|
+
effective_z = z * z_resize * rescale**3
|
|
491
505
|
max_pixels_per_slice = max(
|
|
492
506
|
1, int((max_raw_bytes // 3) // (effective_z * itemsize))
|
|
493
507
|
)
|
|
@@ -18,32 +18,128 @@ _ZARR_V3 = int(zarr.__version__.split(".")[0]) >= 3
|
|
|
18
18
|
_ZARR_V3 = int(zarr.__version__.split(".")[0]) >= 3
|
|
19
19
|
|
|
20
20
|
|
|
21
|
-
def zarr_compressor_kwargs() -> dict:
|
|
21
|
+
def zarr_compressor_kwargs(zarr_format: int = 3) -> dict:
|
|
22
22
|
"""Keyword arguments pinning the compression codec for a new array.
|
|
23
23
|
|
|
24
24
|
zstd is already zarr v3's default, but relying on a library default means
|
|
25
25
|
the stores patchworks writes change silently if that default ever moves.
|
|
26
26
|
Labels in particular are highly compressible, so this is worth stating.
|
|
27
27
|
|
|
28
|
+
The codec *object* depends on the format of the array being written, not
|
|
29
|
+
on the installed zarr: zarr-python 3 can write a zarr-v2 array (which is
|
|
30
|
+
what NGFF 0.4 needs), and a v2 array rejects ``zarr.codecs.ZstdCodec`` --
|
|
31
|
+
it wants the numcodecs one.
|
|
32
|
+
|
|
33
|
+
Parameters
|
|
34
|
+
----------
|
|
35
|
+
zarr_format : int, optional
|
|
36
|
+
Format of the array about to be created, 2 or 3 (default 3).
|
|
37
|
+
|
|
28
38
|
Returns
|
|
29
39
|
-------
|
|
30
40
|
dict
|
|
31
|
-
``compressors=``/``compressor=`` as
|
|
41
|
+
``compressors=``/``compressor=`` as that combination expects, or
|
|
32
42
|
empty if the codec cannot be built (then the default applies).
|
|
33
43
|
"""
|
|
34
44
|
try:
|
|
35
|
-
if _ZARR_V3:
|
|
45
|
+
if _ZARR_V3 and zarr_format != 2:
|
|
36
46
|
from zarr.codecs import ZstdCodec
|
|
37
47
|
|
|
38
48
|
return {"compressors": (ZstdCodec(level=1),)}
|
|
39
49
|
import numcodecs
|
|
40
50
|
|
|
41
|
-
|
|
51
|
+
codec = numcodecs.Zstd(level=1)
|
|
52
|
+
return {"compressors": (codec,)} if _ZARR_V3 else {"compressor": codec}
|
|
42
53
|
except Exception: # pragma: no cover - depends on the installed zarr
|
|
43
54
|
logger.debug("could not pin a compressor; using zarr's default")
|
|
44
55
|
return {}
|
|
45
56
|
|
|
46
57
|
|
|
58
|
+
def open_zarr_source(
|
|
59
|
+
store_path: Union[str, Path],
|
|
60
|
+
) -> tuple[Union[str, "zarr.storage.StoreLike"], str]:
|
|
61
|
+
"""Resolve a store path, transparently opening a ``.zip`` bundle.
|
|
62
|
+
|
|
63
|
+
A store packed by ``pixi run zip`` is one file holding
|
|
64
|
+
``<name>.zarr/...``. zarr reads it in place through a ``ZipStore``, so
|
|
65
|
+
nothing has to be unpacked first -- but a plain path string does not,
|
|
66
|
+
and every reader here takes a path. This returns what zarr and dask
|
|
67
|
+
should actually be handed, plus the prefix to prepend to a component.
|
|
68
|
+
|
|
69
|
+
Parameters
|
|
70
|
+
----------
|
|
71
|
+
store_path : str or Path
|
|
72
|
+
A ``.zarr`` directory, or a ``.zip`` bundle containing one.
|
|
73
|
+
|
|
74
|
+
Returns
|
|
75
|
+
-------
|
|
76
|
+
tuple
|
|
77
|
+
``(source, prefix)``. For a directory, the path and ``""``. For a
|
|
78
|
+
bundle, an open read-only ``ZipStore`` and the store's name inside
|
|
79
|
+
it, so a component is addressed as ``f"{prefix}/{component}"``.
|
|
80
|
+
|
|
81
|
+
Raises
|
|
82
|
+
------
|
|
83
|
+
ValueError
|
|
84
|
+
If a ``.zip`` does not hold exactly one top-level store.
|
|
85
|
+
"""
|
|
86
|
+
text = str(store_path)
|
|
87
|
+
if ".zip" not in text:
|
|
88
|
+
return text, ""
|
|
89
|
+
|
|
90
|
+
import zipfile
|
|
91
|
+
|
|
92
|
+
# The bundle may be addressed with a group path after it, e.g.
|
|
93
|
+
# "scan.zarr.zip/labels/cells" -- callers build those by string-joining.
|
|
94
|
+
head, _, tail = text.partition(".zip")
|
|
95
|
+
archive_path = head + ".zip"
|
|
96
|
+
with zipfile.ZipFile(archive_path) as archive:
|
|
97
|
+
tops = {
|
|
98
|
+
name.split("/", 1)[0] for name in archive.namelist() if "/" in name
|
|
99
|
+
}
|
|
100
|
+
if len(tops) != 1:
|
|
101
|
+
raise ValueError(
|
|
102
|
+
f"{archive_path} must contain exactly one top-level store; found "
|
|
103
|
+
f"{sorted(tops) or 'nothing'}. Bundles written by "
|
|
104
|
+
"`pixi run zip` always do."
|
|
105
|
+
)
|
|
106
|
+
prefix = tops.pop()
|
|
107
|
+
inner = tail.strip("/")
|
|
108
|
+
if inner:
|
|
109
|
+
prefix = f"{prefix}/{inner}"
|
|
110
|
+
return zarr.storage.ZipStore(archive_path, mode="r"), prefix
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def open_group_any(path: Union[str, Path], mode: str = "r"):
|
|
114
|
+
"""``zarr.open_group`` that also accepts a path *inside* a .zip bundle.
|
|
115
|
+
|
|
116
|
+
Callers build group paths by string-joining (``f"{store}/labels"``),
|
|
117
|
+
which a bundle breaks: the archive is a file, not a directory. Split on
|
|
118
|
+
the ``.zip`` instead, so ``bundle.zip/labels/cells`` resolves to the
|
|
119
|
+
right group inside it.
|
|
120
|
+
"""
|
|
121
|
+
source, prefix = open_zarr_source(path)
|
|
122
|
+
return zarr.open_group(source, path=prefix, mode=mode)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def from_zarr_any(path: Union[str, Path], component: str | None = None):
|
|
126
|
+
"""``dask.array.from_zarr`` that also accepts a .zip bundle."""
|
|
127
|
+
import dask.array as _da
|
|
128
|
+
|
|
129
|
+
source, prefix = open_zarr_source(path)
|
|
130
|
+
inner = _component(prefix, component) if component else prefix
|
|
131
|
+
return (
|
|
132
|
+
_da.from_zarr(source, component=inner)
|
|
133
|
+
if inner
|
|
134
|
+
else _da.from_zarr(source)
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _component(prefix: str, name: str) -> str:
|
|
139
|
+
"""Join a bundle prefix and a component, tolerating an empty prefix."""
|
|
140
|
+
return f"{prefix}/{name}" if prefix else name
|
|
141
|
+
|
|
142
|
+
|
|
47
143
|
def load_ome_zarr(
|
|
48
144
|
store_path: Union[str, Path],
|
|
49
145
|
channel: int | None = 0,
|
|
@@ -75,7 +171,8 @@ def load_ome_zarr(
|
|
|
75
171
|
>>> arr.shape
|
|
76
172
|
(128, 2048, 2048)
|
|
77
173
|
"""
|
|
78
|
-
|
|
174
|
+
source, prefix = open_zarr_source(store_path)
|
|
175
|
+
root = zarr.open_group(source, path=prefix, mode="r")
|
|
79
176
|
# OME-ZARR 0.5 nests under "ome" key; older stores use "multiscales" directly
|
|
80
177
|
_attrs = dict(root.attrs)
|
|
81
178
|
_ms = _attrs.get("multiscales") or _attrs.get("ome", {}).get("multiscales")
|
|
@@ -93,7 +190,9 @@ def load_ome_zarr(
|
|
|
93
190
|
if zarr_ndim > len(chunks):
|
|
94
191
|
zarr_chunks = (1,) * (zarr_ndim - len(chunks)) + tuple(chunks)
|
|
95
192
|
|
|
96
|
-
arr = da.from_zarr(
|
|
193
|
+
arr = da.from_zarr(
|
|
194
|
+
source, component=_component(prefix, path), chunks=zarr_chunks
|
|
195
|
+
)
|
|
97
196
|
if channel is not None:
|
|
98
197
|
arr = _select_channel(arr, channel, _ms[0], store_path)
|
|
99
198
|
return arr
|