patchworks 2.6.12__tar.gz → 2.7.0__tar.gz

{patchworks-2.6.12 → patchworks-2.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: patchworks
-Version: 2.6.12
+Version: 2.7.0
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues
@@ -57,6 +57,7 @@ Requires-Dist: cellpose<4,>=3.0; extra == 'cellpose3'
 Provides-Extra: cellpose4
 Requires-Dist: cellpose>=4; extra == 'cellpose4'
 Provides-Extra: dev
+Requires-Dist: jsonschema>=4.18; extra == 'dev'
 Requires-Dist: psutil; extra == 'dev'
 Requires-Dist: pytest; extra == 'dev'
 Requires-Dist: pytest-cov; extra == 'dev'

{patchworks-2.6.12 → patchworks-2.7.0}/docs/api/plugins/ome_zarr.md

@@ -25,6 +25,10 @@ resolution, matching anisotropic microscopy stacks.
 
 ::: patchworks.plugins.ome_zarr.register_labels
 
+## reshard_level
+
+::: patchworks.plugins.ome_zarr.reshard_level
+
 ## read_pixel_size
 
 ::: patchworks.plugins.ome_zarr.read_pixel_size
@@ -37,6 +41,14 @@ matches the store, and reads both.
 
 ::: patchworks.plugins.ome_zarr.ngff_version
 
+Every writer takes an `ngff_version=` keyword: `"auto"` (default) follows the
+installed zarr — 0.5 on v3, 0.4 on v2 — and `"0.4"` pins the older, zarr-v2
+layout. Writing into an existing store always follows *that store's* format,
+so a label pyramid added later can never disagree with the image it sits in.
+NGFF 0.6 is released but not written: RFC-5 replaces `axes` with
+`coordinateSystems` and requires `input`/`output` on every coordinate
+transformation, and no reader supports it yet.
+
 ::: patchworks.plugins.ome_zarr.read_ngff_attr
 
 ::: patchworks.plugins.ome_zarr.write_ngff_attrs

{patchworks-2.6.12 → patchworks-2.7.0}/docs/guide/snakemake.md

@@ -50,7 +50,9 @@ work_dir: "/scratch/results"   # everything is written here
 # conversion (input → pyramidal OME-ZARR)
 reuse_pyramid: true            # .ims: copy its own pyramid (fast)
 convert_chunks: null           # null → bounded auto chunks; or [c,z,y,x]
-shard: false                   # true → pack chunks into shards (fewer files)
+shard: false                   # true → pack chunks into shards (fewer files);
+                               # covers the image and the label pyramids
+ngff_version: "auto"           # OME-ZARR version: "auto" (0.5), or "0.4"
 
 # tiling
 channel: 0                     # channel to segment, 0-based (null = keep all)
@@ -83,6 +85,8 @@ cellpose:
 pyramid_levels: 5
 pyramid_downscale: 2
 sequential_labels: true        # renumber labels to a contiguous 1..N
+shard_labels: false            # true → also reshard label level 0 after the
+                               # merge (one extra pass; see tip below)
 ```
 
 !!! tip "Growing labels after segmentation"
@@ -98,6 +102,77 @@ sequential_labels: true        # renumber labels to a contiguous 1..N
     labels afterwards](custom_segmentation.md#growing-labels-afterwards-dilation)
     for how it works and the equivalent direct-API call.
 
+!!! tip "What `shard: true` covers, and `shard_labels` for label level 0"
+    Without sharding, one chunk is one file, and a fine-chunked level 0 can
+    run to ~950k of them — painful on a shared filesystem at write time and
+    on every read after. `shard: true` packs chunks into far fewer files
+    without changing the chunking or the memory profile.
+
+    `shard: true` applies to **the converted image (all levels)** and to
+    **the label pyramid levels (1..N)**. It cannot apply to **label level
+    0** *while that level is being written*: a shard has to be written whole
+    by a single writer, while level 0 is filled one chunk at a time by
+    concurrent `segment` jobs (or by the merge's own worker pool). Two
+    writers doing a read-modify-write on the same shard file silently lose
+    each other's chunks.
+
+    Once the merge has finished, though, nothing is writing it anymore, so a
+    single-threaded pass *can* rewrite it sharded. That is `shard_labels`:
+
+    ```yaml
+    shard: true
+    shard_labels: true       # or an explicit shape, e.g. [16, 512, 512]
+    ```
+
+    `true` reuses whatever spec `shard` carries; a list overrides it.
+    `shard: false` does not veto it — an unsharded image with sharded labels
+    is a valid combination. It is opt-in because it costs one extra full
+    read+write of level 0, and because it is the level with the most chunks
+    it is also the one worth paying for. The array's attributes (including
+    the merge's own completion marker) are carried across, so a later rerun
+    still sees the merge as done.
+
+    Check whether the file count is actually a problem for your data first —
+    a `(126, 34000, 28500)` image at the `(16, 1024, 1024)` label chunk cap
+    is 7,616 files per label group, well under the 200,000 at which the
+    conversion starts warning. On scicore that is fine; on a filesystem with
+    a tighter inode or per-directory budget it may not be.
+
+!!! tip "Which OME-ZARR version gets written (`ngff_version`)"
+    OME-ZARR is versioned, and the version decides the **zarr format** as
+    well as the metadata layout — the two are not separate choices. NGFF 0.4
+    is specified over zarr v2 and puts `multiscales`, `labels` and
+    `image-label` at the top level of the store's attributes; 0.5 is the
+    zarr-v3 revision and nests them under an `ome` key.
+
+    `ngff_version: "auto"` (the default) follows the installed zarr, which on
+    any current environment means **0.5**. Pin `"0.4"` only when a downstream
+    tool still cannot read 0.5:
+
+    ```yaml
+    ngff_version: "0.4"
+    shard: false           # required: zarr v2 has no sharding codec
+    ```
+
+    That is a real trade-off, not a formality — 0.4 means a zarr-v2 store, and
+    sharding is a zarr-v3 feature, so `shard`/`shard_labels` stop doing
+    anything. The workflow refuses the combination rather than silently
+    ignoring it. The store's own root file changes too (`.zgroup` instead of
+    `zarr.json`), which the rules account for.
+
+    Writing into an **existing** store always follows that store's format,
+    whatever this key says: the label pyramid is written by `merge`, long
+    after `convert` made the image, and a store with v2 arrays and v3
+    metadata is one no reader can open.
+
+    **What about 0.6?** It was released on 14 September 2026 and patchworks
+    does not write it. It is not a version bump but a different metadata
+    document: RFC-5 replaces a multiscale's `axes` with `coordinateSystems`
+    and requires an `input`/`output` pair on every coordinate transformation.
+    Nothing reads it yet either — ome-zarr-py, and so napari, still default
+    to 0.5. Setting `ngff_version: "0.6"` is rejected with that explanation
+    rather than writing a store you could not open.
+
 !!! tip "Dropping objects by size with `min_volume`/`max_volume`"
     `min_volume: N` drops any object smaller than `N` µm³; `max_volume: N`
     drops any object larger than `N` µm³ (e.g. several objects merged into
@@ -362,6 +437,7 @@ input: "/data/scan.ims"
 work_dir: "/scratch/results"
 tile_shape: [16, 1024, 1024]
 shard: false                   # true → far fewer files, same chunks
+ngff_version: "auto"           # convert reads it, so it belongs here
 tiles_per_job: 4
 ```
 
@@ -447,11 +523,21 @@ snakemake --workflow-profile profile/slurm --configfile config/common.yaml confi
 ```
 
 !!! warning "Conversion settings belong in the shared file"
-    `convert` runs **once**, from the first config only. A `shard`, `input` or
-    `pyramid_levels` set on the second config is therefore never read, and
-    nothing logs that it was dropped. `run_multi` refuses to start when those
-    keys disagree across configs and tells you which one — but if you drive
-    the configs by hand, keep them in `common.yaml`.
+    `convert` runs **once**, from the first config only. A `shard`, `input`,
+    `convert_chunks`, `sequence_pattern` or `reuse_pyramid` set on the second
+    config is therefore never read, and nothing logs that it was dropped.
+    `run_multi` refuses to start when those keys disagree across configs and
+    tells you which one — but if you drive the configs by hand, keep them in
+    `common.yaml`.
+
+    `ngff_version` is in that list: it decides the store's zarr format, so a
+    second config disagreeing about it would describe a store that is not the
+    one on disk.
+
+    `merge` runs once **per config**, so the keys it reads —
+    `pyramid_levels`, `pyramid_downscale`, `sequential_labels`,
+    `min_volume`/`max_volume` and `shard_labels` — may legitimately differ
+    between them, and are only in `common.yaml` for convenience.
 
     Splitting the configs is optional: a self-contained config still works,
     and `common:` can simply be left out of `multi.yaml`.

{patchworks-2.6.12 → patchworks-2.7.0}/pyproject.toml

@@ -96,7 +96,16 @@ napari = [
 # openpyxl -> scripts/run_multi.py writes label_relations() output as an
 # Excel workbook (per-object + per-container sheets), not a plain CSV.
 workflow = ["snakemake>=8", "snakemake-executor-plugin-slurm", "openpyxl"]
-dev = ["pytest", "pytest-cov", "scikit-image", "psutil", "tqdm"]
+# jsonschema validates what we write against the vendored official
+# OME-NGFF schemas (tests/ngff_schemas/); without it that one test skips.
+dev = [
+    "pytest",
+    "pytest-cov",
+    "scikit-image",
+    "psutil",
+    "tqdm",
+    "jsonschema>=4.18",
+]
 docs = ["mkdocs-material>=9.0", "mkdocstrings[python]>=0.24"]
 all = [
     "patchworks[io,gpu,bioio,imaris,napari]",

{patchworks-2.6.12 → patchworks-2.7.0}/src/patchworks/_chunks.py

@@ -449,6 +449,18 @@ def auto_tile_shape_cellpose(
     (1, 2048, 2048)
     """
     n_workers = n_workers or cpu_allocation()
+    # Cellpose resizes the tile before the net runs -- by `rescale`
+    # (= 30 / diameter) on every axis, and by `anisotropy` on z as well -- so
+    # the array it actually holds is bigger than the one it was handed, and a
+    # budget computed from the unresized tile under-counts by that factor. A
+    # diameter half the model's 30 px means a 2x upsample per axis: 8x the
+    # voxels, enough to turn a comfortable tile into an OOM.
+    #
+    # Both only ever *shrink* the tile. A predicted downsample (diameter > 30)
+    # would license a bigger one, but these are a safety margin against a
+    # rough memory model, not a measurement to spend headroom on.
+    rescale = max(1.0, 30.0 / diameter) if diameter else 1.0
+    z_resize = max(1.0, anisotropy or 1.0)
     # A tile holds n_channels planes per voxel (e.g. Cellpose's
     # cyto+nuclei pair), so the per-voxel cost -- and every budget
     # derived from it below -- scales with them.
@@ -475,7 +487,8 @@ def auto_tile_shape_cellpose(
     min_tile = int(4 * diameter) if diameter is not None else 1
 
     if n_spatial == 2 or not do_3D:
-        max_pixels_2d = max(1, max_raw_bytes // itemsize)
+        # Two axes resized, so the cost per configured pixel is rescale**2.
+        max_pixels_2d = max(1, int(max_raw_bytes // (itemsize * rescale**2)))
         tile_side = max(min_tile, int(max_pixels_2d**0.5))
         if n_spatial == 2:
             y, x = shape[-2], shape[-1]
@@ -485,9 +498,10 @@ def auto_tile_shape_cellpose(
             chunk_spatial = [1, min(y, tile_side), min(x, tile_side)]
     else:
         z, y, x = shape[-3], shape[-2], shape[-1]
-        # Cellpose resizes z by `anisotropy` before the net runs, so the tile
-        # it actually holds is that much deeper than the one handed to it.
-        effective_z = z * max(1.0, anisotropy or 1.0)
+        # All three axes are resized: z by anisotropy * rescale, y/x by
+        # rescale each -- so one configured voxel costs
+        # anisotropy * rescale**3 of them.
+        effective_z = z * z_resize * rescale**3
         max_pixels_per_slice = max(
             1, int((max_raw_bytes // 3) // (effective_z * itemsize))
         )

{patchworks-2.6.12 → patchworks-2.7.0}/src/patchworks/_io.py

@@ -18,27 +18,38 @@ _ZARR_V3 = int(zarr.__version__.split(".")[0]) >= 3
 _ZARR_V3 = int(zarr.__version__.split(".")[0]) >= 3
 
 
-def zarr_compressor_kwargs() -> dict:
+def zarr_compressor_kwargs(zarr_format: int = 3) -> dict:
     """Keyword arguments pinning the compression codec for a new array.
 
     zstd is already zarr v3's default, but relying on a library default means
     the stores patchworks writes change silently if that default ever moves.
     Labels in particular are highly compressible, so this is worth stating.
 
+    The codec *object* depends on the format of the array being written, not
+    on the installed zarr: zarr-python 3 can write a zarr-v2 array (which is
+    what NGFF 0.4 needs), and a v2 array rejects ``zarr.codecs.ZstdCodec`` --
+    it wants the numcodecs one.
+
+    Parameters
+    ----------
+    zarr_format : int, optional
+        Format of the array about to be created, 2 or 3 (default 3).
+
     Returns
     -------
     dict
-        ``compressors=``/``compressor=`` as the installed zarr expects, or
+        ``compressors=``/``compressor=`` as that combination expects, or
         empty if the codec cannot be built (then the default applies).
     """
     try:
-        if _ZARR_V3:
+        if _ZARR_V3 and zarr_format != 2:
             from zarr.codecs import ZstdCodec
 
             return {"compressors": (ZstdCodec(level=1),)}
         import numcodecs
 
-        return {"compressor": numcodecs.Zstd(level=1)}
+        codec = numcodecs.Zstd(level=1)
+        return {"compressors": (codec,)} if _ZARR_V3 else {"compressor": codec}
     except Exception:  # pragma: no cover - depends on the installed zarr
         logger.debug("could not pin a compressor; using zarr's default")
         return {}